Source code

001package co.codewizards.cloudstore.local.transport;
002
003import static co.codewizards.cloudstore.core.io.StreamUtil.*;
004import static co.codewizards.cloudstore.core.objectfactory.ObjectFactoryUtil.*;
005import static co.codewizards.cloudstore.core.oio.OioFileFactory.*;
006
007import co.codewizards.cloudstore.core.io.ByteArrayInputStream;
008import java.io.IOException;
009import java.io.OutputStream;
010import java.security.NoSuchAlgorithmException;
011import java.util.Collection;
012import java.util.Collections;
013import java.util.Map;
014import java.util.TreeMap;
015
016import org.slf4j.Logger;
017import org.slf4j.LoggerFactory;
018
019import co.codewizards.cloudstore.core.dto.FileChunkDto;
020import co.codewizards.cloudstore.core.dto.TempChunkFileDto;
021import co.codewizards.cloudstore.core.dto.jaxb.TempChunkFileDtoIo;
022import co.codewizards.cloudstore.core.oio.File;
023import co.codewizards.cloudstore.core.repo.local.LocalRepoManager;
024import co.codewizards.cloudstore.core.util.AssertUtil;
025import co.codewizards.cloudstore.core.util.HashUtil;
026import co.codewizards.cloudstore.core.util.IOUtil;
027
028public class TempChunkFileManager {
029
030        private static final Logger logger = LoggerFactory.getLogger(TempChunkFileManager.class);
031
032        private static final String TEMP_CHUNK_FILE_PREFIX = "chunk_";
033        private static final String TEMP_CHUNK_FILE_Dto_FILE_SUFFIX = ".xml";
034
035        private static final class Holder {
036                static final TempChunkFileManager instance = createObject(TempChunkFileManager.class);
037        }
038
039        protected TempChunkFileManager() { }
040
041        public static TempChunkFileManager getInstance() {
042                return Holder.instance;
043        }
044
045        public void writeFileDataToTempChunkFile(final File destFile, final long offset, final byte[] fileData) {
046                AssertUtil.assertNotNull(destFile, "destFile");
047                AssertUtil.assertNotNull(fileData, "fileData");
048                try {
049                        final File tempChunkFile = createTempChunkFile(destFile, offset);
050                        final File tempChunkFileDtoFile = getTempChunkFileDtoFile(tempChunkFile);
051
052                        // Delete the meta-data-file, in case we overwrite an older temp-chunk-file. This way it
053                        // is guaranteed, that if the meta-data-file exists, it is consistent with either
054                        // the temp-chunk-file or the chunk was already written into the final destination.
055                        deleteOrFail(tempChunkFileDtoFile);
056
057                        try (final OutputStream out = castStream(tempChunkFile.createOutputStream())) {
058                                out.write(fileData);
059                        }
060                        final String sha1 = sha1(fileData);
061                        logger.trace("writeFileDataToTempChunkFile: Wrote {} bytes with SHA1 '{}' to '{}'.", fileData.length, sha1, tempChunkFile.getAbsolutePath());
062                        final TempChunkFileDto tempChunkFileDto = createTempChunkFileDto(offset, tempChunkFile, sha1);
063                        new TempChunkFileDtoIo().serialize(tempChunkFileDto, tempChunkFileDtoFile);
064                } catch (final IOException e) {
065                        throw new RuntimeException(e);
066                }
067        }
068
069        protected void deleteOrFail(File file) throws IOException {
070                IOUtil.deleteOrFail(file);
071        }
072
073        public void deleteTempChunkFilesWithoutDtoFile(final Collection<TempChunkFileWithDtoFile> tempChunkFileWithDtoFiles) {
074                for (final TempChunkFileWithDtoFile tempChunkFileWithDtoFile : tempChunkFileWithDtoFiles) {
075                        final File tempChunkFileDtoFile = tempChunkFileWithDtoFile.getTempChunkFileDtoFile();
076                        if (tempChunkFileDtoFile == null || !tempChunkFileDtoFile.exists()) {
077                                final File tempChunkFile = tempChunkFileWithDtoFile.getTempChunkFile();
078                                logger.warn("deleteTempChunkFilesWithoutDtoFile: No Dto-file for temporary chunk-file '{}'! DELETING this temporary file!", tempChunkFile.getAbsolutePath());
079                                try {
080                                        deleteOrFail(tempChunkFile);
081                                } catch (IOException x) {
082                                        throw new RuntimeException(x);
083                                }
084                                continue;
085                        }
086                }
087        }
088
089        public Map<Long, TempChunkFileWithDtoFile> getOffset2TempChunkFileWithDtoFile(final File destFile) {
090                final File[] tempFiles = getTempDir(destFile).listFiles();
091                if (tempFiles == null)
092                        return Collections.emptyMap();
093
094                final String destFileName = destFile.getName();
095                final Map<Long, TempChunkFileWithDtoFile> result = new TreeMap<Long, TempChunkFileWithDtoFile>();
096                for (final File tempFile : tempFiles) {
097                        String tempFileName = tempFile.getName();
098                        if (!tempFileName.startsWith(TEMP_CHUNK_FILE_PREFIX))
099                                continue;
100
101                        final boolean dtoFile;
102                        if (tempFileName.endsWith(TEMP_CHUNK_FILE_Dto_FILE_SUFFIX)) {
103                                dtoFile = true;
104                                tempFileName = tempFileName.substring(0, tempFileName.length() - TEMP_CHUNK_FILE_Dto_FILE_SUFFIX.length());
105                        }
106                        else
107                                dtoFile = false;
108
109                        final int lastUnderscoreIndex = tempFileName.lastIndexOf('_');
110                        if (lastUnderscoreIndex < 0)
111                                throw new IllegalStateException("lastUnderscoreIndex < 0 :: tempFileName='" + tempFileName + '\'');
112
113                        final String tempFileDestFileName = tempFileName.substring(TEMP_CHUNK_FILE_PREFIX.length(), lastUnderscoreIndex);
114                        if (!destFileName.equals(tempFileDestFileName))
115                                continue;
116
117                        final String offsetStr = tempFileName.substring(lastUnderscoreIndex + 1);
118                        final Long offset = Long.valueOf(offsetStr, 36);
119                        TempChunkFileWithDtoFile tempChunkFileWithDtoFile = result.get(offset);
120                        if (tempChunkFileWithDtoFile == null) {
121                                tempChunkFileWithDtoFile = new TempChunkFileWithDtoFile();
122                                result.put(offset, tempChunkFileWithDtoFile);
123                        }
124                        if (dtoFile)
125                                tempChunkFileWithDtoFile.setTempChunkFileDtoFile(tempFile);
126                        else
127                                tempChunkFileWithDtoFile.setTempChunkFile(tempFile);
128                }
129                return Collections.unmodifiableMap(result);
130        }
131
132        public File getTempChunkFileDtoFile(final File file) {
133                return createFile(file.getParentFile(), file.getName() + TEMP_CHUNK_FILE_Dto_FILE_SUFFIX);
134        }
135
136        private String sha1(final byte[] data) {
137                AssertUtil.assertNotNull(data, "data");
138                try {
139                        final byte[] hash = HashUtil.hash(HashUtil.HASH_ALGORITHM_SHA, new ByteArrayInputStream(data));
140                        return HashUtil.encodeHexStr(hash);
141                } catch (final NoSuchAlgorithmException e) {
142                        throw new RuntimeException(e);
143                } catch (final IOException e) {
144                        throw new RuntimeException(e);
145                }
146        }
147
148        /**
149         * Create the temporary file for the given {@code destFile} and {@code offset}.
150         * <p>
151         * The returned file is created, if it does not yet exist; but it is <i>not</i> overwritten,
152         * if it already exists.
153         * <p>
154         * The {@linkplain #getTempDir(File) temporary directory} in which the temporary file is located
155         * is created, if necessary. In order to prevent collisions with code trying to delete the empty
156         * temporary directory, this method and the corresponding {@link #deleteTempDirIfEmpty(File)} are
157         * both synchronized.
158         * @param destFile the destination file for which to resolve and create the temporary file.
159         * Must not be <code>null</code>.
160         * @param offset the offset (inside the final destination file and the source file) of the block to
161         * be temporarily stored in the temporary file created by this method. The temporary file will hold
162         * solely this block (thus the offset in the temporary file is 0).
163         * @return the temporary file. Never <code>null</code>. The file is already created in the file system
164         * (empty), if it did not yet exist.
165         */
166        public synchronized File createTempChunkFile(final File destFile, final long offset) {
167                return createTempChunkFile(destFile, offset, true);
168        }
169        protected synchronized File createTempChunkFile(final File destFile, final long offset, final boolean createNewFile) {
170                final File tempDir = getTempDir(destFile);
171                tempDir.mkdir();
172                if (!tempDir.isDirectory())
173                        throw new IllegalStateException("Creating the directory failed (it does not exist after mkdir): " + tempDir.getAbsolutePath());
174
175                final File tempFile = createFile(tempDir, String.format("%s%s_%s",
176                                TEMP_CHUNK_FILE_PREFIX, destFile.getName(), Long.toString(offset, 36)));
177                if (createNewFile) {
178                        try {
179                                tempFile.createNewFile();
180                        } catch (final IOException e) {
181                                throw new RuntimeException(e);
182                        }
183                }
184                return tempFile;
185        }
186
187        /** If source file was moved, the chunks need to be moved, too. */
188        public void moveChunks(final File oldDestFile, final File newDestFile) {
189                final Map<Long, TempChunkFileWithDtoFile> offset2TempChunkFileWithDtoFile = getOffset2TempChunkFileWithDtoFile(oldDestFile);
190                for (final Map.Entry<Long, TempChunkFileWithDtoFile> entry : offset2TempChunkFileWithDtoFile.entrySet()) {
191                        final Long offset = entry.getKey();
192                        final TempChunkFileWithDtoFile tempChunkFileWithDtoFile = entry.getValue();
193                        final File oldTempChunkFile = tempChunkFileWithDtoFile.getTempChunkFile();
194                        final File newTempChunkFile = createTempChunkFile(newDestFile, offset, false);
195                        final File oldTempChunkFileDtoFile = getTempChunkFileDtoFile(oldTempChunkFile);
196                        final File newTempChunkFileDtoFile = getTempChunkFileDtoFile(newTempChunkFile);
197                        try {
198//                              oldTempChunkFileDtoFile.move(newTempChunkFileDtoFile);
199                                moveOrFail(oldTempChunkFileDtoFile, newTempChunkFileDtoFile);
200                                logger.info("Moved chunkDto from {} to {}", oldTempChunkFileDtoFile, newTempChunkFileDtoFile);
201//                              oldTempChunkFile.move(newTempChunkFile);
202                                moveOrFail(oldTempChunkFile, newTempChunkFile);
203                                logger.info("Moved chunk from {} to {}", oldTempChunkFile, newTempChunkFile);
204                        } catch (final IOException e) {
205                                throw new RuntimeException(e);
206                        }
207                }
208        }
209
210        protected void moveOrFail(File oldFile, File newFile) throws IOException {
211                oldFile.move(newFile);
212        }
213
214        /**
215         * Deletes the {@linkplain #getTempDir(File) temporary directory} for the given {@code destFile},
216         * if this directory is empty.
217         * <p>
218         * This method is synchronized to prevent it from colliding with {@link #createTempChunkFile(File, long)}
219         * which first creates the temporary directory and then the file in it. Without synchronisation, the
220         * newly created directory might be deleted by this method, before the temporary file in it is created.
221         * @param destFile the destination file for which to resolve and delete the temporary directory.
222         * Must not be <code>null</code>.
223         */
224        public synchronized void deleteTempDirIfEmpty(final File destFile) {
225                final File tempDir = getTempDir(destFile);
226                tempDir.delete(); // deletes only empty directories ;-)
227        }
228
229        public File getTempDir(final File destFile) {
230                AssertUtil.assertNotNull(destFile, "destFile");
231                final File parentDir = destFile.getParentFile();
232                return createFile(parentDir, LocalRepoManager.TEMP_DIR_NAME);
233        }
234
235        /**
236         * @param offset the offset in the (real) destination file (<i>not</i> in {@code tempChunkFile}! there the offset is always 0).
237         * @param tempChunkFile the tempChunkFile containing the chunk's data. Must not be <code>null</code>.
238         * @param sha1 the sha1 of the single chunk (in {@code tempChunkFile}). Must not be <code>null</code>.
239         * @return the Dto. Never <code>null</code>.
240         */
241        public TempChunkFileDto createTempChunkFileDto(final long offset, final File tempChunkFile, final String sha1) {
242                AssertUtil.assertNotNull(tempChunkFile, "tempChunkFile");
243                AssertUtil.assertNotNull(sha1, "sha1");
244
245                if (!tempChunkFile.exists())
246                        throw new IllegalArgumentException("The tempChunkFile does not exist: " + tempChunkFile.getAbsolutePath());
247
248                final FileChunkDto fileChunkDto = new FileChunkDto();
249                fileChunkDto.setOffset(offset);
250
251                final long tempChunkFileLength = tempChunkFile.length();
252                if (tempChunkFileLength > Integer.MAX_VALUE)
253                        throw new IllegalStateException("tempChunkFile.length > Integer.MAX_VALUE");
254
255                fileChunkDto.setLength((int) tempChunkFileLength);
256                fileChunkDto.setSha1(sha1);
257
258                final TempChunkFileDto tempChunkFileDto = new TempChunkFileDto();
259                tempChunkFileDto.setFileChunkDto(fileChunkDto);
260                return tempChunkFileDto;
261        }
262
263        public void deleteTempChunkFiles(final Collection<TempChunkFileWithDtoFile> tempChunkFileWithDtoFiles) {
264                for (final TempChunkFileWithDtoFile tempChunkFileWithDtoFile : tempChunkFileWithDtoFiles) {
265                        final File tempChunkFile = tempChunkFileWithDtoFile.getTempChunkFile(); // tempChunkFile may be null!!!
266                        final File tempChunkFileDtoFile = tempChunkFileWithDtoFile.getTempChunkFileDtoFile();
267
268                        try {
269                                if (tempChunkFile != null && tempChunkFile.exists())
270                                        deleteOrFail(tempChunkFile);
271
272                                if (tempChunkFileDtoFile != null && tempChunkFileDtoFile.exists())
273                                        deleteOrFail(tempChunkFileDtoFile);
274                        } catch (IOException x) {
275                                throw new RuntimeException(x);
276                        }
277                }
278        }
279
280}