From 2efac0c96b29a8ca63c2a82ad1e8d1db1ca92219 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 6 Dec 2021 19:53:20 +0000 Subject: [PATCH] Modified zip implementation to preserve filenames of single file resources. Also modified the directory structure of single file resources to make them consistent with multi file resources. For multi file resources, the original folder is renamed to "data", resulting in a layout such as: data/file1.txt data/file2.txt data/dir1/file3.txt For single file resources, the file is now moved into a "data" folder, like so: data/file.txt This is slightly unconventional, but is appropriate within the context of QDN to keep everything consistent. --- .../qortal/arbitrary/ArbitraryDataReader.java | 13 --- .../qortal/arbitrary/ArbitraryDataWriter.java | 4 +- src/main/java/org/qortal/utils/ZipUtils.java | 34 +++++--- .../arbitrary/ArbitraryCompressionTests.java | 79 ++++++++++++++++--- .../test/arbitrary/ArbitraryDataTests.java | 2 +- 5 files changed, 92 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index bc4c8d9a..337766e1 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -420,19 +420,6 @@ public class ArbitraryDataReader { throw new DataException(String.format("Unable to unzip file: %s", e.getMessage())); } - // If unzipped data was a file not a directory, move it into a data/ directory so that the .qortal - // metadata folder is able to be created there too - if (this.uncompressedPath.toFile().isFile()) { - // Rename to temporary filename - Path tempDest = Paths.get(this.uncompressedPath.getParent().toString(), "data2"); - this.uncompressedPath.toFile().renameTo(tempDest.toFile()); - // Create a "data" directory - Files.createDirectories(this.uncompressedPath); - // Move the original file into the newly created directory - Path finalPath = Paths.get(this.uncompressedPath.toString(), "data"); - tempDest.toFile().renameTo(finalPath.toFile()); - } - // Replace filePath pointer with the uncompressed file path if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { if (Files.exists(this.filePath)) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 354f1605..f27a8269 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -184,8 +184,8 @@ public class ArbitraryDataWriter { if (this.compression == Compression.ZIP) { LOGGER.info("Compressing..."); - String fileName = "data"; //isSingleFile ? singleFileName : null; - ZipUtils.zip(this.filePath.toString(), this.compressedPath.toString(), fileName); + String enclosingFolderName = "data"; + ZipUtils.zip(this.filePath.toString(), this.compressedPath.toString(), enclosingFolderName); } else { throw new DataException(String.format("Unknown compression type specified: %s", compression.toString())); diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index 6e614aef..c61723e7 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -21,6 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * + * Code modified in 2021 for Qortal Core + * */ package org.qortal.utils; @@ -31,44 +33,54 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Paths; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; public class ZipUtils { - public static void zip(String sourcePath, String destFilePath, String fileName) throws IOException, InterruptedException { + public static void zip(String sourcePath, String destFilePath, String enclosingFolderName) throws IOException, InterruptedException { File sourceFile = new File(sourcePath); - if (fileName == null) { - fileName = sourceFile.getName(); - } + boolean isSingleFile = Paths.get(sourcePath).toFile().isFile(); FileOutputStream fileOutputStream = new FileOutputStream(destFilePath); ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); - ZipUtils.zip(sourceFile, fileName, zipOutputStream); + ZipUtils.zip(sourceFile, enclosingFolderName, zipOutputStream, isSingleFile); zipOutputStream.close(); fileOutputStream.close(); } - public static void zip(final File fileToZip, final String fileName, final ZipOutputStream zipOut) throws IOException, InterruptedException { + public static void zip(final File fileToZip, final String enclosingFolderName, final ZipOutputStream zipOut, boolean isSingleFile) throws IOException, InterruptedException { if (Controller.isStopping()) { throw new InterruptedException("Controller is stopping"); } + + // Handle single file resources slightly differently + if (isSingleFile) { + // Create enclosing folder + zipOut.putNextEntry(new ZipEntry(enclosingFolderName + "/")); + zipOut.closeEntry(); + // Place the supplied file within the folder + ZipUtils.zip(fileToZip, enclosingFolderName + "/" + fileToZip.getName(), zipOut, false); + return; + } + if (fileToZip.isDirectory()) { - if (fileName.endsWith("/")) { - zipOut.putNextEntry(new ZipEntry(fileName)); + if (enclosingFolderName.endsWith("/")) { + zipOut.putNextEntry(new ZipEntry(enclosingFolderName)); zipOut.closeEntry(); } else { - zipOut.putNextEntry(new ZipEntry(fileName + "/")); + zipOut.putNextEntry(new ZipEntry(enclosingFolderName + "/")); zipOut.closeEntry(); } final File[] children = fileToZip.listFiles(); for (final File childFile : children) { - ZipUtils.zip(childFile, fileName + "/" + childFile.getName(), zipOut); + ZipUtils.zip(childFile, enclosingFolderName + "/" + childFile.getName(), zipOut, false); } return; } final FileInputStream fis = new FileInputStream(fileToZip); - final ZipEntry zipEntry = new ZipEntry(fileName); + final ZipEntry zipEntry = new ZipEntry(enclosingFolderName); zipOut.putNextEntry(zipEntry); final byte[] bytes = new byte[1024]; int length; diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryCompressionTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryCompressionTests.java index 9fa580f6..dd482074 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryCompressionTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryCompressionTests.java @@ -27,10 +27,10 @@ public class ArbitraryCompressionTests extends Common { @Test public void testZipSingleFile() throws IOException, InterruptedException { - String fileName = "data"; + String enclosingFolderName = "data"; Path inputFile = Files.createTempFile("inputFile", null); Path outputDirectory = Files.createTempDirectory("outputDirectory"); - Path outputFile = Paths.get(outputDirectory.toString(), fileName); + Path outputFile = Paths.get(outputDirectory.toString(), enclosingFolderName); inputFile.toFile().deleteOnExit(); outputDirectory.toFile().deleteOnExit(); @@ -42,7 +42,8 @@ public class ArbitraryCompressionTests extends Common { assertTrue(Files.exists(inputFile)); assertFalse(Files.exists(outputFile)); - ZipUtils.zip(inputFile.toString(), outputFile.toString(), fileName); + // Zip... + ZipUtils.zip(inputFile.toString(), outputFile.toString(), enclosingFolderName); assertTrue(Files.exists(inputFile)); assertTrue(Files.exists(outputFile)); @@ -52,8 +53,8 @@ public class ArbitraryCompressionTests extends Common { // Create paths for unzipping Path unzippedDirectory = Files.createTempDirectory("unzippedDirectory"); - // Single file data is unzipped directly, without an enclosing folder - Path unzippedFile = Paths.get(unzippedDirectory.toString(), fileName); + // Single file data is unzipped directly, without an enclosing folder. Original name is maintained. + Path unzippedFile = Paths.get(unzippedDirectory.toString(), enclosingFolderName, inputFile.getFileName().toString()); unzippedDirectory.toFile().deleteOnExit(); assertFalse(Files.exists(unzippedFile)); @@ -68,11 +69,63 @@ public class ArbitraryCompressionTests extends Common { } @Test - public void testZipMultipleFiles() throws IOException, InterruptedException, DataException { - String fileName = "data"; + public void testZipDirectoryWithSingleFile() throws IOException, InterruptedException, DataException { + String enclosingFolderName = "data"; Path inputDirectory = Files.createTempDirectory("inputDirectory"); Path outputDirectory = Files.createTempDirectory("outputDirectory"); - Path outputFile = Paths.get(outputDirectory.toString(), fileName); + Path outputFile = Paths.get(outputDirectory.toString(), enclosingFolderName); + inputDirectory.toFile().deleteOnExit(); + outputDirectory.toFile().deleteOnExit(); + + Path inputFile = Paths.get(inputDirectory.toString(), "file"); + + // Write random data to a file + byte[] data = new byte[1024]; + new Random().nextBytes(data); + Files.write(inputFile, data, StandardOpenOption.CREATE); + + assertTrue(Files.exists(inputDirectory)); + assertTrue(Files.exists(inputFile)); + assertFalse(Files.exists(outputFile)); + + // Zip... + ZipUtils.zip(inputDirectory.toString(), outputFile.toString(), enclosingFolderName); + + assertTrue(Files.exists(inputDirectory)); + assertTrue(Files.exists(outputFile)); + + // Create paths for unzipping + Path unzippedDirectory = Files.createTempDirectory("unzippedDirectory"); + unzippedDirectory.toFile().deleteOnExit(); + Path unzippedFile = Paths.get(unzippedDirectory.toString(), enclosingFolderName, "file"); + assertFalse(Files.exists(unzippedFile)); + + // Now unzip... + ZipUtils.unzip(outputFile.toString(), unzippedDirectory.toString()); + + // Ensure resulting file exists + assertTrue(Files.exists(unzippedFile)); + + // And make sure they match the original input files + assertTrue(Arrays.equals(Crypto.digest(inputFile.toFile()), Crypto.digest(unzippedFile.toFile()))); + + // Unzipped files are placed within a folder named by the supplied enclosingFolderName + Path unzippedInnerDirectory = Paths.get(unzippedDirectory.toString(), enclosingFolderName); + + // Finally, make sure the directory digests match + ArbitraryDataDigest inputDirectoryDigest = new ArbitraryDataDigest(inputDirectory); + inputDirectoryDigest.compute(); + ArbitraryDataDigest unzippedDirectoryDigest = new ArbitraryDataDigest(unzippedInnerDirectory); + unzippedDirectoryDigest.compute(); + assertEquals(inputDirectoryDigest.getHash58(), unzippedDirectoryDigest.getHash58()); + } + + @Test + public void testZipMultipleFiles() throws IOException, InterruptedException, DataException { + String enclosingFolderName = "data"; + Path inputDirectory = Files.createTempDirectory("inputDirectory"); + Path outputDirectory = Files.createTempDirectory("outputDirectory"); + Path outputFile = Paths.get(outputDirectory.toString(), enclosingFolderName); inputDirectory.toFile().deleteOnExit(); outputDirectory.toFile().deleteOnExit(); @@ -91,7 +144,7 @@ public class ArbitraryCompressionTests extends Common { assertFalse(Files.exists(outputFile)); // Zip... - ZipUtils.zip(inputDirectory.toString(), outputFile.toString(), fileName); + ZipUtils.zip(inputDirectory.toString(), outputFile.toString(), enclosingFolderName); assertTrue(Files.exists(inputDirectory)); assertTrue(Files.exists(outputFile)); @@ -99,8 +152,8 @@ public class ArbitraryCompressionTests extends Common { // Create paths for unzipping Path unzippedDirectory = Files.createTempDirectory("unzippedDirectory"); unzippedDirectory.toFile().deleteOnExit(); - Path unzippedFile1 = Paths.get(unzippedDirectory.toString(), fileName, "file1"); - Path unzippedFile2 = Paths.get(unzippedDirectory.toString(), fileName, "file2"); + Path unzippedFile1 = Paths.get(unzippedDirectory.toString(), enclosingFolderName, "file1"); + Path unzippedFile2 = Paths.get(unzippedDirectory.toString(), enclosingFolderName, "file2"); assertFalse(Files.exists(unzippedFile1)); assertFalse(Files.exists(unzippedFile2)); @@ -115,8 +168,8 @@ public class ArbitraryCompressionTests extends Common { assertTrue(Arrays.equals(Crypto.digest(inputFile1.toFile()), Crypto.digest(unzippedFile1.toFile()))); assertTrue(Arrays.equals(Crypto.digest(inputFile2.toFile()), Crypto.digest(unzippedFile2.toFile()))); - // Unzipped files are placed within a folder named by the supplied fileName - Path unzippedInnerDirectory = Paths.get(unzippedDirectory.toString(), fileName); + // Unzipped files are placed within a folder named by the supplied enclosingFolderName + Path unzippedInnerDirectory = Paths.get(unzippedDirectory.toString(), enclosingFolderName); // Finally, make sure the directory digests match ArbitraryDataDigest inputDirectoryDigest = new ArbitraryDataDigest(inputDirectory); diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java index 3ae87716..c8bcc760 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java @@ -352,7 +352,7 @@ public class ArbitraryDataTests extends Common { // Now build the latest data state for this name ArbitraryDataReader arbitraryDataReader1 = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier); arbitraryDataReader1.loadSynchronously(true); - Path builtFilePath = Paths.get(arbitraryDataReader1.getFilePath().toString(), "data"); + Path builtFilePath = Paths.get(arbitraryDataReader1.getFilePath().toString(), path1.getFileName().toString()); byte[] builtFileDigest = Crypto.digest(builtFilePath.toFile()); // Compare it against the hash of the original file