From 055775b13dbae9485a0610b5f0836271659217b9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Oct 2022 18:54:38 +0000 Subject: [PATCH 1/8] Include a list of files in the QDN metadata. --- .../qortal/arbitrary/ArbitraryDataWriter.java | 33 +++++++-- .../ArbitraryDataTransactionMetadata.java | 31 ++++++++ .../ArbitraryTransactionMetadataTests.java | 72 +++++++++++++++++++ 3 files changed, 129 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 33802d4f..8b1d00c3 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -23,16 +23,13 @@ import javax.crypto.NoSuchPaddingException; import javax.crypto.SecretKey; import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class ArbitraryDataWriter { @@ -50,6 +47,7 @@ public class ArbitraryDataWriter { private final String description; private final List tags; private final Category category; + private List files; private int chunkSize = ArbitraryDataFile.CHUNK_SIZE; @@ -80,12 +78,14 @@ public class ArbitraryDataWriter { this.description = ArbitraryDataTransactionMetadata.limitDescription(description); this.tags = ArbitraryDataTransactionMetadata.limitTags(tags); this.category = category; + this.files = new ArrayList<>(); // Populated in buildFileList() } public void save() throws IOException, DataException, InterruptedException, MissingDataException { try { this.preExecute(); this.validateService(); + this.buildFileList(); this.process(); this.compress(); this.encrypt(); @@ -143,6 +143,24 @@ public class ArbitraryDataWriter { } } + private void buildFileList() throws IOException { + // Single file resources consist of a single element in the file list + boolean isSingleFile = this.filePath.toFile().isFile(); + if (isSingleFile) { + this.files.add(this.filePath.getFileName().toString()); + return; + } + + // Multi file resources require a walk through the directory tree + try (Stream stream = Files.walk(this.filePath)) { + this.files = stream + .filter(Files::isRegularFile) + .map(p -> this.filePath.relativize(p).toString()) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + } + private void process() throws DataException, IOException, MissingDataException { switch (this.method) { @@ -285,6 +303,7 @@ public class ArbitraryDataWriter { metadata.setTags(this.tags); metadata.setCategory(this.category); metadata.setChunks(this.arbitraryDataFile.chunkHashList()); + metadata.setFiles(this.files); metadata.write(); // Create an ArbitraryDataFile from the JSON file (we don't have a signature yet) diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java index 0f8b676b..33da343c 100644 --- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java @@ -19,6 +19,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata { private String description; private List tags; private Category category; + private List files; private static int MAX_TITLE_LENGTH = 80; private static int MAX_DESCRIPTION_LENGTH = 500; @@ -77,6 +78,20 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata { } this.chunks = chunksList; } + + List filesList = new ArrayList<>(); + if (metadata.has("files")) { + JSONArray files = metadata.getJSONArray("files"); + if (files != null) { + for (int i=0; i files) { + this.files = files; + } + + public List getFiles() { + return this.files; + } + public boolean containsChunk(byte[] chunk) { for (byte[] c : this.chunks) { if (Arrays.equals(c, chunk)) { diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java index 357046fe..d8071777 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java @@ -25,9 +25,13 @@ import org.qortal.transaction.RegisterNameTransaction; import org.qortal.utils.Base58; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.Arrays; import java.util.List; +import java.util.Random; import static org.junit.Assert.*; @@ -279,6 +283,74 @@ public class ArbitraryTransactionMetadataTests extends Common { } } + @Test + public void testSingleFileList() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + int chunkSize = 100; + int dataLength = 900; // Actual data length will be longer due to encryption + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Add a few files at multiple levels + byte[] data = new byte[1024]; + new Random().nextBytes(data); + Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength); + Path file1 = Paths.get(path1.toString(), "file.txt"); + + // Create PUT transaction + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, file1, name, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); + + // Check the file list metadata is correct + assertEquals(1, arbitraryDataFile.getMetadata().getFiles().size()); + assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("file.txt")); + } + } + + @Test + public void testMultipleFileList() throws DataException, IOException, MissingDataException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = null; // Not used for this test + Service service = Service.ARBITRARY_DATA; + int chunkSize = 100; + int dataLength = 900; // Actual data length will be longer due to encryption + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Add a few files at multiple levels + byte[] data = new byte[1024]; + new Random().nextBytes(data); + Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength); + Files.write(Paths.get(path1.toString(), "image1.jpg"), data, StandardOpenOption.CREATE); + + Path subdirectory = Paths.get(path1.toString(), "subdirectory"); + Files.createDirectories(subdirectory); + Files.write(Paths.get(subdirectory.toString(), "config.json"), data, StandardOpenOption.CREATE); + + // Create PUT transaction + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); + + // Check the file list metadata is correct + assertEquals(3, arbitraryDataFile.getMetadata().getFiles().size()); + assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("file.txt")); + assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("image1.jpg")); + assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("subdirectory/config.json")); + } + } + @Test public void testExistingCategories() { // Matching categories should be correctly located From 8ad46b6344277a7b6869fc71d205c93280f60412 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Jan 2023 09:58:28 +0000 Subject: [PATCH 2/8] Fixed/removed incorrect comments --- .../test/arbitrary/ArbitraryServiceTests.java | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java index f7738c45..acd86eaa 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java @@ -117,7 +117,6 @@ public class ArbitraryServiceTests extends Common { Service service = Service.GIF_REPOSITORY; assertTrue(service.isValidationRequired()); - // There is an index file in the root assertEquals(ValidationResult.OK, service.validate(path)); } @@ -140,7 +139,6 @@ public class ArbitraryServiceTests extends Common { Service service = Service.GIF_REPOSITORY; assertTrue(service.isValidationRequired()); - // There is an index file in the root assertEquals(ValidationResult.DIRECTORIES_NOT_ALLOWED, service.validate(path)); } @@ -151,7 +149,6 @@ public class ArbitraryServiceTests extends Common { Service service = Service.GIF_REPOSITORY; assertTrue(service.isValidationRequired()); - // There is an index file in the root assertEquals(ValidationResult.MISSING_DATA, service.validate(path)); } @@ -171,7 +168,6 @@ public class ArbitraryServiceTests extends Common { Service service = Service.GIF_REPOSITORY; assertTrue(service.isValidationRequired()); - // There is an index file in the root assertEquals(ValidationResult.INVALID_FILE_EXTENSION, service.validate(path)); } @@ -181,7 +177,7 @@ public class ArbitraryServiceTests extends Common { byte[] data = new byte[1024]; new Random().nextBytes(data); - // Write the data to several files in a temp path + // Write the data a single file in a temp path Path path = Files.createTempDirectory("testValidateQChatAttachment"); path.toFile().deleteOnExit(); Files.write(Paths.get(path.toString(), "document.pdf"), data, StandardOpenOption.CREATE); @@ -189,7 +185,6 @@ public class ArbitraryServiceTests extends Common { Service service = Service.QCHAT_ATTACHMENT; assertTrue(service.isValidationRequired()); - // There is an index file in the root assertEquals(ValidationResult.OK, service.validate(path)); } @@ -199,7 +194,7 @@ public class ArbitraryServiceTests extends Common { byte[] data = new byte[1024]; new Random().nextBytes(data); - // Write the data to several files in a temp path + // Write the data a single file in a temp path Path path = Files.createTempDirectory("testValidateInvalidQChatAttachmentFileExtension"); path.toFile().deleteOnExit(); Files.write(Paths.get(path.toString(), "application.exe"), data, StandardOpenOption.CREATE); @@ -207,7 +202,6 @@ public class ArbitraryServiceTests extends Common { Service service = Service.QCHAT_ATTACHMENT; assertTrue(service.isValidationRequired()); - // There is an index file in the root assertEquals(ValidationResult.INVALID_FILE_EXTENSION, service.validate(path)); } @@ -218,7 +212,6 @@ public class ArbitraryServiceTests extends Common { Service service = Service.QCHAT_ATTACHMENT; assertTrue(service.isValidationRequired()); - // There is an index file in the root assertEquals(ValidationResult.INVALID_FILE_COUNT, service.validate(path)); } @@ -241,7 +234,6 @@ public class ArbitraryServiceTests extends Common { Service service = Service.QCHAT_ATTACHMENT; assertTrue(service.isValidationRequired()); - // There is an index file in the root assertEquals(ValidationResult.DIRECTORIES_NOT_ALLOWED, service.validate(path)); } @@ -260,7 +252,6 @@ public class ArbitraryServiceTests extends Common { Service service = Service.QCHAT_ATTACHMENT; assertTrue(service.isValidationRequired()); - // There is an index file in the root assertEquals(ValidationResult.INVALID_FILE_COUNT, service.validate(path)); } From e31515b4a297283374dd026b61f085732729715b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Jan 2023 10:14:42 +0000 Subject: [PATCH 3/8] Fixed bugs preventing single file GIF repositories and QCHAT attachments from passing validation. --- .../org/qortal/arbitrary/misc/Service.java | 8 +++++ .../test/arbitrary/ArbitraryServiceTests.java | 36 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java index dc2deaeb..96934de2 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Service.java +++ b/src/main/java/org/qortal/arbitrary/misc/Service.java @@ -24,6 +24,10 @@ public enum Service { // Custom validation function to require a single file, with a whitelisted extension int fileCount = 0; File[] files = path.toFile().listFiles(); + // If already a single file, replace the list with one that contains that file only + if (files == null && path.toFile().isFile()) { + files = new File[] { path.toFile() }; + } if (files != null) { for (File file : files) { if (file.isDirectory()) { @@ -80,6 +84,10 @@ public enum Service { // Custom validation function to require .gif files only, and at least 1 int gifCount = 0; File[] files = path.toFile().listFiles(); + // If already a single file, replace the list with one that contains that file only + if (files == null && path.toFile().isFile()) { + files = new File[] { path.toFile() }; + } if (files != null) { for (File file : files) { if (file.isDirectory()) { diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java index acd86eaa..bbd17ab7 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java @@ -120,6 +120,24 @@ public class ArbitraryServiceTests extends Common { assertEquals(ValidationResult.OK, service.validate(path)); } + @Test + public void testValidateSingleFileGifRepository() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to a single file in a temp path + Path path = Files.createTempDirectory("testValidateSingleFileGifRepository"); + path.toFile().deleteOnExit(); + Path imagePath = Paths.get(path.toString(), "image1.gif"); + Files.write(imagePath, data, StandardOpenOption.CREATE); + + Service service = Service.GIF_REPOSITORY; + assertTrue(service.isValidationRequired()); + + assertEquals(ValidationResult.OK, service.validate(imagePath)); + } + @Test public void testValidateMultiLayerGifRepository() throws IOException { // Generate some random data @@ -188,6 +206,24 @@ public class ArbitraryServiceTests extends Common { assertEquals(ValidationResult.OK, service.validate(path)); } + @Test + public void testValidateSingleFileQChatAttachment() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data a single file in a temp path + Path path = Files.createTempDirectory("testValidateSingleFileQChatAttachment"); + path.toFile().deleteOnExit(); + Path filePath = Paths.get(path.toString(), "document.pdf"); + Files.write(filePath, data, StandardOpenOption.CREATE); + + Service service = Service.QCHAT_ATTACHMENT; + assertTrue(service.isValidationRequired()); + + assertEquals(ValidationResult.OK, service.validate(filePath)); + } + @Test public void testValidateInvalidQChatAttachmentFileExtension() throws IOException { // Generate some random data From c3f19ea0c1c52507dbaa0872de506c3d408cabd9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Jan 2023 10:21:05 +0000 Subject: [PATCH 4/8] Don't allow the custom validation methods to evade superclass validation. --- .../org/qortal/arbitrary/misc/Service.java | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java index 96934de2..0aeb99ed 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Service.java +++ b/src/main/java/org/qortal/arbitrary/misc/Service.java @@ -20,7 +20,12 @@ public enum Service { ARBITRARY_DATA(100, false, null, null), QCHAT_ATTACHMENT(120, true, 1024*1024L, null) { @Override - public ValidationResult validate(Path path) { + public ValidationResult validate(Path path) throws IOException { + ValidationResult superclassResult = super.validate(path); + if (superclassResult != ValidationResult.OK) { + return superclassResult; + } + // Custom validation function to require a single file, with a whitelisted extension int fileCount = 0; File[] files = path.toFile().listFiles(); @@ -49,7 +54,12 @@ public enum Service { }, WEBSITE(200, true, null, null) { @Override - public ValidationResult validate(Path path) { + public ValidationResult validate(Path path) throws IOException { + ValidationResult superclassResult = super.validate(path); + if (superclassResult != ValidationResult.OK) { + return superclassResult; + } + // Custom validation function to require an index HTML file in the root directory List fileNames = ArbitraryDataRenderer.indexFiles(); String[] files = path.toFile().list(); @@ -80,7 +90,12 @@ public enum Service { METADATA(1100, false, null, null), GIF_REPOSITORY(1200, true, 25*1024*1024L, null) { @Override - public ValidationResult validate(Path path) { + public ValidationResult validate(Path path) throws IOException { + ValidationResult superclassResult = super.validate(path); + if (superclassResult != ValidationResult.OK) { + return superclassResult; + } + // Custom validation function to require .gif files only, and at least 1 int gifCount = 0; File[] files = path.toFile().listFiles(); From 1f7fec6251d095519fa26d6ae65b6e72995d4e43 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 20 Jan 2023 10:40:20 +0000 Subject: [PATCH 5/8] Exclude .qortal directory in validation functions, as it was incorrectly failing with "DIRECTORIES_NOT_ALLOWED". --- .../org/qortal/arbitrary/misc/Service.java | 6 ++ .../test/arbitrary/ArbitraryServiceTests.java | 98 +++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java index 0aeb99ed..5ddccbe5 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Service.java +++ b/src/main/java/org/qortal/arbitrary/misc/Service.java @@ -35,6 +35,9 @@ public enum Service { } if (files != null) { for (File file : files) { + if (file.getName().equals(".qortal")) { + continue; + } if (file.isDirectory()) { return ValidationResult.DIRECTORIES_NOT_ALLOWED; } @@ -105,6 +108,9 @@ public enum Service { } if (files != null) { for (File file : files) { + if (file.getName().equals(".qortal")) { + continue; + } if (file.isDirectory()) { return ValidationResult.DIRECTORIES_NOT_ALLOWED; } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java index bbd17ab7..96843876 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java @@ -1,11 +1,26 @@ package org.qortal.test.arbitrary; +import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.Before; import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.arbitrary.ArbitraryDataFile; +import org.qortal.arbitrary.ArbitraryDataReader; +import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.arbitrary.misc.Service; import org.qortal.arbitrary.misc.Service.ValidationResult; +import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.RegisterNameTransactionData; import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.ArbitraryUtils; import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.utils.Base58; import java.io.IOException; import java.nio.file.Files; @@ -189,6 +204,48 @@ public class ArbitraryServiceTests extends Common { assertEquals(ValidationResult.INVALID_FILE_EXTENSION, service.validate(path)); } + @Test + public void testValidatePublishedGifRepository() throws IOException, DataException, MissingDataException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateGifRepository"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "image1.gif"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "image2.gif"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "image3.gif"), data, StandardOpenOption.CREATE); + + Service service = Service.GIF_REPOSITORY; + assertTrue(service.isValidationRequired()); + + assertEquals(ValidationResult.OK, service.validate(path)); + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = "test_identifier"; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Set difficulty to 1 + FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 1, true); + + // Create PUT transaction + ArbitraryUtils.createAndMintTxn(repository, publicKey58, path, name, identifier, ArbitraryTransactionData.Method.PUT, service, alice); + + // Build the latest data state for this name, and no exceptions should be thrown because validation passes + ArbitraryDataReader arbitraryDataReader1a = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); + arbitraryDataReader1a.loadSynchronously(true); + } + } + @Test public void testValidateQChatAttachment() throws IOException { // Generate some random data @@ -291,4 +348,45 @@ public class ArbitraryServiceTests extends Common { assertEquals(ValidationResult.INVALID_FILE_COUNT, service.validate(path)); } + @Test + public void testValidatePublishedQChatAttachment() throws IOException, DataException, MissingDataException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data a single file in a temp path + Path path = Files.createTempDirectory("testValidateSingleFileQChatAttachment"); + path.toFile().deleteOnExit(); + Path filePath = Paths.get(path.toString(), "document.pdf"); + Files.write(filePath, data, StandardOpenOption.CREATE); + + Service service = Service.QCHAT_ATTACHMENT; + assertTrue(service.isValidationRequired()); + + assertEquals(ValidationResult.OK, service.validate(filePath)); + + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = "test_identifier"; + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Set difficulty to 1 + FieldUtils.writeField(ArbitraryDataManager.getInstance(), "powDifficulty", 1, true); + + // Create PUT transaction + ArbitraryUtils.createAndMintTxn(repository, publicKey58, filePath, name, identifier, ArbitraryTransactionData.Method.PUT, service, alice); + + // Build the latest data state for this name, and no exceptions should be thrown because validation passes + ArbitraryDataReader arbitraryDataReader1a = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); + arbitraryDataReader1a.loadSynchronously(true); + } + } + } \ No newline at end of file From 9f30571b12a3463465547286b47083ee1417b271 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 22 Jan 2023 15:58:53 +0000 Subject: [PATCH 6/8] Use a filename without an extension when publishing data from a string (instead of .tmp) --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 978183c0..25b968f1 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1128,7 +1128,7 @@ public class ArbitraryResource { if (path == null) { // See if we have a string instead if (string != null) { - File tempFile = File.createTempFile("qortal-", ".tmp"); + File tempFile = File.createTempFile("qortal-", ""); tempFile.deleteOnExit(); BufferedWriter writer = new BufferedWriter(new FileWriter(tempFile.toPath().toString())); writer.write(string); @@ -1138,7 +1138,7 @@ public class ArbitraryResource { } // ... or base64 encoded raw data else if (base64 != null) { - File tempFile = File.createTempFile("qortal-", ".tmp"); + File tempFile = File.createTempFile("qortal-", ""); tempFile.deleteOnExit(); Files.write(tempFile.toPath(), Base64.decode(base64)); path = tempFile.toPath().toString(); From 6196841609ce1b11bc10ae653928e0d40e07f11f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 22 Jan 2023 15:59:16 +0000 Subject: [PATCH 7/8] Allow files without extensions in QCHAT_ATTACHMENT validation. --- src/main/java/org/qortal/arbitrary/misc/Service.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java index 5ddccbe5..01419d2f 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Service.java +++ b/src/main/java/org/qortal/arbitrary/misc/Service.java @@ -42,7 +42,8 @@ public enum Service { return ValidationResult.DIRECTORIES_NOT_ALLOWED; } final String extension = FilenameUtils.getExtension(file.getName()).toLowerCase(); - final List allowedExtensions = Arrays.asList("zip", "pdf", "txt", "odt", "ods", "doc", "docx", "xls", "xlsx", "ppt", "pptx"); + // We must allow blank file extensions because these are used by data published from a plaintext or base64-encoded string + final List allowedExtensions = Arrays.asList("zip", "pdf", "txt", "odt", "ods", "doc", "docx", "xls", "xlsx", "ppt", "pptx", ""); if (extension == null || !allowedExtensions.contains(extension)) { return ValidationResult.INVALID_FILE_EXTENSION; } From 1d568fa46245cac370a8f7a79cfb887033a52f93 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 22 Jan 2023 16:29:23 +0000 Subject: [PATCH 8/8] Return file lists via /arbitrary/metadata/* endpoints, but exclude it from /arbitrary/resources/* endpoints. --- .../api/resource/ArbitraryResource.java | 4 ++-- .../arbitrary/ArbitraryResourceMetadata.java | 20 ++++++++++++++---- .../ArbitraryTransactionMetadataTests.java | 21 +++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 25b968f1..0df81d9b 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -719,7 +719,7 @@ public class ArbitraryResource { try { ArbitraryDataTransactionMetadata transactionMetadata = ArbitraryMetadataManager.getInstance().fetchMetadata(resource, false); if (transactionMetadata != null) { - ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata); + ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata, true); if (resourceMetadata != null) { return resourceMetadata; } @@ -1288,7 +1288,7 @@ public class ArbitraryResource { ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ResourceIdType.NAME, resourceInfo.service, resourceInfo.identifier); ArbitraryDataTransactionMetadata transactionMetadata = resource.getLatestTransactionMetadata(); - ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata); + ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata, false); if (resourceMetadata != null) { resourceInfo.metadata = resourceMetadata; } diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java index e2bcaf56..497e214f 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java @@ -15,22 +15,24 @@ public class ArbitraryResourceMetadata { private List tags; private Category category; private String categoryName; + private List files; public ArbitraryResourceMetadata() { } - public ArbitraryResourceMetadata(String title, String description, List tags, Category category) { + public ArbitraryResourceMetadata(String title, String description, List tags, Category category, List files) { this.title = title; this.description = description; this.tags = tags; this.category = category; + this.files = files; if (category != null) { this.categoryName = category.getName(); } } - public static ArbitraryResourceMetadata fromTransactionMetadata(ArbitraryDataTransactionMetadata transactionMetadata) { + public static ArbitraryResourceMetadata fromTransactionMetadata(ArbitraryDataTransactionMetadata transactionMetadata, boolean includeFileList) { if (transactionMetadata == null) { return null; } @@ -39,10 +41,20 @@ public class ArbitraryResourceMetadata { List tags = transactionMetadata.getTags(); Category category = transactionMetadata.getCategory(); - if (title == null && description == null && tags == null && category == null) { + // We don't always want to include the file list as it can be too verbose + List files = null; + if (includeFileList) { + files = transactionMetadata.getFiles(); + } + + if (title == null && description == null && tags == null && category == null && files == null) { return null; } - return new ArbitraryResourceMetadata(title, description, tags, category); + return new ArbitraryResourceMetadata(title, description, tags, category, files); + } + + public List getFiles() { + return this.files; } } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java index d8071777..5d28568d 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java @@ -12,6 +12,7 @@ import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.arbitrary.misc.Category; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.data.arbitrary.ArbitraryResourceMetadata; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.RegisterNameTransactionData; import org.qortal.repository.DataException; @@ -311,6 +312,15 @@ public class ArbitraryTransactionMetadataTests extends Common { // Check the file list metadata is correct assertEquals(1, arbitraryDataFile.getMetadata().getFiles().size()); assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("file.txt")); + + // Ensure the file list can be read back out again, when specified to be included + ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(arbitraryDataFile.getMetadata(), true); + assertTrue(resourceMetadata.getFiles().contains("file.txt")); + + // Ensure it's not returned when specified to be excluded + // The entire object will be null because there is no metadata + ArbitraryResourceMetadata resourceMetadataSimple = ArbitraryResourceMetadata.fromTransactionMetadata(arbitraryDataFile.getMetadata(), false); + assertNull(resourceMetadataSimple); } } @@ -348,6 +358,17 @@ public class ArbitraryTransactionMetadataTests extends Common { assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("file.txt")); assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("image1.jpg")); assertTrue(arbitraryDataFile.getMetadata().getFiles().contains("subdirectory/config.json")); + + // Ensure the file list can be read back out again, when specified to be included + ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(arbitraryDataFile.getMetadata(), true); + assertTrue(resourceMetadata.getFiles().contains("file.txt")); + assertTrue(resourceMetadata.getFiles().contains("image1.jpg")); + assertTrue(resourceMetadata.getFiles().contains("subdirectory/config.json")); + + // Ensure it's not returned when specified to be excluded + // The entire object will be null because there is no metadata + ArbitraryResourceMetadata resourceMetadataSimple = ArbitraryResourceMetadata.fromTransactionMetadata(arbitraryDataFile.getMetadata(), false); + assertNull(resourceMetadataSimple); } }