From f296d5138b99c7b5ee253421a6e35f6c91fee6e0 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 21 Jan 2022 21:14:28 +0000 Subject: [PATCH] Allow metadata to optionally be included with any arbitrary resource. --- .../api/resource/ArbitraryResource.java | 63 ++++++++++++++--- .../qortal/api/resource/RenderResource.java | 4 +- .../qortal/arbitrary/ArbitraryDataFile.java | 4 ++ .../ArbitraryDataTransactionBuilder.java | 18 ++++- .../qortal/arbitrary/ArbitraryDataWriter.java | 52 ++++++++++++-- .../ArbitraryDataTransactionMetadata.java | 69 ++++++++++++++++++- .../ArbitraryDataStoragePolicyTests.java | 3 +- .../ArbitraryTransactionMetadataTests.java | 49 +++++++++++++ .../qortal/test/common/ArbitraryUtils.java | 12 +++- 9 files changed, 250 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 82618152..8d1dfee9 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -642,6 +642,10 @@ public class ArbitraryResource { public String post(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("service") String serviceString, @PathParam("name") String name, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") String tags, + @QueryParam("category") String category, String path) { Security.checkApiCallAllowed(request); @@ -649,7 +653,8 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Path not supplied"); } - return this.upload(Service.valueOf(serviceString), name, null, path, null, null, false); + return this.upload(Service.valueOf(serviceString), name, null, path, null, null, false, + title, description, tags, category); } @POST @@ -682,6 +687,10 @@ public class ArbitraryResource { @PathParam("service") String serviceString, @PathParam("name") String name, @PathParam("identifier") String identifier, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") String tags, + @QueryParam("category") String category, String path) { Security.checkApiCallAllowed(request); @@ -689,7 +698,8 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Path not supplied"); } - return this.upload(Service.valueOf(serviceString), name, identifier, path, null, null, false); + return this.upload(Service.valueOf(serviceString), name, identifier, path, null, null, false, + title, description, tags, category); } @@ -723,6 +733,10 @@ public class ArbitraryResource { public String postBase64EncodedData(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("service") String serviceString, @PathParam("name") String name, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") String tags, + @QueryParam("category") String category, String base64) { Security.checkApiCallAllowed(request); @@ -730,7 +744,8 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied"); } - return this.upload(Service.valueOf(serviceString), name, null, null, null, base64, false); + return this.upload(Service.valueOf(serviceString), name, null, null, null, base64, false, + title, description, tags, category); } @POST @@ -761,6 +776,10 @@ public class ArbitraryResource { @PathParam("service") String serviceString, @PathParam("name") String name, @PathParam("identifier") String identifier, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") String tags, + @QueryParam("category") String category, String base64) { Security.checkApiCallAllowed(request); @@ -768,7 +787,8 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied"); } - return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64, false); + return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64, false, + title, description, tags, category); } @@ -801,6 +821,10 @@ public class ArbitraryResource { public String postZippedData(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("service") String serviceString, @PathParam("name") String name, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") String tags, + @QueryParam("category") String category, String base64Zip) { Security.checkApiCallAllowed(request); @@ -808,7 +832,8 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied"); } - return this.upload(Service.valueOf(serviceString), name, null, null, null, base64Zip, true); + return this.upload(Service.valueOf(serviceString), name, null, null, null, base64Zip, true, + title, description, tags, category); } @POST @@ -839,6 +864,10 @@ public class ArbitraryResource { @PathParam("service") String serviceString, @PathParam("name") String name, @PathParam("identifier") String identifier, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") String tags, + @QueryParam("category") String category, String base64Zip) { Security.checkApiCallAllowed(request); @@ -846,7 +875,8 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied"); } - return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64Zip, true); + return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64Zip, true, + title, description, tags, category); } @@ -882,6 +912,10 @@ public class ArbitraryResource { public String postString(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("service") String serviceString, @PathParam("name") String name, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") String tags, + @QueryParam("category") String category, String string) { Security.checkApiCallAllowed(request); @@ -889,7 +923,8 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data string not supplied"); } - return this.upload(Service.valueOf(serviceString), name, null, null, string, null, false); + return this.upload(Service.valueOf(serviceString), name, null, null, string, null, false, + title, description, tags, category); } @POST @@ -922,6 +957,10 @@ public class ArbitraryResource { @PathParam("service") String serviceString, @PathParam("name") String name, @PathParam("identifier") String identifier, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") String tags, + @QueryParam("category") String category, String string) { Security.checkApiCallAllowed(request); @@ -929,13 +968,16 @@ public class ArbitraryResource { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data string not supplied"); } - return this.upload(Service.valueOf(serviceString), name, identifier, null, string, null, false); + return this.upload(Service.valueOf(serviceString), name, identifier, null, string, null, false, + title, description, tags, category); } // Shared methods - private String upload(Service service, String name, String identifier, String path, String string, String base64, boolean zipped) { + private String upload(Service service, String name, String identifier, + String path, String string, String base64, boolean zipped, + String title, String description, String tags, String category) { // Fetch public key from registered name try (final Repository repository = RepositoryManager.getRepository()) { NameData nameData = repository.getNameRepository().fromName(name); @@ -999,7 +1041,8 @@ public class ArbitraryResource { try { ArbitraryDataTransactionBuilder transactionBuilder = new ArbitraryDataTransactionBuilder( - repository, publicKey58, Paths.get(path), name, null, service, identifier + repository, publicKey58, Paths.get(path), name, null, service, identifier, + title, description, tags, category ); transactionBuilder.build(); diff --git a/src/main/java/org/qortal/api/resource/RenderResource.java b/src/main/java/org/qortal/api/resource/RenderResource.java index 97411e54..7294da0c 100644 --- a/src/main/java/org/qortal/api/resource/RenderResource.java +++ b/src/main/java/org/qortal/api/resource/RenderResource.java @@ -74,7 +74,9 @@ public class RenderResource { Method method = Method.PUT; Compression compression = Compression.ZIP; - ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), null, Service.WEBSITE, null, method, compression); + ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), + null, Service.WEBSITE, null, method, compression, + null, null, null, null); try { arbitraryDataWriter.save(); } catch (IOException | DataException | InterruptedException | MissingDataException e) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 1eaeda3c..e65e7b3c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -728,6 +728,10 @@ public class ArbitraryDataFile { this.loadMetadata(); } + public ArbitraryDataTransactionMetadata getMetadata() { + return this.metadata; + } + @Override public String toString() { return this.shortHash58(); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java index 442461e1..933a5d66 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java @@ -51,13 +51,20 @@ public class ArbitraryDataTransactionBuilder { private final String identifier; private final Repository repository; + // Metadata + private final String title; + private final String description; + private final String tags; + private final String category; + private int chunkSize = ArbitraryDataFile.CHUNK_SIZE; private ArbitraryTransactionData arbitraryTransactionData; private ArbitraryDataFile arbitraryDataFile; public ArbitraryDataTransactionBuilder(Repository repository, String publicKey58, Path path, String name, - Method method, Service service, String identifier) { + Method method, Service service, String identifier, + String title, String description, String tags, String category) { this.repository = repository; this.publicKey58 = publicKey58; this.path = path; @@ -70,6 +77,12 @@ public class ArbitraryDataTransactionBuilder { identifier = null; } this.identifier = identifier; + + // Metadata (optional) + this.title = title; + this.description = description; + this.tags = tags; + this.category = category; } public void build() throws DataException { @@ -200,7 +213,8 @@ public class ArbitraryDataTransactionBuilder { // FUTURE? Use zip compression for directories, or no compression for single files // Compression compression = (path.toFile().isDirectory()) ? Compression.ZIP : Compression.NONE; - ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, compression); + ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, + compression, title, description, tags, category); try { arbitraryDataWriter.setChunkSize(this.chunkSize); arbitraryDataWriter.save(); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 39ba4ade..4a5d10af 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -28,6 +28,7 @@ import java.nio.file.Paths; import java.security.InvalidAlgorithmParameterException; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; +import java.util.Objects; public class ArbitraryDataWriter { @@ -40,6 +41,12 @@ public class ArbitraryDataWriter { private final Method method; private final Compression compression; + // Metadata + private final String title; + private final String description; + private final String tags; + private final String category; + private int chunkSize = ArbitraryDataFile.CHUNK_SIZE; private SecretKey aesKey; @@ -50,7 +57,8 @@ public class ArbitraryDataWriter { private Path compressedPath; private Path encryptedPath; - public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression) { + public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression, + String title, String description, String tags, String category) { this.filePath = filePath; this.name = name; this.service = service; @@ -62,6 +70,12 @@ public class ArbitraryDataWriter { identifier = null; } this.identifier = identifier; + + // Metadata (optional) + this.title = title; + this.description = description; + this.tags = tags; + this.category = category; } public void save() throws IOException, DataException, InterruptedException, MissingDataException { @@ -258,12 +272,16 @@ public class ArbitraryDataWriter { private void createMetadataFile() throws IOException, DataException { // If we have at least one chunk, we need to create an index file containing their hashes - if (this.arbitraryDataFile.chunkCount() > 1) { + if (this.needsMetadataFile()) { // Create the JSON file Path chunkFilePath = Paths.get(this.workingPath.toString(), "metadata.json"); - ArbitraryDataTransactionMetadata chunkMetadata = new ArbitraryDataTransactionMetadata(chunkFilePath); - chunkMetadata.setChunks(this.arbitraryDataFile.chunkHashList()); - chunkMetadata.write(); + ArbitraryDataTransactionMetadata metadata = new ArbitraryDataTransactionMetadata(chunkFilePath); + metadata.setTitle(this.title); + metadata.setDescription(this.description); + metadata.setTags(this.tags); + metadata.setCategory(this.category); + metadata.setChunks(this.arbitraryDataFile.chunkHashList()); + metadata.write(); // Create an ArbitraryDataFile from the JSON file (we don't have a signature yet) ArbitraryDataFile metadataFile = ArbitraryDataFile.fromPath(chunkFilePath, null); @@ -308,6 +326,20 @@ public class ArbitraryDataWriter { throw new DataException(String.format("Missing chunk %s in metadata file", Base58.encode(chunk))); } } + + // Check that the metadata is correct + if (!Objects.equals(metadata.getTitle(), this.title)) { + throw new DataException("Metadata mismatch: title"); + } + if (!Objects.equals(metadata.getDescription(), this.description)) { + throw new DataException("Metadata mismatch: description"); + } + if (!Objects.equals(metadata.getTags(), this.tags)) { + throw new DataException("Metadata mismatch: tags"); + } + if (!Objects.equals(metadata.getCategory(), this.category)) { + throw new DataException("Metadata mismatch: category"); + } } } @@ -330,6 +362,16 @@ public class ArbitraryDataWriter { } } + private boolean needsMetadataFile() { + if (this.arbitraryDataFile.chunkCount() > 1) { + return true; + } + if (this.title != null || this.description != null || this.tags != null || this.category != null) { + return true; + } + return false; + } + public ArbitraryDataFile getArbitraryDataFile() { return this.arbitraryDataFile; diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java index abd47ec9..03dec5b4 100644 --- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java @@ -13,6 +13,10 @@ import java.util.List; public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata { private List chunks; + private String title; + private String description; + private String tags; + private String category; public ArbitraryDataTransactionMetadata(Path filePath) { super(filePath); @@ -25,10 +29,24 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata { throw new DataException("Transaction metadata JSON string is null"); } + JSONObject metadata = new JSONObject(this.jsonString); + + if (metadata.has("title")) { + this.title = metadata.getString("title"); + } + if (metadata.has("description")) { + this.description = metadata.getString("description"); + } + if (metadata.has("tags")) { + this.tags = metadata.getString("tags"); + } + if (metadata.has("category")) { + this.category = metadata.getString("category"); + } + List chunksList = new ArrayList<>(); - JSONObject cache = new JSONObject(this.jsonString); - if (cache.has("chunks")) { - JSONArray chunks = cache.getJSONArray("chunks"); + if (metadata.has("chunks")) { + JSONArray chunks = metadata.getJSONArray("chunks"); if (chunks != null) { for (int i=0; i