AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Allow metadata to optionally be included with any arbitrary resource.

Browse Source
This commit is contained in:
CalDescent 2022-01-21 21:14:28 +00:00
parent b30445c5f8
commit f296d5138b
9 changed files with 250 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -642,6 +642,10 @@ public class ArbitraryResource {
public String post(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") String serviceString,
@PathParam("name") String name,
@QueryParam("title") String title,
@QueryParam("description") String description,
@QueryParam("tags") String tags,
@QueryParam("category") String category,
String path) {
Security.checkApiCallAllowed(request);
@ -649,7 +653,8 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Path not supplied");
}
return this.upload(Service.valueOf(serviceString), name, null, path, null, null, false);
return this.upload(Service.valueOf(serviceString), name, null, path, null, null, false,
title, description, tags, category);
}
@POST
@ -682,6 +687,10 @@ public class ArbitraryResource {
@PathParam("service") String serviceString,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("title") String title,
@QueryParam("description") String description,
@QueryParam("tags") String tags,
@QueryParam("category") String category,
String path) {
Security.checkApiCallAllowed(request);
@ -689,7 +698,8 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Path not supplied");
}
return this.upload(Service.valueOf(serviceString), name, identifier, path, null, null, false);
return this.upload(Service.valueOf(serviceString), name, identifier, path, null, null, false,
title, description, tags, category);
}
@ -723,6 +733,10 @@ public class ArbitraryResource {
public String postBase64EncodedData(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") String serviceString,
@PathParam("name") String name,
@QueryParam("title") String title,
@QueryParam("description") String description,
@QueryParam("tags") String tags,
@QueryParam("category") String category,
String base64) {
Security.checkApiCallAllowed(request);
@ -730,7 +744,8 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied");
}
return this.upload(Service.valueOf(serviceString), name, null, null, null, base64, false);
return this.upload(Service.valueOf(serviceString), name, null, null, null, base64, false,
title, description, tags, category);
}
@POST
@ -761,6 +776,10 @@ public class ArbitraryResource {
@PathParam("service") String serviceString,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("title") String title,
@QueryParam("description") String description,
@QueryParam("tags") String tags,
@QueryParam("category") String category,
String base64) {
Security.checkApiCallAllowed(request);
@ -768,7 +787,8 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied");
}
return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64, false);
return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64, false,
title, description, tags, category);
}
@ -801,6 +821,10 @@ public class ArbitraryResource {
public String postZippedData(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") String serviceString,
@PathParam("name") String name,
@QueryParam("title") String title,
@QueryParam("description") String description,
@QueryParam("tags") String tags,
@QueryParam("category") String category,
String base64Zip) {
Security.checkApiCallAllowed(request);
@ -808,7 +832,8 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied");
}
return this.upload(Service.valueOf(serviceString), name, null, null, null, base64Zip, true);
return this.upload(Service.valueOf(serviceString), name, null, null, null, base64Zip, true,
title, description, tags, category);
}
@POST
@ -839,6 +864,10 @@ public class ArbitraryResource {
@PathParam("service") String serviceString,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("title") String title,
@QueryParam("description") String description,
@QueryParam("tags") String tags,
@QueryParam("category") String category,
String base64Zip) {
Security.checkApiCallAllowed(request);
@ -846,7 +875,8 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied");
}
return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64Zip, true);
return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64Zip, true,
title, description, tags, category);
}
@ -882,6 +912,10 @@ public class ArbitraryResource {
public String postString(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") String serviceString,
@PathParam("name") String name,
@QueryParam("title") String title,
@QueryParam("description") String description,
@QueryParam("tags") String tags,
@QueryParam("category") String category,
String string) {
Security.checkApiCallAllowed(request);
@ -889,7 +923,8 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data string not supplied");
}
return this.upload(Service.valueOf(serviceString), name, null, null, string, null, false);
return this.upload(Service.valueOf(serviceString), name, null, null, string, null, false,
title, description, tags, category);
}
@POST
@ -922,6 +957,10 @@ public class ArbitraryResource {
@PathParam("service") String serviceString,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("title") String title,
@QueryParam("description") String description,
@QueryParam("tags") String tags,
@QueryParam("category") String category,
String string) {
Security.checkApiCallAllowed(request);
@ -929,13 +968,16 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data string not supplied");
}
return this.upload(Service.valueOf(serviceString), name, identifier, null, string, null, false);
return this.upload(Service.valueOf(serviceString), name, identifier, null, string, null, false,
title, description, tags, category);
}
// Shared methods
private String upload(Service service, String name, String identifier, String path, String string, String base64, boolean zipped) {
private String upload(Service service, String name, String identifier,
String path, String string, String base64, boolean zipped,
String title, String description, String tags, String category) {
// Fetch public key from registered name
try (final Repository repository = RepositoryManager.getRepository()) {
NameData nameData = repository.getNameRepository().fromName(name);
@ -999,7 +1041,8 @@ public class ArbitraryResource {
try {
ArbitraryDataTransactionBuilder transactionBuilder = new ArbitraryDataTransactionBuilder(
repository, publicKey58, Paths.get(path), name, null, service, identifier
repository, publicKey58, Paths.get(path), name, null, service, identifier,
title, description, tags, category
);
transactionBuilder.build();

4
src/main/java/org/qortal/api/resource/RenderResource.java
View File

@ -74,7 +74,9 @@ public class RenderResource {
Method method = Method.PUT;
Compression compression = Compression.ZIP;
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), null, Service.WEBSITE, null, method, compression);
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath),
null, Service.WEBSITE, null, method, compression,
null, null, null, null);
try {
arbitraryDataWriter.save();
} catch (IOException | DataException | InterruptedException | MissingDataException e) {

4
src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java
View File

@ -728,6 +728,10 @@ public class ArbitraryDataFile {
this.loadMetadata();
}
public ArbitraryDataTransactionMetadata getMetadata() {
return this.metadata;
}
@Override
public String toString() {
return this.shortHash58();

18
src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java
View File

@ -51,13 +51,20 @@ public class ArbitraryDataTransactionBuilder {
private final String identifier;
private final Repository repository;
// Metadata
private final String title;
private final String description;
private final String tags;
private final String category;
private int chunkSize = ArbitraryDataFile.CHUNK_SIZE;
private ArbitraryTransactionData arbitraryTransactionData;
private ArbitraryDataFile arbitraryDataFile;
public ArbitraryDataTransactionBuilder(Repository repository, String publicKey58, Path path, String name,
Method method, Service service, String identifier) {
Method method, Service service, String identifier,
String title, String description, String tags, String category) {
this.repository = repository;
this.publicKey58 = publicKey58;
this.path = path;
@ -70,6 +77,12 @@ public class ArbitraryDataTransactionBuilder {
identifier = null;
}
this.identifier = identifier;
// Metadata (optional)
this.title = title;
this.description = description;
this.tags = tags;
this.category = category;
}
public void build() throws DataException {
@ -200,7 +213,8 @@ public class ArbitraryDataTransactionBuilder {
// FUTURE? Use zip compression for directories, or no compression for single files
// Compression compression = (path.toFile().isDirectory()) ? Compression.ZIP : Compression.NONE;
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, compression);
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method,
compression, title, description, tags, category);
try {
arbitraryDataWriter.setChunkSize(this.chunkSize);
arbitraryDataWriter.save();

52
src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java
View File

@ -28,6 +28,7 @@ import java.nio.file.Paths;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.Objects;
public class ArbitraryDataWriter {
@ -40,6 +41,12 @@ public class ArbitraryDataWriter {
private final Method method;
private final Compression compression;
// Metadata
private final String title;
private final String description;
private final String tags;
private final String category;
private int chunkSize = ArbitraryDataFile.CHUNK_SIZE;
private SecretKey aesKey;
@ -50,7 +57,8 @@ public class ArbitraryDataWriter {
private Path compressedPath;
private Path encryptedPath;
public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression) {
public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression,
String title, String description, String tags, String category) {
this.filePath = filePath;
this.name = name;
this.service = service;
@ -62,6 +70,12 @@ public class ArbitraryDataWriter {
identifier = null;
}
this.identifier = identifier;
// Metadata (optional)
this.title = title;
this.description = description;
this.tags = tags;
this.category = category;
}
public void save() throws IOException, DataException, InterruptedException, MissingDataException {
@ -258,12 +272,16 @@ public class ArbitraryDataWriter {
private void createMetadataFile() throws IOException, DataException {
// If we have at least one chunk, we need to create an index file containing their hashes
if (this.arbitraryDataFile.chunkCount() > 1) {
if (this.needsMetadataFile()) {
// Create the JSON file
Path chunkFilePath = Paths.get(this.workingPath.toString(), "metadata.json");
ArbitraryDataTransactionMetadata chunkMetadata = new ArbitraryDataTransactionMetadata(chunkFilePath);
chunkMetadata.setChunks(this.arbitraryDataFile.chunkHashList());
chunkMetadata.write();
ArbitraryDataTransactionMetadata metadata = new ArbitraryDataTransactionMetadata(chunkFilePath);
metadata.setTitle(this.title);
metadata.setDescription(this.description);
metadata.setTags(this.tags);
metadata.setCategory(this.category);
metadata.setChunks(this.arbitraryDataFile.chunkHashList());
metadata.write();
// Create an ArbitraryDataFile from the JSON file (we don't have a signature yet)
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromPath(chunkFilePath, null);
@ -308,6 +326,20 @@ public class ArbitraryDataWriter {
throw new DataException(String.format("Missing chunk %s in metadata file", Base58.encode(chunk)));
}
}
// Check that the metadata is correct
if (!Objects.equals(metadata.getTitle(), this.title)) {
throw new DataException("Metadata mismatch: title");
}
if (!Objects.equals(metadata.getDescription(), this.description)) {
throw new DataException("Metadata mismatch: description");
}
if (!Objects.equals(metadata.getTags(), this.tags)) {
throw new DataException("Metadata mismatch: tags");
}
if (!Objects.equals(metadata.getCategory(), this.category)) {
throw new DataException("Metadata mismatch: category");
}
}
}
@ -330,6 +362,16 @@ public class ArbitraryDataWriter {
}
}
private boolean needsMetadataFile() {
if (this.arbitraryDataFile.chunkCount() > 1) {
return true;
}
if (this.title != null || this.description != null || this.tags != null || this.category != null) {
return true;
}
return false;
}
public ArbitraryDataFile getArbitraryDataFile() {
return this.arbitraryDataFile;

69
src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java
View File

@ -13,6 +13,10 @@ import java.util.List;
public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
private List<byte[]> chunks;
private String title;
private String description;
private String tags;
private String category;
public ArbitraryDataTransactionMetadata(Path filePath) {
super(filePath);
@ -25,10 +29,24 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
throw new DataException("Transaction metadata JSON string is null");
}
JSONObject metadata = new JSONObject(this.jsonString);
if (metadata.has("title")) {
this.title = metadata.getString("title");
}
if (metadata.has("description")) {
this.description = metadata.getString("description");
}
if (metadata.has("tags")) {
this.tags = metadata.getString("tags");
}
if (metadata.has("category")) {
this.category = metadata.getString("category");
}
List<byte[]> chunksList = new ArrayList<>();
JSONObject cache = new JSONObject(this.jsonString);
if (cache.has("chunks")) {
JSONArray chunks = cache.getJSONArray("chunks");
if (metadata.has("chunks")) {
JSONArray chunks = metadata.getJSONArray("chunks");
if (chunks != null) {
for (int i=0; i<chunks.length(); i++) {
String chunk = chunks.getString(i);
@ -45,6 +63,19 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
protected void buildJson() {
JSONObject outer = new JSONObject();
if (this.title != null && !this.title.isEmpty()) {
outer.put("title", this.title);
}
if (this.description != null && !this.description.isEmpty()) {
outer.put("description", this.description);
}
if (this.tags != null && !this.tags.isEmpty()) {
outer.put("tags", this.tags);
}
if (this.category != null && !this.category.isEmpty()) {
outer.put("category", this.category);
}
JSONArray chunks = new JSONArray();
if (this.chunks != null) {
for (byte[] chunk : this.chunks) {
@ -66,6 +97,38 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
return this.chunks;
}
public void setTitle(String title) {
this.title = title;
}
public String getTitle() {
return this.title;
}
public void setDescription(String description) {
this.description = description;
}
public String getDescription() {
return this.description;
}
public void setTags(String tags) {
this.tags = tags;
}
public String getTags() {
return this.tags;
}
public void setCategory(String category) {
this.category = category;
}
public String getCategory() {
return this.category;
}
public boolean containsChunk(byte[] chunk) {
for (byte[] c : this.chunks) {
if (Arrays.equals(c, chunk)) {

3
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java
View File

@ -234,7 +234,8 @@ public class ArbitraryDataStoragePolicyTests extends Common {
Path path = Paths.get("src/test/resources/arbitrary/demo1");
ArbitraryDataTransactionBuilder txnBuilder = new ArbitraryDataTransactionBuilder(
repository, publicKey58, path, name, Method.PUT, Service.ARBITRARY_DATA, null);
repository, publicKey58, path, name, Method.PUT, Service.ARBITRARY_DATA, null,
null, null, null, null);
txnBuilder.build();
ArbitraryTransactionData transactionData = txnBuilder.getArbitraryTransactionData();

49
src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java
View File

@ -73,4 +73,53 @@ public class ArbitraryTransactionMetadataTests extends Common {
}
}
@Test
public void testDescriptiveMetadata() throws DataException, IOException, MissingDataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());
String name = "TEST"; // Can be anything for this test
String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA;
int chunkSize = 100;
int dataLength = 900; // Actual data length will be longer due to encryption
String title = "Test title";
String description = "Test description";
String tags = "Test tags";
String category = "Test category";
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength);
ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name,
identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize,
title, description, tags, category);
// Check the chunk count is correct
assertEquals(10, arbitraryDataFile.chunkCount());
// Check the metadata is correct
assertEquals(title, arbitraryDataFile.getMetadata().getTitle());
assertEquals(description, arbitraryDataFile.getMetadata().getDescription());
assertEquals(tags, arbitraryDataFile.getMetadata().getTags());
assertEquals(category, arbitraryDataFile.getMetadata().getCategory());
// Now build the latest data state for this name
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
arbitraryDataReader.loadSynchronously(true);
Path initialLayerPath = arbitraryDataReader.getFilePath();
ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath);
initialLayerDigest.compute();
// Its directory hash should match the original directory hash
ArbitraryDataDigest path1Digest = new ArbitraryDataDigest(path1);
path1Digest.compute();
assertEquals(path1Digest.getHash58(), initialLayerDigest.getHash58());
}
}
}

12
src/test/java/org/qortal/test/common/ArbitraryUtils.java
View File

@ -22,12 +22,20 @@ import static org.junit.Assert.assertEquals;
public class ArbitraryUtils {
public static ArbitraryDataFile createAndMintTxn(Repository repository, String publicKey58, Path path, String name, String identifier,
ArbitraryTransactionData.Method method, Service service, PrivateKeyAccount account,
int chunkSize) throws DataException {
return ArbitraryUtils.createAndMintTxn(repository, publicKey58, path, name, identifier, method, service,
account, chunkSize, null, null, null, null);
}
public static ArbitraryDataFile createAndMintTxn(Repository repository, String publicKey58, Path path, String name, String identifier,
ArbitraryTransactionData.Method method, Service service, PrivateKeyAccount account,
int chunkSize) throws DataException {
int chunkSize, String title, String description, String tags, String category) throws DataException {
ArbitraryDataTransactionBuilder txnBuilder = new ArbitraryDataTransactionBuilder(
repository, publicKey58, path, name, method, service, identifier);
repository, publicKey58, path, name, method, service, identifier, title, description, tags, category);
txnBuilder.setChunkSize(chunkSize);
txnBuilder.build();