Bump version to 4.0.1

This reverts commit ce52b39495.

Q-Apps.md

@@ -240,6 +240,9 @@ Here is a list of currently supported actions:
 - SEARCH_QDN_RESOURCES
 - GET_QDN_RESOURCE_STATUS
 - GET_QDN_RESOURCE_PROPERTIES
+- GET_QDN_RESOURCE_METADATA
+- GET_QDN_RESOURCE_URL
+- LINK_TO_QDN_RESOURCE
 - FETCH_QDN_RESOURCE
 - PUBLISH_QDN_RESOURCE
 - PUBLISH_MULTIPLE_QDN_RESOURCES
@@ -258,8 +261,6 @@ Here is a list of currently supported actions:
 - FETCH_BLOCK_RANGE
 - SEARCH_TRANSACTIONS
 - GET_PRICE
-- GET_QDN_RESOURCE_URL
-- LINK_TO_QDN_RESOURCE
 - GET_LIST_ITEMS
 - ADD_LIST_ITEMS
 - DELETE_LIST_ITEM
@@ -420,6 +421,16 @@ let res = await qortalRequest({
 // Returns: filename, size, mimeType (where available)
 ```
 
+### Get QDN resource metadata
+```
+let res = await qortalRequest({
+    action: "GET_QDN_RESOURCE_METADATA",
+    name: "QortalDemo",
+    service: "THUMBNAIL",
+    identifier: "qortal_avatar" // Optional
+});
+```
+
 ### Publish a single file to QDN
 _Requires user approval_.<br />
 Note: this publishes a single, base64-encoded file. Multi-file resource publishing (such as a WEBSITE or GIF_REPOSITORY) is not yet supported via a Q-App. It will be added in a future update.
@@ -528,6 +539,7 @@ let res = await qortalRequest({
     // reference: "reference", // Optional
     // chatReference: "chatreference", // Optional
     // hasChatReference: true, // Optional
+    encoding: "BASE64", // Optional (defaults to BASE58 if omitted)
     limit: 100,
     offset: 0,
     reverse: true

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>4.0.0</version>
+	<version>4.0.1</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

src/main/java/org/qortal/api/HTMLParser.java

@@ -13,7 +13,8 @@ public class HTMLParser {
 
     private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
 
-    private String linkPrefix;
+    private String qdnBase;
+    private String qdnBaseWithPath;
     private byte[] data;
     private String qdnContext;
     private String resourceId;
@@ -21,10 +22,13 @@ public class HTMLParser {
     private String identifier;
     private String path;
     private String theme;
+    private boolean usingCustomRouting;
 
     public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix, byte[] data,
-                      String qdnContext, Service service, String identifier, String theme) {
-        this.linkPrefix = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
+                      String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting) {
+        String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : "";
+        this.qdnBase = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
+        this.qdnBaseWithPath = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
         this.data = data;
         this.qdnContext = qdnContext;
         this.resourceId = resourceId;
@@ -32,12 +36,12 @@ public class HTMLParser {
         this.identifier = identifier;
         this.path = inPath;
         this.theme = theme;
+        this.usingCustomRouting = usingCustomRouting;
     }
 
     public void addAdditionalHeaderTags() {
         String fileContents = new String(data);
         Document document = Jsoup.parse(fileContents);
-        String baseUrl = this.linkPrefix;
         Elements head = document.getElementsByTag("head");
         if (!head.isEmpty()) {
             // Add q-apps script tag
@@ -56,11 +60,13 @@ public class HTMLParser {
             String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"") : "";
             String path = this.path != null ? this.path.replace("\"","\\\"") : "";
             String theme = this.theme != null ? this.theme.replace("\"","\\\"") : "";
-            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\";</script>", this.qdnContext, theme, service, name, identifier, path, baseUrl);
+            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", this.qdnContext, theme, service, name, identifier, path, this.qdnBase, this.qdnBaseWithPath);
             head.get(0).prepend(qdnContextVar);
 
             // Add base href tag
-            String baseElement = String.format("<base href=\"%s/\">", baseUrl);
+            // Exclude the path if this request was routed back to the index automatically
+            String baseHref = this.usingCustomRouting ? this.qdnBase : this.qdnBaseWithPath;
+            String baseElement = String.format("<base href=\"%s/\">", baseHref);
             head.get(0).prepend(baseElement);
 
             // Add meta charset tag

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -65,10 +65,7 @@ import org.qortal.transaction.Transaction.ValidationResult;
 import org.qortal.transform.TransformationException;
 import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
 import org.qortal.transform.transaction.TransactionTransformer;
-import org.qortal.utils.ArbitraryTransactionUtils;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-import org.qortal.utils.ZipUtils;
+import org.qortal.utils.*;
 
 @Path("/arbitrary")
 @Tag(name = "Arbitrary")
@@ -721,12 +718,9 @@ public class ArbitraryResource {
 			}
 	)
 	@SecurityRequirement(name = "apiKey")
-	public ArbitraryResourceMetadata getMetadata(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-							  @PathParam("service") Service service,
-							  @PathParam("name") String name,
-							  @PathParam("identifier") String identifier) {
-		Security.checkApiCallAllowed(request);
-
+	public ArbitraryResourceMetadata getMetadata(@PathParam("service") Service service,
+							  					 @PathParam("name") String name,
+							  					 @PathParam("identifier") String identifier) {
 		ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
 
 		try {
@@ -1327,20 +1321,50 @@ public class ArbitraryResource {
 				}
 			}
 
-			// TODO: limit file size that can be read into memory
 			java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
 			if (!Files.exists(path)) {
 				String message = String.format("No file exists at filepath: %s", filepath);
 				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message);
 			}
 
-			byte[] data = Files.readAllBytes(path);
+			byte[] data;
+			int fileSize = (int)path.toFile().length();
+			int length = fileSize;
+
+			// Parse "Range" header
+			Integer rangeStart = null;
+			Integer rangeEnd = null;
+			String range = request.getHeader("Range");
+			if (range != null) {
+				range = range.replace("bytes=", "");
+				String[] parts = range.split("-");
+				rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null;
+				rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize;
+			}
+
+			if (rangeStart != null && rangeEnd != null) {
+				// We have a range, so update the requested length
+				length = rangeEnd - rangeStart;
+			}
+
+			if (length < fileSize && encoding == null) {
+				// Partial content requested, and not encoding the data
+				response.setStatus(206);
+				response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize));
+				data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length);
+			}
+			else {
+				// Full content requested (or encoded data)
+				response.setStatus(200);
+				data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory
+			}
 
 			// Encode the data if requested
 			if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) {
 				data = Base64.encode(data);
 			}
 
+			response.addHeader("Accept-Ranges", "bytes");
 			response.setContentType(context.getMimeType(path.toString()));
 			response.setContentLength(data.length);
 			response.getOutputStream().write(data);

src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java

@@ -126,7 +126,8 @@ public class ArbitraryDataRenderer {
         try {
             String filename = this.getFilename(unzippedPath, inPath);
             Path filePath = Paths.get(unzippedPath, filename);
-
+            boolean usingCustomRouting = false;
+            
             // If the file doesn't exist, we may need to route the request elsewhere, or cleanup
             if (!Files.exists(filePath)) {
                 if (inPath.equals("/")) {
@@ -148,6 +149,7 @@ public class ArbitraryDataRenderer {
                             // Forward request to index file
                             filePath = indexPath;
                             filename = indexFile;
+                            usingCustomRouting = true;
                             break;
                         }
                     }
@@ -157,7 +159,7 @@ public class ArbitraryDataRenderer {
             if (HTMLParser.isHtmlFile(filename)) {
                 // HTML file - needs to be parsed
                 byte[] data = Files.readAllBytes(filePath); // TODO: limit file size that can be read into memory
-                HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data, qdnContext, service, identifier, theme);
+                HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data, qdnContext, service, identifier, theme, usingCustomRouting);
                 htmlParser.addAdditionalHeaderTags();
                 response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:;");
                 response.setContentType(context.getMimeType(filename));

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java

@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
@@ -50,7 +51,7 @@ public class ArbitraryDataMetadata {
             this.readJson();
 
         } catch (JSONException e) {
-            throw new DataException(String.format("Unable to read JSON: %s", e.getMessage()));
+            throw new DataException(String.format("Unable to read JSON at path %s: %s", this.filePath, e.getMessage()));
         }
     }
 
@@ -64,6 +65,10 @@ public class ArbitraryDataMetadata {
         writer.close();
     }
 
+    public void delete() throws IOException {
+        Files.delete(this.filePath);
+    }
+
 
     protected void loadJson() throws IOException {
         File metadataFile = new File(this.filePath.toString());
@@ -71,7 +76,7 @@ public class ArbitraryDataMetadata {
             throw new IOException(String.format("Metadata file doesn't exist: %s", this.filePath.toString()));
         }
 
-        this.jsonString = new String(Files.readAllBytes(this.filePath));
+        this.jsonString = new String(Files.readAllBytes(this.filePath), StandardCharsets.UTF_8);
     }
 
 

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java

@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -69,7 +70,7 @@ public class ArbitraryDataQortalMetadata extends ArbitraryDataMetadata {
             throw new IOException(String.format("Patch file doesn't exist: %s", path.toString()));
         }
 
-        this.jsonString = new String(Files.readAllBytes(path));
+        this.jsonString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
     }
 
 

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java

@@ -7,6 +7,7 @@ import org.qortal.arbitrary.misc.Category;
 import org.qortal.repository.DataException;
 import org.qortal.utils.Base58;
 
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -217,6 +218,25 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
 
     // Static helper methods
 
+    public static String trimUTF8String(String string, int maxLength) {
+        byte[] inputBytes = string.getBytes(StandardCharsets.UTF_8);
+        int length = Math.min(inputBytes.length, maxLength);
+        byte[] outputBytes = new byte[length];
+
+        System.arraycopy(inputBytes, 0, outputBytes, 0, length);
+        String result = new String(outputBytes, StandardCharsets.UTF_8);
+
+        // check if last character is truncated
+        int lastIndex = result.length() - 1;
+
+        if (lastIndex > 0 && result.charAt(lastIndex) != string.charAt(lastIndex)) {
+            // last character is truncated so remove the last character
+            return result.substring(0, lastIndex);
+        }
+
+        return result;
+    }
+
     public static String limitTitle(String title) {
         if (title == null) {
             return null;
@@ -225,7 +245,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
             return null;
         }
 
-        return title.substring(0, Math.min(title.length(), MAX_TITLE_LENGTH));
+        return trimUTF8String(title, MAX_TITLE_LENGTH);
     }
 
     public static String limitDescription(String description) {
@@ -236,7 +256,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
             return null;
         }
 
-        return description.substring(0, Math.min(description.length(), MAX_DESCRIPTION_LENGTH));
+        return trimUTF8String(description, MAX_DESCRIPTION_LENGTH);
     }
 
     public static List<String> limitTags(List<String> tags) {

src/main/java/org/qortal/arbitrary/misc/Service.java

@@ -159,7 +159,9 @@ public enum Service {
     DATABASE(1700, false, null, false, null),
     SNAPSHOT(1710, false, null, false, null),
     COMMENT(1800, true, 500*1024L, true, null),
-    CHAIN_COMMENT(1810, true, 239L, true, null);
+    CHAIN_COMMENT(1810, true, 239L, true, null),
+    MAIL(1900, true, 1024*1024L, true, null),
+    MESSAGE(1910, true, 1024*1024L, true, null);
 
     public final int value;
     private final boolean requiresValidation;
@@ -227,7 +229,7 @@ public enum Service {
     }
 
     public static JSONObject toJsonObject(byte[] data) {
-        String dataString = new String(data);
+        String dataString = new String(data, StandardCharsets.UTF_8);
         return new JSONObject(dataString);
     }
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java

@@ -124,29 +124,29 @@ public class ArbitraryDataFileListManager {
             }
         }
 
-        // Then allow another 3 attempts, each 5 minutes apart
-        if (timeSinceLastAttempt > 5 * 60 * 1000L) {
-            // We haven't tried for at least 5 minutes
+        // Then allow another 5 attempts, each 1 minute apart
+        if (timeSinceLastAttempt > 60 * 1000L) {
+            // We haven't tried for at least 1 minute
 
-            if (networkBroadcastCount < 6) {
-                // We've made less than 6 total attempts
+            if (networkBroadcastCount < 8) {
+                // We've made less than 8 total attempts
                 return true;
             }
         }
 
-        // Then allow another 4 attempts, each 30 minutes apart
-        if (timeSinceLastAttempt > 30 * 60 * 1000L) {
-            // We haven't tried for at least 5 minutes
+        // Then allow another 8 attempts, each 15 minutes apart
+        if (timeSinceLastAttempt > 15 * 60 * 1000L) {
+            // We haven't tried for at least 15 minutes
 
-            if (networkBroadcastCount < 10) {
-                // We've made less than 10 total attempts
+            if (networkBroadcastCount < 16) {
+                // We've made less than 16 total attempts
                 return true;
             }
         }
 
-        // From then on, only try once every 24 hours, to reduce network spam
-        if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) {
-            // We haven't tried for at least 24 hours
+        // From then on, only try once every 6 hours, to reduce network spam
+        if (timeSinceLastAttempt > 6 * 60 * 60 * 1000L) {
+            // We haven't tried for at least 6 hours
             return true;
         }
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java

@@ -102,7 +102,14 @@ public class ArbitraryMetadataManager {
                 if (metadataFile.exists()) {
                     // Use local copy
                     ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
-                    transactionMetadata.read();
+                    try {
+                        transactionMetadata.read();
+                    } catch (DataException e) {
+                        // Invalid file, so delete it
+                        LOGGER.info("Deleting invalid metadata file due to exception: {}", e.getMessage());
+                        transactionMetadata.delete();
+                        return null;
+                    }
                     return transactionMetadata;
                 }
             }

src/main/java/org/qortal/list/ResourceList.java

@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -81,7 +82,7 @@ public class ResourceList {
         }
 
         try {
-            String jsonString = new String(Files.readAllBytes(path));
+            String jsonString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
             this.list = ResourceList.listFromJSONString(jsonString);
         } catch (IOException e) {
             throw new IOException(String.format("Couldn't read contents from file %s", path.toString()));

src/main/resources/q-apps/q-apps.js

@@ -1,4 +1,4 @@
-function httpGet(event, url) {
+function httpGet(url) {
     var request = new XMLHttpRequest();
     request.open("GET", url, false);
     request.send(null);
@@ -250,6 +250,11 @@ window.addEventListener("message", (event) => {
             url = "/arbitrary/resource/properties/" + data.service + "/" + data.name + "/" + identifier;
             return httpGetAsyncWithEvent(event, url);
 
+        case "GET_QDN_RESOURCE_METADATA":
+            identifier = (data.identifier != null) ? data.identifier : "default";
+            url = "/arbitrary/metadata/" + data.service + "/" + data.name + "/" + identifier;
+            return httpGetAsyncWithEvent(event, url);
+
         case "SEARCH_CHAT_MESSAGES":
             url = "/chat/messages?";
             if (data.before != null) url = url.concat("&before=" + data.before);
@@ -259,6 +264,7 @@ window.addEventListener("message", (event) => {
             if (data.reference != null) url = url.concat("&reference=" + data.reference);
             if (data.chatReference != null) url = url.concat("&chatreference=" + data.chatReference);
             if (data.hasChatReference != null) url = url.concat("&haschatreference=" + new Boolean(data.hasChatReference).toString());
+            if (data.encoding != null) url = url.concat("&encoding=" + data.encoding);
             if (data.limit != null) url = url.concat("&limit=" + data.limit);
             if (data.offset != null) url = url.concat("&offset=" + data.offset);
             if (data.reverse != null) url = url.concat("&reverse=" + new Boolean(data.reverse).toString());

src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java

@@ -248,6 +248,47 @@ public class ArbitraryTransactionMetadataTests extends Common {
         }
     }
 
+    @Test
+    public void testUTF8Metadata() throws DataException, IOException, MissingDataException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
+            String publicKey58 = Base58.encode(alice.getPublicKey());
+            String name = "TEST"; // Can be anything for this test
+            String identifier = null; // Not used for this test
+            Service service = Service.ARBITRARY_DATA;
+            int chunkSize = 100;
+            int dataLength = 900; // Actual data length will be longer due to encryption
+
+            // Example (modified) strings from real world content
+            String title = "Доля юаня в трансграничных Доля юаня в трансграничных";
+            String description = "Когда рыночек порешал";
+            List<String> tags = Arrays.asList("Доля", "юаня", "трансграничных");
+            Category category = Category.OTHER;
+
+            // Register the name to Alice
+            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+            TransactionUtils.signAndMint(repository, transactionData, alice);
+
+            // Create PUT transaction
+            Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength);
+            ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name,
+                    identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize, 0L, true,
+                    title, description, tags, category);
+
+            // Check the chunk count is correct
+            assertEquals(10, arbitraryDataFile.chunkCount());
+
+            // Check the metadata is correct
+            String expectedTrimmedTitle = "Доля юаня в трансграничных Доля юаня в тран";
+            assertEquals(expectedTrimmedTitle, arbitraryDataFile.getMetadata().getTitle());
+            assertEquals(description, arbitraryDataFile.getMetadata().getDescription());
+            assertEquals(tags, arbitraryDataFile.getMetadata().getTags());
+            assertEquals(category, arbitraryDataFile.getMetadata().getCategory());
+            assertEquals("text/plain", arbitraryDataFile.getMetadata().getMimeType());
+        }
+    }
+
     @Test
     public void testMetadataLengths() throws DataException, IOException, MissingDataException {
         try (final Repository repository = RepositoryManager.getRepository()) {