From 16dc23ddc7fd3a5d5552cc6f480df2817b2f0443 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 21 Apr 2023 21:45:16 +0100
Subject: [PATCH 01/48] Added new actions to gateway handler.

---
 src/main/resources/q-apps/q-apps-gateway.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/main/resources/q-apps/q-apps-gateway.js b/src/main/resources/q-apps/q-apps-gateway.js
index d8d15d06..d5028dca 100644
--- a/src/main/resources/q-apps/q-apps-gateway.js
+++ b/src/main/resources/q-apps/q-apps-gateway.js
@@ -50,12 +50,16 @@ window.addEventListener("message", (event) => {
     switch (data.action) {
         case "GET_USER_ACCOUNT":
         case "PUBLISH_QDN_RESOURCE":
+        case "PUBLISH_MULTIPLE_QDN_RESOURCES":
         case "SEND_CHAT_MESSAGE":
         case "JOIN_GROUP":
         case "DEPLOY_AT":
         case "GET_WALLET_BALANCE":
         case "SEND_COIN":
-            const errorString = "Authentication was requested, but this is not yet supported when viewing via a gateway. To use interactive features, please access using the Qortal UI desktop app. More info at: https://qortal.org";
+        case "GET_LIST_ITEMS":
+        case "ADD_LIST_ITEMS":
+        case "DELETE_LIST_ITEM":
+            const errorString = "Interactive features were requested, but these are not yet supported when viewing via a gateway. To use interactive features, please access using the Qortal UI desktop app. More info at: https://qortal.org";
             response = "{\"error\": \"" + errorString + "\"}"
 
             const modalText = "This app is powered by the Qortal blockchain. You are viewing in read-only mode. To use interactive features, please access using the Qortal UI desktop app.";

From 33aeec7e87557c310eda4def6056d890b1124e55 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sat, 22 Apr 2023 11:00:21 +0100
Subject: [PATCH 02/48] Added various new service types, in preparation for
 Q-Apps release.

---
 Q-Apps.md                                     | 22 ++++++++++++++++-
 .../org/qortal/arbitrary/misc/Service.java    | 24 +++++++++++++++++--
 2 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/Q-Apps.md b/Q-Apps.md
index 0f52c086..77095c72 100644
--- a/Q-Apps.md
+++ b/Q-Apps.md
@@ -46,6 +46,8 @@ IMAGE,
 THUMBNAIL,
 VIDEO,
 AUDIO,
+PODCAST,
+VOICE,
 ARBITRARY_DATA,
 JSON,
 DOCUMENT,
@@ -55,7 +57,25 @@ METADATA,
 BLOG,
 BLOG_POST,
 BLOG_COMMENT,
-GIF_REPOSITORY
+GIF_REPOSITORY,
+ATTACHMENT,
+FILE,
+FILES,
+CHAIN_DATA,
+STORE,
+PRODUCT,
+OFFER,
+COUPON,
+CODE,
+PLUGIN,
+EXTENSION,
+GAME,
+ITEM,
+NFT,
+DATABASE,
+SNAPSHOT,
+COMMENT,
+CHAIN_COMMENT,
 WEBSITE,
 APP,
 QCHAT_ATTACHMENT,
diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java
index fa47f020..3138ccd8 100644
--- a/src/main/java/org/qortal/arbitrary/misc/Service.java
+++ b/src/main/java/org/qortal/arbitrary/misc/Service.java
@@ -47,6 +47,10 @@ public enum Service {
             return ValidationResult.OK;
         }
     },
+    ATTACHMENT(130, false, null, true, null),
+    FILE(140, false, null, true, null),
+    FILES(150, false, null, false, null),
+    CHAIN_DATA(160, false, 239L, true, null),
     WEBSITE(200, true, null, false, null) {
         @Override
         public ValidationResult validate(Path path) throws IOException {
@@ -75,11 +79,13 @@ public enum Service {
     QCHAT_IMAGE(420, true, 500*1024L, true, null),
     VIDEO(500, false, null, true, null),
     AUDIO(600, false, null, true, null),
+    PODCAST(610, false, null, true, null),
     QCHAT_AUDIO(610, true, 10*1024*1024L, true, null),
     QCHAT_VOICE(620, true, 10*1024*1024L, true, null),
+    VOICE(630, true, 10*1024*1024L, true, null),
     BLOG(700, false, null, false, null),
     BLOG_POST(777, false, null, true, null),
-    BLOG_COMMENT(778, false, null, true, null),
+    BLOG_COMMENT(778, false, 500*1024L, true, null),
     DOCUMENT(800, false, null, true, null),
     LIST(900, true, null, true, null),
     PLAYLIST(910, true, null, true, null),
@@ -139,7 +145,21 @@ public enum Service {
             }
             return ValidationResult.OK;
         }
-    };
+    },
+    STORE(1200, false, null, true, null),
+    PRODUCT(1210, false, null, true, null),
+    OFFER(1230, false, null, true, null),
+    COUPON(1240, false, null, true, null),
+    CODE(1300, false, null, true, null),
+    PLUGIN(1310, false, null, true, null),
+    EXTENSION(1320, false, null, true, null),
+    GAME(1400, false, null, false, null),
+    ITEM(1410, false, null, true, null),
+    NFT(1500, false, null, true, null),
+    DATABASE(1600, false, null, false, null),
+    SNAPSHOT(1610, false, null, false, null),
+    COMMENT(1700, false, 500*1024L, true, null),
+    CHAIN_COMMENT(1710, false, 239L, true, null);
 
     public final int value;
     private final boolean requiresValidation;

From 53508f92983c152f0a767733e288034a5a9462d5 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sat, 22 Apr 2023 11:33:59 +0100
Subject: [PATCH 03/48] Fixed problems in last commit.

---
 .../org/qortal/arbitrary/misc/Service.java    | 34 +++++++++----------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java
index 3138ccd8..da089675 100644
--- a/src/main/java/org/qortal/arbitrary/misc/Service.java
+++ b/src/main/java/org/qortal/arbitrary/misc/Service.java
@@ -50,7 +50,7 @@ public enum Service {
     ATTACHMENT(130, false, null, true, null),
     FILE(140, false, null, true, null),
     FILES(150, false, null, false, null),
-    CHAIN_DATA(160, false, 239L, true, null),
+    CHAIN_DATA(160, true, 239L, true, null),
     WEBSITE(200, true, null, false, null) {
         @Override
         public ValidationResult validate(Path path) throws IOException {
@@ -79,13 +79,13 @@ public enum Service {
     QCHAT_IMAGE(420, true, 500*1024L, true, null),
     VIDEO(500, false, null, true, null),
     AUDIO(600, false, null, true, null),
-    PODCAST(610, false, null, true, null),
     QCHAT_AUDIO(610, true, 10*1024*1024L, true, null),
     QCHAT_VOICE(620, true, 10*1024*1024L, true, null),
     VOICE(630, true, 10*1024*1024L, true, null),
+    PODCAST(640, false, null, true, null),
     BLOG(700, false, null, false, null),
     BLOG_POST(777, false, null, true, null),
-    BLOG_COMMENT(778, false, 500*1024L, true, null),
+    BLOG_COMMENT(778, true, 500*1024L, true, null),
     DOCUMENT(800, false, null, true, null),
     LIST(900, true, null, true, null),
     PLAYLIST(910, true, null, true, null),
@@ -146,20 +146,20 @@ public enum Service {
             return ValidationResult.OK;
         }
     },
-    STORE(1200, false, null, true, null),
-    PRODUCT(1210, false, null, true, null),
-    OFFER(1230, false, null, true, null),
-    COUPON(1240, false, null, true, null),
-    CODE(1300, false, null, true, null),
-    PLUGIN(1310, false, null, true, null),
-    EXTENSION(1320, false, null, true, null),
-    GAME(1400, false, null, false, null),
-    ITEM(1410, false, null, true, null),
-    NFT(1500, false, null, true, null),
-    DATABASE(1600, false, null, false, null),
-    SNAPSHOT(1610, false, null, false, null),
-    COMMENT(1700, false, 500*1024L, true, null),
-    CHAIN_COMMENT(1710, false, 239L, true, null);
+    STORE(1300, false, null, true, null),
+    PRODUCT(1310, false, null, true, null),
+    OFFER(1330, false, null, true, null),
+    COUPON(1340, false, null, true, null),
+    CODE(1400, false, null, true, null),
+    PLUGIN(1410, false, null, true, null),
+    EXTENSION(1420, false, null, true, null),
+    GAME(1500, false, null, false, null),
+    ITEM(1510, false, null, true, null),
+    NFT(1600, false, null, true, null),
+    DATABASE(1700, false, null, false, null),
+    SNAPSHOT(1710, false, null, false, null),
+    COMMENT(1800, true, 500*1024L, true, null),
+    CHAIN_COMMENT(1810, true, 239L, true, null);
 
     public final int value;
     private final boolean requiresValidation;

From e48529704c78e091e65d6223622806f34d91113f Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sat, 22 Apr 2023 16:08:09 +0100
Subject: [PATCH 04/48] Bump version to 4.0.0

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 70366ada..bbb7bba3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>3.9.1</version>
+	<version>4.0.0</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

From f27c9193c744d5b32cda3cb8aad1416259fd80d1 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sun, 23 Apr 2023 11:30:42 +0100
Subject: [PATCH 05/48] Auto delete any metadata files that are unreadable
 (e.g. due to being empty, or invalid JSON).

---
 .../qortal/arbitrary/metadata/ArbitraryDataMetadata.java | 6 +++++-
 .../controller/arbitrary/ArbitraryMetadataManager.java   | 9 ++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java
index 07f6032c..06d02340 100644
--- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java
+++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java
@@ -50,7 +50,7 @@ public class ArbitraryDataMetadata {
             this.readJson();
 
         } catch (JSONException e) {
-            throw new DataException(String.format("Unable to read JSON: %s", e.getMessage()));
+            throw new DataException(String.format("Unable to read JSON at path %s: %s", this.filePath, e.getMessage()));
         }
     }
 
@@ -64,6 +64,10 @@ public class ArbitraryDataMetadata {
         writer.close();
     }
 
+    public void delete() throws IOException {
+        Files.delete(this.filePath);
+    }
+
 
     protected void loadJson() throws IOException {
         File metadataFile = new File(this.filePath.toString());
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
index 97d659ad..663bc22a 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
@@ -102,7 +102,14 @@ public class ArbitraryMetadataManager {
                 if (metadataFile.exists()) {
                     // Use local copy
                     ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
-                    transactionMetadata.read();
+                    try {
+                        transactionMetadata.read();
+                    } catch (DataException e) {
+                        // Invalid file, so delete it
+                        LOGGER.info("Deleting invalid metadata file due to exception: {}", e.getMessage());
+                        transactionMetadata.delete();
+                        return null;
+                    }
                     return transactionMetadata;
                 }
             }

From ed6333f82ee260f36d60f9382b41db9fb95ff4e0 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sun, 23 Apr 2023 19:14:28 +0100
Subject: [PATCH 06/48] Allow for faster and more frequent retries when QDN
 data fails to be retrieved (thanks to suggestions from @xspektrex)

---
 .../ArbitraryDataFileListManager.java         | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
index 2fd6033e..5ed8df21 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
@@ -124,29 +124,29 @@ public class ArbitraryDataFileListManager {
             }
         }
 
-        // Then allow another 3 attempts, each 5 minutes apart
-        if (timeSinceLastAttempt > 5 * 60 * 1000L) {
-            // We haven't tried for at least 5 minutes
+        // Then allow another 5 attempts, each 1 minute apart
+        if (timeSinceLastAttempt > 60 * 1000L) {
+            // We haven't tried for at least 1 minute
 
-            if (networkBroadcastCount < 6) {
-                // We've made less than 6 total attempts
+            if (networkBroadcastCount < 8) {
+                // We've made less than 8 total attempts
                 return true;
             }
         }
 
-        // Then allow another 4 attempts, each 30 minutes apart
-        if (timeSinceLastAttempt > 30 * 60 * 1000L) {
-            // We haven't tried for at least 5 minutes
+        // Then allow another 8 attempts, each 15 minutes apart
+        if (timeSinceLastAttempt > 15 * 60 * 1000L) {
+            // We haven't tried for at least 15 minutes
 
-            if (networkBroadcastCount < 10) {
-                // We've made less than 10 total attempts
+            if (networkBroadcastCount < 16) {
+                // We've made less than 16 total attempts
                 return true;
             }
         }
 
-        // From then on, only try once every 24 hours, to reduce network spam
-        if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) {
-            // We haven't tried for at least 24 hours
+        // From then on, only try once every 6 hours, to reduce network spam
+        if (timeSinceLastAttempt > 6 * 60 * 60 * 1000L) {
+            // We haven't tried for at least 6 hours
             return true;
         }
 

From 1ce2dcfb2b339b58c643282e7409025ac8157455 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Tue, 25 Apr 2023 08:33:33 +0100
Subject: [PATCH 07/48] Fixed bug which prevented qortal:// URLs from working
 properly in most cases.

---
 src/main/resources/q-apps/q-apps.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js
index 2274cec0..9ae4e478 100644
--- a/src/main/resources/q-apps/q-apps.js
+++ b/src/main/resources/q-apps/q-apps.js
@@ -1,4 +1,4 @@
-function httpGet(event, url) {
+function httpGet(url) {
     var request = new XMLHttpRequest();
     request.open("GET", url, false);
     request.send(null);

From 5dbacc4db379c9346b7c2d6947a6f70308c82dde Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 28 Apr 2023 10:12:16 +0100
Subject: [PATCH 08/48] Added "Accept-Ranges" header when serving arbitrary
 data.

Allows for video seeking when using URL playback, even though the Range header isn't implemented yet. This could be heavily optimized by adding full support of the Range/Content-Range headers, however this is still a big step forward as it allows for (inefficient) seeking.
---
 src/main/java/org/qortal/api/resource/ArbitraryResource.java | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
index 3d1a6a2e..64ee2a6f 100644
--- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java
+++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
@@ -1341,6 +1341,7 @@ public class ArbitraryResource {
 				data = Base64.encode(data);
 			}
 
+			response.addHeader("Accept-Ranges", "bytes");
 			response.setContentType(context.getMimeType(path.toString()));
 			response.setContentLength(data.length);
 			response.getOutputStream().write(data);

From 0a1ab3d68583b24f9588f376c62416d864b3f87e Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 28 Apr 2023 10:57:04 +0100
Subject: [PATCH 09/48] Added GET_QDN_RESOURCE_METADATA action.

---
 Q-Apps.md                                         | 15 +++++++++++++--
 .../qortal/api/resource/ArbitraryResource.java    |  9 +++------
 src/main/resources/q-apps/q-apps.js               |  5 +++++
 3 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/Q-Apps.md b/Q-Apps.md
index 77095c72..0f5bc7e8 100644
--- a/Q-Apps.md
+++ b/Q-Apps.md
@@ -240,6 +240,9 @@ Here is a list of currently supported actions:
 - SEARCH_QDN_RESOURCES
 - GET_QDN_RESOURCE_STATUS
 - GET_QDN_RESOURCE_PROPERTIES
+- GET_QDN_RESOURCE_METADATA
+- GET_QDN_RESOURCE_URL
+- LINK_TO_QDN_RESOURCE
 - FETCH_QDN_RESOURCE
 - PUBLISH_QDN_RESOURCE
 - PUBLISH_MULTIPLE_QDN_RESOURCES
@@ -258,8 +261,6 @@ Here is a list of currently supported actions:
 - FETCH_BLOCK_RANGE
 - SEARCH_TRANSACTIONS
 - GET_PRICE
-- GET_QDN_RESOURCE_URL
-- LINK_TO_QDN_RESOURCE
 - GET_LIST_ITEMS
 - ADD_LIST_ITEMS
 - DELETE_LIST_ITEM
@@ -420,6 +421,16 @@ let res = await qortalRequest({
 // Returns: filename, size, mimeType (where available)
 ```
 
+### Get QDN resource metadata
+```
+let res = await qortalRequest({
+    action: "GET_QDN_RESOURCE_METADATA",
+    name: "QortalDemo",
+    service: "THUMBNAIL",
+    identifier: "qortal_avatar" // Optional
+});
+```
+
 ### Publish a single file to QDN
 _Requires user approval_.<br />
 Note: this publishes a single, base64-encoded file. Multi-file resource publishing (such as a WEBSITE or GIF_REPOSITORY) is not yet supported via a Q-App. It will be added in a future update.
diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
index 64ee2a6f..dddad594 100644
--- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java
+++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
@@ -721,12 +721,9 @@ public class ArbitraryResource {
 			}
 	)
 	@SecurityRequirement(name = "apiKey")
-	public ArbitraryResourceMetadata getMetadata(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-							  @PathParam("service") Service service,
-							  @PathParam("name") String name,
-							  @PathParam("identifier") String identifier) {
-		Security.checkApiCallAllowed(request);
-
+	public ArbitraryResourceMetadata getMetadata(@PathParam("service") Service service,
+							  					 @PathParam("name") String name,
+							  					 @PathParam("identifier") String identifier) {
 		ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
 
 		try {
diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js
index 9ae4e478..9da494c0 100644
--- a/src/main/resources/q-apps/q-apps.js
+++ b/src/main/resources/q-apps/q-apps.js
@@ -250,6 +250,11 @@ window.addEventListener("message", (event) => {
             url = "/arbitrary/resource/properties/" + data.service + "/" + data.name + "/" + identifier;
             return httpGetAsyncWithEvent(event, url);
 
+        case "GET_QDN_RESOURCE_METADATA":
+            identifier = (data.identifier != null) ? data.identifier : "default";
+            url = "/arbitrary/metadata/" + data.service + "/" + data.name + "/" + identifier;
+            return httpGetAsyncWithEvent(event, url);
+
         case "SEARCH_CHAT_MESSAGES":
             url = "/chat/messages?";
             if (data.before != null) url = url.concat("&before=" + data.before);

From a3518d1f059eeffb2a67ba032f4a97f9add862e9 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 28 Apr 2023 12:13:31 +0100
Subject: [PATCH 10/48] Revert "Fixed bug with base path."

This reverts commit ce52b3949501cccf66240093a077686b9f48c664.
---
 src/main/java/org/qortal/api/HTMLParser.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java
index eac813a9..3cba9a62 100644
--- a/src/main/java/org/qortal/api/HTMLParser.java
+++ b/src/main/java/org/qortal/api/HTMLParser.java
@@ -24,7 +24,8 @@ public class HTMLParser {
 
     public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix, byte[] data,
                       String qdnContext, Service service, String identifier, String theme) {
-        this.linkPrefix = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
+        String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : "";
+        this.linkPrefix = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
         this.data = data;
         this.qdnContext = qdnContext;
         this.resourceId = resourceId;

From 46e2e1043d40d2472c735ab5eed1382abe526688 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 28 Apr 2023 12:18:27 +0100
Subject: [PATCH 11/48] Fixed issue with <base href> introduced in v4.0.0

---
 src/main/java/org/qortal/api/HTMLParser.java | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java
index 3cba9a62..72a35ed9 100644
--- a/src/main/java/org/qortal/api/HTMLParser.java
+++ b/src/main/java/org/qortal/api/HTMLParser.java
@@ -14,6 +14,7 @@ public class HTMLParser {
     private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
 
     private String linkPrefix;
+    private String qdnBase;
     private byte[] data;
     private String qdnContext;
     private String resourceId;
@@ -26,6 +27,7 @@ public class HTMLParser {
                       String qdnContext, Service service, String identifier, String theme) {
         String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : "";
         this.linkPrefix = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
+        this.qdnBase = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
         this.data = data;
         this.qdnContext = qdnContext;
         this.resourceId = resourceId;
@@ -38,7 +40,6 @@ public class HTMLParser {
     public void addAdditionalHeaderTags() {
         String fileContents = new String(data);
         Document document = Jsoup.parse(fileContents);
-        String baseUrl = this.linkPrefix;
         Elements head = document.getElementsByTag("head");
         if (!head.isEmpty()) {
             // Add q-apps script tag
@@ -57,11 +58,11 @@ public class HTMLParser {
             String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"") : "";
             String path = this.path != null ? this.path.replace("\"","\\\"") : "";
             String theme = this.theme != null ? this.theme.replace("\"","\\\"") : "";
-            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\";</script>", this.qdnContext, theme, service, name, identifier, path, baseUrl);
+            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\";</script>", this.qdnContext, theme, service, name, identifier, path, this.qdnBase);
             head.get(0).prepend(qdnContextVar);
 
             // Add base href tag
-            String baseElement = String.format("<base href=\"%s/\">", baseUrl);
+            String baseElement = String.format("<base href=\"%s/\">", this.linkPrefix);
             head.get(0).prepend(baseElement);
 
             // Add meta charset tag

From 45bc2e46d6c43c7d99b1ce15cd86c1826c4b5b34 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 28 Apr 2023 12:48:38 +0100
Subject: [PATCH 12/48] Improved metadata trimming, to better handle multibyte
 UTF-8 characters.

---
 .../ArbitraryDataTransactionMetadata.java     | 24 ++++++++++-
 .../ArbitraryTransactionMetadataTests.java    | 41 +++++++++++++++++++
 2 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java
index 004e0ed3..d9dba037 100644
--- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java
+++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java
@@ -7,6 +7,7 @@ import org.qortal.arbitrary.misc.Category;
 import org.qortal.repository.DataException;
 import org.qortal.utils.Base58;
 
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -217,6 +218,25 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
 
     // Static helper methods
 
+    public static String trimUTF8String(String string, int maxLength) {
+        byte[] inputBytes = string.getBytes(StandardCharsets.UTF_8);
+        int length = Math.min(inputBytes.length, maxLength);
+        byte[] outputBytes = new byte[length];
+
+        System.arraycopy(inputBytes, 0, outputBytes, 0, length);
+        String result = new String(outputBytes, StandardCharsets.UTF_8);
+
+        // check if last character is truncated
+        int lastIndex = result.length() - 1;
+
+        if (lastIndex > 0 && result.charAt(lastIndex) != string.charAt(lastIndex)) {
+            // last character is truncated so remove the last character
+            return result.substring(0, lastIndex);
+        }
+
+        return result;
+    }
+
     public static String limitTitle(String title) {
         if (title == null) {
             return null;
@@ -225,7 +245,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
             return null;
         }
 
-        return title.substring(0, Math.min(title.length(), MAX_TITLE_LENGTH));
+        return trimUTF8String(title, MAX_TITLE_LENGTH);
     }
 
     public static String limitDescription(String description) {
@@ -236,7 +256,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
             return null;
         }
 
-        return description.substring(0, Math.min(description.length(), MAX_DESCRIPTION_LENGTH));
+        return trimUTF8String(description, MAX_DESCRIPTION_LENGTH);
     }
 
     public static List<String> limitTags(List<String> tags) {
diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java
index 37da4e31..47c68b25 100644
--- a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java
+++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java
@@ -248,6 +248,47 @@ public class ArbitraryTransactionMetadataTests extends Common {
         }
     }
 
+    @Test
+    public void testUTF8Metadata() throws DataException, IOException, MissingDataException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
+            String publicKey58 = Base58.encode(alice.getPublicKey());
+            String name = "TEST"; // Can be anything for this test
+            String identifier = null; // Not used for this test
+            Service service = Service.ARBITRARY_DATA;
+            int chunkSize = 100;
+            int dataLength = 900; // Actual data length will be longer due to encryption
+
+            // Example (modified) strings from real world content
+            String title = "Доля юаня в трансграничных Доля юаня в трансграничных";
+            String description = "Когда рыночек порешал";
+            List<String> tags = Arrays.asList("Доля", "юаня", "трансграничных");
+            Category category = Category.OTHER;
+
+            // Register the name to Alice
+            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+            TransactionUtils.signAndMint(repository, transactionData, alice);
+
+            // Create PUT transaction
+            Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength);
+            ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name,
+                    identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize, 0L, true,
+                    title, description, tags, category);
+
+            // Check the chunk count is correct
+            assertEquals(10, arbitraryDataFile.chunkCount());
+
+            // Check the metadata is correct
+            String expectedTrimmedTitle = "Доля юаня в трансграничных Доля юаня в тран";
+            assertEquals(expectedTrimmedTitle, arbitraryDataFile.getMetadata().getTitle());
+            assertEquals(description, arbitraryDataFile.getMetadata().getDescription());
+            assertEquals(tags, arbitraryDataFile.getMetadata().getTags());
+            assertEquals(category, arbitraryDataFile.getMetadata().getCategory());
+            assertEquals("text/plain", arbitraryDataFile.getMetadata().getMimeType());
+        }
+    }
+
     @Test
     public void testMetadataLengths() throws DataException, IOException, MissingDataException {
         try (final Repository repository = RepositoryManager.getRepository()) {

From 6dfaaf0054aaf5847ad3b75b12de368799ac37b4 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 28 Apr 2023 13:06:29 +0100
Subject: [PATCH 13/48] Set charset to UTF-8 in various places that bytes are
 converted to a string.

---
 .../org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java   | 3 ++-
 .../qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java | 3 ++-
 src/main/java/org/qortal/arbitrary/misc/Service.java           | 2 +-
 src/main/java/org/qortal/list/ResourceList.java                | 3 ++-
 4 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java
index 06d02340..498f3296 100644
--- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java
+++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java
@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
@@ -75,7 +76,7 @@ public class ArbitraryDataMetadata {
             throw new IOException(String.format("Metadata file doesn't exist: %s", this.filePath.toString()));
         }
 
-        this.jsonString = new String(Files.readAllBytes(this.filePath));
+        this.jsonString = new String(Files.readAllBytes(this.filePath), StandardCharsets.UTF_8);
     }
 
 
diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java
index df23655c..eb3d6cc9 100644
--- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java
+++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java
@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -69,7 +70,7 @@ public class ArbitraryDataQortalMetadata extends ArbitraryDataMetadata {
             throw new IOException(String.format("Patch file doesn't exist: %s", path.toString()));
         }
 
-        this.jsonString = new String(Files.readAllBytes(path));
+        this.jsonString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
     }
 
 
diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java
index da089675..03c38a56 100644
--- a/src/main/java/org/qortal/arbitrary/misc/Service.java
+++ b/src/main/java/org/qortal/arbitrary/misc/Service.java
@@ -227,7 +227,7 @@ public enum Service {
     }
 
     public static JSONObject toJsonObject(byte[] data) {
-        String dataString = new String(data);
+        String dataString = new String(data, StandardCharsets.UTF_8);
         return new JSONObject(dataString);
     }
 
diff --git a/src/main/java/org/qortal/list/ResourceList.java b/src/main/java/org/qortal/list/ResourceList.java
index 5c12e0f5..855c9068 100644
--- a/src/main/java/org/qortal/list/ResourceList.java
+++ b/src/main/java/org/qortal/list/ResourceList.java
@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -81,7 +82,7 @@ public class ResourceList {
         }
 
         try {
-            String jsonString = new String(Files.readAllBytes(path));
+            String jsonString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
             this.list = ResourceList.listFromJSONString(jsonString);
         } catch (IOException e) {
             throw new IOException(String.format("Couldn't read contents from file %s", path.toString()));

From aed1823afbd852b5f706d94314b3da41386e3a79 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 28 Apr 2023 20:36:06 +0100
Subject: [PATCH 14/48] Added support of simple Range headers when requesting
 QDN data.

---
 .../api/resource/ArbitraryResource.java       | 38 ++++++++++++++++---
 1 file changed, 32 insertions(+), 6 deletions(-)

diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
index dddad594..1101e71d 100644
--- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java
+++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
@@ -65,10 +65,7 @@ import org.qortal.transaction.Transaction.ValidationResult;
 import org.qortal.transform.TransformationException;
 import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
 import org.qortal.transform.transaction.TransactionTransformer;
-import org.qortal.utils.ArbitraryTransactionUtils;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-import org.qortal.utils.ZipUtils;
+import org.qortal.utils.*;
 
 @Path("/arbitrary")
 @Tag(name = "Arbitrary")
@@ -1324,14 +1321,43 @@ public class ArbitraryResource {
 				}
 			}
 
-			// TODO: limit file size that can be read into memory
 			java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
 			if (!Files.exists(path)) {
 				String message = String.format("No file exists at filepath: %s", filepath);
 				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message);
 			}
 
-			byte[] data = Files.readAllBytes(path);
+			byte[] data;
+			int fileSize = (int)path.toFile().length();
+			int length = fileSize;
+
+			// Parse "Range" header
+			Integer rangeStart = null;
+			Integer rangeEnd = null;
+			String range = request.getHeader("Range");
+			if (range != null) {
+				range = range.replace("bytes=", "");
+				String[] parts = range.split("-");
+				rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null;
+				rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize;
+			}
+
+			if (rangeStart != null && rangeEnd != null) {
+				// We have a range, so update the requested length
+				length = rangeEnd - rangeStart;
+			}
+
+			if (length < fileSize && encoding == null) {
+				// Partial content requested, and not encoding the data
+				response.setStatus(206);
+				response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize));
+				data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length);
+			}
+			else {
+				// Full content requested (or encoded data)
+				response.setStatus(200);
+				data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory
+			}
 
 			// Encode the data if requested
 			if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) {

From f044166b81f248dccf51c855037b596f616e2f51 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sat, 29 Apr 2023 17:13:50 +0100
Subject: [PATCH 15/48] More qdnBase improvements, to hopefully handle all
 cases correctly.

---
 src/main/java/org/qortal/api/HTMLParser.java       | 14 +++++++++-----
 .../qortal/arbitrary/ArbitraryDataRenderer.java    |  6 ++++--
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java
index 72a35ed9..03cdb066 100644
--- a/src/main/java/org/qortal/api/HTMLParser.java
+++ b/src/main/java/org/qortal/api/HTMLParser.java
@@ -13,8 +13,8 @@ public class HTMLParser {
 
     private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
 
-    private String linkPrefix;
     private String qdnBase;
+    private String qdnBaseWithPath;
     private byte[] data;
     private String qdnContext;
     private String resourceId;
@@ -22,12 +22,13 @@ public class HTMLParser {
     private String identifier;
     private String path;
     private String theme;
+    private boolean usingCustomRouting;
 
     public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix, byte[] data,
-                      String qdnContext, Service service, String identifier, String theme) {
+                      String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting) {
         String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : "";
-        this.linkPrefix = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
         this.qdnBase = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
+        this.qdnBaseWithPath = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
         this.data = data;
         this.qdnContext = qdnContext;
         this.resourceId = resourceId;
@@ -35,6 +36,7 @@ public class HTMLParser {
         this.identifier = identifier;
         this.path = inPath;
         this.theme = theme;
+        this.usingCustomRouting = usingCustomRouting;
     }
 
     public void addAdditionalHeaderTags() {
@@ -58,11 +60,13 @@ public class HTMLParser {
             String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"") : "";
             String path = this.path != null ? this.path.replace("\"","\\\"") : "";
             String theme = this.theme != null ? this.theme.replace("\"","\\\"") : "";
-            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\";</script>", this.qdnContext, theme, service, name, identifier, path, this.qdnBase);
+            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", this.qdnContext, theme, service, name, identifier, path, this.qdnBase, this.qdnBaseWithPath);
             head.get(0).prepend(qdnContextVar);
 
             // Add base href tag
-            String baseElement = String.format("<base href=\"%s/\">", this.linkPrefix);
+            // Exclude the path if this request was routed back to the index automatically
+            String baseHref = this.usingCustomRouting ? this.qdnBase : this.qdnBaseWithPath;
+            String baseElement = String.format("<base href=\"%s/\">", baseHref);
             head.get(0).prepend(baseElement);
 
             // Add meta charset tag
diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
index 66fc7b98..97641f32 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
@@ -126,7 +126,8 @@ public class ArbitraryDataRenderer {
         try {
             String filename = this.getFilename(unzippedPath, inPath);
             Path filePath = Paths.get(unzippedPath, filename);
-
+            boolean usingCustomRouting = false;
+            
             // If the file doesn't exist, we may need to route the request elsewhere, or cleanup
             if (!Files.exists(filePath)) {
                 if (inPath.equals("/")) {
@@ -148,6 +149,7 @@ public class ArbitraryDataRenderer {
                             // Forward request to index file
                             filePath = indexPath;
                             filename = indexFile;
+                            usingCustomRouting = true;
                             break;
                         }
                     }
@@ -157,7 +159,7 @@ public class ArbitraryDataRenderer {
             if (HTMLParser.isHtmlFile(filename)) {
                 // HTML file - needs to be parsed
                 byte[] data = Files.readAllBytes(filePath); // TODO: limit file size that can be read into memory
-                HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data, qdnContext, service, identifier, theme);
+                HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data, qdnContext, service, identifier, theme, usingCustomRouting);
                 htmlParser.addAdditionalHeaderTags();
                 response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:;");
                 response.setContentType(context.getMimeType(filename));

From 36e944d7e2fb7b3670d6bb36f549bd2d9b96925f Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sat, 29 Apr 2023 17:45:38 +0100
Subject: [PATCH 16/48] Added MAIL and MESSAGE services.

---
 src/main/java/org/qortal/arbitrary/misc/Service.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java
index 03c38a56..fc35bc6f 100644
--- a/src/main/java/org/qortal/arbitrary/misc/Service.java
+++ b/src/main/java/org/qortal/arbitrary/misc/Service.java
@@ -159,7 +159,9 @@ public enum Service {
     DATABASE(1700, false, null, false, null),
     SNAPSHOT(1710, false, null, false, null),
     COMMENT(1800, true, 500*1024L, true, null),
-    CHAIN_COMMENT(1810, true, 239L, true, null);
+    CHAIN_COMMENT(1810, true, 239L, true, null),
+    MAIL(1900, true, null, true, null),
+    MESSAGE(1910, true, null, true, null);
 
     public final int value;
     private final boolean requiresValidation;

From 95a1c6bf8b0f80f143e73f73347d074389106dbe Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sat, 29 Apr 2023 17:48:58 +0100
Subject: [PATCH 17/48] Added "encoding" parameter to the SEARCH_CHAT_MESSAGES
 action.

---
 Q-Apps.md                           | 1 +
 src/main/resources/q-apps/q-apps.js | 1 +
 2 files changed, 2 insertions(+)

diff --git a/Q-Apps.md b/Q-Apps.md
index 0f5bc7e8..94f7414f 100644
--- a/Q-Apps.md
+++ b/Q-Apps.md
@@ -539,6 +539,7 @@ let res = await qortalRequest({
     // reference: "reference", // Optional
     // chatReference: "chatreference", // Optional
     // hasChatReference: true, // Optional
+    encoding: "BASE64", // Optional (defaults to BASE58 if omitted)
     limit: 100,
     offset: 0,
     reverse: true
diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js
index 9da494c0..f6075e8e 100644
--- a/src/main/resources/q-apps/q-apps.js
+++ b/src/main/resources/q-apps/q-apps.js
@@ -264,6 +264,7 @@ window.addEventListener("message", (event) => {
             if (data.reference != null) url = url.concat("&reference=" + data.reference);
             if (data.chatReference != null) url = url.concat("&chatreference=" + data.chatReference);
             if (data.hasChatReference != null) url = url.concat("&haschatreference=" + new Boolean(data.hasChatReference).toString());
+            if (data.encoding != null) url = url.concat("&encoding=" + data.encoding);
             if (data.limit != null) url = url.concat("&limit=" + data.limit);
             if (data.offset != null) url = url.concat("&offset=" + data.offset);
             if (data.reverse != null) url = url.concat("&reverse=" + new Boolean(data.reverse).toString());

From 34c3adf280da2ab263af48bae7bb957979ea2b95 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sat, 29 Apr 2023 19:04:17 +0100
Subject: [PATCH 18/48] Limit MAIL and MESSAGE to 1MB.

---
 src/main/java/org/qortal/arbitrary/misc/Service.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java
index fc35bc6f..27557045 100644
--- a/src/main/java/org/qortal/arbitrary/misc/Service.java
+++ b/src/main/java/org/qortal/arbitrary/misc/Service.java
@@ -160,8 +160,8 @@ public enum Service {
     SNAPSHOT(1710, false, null, false, null),
     COMMENT(1800, true, 500*1024L, true, null),
     CHAIN_COMMENT(1810, true, 239L, true, null),
-    MAIL(1900, true, null, true, null),
-    MESSAGE(1910, true, null, true, null);
+    MAIL(1900, true, 1024*1024L, true, null),
+    MESSAGE(1910, true, 1024*1024L, true, null);
 
     public final int value;
     private final boolean requiresValidation;

From c71dce92b5a43a8759362836f5cefe9bc2354153 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Mon, 1 May 2023 19:34:01 +0100
Subject: [PATCH 19/48] Bump version to 4.0.1

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index bbb7bba3..ff9c9db1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>4.0.0</version>
+	<version>4.0.1</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

From 611240650ed031b3fcf26f8d4b93fcacc8bf7a15 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Wed, 3 May 2023 19:27:59 +0100
Subject: [PATCH 20/48] Added GET /chat/messages/count endpoint, which is
 identical to /chat/messages but returns a count of the messages rather than
 the messages themselves.

---
 .../org/qortal/api/resource/ChatResource.java | 69 +++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/src/main/java/org/qortal/api/resource/ChatResource.java b/src/main/java/org/qortal/api/resource/ChatResource.java
index 986bb03d..22e90a43 100644
--- a/src/main/java/org/qortal/api/resource/ChatResource.java
+++ b/src/main/java/org/qortal/api/resource/ChatResource.java
@@ -119,6 +119,75 @@ public class ChatResource {
 		}
 	}
 
+	@GET
+	@Path("/messages/count")
+	@Operation(
+			summary = "Count chat messages",
+			description = "Returns count of CHAT messages that match criteria. Must provide EITHER 'txGroupId' OR two 'involving' addresses.",
+			responses = {
+					@ApiResponse(
+							description = "count of messages",
+							content = @Content(
+									mediaType = MediaType.TEXT_PLAIN,
+									schema = @Schema(
+											type = "integer"
+									)
+							)
+					)
+			}
+	)
+	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE})
+	public int countChatMessages(@QueryParam("before") Long before, @QueryParam("after") Long after,
+										@QueryParam("txGroupId") Integer txGroupId,
+										@QueryParam("involving") List<String> involvingAddresses,
+										@QueryParam("reference") String reference,
+										@QueryParam("chatreference") String chatReference,
+										@QueryParam("haschatreference") Boolean hasChatReference,
+										@QueryParam("sender") String sender,
+										@QueryParam("encoding") Encoding encoding,
+										@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
+										@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+										@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) {
+		// Check args meet expectations
+		if ((txGroupId == null && involvingAddresses.size() != 2)
+				|| (txGroupId != null && !involvingAddresses.isEmpty()))
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		// Check any provided addresses are valid
+		if (involvingAddresses.stream().anyMatch(address -> !Crypto.isValidAddress(address)))
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
+
+		if (before != null && before < 1500000000000L)
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		if (after != null && after < 1500000000000L)
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		byte[] referenceBytes = null;
+		if (reference != null)
+			referenceBytes = Base58.decode(reference);
+
+		byte[] chatReferenceBytes = null;
+		if (chatReference != null)
+			chatReferenceBytes = Base58.decode(chatReference);
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			return repository.getChatRepository().getMessagesMatchingCriteria(
+					before,
+					after,
+					txGroupId,
+					referenceBytes,
+					chatReferenceBytes,
+					hasChatReference,
+					involvingAddresses,
+					sender,
+					encoding,
+					limit, offset, reverse).size();
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}
+	}
+
 	@GET
 	@Path("/message/{signature}")
 	@Operation(

From e014a207efd2d639797688b5f0e48a90aad92bba Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Wed, 3 May 2023 19:28:26 +0100
Subject: [PATCH 21/48] Escape all vars added by HTML parser

---
 src/main/java/org/qortal/api/HTMLParser.java | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java
index 03cdb066..2bf8947d 100644
--- a/src/main/java/org/qortal/api/HTMLParser.java
+++ b/src/main/java/org/qortal/api/HTMLParser.java
@@ -55,12 +55,15 @@ public class HTMLParser {
             }
 
             // Escape and add vars
+            String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\"","\\\"") : "";
             String service = this.service.toString().replace("\"","\\\"");
             String name = this.resourceId != null ? this.resourceId.replace("\"","\\\"") : "";
             String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"") : "";
             String path = this.path != null ? this.path.replace("\"","\\\"") : "";
             String theme = this.theme != null ? this.theme.replace("\"","\\\"") : "";
-            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", this.qdnContext, theme, service, name, identifier, path, this.qdnBase, this.qdnBaseWithPath);
+            String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\"","\\\"") : "";
+            String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\"","\\\"") : "";
+            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath);
             head.get(0).prepend(qdnContextVar);
 
             // Add base href tag

From 9547a087b25401183c0ca7d87c3a757858bdd862 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Wed, 3 May 2023 19:38:31 +0100
Subject: [PATCH 22/48] Remove all backslashes from vars in HTML parser.

---
 src/main/java/org/qortal/api/HTMLParser.java | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java
index 2bf8947d..4e97e2bd 100644
--- a/src/main/java/org/qortal/api/HTMLParser.java
+++ b/src/main/java/org/qortal/api/HTMLParser.java
@@ -55,14 +55,14 @@ public class HTMLParser {
             }
 
             // Escape and add vars
-            String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\"","\\\"") : "";
-            String service = this.service.toString().replace("\"","\\\"");
-            String name = this.resourceId != null ? this.resourceId.replace("\"","\\\"") : "";
-            String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"") : "";
-            String path = this.path != null ? this.path.replace("\"","\\\"") : "";
-            String theme = this.theme != null ? this.theme.replace("\"","\\\"") : "";
-            String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\"","\\\"") : "";
-            String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\"","\\\"") : "";
+            String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\"","\\\"").replace("\\", "") : "";
+            String service = this.service.toString().replace("\"","\\\"").replace("\\", "");
+            String name = this.resourceId != null ? this.resourceId.replace("\"","\\\"").replace("\\", "") : "";
+            String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"").replace("\\", "") : "";
+            String path = this.path != null ? this.path.replace("\"","\\\"").replace("\\", "") : "";
+            String theme = this.theme != null ? this.theme.replace("\"","\\\"").replace("\\", "") : "";
+            String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\"","\\\"").replace("\\", "") : "";
+            String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\"","\\\"").replace("\\", "") : "";
             String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath);
             head.get(0).prepend(qdnContextVar);
 

From b9d81645f89a6f31f63405eb5bbc722abf281441 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Wed, 3 May 2023 19:40:17 +0100
Subject: [PATCH 23/48] Revert "Remove all backslashes from vars in HTML
 parser."

This reverts commit 9547a087b25401183c0ca7d87c3a757858bdd862.
---
 src/main/java/org/qortal/api/HTMLParser.java | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java
index 4e97e2bd..2bf8947d 100644
--- a/src/main/java/org/qortal/api/HTMLParser.java
+++ b/src/main/java/org/qortal/api/HTMLParser.java
@@ -55,14 +55,14 @@ public class HTMLParser {
             }
 
             // Escape and add vars
-            String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\"","\\\"").replace("\\", "") : "";
-            String service = this.service.toString().replace("\"","\\\"").replace("\\", "");
-            String name = this.resourceId != null ? this.resourceId.replace("\"","\\\"").replace("\\", "") : "";
-            String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"").replace("\\", "") : "";
-            String path = this.path != null ? this.path.replace("\"","\\\"").replace("\\", "") : "";
-            String theme = this.theme != null ? this.theme.replace("\"","\\\"").replace("\\", "") : "";
-            String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\"","\\\"").replace("\\", "") : "";
-            String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\"","\\\"").replace("\\", "") : "";
+            String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\"","\\\"") : "";
+            String service = this.service.toString().replace("\"","\\\"");
+            String name = this.resourceId != null ? this.resourceId.replace("\"","\\\"") : "";
+            String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"") : "";
+            String path = this.path != null ? this.path.replace("\"","\\\"") : "";
+            String theme = this.theme != null ? this.theme.replace("\"","\\\"") : "";
+            String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\"","\\\"") : "";
+            String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\"","\\\"") : "";
             String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath);
             head.get(0).prepend(qdnContextVar);
 

From 2dfee13d86b4f984b8ba8bccf7017568fdc8cb3b Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Wed, 3 May 2023 19:44:54 +0100
Subject: [PATCH 24/48] Remove all backslashes from vars in HTML parser
 (correct order this time)

---
 src/main/java/org/qortal/api/HTMLParser.java | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java
index 2bf8947d..cc3102e8 100644
--- a/src/main/java/org/qortal/api/HTMLParser.java
+++ b/src/main/java/org/qortal/api/HTMLParser.java
@@ -55,14 +55,14 @@ public class HTMLParser {
             }
 
             // Escape and add vars
-            String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\"","\\\"") : "";
-            String service = this.service.toString().replace("\"","\\\"");
-            String name = this.resourceId != null ? this.resourceId.replace("\"","\\\"") : "";
-            String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"") : "";
-            String path = this.path != null ? this.path.replace("\"","\\\"") : "";
-            String theme = this.theme != null ? this.theme.replace("\"","\\\"") : "";
-            String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\"","\\\"") : "";
-            String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\"","\\\"") : "";
+            String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\\", "").replace("\"","\\\"") : "";
+            String service = this.service.toString().replace("\\", "").replace("\"","\\\"");
+            String name = this.resourceId != null ? this.resourceId.replace("\\", "").replace("\"","\\\"") : "";
+            String identifier = this.identifier != null ? this.identifier.replace("\\", "").replace("\"","\\\"") : "";
+            String path = this.path != null ? this.path.replace("\\", "").replace("\"","\\\"") : "";
+            String theme = this.theme != null ? this.theme.replace("\\", "").replace("\"","\\\"") : "";
+            String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\\", "").replace("\"","\\\"") : "";
+            String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\\", "").replace("\"","\\\"") : "";
             String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath);
             head.get(0).prepend(qdnContextVar);
 

From f39b6a15da6e418b40b2a0102019da6ed2d84e14 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 5 May 2023 11:03:13 +0100
Subject: [PATCH 25/48] Fixed refresh bug on Windows.

---
 src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
index 97641f32..089a99ca 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
@@ -67,8 +67,8 @@ public class ArbitraryDataRenderer {
     }
 
     public HttpServletResponse render() {
-        if (!inPath.startsWith(File.separator)) {
-            inPath = File.separator + inPath;
+        if (!inPath.startsWith("/")) {
+            inPath = "/" + inPath;
         }
 
         // Don't render data if QDN is disabled

From 1a5e3b4fb1012c37530b23c2f5596fa33cd3ec17 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 5 May 2023 11:24:52 +0100
Subject: [PATCH 26/48] Added `GET /names/search` endpoint, to search names via
 case insensitive, partial name matching.

---
 .../qortal/api/resource/NamesResource.java    | 32 ++++++++++++
 .../org/qortal/repository/NameRepository.java |  2 +
 .../hsqldb/HSQLDBNameRepository.java          | 51 +++++++++++++++++++
 3 files changed, 85 insertions(+)

diff --git a/src/main/java/org/qortal/api/resource/NamesResource.java b/src/main/java/org/qortal/api/resource/NamesResource.java
index a900d6bf..03dffc08 100644
--- a/src/main/java/org/qortal/api/resource/NamesResource.java
+++ b/src/main/java/org/qortal/api/resource/NamesResource.java
@@ -155,6 +155,38 @@ public class NamesResource {
 		}
 	}
 
+	@GET
+	@Path("/search")
+	@Operation(
+			summary = "Search registered names",
+			responses = {
+					@ApiResponse(
+							description = "registered name info",
+							content = @Content(
+									mediaType = MediaType.APPLICATION_JSON,
+									array = @ArraySchema(schema = @Schema(implementation = NameData.class))
+							)
+					)
+			}
+	)
+	@ApiErrors({ApiError.NAME_UNKNOWN, ApiError.REPOSITORY_ISSUE})
+	public List<NameData> searchNames(@QueryParam("query") String query,
+									  @Parameter(ref = "limit") @QueryParam("limit") Integer limit,
+									  @Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+									  @Parameter(ref="reverse") @QueryParam("reverse") Boolean reverse) {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			if (query == null) {
+				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing query");
+			}
+
+			return repository.getNameRepository().searchNames(query, limit, offset, reverse);
+		} catch (ApiException e) {
+			throw e;
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}
+	}
+
 
 	@POST
 	@Path("/register")
diff --git a/src/main/java/org/qortal/repository/NameRepository.java b/src/main/java/org/qortal/repository/NameRepository.java
index d6c0f33e..a8b2a3db 100644
--- a/src/main/java/org/qortal/repository/NameRepository.java
+++ b/src/main/java/org/qortal/repository/NameRepository.java
@@ -14,6 +14,8 @@ public interface NameRepository {
 
 	public boolean reducedNameExists(String reducedName) throws DataException;
 
+	public List<NameData> searchNames(String query, Integer limit, Integer offset, Boolean reverse) throws DataException;
+
 	public List<NameData> getAllNames(Integer limit, Integer offset, Boolean reverse) throws DataException;
 
 	public default List<NameData> getAllNames() throws DataException {
diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java
index 3a3574ef..3e4a8e11 100644
--- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java
+++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java
@@ -103,6 +103,57 @@ public class HSQLDBNameRepository implements NameRepository {
 		}
 	}
 
+	public List<NameData> searchNames(String query, Integer limit, Integer offset, Boolean reverse) throws DataException {
+		StringBuilder sql = new StringBuilder(512);
+		List<Object> bindParams = new ArrayList<>();
+
+		sql.append("SELECT name, reduced_name, owner, data, registered_when, updated_when, "
+				+ "is_for_sale, sale_price, reference, creation_group_id FROM Names "
+				+ "WHERE LCASE(name) LIKE ? ORDER BY name");
+
+		bindParams.add(String.format("%%%s%%", query.toLowerCase()));
+
+		if (reverse != null && reverse)
+			sql.append(" DESC");
+
+		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
+
+		List<NameData> names = new ArrayList<>();
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
+			if (resultSet == null)
+				return names;
+
+			do {
+				String name = resultSet.getString(1);
+				String reducedName = resultSet.getString(2);
+				String owner = resultSet.getString(3);
+				String data = resultSet.getString(4);
+				long registered = resultSet.getLong(5);
+
+				// Special handling for possibly-NULL "updated" column
+				Long updated = resultSet.getLong(6);
+				if (updated == 0 && resultSet.wasNull())
+					updated = null;
+
+				boolean isForSale = resultSet.getBoolean(7);
+
+				Long salePrice = resultSet.getLong(8);
+				if (salePrice == 0 && resultSet.wasNull())
+					salePrice = null;
+
+				byte[] reference = resultSet.getBytes(9);
+				int creationGroupId = resultSet.getInt(10);
+
+				names.add(new NameData(name, reducedName, owner, data, registered, updated, isForSale, salePrice, reference, creationGroupId));
+			} while (resultSet.next());
+
+			return names;
+		} catch (SQLException e) {
+			throw new DataException("Unable to search names in repository", e);
+		}
+	}
+
 	@Override
 	public List<NameData> getAllNames(Integer limit, Integer offset, Boolean reverse) throws DataException {
 		StringBuilder sql = new StringBuilder(256);

From c172a5764b353cf6d80c1d22d30067dbc5cfbc85 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 5 May 2023 12:26:18 +0100
Subject: [PATCH 27/48] Added `_PRIVATE` services, to allow for
 publishing/validation of encrypted data.

New additions:

QCHAT_ATTACHMENT_PRIVATE
ATTACHMENT_PRIVATE
FILE_PRIVATE
IMAGE_PRIVATE
VIDEO_PRIVATE
AUDIO_PRIVATE
VOICE_PRIVATE
DOCUMENT_PRIVATE
MAIL_PRIVATE
MESSAGE_PRIVATE
---
 .../org/qortal/arbitrary/misc/Service.java    | 123 +++++++++++-------
 .../arbitrary/ArbitraryDataManager.java       |   1 -
 .../test/arbitrary/ArbitraryServiceTests.java |  67 ++++++++++
 3 files changed, 142 insertions(+), 49 deletions(-)

diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java
index 27557045..e0caa2a5 100644
--- a/src/main/java/org/qortal/arbitrary/misc/Service.java
+++ b/src/main/java/org/qortal/arbitrary/misc/Service.java
@@ -9,7 +9,6 @@ import org.qortal.utils.FilesystemUtils;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
@@ -20,9 +19,9 @@ import static java.util.Arrays.stream;
 import static java.util.stream.Collectors.toMap;
 
 public enum Service {
-    AUTO_UPDATE(1, false, null, false, null),
-    ARBITRARY_DATA(100, false, null, false, null),
-    QCHAT_ATTACHMENT(120, true, 1024*1024L, true, null) {
+    AUTO_UPDATE(1, false, null, false, false, null),
+    ARBITRARY_DATA(100, false, null, false, false, null),
+    QCHAT_ATTACHMENT(120, true, 1024*1024L, true, false, null) {
         @Override
         public ValidationResult validate(Path path) throws IOException {
             ValidationResult superclassResult = super.validate(path);
@@ -47,11 +46,14 @@ public enum Service {
             return ValidationResult.OK;
         }
     },
-    ATTACHMENT(130, false, null, true, null),
-    FILE(140, false, null, true, null),
-    FILES(150, false, null, false, null),
-    CHAIN_DATA(160, true, 239L, true, null),
-    WEBSITE(200, true, null, false, null) {
+    QCHAT_ATTACHMENT_PRIVATE(121, true, 1024*1024L, true, true, null),
+    ATTACHMENT(130, false, 50*1024*1024L, true, false, null),
+    ATTACHMENT_PRIVATE(131, true, 50*1024*1024L, true, true, null),
+    FILE(140, false, null, true, false, null),
+    FILE_PRIVATE(141, true, null, true, true, null),
+    FILES(150, false, null, false, false, null),
+    CHAIN_DATA(160, true, 239L, true, false, null),
+    WEBSITE(200, true, null, false, false, null) {
         @Override
         public ValidationResult validate(Path path) throws IOException {
             ValidationResult superclassResult = super.validate(path);
@@ -73,25 +75,30 @@ public enum Service {
             return ValidationResult.MISSING_INDEX_FILE;
         }
     },
-    GIT_REPOSITORY(300, false, null, false, null),
-    IMAGE(400, true, 10*1024*1024L, true, null),
-    THUMBNAIL(410, true, 500*1024L, true, null),
-    QCHAT_IMAGE(420, true, 500*1024L, true, null),
-    VIDEO(500, false, null, true, null),
-    AUDIO(600, false, null, true, null),
-    QCHAT_AUDIO(610, true, 10*1024*1024L, true, null),
-    QCHAT_VOICE(620, true, 10*1024*1024L, true, null),
-    VOICE(630, true, 10*1024*1024L, true, null),
-    PODCAST(640, false, null, true, null),
-    BLOG(700, false, null, false, null),
-    BLOG_POST(777, false, null, true, null),
-    BLOG_COMMENT(778, true, 500*1024L, true, null),
-    DOCUMENT(800, false, null, true, null),
-    LIST(900, true, null, true, null),
-    PLAYLIST(910, true, null, true, null),
-    APP(1000, true, 50*1024*1024L, false, null),
-    METADATA(1100, false, null, true, null),
-    JSON(1110, true, 25*1024L, true, null) {
+    GIT_REPOSITORY(300, false, null, false, false, null),
+    IMAGE(400, true, 10*1024*1024L, true, false, null),
+    IMAGE_PRIVATE(401, true, 10*1024*1024L, true, true, null),
+    THUMBNAIL(410, true, 500*1024L, true, false, null),
+    QCHAT_IMAGE(420, true, 500*1024L, true, false, null),
+    VIDEO(500, false, null, true, false, null),
+    VIDEO_PRIVATE(501, true, null, true, true, null),
+    AUDIO(600, false, null, true, false, null),
+    AUDIO_PRIVATE(601, true, null, true, true, null),
+    QCHAT_AUDIO(610, true, 10*1024*1024L, true, false, null),
+    QCHAT_VOICE(620, true, 10*1024*1024L, true, false, null),
+    VOICE(630, true, 10*1024*1024L, true, false, null),
+    VOICE_PRIVATE(631, true, 10*1024*1024L, true, true, null),
+    PODCAST(640, false, null, true, false, null),
+    BLOG(700, false, null, false, false, null),
+    BLOG_POST(777, false, null, true, false, null),
+    BLOG_COMMENT(778, true, 500*1024L, true, false, null),
+    DOCUMENT(800, false, null, true, false, null),
+    DOCUMENT_PRIVATE(801, true, null, true, true, null),
+    LIST(900, true, null, true, false, null),
+    PLAYLIST(910, true, null, true, false, null),
+    APP(1000, true, 50*1024*1024L, false, false, null),
+    METADATA(1100, false, null, true, false, null),
+    JSON(1110, true, 25*1024L, true, false, null) {
         @Override
         public ValidationResult validate(Path path) throws IOException {
             ValidationResult superclassResult = super.validate(path);
@@ -110,7 +117,7 @@ public enum Service {
             }
         }
     },
-    GIF_REPOSITORY(1200, true, 25*1024*1024L, false, null) {
+    GIF_REPOSITORY(1200, true, 25*1024*1024L, false, false, null) {
         @Override
         public ValidationResult validate(Path path) throws IOException {
             ValidationResult superclassResult = super.validate(path);
@@ -146,27 +153,30 @@ public enum Service {
             return ValidationResult.OK;
         }
     },
-    STORE(1300, false, null, true, null),
-    PRODUCT(1310, false, null, true, null),
-    OFFER(1330, false, null, true, null),
-    COUPON(1340, false, null, true, null),
-    CODE(1400, false, null, true, null),
-    PLUGIN(1410, false, null, true, null),
-    EXTENSION(1420, false, null, true, null),
-    GAME(1500, false, null, false, null),
-    ITEM(1510, false, null, true, null),
-    NFT(1600, false, null, true, null),
-    DATABASE(1700, false, null, false, null),
-    SNAPSHOT(1710, false, null, false, null),
-    COMMENT(1800, true, 500*1024L, true, null),
-    CHAIN_COMMENT(1810, true, 239L, true, null),
-    MAIL(1900, true, 1024*1024L, true, null),
-    MESSAGE(1910, true, 1024*1024L, true, null);
+    STORE(1300, false, null, true, false, null),
+    PRODUCT(1310, false, null, true, false, null),
+    OFFER(1330, false, null, true, false, null),
+    COUPON(1340, false, null, true, false, null),
+    CODE(1400, false, null, true, false, null),
+    PLUGIN(1410, false, null, true, false, null),
+    EXTENSION(1420, false, null, true, false, null),
+    GAME(1500, false, null, false, false, null),
+    ITEM(1510, false, null, true, false, null),
+    NFT(1600, false, null, true, false, null),
+    DATABASE(1700, false, null, false, false, null),
+    SNAPSHOT(1710, false, null, false, false, null),
+    COMMENT(1800, true, 500*1024L, true, false, null),
+    CHAIN_COMMENT(1810, true, 239L, true, false, null),
+    MAIL(1900, true, 1024*1024L, true, false, null),
+    MAIL_PRIVATE(1901, true, 1024*1024L, true, true, null),
+    MESSAGE(1910, true, 1024*1024L, true, false, null),
+    MESSAGE_PRIVATE(1911, true, 1024*1024L, true, true, null);
 
     public final int value;
     private final boolean requiresValidation;
     private final Long maxSize;
     private final boolean single;
+    private final boolean isPrivate;
     private final List<String> requiredKeys;
 
     private static final Map<Integer, Service> map = stream(Service.values())
@@ -175,11 +185,14 @@ public enum Service {
     // For JSON validation
     private static final ObjectMapper objectMapper = new ObjectMapper();
 
-    Service(int value, boolean requiresValidation, Long maxSize, boolean single, List<String> requiredKeys) {
+    private static final String encryptedDataPrefix = "qortalEncryptedData";
+
+    Service(int value, boolean requiresValidation, Long maxSize, boolean single, boolean isPrivate, List<String> requiredKeys) {
         this.value = value;
         this.requiresValidation = requiresValidation;
         this.maxSize = maxSize;
         this.single = single;
+        this.isPrivate = isPrivate;
         this.requiredKeys = requiredKeys;
     }
 
@@ -203,6 +216,17 @@ public enum Service {
             return ValidationResult.INVALID_FILE_COUNT;
         }
 
+        // Validate private data for single file resources
+        if (this.single) {
+            String dataString = new String(data, StandardCharsets.UTF_8);
+            if (this.isPrivate && !dataString.startsWith(encryptedDataPrefix)) {
+                return ValidationResult.DATA_NOT_ENCRYPTED;
+            }
+            if (!this.isPrivate && dataString.startsWith(encryptedDataPrefix)) {
+                return ValidationResult.DATA_ENCRYPTED;
+            }
+        }
+
         // Validate required keys if needed
         if (this.requiredKeys != null) {
             if (data == null) {
@@ -221,7 +245,8 @@ public enum Service {
     }
 
     public boolean isValidationRequired() {
-        return this.requiresValidation;
+        // We must always validate single file resources, to ensure they are actually a single file
+        return this.requiresValidation || this.single;
     }
 
     public static Service valueOf(int value) {
@@ -242,7 +267,9 @@ public enum Service {
         INVALID_FILE_EXTENSION(6),
         MISSING_DATA(7),
         INVALID_FILE_COUNT(8),
-        INVALID_CONTENT(9);
+        INVALID_CONTENT(9),
+        DATA_NOT_ENCRYPTED(10),
+        DATA_ENCRYPTED(10);
 
         public final int value;
 
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
index 567dcdd3..9284e672 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
@@ -16,7 +16,6 @@ import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.Controller;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
-import org.qortal.list.ResourceListManager;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
 import org.qortal.repository.DataException;
diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java
index 940b33a9..45960a25 100644
--- a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java
+++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java
@@ -436,4 +436,71 @@ public class ArbitraryServiceTests extends Common {
         assertEquals(ValidationResult.INVALID_FILE_COUNT, service.validate(path));
     }
 
+    @Test
+    public void testValidPrivateData() throws IOException {
+        String dataString = "qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1rfla5UBoEcoxbtjgZ9G7FLPb8V/Qfr0bfKWfvMmN06U/pgUdLuv2mGL2V0D3qYd1011MUzGdNG1qERjaCDz8GAi63+KnHHjfMtPgYt6bcqjs4CNV+ZZ4dIt3xxHYyVEBNc=";
+
+        // Write the data a single file in a temp path
+        Path path = Files.createTempDirectory("testValidPrivateData");
+        Path filePath = Paths.get(path.toString(), "test");
+        filePath.toFile().deleteOnExit();
+
+        BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile()));
+        writer.write(dataString);
+        writer.close();
+
+        Service service = Service.FILE_PRIVATE;
+        assertTrue(service.isValidationRequired());
+
+        assertEquals(ValidationResult.OK, service.validate(filePath));
+    }
+
+    @Test
+    public void testEncryptedData() throws IOException {
+        String dataString = "qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1rfla5UBoEcoxbtjgZ9G7FLPb8V/Qfr0bfKWfvMmN06U/pgUdLuv2mGL2V0D3qYd1011MUzGdNG1qERjaCDz8GAi63+KnHHjfMtPgYt6bcqjs4CNV+ZZ4dIt3xxHYyVEBNc=";
+
+        // Write the data a single file in a temp path
+        Path path = Files.createTempDirectory("testValidPrivateData");
+        Path filePath = Paths.get(path.toString(), "test");
+        filePath.toFile().deleteOnExit();
+
+        BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile()));
+        writer.write(dataString);
+        writer.close();
+
+        // Validate a private service
+        Service service = Service.FILE_PRIVATE;
+        assertTrue(service.isValidationRequired());
+        assertEquals(ValidationResult.OK, service.validate(filePath));
+
+        // Validate a regular service
+        service = Service.FILE;
+        assertTrue(service.isValidationRequired());
+        assertEquals(ValidationResult.DATA_ENCRYPTED, service.validate(filePath));
+    }
+
+    @Test
+    public void testPlainTextData() throws IOException {
+        String dataString = "plaintext";
+
+        // Write the data a single file in a temp path
+        Path path = Files.createTempDirectory("testInvalidPrivateData");
+        Path filePath = Paths.get(path.toString(), "test");
+        filePath.toFile().deleteOnExit();
+
+        BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile()));
+        writer.write(dataString);
+        writer.close();
+
+        // Validate a private service
+        Service service = Service.FILE_PRIVATE;
+        assertTrue(service.isValidationRequired());
+        assertEquals(ValidationResult.DATA_NOT_ENCRYPTED, service.validate(filePath));
+
+        // Validate a regular service
+        service = Service.FILE;
+        assertTrue(service.isValidationRequired());
+        assertEquals(ValidationResult.OK, service.validate(filePath));
+    }
+
 }
\ No newline at end of file

From 3775135e0cfadc53dd24c891386da896d6127488 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 5 May 2023 12:39:11 +0100
Subject: [PATCH 28/48] Added helper methods to fetch lists of private or
 public service objects.

These can ultimately be used to help inform the cleanup manager on the best order to delete files when the node runs out of space. Public data should be given priority over private data (unless the node is part of a data market contract for that data - this isn't developed yet).
---
 .../org/qortal/arbitrary/misc/Service.java    | 35 +++++++++++++++++++
 .../ArbitraryDataCleanupManager.java          |  4 +++
 .../test/arbitrary/ArbitraryServiceTests.java | 17 +++++++++
 3 files changed, 56 insertions(+)

diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java
index e0caa2a5..b53ab7ca 100644
--- a/src/main/java/org/qortal/arbitrary/misc/Service.java
+++ b/src/main/java/org/qortal/arbitrary/misc/Service.java
@@ -249,6 +249,10 @@ public enum Service {
         return this.requiresValidation || this.single;
     }
 
+    public boolean isPrivate() {
+        return this.isPrivate;
+    }
+
     public static Service valueOf(int value) {
         return map.get(value);
     }
@@ -258,6 +262,37 @@ public enum Service {
         return new JSONObject(dataString);
     }
 
+    public static List<Service> publicServices() {
+        List<Service> privateServices = new ArrayList<>();
+        for (Service service : Service.values()) {
+            if (!service.isPrivate) {
+                privateServices.add(service);
+            }
+        }
+        return privateServices;
+    }
+
+    /**
+     * Fetch a list of Service objects that require encrypted data.
+     *
+     * These can ultimately be used to help inform the cleanup manager
+     * on the best order to delete files when the node runs out of space.
+     * Public data should be given priority over private data (unless
+     * this node is part of a data market contract for that data - this
+     * isn't developed yet).
+     *
+     * @return a list of Service objects that require encrypted data.
+     */
+    public static List<Service> privateServices() {
+        List<Service> privateServices = new ArrayList<>();
+        for (Service service : Service.values()) {
+            if (service.isPrivate) {
+                privateServices.add(service);
+            }
+        }
+        return privateServices;
+    }
+
     public enum ValidationResult {
         OK(1),
         MISSING_KEYS(2),
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
index 9d57ce8a..e0c62acb 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
@@ -346,6 +346,10 @@ public class ArbitraryDataCleanupManager extends Thread {
 	/**
 	 * Iteratively walk through given directory and delete a single random file
 	 *
+	 * TODO: public data should be prioritized over private data
+	 * (unless this node is part of a data market contract for that data).
+	 * See: Service.privateServices() for a list of services containing private data.
+	 *
 	 * @param directory - the base directory
 	 * @return boolean - whether a file was deleted
 	 */
diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java
index 45960a25..33632b4a 100644
--- a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java
+++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java
@@ -29,6 +29,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
+import java.util.List;
 import java.util.Random;
 
 import static org.junit.Assert.*;
@@ -503,4 +504,20 @@ public class ArbitraryServiceTests extends Common {
         assertEquals(ValidationResult.OK, service.validate(filePath));
     }
 
+    @Test
+    public void testGetPrivateServices() {
+        List<Service> privateServices = Service.privateServices();
+        for (Service service : privateServices) {
+            assertTrue(service.isPrivate());
+        }
+    }
+
+    @Test
+    public void testGetPublicServices() {
+        List<Service> publicServices = Service.publicServices();
+        for (Service service : publicServices) {
+            assertFalse(service.isPrivate());
+        }
+    }
+
 }
\ No newline at end of file

From 86b5bae320f5f1d9fe1e731071698982bb320bf0 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 5 May 2023 13:22:14 +0100
Subject: [PATCH 29/48] Set timeout of PUBLISH_MULTIPLE_QDN_RESOURCES to 60
 mins.

---
 src/main/resources/q-apps/q-apps.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js
index f6075e8e..ab82b6b8 100644
--- a/src/main/resources/q-apps/q-apps.js
+++ b/src/main/resources/q-apps/q-apps.js
@@ -440,8 +440,8 @@ function getDefaultTimeout(action) {
                 return 60 * 1000;
 
             case "PUBLISH_QDN_RESOURCE":
+            case "PUBLISH_MULTIPLE_QDN_RESOURCES":
                 // Publishing could take a very long time on slow system, due to the proof-of-work computation
-                // It's best not to timeout
                 return 60 * 60 * 1000;
 
             case "SEND_CHAT_MESSAGE":

From 3f71a63512f6876eea2486c9cf7a514419c9184c Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 5 May 2023 18:30:14 +0100
Subject: [PATCH 30/48] Increased timeout for other new actions.

---
 src/main/resources/q-apps/q-apps.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js
index ab82b6b8..5233c7d5 100644
--- a/src/main/resources/q-apps/q-apps.js
+++ b/src/main/resources/q-apps/q-apps.js
@@ -432,6 +432,8 @@ function getDefaultTimeout(action) {
         // Some actions need longer default timeouts, especially those that create transactions
         switch (action) {
             case "GET_USER_ACCOUNT":
+            case "SAVE_FILE":
+            case "DECRYPT_DATA":
                 // User may take a long time to accept/deny the popup
                 return 60 * 60 * 1000;
 

From 92b983a16e68f4db93e3ab82258e24166229462c Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 5 May 2023 22:25:12 +0100
Subject: [PATCH 31/48] Q-Apps documentation updates.

---
 Q-Apps.md | 65 ++++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 55 insertions(+), 10 deletions(-)

diff --git a/Q-Apps.md b/Q-Apps.md
index 94f7414f..c7579f1d 100644
--- a/Q-Apps.md
+++ b/Q-Apps.md
@@ -42,6 +42,9 @@ A "default" resource refers to one without an identifier. For example, when a we
 
 Here is a list of currently available services that can be used in Q-Apps:
 
+### Public services ###
+The services below are intended to be used for publicly accessible data.
+
 IMAGE,
 THUMBNAIL,
 VIDEO,
@@ -83,6 +86,20 @@ QCHAT_IMAGE,
 QCHAT_AUDIO,
 QCHAT_VOICE
 
+### Private services ###
+For the services below, data is encrypted for a single recipient, and can only be decrypted using the private key of the recipient's wallet.
+
+QCHAT_ATTACHMENT_PRIVATE
+ATTACHMENT_PRIVATE
+FILE_PRIVATE
+IMAGE_PRIVATE
+VIDEO_PRIVATE
+AUDIO_PRIVATE
+VOICE_PRIVATE
+DOCUMENT_PRIVATE
+MAIL_PRIVATE
+MESSAGE_PRIVATE
+
 
 ## Single vs multi-file resources
 
@@ -246,6 +263,8 @@ Here is a list of currently supported actions:
 - FETCH_QDN_RESOURCE
 - PUBLISH_QDN_RESOURCE
 - PUBLISH_MULTIPLE_QDN_RESOURCES
+- DECRYPT_DATA
+- SAVE_FILE
 - GET_WALLET_BALANCE
 - GET_BALANCE
 - SEND_COIN
@@ -435,7 +454,7 @@ let res = await qortalRequest({
 _Requires user approval_.<br />
 Note: this publishes a single, base64-encoded file. Multi-file resource publishing (such as a WEBSITE or GIF_REPOSITORY) is not yet supported via a Q-App. It will be added in a future update.
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "PUBLISH_QDN_RESOURCE",
     name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
     service: "IMAGE",
@@ -449,7 +468,9 @@ await qortalRequest({
     // tag2: "strings", // Optional
     // tag3: "can", // Optional
     // tag4: "go", // Optional
-    // tag5: "here" // Optional
+    // tag5: "here", // Optional
+    // encrypt: true, // Optional - to be used with a private service
+    // recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
 });
 ```
 
@@ -457,7 +478,7 @@ await qortalRequest({
 _Requires user approval_.<br />
 Note: each resource being published consists of a single, base64-encoded file, each in its own transaction. Useful for publishing two or more related things, such as a video and a video thumbnail.
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "PUBLISH_MULTIPLE_QDN_RESOURCES",
     resources: [
         name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
@@ -472,7 +493,9 @@ await qortalRequest({
         // tag2: "strings", // Optional
         // tag3: "can", // Optional
         // tag4: "go", // Optional
-        // tag5: "here" // Optional
+        // tag5: "here", // Optional
+        // encrypt: true, // Optional - to be used with a private service
+        // recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
     ],
     [
         ... more resources here if needed ...
@@ -480,10 +503,32 @@ await qortalRequest({
 });
 ```
 
+### Decrypt encrypted/private data
+```
+let res = await qortalRequest({
+    action: "DECRYPT_DATA",
+    encryptedData: 'qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1r',
+    publicKey: 'publickeygoeshere'
+});
+// Returns base64 encoded string of plaintext data
+```
+
+### Prompt user to save a file to disk
+Note: mimeType not required but recommended. If not specified, saving will fail if the mimeType is unable to be derived from the Blob.
+```
+let res = await qortalRequest({
+    action: "SAVE_FILE",
+    blob: dataBlob,
+    filename: "myfile.pdf",
+    mimeType: "application/pdf" // Optional but recommended
+});
+```
+
+
 ### Get wallet balance (QORT)
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "GET_WALLET_BALANCE",
     coin: "QORT"
 });
@@ -508,7 +553,7 @@ let res = await qortalRequest({
 ### Send QORT to address
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "SEND_COIN",
     coin: "QORT",
     destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
@@ -519,7 +564,7 @@ await qortalRequest({
 ### Send foreign coin to address
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "SEND_COIN",
     coin: "LTC",
     destinationAddress: "LSdTvMHRm8sScqwCi6x9wzYQae8JeZhx6y",
@@ -549,7 +594,7 @@ let res = await qortalRequest({
 ### Send a group chat message
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "SEND_CHAT_MESSAGE",
     groupId: 0,
     message: "Test"
@@ -559,7 +604,7 @@ await qortalRequest({
 ### Send a private chat message
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "SEND_CHAT_MESSAGE",
     destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
     message: "Test"
@@ -579,7 +624,7 @@ let res = await qortalRequest({
 ### Join a group
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "JOIN_GROUP",
     groupId: 100
 });

From b5719311275e8ba780ceb3eed301f612029f2a9b Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Fri, 5 May 2023 22:35:19 +0100
Subject: [PATCH 32/48] Fixed formatting of services list

---
 Q-Apps.md | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/Q-Apps.md b/Q-Apps.md
index c7579f1d..bad9abe0 100644
--- a/Q-Apps.md
+++ b/Q-Apps.md
@@ -89,15 +89,15 @@ QCHAT_VOICE
 ### Private services ###
 For the services below, data is encrypted for a single recipient, and can only be decrypted using the private key of the recipient's wallet.
 
-QCHAT_ATTACHMENT_PRIVATE
-ATTACHMENT_PRIVATE
-FILE_PRIVATE
-IMAGE_PRIVATE
-VIDEO_PRIVATE
-AUDIO_PRIVATE
-VOICE_PRIVATE
-DOCUMENT_PRIVATE
-MAIL_PRIVATE
+QCHAT_ATTACHMENT_PRIVATE,
+ATTACHMENT_PRIVATE,
+FILE_PRIVATE,
+IMAGE_PRIVATE,
+VIDEO_PRIVATE,
+AUDIO_PRIVATE,
+VOICE_PRIVATE,
+DOCUMENT_PRIVATE,
+MAIL_PRIVATE,
 MESSAGE_PRIVATE
 
 

From b693a514fd53a850b1cbbf7ed7edf2c632075e6f Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sat, 6 May 2023 12:13:41 +0100
Subject: [PATCH 33/48] Fixed warnings, and other improvements.

---
 .../api/resource/ArbitraryResource.java       |   8 +-
 .../arbitrary/ArbitraryDataBuilder.java       |   8 -
 .../qortal/arbitrary/ArbitraryDataReader.java |  10 +-
 .../arbitrary/ArbitraryDataResource.java      |   3 +
 .../controller/OnlineAccountsManager.java     | 186 +++++++++---------
 5 files changed, 110 insertions(+), 105 deletions(-)

diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
index 1101e71d..dee27413 100644
--- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java
+++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
@@ -1173,7 +1173,11 @@ public class ArbitraryResource {
 				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, error);
 			}
 
-			final Long minLatestBlockTimestamp = NTP.getTime() - (60 * 60 * 1000L);
+			final Long now = NTP.getTime();
+			if (now == null) {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NO_TIME_SYNC);
+			}
+			final Long minLatestBlockTimestamp = now - (60 * 60 * 1000L);
 			if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp)) {
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC);
 			}
@@ -1231,7 +1235,7 @@ public class ArbitraryResource {
 					// The actual data will be in a randomly-named subfolder of tempDirectory
 					// Remove hidden folders, i.e. starting with "_", as some systems can add them, e.g. "__MACOSX"
 					String[] files = tempDirectory.toFile().list((parent, child) -> !child.startsWith("_"));
-					if (files.length == 1) { // Single directory or file only
+					if (files != null && files.length == 1) { // Single directory or file only
 						path = Paths.get(tempDirectory.toString(), files[0]).toString();
 					}
 				}
diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
index b6b17ea5..fba6a32b 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
@@ -54,10 +54,6 @@ public class ArbitraryDataBuilder {
     /**
      * Process transactions, but do not build anything
      * This is useful for checking the status of a given resource
-     *
-     * @throws DataException
-     * @throws IOException
-     * @throws MissingDataException
      */
     public void process() throws DataException, IOException, MissingDataException {
         this.fetchTransactions();
@@ -69,10 +65,6 @@ public class ArbitraryDataBuilder {
 
     /**
      * Build the latest state of a given resource
-     *
-     * @throws DataException
-     * @throws IOException
-     * @throws MissingDataException
      */
     public void build() throws DataException, IOException, MissingDataException {
         this.process();
diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
index 779e4024..a7876236 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
@@ -9,7 +9,6 @@ import org.qortal.arbitrary.exception.MissingDataException;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
-import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
 import org.qortal.crypto.AES;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ArbitraryTransactionData.*;
@@ -154,9 +153,6 @@ public class ArbitraryDataReader {
      * If no exception is thrown, you can then use getFilePath() to access the data immediately after returning
      *
      * @param overwrite - set to true to force rebuild an existing cache
-     * @throws IOException
-     * @throws DataException
-     * @throws MissingDataException
      */
     public void loadSynchronously(boolean overwrite) throws DataException, IOException, MissingDataException {
         try {
@@ -223,7 +219,6 @@ public class ArbitraryDataReader {
     /**
      * Working directory should only be deleted on failure, since it is currently used to
      * serve a cached version of the resource for subsequent requests.
-     * @throws IOException
      */
     private void deleteWorkingDirectory() {
         try {
@@ -303,7 +298,7 @@ public class ArbitraryDataReader {
                 break;
 
             default:
-                throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType.toString()));
+                throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType));
         }
     }
 
@@ -368,6 +363,9 @@ public class ArbitraryDataReader {
         // Load data file(s)
         ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
         ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
+        if (arbitraryDataFile == null) {
+            throw new DataException(String.format("arbitraryDataFile is null"));
+        }
 
         if (!arbitraryDataFile.allFilesExist()) {
             if (ListUtils.isNameBlocked(transactionData.getName())) {
diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java
index 79bb882b..a4650dfc 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java
@@ -150,6 +150,9 @@ public class ArbitraryDataResource {
 
             for (ArbitraryTransactionData transactionData : transactionDataList) {
                 ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
+                if (arbitraryDataFile == null) {
+                    continue;
+                }
 
                 // Delete any chunks or complete files from each transaction
                 arbitraryDataFile.deleteAll(deleteMetadata);
diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java
index fd2c38df..224228b8 100644
--- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java
+++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java
@@ -504,110 +504,118 @@ public class OnlineAccountsManager {
         computeOurAccountsForTimestamp(onlineAccountsTimestamp);
     }
 
-    private boolean computeOurAccountsForTimestamp(long onlineAccountsTimestamp) {
-        List<MintingAccountData> mintingAccounts;
-        try (final Repository repository = RepositoryManager.getRepository()) {
-            mintingAccounts = repository.getAccountRepository().getMintingAccounts();
+    private boolean computeOurAccountsForTimestamp(Long onlineAccountsTimestamp) {
+        if (onlineAccountsTimestamp != null) {
+            List<MintingAccountData> mintingAccounts;
+            try (final Repository repository = RepositoryManager.getRepository()) {
+                mintingAccounts = repository.getAccountRepository().getMintingAccounts();
 
-            // We have no accounts to send
-            if (mintingAccounts.isEmpty())
+                // We have no accounts to send
+                if (mintingAccounts.isEmpty())
+                    return false;
+
+                // Only active reward-shares allowed
+                Iterator<MintingAccountData> iterator = mintingAccounts.iterator();
+                int i = 0;
+                while (iterator.hasNext()) {
+                    MintingAccountData mintingAccountData = iterator.next();
+
+                    RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
+                    if (rewardShareData == null) {
+                        // Reward-share doesn't even exist - probably not a good sign
+                        iterator.remove();
+                        continue;
+                    }
+
+                    Account mintingAccount = new Account(repository, rewardShareData.getMinter());
+                    if (!mintingAccount.canMint()) {
+                        // Minting-account component of reward-share can no longer mint - disregard
+                        iterator.remove();
+                        continue;
+                    }
+
+                    if (++i > 1 + 1) {
+                        iterator.remove();
+                        continue;
+                    }
+                }
+            } catch (DataException e) {
+                LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
                 return false;
+            }
 
-            // Only active reward-shares allowed
-            Iterator<MintingAccountData> iterator = mintingAccounts.iterator();
-            while (iterator.hasNext()) {
-                MintingAccountData mintingAccountData = iterator.next();
+            byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
+            List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
 
-                RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
-                if (rewardShareData == null) {
-                    // Reward-share doesn't even exist - probably not a good sign
-                    iterator.remove();
+            int remaining = mintingAccounts.size();
+            for (MintingAccountData mintingAccountData : mintingAccounts) {
+                remaining--;
+                byte[] privateKey = mintingAccountData.getPrivateKey();
+                byte[] publicKey = Crypto.toPublicKey(privateKey);
+
+                // We don't want to compute the online account nonce and signature again if it already exists
+                Set<OnlineAccountData> onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountsTimestamp, k -> ConcurrentHashMap.newKeySet());
+                boolean alreadyExists = onlineAccounts.stream().anyMatch(a -> Arrays.equals(a.getPublicKey(), publicKey));
+                if (alreadyExists) {
+                    this.hasOurOnlineAccounts = true;
+
+                    if (remaining > 0) {
+                        // Move on to next account
+                        continue;
+                    } else {
+                        // Everything exists, so return true
+                        return true;
+                    }
+                }
+
+                // Generate bytes for mempow
+                byte[] mempowBytes;
+                try {
+                    mempowBytes = this.getMemoryPoWBytes(publicKey, onlineAccountsTimestamp);
+                } catch (IOException e) {
+                    LOGGER.info("Unable to create bytes for MemoryPoW. Moving on to next account...");
                     continue;
                 }
 
-                Account mintingAccount = new Account(repository, rewardShareData.getMinter());
-                if (!mintingAccount.canMint()) {
-                    // Minting-account component of reward-share can no longer mint - disregard
-                    iterator.remove();
-                    continue;
-                }
-            }
-        } catch (DataException e) {
-            LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
-            return false;
-        }
-
-        byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
-        List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
-
-        int remaining = mintingAccounts.size();
-        for (MintingAccountData mintingAccountData : mintingAccounts) {
-            remaining--;
-            byte[] privateKey = mintingAccountData.getPrivateKey();
-            byte[] publicKey = Crypto.toPublicKey(privateKey);
-
-            // We don't want to compute the online account nonce and signature again if it already exists
-            Set<OnlineAccountData> onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountsTimestamp, k -> ConcurrentHashMap.newKeySet());
-            boolean alreadyExists = onlineAccounts.stream().anyMatch(a -> Arrays.equals(a.getPublicKey(), publicKey));
-            if (alreadyExists) {
-                this.hasOurOnlineAccounts = true;
-
-                if (remaining > 0) {
-                    // Move on to next account
-                    continue;
-                }
-                else {
-                    // Everything exists, so return true
-                    return true;
-                }
-            }
-
-            // Generate bytes for mempow
-            byte[] mempowBytes;
-            try {
-                mempowBytes = this.getMemoryPoWBytes(publicKey, onlineAccountsTimestamp);
-            }
-            catch (IOException e) {
-                LOGGER.info("Unable to create bytes for MemoryPoW. Moving on to next account...");
-                continue;
-            }
-
-            // Compute nonce
-            Integer nonce;
-            try {
-                nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp);
-                if (nonce == null) {
-                    // A nonce is required
+                // Compute nonce
+                Integer nonce;
+                try {
+                    nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp);
+                    if (nonce == null) {
+                        // A nonce is required
+                        return false;
+                    }
+                } catch (TimeoutException e) {
+                    LOGGER.info(String.format("Timed out computing nonce for account %.8s", Base58.encode(publicKey)));
                     return false;
                 }
-            } catch (TimeoutException e) {
-                LOGGER.info(String.format("Timed out computing nonce for account %.8s", Base58.encode(publicKey)));
+
+                byte[] signature = Qortal25519Extras.signForAggregation(privateKey, timestampBytes);
+
+                // Our account is online
+                OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce);
+
+                // Make sure to verify before adding
+                if (verifyMemoryPoW(ourOnlineAccountData, null)) {
+                    ourOnlineAccounts.add(ourOnlineAccountData);
+                }
+            }
+
+            this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty();
+
+            boolean hasInfoChanged = addAccounts(ourOnlineAccounts);
+
+            if (!hasInfoChanged)
                 return false;
-            }
 
-            byte[] signature = Qortal25519Extras.signForAggregation(privateKey, timestampBytes);
+            Network.getInstance().broadcast(peer -> new OnlineAccountsV3Message(ourOnlineAccounts));
 
-            // Our account is online
-            OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce);
+            LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp);
 
-            // Make sure to verify before adding
-            if (verifyMemoryPoW(ourOnlineAccountData, null)) {
-                ourOnlineAccounts.add(ourOnlineAccountData);
-            }
+            return true;
         }
 
-        this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty();
-
-        boolean hasInfoChanged = addAccounts(ourOnlineAccounts);
-
-        if (!hasInfoChanged)
-            return false;
-
-        Network.getInstance().broadcast(peer -> new OnlineAccountsV3Message(ourOnlineAccounts));
-
-        LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp);
-
-        return true;
+        return false;
     }
 
 

From 1f77ee535f99bb98fb5ace444471099d7b4526a0 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sat, 6 May 2023 12:16:59 +0100
Subject: [PATCH 34/48] Added link to example Q-App projects.

---
 Q-Apps.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Q-Apps.md b/Q-Apps.md
index bad9abe0..ea880874 100644
--- a/Q-Apps.md
+++ b/Q-Apps.md
@@ -816,6 +816,9 @@ let res = await qortalRequest({
 
 # Section 4: Examples
 
+Some example projects can be found [here](https://github.com/Qortal/Q-Apps). These can be cloned and modified, or used as a reference when creating a new app.
+
+
 ## Sample App
 
 Here is a sample application to display the logged-in user's avatar:

From 0acf0729e9ead3f12c7344ea7f78e01f461f0eb1 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sat, 6 May 2023 15:10:46 +0100
Subject: [PATCH 35/48] Bump version to 4.0.2

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index ff9c9db1..78df68a7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>4.0.1</version>
+	<version>4.0.2</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

From c941bc6024a8b44e39984ff08279bc85505d56c7 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Sun, 7 May 2023 11:19:42 +0100
Subject: [PATCH 36/48] Catch and log all exceptions when publishing data.

---
 src/main/java/org/qortal/api/resource/ArbitraryResource.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
index dee27413..89008eb2 100644
--- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java
+++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
@@ -1267,7 +1267,8 @@ public class ArbitraryResource {
 				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage());
 			}
 
-		} catch (DataException | IOException e) {
+		} catch (Exception e) {
+			LOGGER.info("Exception when publishing data: ", e);
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
 		}
 	}

From 9490c622421398b11e871181c43302890db55965 Mon Sep 17 00:00:00 2001
From: catbref <misc-github@talk2dom.com>
Date: Mon, 8 May 2023 12:07:02 +0100
Subject: [PATCH 37/48] Improved tx.pl that supports local signing via openssl
 and "deploy_at" transaction type + other minor fixes

---
 tools/tx.pl | 180 ++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 161 insertions(+), 19 deletions(-)

diff --git a/tools/tx.pl b/tools/tx.pl
index fe3cd872..1cb3dd5b 100755
--- a/tools/tx.pl
+++ b/tools/tx.pl
@@ -1,16 +1,23 @@
 #!/usr/bin/env perl
 
+# v4.0.2
+
 use JSON;
 use warnings;
 use strict;
 
 use Getopt::Std;
 use File::Basename;
+use Digest::SHA qw( sha256 sha256_hex );
+use Crypt::RIPEMD160;
 
 our %opt;
 getopts('dpst', \%opt);
 
 my $proc = basename($0);
+my $dirname = dirname($0);
+my $OPENSSL_SIGN = "${dirname}/openssl-sign.sh";
+my $OPENSSL_PRIV_TO_PUB = index(`$ENV{SHELL} -i -c 'openssl version;exit' 2>/dev/null`, 'OpenSSL 3.') != -1;
 
 if (@ARGV < 1) {
 	print STDERR "usage: $proc [-d] [-p] [-s] [-t] <tx-type> <privkey> <values> [<key-value pairs>]\n";
@@ -24,7 +31,15 @@ if (@ARGV < 1) {
 	exit 2;
 }
 
-our $BASE_URL = $ENV{BASE_URL} || $opt{t} ? 'http://localhost:62391' : 'http://localhost:12391';
+our @b58 = qw{
+      1 2 3 4 5 6 7 8 9
+    A B C D E F G H   J K L M N   P Q R S T U V W X Y Z
+    a b c d e f g h i j k   m n o p q r s t u v w x y z
+};
+our %b58 = map { $b58[$_] => $_ } 0 .. 57;
+our %reverseb58 = reverse %b58;
+
+our $BASE_URL = $ENV{BASE_URL} || ($opt{t} ? 'http://localhost:62391' : 'http://localhost:12391');
 our $DEFAULT_FEE = 0.001;
 
 our %TRANSACTION_TYPES = (
@@ -42,6 +57,7 @@ our %TRANSACTION_TYPES = (
 	create_group => {
 		url => 'groups/create',
 		required => [qw(groupName description isOpen approvalThreshold)],
+		defaults => { minimumBlockDelay => 10, maximumBlockDelay => 30 },
 		key_name => 'creatorPublicKey',
 	},
 	update_group => {
@@ -75,10 +91,10 @@ our %TRANSACTION_TYPES = (
 		key_name => 'ownerPublicKey',
 	},
 	remove_group_admin => {
-        url => 'groups/removeadmin',
-        required => [qw(groupId txGroupId admin)],
-        key_name => 'ownerPublicKey',
-    },
+		url => 'groups/removeadmin',
+		required => [qw(groupId txGroupId member)],
+		key_name => 'ownerPublicKey',
+	},
 	group_approval => {
 		url => 'groups/approval',
 		required => [qw(pendingSignature approval)],
@@ -113,7 +129,7 @@ our %TRANSACTION_TYPES = (
 	},
 	update_name => {
 		url => 'names/update',
-		required => [qw(newName newData)],
+		required => [qw(name newName newData)],
 		key_name => 'ownerPublicKey',
 	},
 	# reward-shares
@@ -144,13 +160,21 @@ our %TRANSACTION_TYPES = (
 		key_name => 'senderPublicKey',
 		pow_url => 'addresses/publicize/compute',
 	},
-	# Cross-chain trading
-	build_trade => {
-		url => 'crosschain/build',
-		required => [qw(initialQortAmount finalQortAmount fundingQortAmount secretHash bitcoinAmount)],
-		optional => [qw(tradeTimeout)],
+	# AT
+	deploy_at => {
+		url => 'at',
+		required => [qw(name description aTType tags creationBytes amount)],
+		optional => [qw(assetId)],
 		key_name => 'creatorPublicKey',
-		defaults => { tradeTimeout => 10800 },
+		defaults => { assetId => 0 },
+	},
+	# Cross-chain trading
+	create_trade => {
+		url => 'crosschain/tradebot/create',
+		required => [qw(qortAmount fundingQortAmount foreignAmount receivingAddress)],
+		optional => [qw(tradeTimeout foreignBlockchain)],
+		key_name => 'creatorPublicKey',
+		defaults => { tradeTimeout => 1440, foreignBlockchain => 'LITECOIN' },
 	},
 	trade_recipient => {
 		url => 'crosschain/tradeoffer/recipient',
@@ -196,7 +220,7 @@ if (@ARGV < @required + 1) {
 
 my $priv_key = shift @ARGV;
 
-my $account = account($priv_key);
+my $account;
 my $raw;
 
 if ($tx_type ne 'sign') {
@@ -215,6 +239,8 @@ if ($tx_type ne 'sign') {
 
 	%extras = (%extras, @ARGV);
 
+	$account = account($priv_key, %extras);
+
 	$raw = build_raw($tx_type, $account, %extras);
 	printf "Raw: %s\n", $raw if $opt{d} || (!$opt{s} && !$opt{p});
 
@@ -229,7 +255,7 @@ if ($tx_type ne 'sign') {
 }
 
 if ($opt{s}) {
-	my $signed = sign($account->{private}, $raw);
+	my $signed = sign($priv_key, $raw);
 	printf "Signed: %s\n", $signed if $opt{d} || $tx_type eq 'sign';
 
 	if ($opt{p}) {
@@ -246,15 +272,25 @@ if ($opt{s}) {
 }
 
 sub account {
-	my ($creator) = @_;
+	my ($privkey, %extras) = @_;
 
-	my $account = { private => $creator };
-	$account->{public} = api('utils/publickey', $creator);
-	$account->{address} = api('addresses/convert/{publickey}', '', '{publickey}', $account->{public});
+	my $account = { private => $privkey };
+	$account->{public} = $extras{publickey} || priv_to_pub($privkey);
+	$account->{address} = $extras{address} || pubkey_to_address($account->{public}); # api('addresses/convert/{publickey}', '', '{publickey}', $account->{public});
 
 	return $account;
 }
 
+sub priv_to_pub {
+	my ($privkey) = @_;
+
+	if ($OPENSSL_PRIV_TO_PUB) {
+		return openssl_priv_to_pub($privkey);
+	} else {
+		return api('utils/publickey', $privkey);
+	}
+}
+
 sub build_raw {
 	my ($type, $account, %extras) = @_;
 
@@ -306,6 +342,21 @@ sub build_raw {
 sub sign {
 	my ($private, $raw) = @_;
 
+	if (-x "$OPENSSL_SIGN") {
+		my $private_hex = decode_base58($private);
+		chomp $private_hex;
+
+		my $raw_hex = decode_base58($raw);
+		chomp $raw_hex;
+
+		my $sig = `${OPENSSL_SIGN} ${private_hex} ${raw_hex}`;
+		chomp $sig;
+
+		my $sig58 = encode_base58(${raw_hex} . ${sig});
+		chomp $sig58;
+		return $sig58;
+	}
+
 	my $json = <<"	__JSON__";
 	{
 		"privateKey": "$private",
@@ -344,7 +395,14 @@ sub api {
 	my $curl = "curl --silent --output - --url '$BASE_URL/$url'";
 	if (defined $postdata && $postdata ne '') {
 		$postdata =~ tr|\n| |s;
-		$curl .= " --header 'Content-Type: application/json' --data-binary '$postdata'";
+
+		if ($postdata =~ /^\s*\{/so) {
+			$curl .= " --header 'Content-Type: application/json'";
+		} else {
+			$curl .= " --header 'Content-Type: text/plain'";
+		}
+
+		$curl .= " --data-binary '$postdata'";
 		$method = 'POST';
 	}
 	my $response = `$curl 2>/dev/null`; 
@@ -356,3 +414,87 @@ sub api {
 
 	return $response;
 }
+
+sub encode_base58 {
+    use integer;
+    my @in = map { hex($_) } ($_[0] =~ /(..)/g);
+    my $bzeros = length($1) if join('', @in) =~ /^(0*)/;
+    my @out;
+    my $size = 2 * scalar @in;
+    for my $c (@in) {
+        for (my $j = $size; $j--; ) {
+            $c += 256 * ($out[$j] // 0);
+            $out[$j] = $c % 58;
+            $c /= 58;
+        }
+    }
+    my $out = join('', map { $reverseb58{$_} } @out);
+    return $1 if $out =~ /(1{$bzeros}[^1].*)/;
+    return $1 if $out =~ /(1{$bzeros})/;
+    die "Invalid base58!\n";
+}
+
+
+sub decode_base58 {
+    use integer;
+    my @out;
+    my $azeros = length($1) if $_[0] =~ /^(1*)/;
+    for my $c ( map { $b58{$_} } $_[0] =~ /./g ) {
+	die("Invalid character!\n") unless defined $c;
+        for (my $j = length($_[0]); $j--; ) {
+            $c += 58 * ($out[$j] // 0);
+            $out[$j] = $c % 256;
+            $c /= 256;
+        }
+    }
+    shift @out while @out && $out[0] == 0;
+    unshift(@out, (0) x $azeros);
+    return sprintf('%02x' x @out, @out);
+}
+
+sub openssl_priv_to_pub {
+	my ($privkey) = @_;
+
+	my $privkey_hex = decode_base58($privkey);
+
+	my $key_type = "04"; # hex
+	my $length = "20"; # hex
+
+	my $asn1 = <<"__ASN1__";
+asn1=SEQUENCE:private_key
+
+[private_key]
+version=INTEGER:0
+included=SEQUENCE:key_info
+raw=FORMAT:HEX,OCTETSTRING:${key_type}${length}${privkey_hex}
+
+[key_info]
+type=OBJECT:ED25519
+
+__ASN1__
+
+	my $output = `echo "${asn1}" | openssl asn1parse -i -genconf - -out - | openssl pkey -in - -inform der -noout -text_pub`;
+
+	# remove colons
+	my $pubkey = '';
+	$pubkey .= $1 while $output =~ m/([0-9a-f]{2})(?::|$)/g;
+
+	return encode_base58($pubkey);
+}
+
+sub pubkey_to_address {
+	my ($pubkey) = @_;
+
+	my $pubkey_hex = decode_base58($pubkey);
+	my $pubkey_raw = pack('H*', $pubkey_hex);
+
+	my $pkh_hex = Crypt::RIPEMD160->hexhash(sha256($pubkey_raw));
+	$pkh_hex =~ tr/ //ds;
+
+	my $version = '3a'; # hex
+
+	my $raw = pack('H*', $version . $pkh_hex);
+	my $chksum = substr(sha256_hex(sha256($raw)), 0, 8);
+
+	return encode_base58($version . $pkh_hex . $chksum);
+}

From 923e90ebedf0538f11f1a4cc38f5702f5cd21420 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Mon, 8 May 2023 12:12:40 +0100
Subject: [PATCH 38/48] Fixed occasional NPE

---
 src/main/java/org/qortal/api/resource/ArbitraryResource.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
index 89008eb2..c617b517 100644
--- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java
+++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
@@ -1316,7 +1316,7 @@ public class ArbitraryResource {
 			if (filepath == null || filepath.isEmpty()) {
 				// No file path supplied - so check if this is a single file resource
 				String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal");
-				if (files.length == 1) {
+				if (files != null && files.length == 1) {
 					// This is a single file resource
 					filepath = files[0];
 				}

From 21d1750779f2c56bbfd1a615cde09ae976204009 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Mon, 8 May 2023 12:13:12 +0100
Subject: [PATCH 39/48] Added more debug logging when building resources.

---
 src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
index a7876236..c1d07054 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
@@ -441,6 +441,7 @@ public class ArbitraryDataReader {
                 Path unencryptedPath = Paths.get(this.workingPath.toString(), "zipped.zip");
                 SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, "AES");
                 AES.decryptFile(algorithm, aesKey, this.filePath.toString(), unencryptedPath.toString());
+                LOGGER.debug("Finished decrypting {} using algorithm {}", this.arbitraryDataResource, algorithm);
 
                 // Replace filePath pointer with the encrypted file path
                 // Don't delete the original ArbitraryDataFile, as this is handled in the cleanup phase
@@ -475,7 +476,9 @@ public class ArbitraryDataReader {
 
             // Handle each type of compression
             if (compression == Compression.ZIP) {
+                LOGGER.debug("Unzipping {}...", this.arbitraryDataResource);
                 ZipUtils.unzip(this.filePath.toString(), this.uncompressedPath.getParent().toString());
+                LOGGER.debug("Finished unzipping {}", this.arbitraryDataResource);
             }
             else if (compression == Compression.NONE) {
                 Files.createDirectories(this.uncompressedPath);
@@ -511,10 +514,12 @@ public class ArbitraryDataReader {
 
     private void validate() throws IOException, DataException {
         if (this.service.isValidationRequired()) {
+            LOGGER.debug("Validating {}...", this.arbitraryDataResource);
             Service.ValidationResult result = this.service.validate(this.filePath);
             if (result != Service.ValidationResult.OK) {
                 throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
             }
+            LOGGER.debug("Finished validating {}", this.arbitraryDataResource);
         }
     }
 

From c682fa89fd537b672e5a375f24dcf88d29759722 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Mon, 8 May 2023 12:14:00 +0100
Subject: [PATCH 40/48] Avoid duplicate concurrent QDN builds.

---
 .../qortal/arbitrary/ArbitraryDataReader.java | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
index c1d07054..b9e62e56 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
@@ -34,6 +34,9 @@ import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 
 public class ArbitraryDataReader {
 
@@ -59,6 +62,10 @@ public class ArbitraryDataReader {
     // The resource being read
     ArbitraryDataResource arbitraryDataResource = null;
 
+    // Track resources that are currently being loaded, to avoid duplicate concurrent builds
+    // TODO: all builds could be handled by the build queue (even synchronous ones), to avoid the need for this
+    private static Map<String, Long> inProgress = Collections.synchronizedMap(new HashMap<>());
+
     public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
         // Ensure names are always lowercase
         if (resourceIdType == ResourceIdType.NAME) {
@@ -166,6 +173,12 @@ public class ArbitraryDataReader {
 
             this.arbitraryDataResource = this.createArbitraryDataResource();
 
+            // Don't allow duplicate loads
+            if (!this.canStartLoading()) {
+                LOGGER.debug("Skipping duplicate load of {}", this.arbitraryDataResource);
+                return;
+            }
+
             this.preExecute();
             this.deleteExistingFiles();
             this.fetch();
@@ -193,6 +206,7 @@ public class ArbitraryDataReader {
 
     private void preExecute() throws DataException {
         ArbitraryDataBuildManager.getInstance().setBuildInProgress(true);
+
         this.checkEnabled();
         this.createWorkingDirectory();
         this.createUncompressedDirectory();
@@ -200,6 +214,9 @@ public class ArbitraryDataReader {
 
     private void postExecute() {
         ArbitraryDataBuildManager.getInstance().setBuildInProgress(false);
+
+        this.arbitraryDataResource = this.createArbitraryDataResource();
+        ArbitraryDataReader.inProgress.remove(this.arbitraryDataResource.getUniqueKey());
     }
 
     private void checkEnabled() throws DataException {
@@ -208,6 +225,17 @@ public class ArbitraryDataReader {
         }
     }
 
+    private boolean canStartLoading() {
+        // Avoid duplicate builds if we're already loading this resource
+        String uniqueKey = this.arbitraryDataResource.getUniqueKey();
+        if (ArbitraryDataReader.inProgress.containsKey(uniqueKey)) {
+            return false;
+        }
+        ArbitraryDataReader.inProgress.put(uniqueKey, NTP.getTime());
+
+        return true;
+    }
+
     private void createWorkingDirectory() throws DataException {
         try {
             Files.createDirectories(this.workingPath);

From aba589c0e0e9b65432e2078923af54b12e4c1798 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Mon, 8 May 2023 12:15:53 +0100
Subject: [PATCH 41/48] Added optional "build" parameter to
 GET_QDN_RESOURCE_STATUS.

This triggers an async build when checking the status.
---
 src/main/resources/q-apps/q-apps.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js
index 5233c7d5..86493b48 100644
--- a/src/main/resources/q-apps/q-apps.js
+++ b/src/main/resources/q-apps/q-apps.js
@@ -236,13 +236,15 @@ window.addEventListener("message", (event) => {
             if (data.identifier != null) url = url.concat("/" + data.identifier);
             url = url.concat("?");
             if (data.filepath != null) url = url.concat("&filepath=" + data.filepath);
-            if (data.rebuild != null) url = url.concat("&rebuild=" + new Boolean(data.rebuild).toString())
+            if (data.rebuild != null) url = url.concat("&rebuild=" + new Boolean(data.rebuild).toString());
             if (data.encoding != null) url = url.concat("&encoding=" + data.encoding);
             return httpGetAsyncWithEvent(event, url);
 
         case "GET_QDN_RESOURCE_STATUS":
             url = "/arbitrary/resource/status/" + data.service + "/" + data.name;
             if (data.identifier != null) url = url.concat("/" + data.identifier);
+            url = url.concat("?");
+            if (data.build != null) url = url.concat("&build=" + new Boolean(data.build).toString());
             return httpGetAsyncWithEvent(event, url);
 
         case "GET_QDN_RESOURCE_PROPERTIES":

From 05b4ecd4edf3e559a0d372a8e2647fca1a446a96 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Mon, 8 May 2023 12:16:17 +0100
Subject: [PATCH 42/48] Updated documentation.

---
 Q-Apps.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Q-Apps.md b/Q-Apps.md
index ea880874..177fee2d 100644
--- a/Q-Apps.md
+++ b/Q-Apps.md
@@ -425,7 +425,8 @@ let res = await qortalRequest({
     action: "GET_QDN_RESOURCE_STATUS",
     name: "QortalDemo",
     service: "THUMBNAIL",
-    identifier: "qortal_avatar" // Optional
+    identifier: "qortal_avatar", // Optional
+    build: true // Optional - request that the resource is fetched & built in the background
 });
 ```
 

From fc10b611933c27a109a9d5377ba0324de7a71454 Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Mon, 8 May 2023 12:17:44 +0100
Subject: [PATCH 43/48] Fixed slow validation issue caused by loading the
 entire resource into memory.

---
 src/main/java/org/qortal/arbitrary/misc/Service.java |  6 ++++--
 src/main/java/org/qortal/utils/FilesystemUtils.java  | 12 ++++++++++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java
index b53ab7ca..94ca9252 100644
--- a/src/main/java/org/qortal/arbitrary/misc/Service.java
+++ b/src/main/java/org/qortal/arbitrary/misc/Service.java
@@ -107,7 +107,7 @@ public enum Service {
             }
 
             // Require valid JSON
-            byte[] data = FilesystemUtils.getSingleFileContents(path);
+            byte[] data = FilesystemUtils.getSingleFileContents(path, 25*1024);
             String json = new String(data, StandardCharsets.UTF_8);
             try {
                 objectMapper.readTree(json);
@@ -201,7 +201,9 @@ public enum Service {
             return ValidationResult.OK;
         }
 
-        byte[] data = FilesystemUtils.getSingleFileContents(path);
+        // Load the first 25KB of data. This only needs to be long enough to check the prefix
+        // and also to allow for possible additional future validation of smaller files.
+        byte[] data = FilesystemUtils.getSingleFileContents(path, 25*1024);
         long size = FilesystemUtils.getDirectorySize(path);
 
         // Validate max size if needed
diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java
index 76651000..e9921561 100644
--- a/src/main/java/org/qortal/utils/FilesystemUtils.java
+++ b/src/main/java/org/qortal/utils/FilesystemUtils.java
@@ -228,12 +228,18 @@ public class FilesystemUtils {
      * @throws IOException
      */
     public static byte[] getSingleFileContents(Path path) throws IOException {
+        return getSingleFileContents(path, null);
+    }
+
+    public static byte[] getSingleFileContents(Path path, Integer maxLength) throws IOException {
         byte[] data = null;
         // TODO: limit the file size that can be loaded into memory
 
         // If the path is a file, read the contents directly
         if (path.toFile().isFile()) {
-            data = Files.readAllBytes(path);
+            int fileSize = (int)path.toFile().length();
+            maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize;
+            data = FilesystemUtils.readFromFile(path.toString(), 0, maxLength);
         }
 
         // Or if it's a directory, only load file contents if there is a single file inside it
@@ -242,7 +248,9 @@ public class FilesystemUtils {
             if (files.length == 1) {
                 Path filePath = Paths.get(path.toString(), files[0]);
                 if (filePath.toFile().isFile()) {
-                    data = Files.readAllBytes(filePath);
+                    int fileSize = (int)filePath.toFile().length();
+                    maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize;
+                    data = FilesystemUtils.readFromFile(filePath.toString(), 0, maxLength);
                 }
             }
         }

From df3c68679f20ea3c959a04a3a990bae670ec3f5f Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Mon, 8 May 2023 14:43:00 +0100
Subject: [PATCH 44/48] Log the action to the console, instead of the entire
 event.

---
 src/main/resources/q-apps/q-apps.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js
index 86493b48..a505c1b0 100644
--- a/src/main/resources/q-apps/q-apps.js
+++ b/src/main/resources/q-apps/q-apps.js
@@ -169,7 +169,7 @@ window.addEventListener("message", (event) => {
         return;
     }
 
-    console.log("Core received event: " + JSON.stringify(event.data));
+    console.log("Core received action: " + JSON.stringify(event.data.action));
 
     let url;
     let data = event.data;

From 49063e54ece7d94a60a3ff90ddbc28b15d34899b Mon Sep 17 00:00:00 2001
From: CalDescent <caldescent@protonmail.com>
Date: Mon, 8 May 2023 19:18:38 +0100
Subject: [PATCH 45/48] Bump version to 4.0.3

---
 pom.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pom.xml b/pom.xml
index 78df68a7..0dfa0cf4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>4.0.2</version>
+	<version>4.0.3</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

From cda32a47f182aca1f2563fc5d10f7fbd01e66294 Mon Sep 17 00:00:00 2001
From: QuickMythril <quickmythril@protonmail.com>
Date: Mon, 8 May 2023 20:23:54 -0400
Subject: [PATCH 46/48] Added API call to get votes

---
 .../qortal/api/resource/PollsResource.java    | 30 +++++++++++++++++++
 .../data/transaction/TransactionData.java     |  3 +-
 .../qortal/data/voting/VoteOnPollData.java    | 17 +++++++++++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/src/main/java/org/qortal/api/resource/PollsResource.java b/src/main/java/org/qortal/api/resource/PollsResource.java
index 952cbdc5..ab163342 100644
--- a/src/main/java/org/qortal/api/resource/PollsResource.java
+++ b/src/main/java/org/qortal/api/resource/PollsResource.java
@@ -37,6 +37,7 @@ import javax.ws.rs.PathParam;
 import javax.ws.rs.QueryParam;
 import org.qortal.api.ApiException;
 import org.qortal.data.voting.PollData;
+import org.qortal.data.voting.VoteOnPollData;
 
 @Path("/polls")
 @Tag(name = "Polls")
@@ -102,6 +103,35 @@ public class PollsResource {
             }
     }
 
+    @GET
+    @Path("/votes/{pollName}")
+    @Operation(
+            summary = "Votes on poll",
+            responses = {
+                    @ApiResponse(
+                            description = "poll votes",
+                            content = @Content(
+                                    mediaType = MediaType.APPLICATION_JSON,
+                                    schema = @Schema(implementation = VoteOnPollData.class)
+                            )
+                    )
+            }
+    )
+    @ApiErrors({ApiError.REPOSITORY_ISSUE})
+    public List<VoteOnPollData> getVoteOnPollData(@PathParam("pollName") String pollName) {
+            try (final Repository repository = RepositoryManager.getRepository()) {
+                    if (repository.getVotingRepository().fromPollName(pollName) == null)
+                            throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.POLL_NO_EXISTS);
+
+                    List<VoteOnPollData> voteOnPollData = repository.getVotingRepository().getVotes(pollName);
+                    return voteOnPollData;
+            } catch (ApiException e) {
+                    throw e;
+            } catch (DataException e) {
+                    throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+            }
+    }
+
     @POST
     @Path("/create")
     @Operation(
diff --git a/src/main/java/org/qortal/data/transaction/TransactionData.java b/src/main/java/org/qortal/data/transaction/TransactionData.java
index 838cffd3..4bf3152c 100644
--- a/src/main/java/org/qortal/data/transaction/TransactionData.java
+++ b/src/main/java/org/qortal/data/transaction/TransactionData.java
@@ -13,6 +13,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 import org.eclipse.persistence.oxm.annotations.XmlDiscriminatorNode;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.voting.PollData;
+import org.qortal.data.voting.VoteOnPollData;
 import org.qortal.transaction.Transaction.ApprovalStatus;
 import org.qortal.transaction.Transaction.TransactionType;
 
@@ -30,7 +31,7 @@ import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
 @XmlSeeAlso({GenesisTransactionData.class, PaymentTransactionData.class, RegisterNameTransactionData.class, UpdateNameTransactionData.class,
 	SellNameTransactionData.class, CancelSellNameTransactionData.class, BuyNameTransactionData.class,
 	CreatePollTransactionData.class, VoteOnPollTransactionData.class, ArbitraryTransactionData.class,
-	PollData.class,
+	PollData.class, VoteOnPollData.class,
 	IssueAssetTransactionData.class, TransferAssetTransactionData.class,
 	CreateAssetOrderTransactionData.class, CancelAssetOrderTransactionData.class,
 	MultiPaymentTransactionData.class, DeployAtTransactionData.class, MessageTransactionData.class, ATTransactionData.class,
diff --git a/src/main/java/org/qortal/data/voting/VoteOnPollData.java b/src/main/java/org/qortal/data/voting/VoteOnPollData.java
index 47c06a54..531ed286 100644
--- a/src/main/java/org/qortal/data/voting/VoteOnPollData.java
+++ b/src/main/java/org/qortal/data/voting/VoteOnPollData.java
@@ -9,6 +9,11 @@ public class VoteOnPollData {
 
 	// Constructors
 
+	// For JAXB
+	protected VoteOnPollData() {
+		super();
+	}
+
 	public VoteOnPollData(String pollName, byte[] voterPublicKey, int optionIndex) {
 		this.pollName = pollName;
 		this.voterPublicKey = voterPublicKey;
@@ -21,12 +26,24 @@ public class VoteOnPollData {
 		return this.pollName;
 	}
 
+	public void setPollName(String pollName) {
+		this.pollName = pollName;
+	}
+
 	public byte[] getVoterPublicKey() {
 		return this.voterPublicKey;
 	}
 
+	public void setVoterPublicKey(byte[] voterPublicKey) {
+		this.voterPublicKey = voterPublicKey;
+	}
+
 	public int getOptionIndex() {
 		return this.optionIndex;
 	}
 
+	public void setOptionIndex(int optionIndex) {
+		this.optionIndex = optionIndex;
+	}
+
 }

From 49c0d45bc6ec11766bf89844e0f4b320ffa0be23 Mon Sep 17 00:00:00 2001
From: QuickMythril <quickmythril@protonmail.com>
Date: Mon, 8 May 2023 23:26:23 -0400
Subject: [PATCH 47/48] Added count to get votes API call

---
 .../java/org/qortal/api/model/PollVotes.java  | 56 +++++++++++++++++++
 .../qortal/api/resource/PollsResource.java    | 37 ++++++++++--
 2 files changed, 88 insertions(+), 5 deletions(-)
 create mode 100644 src/main/java/org/qortal/api/model/PollVotes.java

diff --git a/src/main/java/org/qortal/api/model/PollVotes.java b/src/main/java/org/qortal/api/model/PollVotes.java
new file mode 100644
index 00000000..c57ebc37
--- /dev/null
+++ b/src/main/java/org/qortal/api/model/PollVotes.java
@@ -0,0 +1,56 @@
+package org.qortal.api.model;
+
+import java.util.List;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+
+import org.qortal.data.voting.VoteOnPollData;
+
+@Schema(description = "Poll vote info, including voters")
+// All properties to be converted to JSON via JAX-RS
+@XmlAccessorType(XmlAccessType.FIELD)
+public class PollVotes {
+
+    @Schema(description = "List of individual votes")
+    @XmlElement(name = "votes")
+    public List<VoteOnPollData> votes;
+
+    @Schema(description = "Total number of votes")
+    public Integer totalVotes;
+
+    @Schema(description = "List of vote counts for each option")
+    public List<OptionCount> voteCounts;
+
+    // For JAX-RS
+    protected PollVotes() {
+    }
+
+    public PollVotes(List<VoteOnPollData> votes, Integer totalVotes, List<OptionCount> voteCounts) {
+        this.votes = votes;
+        this.totalVotes = totalVotes;
+        this.voteCounts = voteCounts;
+    }
+
+    @Schema(description = "Vote info")
+    // All properties to be converted to JSON via JAX-RS
+    @XmlAccessorType(XmlAccessType.FIELD)
+    public static class OptionCount {
+        @Schema(description = "Option name")
+        public String optionName;
+
+        @Schema(description = "Vote count")
+        public Integer voteCount;
+
+        // For JAX-RS
+        protected OptionCount() {
+        }
+
+        public OptionCount(String optionName, Integer voteCount) {
+            this.optionName = optionName;
+            this.voteCount = voteCount;
+        }
+    }
+}
diff --git a/src/main/java/org/qortal/api/resource/PollsResource.java b/src/main/java/org/qortal/api/resource/PollsResource.java
index ab163342..999fa2fd 100644
--- a/src/main/java/org/qortal/api/resource/PollsResource.java
+++ b/src/main/java/org/qortal/api/resource/PollsResource.java
@@ -31,12 +31,17 @@ import javax.ws.rs.core.MediaType;
 
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.ArraySchema;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 import javax.ws.rs.GET;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.QueryParam;
 import org.qortal.api.ApiException;
+import org.qortal.api.model.PollVotes;
 import org.qortal.data.voting.PollData;
+import org.qortal.data.voting.PollOptionData;
 import org.qortal.data.voting.VoteOnPollData;
 
 @Path("/polls")
@@ -112,19 +117,41 @@ public class PollsResource {
                             description = "poll votes",
                             content = @Content(
                                     mediaType = MediaType.APPLICATION_JSON,
-                                    schema = @Schema(implementation = VoteOnPollData.class)
+                                    schema = @Schema(implementation = PollVotes.class)
                             )
                     )
             }
     )
     @ApiErrors({ApiError.REPOSITORY_ISSUE})
-    public List<VoteOnPollData> getVoteOnPollData(@PathParam("pollName") String pollName) {
+    public PollVotes getPollVotes(@PathParam("pollName") String pollName) {
             try (final Repository repository = RepositoryManager.getRepository()) {
-                    if (repository.getVotingRepository().fromPollName(pollName) == null)
+                    PollData pollData = repository.getVotingRepository().fromPollName(pollName);
+                    if (pollData == null)
                             throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.POLL_NO_EXISTS);
 
-                    List<VoteOnPollData> voteOnPollData = repository.getVotingRepository().getVotes(pollName);
-                    return voteOnPollData;
+                    List<VoteOnPollData> votes = repository.getVotingRepository().getVotes(pollName);
+
+                    // Initialize map for counting votes
+                    Map<String, Integer> voteCountMap = new HashMap<>();
+                    for (PollOptionData optionData : pollData.getPollOptions()) {
+                            voteCountMap.put(optionData.getOptionName(), 0);
+                    }
+
+                    int totalVotes = 0;
+                    for (VoteOnPollData vote : votes) {
+                            String selectedOption = pollData.getPollOptions().get(vote.getOptionIndex()).getOptionName();
+                            if (voteCountMap.containsKey(selectedOption)) {
+                                    voteCountMap.put(selectedOption, voteCountMap.get(selectedOption) + 1);
+                                    totalVotes++;
+                            }
+                    }
+
+                    // Convert map to list of VoteInfo
+                    List<PollVotes.OptionCount> voteCounts = voteCountMap.entrySet().stream()
+                        .map(entry -> new PollVotes.OptionCount(entry.getKey(), entry.getValue()))
+                        .collect(Collectors.toList());
+    
+                    return new PollVotes(votes, totalVotes, voteCounts);
             } catch (ApiException e) {
                     throw e;
             } catch (DataException e) {

From 3e45948646c3e9fa22108c898c7334c51605314c Mon Sep 17 00:00:00 2001
From: QuickMythril <quickmythril@protonmail.com>
Date: Mon, 8 May 2023 23:41:31 -0400
Subject: [PATCH 48/48] Added get votes option to return only counts

---
 src/main/java/org/qortal/api/resource/PollsResource.java | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/java/org/qortal/api/resource/PollsResource.java b/src/main/java/org/qortal/api/resource/PollsResource.java
index 999fa2fd..c64a8caf 100644
--- a/src/main/java/org/qortal/api/resource/PollsResource.java
+++ b/src/main/java/org/qortal/api/resource/PollsResource.java
@@ -123,7 +123,7 @@ public class PollsResource {
             }
     )
     @ApiErrors({ApiError.REPOSITORY_ISSUE})
-    public PollVotes getPollVotes(@PathParam("pollName") String pollName) {
+    public PollVotes getPollVotes(@PathParam("pollName") String pollName, @QueryParam("onlyCounts") Boolean onlyCounts) {
             try (final Repository repository = RepositoryManager.getRepository()) {
                     PollData pollData = repository.getVotingRepository().fromPollName(pollName);
                     if (pollData == null)
@@ -151,7 +151,11 @@ public class PollsResource {
                         .map(entry -> new PollVotes.OptionCount(entry.getKey(), entry.getValue()))
                         .collect(Collectors.toList());
     
-                    return new PollVotes(votes, totalVotes, voteCounts);
+                    if (onlyCounts != null && onlyCounts) {
+                        return new PollVotes(null, totalVotes, voteCounts);
+                    } else {
+                        return new PollVotes(votes, totalVotes, voteCounts);
+                    }
             } catch (ApiException e) {
                     throw e;
             } catch (DataException e) {