Merge branch 'Qortal:master' into upgrade-tls

Q-Apps.md

@@ -42,10 +42,15 @@ A "default" resource refers to one without an identifier. For example, when a we
 
 Here is a list of currently available services that can be used in Q-Apps:
 
+### Public services ###
+The services below are intended to be used for publicly accessible data.
+
 IMAGE,
 THUMBNAIL,
 VIDEO,
 AUDIO,
+PODCAST,
+VOICE,
 ARBITRARY_DATA,
 JSON,
 DOCUMENT,
@@ -55,7 +60,25 @@ METADATA,
 BLOG,
 BLOG_POST,
 BLOG_COMMENT,
-GIF_REPOSITORY
+GIF_REPOSITORY,
+ATTACHMENT,
+FILE,
+FILES,
+CHAIN_DATA,
+STORE,
+PRODUCT,
+OFFER,
+COUPON,
+CODE,
+PLUGIN,
+EXTENSION,
+GAME,
+ITEM,
+NFT,
+DATABASE,
+SNAPSHOT,
+COMMENT,
+CHAIN_COMMENT,
 WEBSITE,
 APP,
 QCHAT_ATTACHMENT,
@@ -63,6 +86,20 @@ QCHAT_IMAGE,
 QCHAT_AUDIO,
 QCHAT_VOICE
 
+### Private services ###
+For the services below, data is encrypted for a single recipient, and can only be decrypted using the private key of the recipient's wallet.
+
+QCHAT_ATTACHMENT_PRIVATE,
+ATTACHMENT_PRIVATE,
+FILE_PRIVATE,
+IMAGE_PRIVATE,
+VIDEO_PRIVATE,
+AUDIO_PRIVATE,
+VOICE_PRIVATE,
+DOCUMENT_PRIVATE,
+MAIL_PRIVATE,
+MESSAGE_PRIVATE
+
 
 ## Single vs multi-file resources
 
@@ -220,9 +257,14 @@ Here is a list of currently supported actions:
 - SEARCH_QDN_RESOURCES
 - GET_QDN_RESOURCE_STATUS
 - GET_QDN_RESOURCE_PROPERTIES
+- GET_QDN_RESOURCE_METADATA
+- GET_QDN_RESOURCE_URL
+- LINK_TO_QDN_RESOURCE
 - FETCH_QDN_RESOURCE
 - PUBLISH_QDN_RESOURCE
 - PUBLISH_MULTIPLE_QDN_RESOURCES
+- DECRYPT_DATA
+- SAVE_FILE
 - GET_WALLET_BALANCE
 - GET_BALANCE
 - SEND_COIN
@@ -238,8 +280,6 @@ Here is a list of currently supported actions:
 - FETCH_BLOCK_RANGE
 - SEARCH_TRANSACTIONS
 - GET_PRICE
-- GET_QDN_RESOURCE_URL
-- LINK_TO_QDN_RESOURCE
 - GET_LIST_ITEMS
 - ADD_LIST_ITEMS
 - DELETE_LIST_ITEM
@@ -385,7 +425,8 @@ let res = await qortalRequest({
     action: "GET_QDN_RESOURCE_STATUS",
     name: "QortalDemo",
     service: "THUMBNAIL",
-    identifier: "qortal_avatar" // Optional
+    identifier: "qortal_avatar", // Optional
+    build: true // Optional - request that the resource is fetched & built in the background
 });
 ```
 
@@ -400,11 +441,21 @@ let res = await qortalRequest({
 // Returns: filename, size, mimeType (where available)
 ```
 
+### Get QDN resource metadata
+```
+let res = await qortalRequest({
+    action: "GET_QDN_RESOURCE_METADATA",
+    name: "QortalDemo",
+    service: "THUMBNAIL",
+    identifier: "qortal_avatar" // Optional
+});
+```
+
 ### Publish a single file to QDN
 _Requires user approval_.<br />
 Note: this publishes a single, base64-encoded file. Multi-file resource publishing (such as a WEBSITE or GIF_REPOSITORY) is not yet supported via a Q-App. It will be added in a future update.
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "PUBLISH_QDN_RESOURCE",
     name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
     service: "IMAGE",
@@ -418,7 +469,9 @@ await qortalRequest({
     // tag2: "strings", // Optional
     // tag3: "can", // Optional
     // tag4: "go", // Optional
-    // tag5: "here" // Optional
+    // tag5: "here", // Optional
+    // encrypt: true, // Optional - to be used with a private service
+    // recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
 });
 ```
 
@@ -426,7 +479,7 @@ await qortalRequest({
 _Requires user approval_.<br />
 Note: each resource being published consists of a single, base64-encoded file, each in its own transaction. Useful for publishing two or more related things, such as a video and a video thumbnail.
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "PUBLISH_MULTIPLE_QDN_RESOURCES",
     resources: [
         name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
@@ -441,7 +494,9 @@ await qortalRequest({
         // tag2: "strings", // Optional
         // tag3: "can", // Optional
         // tag4: "go", // Optional
-        // tag5: "here" // Optional
+        // tag5: "here", // Optional
+        // encrypt: true, // Optional - to be used with a private service
+        // recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
     ],
     [
         ... more resources here if needed ...
@@ -449,10 +504,32 @@ await qortalRequest({
 });
 ```
 
+### Decrypt encrypted/private data
+```
+let res = await qortalRequest({
+    action: "DECRYPT_DATA",
+    encryptedData: 'qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1r',
+    publicKey: 'publickeygoeshere'
+});
+// Returns base64 encoded string of plaintext data
+```
+
+### Prompt user to save a file to disk
+Note: mimeType not required but recommended. If not specified, saving will fail if the mimeType is unable to be derived from the Blob.
+```
+let res = await qortalRequest({
+    action: "SAVE_FILE",
+    blob: dataBlob,
+    filename: "myfile.pdf",
+    mimeType: "application/pdf" // Optional but recommended
+});
+```
+
+
 ### Get wallet balance (QORT)
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "GET_WALLET_BALANCE",
     coin: "QORT"
 });
@@ -477,7 +554,7 @@ let res = await qortalRequest({
 ### Send QORT to address
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "SEND_COIN",
     coin: "QORT",
     destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
@@ -488,7 +565,7 @@ await qortalRequest({
 ### Send foreign coin to address
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "SEND_COIN",
     coin: "LTC",
     destinationAddress: "LSdTvMHRm8sScqwCi6x9wzYQae8JeZhx6y",
@@ -508,6 +585,7 @@ let res = await qortalRequest({
     // reference: "reference", // Optional
     // chatReference: "chatreference", // Optional
     // hasChatReference: true, // Optional
+    encoding: "BASE64", // Optional (defaults to BASE58 if omitted)
     limit: 100,
     offset: 0,
     reverse: true
@@ -517,7 +595,7 @@ let res = await qortalRequest({
 ### Send a group chat message
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "SEND_CHAT_MESSAGE",
     groupId: 0,
     message: "Test"
@@ -527,7 +605,7 @@ await qortalRequest({
 ### Send a private chat message
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "SEND_CHAT_MESSAGE",
     destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
     message: "Test"
@@ -547,7 +625,7 @@ let res = await qortalRequest({
 ### Join a group
 _Requires user approval_
 ```
-await qortalRequest({
+let res = await qortalRequest({
     action: "JOIN_GROUP",
     groupId: 100
 });
@@ -739,6 +817,9 @@ let res = await qortalRequest({
 
 # Section 4: Examples
 
+Some example projects can be found [here](https://github.com/Qortal/Q-Apps). These can be cloned and modified, or used as a reference when creating a new app.
+
+
 ## Sample App
 
 Here is a sample application to display the logged-in user's avatar:

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>3.9.1</version>
+	<version>4.0.3</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

src/main/java/org/qortal/api/HTMLParser.java

@@ -13,7 +13,8 @@ public class HTMLParser {
 
     private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
 
-    private String linkPrefix;
+    private String qdnBase;
+    private String qdnBaseWithPath;
     private byte[] data;
     private String qdnContext;
     private String resourceId;
@@ -21,10 +22,13 @@ public class HTMLParser {
     private String identifier;
     private String path;
     private String theme;
+    private boolean usingCustomRouting;
 
     public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix, byte[] data,
-                      String qdnContext, Service service, String identifier, String theme) {
+                      String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting) {
-        this.linkPrefix = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
+        String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : "";
+        this.qdnBase = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
+        this.qdnBaseWithPath = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
         this.data = data;
         this.qdnContext = qdnContext;
         this.resourceId = resourceId;
@@ -32,12 +36,12 @@ public class HTMLParser {
         this.identifier = identifier;
         this.path = inPath;
         this.theme = theme;
+        this.usingCustomRouting = usingCustomRouting;
     }
 
     public void addAdditionalHeaderTags() {
         String fileContents = new String(data);
         Document document = Jsoup.parse(fileContents);
-        String baseUrl = this.linkPrefix;
         Elements head = document.getElementsByTag("head");
         if (!head.isEmpty()) {
             // Add q-apps script tag
@@ -51,16 +55,21 @@ public class HTMLParser {
             }
 
             // Escape and add vars
-            String service = this.service.toString().replace("\"","\\\"");
+            String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\\", "").replace("\"","\\\"") : "";
-            String name = this.resourceId != null ? this.resourceId.replace("\"","\\\"") : "";
+            String service = this.service.toString().replace("\\", "").replace("\"","\\\"");
-            String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"") : "";
+            String name = this.resourceId != null ? this.resourceId.replace("\\", "").replace("\"","\\\"") : "";
-            String path = this.path != null ? this.path.replace("\"","\\\"") : "";
+            String identifier = this.identifier != null ? this.identifier.replace("\\", "").replace("\"","\\\"") : "";
-            String theme = this.theme != null ? this.theme.replace("\"","\\\"") : "";
+            String path = this.path != null ? this.path.replace("\\", "").replace("\"","\\\"") : "";
-            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\";</script>", this.qdnContext, theme, service, name, identifier, path, baseUrl);
+            String theme = this.theme != null ? this.theme.replace("\\", "").replace("\"","\\\"") : "";
+            String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\\", "").replace("\"","\\\"") : "";
+            String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\\", "").replace("\"","\\\"") : "";
+            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath);
             head.get(0).prepend(qdnContextVar);
 
             // Add base href tag
-            String baseElement = String.format("<base href=\"%s/\">", baseUrl);
+            // Exclude the path if this request was routed back to the index automatically
+            String baseHref = this.usingCustomRouting ? this.qdnBase : this.qdnBaseWithPath;
+            String baseElement = String.format("<base href=\"%s/\">", baseHref);
             head.get(0).prepend(baseElement);
 
             // Add meta charset tag

src/main/java/org/qortal/api/model/PollVotes.java

@@ -0,0 +1,56 @@
+package org.qortal.api.model;
+
+import java.util.List;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+
+import org.qortal.data.voting.VoteOnPollData;
+
+@Schema(description = "Poll vote info, including voters")
+// All properties to be converted to JSON via JAX-RS
+@XmlAccessorType(XmlAccessType.FIELD)
+public class PollVotes {
+
+    @Schema(description = "List of individual votes")
+    @XmlElement(name = "votes")
+    public List<VoteOnPollData> votes;
+
+    @Schema(description = "Total number of votes")
+    public Integer totalVotes;
+
+    @Schema(description = "List of vote counts for each option")
+    public List<OptionCount> voteCounts;
+
+    // For JAX-RS
+    protected PollVotes() {
+    }
+
+    public PollVotes(List<VoteOnPollData> votes, Integer totalVotes, List<OptionCount> voteCounts) {
+        this.votes = votes;
+        this.totalVotes = totalVotes;
+        this.voteCounts = voteCounts;
+    }
+
+    @Schema(description = "Vote info")
+    // All properties to be converted to JSON via JAX-RS
+    @XmlAccessorType(XmlAccessType.FIELD)
+    public static class OptionCount {
+        @Schema(description = "Option name")
+        public String optionName;
+
+        @Schema(description = "Vote count")
+        public Integer voteCount;
+
+        // For JAX-RS
+        protected OptionCount() {
+        }
+
+        public OptionCount(String optionName, Integer voteCount) {
+            this.optionName = optionName;
+            this.voteCount = voteCount;
+        }
+    }
+}

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -65,10 +65,7 @@ import org.qortal.transaction.Transaction.ValidationResult;
 import org.qortal.transform.TransformationException;
 import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
 import org.qortal.transform.transaction.TransactionTransformer;
-import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.*;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-import org.qortal.utils.ZipUtils;
 
 @Path("/arbitrary")
 @Tag(name = "Arbitrary")
@@ -721,12 +718,9 @@ public class ArbitraryResource {
 			}
 	)
 	@SecurityRequirement(name = "apiKey")
-	public ArbitraryResourceMetadata getMetadata(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
+	public ArbitraryResourceMetadata getMetadata(@PathParam("service") Service service,
-							  @PathParam("service") Service service,
+							  					 @PathParam("name") String name,
-							  @PathParam("name") String name,
+							  					 @PathParam("identifier") String identifier) {
-							  @PathParam("identifier") String identifier) {
-		Security.checkApiCallAllowed(request);
-
 		ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
 
 		try {
@@ -1179,7 +1173,11 @@ public class ArbitraryResource {
 				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, error);
 			}
 
-			final Long minLatestBlockTimestamp = NTP.getTime() - (60 * 60 * 1000L);
+			final Long now = NTP.getTime();
+			if (now == null) {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NO_TIME_SYNC);
+			}
+			final Long minLatestBlockTimestamp = now - (60 * 60 * 1000L);
 			if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp)) {
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC);
 			}
@@ -1237,7 +1235,7 @@ public class ArbitraryResource {
 					// The actual data will be in a randomly-named subfolder of tempDirectory
 					// Remove hidden folders, i.e. starting with "_", as some systems can add them, e.g. "__MACOSX"
 					String[] files = tempDirectory.toFile().list((parent, child) -> !child.startsWith("_"));
-					if (files.length == 1) { // Single directory or file only
+					if (files != null && files.length == 1) { // Single directory or file only
 						path = Paths.get(tempDirectory.toString(), files[0]).toString();
 					}
 				}
@@ -1269,7 +1267,8 @@ public class ArbitraryResource {
 				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage());
 			}
 
-		} catch (DataException | IOException e) {
+		} catch (Exception e) {
+			LOGGER.info("Exception when publishing data: ", e);
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
 		}
 	}
@@ -1317,7 +1316,7 @@ public class ArbitraryResource {
 			if (filepath == null || filepath.isEmpty()) {
 				// No file path supplied - so check if this is a single file resource
 				String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal");
-				if (files.length == 1) {
+				if (files != null && files.length == 1) {
 					// This is a single file resource
 					filepath = files[0];
 				}
@@ -1327,20 +1326,50 @@ public class ArbitraryResource {
 				}
 			}
 
-			// TODO: limit file size that can be read into memory
 			java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
 			if (!Files.exists(path)) {
 				String message = String.format("No file exists at filepath: %s", filepath);
 				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message);
 			}
 
-			byte[] data = Files.readAllBytes(path);
+			byte[] data;
+			int fileSize = (int)path.toFile().length();
+			int length = fileSize;
+
+			// Parse "Range" header
+			Integer rangeStart = null;
+			Integer rangeEnd = null;
+			String range = request.getHeader("Range");
+			if (range != null) {
+				range = range.replace("bytes=", "");
+				String[] parts = range.split("-");
+				rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null;
+				rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize;
+			}
+
+			if (rangeStart != null && rangeEnd != null) {
+				// We have a range, so update the requested length
+				length = rangeEnd - rangeStart;
+			}
+
+			if (length < fileSize && encoding == null) {
+				// Partial content requested, and not encoding the data
+				response.setStatus(206);
+				response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize));
+				data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length);
+			}
+			else {
+				// Full content requested (or encoded data)
+				response.setStatus(200);
+				data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory
+			}
 
 			// Encode the data if requested
 			if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) {
 				data = Base64.encode(data);
 			}
 
+			response.addHeader("Accept-Ranges", "bytes");
 			response.setContentType(context.getMimeType(path.toString()));
 			response.setContentLength(data.length);
 			response.getOutputStream().write(data);

src/main/java/org/qortal/api/resource/ChatResource.java

@@ -119,6 +119,75 @@ public class ChatResource {
 		}
 	}
 
+	@GET
+	@Path("/messages/count")
+	@Operation(
+			summary = "Count chat messages",
+			description = "Returns count of CHAT messages that match criteria. Must provide EITHER 'txGroupId' OR two 'involving' addresses.",
+			responses = {
+					@ApiResponse(
+							description = "count of messages",
+							content = @Content(
+									mediaType = MediaType.TEXT_PLAIN,
+									schema = @Schema(
+											type = "integer"
+									)
+							)
+					)
+			}
+	)
+	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE})
+	public int countChatMessages(@QueryParam("before") Long before, @QueryParam("after") Long after,
+										@QueryParam("txGroupId") Integer txGroupId,
+										@QueryParam("involving") List<String> involvingAddresses,
+										@QueryParam("reference") String reference,
+										@QueryParam("chatreference") String chatReference,
+										@QueryParam("haschatreference") Boolean hasChatReference,
+										@QueryParam("sender") String sender,
+										@QueryParam("encoding") Encoding encoding,
+										@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
+										@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+										@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) {
+		// Check args meet expectations
+		if ((txGroupId == null && involvingAddresses.size() != 2)
+				|| (txGroupId != null && !involvingAddresses.isEmpty()))
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		// Check any provided addresses are valid
+		if (involvingAddresses.stream().anyMatch(address -> !Crypto.isValidAddress(address)))
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
+
+		if (before != null && before < 1500000000000L)
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		if (after != null && after < 1500000000000L)
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		byte[] referenceBytes = null;
+		if (reference != null)
+			referenceBytes = Base58.decode(reference);
+
+		byte[] chatReferenceBytes = null;
+		if (chatReference != null)
+			chatReferenceBytes = Base58.decode(chatReference);
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			return repository.getChatRepository().getMessagesMatchingCriteria(
+					before,
+					after,
+					txGroupId,
+					referenceBytes,
+					chatReferenceBytes,
+					hasChatReference,
+					involvingAddresses,
+					sender,
+					encoding,
+					limit, offset, reverse).size();
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}
+	}
+
 	@GET
 	@Path("/message/{signature}")
 	@Operation(

src/main/java/org/qortal/api/resource/NamesResource.java

@@ -155,6 +155,38 @@ public class NamesResource {
 		}
 	}
 
+	@GET
+	@Path("/search")
+	@Operation(
+			summary = "Search registered names",
+			responses = {
+					@ApiResponse(
+							description = "registered name info",
+							content = @Content(
+									mediaType = MediaType.APPLICATION_JSON,
+									array = @ArraySchema(schema = @Schema(implementation = NameData.class))
+							)
+					)
+			}
+	)
+	@ApiErrors({ApiError.NAME_UNKNOWN, ApiError.REPOSITORY_ISSUE})
+	public List<NameData> searchNames(@QueryParam("query") String query,
+									  @Parameter(ref = "limit") @QueryParam("limit") Integer limit,
+									  @Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+									  @Parameter(ref="reverse") @QueryParam("reverse") Boolean reverse) {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			if (query == null) {
+				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing query");
+			}
+
+			return repository.getNameRepository().searchNames(query, limit, offset, reverse);
+		} catch (ApiException e) {
+			throw e;
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}
+	}
+
 
 	@POST
 	@Path("/register")

src/main/java/org/qortal/api/resource/PollsResource.java

@@ -31,12 +31,18 @@ import javax.ws.rs.core.MediaType;
 
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.ArraySchema;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 import javax.ws.rs.GET;
 import javax.ws.rs.PathParam;
 import javax.ws.rs.QueryParam;
 import org.qortal.api.ApiException;
+import org.qortal.api.model.PollVotes;
 import org.qortal.data.voting.PollData;
+import org.qortal.data.voting.PollOptionData;
+import org.qortal.data.voting.VoteOnPollData;
 
 @Path("/polls")
 @Tag(name = "Polls")
@@ -102,6 +108,61 @@ public class PollsResource {
             }
     }
 
+    @GET
+    @Path("/votes/{pollName}")
+    @Operation(
+            summary = "Votes on poll",
+            responses = {
+                    @ApiResponse(
+                            description = "poll votes",
+                            content = @Content(
+                                    mediaType = MediaType.APPLICATION_JSON,
+                                    schema = @Schema(implementation = PollVotes.class)
+                            )
+                    )
+            }
+    )
+    @ApiErrors({ApiError.REPOSITORY_ISSUE})
+    public PollVotes getPollVotes(@PathParam("pollName") String pollName, @QueryParam("onlyCounts") Boolean onlyCounts) {
+            try (final Repository repository = RepositoryManager.getRepository()) {
+                    PollData pollData = repository.getVotingRepository().fromPollName(pollName);
+                    if (pollData == null)
+                            throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.POLL_NO_EXISTS);
+
+                    List<VoteOnPollData> votes = repository.getVotingRepository().getVotes(pollName);
+
+                    // Initialize map for counting votes
+                    Map<String, Integer> voteCountMap = new HashMap<>();
+                    for (PollOptionData optionData : pollData.getPollOptions()) {
+                            voteCountMap.put(optionData.getOptionName(), 0);
+                    }
+
+                    int totalVotes = 0;
+                    for (VoteOnPollData vote : votes) {
+                            String selectedOption = pollData.getPollOptions().get(vote.getOptionIndex()).getOptionName();
+                            if (voteCountMap.containsKey(selectedOption)) {
+                                    voteCountMap.put(selectedOption, voteCountMap.get(selectedOption) + 1);
+                                    totalVotes++;
+                            }
+                    }
+
+                    // Convert map to list of VoteInfo
+                    List<PollVotes.OptionCount> voteCounts = voteCountMap.entrySet().stream()
+                        .map(entry -> new PollVotes.OptionCount(entry.getKey(), entry.getValue()))
+                        .collect(Collectors.toList());
+    
+                    if (onlyCounts != null && onlyCounts) {
+                        return new PollVotes(null, totalVotes, voteCounts);
+                    } else {
+                        return new PollVotes(votes, totalVotes, voteCounts);
+                    }
+            } catch (ApiException e) {
+                    throw e;
+            } catch (DataException e) {
+                    throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+            }
+    }
+
     @POST
     @Path("/create")
     @Operation(

src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java

@@ -54,10 +54,6 @@ public class ArbitraryDataBuilder {
     /**
      * Process transactions, but do not build anything
      * This is useful for checking the status of a given resource
-     *
-     * @throws DataException
-     * @throws IOException
-     * @throws MissingDataException
      */
     public void process() throws DataException, IOException, MissingDataException {
         this.fetchTransactions();
@@ -69,10 +65,6 @@ public class ArbitraryDataBuilder {
 
     /**
      * Build the latest state of a given resource
-     *
-     * @throws DataException
-     * @throws IOException
-     * @throws MissingDataException
      */
     public void build() throws DataException, IOException, MissingDataException {
         this.process();

src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java

@@ -9,7 +9,6 @@ import org.qortal.arbitrary.exception.MissingDataException;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
-import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
 import org.qortal.crypto.AES;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ArbitraryTransactionData.*;
@@ -35,6 +34,9 @@ import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
 
 public class ArbitraryDataReader {
 
@@ -60,6 +62,10 @@ public class ArbitraryDataReader {
     // The resource being read
     ArbitraryDataResource arbitraryDataResource = null;
 
+    // Track resources that are currently being loaded, to avoid duplicate concurrent builds
+    // TODO: all builds could be handled by the build queue (even synchronous ones), to avoid the need for this
+    private static Map<String, Long> inProgress = Collections.synchronizedMap(new HashMap<>());
+
     public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
         // Ensure names are always lowercase
         if (resourceIdType == ResourceIdType.NAME) {
@@ -154,9 +160,6 @@ public class ArbitraryDataReader {
      * If no exception is thrown, you can then use getFilePath() to access the data immediately after returning
      *
      * @param overwrite - set to true to force rebuild an existing cache
-     * @throws IOException
-     * @throws DataException
-     * @throws MissingDataException
      */
     public void loadSynchronously(boolean overwrite) throws DataException, IOException, MissingDataException {
         try {
@@ -170,6 +173,12 @@ public class ArbitraryDataReader {
 
             this.arbitraryDataResource = this.createArbitraryDataResource();
 
+            // Don't allow duplicate loads
+            if (!this.canStartLoading()) {
+                LOGGER.debug("Skipping duplicate load of {}", this.arbitraryDataResource);
+                return;
+            }
+
             this.preExecute();
             this.deleteExistingFiles();
             this.fetch();
@@ -197,6 +206,7 @@ public class ArbitraryDataReader {
 
     private void preExecute() throws DataException {
         ArbitraryDataBuildManager.getInstance().setBuildInProgress(true);
+
         this.checkEnabled();
         this.createWorkingDirectory();
         this.createUncompressedDirectory();
@@ -204,6 +214,9 @@ public class ArbitraryDataReader {
 
     private void postExecute() {
         ArbitraryDataBuildManager.getInstance().setBuildInProgress(false);
+
+        this.arbitraryDataResource = this.createArbitraryDataResource();
+        ArbitraryDataReader.inProgress.remove(this.arbitraryDataResource.getUniqueKey());
     }
 
     private void checkEnabled() throws DataException {
@@ -212,6 +225,17 @@ public class ArbitraryDataReader {
         }
     }
 
+    private boolean canStartLoading() {
+        // Avoid duplicate builds if we're already loading this resource
+        String uniqueKey = this.arbitraryDataResource.getUniqueKey();
+        if (ArbitraryDataReader.inProgress.containsKey(uniqueKey)) {
+            return false;
+        }
+        ArbitraryDataReader.inProgress.put(uniqueKey, NTP.getTime());
+
+        return true;
+    }
+
     private void createWorkingDirectory() throws DataException {
         try {
             Files.createDirectories(this.workingPath);
@@ -223,7 +247,6 @@ public class ArbitraryDataReader {
     /**
      * Working directory should only be deleted on failure, since it is currently used to
      * serve a cached version of the resource for subsequent requests.
-     * @throws IOException
      */
     private void deleteWorkingDirectory() {
         try {
@@ -303,7 +326,7 @@ public class ArbitraryDataReader {
                 break;
 
             default:
-                throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType.toString()));
+                throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType));
         }
     }
 
@@ -368,6 +391,9 @@ public class ArbitraryDataReader {
         // Load data file(s)
         ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
         ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
+        if (arbitraryDataFile == null) {
+            throw new DataException(String.format("arbitraryDataFile is null"));
+        }
 
         if (!arbitraryDataFile.allFilesExist()) {
             if (ListUtils.isNameBlocked(transactionData.getName())) {
@@ -443,6 +469,7 @@ public class ArbitraryDataReader {
                 Path unencryptedPath = Paths.get(this.workingPath.toString(), "zipped.zip");
                 SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, "AES");
                 AES.decryptFile(algorithm, aesKey, this.filePath.toString(), unencryptedPath.toString());
+                LOGGER.debug("Finished decrypting {} using algorithm {}", this.arbitraryDataResource, algorithm);
 
                 // Replace filePath pointer with the encrypted file path
                 // Don't delete the original ArbitraryDataFile, as this is handled in the cleanup phase
@@ -477,7 +504,9 @@ public class ArbitraryDataReader {
 
             // Handle each type of compression
             if (compression == Compression.ZIP) {
+                LOGGER.debug("Unzipping {}...", this.arbitraryDataResource);
                 ZipUtils.unzip(this.filePath.toString(), this.uncompressedPath.getParent().toString());
+                LOGGER.debug("Finished unzipping {}", this.arbitraryDataResource);
             }
             else if (compression == Compression.NONE) {
                 Files.createDirectories(this.uncompressedPath);
@@ -513,10 +542,12 @@ public class ArbitraryDataReader {
 
     private void validate() throws IOException, DataException {
         if (this.service.isValidationRequired()) {
+            LOGGER.debug("Validating {}...", this.arbitraryDataResource);
             Service.ValidationResult result = this.service.validate(this.filePath);
             if (result != Service.ValidationResult.OK) {
                 throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
             }
+            LOGGER.debug("Finished validating {}", this.arbitraryDataResource);
         }
     }
 

src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java

@@ -67,8 +67,8 @@ public class ArbitraryDataRenderer {
     }
 
     public HttpServletResponse render() {
-        if (!inPath.startsWith(File.separator)) {
+        if (!inPath.startsWith("/")) {
-            inPath = File.separator + inPath;
+            inPath = "/" + inPath;
         }
 
         // Don't render data if QDN is disabled
@@ -126,6 +126,7 @@ public class ArbitraryDataRenderer {
         try {
             String filename = this.getFilename(unzippedPath, inPath);
             Path filePath = Paths.get(unzippedPath, filename);
+            boolean usingCustomRouting = false;
             
             // If the file doesn't exist, we may need to route the request elsewhere, or cleanup
             if (!Files.exists(filePath)) {
@@ -148,6 +149,7 @@ public class ArbitraryDataRenderer {
                             // Forward request to index file
                             filePath = indexPath;
                             filename = indexFile;
+                            usingCustomRouting = true;
                             break;
                         }
                     }
@@ -157,7 +159,7 @@ public class ArbitraryDataRenderer {
             if (HTMLParser.isHtmlFile(filename)) {
                 // HTML file - needs to be parsed
                 byte[] data = Files.readAllBytes(filePath); // TODO: limit file size that can be read into memory
-                HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data, qdnContext, service, identifier, theme);
+                HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data, qdnContext, service, identifier, theme, usingCustomRouting);
                 htmlParser.addAdditionalHeaderTags();
                 response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:;");
                 response.setContentType(context.getMimeType(filename));

src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java

@@ -150,6 +150,9 @@ public class ArbitraryDataResource {
 
             for (ArbitraryTransactionData transactionData : transactionDataList) {
                 ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
+                if (arbitraryDataFile == null) {
+                    continue;
+                }
 
                 // Delete any chunks or complete files from each transaction
                 arbitraryDataFile.deleteAll(deleteMetadata);

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java

@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
@@ -50,7 +51,7 @@ public class ArbitraryDataMetadata {
             this.readJson();
 
         } catch (JSONException e) {
-            throw new DataException(String.format("Unable to read JSON: %s", e.getMessage()));
+            throw new DataException(String.format("Unable to read JSON at path %s: %s", this.filePath, e.getMessage()));
         }
     }
 
@@ -64,6 +65,10 @@ public class ArbitraryDataMetadata {
         writer.close();
     }
 
+    public void delete() throws IOException {
+        Files.delete(this.filePath);
+    }
+
 
     protected void loadJson() throws IOException {
         File metadataFile = new File(this.filePath.toString());
@@ -71,7 +76,7 @@ public class ArbitraryDataMetadata {
             throw new IOException(String.format("Metadata file doesn't exist: %s", this.filePath.toString()));
         }
 
-        this.jsonString = new String(Files.readAllBytes(this.filePath));
+        this.jsonString = new String(Files.readAllBytes(this.filePath), StandardCharsets.UTF_8);
     }
 
 

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java

@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -69,7 +70,7 @@ public class ArbitraryDataQortalMetadata extends ArbitraryDataMetadata {
             throw new IOException(String.format("Patch file doesn't exist: %s", path.toString()));
         }
 
-        this.jsonString = new String(Files.readAllBytes(path));
+        this.jsonString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
     }
 
 

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java

@@ -7,6 +7,7 @@ import org.qortal.arbitrary.misc.Category;
 import org.qortal.repository.DataException;
 import org.qortal.utils.Base58;
 
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -217,6 +218,25 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
 
     // Static helper methods
 
+    public static String trimUTF8String(String string, int maxLength) {
+        byte[] inputBytes = string.getBytes(StandardCharsets.UTF_8);
+        int length = Math.min(inputBytes.length, maxLength);
+        byte[] outputBytes = new byte[length];
+
+        System.arraycopy(inputBytes, 0, outputBytes, 0, length);
+        String result = new String(outputBytes, StandardCharsets.UTF_8);
+
+        // check if last character is truncated
+        int lastIndex = result.length() - 1;
+
+        if (lastIndex > 0 && result.charAt(lastIndex) != string.charAt(lastIndex)) {
+            // last character is truncated so remove the last character
+            return result.substring(0, lastIndex);
+        }
+
+        return result;
+    }
+
     public static String limitTitle(String title) {
         if (title == null) {
             return null;
@@ -225,7 +245,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
             return null;
         }
 
-        return title.substring(0, Math.min(title.length(), MAX_TITLE_LENGTH));
+        return trimUTF8String(title, MAX_TITLE_LENGTH);
     }
 
     public static String limitDescription(String description) {
@@ -236,7 +256,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
             return null;
         }
 
-        return description.substring(0, Math.min(description.length(), MAX_DESCRIPTION_LENGTH));
+        return trimUTF8String(description, MAX_DESCRIPTION_LENGTH);
     }
 
     public static List<String> limitTags(List<String> tags) {

src/main/java/org/qortal/arbitrary/misc/Service.java

@@ -9,7 +9,6 @@ import org.qortal.utils.FilesystemUtils;
 import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
@@ -20,9 +19,9 @@ import static java.util.Arrays.stream;
 import static java.util.stream.Collectors.toMap;
 
 public enum Service {
-    AUTO_UPDATE(1, false, null, false, null),
+    AUTO_UPDATE(1, false, null, false, false, null),
-    ARBITRARY_DATA(100, false, null, false, null),
+    ARBITRARY_DATA(100, false, null, false, false, null),
-    QCHAT_ATTACHMENT(120, true, 1024*1024L, true, null) {
+    QCHAT_ATTACHMENT(120, true, 1024*1024L, true, false, null) {
         @Override
         public ValidationResult validate(Path path) throws IOException {
             ValidationResult superclassResult = super.validate(path);
@@ -47,7 +46,14 @@ public enum Service {
             return ValidationResult.OK;
         }
     },
-    WEBSITE(200, true, null, false, null) {
+    QCHAT_ATTACHMENT_PRIVATE(121, true, 1024*1024L, true, true, null),
+    ATTACHMENT(130, false, 50*1024*1024L, true, false, null),
+    ATTACHMENT_PRIVATE(131, true, 50*1024*1024L, true, true, null),
+    FILE(140, false, null, true, false, null),
+    FILE_PRIVATE(141, true, null, true, true, null),
+    FILES(150, false, null, false, false, null),
+    CHAIN_DATA(160, true, 239L, true, false, null),
+    WEBSITE(200, true, null, false, false, null) {
         @Override
         public ValidationResult validate(Path path) throws IOException {
             ValidationResult superclassResult = super.validate(path);
@@ -69,23 +75,30 @@ public enum Service {
             return ValidationResult.MISSING_INDEX_FILE;
         }
     },
-    GIT_REPOSITORY(300, false, null, false, null),
+    GIT_REPOSITORY(300, false, null, false, false, null),
-    IMAGE(400, true, 10*1024*1024L, true, null),
+    IMAGE(400, true, 10*1024*1024L, true, false, null),
-    THUMBNAIL(410, true, 500*1024L, true, null),
+    IMAGE_PRIVATE(401, true, 10*1024*1024L, true, true, null),
-    QCHAT_IMAGE(420, true, 500*1024L, true, null),
+    THUMBNAIL(410, true, 500*1024L, true, false, null),
-    VIDEO(500, false, null, true, null),
+    QCHAT_IMAGE(420, true, 500*1024L, true, false, null),
-    AUDIO(600, false, null, true, null),
+    VIDEO(500, false, null, true, false, null),
-    QCHAT_AUDIO(610, true, 10*1024*1024L, true, null),
+    VIDEO_PRIVATE(501, true, null, true, true, null),
-    QCHAT_VOICE(620, true, 10*1024*1024L, true, null),
+    AUDIO(600, false, null, true, false, null),
-    BLOG(700, false, null, false, null),
+    AUDIO_PRIVATE(601, true, null, true, true, null),
-    BLOG_POST(777, false, null, true, null),
+    QCHAT_AUDIO(610, true, 10*1024*1024L, true, false, null),
-    BLOG_COMMENT(778, false, null, true, null),
+    QCHAT_VOICE(620, true, 10*1024*1024L, true, false, null),
-    DOCUMENT(800, false, null, true, null),
+    VOICE(630, true, 10*1024*1024L, true, false, null),
-    LIST(900, true, null, true, null),
+    VOICE_PRIVATE(631, true, 10*1024*1024L, true, true, null),
-    PLAYLIST(910, true, null, true, null),
+    PODCAST(640, false, null, true, false, null),
-    APP(1000, true, 50*1024*1024L, false, null),
+    BLOG(700, false, null, false, false, null),
-    METADATA(1100, false, null, true, null),
+    BLOG_POST(777, false, null, true, false, null),
-    JSON(1110, true, 25*1024L, true, null) {
+    BLOG_COMMENT(778, true, 500*1024L, true, false, null),
+    DOCUMENT(800, false, null, true, false, null),
+    DOCUMENT_PRIVATE(801, true, null, true, true, null),
+    LIST(900, true, null, true, false, null),
+    PLAYLIST(910, true, null, true, false, null),
+    APP(1000, true, 50*1024*1024L, false, false, null),
+    METADATA(1100, false, null, true, false, null),
+    JSON(1110, true, 25*1024L, true, false, null) {
         @Override
         public ValidationResult validate(Path path) throws IOException {
             ValidationResult superclassResult = super.validate(path);
@@ -94,7 +107,7 @@ public enum Service {
             }
 
             // Require valid JSON
-            byte[] data = FilesystemUtils.getSingleFileContents(path);
+            byte[] data = FilesystemUtils.getSingleFileContents(path, 25*1024);
             String json = new String(data, StandardCharsets.UTF_8);
             try {
                 objectMapper.readTree(json);
@@ -104,7 +117,7 @@ public enum Service {
             }
         }
     },
-    GIF_REPOSITORY(1200, true, 25*1024*1024L, false, null) {
+    GIF_REPOSITORY(1200, true, 25*1024*1024L, false, false, null) {
         @Override
         public ValidationResult validate(Path path) throws IOException {
             ValidationResult superclassResult = super.validate(path);
@@ -139,12 +152,31 @@ public enum Service {
             }
             return ValidationResult.OK;
         }
-    };
+    },
+    STORE(1300, false, null, true, false, null),
+    PRODUCT(1310, false, null, true, false, null),
+    OFFER(1330, false, null, true, false, null),
+    COUPON(1340, false, null, true, false, null),
+    CODE(1400, false, null, true, false, null),
+    PLUGIN(1410, false, null, true, false, null),
+    EXTENSION(1420, false, null, true, false, null),
+    GAME(1500, false, null, false, false, null),
+    ITEM(1510, false, null, true, false, null),
+    NFT(1600, false, null, true, false, null),
+    DATABASE(1700, false, null, false, false, null),
+    SNAPSHOT(1710, false, null, false, false, null),
+    COMMENT(1800, true, 500*1024L, true, false, null),
+    CHAIN_COMMENT(1810, true, 239L, true, false, null),
+    MAIL(1900, true, 1024*1024L, true, false, null),
+    MAIL_PRIVATE(1901, true, 1024*1024L, true, true, null),
+    MESSAGE(1910, true, 1024*1024L, true, false, null),
+    MESSAGE_PRIVATE(1911, true, 1024*1024L, true, true, null);
 
     public final int value;
     private final boolean requiresValidation;
     private final Long maxSize;
     private final boolean single;
+    private final boolean isPrivate;
     private final List<String> requiredKeys;
 
     private static final Map<Integer, Service> map = stream(Service.values())
@@ -153,11 +185,14 @@ public enum Service {
     // For JSON validation
     private static final ObjectMapper objectMapper = new ObjectMapper();
 
-    Service(int value, boolean requiresValidation, Long maxSize, boolean single, List<String> requiredKeys) {
+    private static final String encryptedDataPrefix = "qortalEncryptedData";
+
+    Service(int value, boolean requiresValidation, Long maxSize, boolean single, boolean isPrivate, List<String> requiredKeys) {
         this.value = value;
         this.requiresValidation = requiresValidation;
         this.maxSize = maxSize;
         this.single = single;
+        this.isPrivate = isPrivate;
         this.requiredKeys = requiredKeys;
     }
 
@@ -166,7 +201,9 @@ public enum Service {
             return ValidationResult.OK;
         }
 
-        byte[] data = FilesystemUtils.getSingleFileContents(path);
+        // Load the first 25KB of data. This only needs to be long enough to check the prefix
+        // and also to allow for possible additional future validation of smaller files.
+        byte[] data = FilesystemUtils.getSingleFileContents(path, 25*1024);
         long size = FilesystemUtils.getDirectorySize(path);
 
         // Validate max size if needed
@@ -181,6 +218,17 @@ public enum Service {
             return ValidationResult.INVALID_FILE_COUNT;
         }
 
+        // Validate private data for single file resources
+        if (this.single) {
+            String dataString = new String(data, StandardCharsets.UTF_8);
+            if (this.isPrivate && !dataString.startsWith(encryptedDataPrefix)) {
+                return ValidationResult.DATA_NOT_ENCRYPTED;
+            }
+            if (!this.isPrivate && dataString.startsWith(encryptedDataPrefix)) {
+                return ValidationResult.DATA_ENCRYPTED;
+            }
+        }
+
         // Validate required keys if needed
         if (this.requiredKeys != null) {
             if (data == null) {
@@ -199,7 +247,12 @@ public enum Service {
     }
 
     public boolean isValidationRequired() {
-        return this.requiresValidation;
+        // We must always validate single file resources, to ensure they are actually a single file
+        return this.requiresValidation || this.single;
+    }
+
+    public boolean isPrivate() {
+        return this.isPrivate;
     }
 
     public static Service valueOf(int value) {
@@ -207,10 +260,41 @@ public enum Service {
     }
 
     public static JSONObject toJsonObject(byte[] data) {
-        String dataString = new String(data);
+        String dataString = new String(data, StandardCharsets.UTF_8);
         return new JSONObject(dataString);
     }
 
+    public static List<Service> publicServices() {
+        List<Service> privateServices = new ArrayList<>();
+        for (Service service : Service.values()) {
+            if (!service.isPrivate) {
+                privateServices.add(service);
+            }
+        }
+        return privateServices;
+    }
+
+    /**
+     * Fetch a list of Service objects that require encrypted data.
+     *
+     * These can ultimately be used to help inform the cleanup manager
+     * on the best order to delete files when the node runs out of space.
+     * Public data should be given priority over private data (unless
+     * this node is part of a data market contract for that data - this
+     * isn't developed yet).
+     *
+     * @return a list of Service objects that require encrypted data.
+     */
+    public static List<Service> privateServices() {
+        List<Service> privateServices = new ArrayList<>();
+        for (Service service : Service.values()) {
+            if (service.isPrivate) {
+                privateServices.add(service);
+            }
+        }
+        return privateServices;
+    }
+
     public enum ValidationResult {
         OK(1),
         MISSING_KEYS(2),
@@ -220,7 +304,9 @@ public enum Service {
         INVALID_FILE_EXTENSION(6),
         MISSING_DATA(7),
         INVALID_FILE_COUNT(8),
-        INVALID_CONTENT(9);
+        INVALID_CONTENT(9),
+        DATA_NOT_ENCRYPTED(10),
+        DATA_ENCRYPTED(10);
 
         public final int value;
 

src/main/java/org/qortal/controller/OnlineAccountsManager.java

@@ -504,110 +504,118 @@ public class OnlineAccountsManager {
         computeOurAccountsForTimestamp(onlineAccountsTimestamp);
     }
 
-    private boolean computeOurAccountsForTimestamp(long onlineAccountsTimestamp) {
+    private boolean computeOurAccountsForTimestamp(Long onlineAccountsTimestamp) {
-        List<MintingAccountData> mintingAccounts;
+        if (onlineAccountsTimestamp != null) {
-        try (final Repository repository = RepositoryManager.getRepository()) {
+            List<MintingAccountData> mintingAccounts;
-            mintingAccounts = repository.getAccountRepository().getMintingAccounts();
+            try (final Repository repository = RepositoryManager.getRepository()) {
+                mintingAccounts = repository.getAccountRepository().getMintingAccounts();
 
-            // We have no accounts to send
+                // We have no accounts to send
-            if (mintingAccounts.isEmpty())
+                if (mintingAccounts.isEmpty())
+                    return false;
+
+                // Only active reward-shares allowed
+                Iterator<MintingAccountData> iterator = mintingAccounts.iterator();
+                int i = 0;
+                while (iterator.hasNext()) {
+                    MintingAccountData mintingAccountData = iterator.next();
+
+                    RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
+                    if (rewardShareData == null) {
+                        // Reward-share doesn't even exist - probably not a good sign
+                        iterator.remove();
+                        continue;
+                    }
+
+                    Account mintingAccount = new Account(repository, rewardShareData.getMinter());
+                    if (!mintingAccount.canMint()) {
+                        // Minting-account component of reward-share can no longer mint - disregard
+                        iterator.remove();
+                        continue;
+                    }
+
+                    if (++i > 1 + 1) {
+                        iterator.remove();
+                        continue;
+                    }
+                }
+            } catch (DataException e) {
+                LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
                 return false;
+            }
 
-            // Only active reward-shares allowed
+            byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
-            Iterator<MintingAccountData> iterator = mintingAccounts.iterator();
+            List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
-            while (iterator.hasNext()) {
-                MintingAccountData mintingAccountData = iterator.next();
 
-                RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
+            int remaining = mintingAccounts.size();
-                if (rewardShareData == null) {
+            for (MintingAccountData mintingAccountData : mintingAccounts) {
-                    // Reward-share doesn't even exist - probably not a good sign
+                remaining--;
-                    iterator.remove();
+                byte[] privateKey = mintingAccountData.getPrivateKey();
+                byte[] publicKey = Crypto.toPublicKey(privateKey);
+
+                // We don't want to compute the online account nonce and signature again if it already exists
+                Set<OnlineAccountData> onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountsTimestamp, k -> ConcurrentHashMap.newKeySet());
+                boolean alreadyExists = onlineAccounts.stream().anyMatch(a -> Arrays.equals(a.getPublicKey(), publicKey));
+                if (alreadyExists) {
+                    this.hasOurOnlineAccounts = true;
+
+                    if (remaining > 0) {
+                        // Move on to next account
+                        continue;
+                    } else {
+                        // Everything exists, so return true
+                        return true;
+                    }
+                }
+
+                // Generate bytes for mempow
+                byte[] mempowBytes;
+                try {
+                    mempowBytes = this.getMemoryPoWBytes(publicKey, onlineAccountsTimestamp);
+                } catch (IOException e) {
+                    LOGGER.info("Unable to create bytes for MemoryPoW. Moving on to next account...");
                     continue;
                 }
 
-                Account mintingAccount = new Account(repository, rewardShareData.getMinter());
+                // Compute nonce
-                if (!mintingAccount.canMint()) {
+                Integer nonce;
-                    // Minting-account component of reward-share can no longer mint - disregard
+                try {
-                    iterator.remove();
+                    nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp);
-                    continue;
+                    if (nonce == null) {
-                }
+                        // A nonce is required
-            }
+                        return false;
-        } catch (DataException e) {
+                    }
-            LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
+                } catch (TimeoutException e) {
-            return false;
+                    LOGGER.info(String.format("Timed out computing nonce for account %.8s", Base58.encode(publicKey)));
-        }
-
-        byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
-        List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
-
-        int remaining = mintingAccounts.size();
-        for (MintingAccountData mintingAccountData : mintingAccounts) {
-            remaining--;
-            byte[] privateKey = mintingAccountData.getPrivateKey();
-            byte[] publicKey = Crypto.toPublicKey(privateKey);
-
-            // We don't want to compute the online account nonce and signature again if it already exists
-            Set<OnlineAccountData> onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountsTimestamp, k -> ConcurrentHashMap.newKeySet());
-            boolean alreadyExists = onlineAccounts.stream().anyMatch(a -> Arrays.equals(a.getPublicKey(), publicKey));
-            if (alreadyExists) {
-                this.hasOurOnlineAccounts = true;
-
-                if (remaining > 0) {
-                    // Move on to next account
-                    continue;
-                }
-                else {
-                    // Everything exists, so return true
-                    return true;
-                }
-            }
-
-            // Generate bytes for mempow
-            byte[] mempowBytes;
-            try {
-                mempowBytes = this.getMemoryPoWBytes(publicKey, onlineAccountsTimestamp);
-            }
-            catch (IOException e) {
-                LOGGER.info("Unable to create bytes for MemoryPoW. Moving on to next account...");
-                continue;
-            }
-
-            // Compute nonce
-            Integer nonce;
-            try {
-                nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp);
-                if (nonce == null) {
-                    // A nonce is required
                     return false;
                 }
-            } catch (TimeoutException e) {
+
-                LOGGER.info(String.format("Timed out computing nonce for account %.8s", Base58.encode(publicKey)));
+                byte[] signature = Qortal25519Extras.signForAggregation(privateKey, timestampBytes);
+
+                // Our account is online
+                OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce);
+
+                // Make sure to verify before adding
+                if (verifyMemoryPoW(ourOnlineAccountData, null)) {
+                    ourOnlineAccounts.add(ourOnlineAccountData);
+                }
+            }
+
+            this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty();
+
+            boolean hasInfoChanged = addAccounts(ourOnlineAccounts);
+
+            if (!hasInfoChanged)
                 return false;
-            }
 
-            byte[] signature = Qortal25519Extras.signForAggregation(privateKey, timestampBytes);
+            Network.getInstance().broadcast(peer -> new OnlineAccountsV3Message(ourOnlineAccounts));
 
-            // Our account is online
+            LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp);
-            OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce);
 
-            // Make sure to verify before adding
+            return true;
-            if (verifyMemoryPoW(ourOnlineAccountData, null)) {
-                ourOnlineAccounts.add(ourOnlineAccountData);
-            }
         }
 
-        this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty();
+        return false;
-
-        boolean hasInfoChanged = addAccounts(ourOnlineAccounts);
-
-        if (!hasInfoChanged)
-            return false;
-
-        Network.getInstance().broadcast(peer -> new OnlineAccountsV3Message(ourOnlineAccounts));
-
-        LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp);
-
-        return true;
     }
 
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java

@@ -346,6 +346,10 @@ public class ArbitraryDataCleanupManager extends Thread {
 	/**
 	 * Iteratively walk through given directory and delete a single random file
 	 *
+	 * TODO: public data should be prioritized over private data
+	 * (unless this node is part of a data market contract for that data).
+	 * See: Service.privateServices() for a list of services containing private data.
+	 *
 	 * @param directory - the base directory
 	 * @return boolean - whether a file was deleted
 	 */

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java

@@ -124,29 +124,29 @@ public class ArbitraryDataFileListManager {
             }
         }
 
-        // Then allow another 3 attempts, each 5 minutes apart
+        // Then allow another 5 attempts, each 1 minute apart
-        if (timeSinceLastAttempt > 5 * 60 * 1000L) {
+        if (timeSinceLastAttempt > 60 * 1000L) {
-            // We haven't tried for at least 5 minutes
+            // We haven't tried for at least 1 minute
 
-            if (networkBroadcastCount < 6) {
+            if (networkBroadcastCount < 8) {
-                // We've made less than 6 total attempts
+                // We've made less than 8 total attempts
                 return true;
             }
         }
 
-        // Then allow another 4 attempts, each 30 minutes apart
+        // Then allow another 8 attempts, each 15 minutes apart
-        if (timeSinceLastAttempt > 30 * 60 * 1000L) {
+        if (timeSinceLastAttempt > 15 * 60 * 1000L) {
-            // We haven't tried for at least 5 minutes
+            // We haven't tried for at least 15 minutes
 
-            if (networkBroadcastCount < 10) {
+            if (networkBroadcastCount < 16) {
-                // We've made less than 10 total attempts
+                // We've made less than 16 total attempts
                 return true;
             }
         }
 
-        // From then on, only try once every 24 hours, to reduce network spam
+        // From then on, only try once every 6 hours, to reduce network spam
-        if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) {
+        if (timeSinceLastAttempt > 6 * 60 * 60 * 1000L) {
-            // We haven't tried for at least 24 hours
+            // We haven't tried for at least 6 hours
             return true;
         }
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -16,7 +16,6 @@ import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.Controller;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
-import org.qortal.list.ResourceListManager;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
 import org.qortal.repository.DataException;

src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java

@@ -102,7 +102,14 @@ public class ArbitraryMetadataManager {
                 if (metadataFile.exists()) {
                     // Use local copy
                     ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
-                    transactionMetadata.read();
+                    try {
+                        transactionMetadata.read();
+                    } catch (DataException e) {
+                        // Invalid file, so delete it
+                        LOGGER.info("Deleting invalid metadata file due to exception: {}", e.getMessage());
+                        transactionMetadata.delete();
+                        return null;
+                    }
                     return transactionMetadata;
                 }
             }

src/main/java/org/qortal/data/transaction/TransactionData.java

@@ -13,6 +13,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 import org.eclipse.persistence.oxm.annotations.XmlDiscriminatorNode;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.voting.PollData;
+import org.qortal.data.voting.VoteOnPollData;
 import org.qortal.transaction.Transaction.ApprovalStatus;
 import org.qortal.transaction.Transaction.TransactionType;
 
@@ -30,7 +31,7 @@ import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
 @XmlSeeAlso({GenesisTransactionData.class, PaymentTransactionData.class, RegisterNameTransactionData.class, UpdateNameTransactionData.class,
 	SellNameTransactionData.class, CancelSellNameTransactionData.class, BuyNameTransactionData.class,
 	CreatePollTransactionData.class, VoteOnPollTransactionData.class, ArbitraryTransactionData.class,
-	PollData.class,
+	PollData.class, VoteOnPollData.class,
 	IssueAssetTransactionData.class, TransferAssetTransactionData.class,
 	CreateAssetOrderTransactionData.class, CancelAssetOrderTransactionData.class,
 	MultiPaymentTransactionData.class, DeployAtTransactionData.class, MessageTransactionData.class, ATTransactionData.class,

src/main/java/org/qortal/data/voting/VoteOnPollData.java

@@ -9,6 +9,11 @@ public class VoteOnPollData {
 
 	// Constructors
 
+	// For JAXB
+	protected VoteOnPollData() {
+		super();
+	}
+
 	public VoteOnPollData(String pollName, byte[] voterPublicKey, int optionIndex) {
 		this.pollName = pollName;
 		this.voterPublicKey = voterPublicKey;
@@ -21,12 +26,24 @@ public class VoteOnPollData {
 		return this.pollName;
 	}
 
+	public void setPollName(String pollName) {
+		this.pollName = pollName;
+	}
+
 	public byte[] getVoterPublicKey() {
 		return this.voterPublicKey;
 	}
 
+	public void setVoterPublicKey(byte[] voterPublicKey) {
+		this.voterPublicKey = voterPublicKey;
+	}
+
 	public int getOptionIndex() {
 		return this.optionIndex;
 	}
 
+	public void setOptionIndex(int optionIndex) {
+		this.optionIndex = optionIndex;
+	}
+
 }

src/main/java/org/qortal/list/ResourceList.java

@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -81,7 +82,7 @@ public class ResourceList {
         }
 
         try {
-            String jsonString = new String(Files.readAllBytes(path));
+            String jsonString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
             this.list = ResourceList.listFromJSONString(jsonString);
         } catch (IOException e) {
             throw new IOException(String.format("Couldn't read contents from file %s", path.toString()));

src/main/java/org/qortal/repository/NameRepository.java

@@ -14,6 +14,8 @@ public interface NameRepository {
 
 	public boolean reducedNameExists(String reducedName) throws DataException;
 
+	public List<NameData> searchNames(String query, Integer limit, Integer offset, Boolean reverse) throws DataException;
+
 	public List<NameData> getAllNames(Integer limit, Integer offset, Boolean reverse) throws DataException;
 
 	public default List<NameData> getAllNames() throws DataException {

src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java

@@ -103,6 +103,57 @@ public class HSQLDBNameRepository implements NameRepository {
 		}
 	}
 
+	public List<NameData> searchNames(String query, Integer limit, Integer offset, Boolean reverse) throws DataException {
+		StringBuilder sql = new StringBuilder(512);
+		List<Object> bindParams = new ArrayList<>();
+
+		sql.append("SELECT name, reduced_name, owner, data, registered_when, updated_when, "
+				+ "is_for_sale, sale_price, reference, creation_group_id FROM Names "
+				+ "WHERE LCASE(name) LIKE ? ORDER BY name");
+
+		bindParams.add(String.format("%%%s%%", query.toLowerCase()));
+
+		if (reverse != null && reverse)
+			sql.append(" DESC");
+
+		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
+
+		List<NameData> names = new ArrayList<>();
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
+			if (resultSet == null)
+				return names;
+
+			do {
+				String name = resultSet.getString(1);
+				String reducedName = resultSet.getString(2);
+				String owner = resultSet.getString(3);
+				String data = resultSet.getString(4);
+				long registered = resultSet.getLong(5);
+
+				// Special handling for possibly-NULL "updated" column
+				Long updated = resultSet.getLong(6);
+				if (updated == 0 && resultSet.wasNull())
+					updated = null;
+
+				boolean isForSale = resultSet.getBoolean(7);
+
+				Long salePrice = resultSet.getLong(8);
+				if (salePrice == 0 && resultSet.wasNull())
+					salePrice = null;
+
+				byte[] reference = resultSet.getBytes(9);
+				int creationGroupId = resultSet.getInt(10);
+
+				names.add(new NameData(name, reducedName, owner, data, registered, updated, isForSale, salePrice, reference, creationGroupId));
+			} while (resultSet.next());
+
+			return names;
+		} catch (SQLException e) {
+			throw new DataException("Unable to search names in repository", e);
+		}
+	}
+
 	@Override
 	public List<NameData> getAllNames(Integer limit, Integer offset, Boolean reverse) throws DataException {
 		StringBuilder sql = new StringBuilder(256);

src/main/java/org/qortal/utils/FilesystemUtils.java

@@ -228,12 +228,18 @@ public class FilesystemUtils {
      * @throws IOException
      */
     public static byte[] getSingleFileContents(Path path) throws IOException {
+        return getSingleFileContents(path, null);
+    }
+
+    public static byte[] getSingleFileContents(Path path, Integer maxLength) throws IOException {
         byte[] data = null;
         // TODO: limit the file size that can be loaded into memory
 
         // If the path is a file, read the contents directly
         if (path.toFile().isFile()) {
-            data = Files.readAllBytes(path);
+            int fileSize = (int)path.toFile().length();
+            maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize;
+            data = FilesystemUtils.readFromFile(path.toString(), 0, maxLength);
         }
 
         // Or if it's a directory, only load file contents if there is a single file inside it
@@ -242,7 +248,9 @@ public class FilesystemUtils {
             if (files.length == 1) {
                 Path filePath = Paths.get(path.toString(), files[0]);
                 if (filePath.toFile().isFile()) {
-                    data = Files.readAllBytes(filePath);
+                    int fileSize = (int)filePath.toFile().length();
+                    maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize;
+                    data = FilesystemUtils.readFromFile(filePath.toString(), 0, maxLength);
                 }
             }
         }

src/main/resources/q-apps/q-apps-gateway.js

@@ -50,12 +50,16 @@ window.addEventListener("message", (event) => {
     switch (data.action) {
         case "GET_USER_ACCOUNT":
         case "PUBLISH_QDN_RESOURCE":
+        case "PUBLISH_MULTIPLE_QDN_RESOURCES":
         case "SEND_CHAT_MESSAGE":
         case "JOIN_GROUP":
         case "DEPLOY_AT":
         case "GET_WALLET_BALANCE":
         case "SEND_COIN":
-            const errorString = "Authentication was requested, but this is not yet supported when viewing via a gateway. To use interactive features, please access using the Qortal UI desktop app. More info at: https://qortal.org";
+        case "GET_LIST_ITEMS":
+        case "ADD_LIST_ITEMS":
+        case "DELETE_LIST_ITEM":
+            const errorString = "Interactive features were requested, but these are not yet supported when viewing via a gateway. To use interactive features, please access using the Qortal UI desktop app. More info at: https://qortal.org";
             response = "{\"error\": \"" + errorString + "\"}"
 
             const modalText = "This app is powered by the Qortal blockchain. You are viewing in read-only mode. To use interactive features, please access using the Qortal UI desktop app.";

src/main/resources/q-apps/q-apps.js

@@ -1,4 +1,4 @@
-function httpGet(event, url) {
+function httpGet(url) {
     var request = new XMLHttpRequest();
     request.open("GET", url, false);
     request.send(null);
@@ -169,7 +169,7 @@ window.addEventListener("message", (event) => {
         return;
     }
 
-    console.log("Core received event: " + JSON.stringify(event.data));
+    console.log("Core received action: " + JSON.stringify(event.data.action));
 
     let url;
     let data = event.data;
@@ -236,13 +236,15 @@ window.addEventListener("message", (event) => {
             if (data.identifier != null) url = url.concat("/" + data.identifier);
             url = url.concat("?");
             if (data.filepath != null) url = url.concat("&filepath=" + data.filepath);
-            if (data.rebuild != null) url = url.concat("&rebuild=" + new Boolean(data.rebuild).toString())
+            if (data.rebuild != null) url = url.concat("&rebuild=" + new Boolean(data.rebuild).toString());
             if (data.encoding != null) url = url.concat("&encoding=" + data.encoding);
             return httpGetAsyncWithEvent(event, url);
 
         case "GET_QDN_RESOURCE_STATUS":
             url = "/arbitrary/resource/status/" + data.service + "/" + data.name;
             if (data.identifier != null) url = url.concat("/" + data.identifier);
+            url = url.concat("?");
+            if (data.build != null) url = url.concat("&build=" + new Boolean(data.build).toString());
             return httpGetAsyncWithEvent(event, url);
 
         case "GET_QDN_RESOURCE_PROPERTIES":
@@ -250,6 +252,11 @@ window.addEventListener("message", (event) => {
             url = "/arbitrary/resource/properties/" + data.service + "/" + data.name + "/" + identifier;
             return httpGetAsyncWithEvent(event, url);
 
+        case "GET_QDN_RESOURCE_METADATA":
+            identifier = (data.identifier != null) ? data.identifier : "default";
+            url = "/arbitrary/metadata/" + data.service + "/" + data.name + "/" + identifier;
+            return httpGetAsyncWithEvent(event, url);
+
         case "SEARCH_CHAT_MESSAGES":
             url = "/chat/messages?";
             if (data.before != null) url = url.concat("&before=" + data.before);
@@ -259,6 +266,7 @@ window.addEventListener("message", (event) => {
             if (data.reference != null) url = url.concat("&reference=" + data.reference);
             if (data.chatReference != null) url = url.concat("&chatreference=" + data.chatReference);
             if (data.hasChatReference != null) url = url.concat("&haschatreference=" + new Boolean(data.hasChatReference).toString());
+            if (data.encoding != null) url = url.concat("&encoding=" + data.encoding);
             if (data.limit != null) url = url.concat("&limit=" + data.limit);
             if (data.offset != null) url = url.concat("&offset=" + data.offset);
             if (data.reverse != null) url = url.concat("&reverse=" + new Boolean(data.reverse).toString());
@@ -426,6 +434,8 @@ function getDefaultTimeout(action) {
         // Some actions need longer default timeouts, especially those that create transactions
         switch (action) {
             case "GET_USER_ACCOUNT":
+            case "SAVE_FILE":
+            case "DECRYPT_DATA":
                 // User may take a long time to accept/deny the popup
                 return 60 * 60 * 1000;
 
@@ -434,8 +444,8 @@ function getDefaultTimeout(action) {
                 return 60 * 1000;
 
             case "PUBLISH_QDN_RESOURCE":
+            case "PUBLISH_MULTIPLE_QDN_RESOURCES":
                 // Publishing could take a very long time on slow system, due to the proof-of-work computation
-                // It's best not to timeout
                 return 60 * 60 * 1000;
 
             case "SEND_CHAT_MESSAGE":

src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java

@@ -29,6 +29,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
+import java.util.List;
 import java.util.Random;
 
 import static org.junit.Assert.*;
@@ -436,4 +437,87 @@ public class ArbitraryServiceTests extends Common {
         assertEquals(ValidationResult.INVALID_FILE_COUNT, service.validate(path));
     }
 
+    @Test
+    public void testValidPrivateData() throws IOException {
+        String dataString = "qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1rfla5UBoEcoxbtjgZ9G7FLPb8V/Qfr0bfKWfvMmN06U/pgUdLuv2mGL2V0D3qYd1011MUzGdNG1qERjaCDz8GAi63+KnHHjfMtPgYt6bcqjs4CNV+ZZ4dIt3xxHYyVEBNc=";
+
+        // Write the data a single file in a temp path
+        Path path = Files.createTempDirectory("testValidPrivateData");
+        Path filePath = Paths.get(path.toString(), "test");
+        filePath.toFile().deleteOnExit();
+
+        BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile()));
+        writer.write(dataString);
+        writer.close();
+
+        Service service = Service.FILE_PRIVATE;
+        assertTrue(service.isValidationRequired());
+
+        assertEquals(ValidationResult.OK, service.validate(filePath));
+    }
+
+    @Test
+    public void testEncryptedData() throws IOException {
+        String dataString = "qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1rfla5UBoEcoxbtjgZ9G7FLPb8V/Qfr0bfKWfvMmN06U/pgUdLuv2mGL2V0D3qYd1011MUzGdNG1qERjaCDz8GAi63+KnHHjfMtPgYt6bcqjs4CNV+ZZ4dIt3xxHYyVEBNc=";
+
+        // Write the data a single file in a temp path
+        Path path = Files.createTempDirectory("testValidPrivateData");
+        Path filePath = Paths.get(path.toString(), "test");
+        filePath.toFile().deleteOnExit();
+
+        BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile()));
+        writer.write(dataString);
+        writer.close();
+
+        // Validate a private service
+        Service service = Service.FILE_PRIVATE;
+        assertTrue(service.isValidationRequired());
+        assertEquals(ValidationResult.OK, service.validate(filePath));
+
+        // Validate a regular service
+        service = Service.FILE;
+        assertTrue(service.isValidationRequired());
+        assertEquals(ValidationResult.DATA_ENCRYPTED, service.validate(filePath));
+    }
+
+    @Test
+    public void testPlainTextData() throws IOException {
+        String dataString = "plaintext";
+
+        // Write the data a single file in a temp path
+        Path path = Files.createTempDirectory("testInvalidPrivateData");
+        Path filePath = Paths.get(path.toString(), "test");
+        filePath.toFile().deleteOnExit();
+
+        BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile()));
+        writer.write(dataString);
+        writer.close();
+
+        // Validate a private service
+        Service service = Service.FILE_PRIVATE;
+        assertTrue(service.isValidationRequired());
+        assertEquals(ValidationResult.DATA_NOT_ENCRYPTED, service.validate(filePath));
+
+        // Validate a regular service
+        service = Service.FILE;
+        assertTrue(service.isValidationRequired());
+        assertEquals(ValidationResult.OK, service.validate(filePath));
+    }
+
+    @Test
+    public void testGetPrivateServices() {
+        List<Service> privateServices = Service.privateServices();
+        for (Service service : privateServices) {
+            assertTrue(service.isPrivate());
+        }
+    }
+
+    @Test
+    public void testGetPublicServices() {
+        List<Service> publicServices = Service.publicServices();
+        for (Service service : publicServices) {
+            assertFalse(service.isPrivate());
+        }
+    }
+
 }

src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java

@@ -248,6 +248,47 @@ public class ArbitraryTransactionMetadataTests extends Common {
         }
     }
 
+    @Test
+    public void testUTF8Metadata() throws DataException, IOException, MissingDataException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
+            String publicKey58 = Base58.encode(alice.getPublicKey());
+            String name = "TEST"; // Can be anything for this test
+            String identifier = null; // Not used for this test
+            Service service = Service.ARBITRARY_DATA;
+            int chunkSize = 100;
+            int dataLength = 900; // Actual data length will be longer due to encryption
+
+            // Example (modified) strings from real world content
+            String title = "Доля юаня в трансграничных Доля юаня в трансграничных";
+            String description = "Когда рыночек порешал";
+            List<String> tags = Arrays.asList("Доля", "юаня", "трансграничных");
+            Category category = Category.OTHER;
+
+            // Register the name to Alice
+            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+            TransactionUtils.signAndMint(repository, transactionData, alice);
+
+            // Create PUT transaction
+            Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength);
+            ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name,
+                    identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize, 0L, true,
+                    title, description, tags, category);
+
+            // Check the chunk count is correct
+            assertEquals(10, arbitraryDataFile.chunkCount());
+
+            // Check the metadata is correct
+            String expectedTrimmedTitle = "Доля юаня в трансграничных Доля юаня в тран";
+            assertEquals(expectedTrimmedTitle, arbitraryDataFile.getMetadata().getTitle());
+            assertEquals(description, arbitraryDataFile.getMetadata().getDescription());
+            assertEquals(tags, arbitraryDataFile.getMetadata().getTags());
+            assertEquals(category, arbitraryDataFile.getMetadata().getCategory());
+            assertEquals("text/plain", arbitraryDataFile.getMetadata().getMimeType());
+        }
+    }
+
     @Test
     public void testMetadataLengths() throws DataException, IOException, MissingDataException {
         try (final Repository repository = RepositoryManager.getRepository()) {

tools/tx.pl

@@ -1,16 +1,23 @@
 #!/usr/bin/env perl
 
+# v4.0.2
+
 use JSON;
 use warnings;
 use strict;
 
 use Getopt::Std;
 use File::Basename;
+use Digest::SHA qw( sha256 sha256_hex );
+use Crypt::RIPEMD160;
 
 our %opt;
 getopts('dpst', \%opt);
 
 my $proc = basename($0);
+my $dirname = dirname($0);
+my $OPENSSL_SIGN = "${dirname}/openssl-sign.sh";
+my $OPENSSL_PRIV_TO_PUB = index(`$ENV{SHELL} -i -c 'openssl version;exit' 2>/dev/null`, 'OpenSSL 3.') != -1;
 
 if (@ARGV < 1) {
 	print STDERR "usage: $proc [-d] [-p] [-s] [-t] <tx-type> <privkey> <values> [<key-value pairs>]\n";
@@ -24,7 +31,15 @@ if (@ARGV < 1) {
 	exit 2;
 }
 
-our $BASE_URL = $ENV{BASE_URL} || $opt{t} ? 'http://localhost:62391' : 'http://localhost:12391';
+our @b58 = qw{
+      1 2 3 4 5 6 7 8 9
+    A B C D E F G H   J K L M N   P Q R S T U V W X Y Z
+    a b c d e f g h i j k   m n o p q r s t u v w x y z
+};
+our %b58 = map { $b58[$_] => $_ } 0 .. 57;
+our %reverseb58 = reverse %b58;
+
+our $BASE_URL = $ENV{BASE_URL} || ($opt{t} ? 'http://localhost:62391' : 'http://localhost:12391');
 our $DEFAULT_FEE = 0.001;
 
 our %TRANSACTION_TYPES = (
@@ -42,6 +57,7 @@ our %TRANSACTION_TYPES = (
 	create_group => {
 		url => 'groups/create',
 		required => [qw(groupName description isOpen approvalThreshold)],
+		defaults => { minimumBlockDelay => 10, maximumBlockDelay => 30 },
 		key_name => 'creatorPublicKey',
 	},
 	update_group => {
@@ -75,10 +91,10 @@ our %TRANSACTION_TYPES = (
 		key_name => 'ownerPublicKey',
 	},
 	remove_group_admin => {
-        url => 'groups/removeadmin',
+		url => 'groups/removeadmin',
-        required => [qw(groupId txGroupId admin)],
+		required => [qw(groupId txGroupId member)],
-        key_name => 'ownerPublicKey',
+		key_name => 'ownerPublicKey',
-    },
+	},
 	group_approval => {
 		url => 'groups/approval',
 		required => [qw(pendingSignature approval)],
@@ -113,7 +129,7 @@ our %TRANSACTION_TYPES = (
 	},
 	update_name => {
 		url => 'names/update',
-		required => [qw(newName newData)],
+		required => [qw(name newName newData)],
 		key_name => 'ownerPublicKey',
 	},
 	# reward-shares
@@ -144,13 +160,21 @@ our %TRANSACTION_TYPES = (
 		key_name => 'senderPublicKey',
 		pow_url => 'addresses/publicize/compute',
 	},
-	# Cross-chain trading
+	# AT
-	build_trade => {
+	deploy_at => {
-		url => 'crosschain/build',
+		url => 'at',
-		required => [qw(initialQortAmount finalQortAmount fundingQortAmount secretHash bitcoinAmount)],
+		required => [qw(name description aTType tags creationBytes amount)],
-		optional => [qw(tradeTimeout)],
+		optional => [qw(assetId)],
 		key_name => 'creatorPublicKey',
-		defaults => { tradeTimeout => 10800 },
+		defaults => { assetId => 0 },
+	},
+	# Cross-chain trading
+	create_trade => {
+		url => 'crosschain/tradebot/create',
+		required => [qw(qortAmount fundingQortAmount foreignAmount receivingAddress)],
+		optional => [qw(tradeTimeout foreignBlockchain)],
+		key_name => 'creatorPublicKey',
+		defaults => { tradeTimeout => 1440, foreignBlockchain => 'LITECOIN' },
 	},
 	trade_recipient => {
 		url => 'crosschain/tradeoffer/recipient',
@@ -196,7 +220,7 @@ if (@ARGV < @required + 1) {
 
 my $priv_key = shift @ARGV;
 
-my $account = account($priv_key);
+my $account;
 my $raw;
 
 if ($tx_type ne 'sign') {
@@ -215,6 +239,8 @@ if ($tx_type ne 'sign') {
 
 	%extras = (%extras, @ARGV);
 
+	$account = account($priv_key, %extras);
+
 	$raw = build_raw($tx_type, $account, %extras);
 	printf "Raw: %s\n", $raw if $opt{d} || (!$opt{s} && !$opt{p});
 
@@ -229,7 +255,7 @@ if ($tx_type ne 'sign') {
 }
 
 if ($opt{s}) {
-	my $signed = sign($account->{private}, $raw);
+	my $signed = sign($priv_key, $raw);
 	printf "Signed: %s\n", $signed if $opt{d} || $tx_type eq 'sign';
 
 	if ($opt{p}) {
@@ -246,15 +272,25 @@ if ($opt{s}) {
 }
 
 sub account {
-	my ($creator) = @_;
+	my ($privkey, %extras) = @_;
 
-	my $account = { private => $creator };
+	my $account = { private => $privkey };
-	$account->{public} = api('utils/publickey', $creator);
+	$account->{public} = $extras{publickey} || priv_to_pub($privkey);
-	$account->{address} = api('addresses/convert/{publickey}', '', '{publickey}', $account->{public});
+	$account->{address} = $extras{address} || pubkey_to_address($account->{public}); # api('addresses/convert/{publickey}', '', '{publickey}', $account->{public});
 
 	return $account;
 }
 
+sub priv_to_pub {
+	my ($privkey) = @_;
+
+	if ($OPENSSL_PRIV_TO_PUB) {
+		return openssl_priv_to_pub($privkey);
+	} else {
+		return api('utils/publickey', $privkey);
+	}
+}
+
 sub build_raw {
 	my ($type, $account, %extras) = @_;
 
@@ -306,6 +342,21 @@ sub build_raw {
 sub sign {
 	my ($private, $raw) = @_;
 
+	if (-x "$OPENSSL_SIGN") {
+		my $private_hex = decode_base58($private);
+		chomp $private_hex;
+
+		my $raw_hex = decode_base58($raw);
+		chomp $raw_hex;
+
+		my $sig = `${OPENSSL_SIGN} ${private_hex} ${raw_hex}`;
+		chomp $sig;
+
+		my $sig58 = encode_base58(${raw_hex} . ${sig});
+		chomp $sig58;
+		return $sig58;
+	}
+
 	my $json = <<"	__JSON__";
 	{
 		"privateKey": "$private",
@@ -344,7 +395,14 @@ sub api {
 	my $curl = "curl --silent --output - --url '$BASE_URL/$url'";
 	if (defined $postdata && $postdata ne '') {
 		$postdata =~ tr|\n| |s;
-		$curl .= " --header 'Content-Type: application/json' --data-binary '$postdata'";
+
+		if ($postdata =~ /^\s*\{/so) {
+			$curl .= " --header 'Content-Type: application/json'";
+		} else {
+			$curl .= " --header 'Content-Type: text/plain'";
+		}
+
+		$curl .= " --data-binary '$postdata'";
 		$method = 'POST';
 	}
 	my $response = `$curl 2>/dev/null`; 
@@ -356,3 +414,87 @@ sub api {
 
 	return $response;
 }
+
+sub encode_base58 {
+    use integer;
+    my @in = map { hex($_) } ($_[0] =~ /(..)/g);
+    my $bzeros = length($1) if join('', @in) =~ /^(0*)/;
+    my @out;
+    my $size = 2 * scalar @in;
+    for my $c (@in) {
+        for (my $j = $size; $j--; ) {
+            $c += 256 * ($out[$j] // 0);
+            $out[$j] = $c % 58;
+            $c /= 58;
+        }
+    }
+    my $out = join('', map { $reverseb58{$_} } @out);
+    return $1 if $out =~ /(1{$bzeros}[^1].*)/;
+    return $1 if $out =~ /(1{$bzeros})/;
+    die "Invalid base58!\n";
+}
+
+
+sub decode_base58 {
+    use integer;
+    my @out;
+    my $azeros = length($1) if $_[0] =~ /^(1*)/;
+    for my $c ( map { $b58{$_} } $_[0] =~ /./g ) {
+	die("Invalid character!\n") unless defined $c;
+        for (my $j = length($_[0]); $j--; ) {
+            $c += 58 * ($out[$j] // 0);
+            $out[$j] = $c % 256;
+            $c /= 256;
+        }
+    }
+    shift @out while @out && $out[0] == 0;
+    unshift(@out, (0) x $azeros);
+    return sprintf('%02x' x @out, @out);
+}
+
+sub openssl_priv_to_pub {
+	my ($privkey) = @_;
+
+	my $privkey_hex = decode_base58($privkey);
+
+	my $key_type = "04"; # hex
+	my $length = "20"; # hex
+
+	my $asn1 = <<"__ASN1__";
+asn1=SEQUENCE:private_key
+
+[private_key]
+version=INTEGER:0
+included=SEQUENCE:key_info
+raw=FORMAT:HEX,OCTETSTRING:${key_type}${length}${privkey_hex}
+
+[key_info]
+type=OBJECT:ED25519
+
+__ASN1__
+
+	my $output = `echo "${asn1}" | openssl asn1parse -i -genconf - -out - | openssl pkey -in - -inform der -noout -text_pub`;
+
+	# remove colons
+	my $pubkey = '';
+	$pubkey .= $1 while $output =~ m/([0-9a-f]{2})(?::|$)/g;
+
+	return encode_base58($pubkey);
+}
+
+sub pubkey_to_address {
+	my ($pubkey) = @_;
+
+	my $pubkey_hex = decode_base58($pubkey);
+	my $pubkey_raw = pack('H*', $pubkey_hex);
+
+	my $pkh_hex = Crypt::RIPEMD160->hexhash(sha256($pubkey_raw));
+	$pkh_hex =~ tr/ //ds;
+
+	my $version = '3a'; # hex
+
+	my $raw = pack('H*', $version . $pkh_hex);
+	my $chksum = substr(sha256_hex(sha256($raw)), 0, 8);
+
+	return encode_base58($version . $pkh_hex . $chksum);
+}