diff --git a/Q-Apps.md b/Q-Apps.md
index 0f52c086..177fee2d 100644
--- a/Q-Apps.md
+++ b/Q-Apps.md
@@ -42,10 +42,15 @@ A "default" resource refers to one without an identifier. For example, when a we
Here is a list of currently available services that can be used in Q-Apps:
+### Public services ###
+The services below are intended to be used for publicly accessible data.
+
IMAGE,
THUMBNAIL,
VIDEO,
AUDIO,
+PODCAST,
+VOICE,
ARBITRARY_DATA,
JSON,
DOCUMENT,
@@ -55,7 +60,25 @@ METADATA,
BLOG,
BLOG_POST,
BLOG_COMMENT,
-GIF_REPOSITORY
+GIF_REPOSITORY,
+ATTACHMENT,
+FILE,
+FILES,
+CHAIN_DATA,
+STORE,
+PRODUCT,
+OFFER,
+COUPON,
+CODE,
+PLUGIN,
+EXTENSION,
+GAME,
+ITEM,
+NFT,
+DATABASE,
+SNAPSHOT,
+COMMENT,
+CHAIN_COMMENT,
WEBSITE,
APP,
QCHAT_ATTACHMENT,
@@ -63,6 +86,20 @@ QCHAT_IMAGE,
QCHAT_AUDIO,
QCHAT_VOICE
+### Private services ###
+For the services below, data is encrypted for a single recipient, and can only be decrypted using the private key of the recipient's wallet.
+
+QCHAT_ATTACHMENT_PRIVATE,
+ATTACHMENT_PRIVATE,
+FILE_PRIVATE,
+IMAGE_PRIVATE,
+VIDEO_PRIVATE,
+AUDIO_PRIVATE,
+VOICE_PRIVATE,
+DOCUMENT_PRIVATE,
+MAIL_PRIVATE,
+MESSAGE_PRIVATE
+
## Single vs multi-file resources
@@ -220,9 +257,14 @@ Here is a list of currently supported actions:
- SEARCH_QDN_RESOURCES
- GET_QDN_RESOURCE_STATUS
- GET_QDN_RESOURCE_PROPERTIES
+- GET_QDN_RESOURCE_METADATA
+- GET_QDN_RESOURCE_URL
+- LINK_TO_QDN_RESOURCE
- FETCH_QDN_RESOURCE
- PUBLISH_QDN_RESOURCE
- PUBLISH_MULTIPLE_QDN_RESOURCES
+- DECRYPT_DATA
+- SAVE_FILE
- GET_WALLET_BALANCE
- GET_BALANCE
- SEND_COIN
@@ -238,8 +280,6 @@ Here is a list of currently supported actions:
- FETCH_BLOCK_RANGE
- SEARCH_TRANSACTIONS
- GET_PRICE
-- GET_QDN_RESOURCE_URL
-- LINK_TO_QDN_RESOURCE
- GET_LIST_ITEMS
- ADD_LIST_ITEMS
- DELETE_LIST_ITEM
@@ -385,7 +425,8 @@ let res = await qortalRequest({
action: "GET_QDN_RESOURCE_STATUS",
name: "QortalDemo",
service: "THUMBNAIL",
- identifier: "qortal_avatar" // Optional
+ identifier: "qortal_avatar", // Optional
+ build: true // Optional - request that the resource is fetched & built in the background
});
```
@@ -400,11 +441,21 @@ let res = await qortalRequest({
// Returns: filename, size, mimeType (where available)
```
+### Get QDN resource metadata
+```
+let res = await qortalRequest({
+ action: "GET_QDN_RESOURCE_METADATA",
+ name: "QortalDemo",
+ service: "THUMBNAIL",
+ identifier: "qortal_avatar" // Optional
+});
+```
+
### Publish a single file to QDN
_Requires user approval_.
Note: this publishes a single, base64-encoded file. Multi-file resource publishing (such as a WEBSITE or GIF_REPOSITORY) is not yet supported via a Q-App. It will be added in a future update.
```
-await qortalRequest({
+let res = await qortalRequest({
action: "PUBLISH_QDN_RESOURCE",
name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
service: "IMAGE",
@@ -418,7 +469,9 @@ await qortalRequest({
// tag2: "strings", // Optional
// tag3: "can", // Optional
// tag4: "go", // Optional
- // tag5: "here" // Optional
+ // tag5: "here", // Optional
+ // encrypt: true, // Optional - to be used with a private service
+ // recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
});
```
@@ -426,7 +479,7 @@ await qortalRequest({
_Requires user approval_.
Note: each resource being published consists of a single, base64-encoded file, each in its own transaction. Useful for publishing two or more related things, such as a video and a video thumbnail.
```
-await qortalRequest({
+let res = await qortalRequest({
action: "PUBLISH_MULTIPLE_QDN_RESOURCES",
resources: [
name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
@@ -441,7 +494,9 @@ await qortalRequest({
// tag2: "strings", // Optional
// tag3: "can", // Optional
// tag4: "go", // Optional
- // tag5: "here" // Optional
+ // tag5: "here", // Optional
+ // encrypt: true, // Optional - to be used with a private service
+ // recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
],
[
... more resources here if needed ...
@@ -449,10 +504,32 @@ await qortalRequest({
});
```
+### Decrypt encrypted/private data
+```
+let res = await qortalRequest({
+ action: "DECRYPT_DATA",
+ encryptedData: 'qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1r',
+ publicKey: 'publickeygoeshere'
+});
+// Returns base64 encoded string of plaintext data
+```
+
+### Prompt user to save a file to disk
+Note: mimeType not required but recommended. If not specified, saving will fail if the mimeType is unable to be derived from the Blob.
+```
+let res = await qortalRequest({
+ action: "SAVE_FILE",
+ blob: dataBlob,
+ filename: "myfile.pdf",
+ mimeType: "application/pdf" // Optional but recommended
+});
+```
+
+
### Get wallet balance (QORT)
_Requires user approval_
```
-await qortalRequest({
+let res = await qortalRequest({
action: "GET_WALLET_BALANCE",
coin: "QORT"
});
@@ -477,7 +554,7 @@ let res = await qortalRequest({
### Send QORT to address
_Requires user approval_
```
-await qortalRequest({
+let res = await qortalRequest({
action: "SEND_COIN",
coin: "QORT",
destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
@@ -488,7 +565,7 @@ await qortalRequest({
### Send foreign coin to address
_Requires user approval_
```
-await qortalRequest({
+let res = await qortalRequest({
action: "SEND_COIN",
coin: "LTC",
destinationAddress: "LSdTvMHRm8sScqwCi6x9wzYQae8JeZhx6y",
@@ -508,6 +585,7 @@ let res = await qortalRequest({
// reference: "reference", // Optional
// chatReference: "chatreference", // Optional
// hasChatReference: true, // Optional
+ encoding: "BASE64", // Optional (defaults to BASE58 if omitted)
limit: 100,
offset: 0,
reverse: true
@@ -517,7 +595,7 @@ let res = await qortalRequest({
### Send a group chat message
_Requires user approval_
```
-await qortalRequest({
+let res = await qortalRequest({
action: "SEND_CHAT_MESSAGE",
groupId: 0,
message: "Test"
@@ -527,7 +605,7 @@ await qortalRequest({
### Send a private chat message
_Requires user approval_
```
-await qortalRequest({
+let res = await qortalRequest({
action: "SEND_CHAT_MESSAGE",
destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
message: "Test"
@@ -547,7 +625,7 @@ let res = await qortalRequest({
### Join a group
_Requires user approval_
```
-await qortalRequest({
+let res = await qortalRequest({
action: "JOIN_GROUP",
groupId: 100
});
@@ -739,6 +817,9 @@ let res = await qortalRequest({
# Section 4: Examples
+Some example projects can be found [here](https://github.com/Qortal/Q-Apps). These can be cloned and modified, or used as a reference when creating a new app.
+
+
## Sample App
Here is a sample application to display the logged-in user's avatar:
diff --git a/pom.xml b/pom.xml
index 70366ada..0dfa0cf4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -3,7 +3,7 @@
4.0.0
org.qortal
qortal
- 3.9.1
+ 4.0.3
jar
true
diff --git a/src/main/java/org/qortal/api/HTMLParser.java b/src/main/java/org/qortal/api/HTMLParser.java
index eac813a9..cc3102e8 100644
--- a/src/main/java/org/qortal/api/HTMLParser.java
+++ b/src/main/java/org/qortal/api/HTMLParser.java
@@ -13,7 +13,8 @@ public class HTMLParser {
private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
- private String linkPrefix;
+ private String qdnBase;
+ private String qdnBaseWithPath;
private byte[] data;
private String qdnContext;
private String resourceId;
@@ -21,10 +22,13 @@ public class HTMLParser {
private String identifier;
private String path;
private String theme;
+ private boolean usingCustomRouting;
public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix, byte[] data,
- String qdnContext, Service service, String identifier, String theme) {
- this.linkPrefix = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
+ String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting) {
+ String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : "";
+ this.qdnBase = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
+ this.qdnBaseWithPath = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
this.data = data;
this.qdnContext = qdnContext;
this.resourceId = resourceId;
@@ -32,12 +36,12 @@ public class HTMLParser {
this.identifier = identifier;
this.path = inPath;
this.theme = theme;
+ this.usingCustomRouting = usingCustomRouting;
}
public void addAdditionalHeaderTags() {
String fileContents = new String(data);
Document document = Jsoup.parse(fileContents);
- String baseUrl = this.linkPrefix;
Elements head = document.getElementsByTag("head");
if (!head.isEmpty()) {
// Add q-apps script tag
@@ -51,16 +55,21 @@ public class HTMLParser {
}
// Escape and add vars
- String service = this.service.toString().replace("\"","\\\"");
- String name = this.resourceId != null ? this.resourceId.replace("\"","\\\"") : "";
- String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"") : "";
- String path = this.path != null ? this.path.replace("\"","\\\"") : "";
- String theme = this.theme != null ? this.theme.replace("\"","\\\"") : "";
- String qdnContextVar = String.format("", this.qdnContext, theme, service, name, identifier, path, baseUrl);
+ String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\\", "").replace("\"","\\\"") : "";
+ String service = this.service.toString().replace("\\", "").replace("\"","\\\"");
+ String name = this.resourceId != null ? this.resourceId.replace("\\", "").replace("\"","\\\"") : "";
+ String identifier = this.identifier != null ? this.identifier.replace("\\", "").replace("\"","\\\"") : "";
+ String path = this.path != null ? this.path.replace("\\", "").replace("\"","\\\"") : "";
+ String theme = this.theme != null ? this.theme.replace("\\", "").replace("\"","\\\"") : "";
+ String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\\", "").replace("\"","\\\"") : "";
+ String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\\", "").replace("\"","\\\"") : "";
+ String qdnContextVar = String.format("", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath);
head.get(0).prepend(qdnContextVar);
// Add base href tag
- String baseElement = String.format("", baseUrl);
+ // Exclude the path if this request was routed back to the index automatically
+ String baseHref = this.usingCustomRouting ? this.qdnBase : this.qdnBaseWithPath;
+ String baseElement = String.format("", baseHref);
head.get(0).prepend(baseElement);
// Add meta charset tag
diff --git a/src/main/java/org/qortal/api/model/PollVotes.java b/src/main/java/org/qortal/api/model/PollVotes.java
new file mode 100644
index 00000000..c57ebc37
--- /dev/null
+++ b/src/main/java/org/qortal/api/model/PollVotes.java
@@ -0,0 +1,56 @@
+package org.qortal.api.model;
+
+import java.util.List;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+
+import org.qortal.data.voting.VoteOnPollData;
+
+@Schema(description = "Poll vote info, including voters")
+// All properties to be converted to JSON via JAX-RS
+@XmlAccessorType(XmlAccessType.FIELD)
+public class PollVotes {
+
+ @Schema(description = "List of individual votes")
+ @XmlElement(name = "votes")
+ public List votes;
+
+ @Schema(description = "Total number of votes")
+ public Integer totalVotes;
+
+ @Schema(description = "List of vote counts for each option")
+ public List voteCounts;
+
+ // For JAX-RS
+ protected PollVotes() {
+ }
+
+ public PollVotes(List votes, Integer totalVotes, List voteCounts) {
+ this.votes = votes;
+ this.totalVotes = totalVotes;
+ this.voteCounts = voteCounts;
+ }
+
+ @Schema(description = "Vote info")
+ // All properties to be converted to JSON via JAX-RS
+ @XmlAccessorType(XmlAccessType.FIELD)
+ public static class OptionCount {
+ @Schema(description = "Option name")
+ public String optionName;
+
+ @Schema(description = "Vote count")
+ public Integer voteCount;
+
+ // For JAX-RS
+ protected OptionCount() {
+ }
+
+ public OptionCount(String optionName, Integer voteCount) {
+ this.optionName = optionName;
+ this.voteCount = voteCount;
+ }
+ }
+}
diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
index 3d1a6a2e..c617b517 100644
--- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java
+++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java
@@ -65,10 +65,7 @@ import org.qortal.transaction.Transaction.ValidationResult;
import org.qortal.transform.TransformationException;
import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
import org.qortal.transform.transaction.TransactionTransformer;
-import org.qortal.utils.ArbitraryTransactionUtils;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-import org.qortal.utils.ZipUtils;
+import org.qortal.utils.*;
@Path("/arbitrary")
@Tag(name = "Arbitrary")
@@ -721,12 +718,9 @@ public class ArbitraryResource {
}
)
@SecurityRequirement(name = "apiKey")
- public ArbitraryResourceMetadata getMetadata(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
- @PathParam("service") Service service,
- @PathParam("name") String name,
- @PathParam("identifier") String identifier) {
- Security.checkApiCallAllowed(request);
-
+ public ArbitraryResourceMetadata getMetadata(@PathParam("service") Service service,
+ @PathParam("name") String name,
+ @PathParam("identifier") String identifier) {
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
try {
@@ -1179,7 +1173,11 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, error);
}
- final Long minLatestBlockTimestamp = NTP.getTime() - (60 * 60 * 1000L);
+ final Long now = NTP.getTime();
+ if (now == null) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NO_TIME_SYNC);
+ }
+ final Long minLatestBlockTimestamp = now - (60 * 60 * 1000L);
if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp)) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC);
}
@@ -1237,7 +1235,7 @@ public class ArbitraryResource {
// The actual data will be in a randomly-named subfolder of tempDirectory
// Remove hidden folders, i.e. starting with "_", as some systems can add them, e.g. "__MACOSX"
String[] files = tempDirectory.toFile().list((parent, child) -> !child.startsWith("_"));
- if (files.length == 1) { // Single directory or file only
+ if (files != null && files.length == 1) { // Single directory or file only
path = Paths.get(tempDirectory.toString(), files[0]).toString();
}
}
@@ -1269,7 +1267,8 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage());
}
- } catch (DataException | IOException e) {
+ } catch (Exception e) {
+ LOGGER.info("Exception when publishing data: ", e);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
}
}
@@ -1317,7 +1316,7 @@ public class ArbitraryResource {
if (filepath == null || filepath.isEmpty()) {
// No file path supplied - so check if this is a single file resource
String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal");
- if (files.length == 1) {
+ if (files != null && files.length == 1) {
// This is a single file resource
filepath = files[0];
}
@@ -1327,20 +1326,50 @@ public class ArbitraryResource {
}
}
- // TODO: limit file size that can be read into memory
java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
if (!Files.exists(path)) {
String message = String.format("No file exists at filepath: %s", filepath);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message);
}
- byte[] data = Files.readAllBytes(path);
+ byte[] data;
+ int fileSize = (int)path.toFile().length();
+ int length = fileSize;
+
+ // Parse "Range" header
+ Integer rangeStart = null;
+ Integer rangeEnd = null;
+ String range = request.getHeader("Range");
+ if (range != null) {
+ range = range.replace("bytes=", "");
+ String[] parts = range.split("-");
+ rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null;
+ rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize;
+ }
+
+ if (rangeStart != null && rangeEnd != null) {
+ // We have a range, so update the requested length
+ length = rangeEnd - rangeStart;
+ }
+
+ if (length < fileSize && encoding == null) {
+ // Partial content requested, and not encoding the data
+ response.setStatus(206);
+ response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize));
+ data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length);
+ }
+ else {
+ // Full content requested (or encoded data)
+ response.setStatus(200);
+ data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory
+ }
// Encode the data if requested
if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) {
data = Base64.encode(data);
}
+ response.addHeader("Accept-Ranges", "bytes");
response.setContentType(context.getMimeType(path.toString()));
response.setContentLength(data.length);
response.getOutputStream().write(data);
diff --git a/src/main/java/org/qortal/api/resource/ChatResource.java b/src/main/java/org/qortal/api/resource/ChatResource.java
index 986bb03d..22e90a43 100644
--- a/src/main/java/org/qortal/api/resource/ChatResource.java
+++ b/src/main/java/org/qortal/api/resource/ChatResource.java
@@ -119,6 +119,75 @@ public class ChatResource {
}
}
+ @GET
+ @Path("/messages/count")
+ @Operation(
+ summary = "Count chat messages",
+ description = "Returns count of CHAT messages that match criteria. Must provide EITHER 'txGroupId' OR two 'involving' addresses.",
+ responses = {
+ @ApiResponse(
+ description = "count of messages",
+ content = @Content(
+ mediaType = MediaType.TEXT_PLAIN,
+ schema = @Schema(
+ type = "integer"
+ )
+ )
+ )
+ }
+ )
+ @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE})
+ public int countChatMessages(@QueryParam("before") Long before, @QueryParam("after") Long after,
+ @QueryParam("txGroupId") Integer txGroupId,
+ @QueryParam("involving") List involvingAddresses,
+ @QueryParam("reference") String reference,
+ @QueryParam("chatreference") String chatReference,
+ @QueryParam("haschatreference") Boolean hasChatReference,
+ @QueryParam("sender") String sender,
+ @QueryParam("encoding") Encoding encoding,
+ @Parameter(ref = "limit") @QueryParam("limit") Integer limit,
+ @Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+ @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) {
+ // Check args meet expectations
+ if ((txGroupId == null && involvingAddresses.size() != 2)
+ || (txGroupId != null && !involvingAddresses.isEmpty()))
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+ // Check any provided addresses are valid
+ if (involvingAddresses.stream().anyMatch(address -> !Crypto.isValidAddress(address)))
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
+
+ if (before != null && before < 1500000000000L)
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+ if (after != null && after < 1500000000000L)
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+ byte[] referenceBytes = null;
+ if (reference != null)
+ referenceBytes = Base58.decode(reference);
+
+ byte[] chatReferenceBytes = null;
+ if (chatReference != null)
+ chatReferenceBytes = Base58.decode(chatReference);
+
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ return repository.getChatRepository().getMessagesMatchingCriteria(
+ before,
+ after,
+ txGroupId,
+ referenceBytes,
+ chatReferenceBytes,
+ hasChatReference,
+ involvingAddresses,
+ sender,
+ encoding,
+ limit, offset, reverse).size();
+ } catch (DataException e) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+ }
+ }
+
@GET
@Path("/message/{signature}")
@Operation(
diff --git a/src/main/java/org/qortal/api/resource/NamesResource.java b/src/main/java/org/qortal/api/resource/NamesResource.java
index a900d6bf..03dffc08 100644
--- a/src/main/java/org/qortal/api/resource/NamesResource.java
+++ b/src/main/java/org/qortal/api/resource/NamesResource.java
@@ -155,6 +155,38 @@ public class NamesResource {
}
}
+ @GET
+ @Path("/search")
+ @Operation(
+ summary = "Search registered names",
+ responses = {
+ @ApiResponse(
+ description = "registered name info",
+ content = @Content(
+ mediaType = MediaType.APPLICATION_JSON,
+ array = @ArraySchema(schema = @Schema(implementation = NameData.class))
+ )
+ )
+ }
+ )
+ @ApiErrors({ApiError.NAME_UNKNOWN, ApiError.REPOSITORY_ISSUE})
+ public List searchNames(@QueryParam("query") String query,
+ @Parameter(ref = "limit") @QueryParam("limit") Integer limit,
+ @Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+ @Parameter(ref="reverse") @QueryParam("reverse") Boolean reverse) {
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ if (query == null) {
+ throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing query");
+ }
+
+ return repository.getNameRepository().searchNames(query, limit, offset, reverse);
+ } catch (ApiException e) {
+ throw e;
+ } catch (DataException e) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+ }
+ }
+
@POST
@Path("/register")
diff --git a/src/main/java/org/qortal/api/resource/PollsResource.java b/src/main/java/org/qortal/api/resource/PollsResource.java
index 952cbdc5..c64a8caf 100644
--- a/src/main/java/org/qortal/api/resource/PollsResource.java
+++ b/src/main/java/org/qortal/api/resource/PollsResource.java
@@ -31,12 +31,18 @@ import javax.ws.rs.core.MediaType;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.ArraySchema;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
import javax.ws.rs.GET;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import org.qortal.api.ApiException;
+import org.qortal.api.model.PollVotes;
import org.qortal.data.voting.PollData;
+import org.qortal.data.voting.PollOptionData;
+import org.qortal.data.voting.VoteOnPollData;
@Path("/polls")
@Tag(name = "Polls")
@@ -102,6 +108,61 @@ public class PollsResource {
}
}
+ @GET
+ @Path("/votes/{pollName}")
+ @Operation(
+ summary = "Votes on poll",
+ responses = {
+ @ApiResponse(
+ description = "poll votes",
+ content = @Content(
+ mediaType = MediaType.APPLICATION_JSON,
+ schema = @Schema(implementation = PollVotes.class)
+ )
+ )
+ }
+ )
+ @ApiErrors({ApiError.REPOSITORY_ISSUE})
+ public PollVotes getPollVotes(@PathParam("pollName") String pollName, @QueryParam("onlyCounts") Boolean onlyCounts) {
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ PollData pollData = repository.getVotingRepository().fromPollName(pollName);
+ if (pollData == null)
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.POLL_NO_EXISTS);
+
+ List votes = repository.getVotingRepository().getVotes(pollName);
+
+ // Initialize map for counting votes
+ Map voteCountMap = new HashMap<>();
+ for (PollOptionData optionData : pollData.getPollOptions()) {
+ voteCountMap.put(optionData.getOptionName(), 0);
+ }
+
+ int totalVotes = 0;
+ for (VoteOnPollData vote : votes) {
+ String selectedOption = pollData.getPollOptions().get(vote.getOptionIndex()).getOptionName();
+ if (voteCountMap.containsKey(selectedOption)) {
+ voteCountMap.put(selectedOption, voteCountMap.get(selectedOption) + 1);
+ totalVotes++;
+ }
+ }
+
+ // Convert map to list of VoteInfo
+ List voteCounts = voteCountMap.entrySet().stream()
+ .map(entry -> new PollVotes.OptionCount(entry.getKey(), entry.getValue()))
+ .collect(Collectors.toList());
+
+ if (onlyCounts != null && onlyCounts) {
+ return new PollVotes(null, totalVotes, voteCounts);
+ } else {
+ return new PollVotes(votes, totalVotes, voteCounts);
+ }
+ } catch (ApiException e) {
+ throw e;
+ } catch (DataException e) {
+ throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+ }
+ }
+
@POST
@Path("/create")
@Operation(
diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
index b6b17ea5..fba6a32b 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
@@ -54,10 +54,6 @@ public class ArbitraryDataBuilder {
/**
* Process transactions, but do not build anything
* This is useful for checking the status of a given resource
- *
- * @throws DataException
- * @throws IOException
- * @throws MissingDataException
*/
public void process() throws DataException, IOException, MissingDataException {
this.fetchTransactions();
@@ -69,10 +65,6 @@ public class ArbitraryDataBuilder {
/**
* Build the latest state of a given resource
- *
- * @throws DataException
- * @throws IOException
- * @throws MissingDataException
*/
public void build() throws DataException, IOException, MissingDataException {
this.process();
diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
index 779e4024..b9e62e56 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
@@ -9,7 +9,6 @@ import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
-import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.crypto.AES;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
@@ -35,6 +34,9 @@ import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
public class ArbitraryDataReader {
@@ -60,6 +62,10 @@ public class ArbitraryDataReader {
// The resource being read
ArbitraryDataResource arbitraryDataResource = null;
+ // Track resources that are currently being loaded, to avoid duplicate concurrent builds
+ // TODO: all builds could be handled by the build queue (even synchronous ones), to avoid the need for this
+ private static Map inProgress = Collections.synchronizedMap(new HashMap<>());
+
public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
// Ensure names are always lowercase
if (resourceIdType == ResourceIdType.NAME) {
@@ -154,9 +160,6 @@ public class ArbitraryDataReader {
* If no exception is thrown, you can then use getFilePath() to access the data immediately after returning
*
* @param overwrite - set to true to force rebuild an existing cache
- * @throws IOException
- * @throws DataException
- * @throws MissingDataException
*/
public void loadSynchronously(boolean overwrite) throws DataException, IOException, MissingDataException {
try {
@@ -170,6 +173,12 @@ public class ArbitraryDataReader {
this.arbitraryDataResource = this.createArbitraryDataResource();
+ // Don't allow duplicate loads
+ if (!this.canStartLoading()) {
+ LOGGER.debug("Skipping duplicate load of {}", this.arbitraryDataResource);
+ return;
+ }
+
this.preExecute();
this.deleteExistingFiles();
this.fetch();
@@ -197,6 +206,7 @@ public class ArbitraryDataReader {
private void preExecute() throws DataException {
ArbitraryDataBuildManager.getInstance().setBuildInProgress(true);
+
this.checkEnabled();
this.createWorkingDirectory();
this.createUncompressedDirectory();
@@ -204,6 +214,9 @@ public class ArbitraryDataReader {
private void postExecute() {
ArbitraryDataBuildManager.getInstance().setBuildInProgress(false);
+
+ this.arbitraryDataResource = this.createArbitraryDataResource();
+ ArbitraryDataReader.inProgress.remove(this.arbitraryDataResource.getUniqueKey());
}
private void checkEnabled() throws DataException {
@@ -212,6 +225,17 @@ public class ArbitraryDataReader {
}
}
+ private boolean canStartLoading() {
+ // Avoid duplicate builds if we're already loading this resource
+ String uniqueKey = this.arbitraryDataResource.getUniqueKey();
+ if (ArbitraryDataReader.inProgress.containsKey(uniqueKey)) {
+ return false;
+ }
+ ArbitraryDataReader.inProgress.put(uniqueKey, NTP.getTime());
+
+ return true;
+ }
+
private void createWorkingDirectory() throws DataException {
try {
Files.createDirectories(this.workingPath);
@@ -223,7 +247,6 @@ public class ArbitraryDataReader {
/**
* Working directory should only be deleted on failure, since it is currently used to
* serve a cached version of the resource for subsequent requests.
- * @throws IOException
*/
private void deleteWorkingDirectory() {
try {
@@ -303,7 +326,7 @@ public class ArbitraryDataReader {
break;
default:
- throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType.toString()));
+ throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType));
}
}
@@ -368,6 +391,9 @@ public class ArbitraryDataReader {
// Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
+ if (arbitraryDataFile == null) {
+ throw new DataException(String.format("arbitraryDataFile is null"));
+ }
if (!arbitraryDataFile.allFilesExist()) {
if (ListUtils.isNameBlocked(transactionData.getName())) {
@@ -443,6 +469,7 @@ public class ArbitraryDataReader {
Path unencryptedPath = Paths.get(this.workingPath.toString(), "zipped.zip");
SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, "AES");
AES.decryptFile(algorithm, aesKey, this.filePath.toString(), unencryptedPath.toString());
+ LOGGER.debug("Finished decrypting {} using algorithm {}", this.arbitraryDataResource, algorithm);
// Replace filePath pointer with the encrypted file path
// Don't delete the original ArbitraryDataFile, as this is handled in the cleanup phase
@@ -477,7 +504,9 @@ public class ArbitraryDataReader {
// Handle each type of compression
if (compression == Compression.ZIP) {
+ LOGGER.debug("Unzipping {}...", this.arbitraryDataResource);
ZipUtils.unzip(this.filePath.toString(), this.uncompressedPath.getParent().toString());
+ LOGGER.debug("Finished unzipping {}", this.arbitraryDataResource);
}
else if (compression == Compression.NONE) {
Files.createDirectories(this.uncompressedPath);
@@ -513,10 +542,12 @@ public class ArbitraryDataReader {
private void validate() throws IOException, DataException {
if (this.service.isValidationRequired()) {
+ LOGGER.debug("Validating {}...", this.arbitraryDataResource);
Service.ValidationResult result = this.service.validate(this.filePath);
if (result != Service.ValidationResult.OK) {
throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
}
+ LOGGER.debug("Finished validating {}", this.arbitraryDataResource);
}
}
diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
index 66fc7b98..089a99ca 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
@@ -67,8 +67,8 @@ public class ArbitraryDataRenderer {
}
public HttpServletResponse render() {
- if (!inPath.startsWith(File.separator)) {
- inPath = File.separator + inPath;
+ if (!inPath.startsWith("/")) {
+ inPath = "/" + inPath;
}
// Don't render data if QDN is disabled
@@ -126,7 +126,8 @@ public class ArbitraryDataRenderer {
try {
String filename = this.getFilename(unzippedPath, inPath);
Path filePath = Paths.get(unzippedPath, filename);
-
+ boolean usingCustomRouting = false;
+
// If the file doesn't exist, we may need to route the request elsewhere, or cleanup
if (!Files.exists(filePath)) {
if (inPath.equals("/")) {
@@ -148,6 +149,7 @@ public class ArbitraryDataRenderer {
// Forward request to index file
filePath = indexPath;
filename = indexFile;
+ usingCustomRouting = true;
break;
}
}
@@ -157,7 +159,7 @@ public class ArbitraryDataRenderer {
if (HTMLParser.isHtmlFile(filename)) {
// HTML file - needs to be parsed
byte[] data = Files.readAllBytes(filePath); // TODO: limit file size that can be read into memory
- HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data, qdnContext, service, identifier, theme);
+ HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data, qdnContext, service, identifier, theme, usingCustomRouting);
htmlParser.addAdditionalHeaderTags();
response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:;");
response.setContentType(context.getMimeType(filename));
diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java
index 79bb882b..a4650dfc 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java
@@ -150,6 +150,9 @@ public class ArbitraryDataResource {
for (ArbitraryTransactionData transactionData : transactionDataList) {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
+ if (arbitraryDataFile == null) {
+ continue;
+ }
// Delete any chunks or complete files from each transaction
arbitraryDataFile.deleteAll(deleteMetadata);
diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java
index 07f6032c..498f3296 100644
--- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java
+++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java
@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -50,7 +51,7 @@ public class ArbitraryDataMetadata {
this.readJson();
} catch (JSONException e) {
- throw new DataException(String.format("Unable to read JSON: %s", e.getMessage()));
+ throw new DataException(String.format("Unable to read JSON at path %s: %s", this.filePath, e.getMessage()));
}
}
@@ -64,6 +65,10 @@ public class ArbitraryDataMetadata {
writer.close();
}
+ public void delete() throws IOException {
+ Files.delete(this.filePath);
+ }
+
protected void loadJson() throws IOException {
File metadataFile = new File(this.filePath.toString());
@@ -71,7 +76,7 @@ public class ArbitraryDataMetadata {
throw new IOException(String.format("Metadata file doesn't exist: %s", this.filePath.toString()));
}
- this.jsonString = new String(Files.readAllBytes(this.filePath));
+ this.jsonString = new String(Files.readAllBytes(this.filePath), StandardCharsets.UTF_8);
}
diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java
index df23655c..eb3d6cc9 100644
--- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java
+++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java
@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -69,7 +70,7 @@ public class ArbitraryDataQortalMetadata extends ArbitraryDataMetadata {
throw new IOException(String.format("Patch file doesn't exist: %s", path.toString()));
}
- this.jsonString = new String(Files.readAllBytes(path));
+ this.jsonString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
}
diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java
index 004e0ed3..d9dba037 100644
--- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java
+++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java
@@ -7,6 +7,7 @@ import org.qortal.arbitrary.misc.Category;
import org.qortal.repository.DataException;
import org.qortal.utils.Base58;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
@@ -217,6 +218,25 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
// Static helper methods
+ public static String trimUTF8String(String string, int maxLength) {
+ byte[] inputBytes = string.getBytes(StandardCharsets.UTF_8);
+ int length = Math.min(inputBytes.length, maxLength);
+ byte[] outputBytes = new byte[length];
+
+ System.arraycopy(inputBytes, 0, outputBytes, 0, length);
+ String result = new String(outputBytes, StandardCharsets.UTF_8);
+
+ // check if last character is truncated
+ int lastIndex = result.length() - 1;
+
+ if (lastIndex > 0 && result.charAt(lastIndex) != string.charAt(lastIndex)) {
+ // last character is truncated so remove the last character
+ return result.substring(0, lastIndex);
+ }
+
+ return result;
+ }
+
public static String limitTitle(String title) {
if (title == null) {
return null;
@@ -225,7 +245,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
return null;
}
- return title.substring(0, Math.min(title.length(), MAX_TITLE_LENGTH));
+ return trimUTF8String(title, MAX_TITLE_LENGTH);
}
public static String limitDescription(String description) {
@@ -236,7 +256,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
return null;
}
- return description.substring(0, Math.min(description.length(), MAX_DESCRIPTION_LENGTH));
+ return trimUTF8String(description, MAX_DESCRIPTION_LENGTH);
}
public static List limitTags(List tags) {
diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java
index fa47f020..94ca9252 100644
--- a/src/main/java/org/qortal/arbitrary/misc/Service.java
+++ b/src/main/java/org/qortal/arbitrary/misc/Service.java
@@ -9,7 +9,6 @@ import org.qortal.utils.FilesystemUtils;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
@@ -20,9 +19,9 @@ import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap;
public enum Service {
- AUTO_UPDATE(1, false, null, false, null),
- ARBITRARY_DATA(100, false, null, false, null),
- QCHAT_ATTACHMENT(120, true, 1024*1024L, true, null) {
+ AUTO_UPDATE(1, false, null, false, false, null),
+ ARBITRARY_DATA(100, false, null, false, false, null),
+ QCHAT_ATTACHMENT(120, true, 1024*1024L, true, false, null) {
@Override
public ValidationResult validate(Path path) throws IOException {
ValidationResult superclassResult = super.validate(path);
@@ -47,7 +46,14 @@ public enum Service {
return ValidationResult.OK;
}
},
- WEBSITE(200, true, null, false, null) {
+ QCHAT_ATTACHMENT_PRIVATE(121, true, 1024*1024L, true, true, null),
+ ATTACHMENT(130, false, 50*1024*1024L, true, false, null),
+ ATTACHMENT_PRIVATE(131, true, 50*1024*1024L, true, true, null),
+ FILE(140, false, null, true, false, null),
+ FILE_PRIVATE(141, true, null, true, true, null),
+ FILES(150, false, null, false, false, null),
+ CHAIN_DATA(160, true, 239L, true, false, null),
+ WEBSITE(200, true, null, false, false, null) {
@Override
public ValidationResult validate(Path path) throws IOException {
ValidationResult superclassResult = super.validate(path);
@@ -69,23 +75,30 @@ public enum Service {
return ValidationResult.MISSING_INDEX_FILE;
}
},
- GIT_REPOSITORY(300, false, null, false, null),
- IMAGE(400, true, 10*1024*1024L, true, null),
- THUMBNAIL(410, true, 500*1024L, true, null),
- QCHAT_IMAGE(420, true, 500*1024L, true, null),
- VIDEO(500, false, null, true, null),
- AUDIO(600, false, null, true, null),
- QCHAT_AUDIO(610, true, 10*1024*1024L, true, null),
- QCHAT_VOICE(620, true, 10*1024*1024L, true, null),
- BLOG(700, false, null, false, null),
- BLOG_POST(777, false, null, true, null),
- BLOG_COMMENT(778, false, null, true, null),
- DOCUMENT(800, false, null, true, null),
- LIST(900, true, null, true, null),
- PLAYLIST(910, true, null, true, null),
- APP(1000, true, 50*1024*1024L, false, null),
- METADATA(1100, false, null, true, null),
- JSON(1110, true, 25*1024L, true, null) {
+ GIT_REPOSITORY(300, false, null, false, false, null),
+ IMAGE(400, true, 10*1024*1024L, true, false, null),
+ IMAGE_PRIVATE(401, true, 10*1024*1024L, true, true, null),
+ THUMBNAIL(410, true, 500*1024L, true, false, null),
+ QCHAT_IMAGE(420, true, 500*1024L, true, false, null),
+ VIDEO(500, false, null, true, false, null),
+ VIDEO_PRIVATE(501, true, null, true, true, null),
+ AUDIO(600, false, null, true, false, null),
+ AUDIO_PRIVATE(601, true, null, true, true, null),
+ QCHAT_AUDIO(610, true, 10*1024*1024L, true, false, null),
+ QCHAT_VOICE(620, true, 10*1024*1024L, true, false, null),
+ VOICE(630, true, 10*1024*1024L, true, false, null),
+ VOICE_PRIVATE(631, true, 10*1024*1024L, true, true, null),
+ PODCAST(640, false, null, true, false, null),
+ BLOG(700, false, null, false, false, null),
+ BLOG_POST(777, false, null, true, false, null),
+ BLOG_COMMENT(778, true, 500*1024L, true, false, null),
+ DOCUMENT(800, false, null, true, false, null),
+ DOCUMENT_PRIVATE(801, true, null, true, true, null),
+ LIST(900, true, null, true, false, null),
+ PLAYLIST(910, true, null, true, false, null),
+ APP(1000, true, 50*1024*1024L, false, false, null),
+ METADATA(1100, false, null, true, false, null),
+ JSON(1110, true, 25*1024L, true, false, null) {
@Override
public ValidationResult validate(Path path) throws IOException {
ValidationResult superclassResult = super.validate(path);
@@ -94,7 +107,7 @@ public enum Service {
}
// Require valid JSON
- byte[] data = FilesystemUtils.getSingleFileContents(path);
+ byte[] data = FilesystemUtils.getSingleFileContents(path, 25*1024);
String json = new String(data, StandardCharsets.UTF_8);
try {
objectMapper.readTree(json);
@@ -104,7 +117,7 @@ public enum Service {
}
}
},
- GIF_REPOSITORY(1200, true, 25*1024*1024L, false, null) {
+ GIF_REPOSITORY(1200, true, 25*1024*1024L, false, false, null) {
@Override
public ValidationResult validate(Path path) throws IOException {
ValidationResult superclassResult = super.validate(path);
@@ -139,12 +152,31 @@ public enum Service {
}
return ValidationResult.OK;
}
- };
+ },
+ STORE(1300, false, null, true, false, null),
+ PRODUCT(1310, false, null, true, false, null),
+ OFFER(1330, false, null, true, false, null),
+ COUPON(1340, false, null, true, false, null),
+ CODE(1400, false, null, true, false, null),
+ PLUGIN(1410, false, null, true, false, null),
+ EXTENSION(1420, false, null, true, false, null),
+ GAME(1500, false, null, false, false, null),
+ ITEM(1510, false, null, true, false, null),
+ NFT(1600, false, null, true, false, null),
+ DATABASE(1700, false, null, false, false, null),
+ SNAPSHOT(1710, false, null, false, false, null),
+ COMMENT(1800, true, 500*1024L, true, false, null),
+ CHAIN_COMMENT(1810, true, 239L, true, false, null),
+ MAIL(1900, true, 1024*1024L, true, false, null),
+ MAIL_PRIVATE(1901, true, 1024*1024L, true, true, null),
+ MESSAGE(1910, true, 1024*1024L, true, false, null),
+ MESSAGE_PRIVATE(1911, true, 1024*1024L, true, true, null);
public final int value;
private final boolean requiresValidation;
private final Long maxSize;
private final boolean single;
+ private final boolean isPrivate;
private final List requiredKeys;
private static final Map map = stream(Service.values())
@@ -153,11 +185,14 @@ public enum Service {
// For JSON validation
private static final ObjectMapper objectMapper = new ObjectMapper();
- Service(int value, boolean requiresValidation, Long maxSize, boolean single, List requiredKeys) {
+ private static final String encryptedDataPrefix = "qortalEncryptedData";
+
+ Service(int value, boolean requiresValidation, Long maxSize, boolean single, boolean isPrivate, List requiredKeys) {
this.value = value;
this.requiresValidation = requiresValidation;
this.maxSize = maxSize;
this.single = single;
+ this.isPrivate = isPrivate;
this.requiredKeys = requiredKeys;
}
@@ -166,7 +201,9 @@ public enum Service {
return ValidationResult.OK;
}
- byte[] data = FilesystemUtils.getSingleFileContents(path);
+ // Load the first 25KB of data. This only needs to be long enough to check the prefix
+ // and also to allow for possible additional future validation of smaller files.
+ byte[] data = FilesystemUtils.getSingleFileContents(path, 25*1024);
long size = FilesystemUtils.getDirectorySize(path);
// Validate max size if needed
@@ -181,6 +218,17 @@ public enum Service {
return ValidationResult.INVALID_FILE_COUNT;
}
+ // Validate private data for single file resources
+ if (this.single) {
+ String dataString = new String(data, StandardCharsets.UTF_8);
+ if (this.isPrivate && !dataString.startsWith(encryptedDataPrefix)) {
+ return ValidationResult.DATA_NOT_ENCRYPTED;
+ }
+ if (!this.isPrivate && dataString.startsWith(encryptedDataPrefix)) {
+ return ValidationResult.DATA_ENCRYPTED;
+ }
+ }
+
// Validate required keys if needed
if (this.requiredKeys != null) {
if (data == null) {
@@ -199,7 +247,12 @@ public enum Service {
}
public boolean isValidationRequired() {
- return this.requiresValidation;
+ // We must always validate single file resources, to ensure they are actually a single file
+ return this.requiresValidation || this.single;
+ }
+
+ public boolean isPrivate() {
+ return this.isPrivate;
}
public static Service valueOf(int value) {
@@ -207,10 +260,41 @@ public enum Service {
}
public static JSONObject toJsonObject(byte[] data) {
- String dataString = new String(data);
+ String dataString = new String(data, StandardCharsets.UTF_8);
return new JSONObject(dataString);
}
+ public static List publicServices() {
+ List privateServices = new ArrayList<>();
+ for (Service service : Service.values()) {
+ if (!service.isPrivate) {
+ privateServices.add(service);
+ }
+ }
+ return privateServices;
+ }
+
+ /**
+ * Fetch a list of Service objects that require encrypted data.
+ *
+ * These can ultimately be used to help inform the cleanup manager
+ * on the best order to delete files when the node runs out of space.
+ * Public data should be given priority over private data (unless
+ * this node is part of a data market contract for that data - this
+ * isn't developed yet).
+ *
+ * @return a list of Service objects that require encrypted data.
+ */
+ public static List privateServices() {
+ List privateServices = new ArrayList<>();
+ for (Service service : Service.values()) {
+ if (service.isPrivate) {
+ privateServices.add(service);
+ }
+ }
+ return privateServices;
+ }
+
public enum ValidationResult {
OK(1),
MISSING_KEYS(2),
@@ -220,7 +304,9 @@ public enum Service {
INVALID_FILE_EXTENSION(6),
MISSING_DATA(7),
INVALID_FILE_COUNT(8),
- INVALID_CONTENT(9);
+ INVALID_CONTENT(9),
+ DATA_NOT_ENCRYPTED(10),
+ DATA_ENCRYPTED(10);
public final int value;
diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java
index fd2c38df..224228b8 100644
--- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java
+++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java
@@ -504,110 +504,118 @@ public class OnlineAccountsManager {
computeOurAccountsForTimestamp(onlineAccountsTimestamp);
}
- private boolean computeOurAccountsForTimestamp(long onlineAccountsTimestamp) {
- List mintingAccounts;
- try (final Repository repository = RepositoryManager.getRepository()) {
- mintingAccounts = repository.getAccountRepository().getMintingAccounts();
+ private boolean computeOurAccountsForTimestamp(Long onlineAccountsTimestamp) {
+ if (onlineAccountsTimestamp != null) {
+ List mintingAccounts;
+ try (final Repository repository = RepositoryManager.getRepository()) {
+ mintingAccounts = repository.getAccountRepository().getMintingAccounts();
- // We have no accounts to send
- if (mintingAccounts.isEmpty())
+ // We have no accounts to send
+ if (mintingAccounts.isEmpty())
+ return false;
+
+ // Only active reward-shares allowed
+ Iterator iterator = mintingAccounts.iterator();
+ int i = 0;
+ while (iterator.hasNext()) {
+ MintingAccountData mintingAccountData = iterator.next();
+
+ RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
+ if (rewardShareData == null) {
+ // Reward-share doesn't even exist - probably not a good sign
+ iterator.remove();
+ continue;
+ }
+
+ Account mintingAccount = new Account(repository, rewardShareData.getMinter());
+ if (!mintingAccount.canMint()) {
+ // Minting-account component of reward-share can no longer mint - disregard
+ iterator.remove();
+ continue;
+ }
+
+ if (++i > 1 + 1) {
+ iterator.remove();
+ continue;
+ }
+ }
+ } catch (DataException e) {
+ LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
return false;
+ }
- // Only active reward-shares allowed
- Iterator iterator = mintingAccounts.iterator();
- while (iterator.hasNext()) {
- MintingAccountData mintingAccountData = iterator.next();
+ byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
+ List ourOnlineAccounts = new ArrayList<>();
- RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
- if (rewardShareData == null) {
- // Reward-share doesn't even exist - probably not a good sign
- iterator.remove();
+ int remaining = mintingAccounts.size();
+ for (MintingAccountData mintingAccountData : mintingAccounts) {
+ remaining--;
+ byte[] privateKey = mintingAccountData.getPrivateKey();
+ byte[] publicKey = Crypto.toPublicKey(privateKey);
+
+ // We don't want to compute the online account nonce and signature again if it already exists
+ Set onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountsTimestamp, k -> ConcurrentHashMap.newKeySet());
+ boolean alreadyExists = onlineAccounts.stream().anyMatch(a -> Arrays.equals(a.getPublicKey(), publicKey));
+ if (alreadyExists) {
+ this.hasOurOnlineAccounts = true;
+
+ if (remaining > 0) {
+ // Move on to next account
+ continue;
+ } else {
+ // Everything exists, so return true
+ return true;
+ }
+ }
+
+ // Generate bytes for mempow
+ byte[] mempowBytes;
+ try {
+ mempowBytes = this.getMemoryPoWBytes(publicKey, onlineAccountsTimestamp);
+ } catch (IOException e) {
+ LOGGER.info("Unable to create bytes for MemoryPoW. Moving on to next account...");
continue;
}
- Account mintingAccount = new Account(repository, rewardShareData.getMinter());
- if (!mintingAccount.canMint()) {
- // Minting-account component of reward-share can no longer mint - disregard
- iterator.remove();
- continue;
- }
- }
- } catch (DataException e) {
- LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
- return false;
- }
-
- byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
- List ourOnlineAccounts = new ArrayList<>();
-
- int remaining = mintingAccounts.size();
- for (MintingAccountData mintingAccountData : mintingAccounts) {
- remaining--;
- byte[] privateKey = mintingAccountData.getPrivateKey();
- byte[] publicKey = Crypto.toPublicKey(privateKey);
-
- // We don't want to compute the online account nonce and signature again if it already exists
- Set onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountsTimestamp, k -> ConcurrentHashMap.newKeySet());
- boolean alreadyExists = onlineAccounts.stream().anyMatch(a -> Arrays.equals(a.getPublicKey(), publicKey));
- if (alreadyExists) {
- this.hasOurOnlineAccounts = true;
-
- if (remaining > 0) {
- // Move on to next account
- continue;
- }
- else {
- // Everything exists, so return true
- return true;
- }
- }
-
- // Generate bytes for mempow
- byte[] mempowBytes;
- try {
- mempowBytes = this.getMemoryPoWBytes(publicKey, onlineAccountsTimestamp);
- }
- catch (IOException e) {
- LOGGER.info("Unable to create bytes for MemoryPoW. Moving on to next account...");
- continue;
- }
-
- // Compute nonce
- Integer nonce;
- try {
- nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp);
- if (nonce == null) {
- // A nonce is required
+ // Compute nonce
+ Integer nonce;
+ try {
+ nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp);
+ if (nonce == null) {
+ // A nonce is required
+ return false;
+ }
+ } catch (TimeoutException e) {
+ LOGGER.info(String.format("Timed out computing nonce for account %.8s", Base58.encode(publicKey)));
return false;
}
- } catch (TimeoutException e) {
- LOGGER.info(String.format("Timed out computing nonce for account %.8s", Base58.encode(publicKey)));
+
+ byte[] signature = Qortal25519Extras.signForAggregation(privateKey, timestampBytes);
+
+ // Our account is online
+ OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce);
+
+ // Make sure to verify before adding
+ if (verifyMemoryPoW(ourOnlineAccountData, null)) {
+ ourOnlineAccounts.add(ourOnlineAccountData);
+ }
+ }
+
+ this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty();
+
+ boolean hasInfoChanged = addAccounts(ourOnlineAccounts);
+
+ if (!hasInfoChanged)
return false;
- }
- byte[] signature = Qortal25519Extras.signForAggregation(privateKey, timestampBytes);
+ Network.getInstance().broadcast(peer -> new OnlineAccountsV3Message(ourOnlineAccounts));
- // Our account is online
- OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce);
+ LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp);
- // Make sure to verify before adding
- if (verifyMemoryPoW(ourOnlineAccountData, null)) {
- ourOnlineAccounts.add(ourOnlineAccountData);
- }
+ return true;
}
- this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty();
-
- boolean hasInfoChanged = addAccounts(ourOnlineAccounts);
-
- if (!hasInfoChanged)
- return false;
-
- Network.getInstance().broadcast(peer -> new OnlineAccountsV3Message(ourOnlineAccounts));
-
- LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp);
-
- return true;
+ return false;
}
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
index 9d57ce8a..e0c62acb 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
@@ -346,6 +346,10 @@ public class ArbitraryDataCleanupManager extends Thread {
/**
* Iteratively walk through given directory and delete a single random file
*
+ * TODO: public data should be prioritized over private data
+ * (unless this node is part of a data market contract for that data).
+ * See: Service.privateServices() for a list of services containing private data.
+ *
* @param directory - the base directory
* @return boolean - whether a file was deleted
*/
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
index 2fd6033e..5ed8df21 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
@@ -124,29 +124,29 @@ public class ArbitraryDataFileListManager {
}
}
- // Then allow another 3 attempts, each 5 minutes apart
- if (timeSinceLastAttempt > 5 * 60 * 1000L) {
- // We haven't tried for at least 5 minutes
+ // Then allow another 5 attempts, each 1 minute apart
+ if (timeSinceLastAttempt > 60 * 1000L) {
+ // We haven't tried for at least 1 minute
- if (networkBroadcastCount < 6) {
- // We've made less than 6 total attempts
+ if (networkBroadcastCount < 8) {
+ // We've made less than 8 total attempts
return true;
}
}
- // Then allow another 4 attempts, each 30 minutes apart
- if (timeSinceLastAttempt > 30 * 60 * 1000L) {
- // We haven't tried for at least 5 minutes
+ // Then allow another 8 attempts, each 15 minutes apart
+ if (timeSinceLastAttempt > 15 * 60 * 1000L) {
+ // We haven't tried for at least 15 minutes
- if (networkBroadcastCount < 10) {
- // We've made less than 10 total attempts
+ if (networkBroadcastCount < 16) {
+ // We've made less than 16 total attempts
return true;
}
}
- // From then on, only try once every 24 hours, to reduce network spam
- if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) {
- // We haven't tried for at least 24 hours
+ // From then on, only try once every 6 hours, to reduce network spam
+ if (timeSinceLastAttempt > 6 * 60 * 60 * 1000L) {
+ // We haven't tried for at least 6 hours
return true;
}
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
index 567dcdd3..9284e672 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
@@ -16,7 +16,6 @@ import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
-import org.qortal.list.ResourceListManager;
import org.qortal.network.Network;
import org.qortal.network.Peer;
import org.qortal.repository.DataException;
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
index 97d659ad..663bc22a 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
@@ -102,7 +102,14 @@ public class ArbitraryMetadataManager {
if (metadataFile.exists()) {
// Use local copy
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
- transactionMetadata.read();
+ try {
+ transactionMetadata.read();
+ } catch (DataException e) {
+ // Invalid file, so delete it
+ LOGGER.info("Deleting invalid metadata file due to exception: {}", e.getMessage());
+ transactionMetadata.delete();
+ return null;
+ }
return transactionMetadata;
}
}
diff --git a/src/main/java/org/qortal/data/transaction/TransactionData.java b/src/main/java/org/qortal/data/transaction/TransactionData.java
index 838cffd3..4bf3152c 100644
--- a/src/main/java/org/qortal/data/transaction/TransactionData.java
+++ b/src/main/java/org/qortal/data/transaction/TransactionData.java
@@ -13,6 +13,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.eclipse.persistence.oxm.annotations.XmlDiscriminatorNode;
import org.qortal.crypto.Crypto;
import org.qortal.data.voting.PollData;
+import org.qortal.data.voting.VoteOnPollData;
import org.qortal.transaction.Transaction.ApprovalStatus;
import org.qortal.transaction.Transaction.TransactionType;
@@ -30,7 +31,7 @@ import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
@XmlSeeAlso({GenesisTransactionData.class, PaymentTransactionData.class, RegisterNameTransactionData.class, UpdateNameTransactionData.class,
SellNameTransactionData.class, CancelSellNameTransactionData.class, BuyNameTransactionData.class,
CreatePollTransactionData.class, VoteOnPollTransactionData.class, ArbitraryTransactionData.class,
- PollData.class,
+ PollData.class, VoteOnPollData.class,
IssueAssetTransactionData.class, TransferAssetTransactionData.class,
CreateAssetOrderTransactionData.class, CancelAssetOrderTransactionData.class,
MultiPaymentTransactionData.class, DeployAtTransactionData.class, MessageTransactionData.class, ATTransactionData.class,
diff --git a/src/main/java/org/qortal/data/voting/VoteOnPollData.java b/src/main/java/org/qortal/data/voting/VoteOnPollData.java
index 47c06a54..531ed286 100644
--- a/src/main/java/org/qortal/data/voting/VoteOnPollData.java
+++ b/src/main/java/org/qortal/data/voting/VoteOnPollData.java
@@ -9,6 +9,11 @@ public class VoteOnPollData {
// Constructors
+ // For JAXB
+ protected VoteOnPollData() {
+ super();
+ }
+
public VoteOnPollData(String pollName, byte[] voterPublicKey, int optionIndex) {
this.pollName = pollName;
this.voterPublicKey = voterPublicKey;
@@ -21,12 +26,24 @@ public class VoteOnPollData {
return this.pollName;
}
+ public void setPollName(String pollName) {
+ this.pollName = pollName;
+ }
+
public byte[] getVoterPublicKey() {
return this.voterPublicKey;
}
+ public void setVoterPublicKey(byte[] voterPublicKey) {
+ this.voterPublicKey = voterPublicKey;
+ }
+
public int getOptionIndex() {
return this.optionIndex;
}
+ public void setOptionIndex(int optionIndex) {
+ this.optionIndex = optionIndex;
+ }
+
}
diff --git a/src/main/java/org/qortal/list/ResourceList.java b/src/main/java/org/qortal/list/ResourceList.java
index 5c12e0f5..855c9068 100644
--- a/src/main/java/org/qortal/list/ResourceList.java
+++ b/src/main/java/org/qortal/list/ResourceList.java
@@ -9,6 +9,7 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -81,7 +82,7 @@ public class ResourceList {
}
try {
- String jsonString = new String(Files.readAllBytes(path));
+ String jsonString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
this.list = ResourceList.listFromJSONString(jsonString);
} catch (IOException e) {
throw new IOException(String.format("Couldn't read contents from file %s", path.toString()));
diff --git a/src/main/java/org/qortal/repository/NameRepository.java b/src/main/java/org/qortal/repository/NameRepository.java
index d6c0f33e..a8b2a3db 100644
--- a/src/main/java/org/qortal/repository/NameRepository.java
+++ b/src/main/java/org/qortal/repository/NameRepository.java
@@ -14,6 +14,8 @@ public interface NameRepository {
public boolean reducedNameExists(String reducedName) throws DataException;
+ public List searchNames(String query, Integer limit, Integer offset, Boolean reverse) throws DataException;
+
public List getAllNames(Integer limit, Integer offset, Boolean reverse) throws DataException;
public default List getAllNames() throws DataException {
diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java
index 3a3574ef..3e4a8e11 100644
--- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java
+++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java
@@ -103,6 +103,57 @@ public class HSQLDBNameRepository implements NameRepository {
}
}
+ public List searchNames(String query, Integer limit, Integer offset, Boolean reverse) throws DataException {
+ StringBuilder sql = new StringBuilder(512);
+ List