crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Merge branch 'Qortal:master' into upgrade-tls

Browse Source
This commit is contained in:
QuickMythril 2023-05-09 15:38:20 -04:00 committed by GitHub
commit 1e10bcf3b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1083 additions and 227 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
Q-Apps.md
View File

@ -42,10 +42,15 @@ A "default" resource refers to one without an identifier. For example, when a we
Here is a list of currently available services that can be used in Q-Apps:
### Public services ###
The services below are intended to be used for publicly accessible data.
IMAGE,
THUMBNAIL,
VIDEO,
AUDIO,
PODCAST,
VOICE,
ARBITRARY_DATA,
JSON,
DOCUMENT,
@ -55,7 +60,25 @@ METADATA,
BLOG,
BLOG_POST,
BLOG_COMMENT,
GIF_REPOSITORY
GIF_REPOSITORY,
ATTACHMENT,
FILE,
FILES,
CHAIN_DATA,
STORE,
PRODUCT,
OFFER,
COUPON,
CODE,
PLUGIN,
EXTENSION,
GAME,
ITEM,
NFT,
DATABASE,
SNAPSHOT,
COMMENT,
CHAIN_COMMENT,
WEBSITE,
APP,
QCHAT_ATTACHMENT,
@ -63,6 +86,20 @@ QCHAT_IMAGE,
QCHAT_AUDIO,
QCHAT_VOICE
### Private services ###
For the services below, data is encrypted for a single recipient, and can only be decrypted using the private key of the recipient's wallet.
QCHAT_ATTACHMENT_PRIVATE,
ATTACHMENT_PRIVATE,
FILE_PRIVATE,
IMAGE_PRIVATE,
VIDEO_PRIVATE,
AUDIO_PRIVATE,
VOICE_PRIVATE,
DOCUMENT_PRIVATE,
MAIL_PRIVATE,
MESSAGE_PRIVATE
## Single vs multi-file resources
@ -220,9 +257,14 @@ Here is a list of currently supported actions:
- SEARCH_QDN_RESOURCES
- GET_QDN_RESOURCE_STATUS
- GET_QDN_RESOURCE_PROPERTIES
- GET_QDN_RESOURCE_METADATA
- GET_QDN_RESOURCE_URL
- LINK_TO_QDN_RESOURCE
- FETCH_QDN_RESOURCE
- PUBLISH_QDN_RESOURCE
- PUBLISH_MULTIPLE_QDN_RESOURCES
- DECRYPT_DATA
- SAVE_FILE
- GET_WALLET_BALANCE
- GET_BALANCE
- SEND_COIN
@ -238,8 +280,6 @@ Here is a list of currently supported actions:
- FETCH_BLOCK_RANGE
- SEARCH_TRANSACTIONS
- GET_PRICE
- GET_QDN_RESOURCE_URL
- LINK_TO_QDN_RESOURCE
- GET_LIST_ITEMS
- ADD_LIST_ITEMS
- DELETE_LIST_ITEM
@ -385,7 +425,8 @@ let res = await qortalRequest({
action: "GET_QDN_RESOURCE_STATUS",
name: "QortalDemo",
service: "THUMBNAIL",
identifier: "qortal_avatar" // Optional
identifier: "qortal_avatar", // Optional
build: true // Optional - request that the resource is fetched & built in the background
});
```
@ -400,11 +441,21 @@ let res = await qortalRequest({
// Returns: filename, size, mimeType (where available)
```
### Get QDN resource metadata
```
let res = await qortalRequest({
action: "GET_QDN_RESOURCE_METADATA",
name: "QortalDemo",
service: "THUMBNAIL",
identifier: "qortal_avatar" // Optional
});
```
### Publish a single file to QDN
_Requires user approval_.<br />
Note: this publishes a single, base64-encoded file. Multi-file resource publishing (such as a WEBSITE or GIF_REPOSITORY) is not yet supported via a Q-App. It will be added in a future update.
```
await qortalRequest({
let res = await qortalRequest({
action: "PUBLISH_QDN_RESOURCE",
name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
service: "IMAGE",
@ -418,7 +469,9 @@ await qortalRequest({
// tag2: "strings", // Optional
// tag3: "can", // Optional
// tag4: "go", // Optional
// tag5: "here" // Optional
// tag5: "here", // Optional
// encrypt: true, // Optional - to be used with a private service
// recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
});
```
@ -426,7 +479,7 @@ await qortalRequest({
_Requires user approval_.<br />
Note: each resource being published consists of a single, base64-encoded file, each in its own transaction. Useful for publishing two or more related things, such as a video and a video thumbnail.
```
await qortalRequest({
let res = await qortalRequest({
action: "PUBLISH_MULTIPLE_QDN_RESOURCES",
resources: [
name: "Demo", // Publisher must own the registered name - use GET_ACCOUNT_NAMES for a list
@ -441,7 +494,9 @@ await qortalRequest({
// tag2: "strings", // Optional
// tag3: "can", // Optional
// tag4: "go", // Optional
// tag5: "here" // Optional
// tag5: "here", // Optional
// encrypt: true, // Optional - to be used with a private service
// recipientPublicKey: "publickeygoeshere" // Only required if `encrypt` is set to true
],
[
... more resources here if needed ...
@ -449,10 +504,32 @@ await qortalRequest({
});
```
### Decrypt encrypted/private data
```
let res = await qortalRequest({
action: "DECRYPT_DATA",
encryptedData: 'qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1r',
publicKey: 'publickeygoeshere'
});
// Returns base64 encoded string of plaintext data
```
### Prompt user to save a file to disk
Note: mimeType not required but recommended. If not specified, saving will fail if the mimeType is unable to be derived from the Blob.
```
let res = await qortalRequest({
action: "SAVE_FILE",
blob: dataBlob,
filename: "myfile.pdf",
mimeType: "application/pdf" // Optional but recommended
});
```
### Get wallet balance (QORT)
_Requires user approval_
```
await qortalRequest({
let res = await qortalRequest({
action: "GET_WALLET_BALANCE",
coin: "QORT"
});
@ -477,7 +554,7 @@ let res = await qortalRequest({
### Send QORT to address
_Requires user approval_
```
await qortalRequest({
let res = await qortalRequest({
action: "SEND_COIN",
coin: "QORT",
destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
@ -488,7 +565,7 @@ await qortalRequest({
### Send foreign coin to address
_Requires user approval_
```
await qortalRequest({
let res = await qortalRequest({
action: "SEND_COIN",
coin: "LTC",
destinationAddress: "LSdTvMHRm8sScqwCi6x9wzYQae8JeZhx6y",
@ -508,6 +585,7 @@ let res = await qortalRequest({
// reference: "reference", // Optional
// chatReference: "chatreference", // Optional
// hasChatReference: true, // Optional
encoding: "BASE64", // Optional (defaults to BASE58 if omitted)
limit: 100,
offset: 0,
reverse: true
@ -517,7 +595,7 @@ let res = await qortalRequest({
### Send a group chat message
_Requires user approval_
```
await qortalRequest({
let res = await qortalRequest({
action: "SEND_CHAT_MESSAGE",
groupId: 0,
message: "Test"
@ -527,7 +605,7 @@ await qortalRequest({
### Send a private chat message
_Requires user approval_
```
await qortalRequest({
let res = await qortalRequest({
action: "SEND_CHAT_MESSAGE",
destinationAddress: "QZLJV7wbaFyxaoZQsjm6rb9MWMiDzWsqM2",
message: "Test"
@ -547,7 +625,7 @@ let res = await qortalRequest({
### Join a group
_Requires user approval_
```
await qortalRequest({
let res = await qortalRequest({
action: "JOIN_GROUP",
groupId: 100
});
@ -739,6 +817,9 @@ let res = await qortalRequest({
# Section 4: Examples
Some example projects can be found [here](https://github.com/Qortal/Q-Apps). These can be cloned and modified, or used as a reference when creating a new app.
## Sample App
Here is a sample application to display the logged-in user's avatar:

2
pom.xml
View File

@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.qortal</groupId>
<artifactId>qortal</artifactId>
<version>3.9.1</version>
<version>4.0.3</version>
<packaging>jar</packaging>
<properties>
<skipTests>true</skipTests>

31
src/main/java/org/qortal/api/HTMLParser.java
View File

@ -13,7 +13,8 @@ public class HTMLParser {
private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
private String linkPrefix;
private String qdnBase;
private String qdnBaseWithPath;
private byte[] data;
private String qdnContext;
private String resourceId;
@ -21,10 +22,13 @@ public class HTMLParser {
private String identifier;
private String path;
private String theme;
private boolean usingCustomRouting;
public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix, byte[] data,
String qdnContext, Service service, String identifier, String theme) {
this.linkPrefix = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting) {
String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : "";
this.qdnBase = usePrefix ? String.format("%s/%s", prefix, resourceId) : "";
this.qdnBaseWithPath = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
this.data = data;
this.qdnContext = qdnContext;
this.resourceId = resourceId;
@ -32,12 +36,12 @@ public class HTMLParser {
this.identifier = identifier;
this.path = inPath;
this.theme = theme;
this.usingCustomRouting = usingCustomRouting;
}
public void addAdditionalHeaderTags() {
String fileContents = new String(data);
Document document = Jsoup.parse(fileContents);
String baseUrl = this.linkPrefix;
Elements head = document.getElementsByTag("head");
if (!head.isEmpty()) {
// Add q-apps script tag
@ -51,16 +55,21 @@ public class HTMLParser {
}
// Escape and add vars
String service = this.service.toString().replace("\"","\\\"");
String name = this.resourceId != null ? this.resourceId.replace("\"","\\\"") : "";
String identifier = this.identifier != null ? this.identifier.replace("\"","\\\"") : "";
String path = this.path != null ? this.path.replace("\"","\\\"") : "";
String theme = this.theme != null ? this.theme.replace("\"","\\\"") : "";
String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\";</script>", this.qdnContext, theme, service, name, identifier, path, baseUrl);
String qdnContext = this.qdnContext != null ? this.qdnContext.replace("\\", "").replace("\"","\\\"") : "";
String service = this.service.toString().replace("\\", "").replace("\"","\\\"");
String name = this.resourceId != null ? this.resourceId.replace("\\", "").replace("\"","\\\"") : "";
String identifier = this.identifier != null ? this.identifier.replace("\\", "").replace("\"","\\\"") : "";
String path = this.path != null ? this.path.replace("\\", "").replace("\"","\\\"") : "";
String theme = this.theme != null ? this.theme.replace("\\", "").replace("\"","\\\"") : "";
String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\\", "").replace("\"","\\\"") : "";
String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\\", "").replace("\"","\\\"") : "";
String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath);
head.get(0).prepend(qdnContextVar);
// Add base href tag
String baseElement = String.format("<base href=\"%s/\">", baseUrl);
// Exclude the path if this request was routed back to the index automatically
String baseHref = this.usingCustomRouting ? this.qdnBase : this.qdnBaseWithPath;
String baseElement = String.format("<base href=\"%s/\">", baseHref);
head.get(0).prepend(baseElement);
// Add meta charset tag

56
src/main/java/org/qortal/api/model/PollVotes.java Normal file
View File

@ -0,0 +1,56 @@
package org.qortal.api.model;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.voting.VoteOnPollData;
@Schema(description = "Poll vote info, including voters")
// All properties to be converted to JSON via JAX-RS
@XmlAccessorType(XmlAccessType.FIELD)
public class PollVotes {
@Schema(description = "List of individual votes")
@XmlElement(name = "votes")
public List<VoteOnPollData> votes;
@Schema(description = "Total number of votes")
public Integer totalVotes;
@Schema(description = "List of vote counts for each option")
public List<OptionCount> voteCounts;
// For JAX-RS
protected PollVotes() {
}
public PollVotes(List<VoteOnPollData> votes, Integer totalVotes, List<OptionCount> voteCounts) {
this.votes = votes;
this.totalVotes = totalVotes;
this.voteCounts = voteCounts;
}
@Schema(description = "Vote info")
// All properties to be converted to JSON via JAX-RS
@XmlAccessorType(XmlAccessType.FIELD)
public static class OptionCount {
@Schema(description = "Option name")
public String optionName;
@Schema(description = "Vote count")
public Integer voteCount;
// For JAX-RS
protected OptionCount() {
}
public OptionCount(String optionName, Integer voteCount) {
this.optionName = optionName;
this.voteCount = voteCount;
}
}
}

61
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -65,10 +65,7 @@ import org.qortal.transaction.Transaction.ValidationResult;
import org.qortal.transform.TransformationException;
import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.ArbitraryTransactionUtils;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import org.qortal.utils.ZipUtils;
import org.qortal.utils.*;
@Path("/arbitrary")
@Tag(name = "Arbitrary")
@ -721,12 +718,9 @@ public class ArbitraryResource {
}
)
@SecurityRequirement(name = "apiKey")
public ArbitraryResourceMetadata getMetadata(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier) {
Security.checkApiCallAllowed(request);
public ArbitraryResourceMetadata getMetadata(@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier) {
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
try {
@ -1179,7 +1173,11 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, error);
}
final Long minLatestBlockTimestamp = NTP.getTime() - (60 * 60 * 1000L);
final Long now = NTP.getTime();
if (now == null) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NO_TIME_SYNC);
}
final Long minLatestBlockTimestamp = now - (60 * 60 * 1000L);
if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp)) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC);
}
@ -1237,7 +1235,7 @@ public class ArbitraryResource {
// The actual data will be in a randomly-named subfolder of tempDirectory
// Remove hidden folders, i.e. starting with "_", as some systems can add them, e.g. "__MACOSX"
String[] files = tempDirectory.toFile().list((parent, child) -> !child.startsWith("_"));
if (files.length == 1) { // Single directory or file only
if (files != null && files.length == 1) { // Single directory or file only
path = Paths.get(tempDirectory.toString(), files[0]).toString();
}
}
@ -1269,7 +1267,8 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage());
}
} catch (DataException | IOException e) {
} catch (Exception e) {
LOGGER.info("Exception when publishing data: ", e);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
}
}
@ -1317,7 +1316,7 @@ public class ArbitraryResource {
if (filepath == null || filepath.isEmpty()) {
// No file path supplied - so check if this is a single file resource
String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal");
if (files.length == 1) {
if (files != null && files.length == 1) {
// This is a single file resource
filepath = files[0];
}
@ -1327,20 +1326,50 @@ public class ArbitraryResource {
}
}
// TODO: limit file size that can be read into memory
java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
if (!Files.exists(path)) {
String message = String.format("No file exists at filepath: %s", filepath);
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message);
}
byte[] data = Files.readAllBytes(path);
byte[] data;
int fileSize = (int)path.toFile().length();
int length = fileSize;
// Parse "Range" header
Integer rangeStart = null;
Integer rangeEnd = null;
String range = request.getHeader("Range");
if (range != null) {
range = range.replace("bytes=", "");
String[] parts = range.split("-");
rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null;
rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize;
}
if (rangeStart != null && rangeEnd != null) {
// We have a range, so update the requested length
length = rangeEnd - rangeStart;
}
if (length < fileSize && encoding == null) {
// Partial content requested, and not encoding the data
response.setStatus(206);
response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize));
data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length);
}
else {
// Full content requested (or encoded data)
response.setStatus(200);
data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory
}
// Encode the data if requested
if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) {
data = Base64.encode(data);
}
response.addHeader("Accept-Ranges", "bytes");
response.setContentType(context.getMimeType(path.toString()));
response.setContentLength(data.length);
response.getOutputStream().write(data);

69
src/main/java/org/qortal/api/resource/ChatResource.java
View File

@ -119,6 +119,75 @@ public class ChatResource {
}
}
@GET
@Path("/messages/count")
@Operation(
summary = "Count chat messages",
description = "Returns count of CHAT messages that match criteria. Must provide EITHER 'txGroupId' OR two 'involving' addresses.",
responses = {
@ApiResponse(
description = "count of messages",
content = @Content(
mediaType = MediaType.TEXT_PLAIN,
schema = @Schema(
type = "integer"
)
)
)
}
)
@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE})
public int countChatMessages(@QueryParam("before") Long before, @QueryParam("after") Long after,
@QueryParam("txGroupId") Integer txGroupId,
@QueryParam("involving") List<String> involvingAddresses,
@QueryParam("reference") String reference,
@QueryParam("chatreference") String chatReference,
@QueryParam("haschatreference") Boolean hasChatReference,
@QueryParam("sender") String sender,
@QueryParam("encoding") Encoding encoding,
@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) {
// Check args meet expectations
if ((txGroupId == null && involvingAddresses.size() != 2)
|| (txGroupId != null && !involvingAddresses.isEmpty()))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
// Check any provided addresses are valid
if (involvingAddresses.stream().anyMatch(address -> !Crypto.isValidAddress(address)))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
if (before != null && before < 1500000000000L)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
if (after != null && after < 1500000000000L)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
byte[] referenceBytes = null;
if (reference != null)
referenceBytes = Base58.decode(reference);
byte[] chatReferenceBytes = null;
if (chatReference != null)
chatReferenceBytes = Base58.decode(chatReference);
try (final Repository repository = RepositoryManager.getRepository()) {
return repository.getChatRepository().getMessagesMatchingCriteria(
before,
after,
txGroupId,
referenceBytes,
chatReferenceBytes,
hasChatReference,
involvingAddresses,
sender,
encoding,
limit, offset, reverse).size();
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@GET
@Path("/message/{signature}")
@Operation(

32
src/main/java/org/qortal/api/resource/NamesResource.java
View File

@ -155,6 +155,38 @@ public class NamesResource {
}
}
@GET
@Path("/search")
@Operation(
summary = "Search registered names",
responses = {
@ApiResponse(
description = "registered name info",
content = @Content(
mediaType = MediaType.APPLICATION_JSON,
array = @ArraySchema(schema = @Schema(implementation = NameData.class))
)
)
}
)
@ApiErrors({ApiError.NAME_UNKNOWN, ApiError.REPOSITORY_ISSUE})
public List<NameData> searchNames(@QueryParam("query") String query,
@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
@Parameter(ref="reverse") @QueryParam("reverse") Boolean reverse) {
try (final Repository repository = RepositoryManager.getRepository()) {
if (query == null) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing query");
}
return repository.getNameRepository().searchNames(query, limit, offset, reverse);
} catch (ApiException e) {
throw e;
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@POST
@Path("/register")

61
src/main/java/org/qortal/api/resource/PollsResource.java
View File

@ -31,12 +31,18 @@ import javax.ws.rs.core.MediaType;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.ArraySchema;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.ws.rs.GET;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import org.qortal.api.ApiException;
import org.qortal.api.model.PollVotes;
import org.qortal.data.voting.PollData;
import org.qortal.data.voting.PollOptionData;
import org.qortal.data.voting.VoteOnPollData;
@Path("/polls")
@Tag(name = "Polls")
@ -102,6 +108,61 @@ public class PollsResource {
}
}
@GET
@Path("/votes/{pollName}")
@Operation(
summary = "Votes on poll",
responses = {
@ApiResponse(
description = "poll votes",
content = @Content(
mediaType = MediaType.APPLICATION_JSON,
schema = @Schema(implementation = PollVotes.class)
)
)
}
)
@ApiErrors({ApiError.REPOSITORY_ISSUE})
public PollVotes getPollVotes(@PathParam("pollName") String pollName, @QueryParam("onlyCounts") Boolean onlyCounts) {
try (final Repository repository = RepositoryManager.getRepository()) {
PollData pollData = repository.getVotingRepository().fromPollName(pollName);
if (pollData == null)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.POLL_NO_EXISTS);
List<VoteOnPollData> votes = repository.getVotingRepository().getVotes(pollName);
// Initialize map for counting votes
Map<String, Integer> voteCountMap = new HashMap<>();
for (PollOptionData optionData : pollData.getPollOptions()) {
voteCountMap.put(optionData.getOptionName(), 0);
}
int totalVotes = 0;
for (VoteOnPollData vote : votes) {
String selectedOption = pollData.getPollOptions().get(vote.getOptionIndex()).getOptionName();
if (voteCountMap.containsKey(selectedOption)) {
voteCountMap.put(selectedOption, voteCountMap.get(selectedOption) + 1);
totalVotes++;
}
}
// Convert map to list of VoteInfo
List<PollVotes.OptionCount> voteCounts = voteCountMap.entrySet().stream()
.map(entry -> new PollVotes.OptionCount(entry.getKey(), entry.getValue()))
.collect(Collectors.toList());
if (onlyCounts != null && onlyCounts) {
return new PollVotes(null, totalVotes, voteCounts);
} else {
return new PollVotes(votes, totalVotes, voteCounts);
}
} catch (ApiException e) {
throw e;
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@POST
@Path("/create")
@Operation(

8
src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
View File

@ -54,10 +54,6 @@ public class ArbitraryDataBuilder {
/**
* Process transactions, but do not build anything
* This is useful for checking the status of a given resource
*
* @throws DataException
* @throws IOException
* @throws MissingDataException
*/
public void process() throws DataException, IOException, MissingDataException {
this.fetchTransactions();
@ -69,10 +65,6 @@ public class ArbitraryDataBuilder {
/**
* Build the latest state of a given resource
*
* @throws DataException
* @throws IOException
* @throws MissingDataException
*/
public void build() throws DataException, IOException, MissingDataException {
this.process();

43
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@ -9,7 +9,6 @@ import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.crypto.AES;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
@ -35,6 +34,9 @@ import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
public class ArbitraryDataReader {
@ -60,6 +62,10 @@ public class ArbitraryDataReader {
// The resource being read
ArbitraryDataResource arbitraryDataResource = null;
// Track resources that are currently being loaded, to avoid duplicate concurrent builds
// TODO: all builds could be handled by the build queue (even synchronous ones), to avoid the need for this
private static Map<String, Long> inProgress = Collections.synchronizedMap(new HashMap<>());
public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
// Ensure names are always lowercase
if (resourceIdType == ResourceIdType.NAME) {
@ -154,9 +160,6 @@ public class ArbitraryDataReader {
* If no exception is thrown, you can then use getFilePath() to access the data immediately after returning
*
* @param overwrite - set to true to force rebuild an existing cache
* @throws IOException
* @throws DataException
* @throws MissingDataException
*/
public void loadSynchronously(boolean overwrite) throws DataException, IOException, MissingDataException {
try {
@ -170,6 +173,12 @@ public class ArbitraryDataReader {
this.arbitraryDataResource = this.createArbitraryDataResource();
// Don't allow duplicate loads
if (!this.canStartLoading()) {
LOGGER.debug("Skipping duplicate load of {}", this.arbitraryDataResource);
return;
}
this.preExecute();
this.deleteExistingFiles();
this.fetch();
@ -197,6 +206,7 @@ public class ArbitraryDataReader {
private void preExecute() throws DataException {
ArbitraryDataBuildManager.getInstance().setBuildInProgress(true);
this.checkEnabled();
this.createWorkingDirectory();
this.createUncompressedDirectory();
@ -204,6 +214,9 @@ public class ArbitraryDataReader {
private void postExecute() {
ArbitraryDataBuildManager.getInstance().setBuildInProgress(false);
this.arbitraryDataResource = this.createArbitraryDataResource();
ArbitraryDataReader.inProgress.remove(this.arbitraryDataResource.getUniqueKey());
}
private void checkEnabled() throws DataException {
@ -212,6 +225,17 @@ public class ArbitraryDataReader {
}
}
private boolean canStartLoading() {
// Avoid duplicate builds if we're already loading this resource
String uniqueKey = this.arbitraryDataResource.getUniqueKey();
if (ArbitraryDataReader.inProgress.containsKey(uniqueKey)) {
return false;
}
ArbitraryDataReader.inProgress.put(uniqueKey, NTP.getTime());
return true;
}
private void createWorkingDirectory() throws DataException {
try {
Files.createDirectories(this.workingPath);
@ -223,7 +247,6 @@ public class ArbitraryDataReader {
/**
* Working directory should only be deleted on failure, since it is currently used to
* serve a cached version of the resource for subsequent requests.
* @throws IOException
*/
private void deleteWorkingDirectory() {
try {
@ -303,7 +326,7 @@ public class ArbitraryDataReader {
break;
default:
throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType.toString()));
throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType));
}
}
@ -368,6 +391,9 @@ public class ArbitraryDataReader {
// Load data file(s)
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
if (arbitraryDataFile == null) {
throw new DataException(String.format("arbitraryDataFile is null"));
}
if (!arbitraryDataFile.allFilesExist()) {
if (ListUtils.isNameBlocked(transactionData.getName())) {
@ -443,6 +469,7 @@ public class ArbitraryDataReader {
Path unencryptedPath = Paths.get(this.workingPath.toString(), "zipped.zip");
SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, "AES");
AES.decryptFile(algorithm, aesKey, this.filePath.toString(), unencryptedPath.toString());
LOGGER.debug("Finished decrypting {} using algorithm {}", this.arbitraryDataResource, algorithm);
// Replace filePath pointer with the encrypted file path
// Don't delete the original ArbitraryDataFile, as this is handled in the cleanup phase
@ -477,7 +504,9 @@ public class ArbitraryDataReader {
// Handle each type of compression
if (compression == Compression.ZIP) {
LOGGER.debug("Unzipping {}...", this.arbitraryDataResource);
ZipUtils.unzip(this.filePath.toString(), this.uncompressedPath.getParent().toString());
LOGGER.debug("Finished unzipping {}", this.arbitraryDataResource);
}
else if (compression == Compression.NONE) {
Files.createDirectories(this.uncompressedPath);
@ -513,10 +542,12 @@ public class ArbitraryDataReader {
private void validate() throws IOException, DataException {
if (this.service.isValidationRequired()) {
LOGGER.debug("Validating {}...", this.arbitraryDataResource);
Service.ValidationResult result = this.service.validate(this.filePath);
if (result != Service.ValidationResult.OK) {
throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
}
LOGGER.debug("Finished validating {}", this.arbitraryDataResource);
}
}

10
src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
View File

@ -67,8 +67,8 @@ public class ArbitraryDataRenderer {
}
public HttpServletResponse render() {
if (!inPath.startsWith(File.separator)) {
inPath = File.separator + inPath;
if (!inPath.startsWith("/")) {
inPath = "/" + inPath;
}
// Don't render data if QDN is disabled
@ -126,7 +126,8 @@ public class ArbitraryDataRenderer {
try {
String filename = this.getFilename(unzippedPath, inPath);
Path filePath = Paths.get(unzippedPath, filename);
boolean usingCustomRouting = false;
// If the file doesn't exist, we may need to route the request elsewhere, or cleanup
if (!Files.exists(filePath)) {
if (inPath.equals("/")) {
@ -148,6 +149,7 @@ public class ArbitraryDataRenderer {
// Forward request to index file
filePath = indexPath;
filename = indexFile;
usingCustomRouting = true;
break;
}
}
@ -157,7 +159,7 @@ public class ArbitraryDataRenderer {
if (HTMLParser.isHtmlFile(filename)) {
// HTML file - needs to be parsed
byte[] data = Files.readAllBytes(filePath); // TODO: limit file size that can be read into memory
HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data, qdnContext, service, identifier, theme);
HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data, qdnContext, service, identifier, theme, usingCustomRouting);
htmlParser.addAdditionalHeaderTags();
response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:;");
response.setContentType(context.getMimeType(filename));

3
src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java
View File

@ -150,6 +150,9 @@ public class ArbitraryDataResource {
for (ArbitraryTransactionData transactionData : transactionDataList) {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
if (arbitraryDataFile == null) {
continue;
}
// Delete any chunks or complete files from each transaction
arbitraryDataFile.deleteAll(deleteMetadata);

9
src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java
View File

@ -9,6 +9,7 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@ -50,7 +51,7 @@ public class ArbitraryDataMetadata {
this.readJson();
} catch (JSONException e) {
throw new DataException(String.format("Unable to read JSON: %s", e.getMessage()));
throw new DataException(String.format("Unable to read JSON at path %s: %s", this.filePath, e.getMessage()));
}
}
@ -64,6 +65,10 @@ public class ArbitraryDataMetadata {
writer.close();
}
public void delete() throws IOException {
Files.delete(this.filePath);
}
protected void loadJson() throws IOException {
File metadataFile = new File(this.filePath.toString());
@ -71,7 +76,7 @@ public class ArbitraryDataMetadata {
throw new IOException(String.format("Metadata file doesn't exist: %s", this.filePath.toString()));
}
this.jsonString = new String(Files.readAllBytes(this.filePath));
this.jsonString = new String(Files.readAllBytes(this.filePath), StandardCharsets.UTF_8);
}

3
src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java
View File

@ -9,6 +9,7 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -69,7 +70,7 @@ public class ArbitraryDataQortalMetadata extends ArbitraryDataMetadata {
throw new IOException(String.format("Patch file doesn't exist: %s", path.toString()));
}
this.jsonString = new String(Files.readAllBytes(path));
this.jsonString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
}

24
src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java
View File

@ -7,6 +7,7 @@ import org.qortal.arbitrary.misc.Category;
import org.qortal.repository.DataException;
import org.qortal.utils.Base58;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
@ -217,6 +218,25 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
// Static helper methods
public static String trimUTF8String(String string, int maxLength) {
byte[] inputBytes = string.getBytes(StandardCharsets.UTF_8);
int length = Math.min(inputBytes.length, maxLength);
byte[] outputBytes = new byte[length];
System.arraycopy(inputBytes, 0, outputBytes, 0, length);
String result = new String(outputBytes, StandardCharsets.UTF_8);
// check if last character is truncated
int lastIndex = result.length() - 1;
if (lastIndex > 0 && result.charAt(lastIndex) != string.charAt(lastIndex)) {
// last character is truncated so remove the last character
return result.substring(0, lastIndex);
}
return result;
}
public static String limitTitle(String title) {
if (title == null) {
return null;
@ -225,7 +245,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
return null;
}
return title.substring(0, Math.min(title.length(), MAX_TITLE_LENGTH));
return trimUTF8String(title, MAX_TITLE_LENGTH);
}
public static String limitDescription(String description) {
@ -236,7 +256,7 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
return null;
}
return description.substring(0, Math.min(description.length(), MAX_DESCRIPTION_LENGTH));
return trimUTF8String(description, MAX_DESCRIPTION_LENGTH);
}
public static List<String> limitTags(List<String> tags) {

146
src/main/java/org/qortal/arbitrary/misc/Service.java
View File

@ -9,7 +9,6 @@ import org.qortal.utils.FilesystemUtils;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
@ -20,9 +19,9 @@ import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap;
public enum Service {
AUTO_UPDATE(1, false, null, false, null),
ARBITRARY_DATA(100, false, null, false, null),
QCHAT_ATTACHMENT(120, true, 1024*1024L, true, null) {
AUTO_UPDATE(1, false, null, false, false, null),
ARBITRARY_DATA(100, false, null, false, false, null),
QCHAT_ATTACHMENT(120, true, 1024*1024L, true, false, null) {
@Override
public ValidationResult validate(Path path) throws IOException {
ValidationResult superclassResult = super.validate(path);
@ -47,7 +46,14 @@ public enum Service {
return ValidationResult.OK;
}
},
WEBSITE(200, true, null, false, null) {
QCHAT_ATTACHMENT_PRIVATE(121, true, 1024*1024L, true, true, null),
ATTACHMENT(130, false, 50*1024*1024L, true, false, null),
ATTACHMENT_PRIVATE(131, true, 50*1024*1024L, true, true, null),
FILE(140, false, null, true, false, null),
FILE_PRIVATE(141, true, null, true, true, null),
FILES(150, false, null, false, false, null),
CHAIN_DATA(160, true, 239L, true, false, null),
WEBSITE(200, true, null, false, false, null) {
@Override
public ValidationResult validate(Path path) throws IOException {
ValidationResult superclassResult = super.validate(path);
@ -69,23 +75,30 @@ public enum Service {
return ValidationResult.MISSING_INDEX_FILE;
}
},
GIT_REPOSITORY(300, false, null, false, null),
IMAGE(400, true, 10*1024*1024L, true, null),
THUMBNAIL(410, true, 500*1024L, true, null),
QCHAT_IMAGE(420, true, 500*1024L, true, null),
VIDEO(500, false, null, true, null),
AUDIO(600, false, null, true, null),
QCHAT_AUDIO(610, true, 10*1024*1024L, true, null),
QCHAT_VOICE(620, true, 10*1024*1024L, true, null),
BLOG(700, false, null, false, null),
BLOG_POST(777, false, null, true, null),
BLOG_COMMENT(778, false, null, true, null),
DOCUMENT(800, false, null, true, null),
LIST(900, true, null, true, null),
PLAYLIST(910, true, null, true, null),
APP(1000, true, 50*1024*1024L, false, null),
METADATA(1100, false, null, true, null),
JSON(1110, true, 25*1024L, true, null) {
GIT_REPOSITORY(300, false, null, false, false, null),
IMAGE(400, true, 10*1024*1024L, true, false, null),
IMAGE_PRIVATE(401, true, 10*1024*1024L, true, true, null),
THUMBNAIL(410, true, 500*1024L, true, false, null),
QCHAT_IMAGE(420, true, 500*1024L, true, false, null),
VIDEO(500, false, null, true, false, null),
VIDEO_PRIVATE(501, true, null, true, true, null),
AUDIO(600, false, null, true, false, null),
AUDIO_PRIVATE(601, true, null, true, true, null),
QCHAT_AUDIO(610, true, 10*1024*1024L, true, false, null),
QCHAT_VOICE(620, true, 10*1024*1024L, true, false, null),
VOICE(630, true, 10*1024*1024L, true, false, null),
VOICE_PRIVATE(631, true, 10*1024*1024L, true, true, null),
PODCAST(640, false, null, true, false, null),
BLOG(700, false, null, false, false, null),
BLOG_POST(777, false, null, true, false, null),
BLOG_COMMENT(778, true, 500*1024L, true, false, null),
DOCUMENT(800, false, null, true, false, null),
DOCUMENT_PRIVATE(801, true, null, true, true, null),
LIST(900, true, null, true, false, null),
PLAYLIST(910, true, null, true, false, null),
APP(1000, true, 50*1024*1024L, false, false, null),
METADATA(1100, false, null, true, false, null),
JSON(1110, true, 25*1024L, true, false, null) {
@Override
public ValidationResult validate(Path path) throws IOException {
ValidationResult superclassResult = super.validate(path);
@ -94,7 +107,7 @@ public enum Service {
}
// Require valid JSON
byte[] data = FilesystemUtils.getSingleFileContents(path);
byte[] data = FilesystemUtils.getSingleFileContents(path, 25*1024);
String json = new String(data, StandardCharsets.UTF_8);
try {
objectMapper.readTree(json);
@ -104,7 +117,7 @@ public enum Service {
}
}
},
GIF_REPOSITORY(1200, true, 25*1024*1024L, false, null) {
GIF_REPOSITORY(1200, true, 25*1024*1024L, false, false, null) {
@Override
public ValidationResult validate(Path path) throws IOException {
ValidationResult superclassResult = super.validate(path);
@ -139,12 +152,31 @@ public enum Service {
}
return ValidationResult.OK;
}
};
},
STORE(1300, false, null, true, false, null),
PRODUCT(1310, false, null, true, false, null),
OFFER(1330, false, null, true, false, null),
COUPON(1340, false, null, true, false, null),
CODE(1400, false, null, true, false, null),
PLUGIN(1410, false, null, true, false, null),
EXTENSION(1420, false, null, true, false, null),
GAME(1500, false, null, false, false, null),
ITEM(1510, false, null, true, false, null),
NFT(1600, false, null, true, false, null),
DATABASE(1700, false, null, false, false, null),
SNAPSHOT(1710, false, null, false, false, null),
COMMENT(1800, true, 500*1024L, true, false, null),
CHAIN_COMMENT(1810, true, 239L, true, false, null),
MAIL(1900, true, 1024*1024L, true, false, null),
MAIL_PRIVATE(1901, true, 1024*1024L, true, true, null),
MESSAGE(1910, true, 1024*1024L, true, false, null),
MESSAGE_PRIVATE(1911, true, 1024*1024L, true, true, null);
public final int value;
private final boolean requiresValidation;
private final Long maxSize;
private final boolean single;
private final boolean isPrivate;
private final List<String> requiredKeys;
private static final Map<Integer, Service> map = stream(Service.values())
@ -153,11 +185,14 @@ public enum Service {
// For JSON validation
private static final ObjectMapper objectMapper = new ObjectMapper();
Service(int value, boolean requiresValidation, Long maxSize, boolean single, List<String> requiredKeys) {
private static final String encryptedDataPrefix = "qortalEncryptedData";
Service(int value, boolean requiresValidation, Long maxSize, boolean single, boolean isPrivate, List<String> requiredKeys) {
this.value = value;
this.requiresValidation = requiresValidation;
this.maxSize = maxSize;
this.single = single;
this.isPrivate = isPrivate;
this.requiredKeys = requiredKeys;
}
@ -166,7 +201,9 @@ public enum Service {
return ValidationResult.OK;
}
byte[] data = FilesystemUtils.getSingleFileContents(path);
// Load the first 25KB of data. This only needs to be long enough to check the prefix
// and also to allow for possible additional future validation of smaller files.
byte[] data = FilesystemUtils.getSingleFileContents(path, 25*1024);
long size = FilesystemUtils.getDirectorySize(path);
// Validate max size if needed
@ -181,6 +218,17 @@ public enum Service {
return ValidationResult.INVALID_FILE_COUNT;
}
// Validate private data for single file resources
if (this.single) {
String dataString = new String(data, StandardCharsets.UTF_8);
if (this.isPrivate && !dataString.startsWith(encryptedDataPrefix)) {
return ValidationResult.DATA_NOT_ENCRYPTED;
}
if (!this.isPrivate && dataString.startsWith(encryptedDataPrefix)) {
return ValidationResult.DATA_ENCRYPTED;
}
}
// Validate required keys if needed
if (this.requiredKeys != null) {
if (data == null) {
@ -199,7 +247,12 @@ public enum Service {
}
public boolean isValidationRequired() {
return this.requiresValidation;
// We must always validate single file resources, to ensure they are actually a single file
return this.requiresValidation || this.single;
}
public boolean isPrivate() {
return this.isPrivate;
}
public static Service valueOf(int value) {
@ -207,10 +260,41 @@ public enum Service {
}
public static JSONObject toJsonObject(byte[] data) {
String dataString = new String(data);
String dataString = new String(data, StandardCharsets.UTF_8);
return new JSONObject(dataString);
}
public static List<Service> publicServices() {
List<Service> privateServices = new ArrayList<>();
for (Service service : Service.values()) {
if (!service.isPrivate) {
privateServices.add(service);
}
}
return privateServices;
}
/**
* Fetch a list of Service objects that require encrypted data.
*
* These can ultimately be used to help inform the cleanup manager
* on the best order to delete files when the node runs out of space.
* Public data should be given priority over private data (unless
* this node is part of a data market contract for that data - this
* isn't developed yet).
*
* @return a list of Service objects that require encrypted data.
*/
public static List<Service> privateServices() {
List<Service> privateServices = new ArrayList<>();
for (Service service : Service.values()) {
if (service.isPrivate) {
privateServices.add(service);
}
}
return privateServices;
}
public enum ValidationResult {
OK(1),
MISSING_KEYS(2),
@ -220,7 +304,9 @@ public enum Service {
INVALID_FILE_EXTENSION(6),
MISSING_DATA(7),
INVALID_FILE_COUNT(8),
INVALID_CONTENT(9);
INVALID_CONTENT(9),
DATA_NOT_ENCRYPTED(10),
DATA_ENCRYPTED(10);
public final int value;

186
src/main/java/org/qortal/controller/OnlineAccountsManager.java
View File

@ -504,110 +504,118 @@ public class OnlineAccountsManager {
computeOurAccountsForTimestamp(onlineAccountsTimestamp);
}
private boolean computeOurAccountsForTimestamp(long onlineAccountsTimestamp) {
List<MintingAccountData> mintingAccounts;
try (final Repository repository = RepositoryManager.getRepository()) {
mintingAccounts = repository.getAccountRepository().getMintingAccounts();
private boolean computeOurAccountsForTimestamp(Long onlineAccountsTimestamp) {
if (onlineAccountsTimestamp != null) {
List<MintingAccountData> mintingAccounts;
try (final Repository repository = RepositoryManager.getRepository()) {
mintingAccounts = repository.getAccountRepository().getMintingAccounts();
// We have no accounts to send
if (mintingAccounts.isEmpty())
// We have no accounts to send
if (mintingAccounts.isEmpty())
return false;
// Only active reward-shares allowed
Iterator<MintingAccountData> iterator = mintingAccounts.iterator();
int i = 0;
while (iterator.hasNext()) {
MintingAccountData mintingAccountData = iterator.next();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
if (rewardShareData == null) {
// Reward-share doesn't even exist - probably not a good sign
iterator.remove();
continue;
}
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
iterator.remove();
continue;
}
if (++i > 1 + 1) {
iterator.remove();
continue;
}
}
} catch (DataException e) {
LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
return false;
}
// Only active reward-shares allowed
Iterator<MintingAccountData> iterator = mintingAccounts.iterator();
while (iterator.hasNext()) {
MintingAccountData mintingAccountData = iterator.next();
byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
if (rewardShareData == null) {
// Reward-share doesn't even exist - probably not a good sign
iterator.remove();
int remaining = mintingAccounts.size();
for (MintingAccountData mintingAccountData : mintingAccounts) {
remaining--;
byte[] privateKey = mintingAccountData.getPrivateKey();
byte[] publicKey = Crypto.toPublicKey(privateKey);
// We don't want to compute the online account nonce and signature again if it already exists
Set<OnlineAccountData> onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountsTimestamp, k -> ConcurrentHashMap.newKeySet());
boolean alreadyExists = onlineAccounts.stream().anyMatch(a -> Arrays.equals(a.getPublicKey(), publicKey));
if (alreadyExists) {
this.hasOurOnlineAccounts = true;
if (remaining > 0) {
// Move on to next account
continue;
} else {
// Everything exists, so return true
return true;
}
}
// Generate bytes for mempow
byte[] mempowBytes;
try {
mempowBytes = this.getMemoryPoWBytes(publicKey, onlineAccountsTimestamp);
} catch (IOException e) {
LOGGER.info("Unable to create bytes for MemoryPoW. Moving on to next account...");
continue;
}
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
iterator.remove();
continue;
}
}
} catch (DataException e) {
LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
return false;
}
byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
int remaining = mintingAccounts.size();
for (MintingAccountData mintingAccountData : mintingAccounts) {
remaining--;
byte[] privateKey = mintingAccountData.getPrivateKey();
byte[] publicKey = Crypto.toPublicKey(privateKey);
// We don't want to compute the online account nonce and signature again if it already exists
Set<OnlineAccountData> onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountsTimestamp, k -> ConcurrentHashMap.newKeySet());
boolean alreadyExists = onlineAccounts.stream().anyMatch(a -> Arrays.equals(a.getPublicKey(), publicKey));
if (alreadyExists) {
this.hasOurOnlineAccounts = true;
if (remaining > 0) {
// Move on to next account
continue;
}
else {
// Everything exists, so return true
return true;
}
}
// Generate bytes for mempow
byte[] mempowBytes;
try {
mempowBytes = this.getMemoryPoWBytes(publicKey, onlineAccountsTimestamp);
}
catch (IOException e) {
LOGGER.info("Unable to create bytes for MemoryPoW. Moving on to next account...");
continue;
}
// Compute nonce
Integer nonce;
try {
nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp);
if (nonce == null) {
// A nonce is required
// Compute nonce
Integer nonce;
try {
nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp);
if (nonce == null) {
// A nonce is required
return false;
}
} catch (TimeoutException e) {
LOGGER.info(String.format("Timed out computing nonce for account %.8s", Base58.encode(publicKey)));
return false;
}
} catch (TimeoutException e) {
LOGGER.info(String.format("Timed out computing nonce for account %.8s", Base58.encode(publicKey)));
byte[] signature = Qortal25519Extras.signForAggregation(privateKey, timestampBytes);
// Our account is online
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce);
// Make sure to verify before adding
if (verifyMemoryPoW(ourOnlineAccountData, null)) {
ourOnlineAccounts.add(ourOnlineAccountData);
}
}
this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty();
boolean hasInfoChanged = addAccounts(ourOnlineAccounts);
if (!hasInfoChanged)
return false;
}
byte[] signature = Qortal25519Extras.signForAggregation(privateKey, timestampBytes);
Network.getInstance().broadcast(peer -> new OnlineAccountsV3Message(ourOnlineAccounts));
// Our account is online
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, nonce);
LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp);
// Make sure to verify before adding
if (verifyMemoryPoW(ourOnlineAccountData, null)) {
ourOnlineAccounts.add(ourOnlineAccountData);
}
return true;
}
this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty();
boolean hasInfoChanged = addAccounts(ourOnlineAccounts);
if (!hasInfoChanged)
return false;
Network.getInstance().broadcast(peer -> new OnlineAccountsV3Message(ourOnlineAccounts));
LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp);
return true;
return false;
}

4
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@ -346,6 +346,10 @@ public class ArbitraryDataCleanupManager extends Thread {
/**
* Iteratively walk through given directory and delete a single random file
*
* TODO: public data should be prioritized over private data
* (unless this node is part of a data market contract for that data).
* See: Service.privateServices() for a list of services containing private data.
*
* @param directory - the base directory
* @return boolean - whether a file was deleted
*/

26
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
View File

@ -124,29 +124,29 @@ public class ArbitraryDataFileListManager {
}
}
// Then allow another 3 attempts, each 5 minutes apart
if (timeSinceLastAttempt > 5 * 60 * 1000L) {
// We haven't tried for at least 5 minutes
// Then allow another 5 attempts, each 1 minute apart
if (timeSinceLastAttempt > 60 * 1000L) {
// We haven't tried for at least 1 minute
if (networkBroadcastCount < 6) {
// We've made less than 6 total attempts
if (networkBroadcastCount < 8) {
// We've made less than 8 total attempts
return true;
}
}
// Then allow another 4 attempts, each 30 minutes apart
if (timeSinceLastAttempt > 30 * 60 * 1000L) {
// We haven't tried for at least 5 minutes
// Then allow another 8 attempts, each 15 minutes apart
if (timeSinceLastAttempt > 15 * 60 * 1000L) {
// We haven't tried for at least 15 minutes
if (networkBroadcastCount < 10) {
// We've made less than 10 total attempts
if (networkBroadcastCount < 16) {
// We've made less than 16 total attempts
return true;
}
}
// From then on, only try once every 24 hours, to reduce network spam
if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) {
// We haven't tried for at least 24 hours
// From then on, only try once every 6 hours, to reduce network spam
if (timeSinceLastAttempt > 6 * 60 * 60 * 1000L) {
// We haven't tried for at least 6 hours
return true;
}

1
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -16,7 +16,6 @@ import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.list.ResourceListManager;
import org.qortal.network.Network;
import org.qortal.network.Peer;
import org.qortal.repository.DataException;

9
src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
View File

@ -102,7 +102,14 @@ public class ArbitraryMetadataManager {
if (metadataFile.exists()) {
// Use local copy
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
transactionMetadata.read();
try {
transactionMetadata.read();
} catch (DataException e) {
// Invalid file, so delete it
LOGGER.info("Deleting invalid metadata file due to exception: {}", e.getMessage());
transactionMetadata.delete();
return null;
}
return transactionMetadata;
}
}

3
src/main/java/org/qortal/data/transaction/TransactionData.java
View File

@ -13,6 +13,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import org.eclipse.persistence.oxm.annotations.XmlDiscriminatorNode;
import org.qortal.crypto.Crypto;
import org.qortal.data.voting.PollData;
import org.qortal.data.voting.VoteOnPollData;
import org.qortal.transaction.Transaction.ApprovalStatus;
import org.qortal.transaction.Transaction.TransactionType;
@ -30,7 +31,7 @@ import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
@XmlSeeAlso({GenesisTransactionData.class, PaymentTransactionData.class, RegisterNameTransactionData.class, UpdateNameTransactionData.class,
SellNameTransactionData.class, CancelSellNameTransactionData.class, BuyNameTransactionData.class,
CreatePollTransactionData.class, VoteOnPollTransactionData.class, ArbitraryTransactionData.class,
PollData.class,
PollData.class, VoteOnPollData.class,
IssueAssetTransactionData.class, TransferAssetTransactionData.class,
CreateAssetOrderTransactionData.class, CancelAssetOrderTransactionData.class,
MultiPaymentTransactionData.class, DeployAtTransactionData.class, MessageTransactionData.class, ATTransactionData.class,

17
src/main/java/org/qortal/data/voting/VoteOnPollData.java
View File

@ -9,6 +9,11 @@ public class VoteOnPollData {
// Constructors
// For JAXB
protected VoteOnPollData() {
super();
}
public VoteOnPollData(String pollName, byte[] voterPublicKey, int optionIndex) {
this.pollName = pollName;
this.voterPublicKey = voterPublicKey;
@ -21,12 +26,24 @@ public class VoteOnPollData {
return this.pollName;
}
public void setPollName(String pollName) {
this.pollName = pollName;
}
public byte[] getVoterPublicKey() {
return this.voterPublicKey;
}
public void setVoterPublicKey(byte[] voterPublicKey) {
this.voterPublicKey = voterPublicKey;
}
public int getOptionIndex() {
return this.optionIndex;
}
public void setOptionIndex(int optionIndex) {
this.optionIndex = optionIndex;
}
}

3
src/main/java/org/qortal/list/ResourceList.java
View File

@ -9,6 +9,7 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@ -81,7 +82,7 @@ public class ResourceList {
}
try {
String jsonString = new String(Files.readAllBytes(path));
String jsonString = new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
this.list = ResourceList.listFromJSONString(jsonString);
} catch (IOException e) {
throw new IOException(String.format("Couldn't read contents from file %s", path.toString()));

2
src/main/java/org/qortal/repository/NameRepository.java
View File

@ -14,6 +14,8 @@ public interface NameRepository {
public boolean reducedNameExists(String reducedName) throws DataException;
public List<NameData> searchNames(String query, Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<NameData> getAllNames(Integer limit, Integer offset, Boolean reverse) throws DataException;
public default List<NameData> getAllNames() throws DataException {

51
src/main/java/org/qortal/repository/hsqldb/HSQLDBNameRepository.java
View File

@ -103,6 +103,57 @@ public class HSQLDBNameRepository implements NameRepository {
}
}
public List<NameData> searchNames(String query, Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
sql.append("SELECT name, reduced_name, owner, data, registered_when, updated_when, "
+ "is_for_sale, sale_price, reference, creation_group_id FROM Names "
+ "WHERE LCASE(name) LIKE ? ORDER BY name");
bindParams.add(String.format("%%%s%%", query.toLowerCase()));
if (reverse != null && reverse)
sql.append(" DESC");
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
List<NameData> names = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return names;
do {
String name = resultSet.getString(1);
String reducedName = resultSet.getString(2);
String owner = resultSet.getString(3);
String data = resultSet.getString(4);
long registered = resultSet.getLong(5);
// Special handling for possibly-NULL "updated" column
Long updated = resultSet.getLong(6);
if (updated == 0 && resultSet.wasNull())
updated = null;
boolean isForSale = resultSet.getBoolean(7);
Long salePrice = resultSet.getLong(8);
if (salePrice == 0 && resultSet.wasNull())
salePrice = null;
byte[] reference = resultSet.getBytes(9);
int creationGroupId = resultSet.getInt(10);
names.add(new NameData(name, reducedName, owner, data, registered, updated, isForSale, salePrice, reference, creationGroupId));
} while (resultSet.next());
return names;
} catch (SQLException e) {
throw new DataException("Unable to search names in repository", e);
}
}
@Override
public List<NameData> getAllNames(Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(256);

12
src/main/java/org/qortal/utils/FilesystemUtils.java
View File

@ -228,12 +228,18 @@ public class FilesystemUtils {
* @throws IOException
*/
public static byte[] getSingleFileContents(Path path) throws IOException {
return getSingleFileContents(path, null);
}
public static byte[] getSingleFileContents(Path path, Integer maxLength) throws IOException {
byte[] data = null;
// TODO: limit the file size that can be loaded into memory
// If the path is a file, read the contents directly
if (path.toFile().isFile()) {
data = Files.readAllBytes(path);
int fileSize = (int)path.toFile().length();
maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize;
data = FilesystemUtils.readFromFile(path.toString(), 0, maxLength);
}
// Or if it's a directory, only load file contents if there is a single file inside it
@ -242,7 +248,9 @@ public class FilesystemUtils {
if (files.length == 1) {
Path filePath = Paths.get(path.toString(), files[0]);
if (filePath.toFile().isFile()) {
data = Files.readAllBytes(filePath);
int fileSize = (int)filePath.toFile().length();
maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize;
data = FilesystemUtils.readFromFile(filePath.toString(), 0, maxLength);
}
}
}

6
src/main/resources/q-apps/q-apps-gateway.js
View File

@ -50,12 +50,16 @@ window.addEventListener("message", (event) => {
switch (data.action) {
case "GET_USER_ACCOUNT":
case "PUBLISH_QDN_RESOURCE":
case "PUBLISH_MULTIPLE_QDN_RESOURCES":
case "SEND_CHAT_MESSAGE":
case "JOIN_GROUP":
case "DEPLOY_AT":
case "GET_WALLET_BALANCE":
case "SEND_COIN":
const errorString = "Authentication was requested, but this is not yet supported when viewing via a gateway. To use interactive features, please access using the Qortal UI desktop app. More info at: https://qortal.org";
case "GET_LIST_ITEMS":
case "ADD_LIST_ITEMS":
case "DELETE_LIST_ITEM":
const errorString = "Interactive features were requested, but these are not yet supported when viewing via a gateway. To use interactive features, please access using the Qortal UI desktop app. More info at: https://qortal.org";
response = "{\"error\": \"" + errorString + "\"}"
const modalText = "This app is powered by the Qortal blockchain. You are viewing in read-only mode. To use interactive features, please access using the Qortal UI desktop app.";

18
src/main/resources/q-apps/q-apps.js
View File

@ -1,4 +1,4 @@
function httpGet(event, url) {
function httpGet(url) {
var request = new XMLHttpRequest();
request.open("GET", url, false);
request.send(null);
@ -169,7 +169,7 @@ window.addEventListener("message", (event) => {
return;
}
console.log("Core received event: " + JSON.stringify(event.data));
console.log("Core received action: " + JSON.stringify(event.data.action));
let url;
let data = event.data;
@ -236,13 +236,15 @@ window.addEventListener("message", (event) => {
if (data.identifier != null) url = url.concat("/" + data.identifier);
url = url.concat("?");
if (data.filepath != null) url = url.concat("&filepath=" + data.filepath);
if (data.rebuild != null) url = url.concat("&rebuild=" + new Boolean(data.rebuild).toString())
if (data.rebuild != null) url = url.concat("&rebuild=" + new Boolean(data.rebuild).toString());
if (data.encoding != null) url = url.concat("&encoding=" + data.encoding);
return httpGetAsyncWithEvent(event, url);
case "GET_QDN_RESOURCE_STATUS":
url = "/arbitrary/resource/status/" + data.service + "/" + data.name;
if (data.identifier != null) url = url.concat("/" + data.identifier);
url = url.concat("?");
if (data.build != null) url = url.concat("&build=" + new Boolean(data.build).toString());
return httpGetAsyncWithEvent(event, url);
case "GET_QDN_RESOURCE_PROPERTIES":
@ -250,6 +252,11 @@ window.addEventListener("message", (event) => {
url = "/arbitrary/resource/properties/" + data.service + "/" + data.name + "/" + identifier;
return httpGetAsyncWithEvent(event, url);
case "GET_QDN_RESOURCE_METADATA":
identifier = (data.identifier != null) ? data.identifier : "default";
url = "/arbitrary/metadata/" + data.service + "/" + data.name + "/" + identifier;
return httpGetAsyncWithEvent(event, url);
case "SEARCH_CHAT_MESSAGES":
url = "/chat/messages?";
if (data.before != null) url = url.concat("&before=" + data.before);
@ -259,6 +266,7 @@ window.addEventListener("message", (event) => {
if (data.reference != null) url = url.concat("&reference=" + data.reference);
if (data.chatReference != null) url = url.concat("&chatreference=" + data.chatReference);
if (data.hasChatReference != null) url = url.concat("&haschatreference=" + new Boolean(data.hasChatReference).toString());
if (data.encoding != null) url = url.concat("&encoding=" + data.encoding);
if (data.limit != null) url = url.concat("&limit=" + data.limit);
if (data.offset != null) url = url.concat("&offset=" + data.offset);
if (data.reverse != null) url = url.concat("&reverse=" + new Boolean(data.reverse).toString());
@ -426,6 +434,8 @@ function getDefaultTimeout(action) {
// Some actions need longer default timeouts, especially those that create transactions
switch (action) {
case "GET_USER_ACCOUNT":
case "SAVE_FILE":
case "DECRYPT_DATA":
// User may take a long time to accept/deny the popup
return 60 * 60 * 1000;
@ -434,8 +444,8 @@ function getDefaultTimeout(action) {
return 60 * 1000;
case "PUBLISH_QDN_RESOURCE":
case "PUBLISH_MULTIPLE_QDN_RESOURCES":
// Publishing could take a very long time on slow system, due to the proof-of-work computation
// It's best not to timeout
return 60 * 60 * 1000;
case "SEND_CHAT_MESSAGE":

84
src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java
View File

@ -29,6 +29,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.List;
import java.util.Random;
import static org.junit.Assert.*;
@ -436,4 +437,87 @@ public class ArbitraryServiceTests extends Common {
assertEquals(ValidationResult.INVALID_FILE_COUNT, service.validate(path));
}
@Test
public void testValidPrivateData() throws IOException {
String dataString = "qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1rfla5UBoEcoxbtjgZ9G7FLPb8V/Qfr0bfKWfvMmN06U/pgUdLuv2mGL2V0D3qYd1011MUzGdNG1qERjaCDz8GAi63+KnHHjfMtPgYt6bcqjs4CNV+ZZ4dIt3xxHYyVEBNc=";
// Write the data a single file in a temp path
Path path = Files.createTempDirectory("testValidPrivateData");
Path filePath = Paths.get(path.toString(), "test");
filePath.toFile().deleteOnExit();
BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile()));
writer.write(dataString);
writer.close();
Service service = Service.FILE_PRIVATE;
assertTrue(service.isValidationRequired());
assertEquals(ValidationResult.OK, service.validate(filePath));
}
@Test
public void testEncryptedData() throws IOException {
String dataString = "qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1rfla5UBoEcoxbtjgZ9G7FLPb8V/Qfr0bfKWfvMmN06U/pgUdLuv2mGL2V0D3qYd1011MUzGdNG1qERjaCDz8GAi63+KnHHjfMtPgYt6bcqjs4CNV+ZZ4dIt3xxHYyVEBNc=";
// Write the data a single file in a temp path
Path path = Files.createTempDirectory("testValidPrivateData");
Path filePath = Paths.get(path.toString(), "test");
filePath.toFile().deleteOnExit();
BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile()));
writer.write(dataString);
writer.close();
// Validate a private service
Service service = Service.FILE_PRIVATE;
assertTrue(service.isValidationRequired());
assertEquals(ValidationResult.OK, service.validate(filePath));
// Validate a regular service
service = Service.FILE;
assertTrue(service.isValidationRequired());
assertEquals(ValidationResult.DATA_ENCRYPTED, service.validate(filePath));
}
@Test
public void testPlainTextData() throws IOException {
String dataString = "plaintext";
// Write the data a single file in a temp path
Path path = Files.createTempDirectory("testInvalidPrivateData");
Path filePath = Paths.get(path.toString(), "test");
filePath.toFile().deleteOnExit();
BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile()));
writer.write(dataString);
writer.close();
// Validate a private service
Service service = Service.FILE_PRIVATE;
assertTrue(service.isValidationRequired());
assertEquals(ValidationResult.DATA_NOT_ENCRYPTED, service.validate(filePath));
// Validate a regular service
service = Service.FILE;
assertTrue(service.isValidationRequired());
assertEquals(ValidationResult.OK, service.validate(filePath));
}
@Test
public void testGetPrivateServices() {
List<Service> privateServices = Service.privateServices();
for (Service service : privateServices) {
assertTrue(service.isPrivate());
}
}
@Test
public void testGetPublicServices() {
List<Service> publicServices = Service.publicServices();
for (Service service : publicServices) {
assertFalse(service.isPrivate());
}
}
}

41
src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java
View File

@ -248,6 +248,47 @@ public class ArbitraryTransactionMetadataTests extends Common {
}
}
@Test
public void testUTF8Metadata() throws DataException, IOException, MissingDataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());
String name = "TEST"; // Can be anything for this test
String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA;
int chunkSize = 100;
int dataLength = 900; // Actual data length will be longer due to encryption
// Example (modified) strings from real world content
String title = "Доля юаня в трансграничных Доля юаня в трансграничных";
String description = "Когда рыночек порешал";
List<String> tags = Arrays.asList("Доля", "юаня", "трансграничных");
Category category = Category.OTHER;
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength);
ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name,
identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize, 0L, true,
title, description, tags, category);
// Check the chunk count is correct
assertEquals(10, arbitraryDataFile.chunkCount());
// Check the metadata is correct
String expectedTrimmedTitle = "Доля юаня в трансграничных Доля юаня в тран";
assertEquals(expectedTrimmedTitle, arbitraryDataFile.getMetadata().getTitle());
assertEquals(description, arbitraryDataFile.getMetadata().getDescription());
assertEquals(tags, arbitraryDataFile.getMetadata().getTags());
assertEquals(category, arbitraryDataFile.getMetadata().getCategory());
assertEquals("text/plain", arbitraryDataFile.getMetadata().getMimeType());
}
}
@Test
public void testMetadataLengths() throws DataException, IOException, MissingDataException {
try (final Repository repository = RepositoryManager.getRepository()) {

180
tools/tx.pl
View File

@ -1,16 +1,23 @@
#!/usr/bin/env perl
# v4.0.2
use JSON;
use warnings;
use strict;
use Getopt::Std;
use File::Basename;
use Digest::SHA qw( sha256 sha256_hex );
use Crypt::RIPEMD160;
our %opt;
getopts('dpst', \%opt);
my $proc = basename($0);
my $dirname = dirname($0);
my $OPENSSL_SIGN = "${dirname}/openssl-sign.sh";
my $OPENSSL_PRIV_TO_PUB = index(`$ENV{SHELL} -i -c 'openssl version;exit' 2>/dev/null`, 'OpenSSL 3.') != -1;
if (@ARGV < 1) {
print STDERR "usage: $proc [-d] [-p] [-s] [-t] <tx-type> <privkey> <values> [<key-value pairs>]\n";
@ -24,7 +31,15 @@ if (@ARGV < 1) {
exit 2;
}
our $BASE_URL = $ENV{BASE_URL} || $opt{t} ? 'http://localhost:62391' : 'http://localhost:12391';
our @b58 = qw{
1 2 3 4 5 6 7 8 9
A B C D E F G H J K L M N P Q R S T U V W X Y Z
a b c d e f g h i j k m n o p q r s t u v w x y z
};
our %b58 = map { $b58[$_] => $_ } 0 .. 57;
our %reverseb58 = reverse %b58;
our $BASE_URL = $ENV{BASE_URL} || ($opt{t} ? 'http://localhost:62391' : 'http://localhost:12391');
our $DEFAULT_FEE = 0.001;
our %TRANSACTION_TYPES = (
@ -42,6 +57,7 @@ our %TRANSACTION_TYPES = (
create_group => {
url => 'groups/create',
required => [qw(groupName description isOpen approvalThreshold)],
defaults => { minimumBlockDelay => 10, maximumBlockDelay => 30 },
key_name => 'creatorPublicKey',
},
update_group => {
@ -75,10 +91,10 @@ our %TRANSACTION_TYPES = (
key_name => 'ownerPublicKey',
},
remove_group_admin => {
url => 'groups/removeadmin',
required => [qw(groupId txGroupId admin)],
key_name => 'ownerPublicKey',
},
url => 'groups/removeadmin',
required => [qw(groupId txGroupId member)],
key_name => 'ownerPublicKey',
},
group_approval => {
url => 'groups/approval',
required => [qw(pendingSignature approval)],
@ -113,7 +129,7 @@ our %TRANSACTION_TYPES = (
},
update_name => {
url => 'names/update',
required => [qw(newName newData)],
required => [qw(name newName newData)],
key_name => 'ownerPublicKey',
},
# reward-shares
@ -144,13 +160,21 @@ our %TRANSACTION_TYPES = (
key_name => 'senderPublicKey',
pow_url => 'addresses/publicize/compute',
},
# Cross-chain trading
build_trade => {
url => 'crosschain/build',
required => [qw(initialQortAmount finalQortAmount fundingQortAmount secretHash bitcoinAmount)],
optional => [qw(tradeTimeout)],
# AT
deploy_at => {
url => 'at',
required => [qw(name description aTType tags creationBytes amount)],
optional => [qw(assetId)],
key_name => 'creatorPublicKey',
defaults => { tradeTimeout => 10800 },
defaults => { assetId => 0 },
},
# Cross-chain trading
create_trade => {
url => 'crosschain/tradebot/create',
required => [qw(qortAmount fundingQortAmount foreignAmount receivingAddress)],
optional => [qw(tradeTimeout foreignBlockchain)],
key_name => 'creatorPublicKey',
defaults => { tradeTimeout => 1440, foreignBlockchain => 'LITECOIN' },
},
trade_recipient => {
url => 'crosschain/tradeoffer/recipient',
@ -196,7 +220,7 @@ if (@ARGV < @required + 1) {
my $priv_key = shift @ARGV;
my $account = account($priv_key);
my $account;
my $raw;
if ($tx_type ne 'sign') {
@ -215,6 +239,8 @@ if ($tx_type ne 'sign') {
%extras = (%extras, @ARGV);
$account = account($priv_key, %extras);
$raw = build_raw($tx_type, $account, %extras);
printf "Raw: %s\n", $raw if $opt{d} || (!$opt{s} && !$opt{p});
@ -229,7 +255,7 @@ if ($tx_type ne 'sign') {
}
if ($opt{s}) {
my $signed = sign($account->{private}, $raw);
my $signed = sign($priv_key, $raw);
printf "Signed: %s\n", $signed if $opt{d} || $tx_type eq 'sign';
if ($opt{p}) {
@ -246,15 +272,25 @@ if ($opt{s}) {
}
sub account {
my ($creator) = @_;
my ($privkey, %extras) = @_;
my $account = { private => $creator };
$account->{public} = api('utils/publickey', $creator);
$account->{address} = api('addresses/convert/{publickey}', '', '{publickey}', $account->{public});
my $account = { private => $privkey };
$account->{public} = $extras{publickey} || priv_to_pub($privkey);
$account->{address} = $extras{address} || pubkey_to_address($account->{public}); # api('addresses/convert/{publickey}', '', '{publickey}', $account->{public});
return $account;
}
sub priv_to_pub {
my ($privkey) = @_;
if ($OPENSSL_PRIV_TO_PUB) {
return openssl_priv_to_pub($privkey);
} else {
return api('utils/publickey', $privkey);
}
}
sub build_raw {
my ($type, $account, %extras) = @_;
@ -306,6 +342,21 @@ sub build_raw {
sub sign {
my ($private, $raw) = @_;
if (-x "$OPENSSL_SIGN") {
my $private_hex = decode_base58($private);
chomp $private_hex;
my $raw_hex = decode_base58($raw);
chomp $raw_hex;
my $sig = `${OPENSSL_SIGN} ${private_hex} ${raw_hex}`;
chomp $sig;
my $sig58 = encode_base58(${raw_hex} . ${sig});
chomp $sig58;
return $sig58;
}
my $json = <<" __JSON__";
{
"privateKey": "$private",
@ -344,7 +395,14 @@ sub api {
my $curl = "curl --silent --output - --url '$BASE_URL/$url'";
if (defined $postdata && $postdata ne '') {
$postdata =~ tr|\n| |s;
$curl .= " --header 'Content-Type: application/json' --data-binary '$postdata'";
if ($postdata =~ /^\s*\{/so) {
$curl .= " --header 'Content-Type: application/json'";
} else {
$curl .= " --header 'Content-Type: text/plain'";
}
$curl .= " --data-binary '$postdata'";
$method = 'POST';
}
my $response = `$curl 2>/dev/null`;
@ -356,3 +414,87 @@ sub api {
return $response;
}
sub encode_base58 {
use integer;
my @in = map { hex($_) } ($_[0] =~ /(..)/g);
my $bzeros = length($1) if join('', @in) =~ /^(0*)/;
my @out;
my $size = 2 * scalar @in;
for my $c (@in) {
for (my $j = $size; $j--; ) {
$c += 256 * ($out[$j] // 0);
$out[$j] = $c % 58;
$c /= 58;
}
}
my $out = join('', map { $reverseb58{$_} } @out);
return $1 if $out =~ /(1{$bzeros}[^1].*)/;
return $1 if $out =~ /(1{$bzeros})/;
die "Invalid base58!\n";
}
sub decode_base58 {
use integer;
my @out;
my $azeros = length($1) if $_[0] =~ /^(1*)/;
for my $c ( map { $b58{$_} } $_[0] =~ /./g ) {
die("Invalid character!\n") unless defined $c;
for (my $j = length($_[0]); $j--; ) {
$c += 58 * ($out[$j] // 0);
$out[$j] = $c % 256;
$c /= 256;
}
}
shift @out while @out && $out[0] == 0;
unshift(@out, (0) x $azeros);
return sprintf('%02x' x @out, @out);
}
sub openssl_priv_to_pub {
my ($privkey) = @_;
my $privkey_hex = decode_base58($privkey);
my $key_type = "04"; # hex
my $length = "20"; # hex
my $asn1 = <<"__ASN1__";
asn1=SEQUENCE:private_key
[private_key]
version=INTEGER:0
included=SEQUENCE:key_info
raw=FORMAT:HEX,OCTETSTRING:${key_type}${length}${privkey_hex}
[key_info]
type=OBJECT:ED25519
__ASN1__
my $output = `echo "${asn1}" | openssl asn1parse -i -genconf - -out - | openssl pkey -in - -inform der -noout -text_pub`;
# remove colons
my $pubkey = '';
$pubkey .= $1 while $output =~ m/([0-9a-f]{2})(?::|$)/g;
return encode_base58($pubkey);
}
sub pubkey_to_address {
my ($pubkey) = @_;
my $pubkey_hex = decode_base58($pubkey);
my $pubkey_raw = pack('H*', $pubkey_hex);
my $pkh_hex = Crypt::RIPEMD160->hexhash(sha256($pubkey_raw));
$pkh_hex =~ tr/ //ds;
my $version = '3a'; # hex
my $raw = pack('H*', $version . $pkh_hex);
my $chksum = substr(sha256_hex(sha256($raw)), 0, 8);
return encode_base58($version . $pkh_hex . $chksum);
}