crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Merge pull request #1 from AlphaX-Projects/arbitrary-resources-cache

Browse Source 
Arbitrary resources cache
This commit is contained in:
AlphaX-Projects 2023-11-04 16:21:24 +01:00 committed by GitHub
commit 050886a496
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 1514 additions and 350 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Q-Apps.md
View File

@ -375,11 +375,15 @@ let res = await qortalRequest({
prefix: false, // Optional - if true, only the beginning of fields are matched in all of the above filters
exactMatchNames: true, // Optional - if true, partial name matches are excluded
default: false, // Optional - if true, only resources without identifiers are returned
mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST
minLevel: 1, // Optional - whether to filter results by minimum account level
includeStatus: false, // Optional - will take time to respond, so only request if necessary
includeMetadata: false, // Optional - will take time to respond, so only request if necessary
nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list
followedOnly: false, // Optional - include followed names only
excludeBlocked: false, // Optional - exclude blocked content
// before: 1683546000000, // Optional - limit to resources created before timestamp
// after: 1683546000000, // Optional - limit to resources created after timestamp
limit: 100,
offset: 0,
reverse: true
@ -395,12 +399,16 @@ let res = await qortalRequest({
identifier: "search query goes here", // Optional - searches only the "identifier" field
names: ["QortalDemo", "crowetic", "AlphaX"], // Optional - searches only the "name" field for any of the supplied names
prefix: false, // Optional - if true, only the beginning of fields are matched in all of the above filters
exactMatchNames: true, // Optional - if true, partial name matches are excluded
default: false, // Optional - if true, only resources without identifiers are returned
mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST
includeStatus: false, // Optional - will take time to respond, so only request if necessary
includeMetadata: false, // Optional - will take time to respond, so only request if necessary
nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list
followedOnly: false, // Optional - include followed names only
excludeBlocked: false, // Optional - exclude blocked content
// before: 1683546000000, // Optional - limit to resources created before timestamp
// after: 1683546000000, // Optional - limit to resources created after timestamp
limit: 100,
offset: 0,
reverse: true

6
src/main/java/org/qortal/api/SearchMode.java Normal file
View File

@ -0,0 +1,6 @@
package org.qortal.api;
public enum SearchMode {
LATEST,
ALL;
}

40
src/main/java/org/qortal/api/gateway/resource/GatewayResource.java
View File

@ -3,6 +3,8 @@ package org.qortal.api.gateway.resource;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.commons.lang3.StringUtils;
import org.qortal.api.ApiError;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
@ -11,6 +13,9 @@ import org.qortal.arbitrary.ArbitraryDataRenderer;
import org.qortal.arbitrary.ArbitraryDataResource;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
@ -31,36 +36,12 @@ public class GatewayResource {
@Context HttpServletResponse response;
@Context ServletContext context;
/**
* We need to allow resource status checking (and building) via the gateway, as the node's API port
* may not be forwarded and will almost certainly not be authenticated. Since gateways allow for
* all resources to be loaded except those that are blocked, there is no need for authentication.
*/
@GET
@Path("/arbitrary/resource/status/{service}/{name}")
public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service,
@PathParam("name") String name,
@QueryParam("build") Boolean build) {
return this.getStatus(service, name, null, build);
}
@GET
@Path("/arbitrary/resource/status/{service}/{name}/{identifier}")
public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("build") Boolean build) {
return this.getStatus(service, name, identifier, build);
}
private ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) {
// If "build=true" has been specified in the query string, build the resource before returning its status
if (build != null && build == true) {
ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null);
try {
ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null);
if (!reader.isBuilding()) {
reader.loadSynchronously(false);
}
@ -69,8 +50,13 @@ public class GatewayResource {
}
}
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.getStatus(false);
try (final Repository repository = RepositoryManager.getRepository()) {
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.getStatus(repository);
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}

150
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -45,6 +45,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Category;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.controller.arbitrary.ArbitraryDataCacheManager;
import org.qortal.controller.arbitrary.ArbitraryDataRenderManager;
import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.controller.arbitrary.ArbitraryMetadataManager;
@ -86,12 +87,12 @@ public class ArbitraryResource {
"- If default is set to true, only resources without identifiers will be returned.",
responses = {
@ApiResponse(
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
)
}
)
@ApiErrors({ApiError.REPOSITORY_ISSUE})
public List<ArbitraryResourceInfo> getResources(
public List<ArbitraryResourceData> getResources(
@QueryParam("service") Service service,
@QueryParam("name") String name,
@QueryParam("identifier") String identifier,
@ -133,20 +134,14 @@ public class ArbitraryResource {
}
}
List<ArbitraryResourceInfo> resources = repository.getArbitraryRepository()
.getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse);
List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
.getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked,
includeMetadata, includeStatus, limit, offset, reverse);
if (resources == null) {
return new ArrayList<>();
}
if (includeStatus != null && includeStatus) {
resources = ArbitraryTransactionUtils.addStatusToResources(resources);
}
if (includeMetadata != null && includeMetadata) {
resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
}
return resources;
} catch (DataException e) {
@ -161,24 +156,30 @@ public class ArbitraryResource {
"If default is set to true, only resources without identifiers will be returned.",
responses = {
@ApiResponse(
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
)
}
)
@ApiErrors({ApiError.REPOSITORY_ISSUE})
public List<ArbitraryResourceInfo> searchResources(
public List<ArbitraryResourceData> searchResources(
@QueryParam("service") Service service,
@Parameter(description = "Query (searches both name and identifier fields)") @QueryParam("query") String query,
@Parameter(description = "Query (searches name, identifier, title and description fields)") @QueryParam("query") String query,
@Parameter(description = "Identifier (searches identifier field only)") @QueryParam("identifier") String identifier,
@Parameter(description = "Name (searches name field only)") @QueryParam("name") List<String> names,
@Parameter(description = "Title (searches title metadata field only)") @QueryParam("title") String title,
@Parameter(description = "Description (searches description metadata field only)") @QueryParam("description") String description,
@Parameter(description = "Prefix only (if true, only the beginning of fields are matched)") @QueryParam("prefix") Boolean prefixOnly,
@Parameter(description = "Exact match names only (if true, partial name matches are excluded)") @QueryParam("exactmatchnames") Boolean exactMatchNamesOnly,
@Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource,
@Parameter(description = "Search mode") @QueryParam("mode") SearchMode mode,
@Parameter(description = "Min level") @QueryParam("minlevel") Integer minLevel,
@Parameter(description = "Filter names by list (exact matches only)") @QueryParam("namefilter") String nameListFilter,
@Parameter(description = "Include followed names only") @QueryParam("followedonly") Boolean followedOnly,
@Parameter(description = "Exclude blocked content") @QueryParam("excludeblocked") Boolean excludeBlocked,
@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
@Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata,
@Parameter(description = "Creation date before timestamp") @QueryParam("before") Long before,
@Parameter(description = "Creation date after timestamp") @QueryParam("after") Long after,
@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) {
@ -206,20 +207,15 @@ public class ArbitraryResource {
names = null;
}
List<ArbitraryResourceInfo> resources = repository.getArbitraryRepository()
.searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse);
List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
.searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly,
exactMatchNames, defaultRes, mode, minLevel, followedOnly, excludeBlocked, includeMetadata, includeStatus,
before, after, limit, offset, reverse);
if (resources == null) {
return new ArrayList<>();
}
if (includeStatus != null && includeStatus) {
resources = ArbitraryTransactionUtils.addStatusToResources(resources);
}
if (includeMetadata != null && includeMetadata) {
resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
}
return resources;
} catch (DataException e) {
@ -238,16 +234,14 @@ public class ArbitraryResource {
)
}
)
@SecurityRequirement(name = "apiKey")
public ArbitraryResourceStatus getDefaultResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") Service service,
public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service,
@PathParam("name") String name,
@QueryParam("build") Boolean build) {
if (!Settings.getInstance().isQDNAuthBypassEnabled())
Security.requirePriorAuthorizationOrApiKey(request, name, service, null, apiKey);
Security.requirePriorAuthorizationOrApiKey(request, name, service, null, null);
return ArbitraryTransactionUtils.getStatus(service, name, null, build);
return ArbitraryTransactionUtils.getStatus(service, name, null, build, true);
}
@GET
@ -261,14 +255,12 @@ public class ArbitraryResource {
)
}
)
@SecurityRequirement(name = "apiKey")
public FileProperties getResourceProperties(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier) {
public FileProperties getResourceProperties(@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier) {
if (!Settings.getInstance().isQDNAuthBypassEnabled())
Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, apiKey);
Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, null);
return this.getFileProperties(service, name, identifier);
}
@ -284,17 +276,15 @@ public class ArbitraryResource {
)
}
)
@SecurityRequirement(name = "apiKey")
public ArbitraryResourceStatus getResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") Service service,
public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("build") Boolean build) {
if (!Settings.getInstance().isQDNAuthBypassEnabled())
Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, apiKey);
Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, null);
return ArbitraryTransactionUtils.getStatus(service, name, identifier, build);
return ArbitraryTransactionUtils.getStatus(service, name, identifier, build, true);
}
@ -479,21 +469,19 @@ public class ArbitraryResource {
summary = "List arbitrary resources hosted by this node",
responses = {
@ApiResponse(
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
)
}
)
@ApiErrors({ApiError.REPOSITORY_ISSUE})
public List<ArbitraryResourceInfo> getHostedResources(
public List<ArbitraryResourceData> getHostedResources(
@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
@Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata,
@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
@QueryParam("query") String query) {
Security.checkApiCallAllowed(request);
List<ArbitraryResourceInfo> resources = new ArrayList<>();
List<ArbitraryResourceData> resources = new ArrayList<>();
try (final Repository repository = RepositoryManager.getRepository()) {
@ -509,22 +497,15 @@ public class ArbitraryResource {
if (transactionData.getService() == null) {
continue;
}
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
arbitraryResourceInfo.name = transactionData.getName();
arbitraryResourceInfo.service = transactionData.getService();
arbitraryResourceInfo.identifier = transactionData.getIdentifier();
if (!resources.contains(arbitraryResourceInfo)) {
resources.add(arbitraryResourceInfo);
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.name = transactionData.getName();
arbitraryResourceData.service = transactionData.getService();
arbitraryResourceData.identifier = transactionData.getIdentifier();
if (!resources.contains(arbitraryResourceData)) {
resources.add(arbitraryResourceData);
}
}
if (includeStatus != null && includeStatus) {
resources = ArbitraryTransactionUtils.addStatusToResources(resources);
}
if (includeMetadata != null && includeMetadata) {
resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
}
return resources;
} catch (DataException e) {
@ -551,8 +532,14 @@ public class ArbitraryResource {
@PathParam("identifier") String identifier) {
Security.checkApiCallAllowed(request);
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.delete(false);
try (final Repository repository = RepositoryManager.getRepository()) {
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.delete(repository, false);
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@POST
@ -644,9 +631,7 @@ public class ArbitraryResource {
)
}
)
@SecurityRequirement(name = "apiKey")
public HttpServletResponse get(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") Service service,
public HttpServletResponse get(@PathParam("service") Service service,
@PathParam("name") String name,
@QueryParam("filepath") String filepath,
@QueryParam("encoding") String encoding,
@ -679,9 +664,7 @@ public class ArbitraryResource {
)
}
)
@SecurityRequirement(name = "apiKey")
public HttpServletResponse get(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") Service service,
public HttpServletResponse get(@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("filepath") String filepath,
@ -692,7 +675,7 @@ public class ArbitraryResource {
// Authentication can be bypassed in the settings, for those running public QDN nodes
if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
Security.checkApiCallAllowed(request, apiKey);
Security.checkApiCallAllowed(request, null);
}
return this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts);
@ -717,7 +700,6 @@ public class ArbitraryResource {
)
}
)
@SecurityRequirement(name = "apiKey")
public ArbitraryResourceMetadata getMetadata(@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier) {
@ -1127,6 +1109,36 @@ public class ArbitraryResource {
}
@POST
@Path("/resources/cache/rebuild")
@Operation(
summary = "Rebuild arbitrary resources cache from transactions",
responses = {
@ApiResponse(
description = "true on success",
content = @Content(
mediaType = MediaType.TEXT_PLAIN,
schema = @Schema(
type = "boolean"
)
)
)
}
)
@SecurityRequirement(name = "apiKey")
public String rebuildCache(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
Security.checkApiCallAllowed(request);
try (final Repository repository = RepositoryManager.getRepository()) {
ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, true);
return "true";
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
}
}
// Shared methods
private String preview(String directoryPath, Service service) {
@ -1275,8 +1287,8 @@ public class ArbitraryResource {
private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
try {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
int attempts = 0;
if (maxAttempts == null) {
@ -1382,8 +1394,8 @@ public class ArbitraryResource {
}
private FileProperties getFileProperties(Service service, String name, String identifier) {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
try {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
arbitraryDataReader.loadSynchronously(false);
java.nio.file.Path outputPath = arbitraryDataReader.getFilePath();
if (outputPath == null) {

4
src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
View File

@ -4,6 +4,7 @@ import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.arbitrary.misc.Service;
import org.qortal.repository.DataException;
import org.qortal.utils.ArbitraryTransactionUtils;
import org.qortal.utils.NTP;
import java.io.IOException;
@ -51,6 +52,9 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
arbitraryDataReader.loadSynchronously(true);
} finally {
this.buildEndTimestamp = NTP.getTime();
// Update status after build
ArbitraryTransactionUtils.getStatus(service, resourceId, identifier, false, true);
}
}

13
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@ -66,7 +66,7 @@ public class ArbitraryDataReader {
// TODO: all builds could be handled by the build queue (even synchronous ones), to avoid the need for this
private static Map<String, Long> inProgress = Collections.synchronizedMap(new HashMap<>());
public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) throws DataException {
// Ensure names are always lowercase
if (resourceIdType == ResourceIdType.NAME) {
resourceId = resourceId.toLowerCase();
@ -90,11 +90,16 @@ public class ArbitraryDataReader {
this.canRequestMissingFiles = true;
}
private Path buildWorkingPath() {
private Path buildWorkingPath() throws DataException {
// Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
String baseDir = Settings.getInstance().getTempDataPath();
String identifier = this.identifier != null ? this.identifier : "default";
return Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier);
try {
return Paths.get(baseDir, "reader", this.resourceIdType.toString(), StringUtils.sanitizeString(this.resourceId), this.service.toString(), StringUtils.sanitizeString(identifier));
} catch (InvalidPathException e) {
throw new DataException(String.format("Invalid path: %s", e.getMessage()));
}
}
public boolean isCachedDataAvailable() {
@ -240,7 +245,7 @@ public class ArbitraryDataReader {
try {
Files.createDirectories(this.workingPath);
} catch (IOException e) {
throw new DataException("Unable to create temp directory");
throw new DataException(String.format("Unable to create temp directory %s: %s", this.workingPath, e.getMessage()));
}
}

6
src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
View File

@ -76,9 +76,11 @@ public class ArbitraryDataRenderer {
return ArbitraryDataRenderer.getResponse(response, 500, "QDN is disabled in settings");
}
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, identifier);
arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
ArbitraryDataReader arbitraryDataReader;
try {
arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, identifier);
arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
if (!arbitraryDataReader.isCachedDataAvailable()) {
// If async is requested, show a loading screen whilst build is in progress
if (async) {

99
src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java
View File

@ -9,6 +9,7 @@ import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.repository.DataException;
@ -57,15 +58,39 @@ public class ArbitraryDataResource {
this.identifier = identifier;
}
public ArbitraryResourceStatus getStatus(boolean quick) {
// Calculate the chunk counts
// Avoid this for "quick" statuses, to speed things up
if (!quick) {
this.calculateChunkCounts();
public ArbitraryResourceStatus getStatusAndUpdateCache(boolean updateCache) {
ArbitraryResourceStatus arbitraryResourceStatus = null;
if (!this.exists) {
return new ArbitraryResourceStatus(Status.NOT_PUBLISHED, this.localChunkCount, this.totalChunkCount);
try (final Repository repository = RepositoryManager.getRepository()) {
arbitraryResourceStatus = this.getStatus(repository);
if (updateCache) {
// Update cache if possible
ArbitraryResourceStatus.Status status = arbitraryResourceStatus != null ? arbitraryResourceStatus.getStatus() : null;
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(this.service, this.resourceId, this.identifier);
repository.discardChanges();
repository.getArbitraryRepository().setStatus(arbitraryResourceData, status);
repository.saveChanges();
}
} catch (DataException e) {
LOGGER.info("Unable to update status cache for resource {}: {}", this.toString(), e.getMessage());
}
return arbitraryResourceStatus;
}
/**
* Get current status of resource
*
* @param repository
* @return the resource's status
*/
public ArbitraryResourceStatus getStatus(Repository repository) {
// Calculate the chunk counts
this.calculateChunkCounts(repository);
if (!this.exists) {
return new ArbitraryResourceStatus(Status.NOT_PUBLISHED, this.localChunkCount, this.totalChunkCount);
}
if (resourceIdType != ResourceIdType.NAME) {
@ -86,18 +111,23 @@ public class ArbitraryDataResource {
}
// Firstly check the cache to see if it's already built
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(
resourceId, resourceIdType, service, identifier);
if (arbitraryDataReader.isCachedDataAvailable()) {
return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount);
try {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(
resourceId, resourceIdType, service, identifier);
if (arbitraryDataReader.isCachedDataAvailable()) {
return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount);
}
} catch (DataException e) {
// Assume no usable data
return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount);
}
// Check if we have all data locally for this resource
if (!this.allFilesDownloaded()) {
if (this.isDownloading()) {
if (!this.allFilesDownloaded(repository)) {
if (this.isDownloading(repository)) {
return new ArbitraryResourceStatus(Status.DOWNLOADING, this.localChunkCount, this.totalChunkCount);
}
else if (this.isDataPotentiallyAvailable()) {
else if (this.isDataPotentiallyAvailable(repository)) {
return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount);
}
return new ArbitraryResourceStatus(Status.MISSING_DATA, this.localChunkCount, this.totalChunkCount);
@ -139,9 +169,9 @@ public class ArbitraryDataResource {
return null;
}
public boolean delete(boolean deleteMetadata) {
public boolean delete(Repository repository, boolean deleteMetadata) {
try {
this.fetchTransactions();
this.fetchTransactions(repository);
if (this.transactions == null) {
return false;
}
@ -190,7 +220,7 @@ public class ArbitraryDataResource {
}
}
private boolean allFilesDownloaded() {
private boolean allFilesDownloaded(Repository repository) {
// Use chunk counts to speed things up if we can
if (this.localChunkCount != null && this.totalChunkCount != null &&
this.localChunkCount >= this.totalChunkCount) {
@ -198,7 +228,7 @@ public class ArbitraryDataResource {
}
try {
this.fetchTransactions();
this.fetchTransactions(repository);
if (this.transactions == null) {
return false;
}
@ -218,9 +248,14 @@ public class ArbitraryDataResource {
}
}
private void calculateChunkCounts() {
/**
* Calculate chunk counts of a resource
*
* @param repository optional - a new instance will be created if null
*/
private void calculateChunkCounts(Repository repository) {
try {
this.fetchTransactions();
this.fetchTransactions(repository);
if (this.transactions == null) {
this.exists = false;
this.localChunkCount = 0;
@ -245,9 +280,9 @@ public class ArbitraryDataResource {
} catch (DataException e) {}
}
private boolean isRateLimited() {
private boolean isRateLimited(Repository repository) {
try {
this.fetchTransactions();
this.fetchTransactions(repository);
if (this.transactions == null) {
return true;
}
@ -271,9 +306,9 @@ public class ArbitraryDataResource {
* This is only used to give an indication to the user of progress
* @return - whether data might be available on the network
*/
private boolean isDataPotentiallyAvailable() {
private boolean isDataPotentiallyAvailable(Repository repository) {
try {
this.fetchTransactions();
this.fetchTransactions(repository);
if (this.transactions == null) {
return false;
}
@ -306,9 +341,9 @@ public class ArbitraryDataResource {
* This is only used to give an indication to the user of progress
* @return - whether we are trying to download the resource
*/
private boolean isDownloading() {
private boolean isDownloading(Repository repository) {
try {
this.fetchTransactions();
this.fetchTransactions(repository);
if (this.transactions == null) {
return false;
}
@ -339,15 +374,19 @@ public class ArbitraryDataResource {
}
private void fetchTransactions() throws DataException {
/**
* Fetch relevant arbitrary transactions for resource
*
* @param repository
* @throws DataException
*/
private void fetchTransactions(Repository repository) throws DataException {
if (this.transactions != null && !this.transactions.isEmpty()) {
// Already fetched
return;
}
try (final Repository repository = RepositoryManager.getRepository()) {
try {
// Get the most recent PUT
ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
.getLatestTransaction(this.resourceId, this.service, ArbitraryTransactionData.Method.PUT, this.identifier);

3
src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java
View File

@ -117,8 +117,9 @@ public class ArbitraryDataTransactionBuilder {
}
private Method determineMethodAutomatically() throws DataException {
ArbitraryDataReader reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier);
ArbitraryDataReader reader;
try {
reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier);
reader.loadSynchronously(true);
} catch (Exception e) {
// Catch all exceptions if the existing resource cannot be loaded first time

5
src/main/java/org/qortal/arbitrary/misc/Category.java
View File

@ -67,9 +67,12 @@ public enum Category {
/**
* Same as valueOf() but with fallback to UNCATEGORIZED if there's no match
* @param name
* @return a Category (using UNCATEGORIZED if no match found)
* @return a Category (using UNCATEGORIZED if no match found), or null if null name passed
*/
public static Category uncategorizedValueOf(String name) {
if (name == null) {
return null;
}
try {
return Category.valueOf(name);
}

27
src/main/java/org/qortal/controller/Controller.java
View File

@ -47,6 +47,7 @@ import org.qortal.data.block.BlockData;
import org.qortal.data.block.BlockSummaryData;
import org.qortal.data.naming.NameData;
import org.qortal.data.network.PeerData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ChatTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.event.Event;
@ -403,6 +404,7 @@ public class Controller extends Thread {
try (final Repository repository = RepositoryManager.getRepository()) {
RepositoryManager.rebuildTransactionSequences(repository);
ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, false);
}
} catch (DataException e) {
// If exception has no cause or message then repository is in use by some other process.
@ -448,6 +450,13 @@ public class Controller extends Thread {
Gui.getInstance().fatalError("Database upgrade needed", "Please restart the core to complete the upgrade process.");
return;
}
if (ArbitraryDataCacheManager.getInstance().needsArbitraryResourcesCacheRebuild(repository)) {
// Don't allow the node to start if arbitrary resources cache hasn't been built yet
// This is needed to handle a case when bootstrapping
LOGGER.error("Database upgrade needed. Please restart the core to complete the upgrade process.");
Gui.getInstance().fatalError("Database upgrade needed", "Please restart the core to complete the upgrade process.");
return;
}
} catch (DataException e) {
LOGGER.error("Error checking transaction sequences in repository", e);
return;
@ -496,6 +505,7 @@ public class Controller extends Thread {
LOGGER.info("Starting arbitrary-transaction controllers");
ArbitraryDataManager.getInstance().start();
ArbitraryDataFileManager.getInstance().start();
ArbitraryDataCacheManager.getInstance().start();
ArbitraryDataBuildManager.getInstance().start();
ArbitraryDataCleanupManager.getInstance().start();
ArbitraryDataStorageManager.getInstance().start();
@ -907,6 +917,7 @@ public class Controller extends Thread {
if (now >= transaction.getDeadline()) {
LOGGER.debug(() -> String.format("Deleting expired, unconfirmed transaction %s", Base58.encode(transactionData.getSignature())));
repository.getTransactionRepository().delete(transactionData);
this.onExpiredTransaction(transactionData);
deletedCount++;
}
}
@ -949,6 +960,7 @@ public class Controller extends Thread {
LOGGER.info("Shutting down arbitrary-transaction controllers");
ArbitraryDataManager.getInstance().shutdown();
ArbitraryDataFileManager.getInstance().shutdown();
ArbitraryDataCacheManager.getInstance().shutdown();
ArbitraryDataBuildManager.getInstance().shutdown();
ArbitraryDataCleanupManager.getInstance().shutdown();
ArbitraryDataStorageManager.getInstance().shutdown();
@ -1219,6 +1231,21 @@ public class Controller extends Thread {
});
}
/**
* Callback for when we've deleted an expired, unconfirmed transaction.
* <p>
* @implSpec performs actions in a new thread
*/
public void onExpiredTransaction(TransactionData transactionData) {
this.callbackExecutor.execute(() -> {
// If this is an ARBITRARY transaction, we may need to update the cache
if (transactionData.getType() == TransactionType.ARBITRARY) {
ArbitraryDataManager.getInstance().onExpiredArbitraryTransaction((ArbitraryTransactionData)transactionData);
}
});
}
public void onPeerHandshakeCompleted(Peer peer) {
// Only send if outbound
if (peer.isOutbound()) {

2
src/main/java/org/qortal/controller/PirateChainWalletController.java
View File

@ -187,7 +187,7 @@ public class PirateChainWalletController extends Thread {
// Check its status
ArbitraryResourceStatus status = ArbitraryTransactionUtils.getStatus(
t.getService(), t.getName(), t.getIdentifier(), false);
t.getService(), t.getName(), t.getIdentifier(), false, true);
if (status.getStatus() != ArbitraryResourceStatus.Status.READY) {
LOGGER.info("Not ready yet: {}", status.getTitle());

12
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java
View File

@ -5,13 +5,17 @@ import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.repository.DataException;
import org.qortal.utils.ArbitraryTransactionUtils;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.util.Comparator;
import java.util.Map;
import static org.qortal.data.arbitrary.ArbitraryResourceStatus.Status.*;
public class ArbitraryDataBuilderThread implements Runnable {
@ -69,6 +73,14 @@ public class ArbitraryDataBuilderThread implements Runnable {
continue;
}
// Get status before build
ArbitraryResourceStatus arbitraryResourceStatus = ArbitraryTransactionUtils.getStatus(queueItem.getService(), queueItem.getResourceId(), queueItem.getIdentifier(), false, true);
if (arbitraryResourceStatus.getStatus() == NOT_PUBLISHED) {
// No point in building a non-existent resource
this.removeFromQueue(queueItem);
continue;
}
// Set the start timestamp, to prevent other threads from building it at the same time
queueItem.prepareForBuild();
}

236
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java Normal file
View File

@ -0,0 +1,236 @@
package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.gui.SplashFrame;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.utils.Base58;
import java.util.*;
public class ArbitraryDataCacheManager extends Thread {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCacheManager.class);
private static ArbitraryDataCacheManager instance;
private volatile boolean isStopping = false;
/** Queue of arbitrary transactions that require cache updates */
private final List<ArbitraryTransactionData> updateQueue = Collections.synchronizedList(new ArrayList<>());
public static synchronized ArbitraryDataCacheManager getInstance() {
if (instance == null) {
instance = new ArbitraryDataCacheManager();
}
return instance;
}
@Override
public void run() {
Thread.currentThread().setName("Arbitrary Data Cache Manager");
try {
while (!Controller.isStopping()) {
Thread.sleep(500L);
// Process queue
processResourceQueue();
}
} catch (InterruptedException e) {
// Fall through to exit thread
}
// Clear queue before terminating thread
processResourceQueue();
}
public void shutdown() {
isStopping = true;
this.interrupt();
}
private void processResourceQueue() {
if (this.updateQueue.isEmpty()) {
// Nothing to do
return;
}
try (final Repository repository = RepositoryManager.getRepository()) {
// Take a snapshot of resourceQueue, so we don't need to lock it while processing
List<ArbitraryTransactionData> resourceQueueCopy = List.copyOf(this.updateQueue);
for (ArbitraryTransactionData transactionData : resourceQueueCopy) {
// Best not to return when controller is stopping, as ideally we need to finish processing
LOGGER.debug(() -> String.format("Processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature())));
// Remove from the queue regardless of outcome
this.updateQueue.remove(transactionData);
// Update arbitrary resource caches
try {
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCache(repository);
arbitraryTransaction.updateArbitraryMetadataCache(repository);
repository.saveChanges();
// Update status as separate commit, as this is more prone to failure
arbitraryTransaction.updateArbitraryResourceStatus(repository);
repository.saveChanges();
LOGGER.debug(() -> String.format("Finished processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature())));
} catch (DataException e) {
repository.discardChanges();
LOGGER.error("Repository issue while updating arbitrary resource caches", e);
}
}
} catch (DataException e) {
LOGGER.error("Repository issue while processing arbitrary resource cache updates", e);
}
}
public void addToUpdateQueue(ArbitraryTransactionData transactionData) {
this.updateQueue.add(transactionData);
LOGGER.debug(() -> String.format("Transaction %.8s added to queue", Base58.encode(transactionData.getSignature())));
}
public boolean needsArbitraryResourcesCacheRebuild(Repository repository) throws DataException {
// Check if we have an entry in the cache for the oldest ARBITRARY transaction with a name
List<ArbitraryTransactionData> oldestCacheableTransactions = repository.getArbitraryRepository().getArbitraryTransactions(true, 1, 0, false);
if (oldestCacheableTransactions == null || oldestCacheableTransactions.isEmpty()) {
// No relevant arbitrary transactions yet on this chain
LOGGER.debug("No relevant arbitrary transactions exist to build cache from");
return false;
}
// We have an arbitrary transaction, so check if it's in the cache
ArbitraryTransactionData txn = oldestCacheableTransactions.get(0);
ArbitraryResourceData cachedResource = repository.getArbitraryRepository().getArbitraryResource(txn.getService(), txn.getName(), txn.getIdentifier());
if (cachedResource != null) {
// Earliest resource exists in the cache, so assume it has been built.
// We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so
// we shouldn't ever be left in a partially rebuilt state.
LOGGER.debug("Arbitrary resources cache already built");
return false;
}
return true;
}
public boolean buildArbitraryResourcesCache(Repository repository, boolean forceRebuild) throws DataException {
if (Settings.getInstance().isLite()) {
// Lite nodes have no blockchain
return false;
}
try {
// Skip if already built
if (!needsArbitraryResourcesCacheRebuild(repository) && !forceRebuild) {
LOGGER.debug("Arbitrary resources cache already built");
return false;
}
LOGGER.info("Building arbitrary resources cache...");
SplashFrame.getInstance().updateStatus("Building QDN cache - please wait...");
final int batchSize = 100;
int offset = 0;
// Loop through all ARBITRARY transactions, and determine latest state
while (!Controller.isStopping()) {
LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false);
if (signatures.isEmpty()) {
// Complete
break;
}
// Expand signatures to transactions
for (byte[] signature : signatures) {
ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository
.getTransactionRepository().fromSignature(signature);
if (transactionData.getService() == null) {
// Unsupported service - ignore this resource
continue;
}
// Update arbitrary resource caches
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCache(repository);
arbitraryTransaction.updateArbitraryMetadataCache(repository);
repository.saveChanges();
}
offset += batchSize;
}
// Now refresh all statuses
refreshArbitraryStatuses(repository);
LOGGER.info("Completed build of arbitrary resources cache.");
return true;
}
catch (DataException e) {
LOGGER.info("Unable to build arbitrary resources cache: {}. The database may have been left in an inconsistent state.", e.getMessage());
// Throw an exception so that the node startup is halted, allowing for a retry next time.
repository.discardChanges();
throw new DataException("Build of arbitrary resources cache failed.");
}
}
private boolean refreshArbitraryStatuses(Repository repository) throws DataException {
try {
LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions...");
SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait...");
final int batchSize = 100;
int offset = 0;
// Loop through all ARBITRARY transactions, and determine latest state
while (!Controller.isStopping()) {
LOGGER.info("Fetching hosted transactions {} - {}", offset, offset+batchSize-1);
List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, batchSize, offset);
if (hostedTransactions.isEmpty()) {
// Complete
break;
}
// Loop through hosted transactions
for (ArbitraryTransactionData transactionData : hostedTransactions) {
// Determine status and update cache
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceStatus(repository);
repository.saveChanges();
}
offset += batchSize;
}
LOGGER.info("Completed refresh of arbitrary resource statuses.");
return true;
}
catch (DataException e) {
LOGGER.info("Unable to refresh arbitrary resource statuses: {}. The database may have been left in an inconsistent state.", e.getMessage());
// Throw an exception so that the node startup is halted, allowing for a retry next time.
repository.discardChanges();
throw new DataException("Refresh of arbitrary resource statuses failed.");
}
}
}

15
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java
View File

@ -146,7 +146,7 @@ public class ArbitraryDataFileManager extends Thread {
if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
LOGGER.debug("Requesting data file {} from peer {}", hash58, peer);
Long startTime = NTP.getTime();
ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, null, signature, hash, null);
ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, null, arbitraryTransactionData, signature, hash, null);
Long endTime = NTP.getTime();
if (receivedArbitraryDataFile != null) {
LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFile.getHash58(), peer, (endTime-startTime));
@ -191,7 +191,7 @@ public class ArbitraryDataFileManager extends Thread {
return receivedAtLeastOneFile;
}
private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer, ArbitraryTransactionData arbitraryTransactionData, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature);
boolean fileAlreadyExists = existingFile.exists();
String hash58 = Base58.encode(hash);
@ -250,6 +250,13 @@ public class ArbitraryDataFileManager extends Thread {
}
}
// If this is a metadata file then we need to update the cache
if (arbitraryTransactionData != null && arbitraryTransactionData.getMetadataHash() != null) {
if (Arrays.equals(arbitraryTransactionData.getMetadataHash(), hash)) {
ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
}
}
return arbitraryDataFile;
}
@ -585,7 +592,9 @@ public class ArbitraryDataFileManager extends Thread {
// Forward the message to this peer
LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58);
this.fetchArbitraryDataFile(peerToAsk, peer, signature, hash, message);
// No need to pass arbitraryTransactionData below because this is only used for metadata caching,
// and metadata isn't retained when relaying.
this.fetchArbitraryDataFile(peerToAsk, peer, null, signature, hash, message);
}
else {
LOGGER.debug("Peer {} not found in relay info", peer);

12
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -14,6 +14,7 @@ import org.qortal.arbitrary.ArbitraryDataResource;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.network.Network;
@ -542,6 +543,17 @@ public class ArbitraryDataManager extends Thread {
return true;
}
public void onExpiredArbitraryTransaction(ArbitraryTransactionData arbitraryTransactionData) {
if (arbitraryTransactionData.getName() == null) {
// No name, so we don't care about this transaction
return;
}
// Add to queue for update/deletion
ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
}
public int getPowDifficulty() {
return this.powDifficulty;
}

55
src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
View File

@ -320,41 +320,46 @@ public class ArbitraryMetadataManager {
return;
}
// Update requests map to reflect that we've received all chunks
// Update requests map to reflect that we've received this metadata
Triple<String, Peer, Long> newEntry = new Triple<>(null, null, request.getC());
arbitraryMetadataRequests.put(message.getId(), newEntry);
ArbitraryTransactionData arbitraryTransactionData = null;
// Forwarding
if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
// Get transaction info
try (final Repository repository = RepositoryManager.getRepository()) {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (!(transactionData instanceof ArbitraryTransactionData))
return;
arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e);
// Get transaction info
try (final Repository repository = RepositoryManager.getRepository()) {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (!(transactionData instanceof ArbitraryTransactionData)) {
return;
}
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
// Check if the name is blocked
boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
if (!isBlocked) {
Peer requestingPeer = request.getB();
if (requestingPeer != null) {
// Forwarding
if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile());
forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId());
// Check if the name is blocked
boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
if (!isBlocked) {
Peer requestingPeer = request.getB();
if (requestingPeer != null) {
// Forward to requesting peer
LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
requestingPeer.disconnect("failed to forward arbitrary metadata");
ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile());
forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId());
// Forward to requesting peer
LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
requestingPeer.disconnect("failed to forward arbitrary metadata");
}
}
}
}
// Add to resource queue to update arbitrary resource caches
if (arbitraryTransactionData != null) {
ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
}
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while saving arbitrary transaction metadata from peer %s", peer), e);
}
}

31
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java → src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java
View File

@ -6,8 +6,10 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.Objects;
import static org.qortal.data.arbitrary.ArbitraryResourceStatus.Status;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryResourceInfo {
public class ArbitraryResourceData {
public String name;
public Service service;
@ -15,11 +17,21 @@ public class ArbitraryResourceInfo {
public ArbitraryResourceStatus status;
public ArbitraryResourceMetadata metadata;
public Long size;
public Integer size;
public Long created;
public Long updated;
public ArbitraryResourceInfo() {
public ArbitraryResourceData() {
}
public ArbitraryResourceData(Service service, String name, String identifier) {
if (identifier == null) {
identifier = "default";
}
this.service = service;
this.name = name;
this.identifier = identifier;
}
@Override
@ -27,15 +39,24 @@ public class ArbitraryResourceInfo {
return String.format("%s %s %s", name, service, identifier);
}
public void setStatus(Status status) {
if (status == null) {
this.status = null;
}
else {
this.status = new ArbitraryResourceStatus(status);
}
}
@Override
public boolean equals(Object o) {
if (o == this)
return true;
if (!(o instanceof ArbitraryResourceInfo))
if (!(o instanceof ArbitraryResourceData))
return false;
ArbitraryResourceInfo other = (ArbitraryResourceInfo) o;
ArbitraryResourceData other = (ArbitraryResourceData) o;
return Objects.equals(this.name, other.name) &&
Objects.equals(this.service, other.service) &&

51
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java
View File

@ -18,6 +18,9 @@ public class ArbitraryResourceMetadata {
private List<String> files;
private String mimeType;
// Only included when updating database
private ArbitraryResourceData arbitraryResourceData;
public ArbitraryResourceMetadata() {
}
@ -60,4 +63,52 @@ public class ArbitraryResourceMetadata {
public List<String> getFiles() {
return this.files;
}
public void setTitle(String title) {
this.title = title;
}
public String getTitle() {
return this.title;
}
public void setDescription(String description) {
this.description = description;
}
public String getDescription() {
return this.description;
}
public void setTags(List<String> tags) {
this.tags = tags;
}
public List<String> getTags() {
return this.tags;
}
public void setCategory(Category category) {
this.category = category;
// Also set categoryName
if (category != null) {
this.categoryName = category.getName();
}
}
public Category getCategory() {
return this.category;
}
public boolean hasMetadata() {
return title != null || description != null || tags != null || category != null || files != null || mimeType != null;
}
public void setArbitraryResourceData(ArbitraryResourceData arbitraryResourceData) {
this.arbitraryResourceData = arbitraryResourceData;
}
public ArbitraryResourceData getArbitraryResourceData() {
return this.arbitraryResourceData;
}
}

2
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java
View File

@ -9,7 +9,7 @@ import java.util.List;
public class ArbitraryResourceNameInfo {
public String name;
public List<ArbitraryResourceInfo> resources = new ArrayList<>();
public List<ArbitraryResourceData> resources = new ArrayList<>();
public ArbitraryResourceNameInfo() {
}

39
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java
View File

@ -2,29 +2,46 @@ package org.qortal.data.arbitrary;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.Map;
import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryResourceStatus {
public enum Status {
PUBLISHED("Published", "Published but not yet downloaded"),
NOT_PUBLISHED("Not published", "Resource does not exist"),
DOWNLOADING("Downloading", "Locating and downloading files..."),
DOWNLOADED("Downloaded", "Files downloaded"),
BUILDING("Building", "Building..."),
READY("Ready", "Ready"),
MISSING_DATA("Missing data", "Unable to locate all files. Please try again later"),
BUILD_FAILED("Build failed", "Build failed. Please try again later"),
UNSUPPORTED("Unsupported", "Unsupported request"),
BLOCKED("Blocked", "Name is blocked so content cannot be served");
// Note: integer values must not be updated, as they are stored in the db
PUBLISHED(1, "Published", "Published but not yet downloaded"),
NOT_PUBLISHED(2, "Not published", "Resource does not exist"),
DOWNLOADING(3, "Downloading", "Locating and downloading files..."),
DOWNLOADED(4, "Downloaded", "Files downloaded"),
BUILDING(5, "Building", "Building..."),
READY(6, "Ready", "Ready"),
MISSING_DATA(7, "Missing data", "Unable to locate all files. Please try again later"),
BUILD_FAILED(8, "Build failed", "Build failed. Please try again later"),
UNSUPPORTED(9, "Unsupported", "Unsupported request"),
BLOCKED(10, "Blocked", "Name is blocked so content cannot be served");
public int value;
private String title;
private String description;
Status(String title, String description) {
private static final Map<Integer, Status> map = stream(Status.values())
.collect(toMap(status -> status.value, status -> status));
Status(int value, String title, String description) {
this.value = value;
this.title = title;
this.description = description;
}
public static Status valueOf(Integer value) {
if (value == null) {
return null;
}
return map.get(value);
}
}
private Status status;

35
src/main/java/org/qortal/repository/ArbitraryRepository.java
View File

@ -1,9 +1,10 @@
package org.qortal.repository;
import org.qortal.api.SearchMode;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
import org.qortal.data.network.ArbitraryPeerData;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
@ -11,23 +12,45 @@ import java.util.List;
public interface ArbitraryRepository {
// Utils
public boolean isDataLocal(byte[] signature) throws DataException;
public byte[] fetchData(byte[] signature) throws DataException;
// Transaction related
public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException;
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
public List<ArbitraryTransactionData> getArbitraryTransactions(boolean requireName, Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceInfo> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException;
// Resource related
public List<ArbitraryResourceNameInfo> getArbitraryResourceCreatorNames(Service service, String identifier, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException;
public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException;
public List<ArbitraryResourceData> getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException;
// Arbitrary resources cache save/load
public void save(ArbitraryResourceData arbitraryResourceData) throws DataException;
public void setStatus(ArbitraryResourceData arbitraryResourceData, ArbitraryResourceStatus.Status status) throws DataException;
public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException;
public void save(ArbitraryResourceMetadata metadata) throws DataException;
public void delete(ArbitraryResourceMetadata metadata) throws DataException;
}

677
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@ -2,10 +2,13 @@ package org.qortal.repository.hsqldb;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.util.Longs;
import org.qortal.api.SearchMode;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Category;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.data.transaction.BaseTransactionData;
@ -22,6 +25,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class HSQLDBArbitraryRepository implements ArbitraryRepository {
@ -41,6 +45,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
return (ArbitraryTransactionData) transactionData;
}
// Utils
@Override
public boolean isDataLocal(byte[] signature) throws DataException {
ArbitraryTransactionData transactionData = getTransactionData(signature);
@ -113,6 +120,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
return null;
}
// Transaction related
@Override
public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
// Already hashed? Nothing to do
@ -211,8 +221,12 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
}
@Override
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException {
if (name == null || service == null) {
// Required fields
return null;
}
StringBuilder sql = new StringBuilder(1024);
sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
@ -228,7 +242,16 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
sql.append(method.value);
}
sql.append("ORDER BY created_when DESC LIMIT 1");
sql.append(" ORDER BY created_when");
if (firstNotLast) {
sql.append(" ASC");
}
else {
sql.append(" DESC");
}
sql.append(" LIMIT 1");
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) {
if (resultSet == null)
@ -284,22 +307,286 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
@Override
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, String identifier, List<String> names,
boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked,
public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException {
return this.getSingleTransaction(name, service, method, identifier, true);
}
@Override
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
return this.getSingleTransaction(name, service, method, identifier, false);
}
public List<ArbitraryTransactionData> getArbitraryTransactions(boolean requireName, Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature)");
if (requireName) {
sql.append(" WHERE name IS NOT NULL");
}
sql.append(" ORDER BY created_when");
if (reverse != null && reverse) {
sql.append(" DESC");
}
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) {
if (resultSet == null)
return null;
do {
//TransactionType type = TransactionType.valueOf(resultSet.getInt(1));
byte[] reference = resultSet.getBytes(2);
byte[] signature = resultSet.getBytes(3);
byte[] creatorPublicKey = resultSet.getBytes(4);
long timestamp = resultSet.getLong(5);
Long fee = resultSet.getLong(6);
if (fee == 0 && resultSet.wasNull())
fee = null;
int txGroupId = resultSet.getInt(7);
Integer blockHeight = resultSet.getInt(8);
if (blockHeight == 0 && resultSet.wasNull())
blockHeight = null;
ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
Integer approvalHeight = resultSet.getInt(10);
if (approvalHeight == 0 && resultSet.wasNull())
approvalHeight = null;
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
int version = resultSet.getInt(11);
int nonce = resultSet.getInt(12);
int serviceInt = resultSet.getInt(13);
int size = resultSet.getInt(14);
boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
byte[] data = resultSet.getBytes(16);
byte[] metadataHash = resultSet.getBytes(17);
String nameResult = resultSet.getString(18);
String identifierResult = resultSet.getString(19);
Method method = Method.valueOf(resultSet.getInt(20));
byte[] secret = resultSet.getBytes(21);
Compression compression = Compression.valueOf(resultSet.getInt(22));
// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
compression, data, dataType, metadataHash, null);
arbitraryTransactionData.add(transactionData);
} while (resultSet.next());
return arbitraryTransactionData;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
}
}
// Resource related
@Override
public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException {
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
// Name is required
if (name == null) {
return null;
}
sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " +
"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
"FROM ArbitraryResourcesCache " +
"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
"WHERE ArbitraryResourcesCache.service = ? AND ArbitraryResourcesCache.name = ?");
bindParams.add(service.value);
bindParams.add(name);
if (identifier != null) {
sql.append(" AND identifier = ?");
bindParams.add(identifier);
}
else {
sql.append(" AND identifier IS NULL");
}
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return null;
String nameResult = resultSet.getString(1);
Service serviceResult = Service.valueOf(resultSet.getInt(2));
String identifierResult = resultSet.getString(3);
Integer sizeResult = resultSet.getInt(4);
Integer status = resultSet.getInt(5);
Long created = resultSet.getLong(6);
Long updated = resultSet.getLong(7);
// Optional metadata fields
String title = resultSet.getString(8);
String description = resultSet.getString(9);
String category = resultSet.getString(10);
String tag1 = resultSet.getString(11);
String tag2 = resultSet.getString(12);
String tag3 = resultSet.getString(13);
String tag4 = resultSet.getString(14);
String tag5 = resultSet.getString(15);
if (Objects.equals(identifierResult, "default")) {
// Map "default" back to null. This is optional but probably less confusing than returning "default".
identifierResult = null;
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.name = nameResult;
arbitraryResourceData.service = serviceResult;
arbitraryResourceData.identifier = identifierResult;
arbitraryResourceData.size = sizeResult;
arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status));
arbitraryResourceData.created = created;
arbitraryResourceData.updated = (updated == 0) ? null : updated;
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setTitle(title);
metadata.setDescription(description);
metadata.setCategory(Category.uncategorizedValueOf(category));
List<String> tags = new ArrayList<>();
if (tag1 != null) tags.add(tag1);
if (tag2 != null) tags.add(tag2);
if (tag3 != null) tags.add(tag3);
if (tag4 != null) tags.add(tag4);
if (tag5 != null) tags.add(tag5);
metadata.setTags(!tags.isEmpty() ? tags : null);
if (metadata.hasMetadata()) {
arbitraryResourceData.metadata = metadata;
}
return arbitraryResourceData;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary resource from repository", e);
}
}
@Override
public List<ArbitraryResourceData> getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " +
"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
"FROM ArbitraryResourcesCache " +
"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
"WHERE name IS NOT NULL ORDER BY created_when");
if (reverse != null && reverse) {
sql.append(" DESC");
}
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return arbitraryResources;
do {
String nameResult = resultSet.getString(1);
Service serviceResult = Service.valueOf(resultSet.getInt(2));
String identifierResult = resultSet.getString(3);
Integer sizeResult = resultSet.getInt(4);
Integer status = resultSet.getInt(5);
Long created = resultSet.getLong(6);
Long updated = resultSet.getLong(7);
// Optional metadata fields
String title = resultSet.getString(8);
String description = resultSet.getString(9);
String category = resultSet.getString(10);
String tag1 = resultSet.getString(11);
String tag2 = resultSet.getString(12);
String tag3 = resultSet.getString(13);
String tag4 = resultSet.getString(14);
String tag5 = resultSet.getString(15);
if (Objects.equals(identifierResult, "default")) {
// Map "default" back to null. This is optional but probably less confusing than returning "default".
identifierResult = null;
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.name = nameResult;
arbitraryResourceData.service = serviceResult;
arbitraryResourceData.identifier = identifierResult;
arbitraryResourceData.size = sizeResult;
arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status));
arbitraryResourceData.created = created;
arbitraryResourceData.updated = (updated == 0) ? null : updated;
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setTitle(title);
metadata.setDescription(description);
metadata.setCategory(Category.uncategorizedValueOf(category));
List<String> tags = new ArrayList<>();
if (tag1 != null) tags.add(tag1);
if (tag2 != null) tags.add(tag2);
if (tag3 != null) tags.add(tag3);
if (tag4 != null) tags.add(tag4);
if (tag5 != null) tags.add(tag5);
metadata.setTags(!tags.isEmpty() ? tags : null);
if (metadata.hasMetadata()) {
arbitraryResourceData.metadata = metadata;
}
arbitraryResources.add(arbitraryResourceData);
} while (resultSet.next());
return arbitraryResources;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary resources from repository", e);
}
}
@Override
public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names,
boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked,
Boolean includeMetadata, Boolean includeStatus,
Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
sql.append("SELECT name, service, identifier, MAX(size) AS max_size FROM ArbitraryTransactions WHERE 1=1");
sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " +
"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
"FROM ArbitraryResourcesCache " +
"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
"WHERE name IS NOT NULL");
if (service != null) {
sql.append(" AND service = ");
sql.append(service.value);
sql.append(" AND service = ?");
bindParams.add(service.value);
}
if (defaultResource) {
// Default resource requested - use NULL identifier
sql.append(" AND identifier IS NULL");
sql.append(" AND identifier='default'");
}
else {
// Non-default resource requested
@ -351,7 +638,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
}
sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
sql.append(" ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
if (reverse != null && reverse) {
sql.append(" DESC");
@ -359,53 +646,122 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
List<ArbitraryResourceInfo> arbitraryResources = new ArrayList<>();
List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return null;
return arbitraryResources;
do {
String nameResult = resultSet.getString(1);
Service serviceResult = Service.valueOf(resultSet.getInt(2));
String identifierResult = resultSet.getString(3);
Integer sizeResult = resultSet.getInt(4);
Integer status = resultSet.getInt(5);
Long created = resultSet.getLong(6);
Long updated = resultSet.getLong(7);
// We should filter out resources without names
if (nameResult == null) {
continue;
// Optional metadata fields
String title = resultSet.getString(8);
String description = resultSet.getString(9);
String category = resultSet.getString(10);
String tag1 = resultSet.getString(11);
String tag2 = resultSet.getString(12);
String tag3 = resultSet.getString(13);
String tag4 = resultSet.getString(14);
String tag5 = resultSet.getString(15);
if (Objects.equals(identifierResult, "default")) {
// Map "default" back to null. This is optional but probably less confusing than returning "default".
identifierResult = null;
}
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
arbitraryResourceInfo.name = nameResult;
arbitraryResourceInfo.service = serviceResult;
arbitraryResourceInfo.identifier = identifierResult;
arbitraryResourceInfo.size = Longs.valueOf(sizeResult);
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.name = nameResult;
arbitraryResourceData.service = serviceResult;
arbitraryResourceData.identifier = identifierResult;
arbitraryResourceData.size = sizeResult;
arbitraryResourceData.created = created;
arbitraryResourceData.updated = (updated == 0) ? null : updated;
arbitraryResources.add(arbitraryResourceInfo);
if (includeStatus != null && includeStatus) {
arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status));
}
if (includeMetadata != null && includeMetadata) {
// TODO: we could avoid the join altogether
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setTitle(title);
metadata.setDescription(description);
metadata.setCategory(Category.uncategorizedValueOf(category));
List<String> tags = new ArrayList<>();
if (tag1 != null) tags.add(tag1);
if (tag2 != null) tags.add(tag2);
if (tag3 != null) tags.add(tag3);
if (tag4 != null) tags.add(tag4);
if (tag5 != null) tags.add(tag5);
metadata.setTags(!tags.isEmpty() ? tags : null);
if (metadata.hasMetadata()) {
arbitraryResourceData.metadata = metadata;
}
}
arbitraryResources.add(arbitraryResourceData);
} while (resultSet.next());
return arbitraryResources;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
throw new DataException("Unable to fetch arbitrary resources from repository", e);
}
}
@Override
public List<ArbitraryResourceInfo> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly,
List<String> exactMatchNames, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked,
Integer limit, Integer offset, Boolean reverse) throws DataException {
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, boolean prefixOnly,
List<String> exactMatchNames, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked,
Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
sql.append("SELECT name, service, identifier, MAX(size) AS max_size, MIN(created_when) AS date_created, MAX(created_when) AS date_updated " +
"FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE 1=1");
sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " +
"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
"FROM ArbitraryResourcesCache");
// Default to "latest" mode
if (mode == null) {
mode = SearchMode.LATEST;
}
switch (mode) {
case LATEST:
// Include latest item only for a name/service combination
sql.append(" JOIN (SELECT name, service, MAX(created_when) AS latest " +
"FROM ArbitraryResourcesCache GROUP BY name, service) LatestResources " +
"ON name=LatestResources.name AND service=LatestResources.service " +
"AND created_when=LatestResources.latest");
break;
case ALL:
break;
}
if (minLevel != null) {
// Join tables necessary for level filter
sql.append(" JOIN Names USING (name) JOIN Accounts ON Accounts.account=Names.owner");
}
sql.append(" LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) WHERE name IS NOT NULL");
if (minLevel != null) {
// Add level filter
sql.append(" AND Accounts.level >= ?");
bindParams.add(minLevel);
}
if (service != null) {
sql.append(" AND service = ");
sql.append(service.value);
sql.append(" AND service = ?");
bindParams.add(service.value);
}
// Handle general query matches
@ -417,14 +773,13 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
if (defaultResource) {
// Default resource requested - use NULL identifier and search name only
sql.append(" AND LCASE(name) LIKE ? AND identifier IS NULL");
sql.append(" AND LCASE(name) LIKE ? AND identifier='default'");
bindParams.add(queryWildcard);
} else {
// Non-default resource requested
// In this case we search the identifier as well as the name
sql.append(" AND (LCASE(name) LIKE ? OR LCASE(identifier) LIKE ?)");
bindParams.add(queryWildcard);
bindParams.add(queryWildcard);
sql.append(" AND (LCASE(name) LIKE ? OR LCASE(identifier) LIKE ? OR LCASE(title) LIKE ? OR LCASE(description) LIKE ?)");
bindParams.add(queryWildcard); bindParams.add(queryWildcard); bindParams.add(queryWildcard); bindParams.add(queryWildcard);
}
}
@ -436,6 +791,22 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
bindParams.add(queryWildcard);
}
// Handle title metadata matches
if (title != null) {
// Search anywhere in the title, unless "prefixOnly" has been requested
String queryWildcard = prefixOnly ? String.format("%s%%", title.toLowerCase()) : String.format("%%%s%%", title.toLowerCase());
sql.append(" AND LCASE(title) LIKE ?");
bindParams.add(queryWildcard);
}
// Handle description metadata matches
if (description != null) {
// Search anywhere in the description, unless "prefixOnly" has been requested
String queryWildcard = prefixOnly ? String.format("%s%%", description.toLowerCase()) : String.format("%%%s%%", description.toLowerCase());
sql.append(" AND LCASE(description) LIKE ?");
bindParams.add(queryWildcard);
}
// Handle name searches
if (names != null && !names.isEmpty()) {
sql.append(" AND (");
@ -462,6 +833,16 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
sql.append(")");
}
// Timestamp range
if (before != null) {
sql.append(" AND created_when < ?");
bindParams.add(before);
}
if (after != null) {
sql.append(" AND created_when > ?");
bindParams.add(after);
}
// Handle "followed only"
if (followedOnly != null && followedOnly) {
List<String> followedNames = ListUtils.followedNames();
@ -492,7 +873,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
}
sql.append(" GROUP BY name, service, identifier ORDER BY date_created");
sql.append(" ORDER BY created_when");
if (reverse != null && reverse) {
sql.append(" DESC");
@ -500,98 +881,182 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
List<ArbitraryResourceInfo> arbitraryResources = new ArrayList<>();
List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return null;
return arbitraryResources;
do {
String nameResult = resultSet.getString(1);
Service serviceResult = Service.valueOf(resultSet.getInt(2));
String identifierResult = resultSet.getString(3);
Integer sizeResult = resultSet.getInt(4);
long dateCreated = resultSet.getLong(5);
long dateUpdated = resultSet.getLong(6);
Integer status = resultSet.getInt(5);
Long created = resultSet.getLong(6);
Long updated = resultSet.getLong(7);
// We should filter out resources without names
if (nameResult == null) {
continue;
// Optional metadata fields
String titleResult = resultSet.getString(8);
String descriptionResult = resultSet.getString(9);
String category = resultSet.getString(10);
String tag1 = resultSet.getString(11);
String tag2 = resultSet.getString(12);
String tag3 = resultSet.getString(13);
String tag4 = resultSet.getString(14);
String tag5 = resultSet.getString(15);
if (Objects.equals(identifierResult, "default")) {
// Map "default" back to null. This is optional but probably less confusing than returning "default".
identifierResult = null;
}
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
arbitraryResourceInfo.name = nameResult;
arbitraryResourceInfo.service = serviceResult;
arbitraryResourceInfo.identifier = identifierResult;
arbitraryResourceInfo.size = Longs.valueOf(sizeResult);
arbitraryResourceInfo.created = dateCreated;
arbitraryResourceInfo.updated = dateUpdated;
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.name = nameResult;
arbitraryResourceData.service = serviceResult;
arbitraryResourceData.identifier = identifierResult;
arbitraryResourceData.size = sizeResult;
arbitraryResourceData.created = created;
arbitraryResourceData.updated = (updated == 0) ? null : updated;
arbitraryResources.add(arbitraryResourceInfo);
if (includeStatus != null && includeStatus) {
arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status));
}
if (includeMetadata != null && includeMetadata) {
// TODO: we could avoid the join altogether
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setTitle(titleResult);
metadata.setDescription(descriptionResult);
metadata.setCategory(Category.uncategorizedValueOf(category));
List<String> tags = new ArrayList<>();
if (tag1 != null) tags.add(tag1);
if (tag2 != null) tags.add(tag2);
if (tag3 != null) tags.add(tag3);
if (tag4 != null) tags.add(tag4);
if (tag5 != null) tags.add(tag5);
metadata.setTags(!tags.isEmpty() ? tags : null);
if (metadata.hasMetadata()) {
arbitraryResourceData.metadata = metadata;
}
}
arbitraryResources.add(arbitraryResourceData);
} while (resultSet.next());
return arbitraryResources;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
throw new DataException("Unable to fetch arbitrary resources from repository", e);
}
}
// Arbitrary resources cache save/load
@Override
public void save(ArbitraryResourceData arbitraryResourceData) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryResourcesCache");
// "status" isn't saved here as we update this field separately
saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name)
.bind("identifier", arbitraryResourceData.identifier).bind("size", arbitraryResourceData.size)
.bind("created_when", arbitraryResourceData.created).bind("updated_when", arbitraryResourceData.updated);
try {
saveHelper.execute(this.repository);
} catch (SQLException e) {
throw new DataException("Unable to save arbitrary resource info into repository", e);
}
}
@Override
public List<ArbitraryResourceNameInfo> getArbitraryResourceCreatorNames(Service service, String identifier,
boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
sql.append("SELECT name FROM ArbitraryTransactions WHERE 1=1");
if (service != null) {
sql.append(" AND service = ");
sql.append(service.value);
public void setStatus(ArbitraryResourceData arbitraryResourceData, ArbitraryResourceStatus.Status status) throws DataException {
if (status == null) {
return;
}
String updateSql = "UPDATE ArbitraryResourcesCache SET status = ? WHERE service = ? AND LCASE(name) = ? AND LCASE(identifier) = ?";
if (defaultResource) {
// Default resource requested - use NULL identifier
// The AND ? IS NULL AND ? IS NULL is a hack to make use of the identifier params in checkedExecute()
identifier = null;
sql.append(" AND (identifier IS NULL AND ? IS NULL AND ? IS NULL)");
}
else {
// Non-default resource requested
// Use an exact match identifier, or list all if supplied identifier is null
sql.append(" AND (identifier = ? OR (? IS NULL))");
}
sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
if (reverse != null && reverse) {
sql.append(" DESC");
}
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
List<ArbitraryResourceNameInfo> arbitraryResources = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), identifier, identifier)) {
if (resultSet == null)
return null;
do {
String name = resultSet.getString(1);
// We should filter out resources without names
if (name == null) {
continue;
}
ArbitraryResourceNameInfo arbitraryResourceNameInfo = new ArbitraryResourceNameInfo();
arbitraryResourceNameInfo.name = name;
arbitraryResources.add(arbitraryResourceNameInfo);
} while (resultSet.next());
return arbitraryResources;
try {
this.repository.executeCheckedUpdate(updateSql, status.value, arbitraryResourceData.service.value, arbitraryResourceData.name.toLowerCase(), arbitraryResourceData.identifier.toLowerCase());
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
throw new DataException("Unable to set status for arbitrary resource", e);
}
}
@Override
public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException {
// NOTE: arbitrary metadata are deleted automatically by the database thanks to "ON DELETE CASCADE"
// in ArbitraryMetadataCache' FOREIGN KEY definition.
try {
this.repository.delete("ArbitraryResourcesCache", "service = ? AND name = ? AND identifier = ?",
arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier);
} catch (SQLException e) {
throw new DataException("Unable to delete account from repository", e);
}
}
/* Arbitrary metadata cache */
@Override
public void save(ArbitraryResourceMetadata metadata) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryMetadataCache");
ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData();
if (arbitraryResourceData == null) {
throw new DataException("Can't save metadata without a referenced resource");
}
// Trim metadata values if they are too long to fit in the db
String title = ArbitraryDataTransactionMetadata.limitTitle(metadata.getTitle());
String description = ArbitraryDataTransactionMetadata.limitDescription(metadata.getDescription());
List<String> tags = ArbitraryDataTransactionMetadata.limitTags(metadata.getTags());
String tag1 = null;
String tag2 = null;
String tag3 = null;
String tag4 = null;
String tag5 = null;
if (tags != null) {
if (tags.size() > 0) tag1 = tags.get(0);
if (tags.size() > 1) tag2 = tags.get(1);
if (tags.size() > 2) tag3 = tags.get(2);
if (tags.size() > 3) tag4 = tags.get(3);
if (tags.size() > 4) tag5 = tags.get(4);
}
String category = metadata.getCategory() != null ? metadata.getCategory().toString() : null;
saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name)
.bind("identifier", arbitraryResourceData.identifier).bind("title", title)
.bind("description", description).bind("category", category)
.bind("tag1", tag1).bind("tag2", tag2).bind("tag3", tag3).bind("tag4", tag4)
.bind("tag5", tag5);
try {
saveHelper.execute(this.repository);
} catch (SQLException e) {
throw new DataException("Unable to save arbitrary metadata into repository", e);
}
}
@Override
public void delete(ArbitraryResourceMetadata metadata) throws DataException {
ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData();
if (arbitraryResourceData == null) {
throw new DataException("Can't delete metadata without a referenced resource");
}
try {
this.repository.delete("ArbitraryMetadataCache", "service = ? AND name = ? AND identifier = ?",
arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier);
} catch (SQLException e) {
throw new DataException("Unable to delete account from repository", e);
}
}
}

45
src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
View File

@ -901,7 +901,7 @@ public class HSQLDBDatabaseUpdates {
case 37:
// ARBITRARY transaction updates for off-chain data storage
// We may want to use a nonce rather than a transaction fee on the data chain
// We may want to use a nonce rather than a transaction fee for ARBITRARY transactions
stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0");
// We need to know the total size of the data file(s) associated with each transaction
stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0");
@ -1004,6 +1004,49 @@ public class HSQLDBDatabaseUpdates {
stmt.execute("CREATE INDEX TransactionHeightSequenceIndex on Transactions (block_height, block_sequence)");
break;
case 48:
// We need to keep a local cache of arbitrary resources (items published to QDN), for easier searching.
// IMPORTANT: this is a cache of the last known state of a resource (both confirmed
// and valid unconfirmed). It cannot be assumed that all nodes will contain the same state at a
// given block height, and therefore must NOT be used for any consensus/validation code. It is
// simply a cache, to avoid having to query the raw transactions and the metadata in flat files
// when serving API requests.
// ARBITRARY transactions aren't really suitable for updating resources in the same way we'd update
// names or groups for instance, as there is no distinction between creations and updates, and metadata
// is off-chain. Plus, QDN allows (valid) unconfirmed data to be queried and viewed. It is very
// easy to keep a cache of the latest transaction's data, but anything more than that would need
// considerable thought (and most likely a rewrite).
stmt.execute("CREATE TABLE ArbitraryResourcesCache (service SMALLINT NOT NULL, "
+ "name RegisteredName NOT NULL, identifier VARCHAR(64), size INT NOT NULL, "
+ "status INTEGER, created_when EpochMillis NOT NULL, updated_when EpochMillis, "
+ "PRIMARY KEY (service, name, identifier))");
// For finding resources by service.
stmt.execute("CREATE INDEX ArbitraryResourcesServiceIndex ON ArbitraryResourcesCache (service)");
// For finding resources by name.
stmt.execute("CREATE INDEX ArbitraryResourcesNameIndex ON ArbitraryResourcesCache (name)");
// For finding resources by identifier.
stmt.execute("CREATE INDEX ArbitraryResourcesIdentifierIndex ON ArbitraryResourcesCache (identifier)");
// For finding resources by creation date (the default column when ordering).
stmt.execute("CREATE INDEX ArbitraryResourcesCreatedIndex ON ArbitraryResourcesCache (created_when)");
// Use a separate table space as this table will be very large.
stmt.execute("SET TABLE ArbitraryResourcesCache NEW SPACE");
stmt.execute("CREATE TABLE ArbitraryMetadataCache (service SMALLINT NOT NULL, "
+ "name RegisteredName NOT NULL, identifier VARCHAR(64), "
+ "title VARCHAR(80), description VARCHAR(240), category VARCHAR(64), "
+ "tag1 VARCHAR(20), tag2 VARCHAR(20), tag3 VARCHAR(20), tag4 VARCHAR(20), tag5 VARCHAR(20), "
+ "PRIMARY KEY (service, name, identifier), FOREIGN KEY (service, name, identifier) "
+ "REFERENCES ArbitraryResourcesCache (service, name, identifier) ON DELETE CASCADE)");
// For finding metadata by title.
stmt.execute("CREATE INDEX ArbitraryMetadataTitleIndex ON ArbitraryMetadataCache (title)");
// For finding arbitrary transactions by service
stmt.execute("CREATE INDEX ArbitraryServiceIndex ON ArbitraryTransactions (service)");
// For finding arbitrary transactions by identifier
stmt.execute("CREATE INDEX ArbitraryIdentifierIndex ON ArbitraryTransactions (identifier)");
break;
default:
// nothing to do
return false;

214
src/main/java/org/qortal/transaction/ArbitraryTransaction.java
View File

@ -1,16 +1,26 @@
package org.qortal.transaction;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.Account;
import org.qortal.arbitrary.ArbitraryDataResource;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Service;
import org.qortal.block.BlockChain;
import org.qortal.controller.arbitrary.ArbitraryDataCacheManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
import org.qortal.crypto.Crypto;
import org.qortal.crypto.MemoryPoW;
import org.qortal.data.PaymentData;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
@ -18,6 +28,7 @@ import org.qortal.payment.Payment;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.repository.RepositoryManager;
import org.qortal.transform.TransformationException;
import org.qortal.transform.Transformer;
import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
@ -27,6 +38,8 @@ import org.qortal.utils.NTP;
public class ArbitraryTransaction extends Transaction {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryTransaction.class);
// Properties
private ArbitraryTransactionData arbitraryTransactionData;
@ -250,14 +263,8 @@ public class ArbitraryTransaction extends Transaction {
// We may need to move files from the misc_ folder
ArbitraryTransactionUtils.checkAndRelocateMiscFiles(arbitraryTransactionData);
// If the data is local, we need to perform a few actions
if (isDataLocal()) {
// We have the data for this transaction, so invalidate the cache
if (arbitraryTransactionData.getName() != null) {
ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData);
}
}
// Update caches
updateCaches();
}
@Override
@ -277,6 +284,35 @@ public class ArbitraryTransaction extends Transaction {
public void process() throws DataException {
// Wrap and delegate payment processing to Payment class.
new Payment(this.repository).process(arbitraryTransactionData.getSenderPublicKey(), arbitraryTransactionData.getPayments());
// Update caches
this.updateCaches();
}
private void updateCaches() {
// Very important to use a separate repository instance from the one being used for validation/processing
try (final Repository repository = RepositoryManager.getRepository()) {
// If the data is local, we need to perform a few actions
if (isDataLocal()) {
// We have the data for this transaction, so invalidate the file cache
if (arbitraryTransactionData.getName() != null) {
ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData);
}
}
// Add/update arbitrary resource caches, but don't update the status as this involves time-consuming
// disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when
// accessing the resource.
this.updateArbitraryResourceCache(repository);
this.updateArbitraryMetadataCache(repository);
repository.saveChanges();
} catch (Exception e) {
// Log and ignore all exceptions. The cache is updated from other places too, and can be rebuilt if needed.
LOGGER.info("Unable to update arbitrary caches", e);
}
}
@Override
@ -314,4 +350,166 @@ public class ArbitraryTransaction extends Transaction {
return null;
}
/**
* Update the arbitrary resources cache.
* This finds the latest transaction and replaces the
* majority of the data in the cache. The current
* transaction is used for the created time,
* if it has a lower timestamp than the existing value.
* It's also used to identify the correct
* service/name/identifier combination.
*
* @throws DataException
*/
public void updateArbitraryResourceCache(Repository repository) throws DataException {
// Don't cache resources without a name (such as auto updates)
if (arbitraryTransactionData.getName() == null) {
return;
}
Service service = arbitraryTransactionData.getService();
String name = arbitraryTransactionData.getName();
String identifier = arbitraryTransactionData.getIdentifier();
if (service == null) {
// Unsupported service - ignore this resource
return;
}
// In the cache we store null identifiers as "default", as it is part of the primary key
if (identifier == null) {
identifier = "default";
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.service = service;
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
// Get the latest transaction
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
// We don't have a latest transaction, so delete from cache
repository.getArbitraryRepository().delete(arbitraryResourceData);
return;
}
// Get existing cached entry if it exists
ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository()
.getArbitraryResource(service, name, identifier);
// Check for existing cached data
if (existingArbitraryResourceData == null) {
// Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed)
arbitraryResourceData.created = arbitraryTransactionData.getTimestamp();
arbitraryResourceData.updated = null;
}
else {
// An entry already exists - update created time from current transaction if this is older
arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp());
// Set updated time to the latest transaction's timestamp, unless it matches the creation time
if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) {
// Latest transaction matches created time, so it hasn't been updated
arbitraryResourceData.updated = null;
}
else {
arbitraryResourceData.updated = latestTransactionData.getTimestamp();
}
}
arbitraryResourceData.size = latestTransactionData.getSize();
// Save
repository.getArbitraryRepository().save(arbitraryResourceData);
}
public void updateArbitraryResourceStatus(Repository repository) throws DataException {
// Don't cache resources without a name (such as auto updates)
if (arbitraryTransactionData.getName() == null) {
return;
}
Service service = arbitraryTransactionData.getService();
String name = arbitraryTransactionData.getName();
String identifier = arbitraryTransactionData.getIdentifier();
if (service == null) {
// Unsupported service - ignore this resource
return;
}
// In the cache we store null identifiers as "default", as it is part of the primary key
if (identifier == null) {
identifier = "default";
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.service = service;
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
// Update status
ArbitraryDataResource resource = new ArbitraryDataResource(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
ArbitraryResourceStatus arbitraryResourceStatus = resource.getStatus(repository);
ArbitraryResourceStatus.Status status = arbitraryResourceStatus != null ? arbitraryResourceStatus.getStatus() : null;
repository.getArbitraryRepository().setStatus(arbitraryResourceData, status);
}
public void updateArbitraryMetadataCache(Repository repository) throws DataException {
// Get the latest transaction
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
// We don't have a latest transaction, so give up
return;
}
Service service = latestTransactionData.getService();
String name = latestTransactionData.getName();
String identifier = latestTransactionData.getIdentifier();
if (service == null) {
// Unsupported service - ignore this resource
return;
}
// In the cache we store null identifiers as "default", as it is part of the primary key
if (identifier == null) {
identifier = "default";
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.service = service;
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
// Update metadata for latest transaction if it is local
if (latestTransactionData.getMetadataHash() != null) {
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
if (metadataFile.exists()) {
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
try {
transactionMetadata.read();
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
metadata.setTitle(transactionMetadata.getTitle());
metadata.setDescription(transactionMetadata.getDescription());
metadata.setCategory(transactionMetadata.getCategory());
metadata.setTags(transactionMetadata.getTags());
repository.getArbitraryRepository().save(metadata);
} catch (IOException e) {
// Ignore, as we can add it again later
}
} else {
// We don't have a local copy of this metadata file, so delete it from the cache
// It will be re-added if the file later arrives via the network
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
repository.getArbitraryRepository().delete(metadata);
}
}
}
}

58
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@ -4,10 +4,8 @@ import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.*;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
@ -258,8 +256,7 @@ public class ArbitraryTransactionUtils {
"chunks if needed", Base58.encode(completeHash));
ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, cleanupAfter);
}
else {
} else {
// File might be in use. It's best to leave it and it it will be cleaned up later.
}
}
@ -271,6 +268,7 @@ public class ArbitraryTransactionUtils {
* When first uploaded, files go into a _misc folder as they are not yet associated with a
* transaction signature. Once the transaction is broadcast, they need to be moved to the
* correct location, keyed by the transaction signature.
*
* @param arbitraryTransactionData
* @return
* @throws DataException
@ -356,8 +354,7 @@ public class ArbitraryTransactionUtils {
file.createNewFile();
}
}
}
catch (DataException | IOException e) {
} catch (DataException | IOException e) {
LOGGER.info("Unable to check and relocate all files for signature {}: {}",
Base58.encode(arbitraryTransactionData.getSignature()), e.getMessage());
}
@ -366,7 +363,7 @@ public class ArbitraryTransactionUtils {
}
public static List<ArbitraryTransactionData> limitOffsetTransactions(List<ArbitraryTransactionData> transactions,
Integer limit, Integer offset) {
Integer limit, Integer offset) {
if (limit != null && limit == 0) {
limit = null;
}
@ -389,18 +386,19 @@ public class ArbitraryTransactionUtils {
/**
* Lookup status of resource
*
* @param service
* @param name
* @param identifier
* @param build
* @return
*/
public static ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) {
public static ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build, boolean updateCache) {
// If "build" has been specified, build the resource before returning its status
if (build != null && build == true) {
ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
try {
ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
if (!reader.isBuilding()) {
reader.loadSynchronously(false);
}
@ -410,44 +408,6 @@ public class ArbitraryTransactionUtils {
}
ArbitraryDataResource resource = new ArbitraryDataResource(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
return resource.getStatus(false);
return resource.getStatusAndUpdateCache(updateCache);
}
public static List<ArbitraryResourceInfo> addStatusToResources(List<ArbitraryResourceInfo> resources) {
// Determine and add the status of each resource
List<ArbitraryResourceInfo> updatedResources = new ArrayList<>();
for (ArbitraryResourceInfo resourceInfo : resources) {
try {
ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME,
resourceInfo.service, resourceInfo.identifier);
ArbitraryResourceStatus status = resource.getStatus(true);
if (status != null) {
resourceInfo.status = status;
}
updatedResources.add(resourceInfo);
} catch (Exception e) {
// Catch and log all exceptions, since some systems are experiencing 500 errors when including statuses
LOGGER.info("Caught exception when adding status to resource {}: {}", resourceInfo, e.toString());
}
}
return updatedResources;
}
public static List<ArbitraryResourceInfo> addMetadataToResources(List<ArbitraryResourceInfo> resources) {
// Add metadata fields to each resource if they exist
List<ArbitraryResourceInfo> updatedResources = new ArrayList<>();
for (ArbitraryResourceInfo resourceInfo : resources) {
ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME,
resourceInfo.service, resourceInfo.identifier);
ArbitraryDataTransactionMetadata transactionMetadata = resource.getLatestTransactionMetadata();
ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata, false);
if (resourceMetadata != null) {
resourceInfo.metadata = resourceMetadata;
}
updatedResources.add(resourceInfo);
}
return updatedResources;
}
}

13
src/main/java/org/qortal/utils/StringUtils.java Normal file
View File

@ -0,0 +1,13 @@
package org.qortal.utils;
public class StringUtils {
public static String sanitizeString(String input) {
String sanitized = input
.replaceAll("[<>:\"/\\\\|?*]", "") // Remove invalid characters
.replaceAll("^\\s+|\\s+$", "") // Trim leading and trailing whitespace
.replaceAll("\\s+", "_"); // Replace consecutive whitespace with underscores
return sanitized;
}
}

6
src/main/resources/q-apps/q-apps.js
View File

@ -227,14 +227,20 @@ window.addEventListener("message", (event) => {
if (data.identifier != null) url = url.concat("&identifier=" + data.identifier);
if (data.name != null) url = url.concat("&name=" + data.name);
if (data.names != null) data.names.forEach((x, i) => url = url.concat("&name=" + x));
if (data.title != null) url = url.concat("&title=" + data.title);
if (data.description != null) url = url.concat("&description=" + data.description);
if (data.prefix != null) url = url.concat("&prefix=" + new Boolean(data.prefix).toString());
if (data.exactMatchNames != null) url = url.concat("&exactmatchnames=" + new Boolean(data.exactMatchNames).toString());
if (data.default != null) url = url.concat("&default=" + new Boolean(data.default).toString());
if (data.mode != null) url = url.concat("&mode=" + data.mode);
if (data.minLevel != null) url = url.concat("&minlevel=" + data.minLevel);
if (data.includeStatus != null) url = url.concat("&includestatus=" + new Boolean(data.includeStatus).toString());
if (data.includeMetadata != null) url = url.concat("&includemetadata=" + new Boolean(data.includeMetadata).toString());
if (data.nameListFilter != null) url = url.concat("&namefilter=" + data.nameListFilter);
if (data.followedOnly != null) url = url.concat("&followedonly=" + new Boolean(data.followedOnly).toString());
if (data.excludeBlocked != null) url = url.concat("&excludeblocked=" + new Boolean(data.excludeBlocked).toString());
if (data.before != null) url = url.concat("&before=" + data.before);
if (data.after != null) url = url.concat("&after=" + data.after);
if (data.limit != null) url = url.concat("&limit=" + data.limit);
if (data.offset != null) url = url.concat("&offset=" + data.offset);
if (data.reverse != null) url = url.concat("&reverse=" + new Boolean(data.reverse).toString());