crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Initial attempt at a database cache to hold arbitrary resources and metadata.

Browse Source
This commit is contained in:
CalDescent 2023-04-22 16:05:12 +01:00
parent 81788610c4
commit 9dba4b2968
14 changed files with 816 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -86,12 +86,12 @@ public class ArbitraryResource {
"- If default is set to true, only resources without identifiers will be returned.",
responses = {
@ApiResponse(
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
)
}
)
@ApiErrors({ApiError.REPOSITORY_ISSUE})
public List<ArbitraryResourceInfo> getResources(
public List<ArbitraryResourceData> getResources(
@QueryParam("service") Service service,
@QueryParam("name") String name,
@QueryParam("identifier") String identifier,
@ -133,8 +133,9 @@ public class ArbitraryResource {
}
}
List<ArbitraryResourceInfo> resources = repository.getArbitraryRepository()
.getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse);
List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
.getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked,
includeMetadata, limit, offset, reverse);
if (resources == null) {
return new ArrayList<>();
@ -143,9 +144,6 @@ public class ArbitraryResource {
if (includeStatus != null && includeStatus) {
resources = ArbitraryTransactionUtils.addStatusToResources(resources);
}
if (includeMetadata != null && includeMetadata) {
resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
}
return resources;
@ -161,12 +159,12 @@ public class ArbitraryResource {
"If default is set to true, only resources without identifiers will be returned.",
responses = {
@ApiResponse(
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
)
}
)
@ApiErrors({ApiError.REPOSITORY_ISSUE})
public List<ArbitraryResourceInfo> searchResources(
public List<ArbitraryResourceData> searchResources(
@QueryParam("service") Service service,
@Parameter(description = "Query (searches both name and identifier fields)") @QueryParam("query") String query,
@Parameter(description = "Identifier (searches identifier field only)") @QueryParam("identifier") String identifier,
@ -206,8 +204,9 @@ public class ArbitraryResource {
names = null;
}
List<ArbitraryResourceInfo> resources = repository.getArbitraryRepository()
.searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse);
List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
.searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames,
defaultRes, followedOnly, excludeBlocked, includeMetadata, limit, offset, reverse);
if (resources == null) {
return new ArrayList<>();
@ -216,9 +215,6 @@ public class ArbitraryResource {
if (includeStatus != null && includeStatus) {
resources = ArbitraryTransactionUtils.addStatusToResources(resources);
}
if (includeMetadata != null && includeMetadata) {
resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
}
return resources;
@ -479,21 +475,20 @@ public class ArbitraryResource {
summary = "List arbitrary resources hosted by this node",
responses = {
@ApiResponse(
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
)
}
)
@ApiErrors({ApiError.REPOSITORY_ISSUE})
public List<ArbitraryResourceInfo> getHostedResources(
public List<ArbitraryResourceData> getHostedResources(
@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
@Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata,
@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
@QueryParam("query") String query) {
Security.checkApiCallAllowed(request);
List<ArbitraryResourceInfo> resources = new ArrayList<>();
List<ArbitraryResourceData> resources = new ArrayList<>();
try (final Repository repository = RepositoryManager.getRepository()) {
@ -509,21 +504,18 @@ public class ArbitraryResource {
if (transactionData.getService() == null) {
continue;
}
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
arbitraryResourceInfo.name = transactionData.getName();
arbitraryResourceInfo.service = transactionData.getService();
arbitraryResourceInfo.identifier = transactionData.getIdentifier();
if (!resources.contains(arbitraryResourceInfo)) {
resources.add(arbitraryResourceInfo);
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.name = transactionData.getName();
arbitraryResourceData.service = transactionData.getService();
arbitraryResourceData.identifier = transactionData.getIdentifier();
if (!resources.contains(arbitraryResourceData)) {
resources.add(arbitraryResourceData);
}
}
if (includeStatus != null && includeStatus) {
resources = ArbitraryTransactionUtils.addStatusToResources(resources);
}
if (includeMetadata != null && includeMetadata) {
resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
}
return resources;

5
src/main/java/org/qortal/arbitrary/misc/Category.java
View File

@ -67,9 +67,12 @@ public enum Category {
/**
* Same as valueOf() but with fallback to UNCATEGORIZED if there's no match
* @param name
* @return a Category (using UNCATEGORIZED if no match found)
* @return a Category (using UNCATEGORIZED if no match found), or null if null name passed
*/
public static Category uncategorizedValueOf(String name) {
if (name == null) {
return null;
}
try {
return Category.valueOf(name);
}

21
src/main/java/org/qortal/controller/Controller.java
View File

@ -47,6 +47,7 @@ import org.qortal.data.block.BlockData;
import org.qortal.data.block.BlockSummaryData;
import org.qortal.data.naming.NameData;
import org.qortal.data.network.PeerData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ChatTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.event.Event;
@ -400,6 +401,10 @@ public class Controller extends Thread {
RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl());
RepositoryManager.setRepositoryFactory(repositoryFactory);
RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
try (final Repository repository = RepositoryManager.getRepository()) {
RepositoryManager.buildInitialArbitraryResourcesCache(repository);
}
}
catch (DataException e) {
// If exception has no cause then repository is in use by some other process.
@ -891,6 +896,7 @@ public class Controller extends Thread {
if (now >= transaction.getDeadline()) {
LOGGER.debug(() -> String.format("Deleting expired, unconfirmed transaction %s", Base58.encode(transactionData.getSignature())));
repository.getTransactionRepository().delete(transactionData);
this.onExpiredTransaction(transactionData);
deletedCount++;
}
}
@ -1203,6 +1209,21 @@ public class Controller extends Thread {
});
}
/**
* Callback for when we've deleted an expired, unconfirmed transaction.
* <p>
* @implSpec performs actions in a new thread
*/
public void onExpiredTransaction(TransactionData transactionData) {
this.callbackExecutor.execute(() -> {
// If this is an ARBITRARY transaction, we may need to update the cache
if (transactionData.getType() == TransactionType.ARBITRARY) {
ArbitraryDataManager.getInstance().onExpiredArbitraryTransaction((ArbitraryTransactionData)transactionData);
}
});
}
public void onPeerHandshakeCompleted(Peer peer) {
// Only send if outbound
if (peer.isOutbound()) {

36
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -14,6 +14,7 @@ import org.qortal.arbitrary.ArbitraryDataResource;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.network.Network;
@ -539,6 +540,41 @@ public class ArbitraryDataManager extends Thread {
return true;
}
public void onExpiredArbitraryTransaction(ArbitraryTransactionData arbitraryTransactionData) {
if (arbitraryTransactionData.getName() == null) {
// No name, so we don't care about this transaction
return;
}
Service service = arbitraryTransactionData.getService();
String name = arbitraryTransactionData.getName();
String identifier = arbitraryTransactionData.getIdentifier();
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.service = service;
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
try (final Repository repository = RepositoryManager.getRepository()) {
// Find next oldest transaction (which is now the latest transaction)
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(name, service, null, identifier);
if (latestTransactionData == null) {
// There are no transactions anymore, so we can delete from the cache entirely (this deletes metadata too)
repository.getArbitraryRepository().delete(arbitraryResourceData);
}
else {
// We found the next oldest transaction, so we can update the cache
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, latestTransactionData);
arbitraryTransaction.updateArbitraryResourceCache();
arbitraryTransaction.updateArbitraryMetadataCache();
}
;
} catch (DataException e) {
// Not much we can do, so ignore for now
}
}
public int getPowDifficulty() {
return this.powDifficulty;
}

56
src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
View File

@ -15,6 +15,7 @@ import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.utils.Base58;
import org.qortal.utils.ListUtils;
import org.qortal.utils.NTP;
@ -324,37 +325,44 @@ public class ArbitraryMetadataManager {
Triple<String, Peer, Long> newEntry = new Triple<>(null, null, request.getC());
arbitraryMetadataRequests.put(message.getId(), newEntry);
ArbitraryTransactionData arbitraryTransactionData = null;
// Forwarding
if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
// Get transaction info
try (final Repository repository = RepositoryManager.getRepository()) {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (!(transactionData instanceof ArbitraryTransactionData))
return;
arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e);
// Get transaction info
try (final Repository repository = RepositoryManager.getRepository()) {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (!(transactionData instanceof ArbitraryTransactionData)) {
return;
}
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
// Check if the name is blocked
boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
if (!isBlocked) {
Peer requestingPeer = request.getB();
if (requestingPeer != null) {
// Forwarding
if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile());
forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId());
// Check if the name is blocked
boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
if (!isBlocked) {
Peer requestingPeer = request.getB();
if (requestingPeer != null) {
// Forward to requesting peer
LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
requestingPeer.disconnect("failed to forward arbitrary metadata");
ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile());
forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId());
// Forward to requesting peer
LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
requestingPeer.disconnect("failed to forward arbitrary metadata");
}
}
}
}
// Update arbitrary resource caches
if (arbitraryTransactionData != null) {
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, arbitraryTransactionData);
arbitraryTransaction.updateArbitraryResourceCache();
arbitraryTransaction.updateArbitraryMetadataCache();
}
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e);
}
}

10
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java → src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java
View File

@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlAccessorType;
import java.util.Objects;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryResourceInfo {
public class ArbitraryResourceData {
public String name;
public Service service;
@ -15,11 +15,11 @@ public class ArbitraryResourceInfo {
public ArbitraryResourceStatus status;
public ArbitraryResourceMetadata metadata;
public Long size;
public Integer size;
public Long created;
public Long updated;
public ArbitraryResourceInfo() {
public ArbitraryResourceData() {
}
@Override
@ -32,10 +32,10 @@ public class ArbitraryResourceInfo {
if (o == this)
return true;
if (!(o instanceof ArbitraryResourceInfo))
if (!(o instanceof ArbitraryResourceData))
return false;
ArbitraryResourceInfo other = (ArbitraryResourceInfo) o;
ArbitraryResourceData other = (ArbitraryResourceData) o;
return Objects.equals(this.name, other.name) &&
Objects.equals(this.service, other.service) &&

47
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java
View File

@ -18,6 +18,9 @@ public class ArbitraryResourceMetadata {
private List<String> files;
private String mimeType;
// Only included when updating database
private ArbitraryResourceData arbitraryResourceData;
public ArbitraryResourceMetadata() {
}
@ -60,4 +63,48 @@ public class ArbitraryResourceMetadata {
public List<String> getFiles() {
return this.files;
}
public void setTitle(String title) {
this.title = title;
}
public String getTitle() {
return this.title;
}
public void setDescription(String description) {
this.description = description;
}
public String getDescription() {
return this.description;
}
public void setTags(List<String> tags) {
this.tags = tags;
}
public List<String> getTags() {
return this.tags;
}
public void setCategory(Category category) {
this.category = category;
// Also set categoryName
if (category != null) {
this.categoryName = category.getName();
}
}
public Category getCategory() {
return this.category;
}
public void setArbitraryResourceData(ArbitraryResourceData arbitraryResourceData) {
this.arbitraryResourceData = arbitraryResourceData;
}
public ArbitraryResourceData getArbitraryResourceData() {
return this.arbitraryResourceData;
}
}

2
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java
View File

@ -9,7 +9,7 @@ import java.util.List;
public class ArbitraryResourceNameInfo {
public String name;
public List<ArbitraryResourceInfo> resources = new ArrayList<>();
public List<ArbitraryResourceData> resources = new ArrayList<>();
public ArbitraryResourceNameInfo() {
}

30
src/main/java/org/qortal/repository/ArbitraryRepository.java
View File

@ -1,9 +1,8 @@
package org.qortal.repository;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
import org.qortal.data.network.ArbitraryPeerData;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
@ -11,23 +10,42 @@ import java.util.List;
public interface ArbitraryRepository {
// Utils
public boolean isDataLocal(byte[] signature) throws DataException;
public byte[] fetchData(byte[] signature) throws DataException;
// Transaction related
public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException;
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException;
// Resource related
public List<ArbitraryResourceInfo> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException;
public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException;
public List<ArbitraryResourceNameInfo> getArbitraryResourceCreatorNames(Service service, String identifier, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceData> getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException;
// Arbitrary resources cache save/load
public void save(ArbitraryResourceData arbitraryResourceData) throws DataException;
public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException;
public void save(ArbitraryResourceMetadata metadata) throws DataException;
public void delete(ArbitraryResourceMetadata metadata) throws DataException;
}

71
src/main/java/org/qortal/repository/RepositoryManager.java
View File

@ -2,8 +2,16 @@ package org.qortal.repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.settings.Settings;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction;
import java.sql.SQLException;
import java.util.List;
import java.util.concurrent.TimeoutException;
public abstract class RepositoryManager {
@ -56,6 +64,69 @@ public abstract class RepositoryManager {
}
}
public static boolean buildInitialArbitraryResourcesCache(Repository repository) throws DataException {
if (Settings.getInstance().isLite()) {
// Lite nodes have no blockchain
return false;
}
try {
// Check if QDNResources table is empty
List<ArbitraryResourceData> resources = repository.getArbitraryRepository().getArbitraryResources(10, 0, false);
if (!resources.isEmpty()) {
// Resources exist in the cache, so assume complete.
// We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so
// we shouldn't ever be left in a partially rebuilt state.
LOGGER.debug("Arbitrary resources cache already built");
return false;
}
LOGGER.info("Building arbitrary resources cache...");
final int batchSize = 100;
int offset = 0;
// Loop through all ARBITRARY transactions, and determine latest state
while (!Controller.isStopping()) {
LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false);
if (signatures.isEmpty()) {
// Complete
break;
}
// Expand signatures to transactions
for (byte[] signature : signatures) {
ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository
.getTransactionRepository().fromSignature(signature);
if (transactionData.getService() == null) {
// Unsupported service - ignore this resource
continue;
}
// Update arbitrary resource caches
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCache();
arbitraryTransaction.updateArbitraryMetadataCache();
}
offset += batchSize;
}
repository.saveChanges();
LOGGER.info("Completed build of initial arbitrary resources cache.");
return true;
}
catch (DataException e) {
LOGGER.info("Unable to build initial arbitrary resources cache: {}. The database may have been left in an inconsistent state.", e.getMessage());
// Throw an exception so that the node startup is halted, allowing for a retry next time.
repository.discardChanges();
throw new DataException("Build of initial arbitrary resources cache failed.");
}
}
public static void setRequestedCheckpoint(Boolean quick) {
quickCheckpointRequested = quick;
}

478
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@ -2,10 +2,10 @@ package org.qortal.repository.hsqldb;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.util.Longs;
import org.qortal.arbitrary.misc.Category;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.data.transaction.BaseTransactionData;
@ -22,6 +22,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class HSQLDBArbitraryRepository implements ArbitraryRepository {
@ -41,6 +42,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
return (ArbitraryTransactionData) transactionData;
}
// Utils
@Override
public boolean isDataLocal(byte[] signature) throws DataException {
ArbitraryTransactionData transactionData = getTransactionData(signature);
@ -113,6 +117,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
return null;
}
// Transaction related
@Override
public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
// Already hashed? Nothing to do
@ -211,8 +218,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
}
@Override
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException {
StringBuilder sql = new StringBuilder(1024);
sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
@ -228,7 +234,16 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
sql.append(method.value);
}
sql.append("ORDER BY created_when DESC LIMIT 1");
sql.append(" ORDER BY created_when");
if (firstNotLast) {
sql.append(" ASC");
}
else {
sql.append(" DESC");
}
sql.append(" LIMIT 1");
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) {
if (resultSet == null)
@ -284,13 +299,189 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
@Override
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, String identifier, List<String> names,
boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked,
Integer limit, Integer offset, Boolean reverse) throws DataException {
public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException {
return this.getSingleTransaction(name, service, method, identifier, true);
}
@Override
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
return this.getSingleTransaction(name, service, method, identifier, false);
}
// Resource related
@Override
public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException {
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
sql.append("SELECT name, service, identifier, MAX(size) AS max_size FROM ArbitraryTransactions WHERE 1=1");
// Name is required
if (name == null) {
return null;
}
sql.append("SELECT name, service, identifier, size, created_when, updated_when, " +
"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
"FROM ArbitraryResourcesCache " +
"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
"WHERE service = ? AND name = ?");
bindParams.add(service.value);
bindParams.add(name);
if (identifier != null) {
sql.append(" AND identifier = ?");
bindParams.add(identifier);
}
else {
sql.append(" AND identifier IS NULL");
}
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return null;
String nameResult = resultSet.getString(1);
Service serviceResult = Service.valueOf(resultSet.getInt(2));
String identifierResult = resultSet.getString(3);
Integer sizeResult = resultSet.getInt(4);
Long created = resultSet.getLong(5);
Long updated = resultSet.getLong(6);
// Optional metadata fields
String title = resultSet.getString(7);
String description = resultSet.getString(8);
String category = resultSet.getString(9);
String tag1 = resultSet.getString(10);
String tag2 = resultSet.getString(11);
String tag3 = resultSet.getString(12);
String tag4 = resultSet.getString(13);
String tag5 = resultSet.getString(14);
if (Objects.equals(identifierResult, "default")) {
// Map "default" back to null. This is optional but probably less confusing than returning "default".
identifierResult = null;
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.name = nameResult;
arbitraryResourceData.service = serviceResult;
arbitraryResourceData.identifier = identifierResult;
arbitraryResourceData.size = sizeResult;
arbitraryResourceData.created = created;
arbitraryResourceData.updated = (updated == 0) ? null : updated;
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setTitle(title);
metadata.setDescription(description);
metadata.setCategory(Category.uncategorizedValueOf(category));
List<String> tags = new ArrayList<>();
if (tag1 != null) tags.add(tag1);
if (tag2 != null) tags.add(tag2);
if (tag3 != null) tags.add(tag3);
if (tag4 != null) tags.add(tag4);
if (tag5 != null) tags.add(tag5);
metadata.setTags(!tags.isEmpty() ? tags : null);
arbitraryResourceData.metadata = metadata;
return arbitraryResourceData;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary resource from repository", e);
}
}
@Override
public List<ArbitraryResourceData> getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
sql.append("SELECT name, service, identifier, size, created_when, updated_when, " +
"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
"FROM ArbitraryResourcesCache " +
"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
"WHERE name IS NOT NULL ORDER BY created_when");
if (reverse != null && reverse) {
sql.append(" DESC");
}
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return arbitraryResources;
do {
String nameResult = resultSet.getString(1);
Service serviceResult = Service.valueOf(resultSet.getInt(2));
String identifierResult = resultSet.getString(3);
Integer sizeResult = resultSet.getInt(4);
Long created = resultSet.getLong(5);
Long updated = resultSet.getLong(6);
// Optional metadata fields
String title = resultSet.getString(7);
String description = resultSet.getString(8);
String category = resultSet.getString(9);
String tag1 = resultSet.getString(10);
String tag2 = resultSet.getString(11);
String tag3 = resultSet.getString(12);
String tag4 = resultSet.getString(13);
String tag5 = resultSet.getString(14);
if (Objects.equals(identifierResult, "default")) {
// Map "default" back to null. This is optional but probably less confusing than returning "default".
identifierResult = null;
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.name = nameResult;
arbitraryResourceData.service = serviceResult;
arbitraryResourceData.identifier = identifierResult;
arbitraryResourceData.size = sizeResult;
arbitraryResourceData.created = created;
arbitraryResourceData.updated = (updated == 0) ? null : updated;
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setTitle(title);
metadata.setDescription(description);
metadata.setCategory(Category.uncategorizedValueOf(category));
List<String> tags = new ArrayList<>();
if (tag1 != null) tags.add(tag1);
if (tag2 != null) tags.add(tag2);
if (tag3 != null) tags.add(tag3);
if (tag4 != null) tags.add(tag4);
if (tag5 != null) tags.add(tag5);
metadata.setTags(!tags.isEmpty() ? tags : null);
arbitraryResourceData.metadata = metadata;
arbitraryResources.add(arbitraryResourceData);
} while (resultSet.next());
return arbitraryResources;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary resources from repository", e);
}
}
@Override
public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names,
boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked,
Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
sql.append("SELECT name, service, identifier, size, created_when, updated_when, " +
"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
"FROM ArbitraryResourcesCache " +
"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
"WHERE name IS NOT NULL");
if (service != null) {
sql.append(" AND service = ");
@ -351,7 +542,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
}
sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
sql.append(" ORDER BY created_when");
if (reverse != null && reverse) {
sql.append(" DESC");
@ -359,49 +550,82 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
List<ArbitraryResourceInfo> arbitraryResources = new ArrayList<>();
List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return null;
return arbitraryResources;
do {
String nameResult = resultSet.getString(1);
Service serviceResult = Service.valueOf(resultSet.getInt(2));
String identifierResult = resultSet.getString(3);
Integer sizeResult = resultSet.getInt(4);
Long created = resultSet.getLong(5);
Long updated = resultSet.getLong(6);
// We should filter out resources without names
if (nameResult == null) {
continue;
// Optional metadata fields
String title = resultSet.getString(7);
String description = resultSet.getString(8);
String category = resultSet.getString(9);
String tag1 = resultSet.getString(10);
String tag2 = resultSet.getString(11);
String tag3 = resultSet.getString(12);
String tag4 = resultSet.getString(13);
String tag5 = resultSet.getString(14);
if (Objects.equals(identifierResult, "default")) {
// Map "default" back to null. This is optional but probably less confusing than returning "default".
identifierResult = null;
}
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
arbitraryResourceInfo.name = nameResult;
arbitraryResourceInfo.service = serviceResult;
arbitraryResourceInfo.identifier = identifierResult;
arbitraryResourceInfo.size = Longs.valueOf(sizeResult);
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.name = nameResult;
arbitraryResourceData.service = serviceResult;
arbitraryResourceData.identifier = identifierResult;
arbitraryResourceData.size = sizeResult;
arbitraryResourceData.created = created;
arbitraryResourceData.updated = (updated == 0) ? null : updated;
arbitraryResources.add(arbitraryResourceInfo);
if (includeMetadata != null && includeMetadata) {
// TODO: we could avoid the join altogether
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setTitle(title);
metadata.setDescription(description);
metadata.setCategory(Category.uncategorizedValueOf(category));
List<String> tags = new ArrayList<>();
if (tag1 != null) tags.add(tag1);
if (tag2 != null) tags.add(tag2);
if (tag3 != null) tags.add(tag3);
if (tag4 != null) tags.add(tag4);
if (tag5 != null) tags.add(tag5);
metadata.setTags(!tags.isEmpty() ? tags : null);
arbitraryResourceData.metadata = metadata;
}
arbitraryResources.add(arbitraryResourceData);
} while (resultSet.next());
return arbitraryResources;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
throw new DataException("Unable to fetch arbitrary resources from repository", e);
}
}
@Override
public List<ArbitraryResourceInfo> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly,
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly,
List<String> exactMatchNames, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked,
Integer limit, Integer offset, Boolean reverse) throws DataException {
Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
sql.append("SELECT name, service, identifier, MAX(size) AS max_size, MIN(created_when) AS date_created, MAX(created_when) AS date_updated " +
"FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE 1=1");
sql.append("SELECT name, service, identifier, size, created_when, updated_when, " +
"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
"FROM ArbitraryResourcesCache " +
"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
"WHERE name IS NOT NULL");
if (service != null) {
sql.append(" AND service = ");
@ -492,7 +716,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
}
sql.append(" GROUP BY name, service, identifier ORDER BY date_created");
sql.append(" ORDER BY created_when");
if (reverse != null && reverse) {
sql.append(" DESC");
@ -500,98 +724,156 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
List<ArbitraryResourceInfo> arbitraryResources = new ArrayList<>();
List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return null;
return arbitraryResources;
do {
String nameResult = resultSet.getString(1);
Service serviceResult = Service.valueOf(resultSet.getInt(2));
String identifierResult = resultSet.getString(3);
Integer sizeResult = resultSet.getInt(4);
long dateCreated = resultSet.getLong(5);
long dateUpdated = resultSet.getLong(6);
Long created = resultSet.getLong(5);
Long updated = resultSet.getLong(6);
// We should filter out resources without names
if (nameResult == null) {
continue;
// Optional metadata fields
String title = resultSet.getString(7);
String description = resultSet.getString(8);
String category = resultSet.getString(9);
String tag1 = resultSet.getString(10);
String tag2 = resultSet.getString(11);
String tag3 = resultSet.getString(12);
String tag4 = resultSet.getString(13);
String tag5 = resultSet.getString(14);
if (Objects.equals(identifierResult, "default")) {
// Map "default" back to null. This is optional but probably less confusing than returning "default".
identifierResult = null;
}
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
arbitraryResourceInfo.name = nameResult;
arbitraryResourceInfo.service = serviceResult;
arbitraryResourceInfo.identifier = identifierResult;
arbitraryResourceInfo.size = Longs.valueOf(sizeResult);
arbitraryResourceInfo.created = dateCreated;
arbitraryResourceInfo.updated = dateUpdated;
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.name = nameResult;
arbitraryResourceData.service = serviceResult;
arbitraryResourceData.identifier = identifierResult;
arbitraryResourceData.size = sizeResult;
arbitraryResourceData.created = created;
arbitraryResourceData.updated = (updated == 0) ? null : updated;
arbitraryResources.add(arbitraryResourceInfo);
if (includeMetadata != null && includeMetadata) {
// TODO: we could avoid the join altogether
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setTitle(title);
metadata.setDescription(description);
metadata.setCategory(Category.uncategorizedValueOf(category));
List<String> tags = new ArrayList<>();
if (tag1 != null) tags.add(tag1);
if (tag2 != null) tags.add(tag2);
if (tag3 != null) tags.add(tag3);
if (tag4 != null) tags.add(tag4);
if (tag5 != null) tags.add(tag5);
metadata.setTags(!tags.isEmpty() ? tags : null);
arbitraryResourceData.metadata = metadata;
}
arbitraryResources.add(arbitraryResourceData);
} while (resultSet.next());
return arbitraryResources;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
throw new DataException("Unable to fetch arbitrary resources from repository", e);
}
}
// Arbitrary resources cache save/load
@Override
public void save(ArbitraryResourceData arbitraryResourceData) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryResourcesCache");
saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name)
.bind("identifier", arbitraryResourceData.identifier).bind("size", arbitraryResourceData.size)
.bind("created_when", arbitraryResourceData.created).bind("updated_when", arbitraryResourceData.updated);
try {
saveHelper.execute(this.repository);
} catch (SQLException e) {
throw new DataException("Unable to save arbitrary resource info into repository", e);
}
}
@Override
public List<ArbitraryResourceNameInfo> getArbitraryResourceCreatorNames(Service service, String identifier,
boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException {
// NOTE: arbitrary metadata are deleted automatically by the database thanks to "ON DELETE CASCADE"
// in ArbitraryMetadataCache' FOREIGN KEY definition.
try {
this.repository.delete("ArbitraryResourcesCache", "service = ? AND name = ? AND identifier = ?",
arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier);
sql.append("SELECT name FROM ArbitraryTransactions WHERE 1=1");
if (service != null) {
sql.append(" AND service = ");
sql.append(service.value);
}
if (defaultResource) {
// Default resource requested - use NULL identifier
// The AND ? IS NULL AND ? IS NULL is a hack to make use of the identifier params in checkedExecute()
identifier = null;
sql.append(" AND (identifier IS NULL AND ? IS NULL AND ? IS NULL)");
}
else {
// Non-default resource requested
// Use an exact match identifier, or list all if supplied identifier is null
sql.append(" AND (identifier = ? OR (? IS NULL))");
}
sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
if (reverse != null && reverse) {
sql.append(" DESC");
}
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
List<ArbitraryResourceNameInfo> arbitraryResources = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), identifier, identifier)) {
if (resultSet == null)
return null;
do {
String name = resultSet.getString(1);
// We should filter out resources without names
if (name == null) {
continue;
}
ArbitraryResourceNameInfo arbitraryResourceNameInfo = new ArbitraryResourceNameInfo();
arbitraryResourceNameInfo.name = name;
arbitraryResources.add(arbitraryResourceNameInfo);
} while (resultSet.next());
return arbitraryResources;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
throw new DataException("Unable to delete account from repository", e);
}
}
/* Arbitrary metadata cache */
@Override
public void save(ArbitraryResourceMetadata metadata) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryMetadataCache");
ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData();
if (arbitraryResourceData == null) {
throw new DataException("Can't save metadata without a referenced resource");
}
String tag1 = null;
String tag2 = null;
String tag3 = null;
String tag4 = null;
String tag5 = null;
List<String> tags = metadata.getTags();
if (tags != null) {
if (tags.size() > 0) tag1 = tags.get(0);
if (tags.size() > 1) tag2 = tags.get(1);
if (tags.size() > 2) tag3 = tags.get(2);
if (tags.size() > 3) tag4 = tags.get(3);
if (tags.size() > 4) tag5 = tags.get(4);
}
String category = metadata.getCategory() != null ? metadata.getCategory().toString() : null;
saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name)
.bind("identifier", arbitraryResourceData.identifier).bind("title", metadata.getTitle())
.bind("description", metadata.getDescription()).bind("category", category)
.bind("tag1", tag1).bind("tag2", tag2).bind("tag3", tag3).bind("tag4", tag4)
.bind("tag5", tag5);
try {
saveHelper.execute(this.repository);
} catch (SQLException e) {
throw new DataException("Unable to save arbitrary metadata into repository", e);
}
}
@Override
public void delete(ArbitraryResourceMetadata metadata) throws DataException {
ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData();
if (arbitraryResourceData == null) {
throw new DataException("Can't delete metadata without a referenced resource");
}
try {
this.repository.delete("ArbitraryMetadataCache", "service = ? AND name = ? AND identifier = ?",
arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier);
} catch (SQLException e) {
throw new DataException("Unable to delete account from repository", e);
}
}
}

40
src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
View File

@ -901,7 +901,7 @@ public class HSQLDBDatabaseUpdates {
case 37:
// ARBITRARY transaction updates for off-chain data storage
// We may want to use a nonce rather than a transaction fee on the data chain
// We may want to use a nonce rather than a transaction fee for ARBITRARY transactions
stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0");
// We need to know the total size of the data file(s) associated with each transaction
stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0");
@ -993,6 +993,44 @@ public class HSQLDBDatabaseUpdates {
stmt.execute("ALTER TABLE CancelSellNameTransactions ADD sale_price QortalAmount");
break;
case 47:
// We need to keep a local cache of arbitrary resources (items published to QDN), for easier searching.
// IMPORTANT: this is a cache of the last known state of a resource (both confirmed
// and valid unconfirmed). It cannot be assumed that all nodes will contain the same state at a
// given block height, and therefore must NOT be used for any consensus/validation code. It is
// simply a cache, to avoid having to query the raw transactions and the metadata in flat files
// when serving API requests.
// ARBITRARY transactions aren't really suitable for updating resources in the same way we'd update
// names or groups for instance, as there is no distinction between creations and updates, and metadata
// is off-chain. Plus, QDN allows (valid) unconfirmed data to be queried and viewed. It is very
// easy to keep a cache of the latest transaction's data, but anything more than that would need
// considerable thought (and most likely a rewrite).
stmt.execute("CREATE TABLE ArbitraryResourcesCache (service SMALLINT NOT NULL, "
+ "name RegisteredName NOT NULL, identifier VARCHAR(64), size INT NOT NULL, "
+ "created_when EpochMillis NOT NULL, updated_when EpochMillis, "
+ "PRIMARY KEY (service, name, identifier))");
// For finding resources by service.
stmt.execute("CREATE INDEX ArbitraryResourcesServiceIndex ON ArbitraryResourcesCache (service)");
// For finding resources by name.
stmt.execute("CREATE INDEX ArbitraryResourcesNameIndex ON ArbitraryResourcesCache (name)");
// For finding resources by identifier.
stmt.execute("CREATE INDEX ArbitraryResourcesIdentifierIndex ON ArbitraryResourcesCache (identifier)");
// For finding resources by creation date (the default column when ordering).
stmt.execute("CREATE INDEX ArbitraryResourcesCreatedIndex ON ArbitraryResourcesCache (created_when)");
// Use a separate table space as this table will be very large.
stmt.execute("SET TABLE ArbitraryResourcesCache NEW SPACE");
stmt.execute("CREATE TABLE ArbitraryMetadataCache (service SMALLINT NOT NULL, "
+ "name RegisteredName NOT NULL, identifier VARCHAR(64), "
+ "title VARCHAR(80), description VARCHAR(240), category VARCHAR(64), "
+ "tag1 VARCHAR(20), tag2 VARCHAR(20), tag3 VARCHAR(20), tag4 VARCHAR(20), tag5 VARCHAR(20), "
+ "PRIMARY KEY (service, name, identifier), FOREIGN KEY (service, name, identifier) "
+ "REFERENCES ArbitraryResourcesCache (service, name, identifier) ON DELETE CASCADE)");
// For finding metadata by title.
stmt.execute("CREATE INDEX ArbitraryMetadataTitleIndex ON ArbitraryMetadataCache (title)");
break;
default:
// nothing to do
return false;

130
src/main/java/org/qortal/transaction/ArbitraryTransaction.java
View File

@ -1,18 +1,21 @@
package org.qortal.transaction;
import java.util.Arrays;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import org.qortal.account.Account;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Service;
import org.qortal.block.BlockChain;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
import org.qortal.crypto.Crypto;
import org.qortal.crypto.MemoryPoW;
import org.qortal.data.PaymentData;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
@ -248,6 +251,10 @@ public class ArbitraryTransaction extends Transaction {
ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData);
}
}
// Add to arbitrary resource caches
this.updateArbitraryResourceCache();
this.updateArbitraryMetadataCache();
}
@Override
@ -304,4 +311,123 @@ public class ArbitraryTransaction extends Transaction {
return null;
}
/**
* Update the arbitrary resources cache.
* This finds the latest transaction and replaces the
* majority of the data in the cache. The current
* transaction is used for the created time,
* if it has a lower timestamp than the existing value.
* It's also used to identify the correct
* service/name/identifier combination.
*
* @throws DataException
*/
public void updateArbitraryResourceCache() throws DataException {
// Don't cache resources without a name (such as auto updates)
if (arbitraryTransactionData.getName() == null) {
return;
}
// Get the latest transaction
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
// We don't have a latest transaction, so give up
return;
}
Service service = arbitraryTransactionData.getService();
String name = arbitraryTransactionData.getName();
String identifier = arbitraryTransactionData.getIdentifier();
// In the cache we store null identifiers as "default", as it is part of the primary key
if (identifier == null) {
identifier = "default";
}
// Get existing cached entry if it exists
ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository()
.getArbitraryResource(service, name, identifier);
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.service = service;
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
// Check for existing cached data
if (existingArbitraryResourceData == null) {
// Nothing exists yet, so set everything from the newest transaction
arbitraryResourceData.created = latestTransactionData.getTimestamp();
arbitraryResourceData.updated = null;
}
else {
// An entry already exists - update created time from current transaction if this is older
arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp());
// Set updated time to the latest transaction's timestamp, unless it matches the creation time
if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) {
// Latest transaction matches created time, so it hasn't been updated
arbitraryResourceData.updated = null;
}
else {
arbitraryResourceData.updated = latestTransactionData.getTimestamp();
}
}
arbitraryResourceData.size = latestTransactionData.getSize();
// Save
repository.getArbitraryRepository().save(arbitraryResourceData);
}
public void updateArbitraryMetadataCache() throws DataException {
// Get the latest transaction
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
// We don't have a latest transaction, so give up
return;
}
Service service = latestTransactionData.getService();
String name = latestTransactionData.getName();
String identifier = latestTransactionData.getIdentifier();
// In the cache we store null identifiers as "default", as it is part of the primary key
if (identifier == null) {
identifier = "default";
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.service = service;
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
// Update metadata for latest transaction if it is local
if (latestTransactionData.getMetadataHash() != null) {
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
if (metadataFile.exists()) {
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
try {
transactionMetadata.read();
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
metadata.setTitle(transactionMetadata.getTitle());
metadata.setDescription(transactionMetadata.getDescription());
metadata.setCategory(transactionMetadata.getCategory());
metadata.setTags(transactionMetadata.getTags());
repository.getArbitraryRepository().save(metadata);
} catch (IOException e) {
// Ignore, as we can add it again later
}
} else {
// We don't have a local copy of this metadata file, so delete it from the cache
// It will be re-added if the file later arrives via the network
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
repository.getArbitraryRepository().delete(metadata);
}
}
}
}

37
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@ -4,10 +4,8 @@ import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.*;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
@ -258,8 +256,7 @@ public class ArbitraryTransactionUtils {
"chunks if needed", Base58.encode(completeHash));
ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, cleanupAfter);
}
else {
} else {
// File might be in use. It's best to leave it and it it will be cleaned up later.
}
}
@ -271,6 +268,7 @@ public class ArbitraryTransactionUtils {
* When first uploaded, files go into a _misc folder as they are not yet associated with a
* transaction signature. Once the transaction is broadcast, they need to be moved to the
* correct location, keyed by the transaction signature.
*
* @param arbitraryTransactionData
* @return
* @throws DataException
@ -356,8 +354,7 @@ public class ArbitraryTransactionUtils {
file.createNewFile();
}
}
}
catch (DataException | IOException e) {
} catch (DataException | IOException e) {
LOGGER.info("Unable to check and relocate all files for signature {}: {}",
Base58.encode(arbitraryTransactionData.getSignature()), e.getMessage());
}
@ -366,7 +363,7 @@ public class ArbitraryTransactionUtils {
}
public static List<ArbitraryTransactionData> limitOffsetTransactions(List<ArbitraryTransactionData> transactions,
Integer limit, Integer offset) {
Integer limit, Integer offset) {
if (limit != null && limit == 0) {
limit = null;
}
@ -389,6 +386,7 @@ public class ArbitraryTransactionUtils {
/**
* Lookup status of resource
*
* @param service
* @param name
* @param identifier
@ -413,10 +411,10 @@ public class ArbitraryTransactionUtils {
return resource.getStatus(false);
}
public static List<ArbitraryResourceInfo> addStatusToResources(List<ArbitraryResourceInfo> resources) {
public static List<ArbitraryResourceData> addStatusToResources(List<ArbitraryResourceData> resources) {
// Determine and add the status of each resource
List<ArbitraryResourceInfo> updatedResources = new ArrayList<>();
for (ArbitraryResourceInfo resourceInfo : resources) {
List<ArbitraryResourceData> updatedResources = new ArrayList<>();
for (ArbitraryResourceData resourceInfo : resources) {
try {
ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME,
resourceInfo.service, resourceInfo.identifier);
@ -433,21 +431,4 @@ public class ArbitraryTransactionUtils {
}
return updatedResources;
}
public static List<ArbitraryResourceInfo> addMetadataToResources(List<ArbitraryResourceInfo> resources) {
// Add metadata fields to each resource if they exist
List<ArbitraryResourceInfo> updatedResources = new ArrayList<>();
for (ArbitraryResourceInfo resourceInfo : resources) {
ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME,
resourceInfo.service, resourceInfo.identifier);
ArbitraryDataTransactionMetadata transactionMetadata = resource.getLatestTransactionMetadata();
ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata, false);
if (resourceMetadata != null) {
resourceInfo.metadata = resourceMetadata;
}
updatedResources.add(resourceInfo);
}
return updatedResources;
}
}