Qortal/qortal
1
0
Fork 1
mirror of https://github.com/Qortal/qortal.git synced 2025-07-23 04:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Connected the rest of the system up to the recently added "identifier" feature.

Browse Source
This commit is contained in:
CalDescent
2021-11-11 09:12:54 +00:00
parent a364206159
commit 4b1a5a5e14
13 changed files with 202 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@@ -290,39 +290,34 @@ public class ArbitraryResource {
@QueryParam("rebuild") boolean rebuild) {
Security.checkApiCallAllowed(request);
if (filepath == null) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing filepath");
}
return this.download(serviceString, name, null, filepath, rebuild);
}
Service service = Service.valueOf(serviceString);
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service);
try {
// Loop until we have data
while (!Controller.isStopping()) {
try {
arbitraryDataReader.loadSynchronously(rebuild);
break;
} catch (MissingDataException e) {
continue;
}
@GET
@Path("/{service}/{name}/{identifier}")
@Operation(
summary = "Fetch raw data from file with supplied service, name, identifier, and relative path",
description = "An optional rebuild boolean can be supplied. If true, any existing cached data will be invalidated.",
responses = {
@ApiResponse(
description = "Path to file structure containing requested data",
content = @Content(
mediaType = MediaType.TEXT_PLAIN,
schema = @Schema(
type = "string"
)
)
)
}
)
public HttpServletResponse get(@PathParam("service") String serviceString,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("filepath") String filepath,
@QueryParam("rebuild") boolean rebuild) {
Security.checkApiCallAllowed(request);
// TODO: limit file size that can be read into memory
java.nio.file.Path path = Paths.get(arbitraryDataReader.getFilePath().toString(), filepath);
if (!Files.exists(path)) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
}
byte[] data = Files.readAllBytes(path);
response.setContentType(context.getMimeType(path.toString()));
response.setContentLength(data.length);
response.getOutputStream().write(data);
return response;
} catch (Exception e) {
LOGGER.info(String.format("Unable to load %s %s: %s", service, name, e.getMessage()));
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
}
return this.download(serviceString, name, identifier, filepath, rebuild);
}
@POST
@@ -432,6 +427,7 @@ public class ArbitraryResource {
return this.upload(Method.PATCH, Service.valueOf(serviceString), name, identifier, path);
}
private String upload(Method method, Service service, String name, String identifier, String path) {
// It's too dangerous to allow user-supplied file paths in weaker security contexts
if (Settings.getInstance().isApiRestricted()) {
@@ -470,6 +466,43 @@ public class ArbitraryResource {
}
}
private HttpServletResponse download(String serviceString, String name, String identifier, String filepath, boolean rebuild) {
if (filepath == null) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Missing filepath");
}
Service service = Service.valueOf(serviceString);
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
try {
// Loop until we have data
while (!Controller.isStopping()) {
try {
arbitraryDataReader.loadSynchronously(rebuild);
break;
} catch (MissingDataException e) {
continue;
}
}
// TODO: limit file size that can be read into memory
java.nio.file.Path path = Paths.get(arbitraryDataReader.getFilePath().toString(), filepath);
if (!Files.exists(path)) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
}
byte[] data = Files.readAllBytes(path);
response.setContentType(context.getMimeType(path.toString()));
response.setContentLength(data.length);
response.getOutputStream().write(data);
return response;
} catch (Exception e) {
LOGGER.info(String.format("Unable to load %s %s: %s", service, name, e.getMessage()));
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
}
}
@DELETE
@Path("/file")

4
src/main/java/org/qortal/api/resource/WebsiteResource.java
View File

@@ -93,7 +93,7 @@ public class WebsiteResource {
Method method = Method.PUT;
Compression compression = Compression.ZIP;
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, method, compression);
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, null, method, compression);
try {
arbitraryDataWriter.save();
} catch (IOException | DataException | InterruptedException | MissingDataException e) {
@@ -178,7 +178,7 @@ public class WebsiteResource {
}
Service service = Service.WEBSITE;
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service);
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, null);
arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
try {
if (!arbitraryDataReader.isCachedDataAvailable()) {

8
src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
View File

@@ -13,6 +13,7 @@ public class ArbitraryDataBuildQueueItem {
private String resourceId;
private ResourceIdType resourceIdType;
private Service service;
private String identifier;
private Long creationTimestamp = null;
private Long buildStartTimestamp = null;
private Long buildEndTimestamp = null;
@@ -24,10 +25,11 @@ public class ArbitraryDataBuildQueueItem {
/* The amount of time to remember that a build has failed, to avoid retries */
public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes
public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) {
public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
this.resourceId = resourceId.toLowerCase();
this.resourceIdType = resourceIdType;
this.service = service;
this.identifier = identifier;
this.creationTimestamp = NTP.getTime();
}
@@ -39,7 +41,7 @@ public class ArbitraryDataBuildQueueItem {
this.buildStartTimestamp = now;
ArbitraryDataReader arbitraryDataReader =
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service);
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);
try {
arbitraryDataReader.loadSynchronously(true);
@@ -86,7 +88,7 @@ public class ArbitraryDataBuildQueueItem {
@Override
public String toString() {
return String.format("%s %s", this.service, this.resourceId);
return String.format("%s %s %s", this.service, this.resourceId, this.identifier);
}
}

27
src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
View File

@@ -28,6 +28,7 @@ public class ArbitraryDataBuilder {
private String name;
private Service service;
private String identifier;
private List<ArbitraryTransactionData> transactions;
private ArbitraryTransactionData latestPutTransaction;
@@ -35,9 +36,10 @@ public class ArbitraryDataBuilder {
private byte[] latestSignature;
private Path finalPath;
public ArbitraryDataBuilder(String name, Service service) {
public ArbitraryDataBuilder(String name, Service service, String identifier) {
this.name = name;
this.service = service;
this.identifier = identifier;
this.paths = new ArrayList<>();
}
@@ -56,16 +58,17 @@ public class ArbitraryDataBuilder {
// Get the most recent PUT
ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
.getLatestTransaction(this.name, this.service, Method.PUT);
.getLatestTransaction(this.name, this.service, Method.PUT, this.identifier);
if (latestPut == null) {
throw new IllegalStateException(String.format(
"Couldn't find PUT transaction for name %s and service %s", this.name, this.service));
String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s",
this.name, this.service, this.identifierString());
throw new IllegalStateException(message);
}
this.latestPutTransaction = latestPut;
// Load all transactions since the latest PUT
List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository()
.getArbitraryTransactions(this.name, this.service, latestPut.getTimestamp());
.getArbitraryTransactions(this.name, this.service, this.identifier, latestPut.getTimestamp());
this.transactions = transactionDataList;
}
}
@@ -81,8 +84,8 @@ public class ArbitraryDataBuilder {
throw new IllegalStateException("Expected PUT but received PATCH");
}
if (transactionDataList.size() == 0) {
throw new IllegalStateException(String.format("No transactions found for name %s, service %s, since %d",
name, service, latestPut.getTimestamp()));
throw new IllegalStateException(String.format("No transactions found for name %s, service %s, " +
"identifier: %s, since %d", name, service, this.identifierString(), latestPut.getTimestamp()));
}
// Verify that the signature of the first transaction matches the latest PUT
@@ -115,7 +118,8 @@ public class ArbitraryDataBuilder {
// Build the data file, overwriting anything that was previously there
String sig58 = Base58.encode(transactionData.getSignature());
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service);
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA,
this.service, this.identifier);
arbitraryDataReader.setTransactionData(transactionData);
boolean hasMissingData = false;
try {
@@ -179,7 +183,8 @@ public class ArbitraryDataBuilder {
// Loop from the second path onwards
for (int i=1; i<paths.size(); i++) {
LOGGER.info(String.format("[%s][%s] Applying layer %d...", this.service, this.name, i));
String identifierPrefix = this.identifier != null ? String.format("[%s]", this.identifier) : "";
LOGGER.info(String.format("[%s][%s]%s Applying layer %d...", this.service, this.name, identifierPrefix, i));
// Create an instance of ArbitraryDataCombiner
Path pathAfter = this.paths.get(i);
@@ -211,6 +216,10 @@ public class ArbitraryDataBuilder {
cache.write();
}
private String identifierString() {
return identifier != null ? identifier : "";
}
public Path getFinalPath() {
return this.finalPath;
}

6
src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java
View File

@@ -22,14 +22,16 @@ public class ArbitraryDataCache {
private String resourceId;
private ResourceIdType resourceIdType;
private Service service;
private String identifier;
public ArbitraryDataCache(Path filePath, boolean overwrite, String resourceId,
ResourceIdType resourceIdType, Service service) {
ResourceIdType resourceIdType, Service service, String identifier) {
this.filePath = filePath;
this.overwrite = overwrite;
this.resourceId = resourceId;
this.resourceIdType = resourceIdType;
this.service = service;
this.identifier = identifier;
}
public boolean isCachedDataAvailable() {
@@ -134,7 +136,7 @@ public class ArbitraryDataCache {
// Find latest transaction for name and service, with any method
ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
.getLatestTransaction(this.resourceId, this.service, null);
.getLatestTransaction(this.resourceId, this.service, null, this.identifier);
if (latestTransaction != null) {
return latestTransaction.getSignature();

23
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@@ -42,6 +42,7 @@ public class ArbitraryDataReader {
private String resourceId;
private ResourceIdType resourceIdType;
private Service service;
private String identifier;
private ArbitraryTransactionData transactionData;
private String secret58;
private Path filePath;
@@ -51,7 +52,7 @@ public class ArbitraryDataReader {
private Path uncompressedPath;
private Path unencryptedPath;
public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service) {
public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
// Ensure names are always lowercase
if (resourceIdType == ResourceIdType.NAME) {
resourceId = resourceId.toLowerCase();
@@ -60,25 +61,31 @@ public class ArbitraryDataReader {
this.resourceId = resourceId;
this.resourceIdType = resourceIdType;
this.service = service;
this.identifier = identifier;
this.workingPath = this.buildWorkingPath();
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
}
private Path buildWorkingPath() {
// Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
String baseDir = Settings.getInstance().getTempDataPath();
this.workingPath = Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString());
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
String identifier = this.identifier != null ? this.identifier : "default";
return Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier);
}
public boolean isCachedDataAvailable() {
// If this resource is in the build queue then we shouldn't attempt to serve
// cached data, as it may not be fully built
ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service);
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service, this.identifier);
if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
return false;
}
// Not in the build queue - so check the cache itself
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false,
this.resourceId, this.resourceIdType, this.service);
this.resourceId, this.resourceIdType, this.service, this.identifier);
if (cache.isCachedDataAvailable()) {
this.filePath = this.uncompressedPath;
return true;
@@ -98,7 +105,7 @@ public class ArbitraryDataReader {
*/
public boolean loadAsynchronously() {
ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service);
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service, this.identifier);
return ArbitraryDataBuildManager.getInstance().addToBuildQueue(queueItem);
}
@@ -117,7 +124,7 @@ public class ArbitraryDataReader {
public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException, MissingDataException {
try {
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
this.resourceId, this.resourceIdType, this.service);
this.resourceId, this.resourceIdType, this.service, this.identifier);
if (cache.isCachedDataAvailable()) {
// Use cached data
this.filePath = this.uncompressedPath;
@@ -233,7 +240,7 @@ public class ArbitraryDataReader {
try {
// Build the existing state using past transactions
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service);
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service, this.identifier);
builder.build();
Path builtPath = builder.getFinalPath();
if (builtPath == null) {

4
src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java
View File

@@ -15,9 +15,7 @@ import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.transform.TransformationException;
import org.qortal.transform.Transformer;
import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
@@ -69,7 +67,7 @@ public class ArbitraryDataTransactionBuilder {
ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP;
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, method, compression);
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, compression);
try {
arbitraryDataWriter.save();
} catch (IOException | DataException | InterruptedException | RuntimeException | MissingDataException e) {

6
src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java
View File

@@ -34,6 +34,7 @@ public class ArbitraryDataWriter {
private Path filePath;
private String name;
private Service service;
private String identifier;
private Method method;
private Compression compression;
@@ -45,10 +46,11 @@ public class ArbitraryDataWriter {
private Path compressedPath;
private Path encryptedPath;
public ArbitraryDataWriter(Path filePath, String name, Service service, Method method, Compression compression) {
public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression) {
this.filePath = filePath;
this.name = name;
this.service = service;
this.identifier = identifier;
this.method = method;
this.compression = compression;
}
@@ -114,7 +116,7 @@ public class ArbitraryDataWriter {
private void processPatch() throws DataException, IOException, MissingDataException {
// Build the existing state using past transactions
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service);
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service, this.identifier);
builder.build();
Path builtPath = builder.getFinalPath();

1
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java
View File

@@ -10,6 +10,7 @@ public class ArbitraryResourceInfo {
public String name;
public ArbitraryTransactionData.Service service;
public String identifier;
public ArbitraryResourceInfo() {
}

4
src/main/java/org/qortal/repository/ArbitraryRepository.java
View File

@@ -17,9 +17,9 @@ public interface ArbitraryRepository {
public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, long since) throws DataException;
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException;
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException;

22
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@@ -153,17 +153,18 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
@Override
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, long since) throws DataException {
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException {
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE lower(name) = ? AND service = ? AND created_when >= ? " +
"ORDER BY created_when ASC";
"WHERE lower(name) = ? AND service = ?" +
"AND (identifier = ? OR (identifier IS NULL AND ? IS NULL))" +
"AND created_when >= ? ORDER BY created_when ASC";
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql, name.toLowerCase(), service.value, since)) {
try (ResultSet resultSet = this.repository.checkedExecute(sql, name.toLowerCase(), service.value, identifier, identifier, since)) {
if (resultSet == null)
return null;
@@ -221,7 +222,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
@Override
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException {
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
StringBuilder sql = new StringBuilder(1024);
sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
@@ -229,7 +230,8 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
"version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE lower(name) = ? AND service = ?");
"WHERE lower(name) = ? AND service = ? " +
"AND (identifier = ? OR (identifier IS NULL AND ? IS NULL))");
if (method != null) {
sql.append(" AND update_method = ");
@@ -238,7 +240,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
sql.append("ORDER BY created_when DESC LIMIT 1");
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value)) {
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) {
if (resultSet == null)
return null;
@@ -295,14 +297,14 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
public List<ArbitraryResourceInfo> getArbitraryResources(Service service, Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
sql.append("SELECT name, service FROM ArbitraryTransactions");
sql.append("SELECT name, service, identifier FROM ArbitraryTransactions");
if (service != null) {
sql.append(" WHERE service = ");
sql.append(service.value);
}
sql.append(" GROUP BY name, service ORDER BY name");
sql.append(" GROUP BY name, service, identifier ORDER BY name");
if (reverse != null && reverse) {
sql.append(" DESC");
@@ -319,6 +321,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
do {
String name = resultSet.getString(1);
Service serviceResult = Service.valueOf(resultSet.getInt(2));
String identifier = resultSet.getString(3);
// We should filter out resources without names
if (name == null) {
@@ -328,6 +331,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
arbitraryResourceInfo.name = name;
arbitraryResourceInfo.service = serviceResult;
arbitraryResourceInfo.identifier = identifier;
arbitraryResources.add(arbitraryResourceInfo);
} while (resultSet.next());

3
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@@ -39,6 +39,7 @@ public class ArbitraryTransactionUtils {
String name = arbitraryTransactionData.getName();
ArbitraryTransactionData.Service service = arbitraryTransactionData.getService();
String identifier = arbitraryTransactionData.getIdentifier();
if (name == null || service == null) {
return null;
@@ -48,7 +49,7 @@ public class ArbitraryTransactionUtils {
ArbitraryTransactionData latestPut;
try {
latestPut = repository.getArbitraryRepository()
.getLatestTransaction(name, service, ArbitraryTransactionData.Method.PUT);
.getLatestTransaction(name, service, ArbitraryTransactionData.Method.PUT, identifier);
} catch (DataException e) {
return null;
}