Qortal/qortal
1
0
Fork 1
mirror of https://github.com/Qortal/qortal.git synced 2025-07-23 04:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework of "Service" types to allow for validation

Browse Source 
Each service supports basic validation params, plus has the option for an entirely custom validation function.

Initial validation settings:
- IMAGE must be less than 10MiB
- THUMBNAIL must be less than 500KiB
- METADATA must be less than 10KiB and must contain JSON keys "title", "description", and "tags"
This commit is contained in:
CalDescent
2021-11-16 19:28:25 +00:00
parent 9c952785e6
commit fb09d77cdc
25 changed files with 266 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@@ -30,6 +30,7 @@ import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.ArbitraryDataTransactionBuilder;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.account.AccountData;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;

1
src/main/java/org/qortal/api/resource/WebsiteResource.java
View File

@@ -22,6 +22,7 @@ import org.apache.logging.log4j.Logger;
import org.qortal.api.ApiError;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.arbitrary.misc.Service;
import org.qortal.arbitrary.*;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.data.transaction.ArbitraryTransactionData.*;

2
src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
View File

@@ -1,8 +1,8 @@
package org.qortal.arbitrary;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.arbitrary.misc.Service;
import org.qortal.repository.DataException;
import org.qortal.utils.NTP;

2
src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
View File

@@ -4,9 +4,9 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.Method;
import org.qortal.data.transaction.ArbitraryTransactionData.Service;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;

2
src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java
View File

@@ -2,9 +2,9 @@ package org.qortal.arbitrary;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;

18
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@@ -1,10 +1,12 @@
package org.qortal.arbitrary;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
@@ -26,6 +28,7 @@ import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import javax.xml.crypto.Data;
import java.io.File;
import java.io.IOException;
import java.io.InvalidObjectException;
@@ -146,6 +149,7 @@ public class ArbitraryDataReader {
this.fetch();
this.decrypt();
this.uncompress();
this.validate();
} finally {
this.postExecute();
@@ -425,6 +429,20 @@ public class ArbitraryDataReader {
this.filePath = this.uncompressedPath;
}
private void validate() throws IOException, DataException {
if (this.service.isValidationRequired()) {
byte[] data = FilesystemUtils.getSingleFileContents(this.filePath);
long size = FilesystemUtils.getDirectorySize(this.filePath);
Service.ValidationResult result = this.service.validate(data, size);
if (result != Service.ValidationResult.OK) {
throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
}
}
}
private void moveFilePathToFinalDestination() throws IOException {
if (this.filePath.compareTo(this.uncompressedPath) != 0) {
File source = new File(this.filePath.toString());

5
src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
View File

@@ -6,8 +6,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.HTMLParser;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.arbitrary.misc.Service;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
@@ -60,7 +59,7 @@ public class ArbitraryDataRenderer {
inPath = File.separator + inPath;
}
ArbitraryTransactionData.Service service = Service.WEBSITE;
Service service = Service.WEBSITE;
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, null);
arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
try {

1
src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java
View File

@@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
import org.qortal.arbitrary.misc.Service;
import org.qortal.block.BlockChain;
import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData;

15
src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java
View File

@@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.crypto.Crypto;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.crypto.AES;
@@ -58,6 +59,7 @@ public class ArbitraryDataWriter {
public void save() throws IllegalStateException, IOException, DataException, InterruptedException, MissingDataException {
try {
this.preExecute();
this.validateService();
this.process();
this.compress();
this.encrypt();
@@ -97,6 +99,19 @@ public class ArbitraryDataWriter {
this.workingPath = tempDir;
}
private void validateService() throws IOException, DataException {
if (this.service.isValidationRequired()) {
byte[] data = FilesystemUtils.getSingleFileContents(this.filePath);
long size = FilesystemUtils.getDirectorySize(this.filePath);
Service.ValidationResult result = this.service.validate(data, size);
if (result != Service.ValidationResult.OK) {
throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
}
}
}
private void process() throws DataException, IOException, MissingDataException {
switch (this.method) {

105
src/main/java/org/qortal/arbitrary/misc/Service.java Normal file
View File

@@ -0,0 +1,105 @@
package org.qortal.arbitrary.misc;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
import org.qortal.transaction.Transaction;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap;
public enum Service {
AUTO_UPDATE(1, false, null, null),
ARBITRARY_DATA(100, false, null, null),
WEBSITE(200, false, null, null),
GIT_REPOSITORY(300, false, null, null),
IMAGE(400, true, 10*1024*1024L, null),
THUMBNAIL(410, true, 500*1024L, null),
VIDEO(500, false, null, null),
AUDIO(600, false, null, null),
BLOG(700, false, null, null),
BLOG_POST(777, false, null, null),
BLOG_COMMENT(778, false, null, null),
DOCUMENT(800, false, null, null),
PLAYLIST(900, true, null, null),
APP(1000, false, null, null),
METADATA(1100, true, 10*1024L, Arrays.asList("title", "description", "tags"));
public final int value;
private final boolean requiresValidation;
private final Long maxSize;
private final List<String> requiredKeys;
private static final Logger LOGGER = LogManager.getLogger(Service.class);
private static final Map<Integer, Service> map = stream(Service.values())
.collect(toMap(service -> service.value, service -> service));
Service(int value, boolean requiresValidation, Long maxSize, List<String> requiredKeys) {
this.value = value;
this.requiresValidation = requiresValidation;
this.maxSize = maxSize;
this.requiredKeys = requiredKeys;
}
public ValidationResult validate(byte[] data, long size) {
if (!this.isValidationRequired()) {
return ValidationResult.OK;
}
// Validate max size if needed
if (this.maxSize != null) {
if (size > this.maxSize || data.length > this.maxSize) {
return ValidationResult.EXCEEDS_SIZE_LIMIT;
}
}
// Validate required keys if needed
if (this.requiredKeys != null) {
JSONObject json = Service.toJsonObject(data);
for (String key : this.requiredKeys) {
if (!json.has(key)) {
return ValidationResult.MISSING_KEYS;
}
}
}
// Validation passed
return ValidationResult.OK;
}
public boolean isValidationRequired() {
return this.requiresValidation;
}
public static Service valueOf(int value) {
return map.get(value);
}
public static JSONObject toJsonObject(byte[] data) {
String dataString = new String(data);
return new JSONObject(dataString);
}
public enum ValidationResult {
OK(1),
MISSING_KEYS(2),
EXCEEDS_SIZE_LIMIT(3);
public final int value;
private static final Map<Integer, Transaction.ValidationResult> map = stream(Transaction.ValidationResult.values()).collect(toMap(result -> result.value, result -> result));
ValidationResult(int value) {
this.value = value;
}
public static Transaction.ValidationResult valueOf(int value) {
return map.get(value);
}
}
}

4
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java
View File

@@ -1,6 +1,6 @@
package org.qortal.data.arbitrary;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -9,7 +9,7 @@ import javax.xml.bind.annotation.XmlAccessorType;
public class ArbitraryResourceInfo {
public String name;
public ArbitraryTransactionData.Service service;
public Service service;
public String identifier;
public ArbitraryResourceInfo() {

33
src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java
View File

@@ -8,6 +8,7 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import org.eclipse.persistence.oxm.annotations.XmlDiscriminatorValue;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.PaymentData;
import org.qortal.transaction.Transaction.TransactionType;
@@ -29,38 +30,6 @@ public class ArbitraryTransactionData extends TransactionData {
DATA_HASH;
}
// Service types
public enum Service {
AUTO_UPDATE(1),
ARBITRARY_DATA(100),
WEBSITE(200),
GIT_REPOSITORY(300),
IMAGE(400),
THUMBNAIL(410),
VIDEO(500),
AUDIO(600),
BLOG(700),
BLOG_POST(777),
BLOG_COMMENT(778),
DOCUMENT(800),
PLAYLIST(900),
APP(1000),
METADATA(1100);
public final int value;
private static final Map<Integer, Service> map = stream(Service.values())
.collect(toMap(service -> service.value, service -> service));
Service(int value) {
this.value = value;
}
public static Service valueOf(int value) {
return map.get(value);
}
}
// Methods
public enum Method {
PUT(0), // A complete replacement of a resource

1
src/main/java/org/qortal/repository/ArbitraryRepository.java
View File

@@ -1,5 +1,6 @@
package org.qortal.repository;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.data.network.ArbitraryPeerData;
import org.qortal.data.transaction.ArbitraryTransactionData;

2
src/main/java/org/qortal/repository/TransactionRepository.java
View File

@@ -5,8 +5,8 @@ import java.util.List;
import java.util.Map;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.group.GroupApprovalData;
import org.qortal.data.transaction.ArbitraryTransactionData.Service;
import org.qortal.data.transaction.GroupApprovalTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.data.transaction.TransferAssetTransactionData;

2
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@@ -1,8 +1,8 @@
package org.qortal.repository.hsqldb;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.arbitrary.ArbitraryResourceInfo;
import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData;
import org.qortal.data.network.ArbitraryPeerData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;

3
src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java
View File

@@ -4,6 +4,7 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.PaymentData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.BaseTransactionData;
@@ -30,7 +31,7 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
int version = resultSet.getInt(1);
int nonce = resultSet.getInt(2);
ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.valueOf(resultSet.getInt(3));
Service service = Service.valueOf(resultSet.getInt(3));
int size = resultSet.getInt(4);
boolean isDataRaw = resultSet.getBoolean(5); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;

2
src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java
View File

@@ -16,13 +16,13 @@ import java.util.Map;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.PaymentData;
import org.qortal.data.group.GroupApprovalData;
import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.GroupApprovalTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.data.transaction.TransferAssetTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.Service;
import org.qortal.repository.DataException;
import org.qortal.repository.TransactionRepository;
import org.qortal.repository.hsqldb.HSQLDBRepository;

3
src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java
View File

@@ -7,6 +7,7 @@ import java.util.ArrayList;
import java.util.List;
import com.google.common.base.Utf8;
import org.qortal.arbitrary.misc.Service;
import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData;
import org.qortal.data.transaction.ArbitraryTransactionData;
@@ -130,7 +131,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
payments.add(PaymentTransformer.fromByteBuffer(byteBuffer));
}
ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.valueOf(byteBuffer.getInt());
Service service = Service.valueOf(byteBuffer.getInt());
// We might be receiving hash of data instead of actual raw data
boolean isRaw = byteBuffer.get() != 0;

3
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@@ -3,6 +3,7 @@ package org.qortal.utils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.DataException;
@@ -38,7 +39,7 @@ public class ArbitraryTransactionUtils {
}
String name = arbitraryTransactionData.getName();
ArbitraryTransactionData.Service service = arbitraryTransactionData.getService();
Service service = arbitraryTransactionData.getService();
String identifier = arbitraryTransactionData.getIdentifier();
if (name == null || service == null) {

33
src/main/java/org/qortal/utils/FilesystemUtils.java
View File

@@ -1,6 +1,7 @@
package org.qortal.utils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.qortal.settings.Settings;
import java.io.File;
@@ -190,4 +191,36 @@ public class FilesystemUtils {
.sum();
}
/**
* getSingleFileContents
* Return the content of the file at given path.
* If the path is a directory, the contents will be returned
* only if it contains a single file.
*
* @param path
* @return
* @throws IOException
*/
public static byte[] getSingleFileContents(Path path) throws IOException {
byte[] data = null;
// TODO: limit the file size that can be loaded into memory
// If the path is a file, read the contents directly
if (path.toFile().isFile()) {
data = Files.readAllBytes(path);
}
// Or if it's a directory, only load file contents if there is a single file inside it
else if (path.toFile().isDirectory()) {
String[] files = ArrayUtils.removeElement(path.toFile().list(), ".qortal");
if (files.length == 1) {
Path filePath = Paths.get(path.toString(), files[0]);
data = Files.readAllBytes(filePath);
}
}
return data;
}
}