From df39819de009bb7d208dbfad671f630fe52873f9 Mon Sep 17 00:00:00 2001
From: AlphaX-Projects <77661270+AlphaX-Projects@users.noreply.github.com>
Date: Mon, 25 Mar 2024 07:34:26 +0100
Subject: [PATCH] Update QDN file management

- Allow files to move around the network more quickly
- Ensuring that metadata isn't saved for blocked names
- More selective disk usage
- Minor fixes
---
 pom.xml                                       | 61 ++++++++-----------
 .../qortal/arbitrary/ArbitraryDataFile.java   | 21 ++++++-
 .../arbitrary/ArbitraryDataFileManager.java   |  9 +--
 .../arbitrary/ArbitraryMetadataManager.java   | 12 +++-
 .../arbitrary/ArbitraryDataFileTests.java     |  2 +
 5 files changed, 61 insertions(+), 44 deletions(-)

diff --git a/pom.xml b/pom.xml
index bbc044a2..4eaaa408 100644
--- a/pom.xml
+++ b/pom.xml
@@ -10,18 +10,17 @@
 		<altcoinj.version>7dc8c6f</altcoinj.version>
 		<bitcoinj.version>0.15.10</bitcoinj.version>
 		<bouncycastle.version>1.69</bouncycastle.version>
-		<build-helper-maven-plugin.version>3.5.0</build-helper-maven-plugin.version>
 		<build.timestamp>${maven.build.timestamp}</build.timestamp>
 		<ciyam-at.version>1.4.1</ciyam-at.version>
 		<commons-net.version>3.8.0</commons-net.version>
 		<commons-text.version>1.11.0</commons-text.version>
-		<commons-io.version>2.11.0</commons-io.version>
-		<commons-compress.version>1.25.0</commons-compress.version>
+		<commons-io.version>2.15.1</commons-io.version>
+		<commons-compress.version>1.26.1</commons-compress.version>
 		<commons-lang3.version>3.14.0</commons-lang3.version>
 		<dagger.version>1.2.2</dagger.version>
 		<extendedset.version>0.12.3</extendedset.version>
 		<git-commit-id-plugin.version>4.9.10</git-commit-id-plugin.version>
-		<grpc.version>1.61.1</grpc.version>
+		<grpc.version>1.62.2</grpc.version>
 		<guava.version>33.0.0-jre</guava.version>
 		<hamcrest-library.version>2.2</hamcrest-library.version>
 		<homoglyph.version>1.2.1</homoglyph.version>
@@ -33,28 +32,30 @@
 		<jersey.version>2.41</jersey.version>
 		<jetty.version>9.4.54.v20240208</jetty.version>
 		<json-simple.version>1.1.1</json-simple.version>
-		<json.version>20240205</json.version>
+		<json.version>20240303</json.version>
 		<jsoup.version>1.17.2</jsoup.version>
 		<junit-jupiter-engine.version>5.10.0</junit-jupiter-engine.version>
 		<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
-		<log4j.version>2.22.1</log4j.version>
+		<log4j.version>2.23.0</log4j.version>
 		<mail.version>1.5.0-b01</mail.version>
+		<maven-build-helper-plugin.version>3.5.0</maven-build-helper-plugin.version>
 		<maven-compiler-plugin.version>3.12.1</maven-compiler-plugin.version>
+		<maven-dependency-plugin.version>3.6.1</maven-dependency-plugin.version>
 		<maven-jar-plugin.version>3.3.0</maven-jar-plugin.version>
+		<maven-package-info-plugin.version>1.1.0</maven-package-info-plugin.version>
+		<maven-plugin.version>2.16.2</maven-plugin.version>
+		<maven-reproducible-build-plugin.version>0.16</maven-reproducible-build-plugin.version>
 		<maven-resources-plugin.version>3.3.1</maven-resources-plugin.version>
-		<maven-shade-plugin.version>3.5.1</maven-shade-plugin.version>
+		<maven-shade-plugin.version>3.5.2</maven-shade-plugin.version>
 		<maven-surefire-plugin.version>3.2.5</maven-surefire-plugin.version>
-		<package-info-maven-plugin.version>1.1.0</package-info-maven-plugin.version>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<protobuf.version>3.25.2</protobuf.version>
+		<protobuf.version>3.25.3</protobuf.version>
 		<replacer.version>1.5.3</replacer.version>
-		<reproducible-build-maven-plugin.version>0.16</reproducible-build-maven-plugin.version>
 		<simplemagic.version>1.17</simplemagic.version>
 		<slf4j.version>1.7.36</slf4j.version>
 		<swagger-api.version>2.0.10</swagger-api.version>
-		<swagger-ui.version>5.10.3</swagger-ui.version>
+		<swagger-ui.version>5.11.8</swagger-ui.version>
 		<upnp.version>1.2</upnp.version>
-		<versions-maven-plugin.version>2.16.2</versions-maven-plugin.version>
 		<xz.version>1.9</xz.version>
 	</properties>
 	<build>
@@ -70,7 +71,7 @@
 			<plugin>
 				<groupId>org.codehaus.mojo</groupId>
 				<artifactId>versions-maven-plugin</artifactId>
-				<version>${versions-maven-plugin.version}</version>
+				<version>${maven-plugin.version}</version>
 				<configuration>
 					<generateBackupPoms>false</generateBackupPoms>
 				</configuration>
@@ -238,7 +239,7 @@
 			<plugin>
 				<groupId>com.github.bohnman</groupId>
 				<artifactId>package-info-maven-plugin</artifactId>
-				<version>${package-info-maven-plugin.version}</version>
+				<version>${maven-package-info-plugin.version}</version>
 				<configuration>
 					<packages>
 						<package>
@@ -268,7 +269,7 @@
 			<plugin>
 				<groupId>org.codehaus.mojo</groupId>
 				<artifactId>build-helper-maven-plugin</artifactId>
-				<version>${build-helper-maven-plugin.version}</version>
+				<version>${maven-build-helper-plugin.version}</version>
 				<executions>
 					<execution>
 						<phase>generate-sources</phase>
@@ -353,7 +354,7 @@
 			<plugin>
 				<groupId>io.github.zlika</groupId>
 				<artifactId>reproducible-build-maven-plugin</artifactId>
-				<version>${reproducible-build-maven-plugin.version}</version>
+				<version>${maven-reproducible-build-plugin.version}</version>
 				<executions>
 					<execution>
 						<phase>package</phase>
@@ -388,15 +389,9 @@
 							<pluginExecutions>
 								<pluginExecution>
 									<pluginExecutionFilter>
-										<groupId>
-											org.apache.maven.plugins
-										</groupId>
-										<artifactId>
-											maven-dependency-plugin
-										</artifactId>
-										<versionRange>
-											[3.6.0,)
-										</versionRange>
+										<groupId>org.apache.maven.plugins</groupId>
+										<artifactId>maven-dependency-plugin</artifactId>
+										<version>${maven-dependency-plugin.version}</version>
 										<goals>
 											<goal>unpack</goal>
 										</goals>
@@ -407,15 +402,9 @@
 								</pluginExecution>
 								<pluginExecution>
 									<pluginExecutionFilter>
-										<groupId>
-											com.google.code.maven-replacer-plugin
-										</groupId>
-										<artifactId>
-											replacer
-										</artifactId>
-										<versionRange>
-											[1.5.3,)
-										</versionRange>
+										<groupId>com.google.code.maven-replacer-plugin</groupId>
+										<artifactId>replacer</artifactId>
+										<version>${replacer.version}</version>
 										<goals>
 											<goal>replace</goal>
 										</goals>
@@ -448,7 +437,7 @@
 		<dependency>
 			<groupId>org.codehaus.mojo</groupId>
 			<artifactId>build-helper-maven-plugin</artifactId>
-			<version>${build-helper-maven-plugin.version}</version>
+			<version>${maven-build-helper-plugin.version}</version>
 			<scope>provided</scope>
 			<!-- needed for build, not for runtime -->
 		</dependency>
@@ -456,7 +445,7 @@
 		<dependency>
 			<groupId>com.github.bohnman</groupId>
 			<artifactId>package-info-maven-plugin</artifactId>
-			<version>${package-info-maven-plugin.version}</version>
+			<version>${maven-package-info-plugin.version}</version>
 			<scope>provided</scope>
 			<!-- needed for build, not for runtime -->
 		</dependency>
diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java
index 71378461..1bba1a73 100644
--- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java
+++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java
@@ -58,6 +58,9 @@ public class ArbitraryDataFile {
     public static int SHORT_DIGEST_LENGTH = 8;
 
     protected Path filePath;
+    protected byte[] fileContent;
+    private boolean useTemporaryFile;
+
     protected String hash58;
     protected byte[] signature;
     private ArrayList<ArbitraryDataFileChunk> chunks;
@@ -90,8 +93,14 @@ public class ArbitraryDataFile {
         this.signature = signature;
         LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length));
 
+        this.fileContent = fileContent;
+        this.useTemporaryFile = useTemporaryFile;
+    }
+
+    public void save() throws DataException {
+
         Path outputFilePath;
-        if (useTemporaryFile) {
+        if (this.useTemporaryFile) {
             try {
                 outputFilePath = Files.createTempFile("qortalRawData", null);
                 outputFilePath.toFile().deleteOnExit();
@@ -149,6 +158,7 @@ public class ArbitraryDataFile {
 
             case RAW_DATA:
                 arbitraryDataFile = ArbitraryDataFile.fromRawData(data, signature);
+                arbitraryDataFile.save();
                 break;
         }
 
@@ -324,6 +334,7 @@ public class ArbitraryDataFile {
                             out.flush();
 
                             ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray(), this.signature);
+                            chunk.save();
                             ValidationResult validationResult = chunk.isValid();
                             if (validationResult == ValidationResult.OK) {
                                 this.chunks.add(chunk);
@@ -406,6 +417,10 @@ public class ArbitraryDataFile {
     }
 
     public boolean delete(int attempts) {
+        if (this.filePath == null) {
+            return false;
+        }
+
         // Keep trying to delete the data until it is deleted, or we reach 10 attempts
         for (int i=0; i<attempts; i++) {
             if (this.delete()) {
@@ -467,6 +482,10 @@ public class ArbitraryDataFile {
     }
 
     public byte[] getBytes() {
+        if (this.fileContent != null) {
+            return this.fileContent;
+        }
+
         try {
             return Files.readAllBytes(this.filePath);
         } catch (IOException e) {
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java
index 69a1150a..ed80d85c 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java
@@ -242,13 +242,14 @@ public class ArbitraryDataFileManager extends Thread {
         boolean isRelayRequest = (requestingPeer != null);
         if (isRelayRequest) {
             if (!fileAlreadyExists) {
-                // File didn't exist locally before the request, and it's a forwarding request, so delete it
-                LOGGER.debug("Deleting file {} because it was needed for forwarding only", Base58.encode(hash));
-
-                // Keep trying to delete the data until it is deleted, or we reach 10 attempts
+                // File didn't exist locally before the request, and it's a forwarding request, so delete it if it exists.
+                // It shouldn't exist on the filesystem yet, but leaving this here just in case.
                 arbitraryDataFile.delete(10);
             }
         }
+        else {
+            arbitraryDataFile.save();
+        }
 
         // If this is a metadata file then we need to update the cache
         if (arbitraryTransactionData != null && arbitraryTransactionData.getMetadataHash() != null) {
diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
index 9133e86c..993a2b72 100644
--- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
+++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java
@@ -334,11 +334,17 @@ public class ArbitraryMetadataManager {
             }
             ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
 
+            // Check if the name is blocked
+            boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
+
+            // Save if not blocked
+            ArbitraryDataFile arbitraryMetadataFile = arbitraryMetadataMessage.getArbitraryMetadataFile();
+            if (!isBlocked && arbitraryMetadataFile != null) {
+                arbitraryMetadataFile.save();
+            }
+
             // Forwarding
             if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
-
-                // Check if the name is blocked
-                boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
                 if (!isBlocked) {
                     Peer requestingPeer = request.getB();
                     if (requestingPeer != null) {
diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java
index ac633717..7703c062 100644
--- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java
+++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java
@@ -21,6 +21,7 @@ public class ArbitraryDataFileTests extends Common {
 	public void testSplitAndJoin() throws DataException {
 		String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
 		ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes(), null, false);
+		arbitraryDataFile.save();
 		assertTrue(arbitraryDataFile.exists());
 		assertEquals(62, arbitraryDataFile.size());
 		assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58());
@@ -51,6 +52,7 @@ public class ArbitraryDataFileTests extends Common {
 		new Random().nextBytes(randomData); // No need for SecureRandom here
 
 		ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData, null, false);
+		arbitraryDataFile.save();
 		assertTrue(arbitraryDataFile.exists());
 		assertEquals(fileSize, arbitraryDataFile.size());
 		String originalFileDigest = arbitraryDataFile.digest58();