Merge pull request #254 from jschulthess/reticulum

Reticulum branch updates

.github/workflows/pr-testing.yml

@@ -1,7 +1,7 @@
 name: PR testing
 
 on:
-  push:
+  pull_request:
     branches: [ master ]
 
 jobs:
@@ -22,10 +22,6 @@ jobs:
         java-version: '11'
         distribution: 'adopt'
 
-    - name: Load custom deps
-      run: |
-        mvn install -DskipTests=true --file pom.xml
-
     - name: Run all tests
       run: |
         mvn -B clean test -DskipTests=false --file pom.xml

.vscode/settings.json

@@ -1,3 +0,0 @@
-{
-    "java.compile.nullAnalysis.mode": "automatic"
-}

README.md

@@ -15,31 +15,20 @@ Building the future one block at a time. Welcome to Qortal.
 
 # Building the Qortal Core from Source
 
-## Build / Run
+## Build / run
 
 - Requires Java 11. OpenJDK 11 recommended over Java SE.
 - Install Maven
 - Use Maven to fetch dependencies and build: `mvn clean package`
-- Update Maven dependencies: `mvn install` 
 - Built JAR should be something like `target/qortal-1.0.jar`
 - Create basic *settings.json* file: `echo '{}' > settings.json`
 - Run JAR in same working directory as *settings.json*: `java -jar target/qortal-1.0.jar`
 - Wrap in shell script, add JVM flags, redirection, backgrounding, etc. as necessary.
 - Or use supplied example shell script: *start.sh*
 
-## IntelliJ IDEA Configuration
-
-- Run -> Edit Configurations
-- Add New Application
-- Name: qortal
-- SDK: java 11
-- Main Class: org.qortal.controller.Controller
-- Program arguments: settings.json -Dlog4j.configurationFile=log4j2.properties -ea
-- Environment variables: Djava.net.preferIPv4Stack=false
-
 # Using a pre-built Qortal 'jar' binary
 
-If you prefer to utilize a released version of Qortal, you may do so by downloading one of the available releases from the releases page, that are also linked on https://qortal.org and https://qortal.dev. 
+If you would prefer to utilize a released version of Qortal, you may do so by downloading one of the available releases from the releases page, that are also linked on https://qortal.org and https://qortal.dev. 
 
 # Learning Q-App Development
 

lib/org/ciyam/AT/1.3.7/AT-1.3.7.pom

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.ciyam</groupId>
+  <artifactId>AT</artifactId>
+  <version>1.3.7</version>
+  <description>POM was created from install:install-file</description>
+</project>

lib/org/ciyam/AT/1.3.8/AT-1.3.8.pom

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.ciyam</groupId>
+  <artifactId>AT</artifactId>
+  <version>1.3.8</version>
+  <description>POM was created from install:install-file</description>
+</project>

lib/org/ciyam/AT/1.4.0/AT-1.4.0.pom

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.ciyam</groupId>
+  <artifactId>AT</artifactId>
+  <version>1.4.0</version>
+  <description>POM was created from install:install-file</description>
+</project>

lib/org/ciyam/AT/1.4.1/AT-1.4.1.pom

@@ -0,0 +1,123 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>org.ciyam</groupId>
+	<artifactId>AT</artifactId>
+	<version>1.4.1</version>
+	<packaging>jar</packaging>
+
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<skipTests>false</skipTests>
+		<bouncycastle.version>1.69</bouncycastle.version>
+		<junit.version>4.13.2</junit.version>
+		<maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>
+		<maven-jar-plugin.version>3.3.0</maven-jar-plugin.version>
+		<maven-javadoc-plugin.version>3.6.3</maven-javadoc-plugin.version>
+		<maven-source-plugin.version>3.3.0</maven-source-plugin.version>
+		<maven-surefire-plugin.version>3.2.2</maven-surefire-plugin.version>
+	</properties>
+
+	<build>
+		<sourceDirectory>src/main/java</sourceDirectory>
+		<testSourceDirectory>src/test/java</testSourceDirectory>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>${maven-compiler-plugin.version}</version>
+				<configuration>
+					<source>11</source>
+					<target>11</target>
+				</configuration>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<version>${maven-surefire-plugin.version}</version>
+				<configuration>
+					<skipTests>${skipTests}</skipTests>
+				</configuration>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-source-plugin</artifactId>
+				<version>${maven-source-plugin.version}</version>
+				<executions>
+					<execution>
+						<id>attach-sources</id>
+						<goals>
+							<goal>jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-javadoc-plugin</artifactId>
+				<version>${maven-javadoc-plugin.version}</version>
+				<executions>
+					<execution>
+						<id>attach-javadoc</id>
+						<goals>
+							<goal>jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+				<version>${maven-jar-plugin.version}</version>
+				<executions>
+					<execution>
+						<goals>
+							<goal>test-jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-compiler-plugin</artifactId>
+			<version>${maven-compiler-plugin.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-surefire-plugin</artifactId>
+			<version>${maven-surefire-plugin.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-source-plugin</artifactId>
+			<version>${maven-source-plugin.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-javadoc-plugin</artifactId>
+			<version>${maven-javadoc-plugin.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-jar-plugin</artifactId>
+			<version>${maven-jar-plugin.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.bouncycastle</groupId>
+			<artifactId>bcprov-jdk15on</artifactId>
+			<version>${bouncycastle.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>${junit.version}</version>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+</project>

lib/org/ciyam/AT/1.4.2/AT-1.4.2.pom

@@ -0,0 +1,123 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<groupId>org.ciyam</groupId>
+	<artifactId>AT</artifactId>
+	<version>1.4.2</version>
+	<packaging>jar</packaging>
+
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+		<skipTests>false</skipTests>
+		<bouncycastle.version>1.70</bouncycastle.version>
+		<junit.version>4.13.2</junit.version>
+		<maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
+		<maven-source-plugin.version>3.3.0</maven-source-plugin.version>
+		<maven-javadoc-plugin.version>3.6.3</maven-javadoc-plugin.version>
+		<maven-surefire-plugin.version>3.2.5</maven-surefire-plugin.version>
+		<maven-jar-plugin.version>3.4.1</maven-jar-plugin.version>
+	</properties>
+
+	<build>
+		<sourceDirectory>src/main/java</sourceDirectory>
+		<testSourceDirectory>src/test/java</testSourceDirectory>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>${maven-compiler-plugin.version}</version>
+				<configuration>
+					<source>11</source>
+					<target>11</target>
+				</configuration>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-surefire-plugin</artifactId>
+				<version>${maven-surefire-plugin.version}</version>
+				<configuration>
+					<skipTests>${skipTests}</skipTests>
+				</configuration>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-source-plugin</artifactId>
+				<version>${maven-source-plugin.version}</version>
+				<executions>
+					<execution>
+						<id>attach-sources</id>
+						<goals>
+							<goal>jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-javadoc-plugin</artifactId>
+				<version>${maven-javadoc-plugin.version}</version>
+				<executions>
+					<execution>
+						<id>attach-javadoc</id>
+						<goals>
+							<goal>jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-jar-plugin</artifactId>
+				<version>${maven-jar-plugin.version}</version>
+				<executions>
+					<execution>
+						<goals>
+							<goal>test-jar</goal>
+						</goals>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-compiler-plugin</artifactId>
+			<version>${maven-compiler-plugin.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-surefire-plugin</artifactId>
+			<version>${maven-surefire-plugin.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-source-plugin</artifactId>
+			<version>${maven-source-plugin.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-javadoc-plugin</artifactId>
+			<version>${maven-javadoc-plugin.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.maven.plugins</groupId>
+			<artifactId>maven-jar-plugin</artifactId>
+			<version>${maven-jar-plugin.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.bouncycastle</groupId>
+			<artifactId>bcprov-jdk15on</artifactId>
+			<version>${bouncycastle.version}</version>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<version>${junit.version}</version>
+			<scope>test</scope>
+		</dependency>
+	</dependencies>
+</project>

lib/org/ciyam/AT/maven-metadata-local.xml

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>org.ciyam</groupId>
+  <artifactId>AT</artifactId>
+  <versioning>
+    <release>1.4.2</release>
+    <versions>
+      <version>1.3.7</version>
+      <version>1.3.8</version>
+      <version>1.4.0</version>
+      <version>1.4.1</version>
+      <version>1.4.2</version>
+    </versions>
+    <lastUpdated>20240426084210</lastUpdated>
+  </versioning>
+</metadata>

lib/org/hsqldb/hsqldb/2.7.4/Notes.txt

@@ -1,5 +0,0 @@
-This is the production hsqldb-2.7.4 with the manifest file updated 
-
-Sealed: false
-
-Allows the addition of the custom Qortal HSQLDBPool and Monitoring Classes

log4j2.properties

@@ -6,6 +6,34 @@ rootLogger.level = info
 rootLogger.appenderRef.console.ref = stdout
 rootLogger.appenderRef.rolling.ref = FILE
 
+### additional debug options (in case of problems eg. 202411
+
+#to see more QDN details - add the stuff below
+#logger.arbitrary.name = org.qortal.arbitrary
+#logger.arbitrary.level = trace
+
+#to see more QDN networking details - add stuff below
+#logger.arbitrarycontroller.name = org.qortal.controller.arbitrary
+#logger.arbitrarycontroller.level = debug
+
+# Support optional, Network Task debugging
+#logger.networkTask.name = org.qortal.network.task
+#logger.networkTask.level = debug
+
+# Support optional, Network Task tracing
+#logger.networkTask.name = org.qortal.network.task
+#logger.networkTask.level = trace
+
+# Support optional, Block debugging
+#logger.block.name = org.qortal.block
+#logger.block.level = debug
+
+# Support optional, Block tracing
+#logger.block.name = org.qortal.block
+#logger.block.level = trace
+
+### end additional debug options
+
 # Suppress extraneous bitcoinj library output
 logger.bitcoinj.name = org.bitcoinj
 logger.bitcoinj.level = error
@@ -18,6 +46,10 @@ logger.hsqldb.level = warn
 logger.hsqldbRepository.name = org.qortal.repository.hsqldb
 logger.hsqldbRepository.level = debug
 
+## Support optional, controller repository debugging
+#logger.controllerRepository.name = org.qortal.controller.repository
+#logger.controllerRepository.level = debug
+
 # Suppress extraneous Jersey warning
 logger.jerseyInject.name = org.glassfish.jersey.internal.inject.Providers
 logger.jerseyInject.level = off

pom.xml

@@ -3,19 +3,18 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>5.1.0</version> <!-- Version must be <X.Y.Z> -->
+	<version>4.7.1</version>
 	<packaging>jar</packaging>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<skipJUnitTests>true</skipJUnitTests>
-		
-		<altcoinj.version>d7cf6ac</altcoinj.version> <!-- BC v16 / Updated Abstract Classes / alertSigningKey -->
-		<bitcoinj.version>0.16.3</bitcoinj.version>
-		<bouncycastle.version>1.73</bouncycastle.version>
+		<skipTests>true</skipTests>
+
+		<altcoinj.version>7dc8c6f</altcoinj.version>
+		<bitcoinj.version>0.15.10</bitcoinj.version>
+		<bouncycastle.version>1.70</bouncycastle.version>
 		<build.timestamp>${maven.build.timestamp}</build.timestamp>
-		<ciyam-at.version>1b731d1</ciyam-at.version> <!-- This is the hash for v1.4.3 -->
+		<ciyam-at.version>1.4.2</ciyam-at.version>
 		<commons-net.version>3.8.0</commons-net.version>
-		<!-- <commons-net.version>3.9.0</commons-net.version> v5.2.0 coming soon -->
 		<commons-text.version>1.12.0</commons-text.version>
 		<commons-io.version>2.18.0</commons-io.version>
 		<commons-compress.version>1.27.1</commons-compress.version>
@@ -24,7 +23,6 @@
 		<extendedset.version>0.12.3</extendedset.version>
 		<git-commit-id-plugin.version>4.9.10</git-commit-id-plugin.version>
 		<grpc.version>1.68.1</grpc.version>
-		<!-- <grpc.version>1.68.3</grpc.version> v5.2.0 coming soon -->
 		<guava.version>33.3.1-jre</guava.version>
 		<hamcrest-library.version>2.2</hamcrest-library.version>
 		<homoglyph.version>1.2.1</homoglyph.version>
@@ -35,7 +33,6 @@
 		<jaxb-runtime.version>2.3.9</jaxb-runtime.version>
 		<jersey.version>2.42</jersey.version>
 		<jetty.version>9.4.56.v20240826</jetty.version>
-		<!-- <jetty.version>9.4.57.v20241219</jetty.version> v5.2.0 Coming Soon -->
 		<json-simple.version>1.1.1</json-simple.version>
 		<json.version>20240303</json.version>
 		<jsoup.version>1.18.1</jsoup.version>
@@ -52,20 +49,20 @@
 		<maven-reproducible-build-plugin.version>0.17</maven-reproducible-build-plugin.version>
 		<maven-resources-plugin.version>3.3.1</maven-resources-plugin.version>
 		<maven-shade-plugin.version>3.6.0</maven-shade-plugin.version>
-		<maven-install-plugin.version>3.1.3</maven-install-plugin.version>
 		<maven-surefire-plugin.version>3.5.2</maven-surefire-plugin.version>
-		<!-- <maven-surefire-plugin.version>3.5.3</maven-surefire-plugin.version> v5.2.0 Coming Soon -->
 		<protobuf.version>3.25.3</protobuf.version>
-		<!-- <protobuf.version>3.25.7</protobuf.version> v 5.1 -->
 		<replacer.version>1.5.3</replacer.version>
+		<reticulum.version>8623ce3</reticulum.version>
 		<simplemagic.version>1.17</simplemagic.version>
-		<slf4j.version>1.7.36</slf4j.version>
-		<!-- <swagger-api.version>2.2.30</swagger-api.version> need code upgrade Future Release -->
-		<!-- <swagger-api.version>2.1.13</swagger-api.version> need code upgrade Future Release -->
 		<swagger-api.version>2.0.10</swagger-api.version>
 		<swagger-ui.version>5.18.2</swagger-ui.version>
 		<upnp.version>1.2</upnp.version>
 		<xz.version>1.10</xz.version>
+   	<lombok.version>1.18.30</lombok.version>
+   	<jackson.version>2.16.1</jackson.version>
+		<slf4j.version>2.0.12</slf4j.version>
+    <nitrite.version>4.3.0</nitrite.version>
+    <junit.version>5.9.2</junit.version>
 	</properties>
 	<build>
 		<sourceDirectory>src/main/java</sourceDirectory>
@@ -297,48 +294,20 @@
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-jar-plugin</artifactId>
 				<version>${maven-jar-plugin.version}</version>
-				<executions>
-					<execution>
-						<configuration>
-							<archive>
-								<manifest>
-									<addDefaultEntries>false</addDefaultEntries>
-									<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
-								</manifest>
-								<manifestEntries>
-									<Last-Commit-Id>${git.commit.id.full}</Last-Commit-Id>
-									<Last-Commit-Time>${git.commit.time}</Last-Commit-Time>
-									<Reproducible-Build>true</Reproducible-Build>
-								</manifestEntries>
-							</archive>
-						</configuration>
-					</execution>
-				</executions>
-			</plugin>
-			<!-- Copy modified hsqldb.jar to install / modified MANIFEST.MF-->
-			<plugin>
-				<groupId>org.apache.maven.plugins</groupId>
-				<artifactId>maven-install-plugin</artifactId>
-				<version>${maven-install-plugin.version}</version>
 				<configuration>
-					<groupId>org.hsqldb</groupId>
-					<artifactId>hsqldb</artifactId>
-					<version>${hsqldb.version}</version>
-					<packaging>jar</packaging>
+					<archive>
+						<manifest>
+							<addDefaultEntries>false</addDefaultEntries>
+							<addDefaultImplementationEntries>true</addDefaultImplementationEntries>
+						</manifest>
+						<manifestEntries>
+							<Last-Commit-Id>${git.commit.id.full}</Last-Commit-Id>
+							<Last-Commit-Time>${git.commit.time}</Last-Commit-Time>
+							<Reproducible-Build>true</Reproducible-Build>
+						</manifestEntries>
+					</archive>
 				</configuration>
-				<executions>
-					<execution>
-						<phase>install</phase>
-						<goals>
-							<goal>install-file</goal>
-						</goals>
-						<configuration>
-							<file>${project.basedir}/lib/org/hsqldb/hsqldb/${hsqldb.version}/hsqldb.jar</file>
-						</configuration>
-					</execution>
-				</executions>
 			</plugin>
-
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-shade-plugin</artifactId>
@@ -388,7 +357,6 @@
 					</execution>
 				</executions>
 			</plugin>
-			<!-- Removed, now use Maven reproducible by default v4.0, IntelliJ v2025.1 and later -->
 			<plugin>
 				<groupId>io.github.zlika</groupId>
 				<artifactId>reproducible-build-maven-plugin</artifactId>
@@ -411,7 +379,7 @@
 				<artifactId>maven-surefire-plugin</artifactId>
 				<version>${maven-surefire-plugin.version}</version>
 				<configuration>
-					<skipTests>${skipJUnitTests}</skipTests>
+					<skipTests>${skipTests}</skipTests>
 				</configuration>
 			</plugin>
 		</plugins>
@@ -463,13 +431,41 @@
 			<id>project.local</id>
 			<name>project</name>
 			<url>file:${project.basedir}/lib</url>
+      <snapshots>
+        <enabled>true</enabled>
+        <updatePolicy>always</updatePolicy>
+      </snapshots>
 		</repository>
 		<!-- jitpack for build-on-demand of altcoinj -->
 		<repository>
 			<id>jitpack.io</id>
 			<url>https://jitpack.io</url>
+      <snapshots>
+        <enabled>true</enabled>
+        <updatePolicy>always</updatePolicy>
+      </snapshots>
 		</repository>
 	</repositories>
+	<!--
+    <dependencyManagement>
+      <dependencies>
+        <dependency>
+          <groupId>org.dizitart</groupId>
+          <artifactId>nitrite-bom</artifactId>
+          <version>${nitrite.version}</version>
+          <scope>import</scope>
+          <type>pom</type>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-bom</artifactId>
+          <version>${log4j.version}</version>
+          <scope>import</scope>
+          <type>pom</type>
+        </dependency>
+      </dependencies>
+  	</dependencyManagement>
+    -->
 	<dependencies>
 		<!-- https://mvnrepository.com/artifact/org.codehaus.mojo/build-helper-maven-plugin -->
 		<dependency>
@@ -487,7 +483,7 @@
 			<scope>provided</scope>
 			<!-- needed for build, not for runtime -->
 		</dependency>
-		<!-- HSQLDB for repository should use local version with Sealed: false -->
+		<!-- HSQLDB for repository -->
 		<dependency>
 			<groupId>org.hsqldb</groupId>
 			<artifactId>hsqldb</artifactId>
@@ -495,7 +491,7 @@
 		</dependency>
 		<!-- CIYAM AT (automated transactions) -->
 		<dependency>
-			<groupId>com.github.iceburst</groupId>
+			<groupId>org.ciyam</groupId>
 			<artifactId>AT</artifactId>
 			<version>${ciyam-at.version}</version>
 		</dependency>
@@ -513,10 +509,17 @@
 		</dependency>
 		<!-- For Litecoin, etc. support, requires bitcoinj -->
 		<dependency>
-			<groupId>com.github.iceburst</groupId>
+			<groupId>com.github.qortal</groupId>
 			<artifactId>altcoinj</artifactId>
 			<version>${altcoinj.version}</version>
 		</dependency>
+		<!-- Build Reticulum from Source -->
+		<dependency>
+			<!--<groupId>com.github.sergst83</groupId>-->
+			<groupId>com.github.jschulthess</groupId>
+			<artifactId>reticulum-network-stack</artifactId>
+			<version>${reticulum.version}</version>
+		</dependency>
 		<!-- Utilities -->
 		<dependency>
 			<groupId>com.googlecode.json-simple</groupId>
@@ -595,35 +598,33 @@
 			<artifactId>guava</artifactId>
 			<version>${guava.version}</version>
 		</dependency>
-		<!-- Logging: log4j2 -->
-		<dependency>
-			<groupId>org.apache.logging.log4j</groupId>
-			<artifactId>log4j-core</artifactId>
-			<version>${log4j.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.logging.log4j</groupId>
-			<artifactId>log4j-api</artifactId>
-			<version>${log4j.version}</version>
-		</dependency>
-		<!-- redirect slf4j to log4j2 -->
-		<dependency>
-			<groupId>org.apache.logging.log4j</groupId>
-			<artifactId>log4j-slf4j-impl</artifactId>
-			<version>${log4j.version}</version>
-		</dependency>
-		<!-- redirect java.utils.logging to log4j2 -->
-		<dependency>
-			<groupId>org.apache.logging.log4j</groupId>
-			<artifactId>log4j-jul</artifactId>
-			<version>${log4j.version}</version>
-		</dependency>
-		<!-- Logging: slf4j used by Jetty/Jersey -->
+    <!-- logging: slf4j -->
 		<dependency>
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
 			<version>${slf4j.version}</version>
 		</dependency>
+    <dependency>
+       <groupId>org.apache.logging.log4j</groupId>
+       <artifactId>log4j-slf4j2-impl</artifactId>
+       <version>${log4j.version}</version>
+    </dependency>
+		<!-- Logging: log4j2 -->
+    <dependency>
+       <groupId>org.apache.logging.log4j</groupId>
+       <artifactId>log4j-core</artifactId>
+       <version>${log4j.version}</version>
+    </dependency>
+    <dependency>
+    <groupId>org.apache.logging.log4j</groupId>
+       <artifactId>log4j-api</artifactId>
+       <version>${log4j.version}</version>
+    </dependency>
+    <dependency>
+       <groupId>org.apache.logging.log4j</groupId>
+       <artifactId>log4j-jul</artifactId>
+       <version>${log4j.version}</version>
+     </dependency> 
 		<!-- Servlet related -->
 		<dependency>
 			<groupId>javax.servlet</groupId>
@@ -758,13 +759,18 @@
 		<!-- BouncyCastle for crypto, including TLS secure networking -->
 		<dependency>
 			<groupId>org.bouncycastle</groupId>
-			<artifactId>bcprov-jdk15to18</artifactId>
+			<artifactId>bcprov-jdk15on</artifactId>
 			<version>${bouncycastle.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.bouncycastle</groupId>
-			<artifactId>bctls-jdk15to18</artifactId>
+			<artifactId>bctls-jdk15on</artifactId>
 			<version>${bouncycastle.version}</version>
+		</dependency><!-- https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15to18 -->
+		<dependency>
+		    <groupId>org.bouncycastle</groupId>
+		    <artifactId>bcprov-jdk15to18</artifactId>
+		    <version>${bouncycastle.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.jsoup</groupId>
@@ -807,10 +813,123 @@
 			<artifactId>jaxb-runtime</artifactId>
 			<version>${jaxb-runtime.version}</version>
 		</dependency>
-		<dependency>
-<groupId>org.apache.tika</groupId>
-<artifactId>tika-core</artifactId>
-<version>3.1.0</version> 
-</dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <version>1.16.1</version>
+    </dependency>
+    <!-- already declared earlier
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.14.0</version>
+    </dependency>
+    -->
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-collections4</artifactId>
+      <version>4.4</version>
+    </dependency>
+    <!-- already declared earlier
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-compress</artifactId>
+      <version>1.26.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+    -->
+    <!-- note: covered ? -->
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.yaml</groupId>
+      <artifactId>snakeyaml</artifactId>
+      <version>2.2</version>
+    </dependency>
+    <dependency>
+      <groupId>org.msgpack</groupId>
+      <artifactId>jackson-dataformat-msgpack</artifactId>
+      <version>0.9.8</version>
+    </dependency>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <version>${lombok.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcpkix-jdk15on</artifactId>
+      <version>1.70</version>
+    </dependency>
+    <dependency>
+      <groupId>com.macasaet.fernet</groupId>
+      <artifactId>fernet-java8</artifactId>
+      <version>1.5.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.igormaznitsa</groupId>
+      <artifactId>jbbp</artifactId>
+      <version>2.0.6</version>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-all</artifactId>
+      <!--<version>4.1.106.Final</version>-->
+      <version>5.0.0.Alpha2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.github.seancfoley</groupId>
+      <artifactId>ipaddress</artifactId>
+      <version>5.4.2</version>
+    </dependency>
+    <!-- Nitrite Modules -->
+    <dependency>
+        <groupId>org.dizitart</groupId>
+        <artifactId>nitrite</artifactId>
+        <version>${nitrite.version}</version>
+    </dependency>
+    <dependency>
+        <groupId>org.dizitart</groupId>
+        <artifactId>nitrite-mvstore-adapter</artifactId>
+        <version>${nitrite.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.h2database</groupId>
+      <artifactId>h2</artifactId>
+      <version>2.3.230</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.h2database</groupId>
+      <artifactId>h2-mvstore</artifactId>
+      <version>2.3.230</version>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-junit-jupiter</artifactId>
+      <version>5.10.0</version>
+      <scope>test</scope>
+    </dependency>
+
 	</dependencies>
 </project>

src/main/java/org/qortal/account/Account.java

@@ -2,14 +2,12 @@ package org.qortal.account;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.qortal.api.resource.TransactionsResource;
 import org.qortal.block.BlockChain;
 import org.qortal.controller.LiteNode;
 import org.qortal.data.account.AccountBalanceData;
 import org.qortal.data.account.AccountData;
 import org.qortal.data.account.RewardShareData;
 import org.qortal.data.naming.NameData;
-import org.qortal.data.transaction.TransactionData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.GroupRepository;
 import org.qortal.repository.NameRepository;
@@ -21,11 +19,7 @@ import org.qortal.utils.Groups;
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 
-import java.util.Comparator;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
-import java.util.Optional;
 
 import static org.qortal.utils.Amounts.prettyAmount;
 
@@ -367,142 +361,6 @@ public class Account {
 		return accountData.getLevel();
 	}
 
-	/**
-	 * Get Primary Name
-	 *
-	 * @return the primary name for this address if present, otherwise empty
-	 *
-	 * @throws DataException
-	 */
-	public Optional<String> getPrimaryName() throws DataException {
-
-		return this.repository.getNameRepository().getPrimaryName(this.address);
-	}
-
-	/**
-	 * Remove Primary Name
-	 *
-	 * @throws DataException
-	 */
-	public void removePrimaryName() throws DataException {
-		this.repository.getNameRepository().removePrimaryName(this.address);
-	}
-
-	/**
-	 * Reset Primary Name
-	 *
-	 * Set primary name based on the names (and their history) this account owns.
-	 *
-	 * @param confirmationStatus the status of the transactions for the determining the primary name
-	 *
-	 * @return the primary name, empty if their isn't one
-	 *
-	 * @throws DataException
-	 */
-	public Optional<String> resetPrimaryName(TransactionsResource.ConfirmationStatus confirmationStatus) throws DataException {
-		Optional<String> primaryName = determinePrimaryName(confirmationStatus);
-
-		if(primaryName.isPresent()) {
-			return setPrimaryName(primaryName.get());
-		}
-		else {
-			return primaryName;
-		}
-	}
-
-	/**
-	 * Determine Primary Name
-	 *
-	 * Determine primary name based on a list of registered names.
-	 *
-	 * @param confirmationStatus the status of the transactions for this determination
-	 *
-	 * @return the primary name, empty if there is no primary name
-	 *
-	 * @throws DataException
-	 */
-	public Optional<String> determinePrimaryName(TransactionsResource.ConfirmationStatus confirmationStatus) throws DataException {
-
-		// all registered names for the owner
-		List<NameData> names = this.repository.getNameRepository().getNamesByOwner(this.address);
-
-		Optional<String> primaryName;
-
-		// if no registered names, the no primary name possible
-		if (names.isEmpty()) {
-			primaryName = Optional.empty();
-		}
-		// if names
-		else {
-			// if one name, then that is the primary name
-			if (names.size() == 1) {
-				primaryName = Optional.of( names.get(0).getName() );
-			}
-			// if more than one name, then seek the earliest name acquisition that was never released
-			else {
-				Map<String, TransactionData> txByName = new HashMap<>(names.size());
-
-				// for each name, get the latest transaction
-				for (NameData nameData : names) {
-
-					// since the name is currently registered to the owner,
-					// we assume the latest transaction involving this name was the transaction that the acquired
-					// name through registration, purchase or update
-					Optional<TransactionData> latestTransaction
-							= this.repository
-							.getTransactionRepository()
-							.getTransactionsInvolvingName(
-									nameData.getName(),
-									confirmationStatus
-							)
-							.stream()
-							.sorted(Comparator.comparing(
-									TransactionData::getTimestamp).reversed()
-							)
-							.findFirst(); // first is the last, since it was reversed
-
-					// if there is a latest transaction, expected for all registered names
-					if (latestTransaction.isPresent()) {
-						txByName.put(nameData.getName(), latestTransaction.get());
-					}
-					// if there is no latest transaction, then
-					else {
-						LOGGER.warn("No matching transaction for name: " + nameData.getName());
-					}
-				}
-
-				// get the first name aqcuistion for this address
-				Optional<Map.Entry<String, TransactionData>> firstNameEntry
-						= txByName.entrySet().stream().sorted(Comparator.comparing(entry -> entry.getValue().getTimestamp())).findFirst();
-
-				// if their is a name acquisition, then the first one is the primary name
-				if (firstNameEntry.isPresent()) {
-					primaryName = Optional.of( firstNameEntry.get().getKey() );
-				}
-				// if there is no nameacquistion, then there is no primary name
-				else {
-					primaryName =  Optional.empty();
-				}
-			}
-		}
-		return primaryName;
-	}
-
-	/**
-	 * Set Primary Name
-	 *
-	 * @param primaryName the primary to set to this address
-	 *
-	 * @return the primary name if successful, empty if unsuccessful
-	 *
-	 * @throws DataException
-	 */
-	public Optional<String> setPrimaryName( String primaryName ) throws DataException {
-		int changed = this.repository.getNameRepository().setPrimaryName(this.address, primaryName);
-
-		return changed > 0 ? Optional.of(primaryName) : Optional.empty();
-	}
-
 	/**
 	 * Returns reward-share minting address, or unknown if reward-share does not exist.
 	 * 

src/main/java/org/qortal/account/PublicKeyAccount.java

@@ -1,41 +1,17 @@
 package org.qortal.account;
 
-import org.bouncycastle.crypto.generators.Ed25519KeyPairGenerator;
-import org.bouncycastle.crypto.params.Ed25519KeyGenerationParameters;
 import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.account.AccountData;
 import org.qortal.repository.Repository;
 
-import java.security.SecureRandom;
-
 public class PublicKeyAccount extends Account {
 
 	protected final byte[] publicKey;
 	protected final Ed25519PublicKeyParameters edPublicKeyParams;
 
-	/** <p>Constructor for generating a PublicKeyAccount</p>
-	 *
-	 * @param repository Block Chain
-	 * @param publicKey 32 byte Public Key
-	 * @since v4.7.3
-	 */
 	public PublicKeyAccount(Repository repository, byte[] publicKey) {
-		super(repository, Crypto.toAddress(publicKey));
-
-		Ed25519PublicKeyParameters t = null;
-		try {
-			t = new Ed25519PublicKeyParameters(publicKey, 0);
-		} catch (Exception e) {
-			var gen = new Ed25519KeyPairGenerator();
-			gen.init(new Ed25519KeyGenerationParameters(new SecureRandom()));
-			var keyPair = gen.generateKeyPair();
-			t = (Ed25519PublicKeyParameters) keyPair.getPublic();
-		} finally {
-			this.edPublicKeyParams = t;
-		}
-
-		this.publicKey = publicKey;
+		this(repository, new Ed25519PublicKeyParameters(publicKey, 0));
 	}
 
 	protected PublicKeyAccount(Repository repository, Ed25519PublicKeyParameters edPublicKeyParams) {

src/main/java/org/qortal/api/ApiService.java

@@ -46,7 +46,6 @@ public class ApiService {
 	private ApiService() {
 		this.config = new ResourceConfig();
 		this.config.packages("org.qortal.api.resource", "org.qortal.api.restricted.resource");
-		this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
 		this.config.register(OpenApiResource.class);
 		this.config.register(ApiDefinition.class);
 		this.config.register(AnnotationPostProcessor.class);
@@ -198,7 +197,6 @@ public class ApiService {
 			context.addServlet(DataMonitorSocket.class, "/websockets/datamonitor");
 			context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*");
 			context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages");
-			context.addServlet(UnsignedFeesSocket.class, "/websockets/crosschain/unsignedfees");
 			context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers");
 			context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot");
 			context.addServlet(TradePresenceWebSocket.class, "/websockets/crosschain/tradepresence");

src/main/java/org/qortal/api/DevProxyService.java

@@ -40,7 +40,6 @@ public class DevProxyService {
 	private DevProxyService() {
 		this.config = new ResourceConfig();
 		this.config.packages("org.qortal.api.proxy.resource", "org.qortal.api.resource");
-		this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
 		this.config.register(OpenApiResource.class);
 		this.config.register(ApiDefinition.class);
 		this.config.register(AnnotationPostProcessor.class);

src/main/java/org/qortal/api/DomainMapService.java

@@ -39,7 +39,6 @@ public class DomainMapService {
 	private DomainMapService() {
 		this.config = new ResourceConfig();
 		this.config.packages("org.qortal.api.resource", "org.qortal.api.domainmap.resource");
-		this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
 		this.config.register(OpenApiResource.class);
 		this.config.register(ApiDefinition.class);
 		this.config.register(AnnotationPostProcessor.class);

src/main/java/org/qortal/api/GatewayService.java

@@ -39,7 +39,6 @@ public class GatewayService {
 	private GatewayService() {
 		this.config = new ResourceConfig();
 		this.config.packages("org.qortal.api.resource", "org.qortal.api.gateway.resource");
-		this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
 		this.config.register(OpenApiResource.class);
 		this.config.register(ApiDefinition.class);
 		this.config.register(AnnotationPostProcessor.class);

src/main/java/org/qortal/api/HTMLParser.java

@@ -1,13 +1,14 @@
 package org.qortal.api;
 
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.select.Elements;
 import org.qortal.arbitrary.misc.Service;
+
 import java.util.Objects;
+
 public class HTMLParser {
 
     private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
@@ -21,11 +22,10 @@ public class HTMLParser {
     private String identifier;
     private String path;
     private String theme;
-    private String lang;
     private boolean usingCustomRouting;
 
     public HTMLParser(String resourceId, String inPath, String prefix, boolean includeResourceIdInPrefix, byte[] data,
-                      String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting,  String lang) {
+                      String qdnContext, Service service, String identifier, String theme, boolean usingCustomRouting) {
         String inPathWithoutFilename = inPath.contains("/") ? inPath.substring(0, inPath.lastIndexOf('/')) : String.format("/%s",inPath);
         this.qdnBase = includeResourceIdInPrefix ? String.format("%s/%s", prefix, resourceId) : prefix;
         this.qdnBaseWithPath = includeResourceIdInPrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : String.format("%s%s", prefix, inPathWithoutFilename);
@@ -36,7 +36,6 @@ public class HTMLParser {
         this.identifier = identifier;
         this.path = inPath;
         this.theme = theme;
-        this.lang = lang;
         this.usingCustomRouting = usingCustomRouting;
     }
 
@@ -62,13 +61,9 @@ public class HTMLParser {
             String identifier = this.identifier != null ? this.identifier.replace("\\", "").replace("\"","\\\"") : "";
             String path = this.path != null ? this.path.replace("\\", "").replace("\"","\\\"") : "";
             String theme = this.theme != null ? this.theme.replace("\\", "").replace("\"","\\\"") : "";
-            String lang = this.lang != null ? this.lang.replace("\\", "").replace("\"", "\\\"") : "";
             String qdnBase = this.qdnBase != null ? this.qdnBase.replace("\\", "").replace("\"","\\\"") : "";
             String qdnBaseWithPath = this.qdnBaseWithPath != null ? this.qdnBaseWithPath.replace("\\", "").replace("\"","\\\"") : "";
-            String qdnContextVar = String.format(
-                "<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnLang=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>",
-                qdnContext, theme, lang, service, name, identifier, path, qdnBase, qdnBaseWithPath
-              );
+            String qdnContextVar = String.format("<script>var _qdnContext=\"%s\"; var _qdnTheme=\"%s\"; var _qdnService=\"%s\"; var _qdnName=\"%s\"; var _qdnIdentifier=\"%s\"; var _qdnPath=\"%s\"; var _qdnBase=\"%s\"; var _qdnBaseWithPath=\"%s\";</script>", qdnContext, theme, service, name, identifier, path, qdnBase, qdnBaseWithPath);
             head.get(0).prepend(qdnContextVar);
 
             // Add base href tag

src/main/java/org/qortal/api/model/crosschain/BitcoinyTBDRequest.java

@@ -304,11 +304,11 @@ public class BitcoinyTBDRequest {
     private String networkName;
 
     /**
-     * Fee Required
+     * Fee Ceiling
      *
-     *  web search, LTC fee required = 1000L
+     *  web search, LTC fee ceiling = 1000L
      */
-    private long feeRequired;
+    private long feeCeiling;
 
     /**
      * Extended Public Key
@@ -570,8 +570,8 @@ public class BitcoinyTBDRequest {
         return this.networkName;
     }
 
-    public long getFeeRequired() {
-        return this.feeRequired;
+    public long getFeeCeiling() {
+        return this.feeCeiling;
     }
 
     public String getExtendedPublicKey() {
@@ -671,7 +671,7 @@ public class BitcoinyTBDRequest {
                 ", minimumOrderAmount=" + minimumOrderAmount +
                 ", feePerKb=" + feePerKb +
                 ", networkName='" + networkName + '\'' +
-                ", feeRequired=" + feeRequired +
+                ", feeCeiling=" + feeCeiling +
                 ", extendedPublicKey='" + extendedPublicKey + '\'' +
                 ", sendAmount=" + sendAmount +
                 ", sendingFeePerByte=" + sendingFeePerByte +

src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java

@@ -142,20 +142,10 @@ public class DevProxyServerResource {
             }
         }
 
-        String lang = request.getParameter("lang");
-        if (lang == null || lang.isBlank()) {
-            lang = "en"; // fallback
-        }
-
-        String theme = request.getParameter("theme");
-        if (theme == null || theme.isBlank()) {
-            theme = "light";
-        }
-
         // Parse and modify output if needed
         if (HTMLParser.isHtmlFile(filename)) {
             // HTML file - needs to be parsed
-            HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, theme , true, lang);
+            HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true);
             htmlParser.addAdditionalHeaderTags();
             response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;");
             response.setContentType(con.getContentType());

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -3,7 +3,6 @@ package org.qortal.api.resource;
 import com.google.common.primitives.Bytes;
 import com.j256.simplemagic.ContentInfo;
 import com.j256.simplemagic.ContentInfoUtil;
-
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.ArraySchema;
@@ -13,7 +12,6 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.logging.log4j.LogManager;
@@ -65,19 +63,14 @@ import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
-
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
 import java.net.FileNameMap;
 import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -85,16 +78,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
-import java.util.zip.GZIPOutputStream;
-
-import org.apache.tika.Tika;
-import org.apache.tika.mime.MimeTypeException;
-import org.apache.tika.mime.MimeTypes;
-
-import javax.ws.rs.core.Response;
-
-import org.glassfish.jersey.media.multipart.FormDataParam;
-import static org.qortal.api.ApiError.REPOSITORY_ISSUE;
 
 @Path("/arbitrary")
 @Tag(name = "Arbitrary")
@@ -703,20 +686,20 @@ public class ArbitraryResource {
 					)
 			}
 	)
-	public void get(@PathParam("service") Service service,
+	public HttpServletResponse get(@PathParam("service") Service service,
 								   @PathParam("name") String name,
 								   @QueryParam("filepath") String filepath,
 								   @QueryParam("encoding") String encoding,
 								   @QueryParam("rebuild") boolean rebuild,
 								   @QueryParam("async") boolean async,
-								   @QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) {
+								   @QueryParam("attempts") Integer attempts) {
 
 		// Authentication can be bypassed in the settings, for those running public QDN nodes
 		if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
 			Security.checkApiCallAllowed(request);
 		}
 
-		 this.download(service, name, null, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename);
+		return this.download(service, name, null, filepath, encoding, rebuild, async, attempts);
 	}
 
 	@GET
@@ -736,21 +719,21 @@ public class ArbitraryResource {
 					)
 			}
 	)
-	public void get(@PathParam("service") Service service,
+	public HttpServletResponse get(@PathParam("service") Service service,
 								   @PathParam("name") String name,
 								   @PathParam("identifier") String identifier,
 								   @QueryParam("filepath") String filepath,
 								   @QueryParam("encoding") String encoding,
 								   @QueryParam("rebuild") boolean rebuild,
 								   @QueryParam("async") boolean async,
-								   @QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) {
+								   @QueryParam("attempts") Integer attempts) {
 
 		// Authentication can be bypassed in the settings, for those running public QDN nodes
 		if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
 			Security.checkApiCallAllowed(request, null);
 		}
 
-		this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename);
+		return this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts);
 	}
 
 
@@ -895,464 +878,6 @@ public class ArbitraryResource {
 	}
 
 
-	@GET
-	@Path("/check/tmp")
-	@Produces(MediaType.TEXT_PLAIN)
-	@Operation(
-		summary = "Check if the disk has enough disk space for an upcoming upload",
-		responses = {
-			@ApiResponse(description = "OK if sufficient space", responseCode = "200"),
-			@ApiResponse(description = "Insufficient space", responseCode = "507") // 507 = Insufficient Storage
-		}
-	)
-	@SecurityRequirement(name = "apiKey")
-	public Response checkUploadSpace(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-									 @QueryParam("totalSize") Long totalSize) {
-		Security.checkApiCallAllowed(request);
-	
-		if (totalSize == null || totalSize <= 0) {
-			return Response.status(Response.Status.BAD_REQUEST)
-					.entity("Missing or invalid totalSize parameter").build();
-		}
-	
-		File uploadDir = new File("uploads-temp");
-		if (!uploadDir.exists()) {
-			uploadDir.mkdirs(); // ensure the folder exists
-		}
-
-		long usableSpace = uploadDir.getUsableSpace();
-		long requiredSpace = (long)(((double)totalSize) * 2.2); // estimate for chunks + merge
-
-		if (usableSpace < requiredSpace) {
-			return Response.status(507).entity("Insufficient disk space").build();
-		}
-
-		return Response.ok("Sufficient disk space").build();
-	}
-
-	@POST
-@Path("/{service}/{name}/chunk")
-@Consumes(MediaType.MULTIPART_FORM_DATA)
-@Operation(
-    summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource (no identifier)",
-    requestBody = @RequestBody(
-        required = true,
-        content = @Content(
-            mediaType = MediaType.MULTIPART_FORM_DATA,
-            schema = @Schema(
-                implementation = Object.class
-            )
-        )
-    ),
-    responses = {
-        @ApiResponse(
-            description = "Chunk uploaded successfully",
-            responseCode = "200"
-        ),
-        @ApiResponse(
-            description = "Error writing chunk",
-            responseCode = "500"
-        )
-    }
-)
-@SecurityRequirement(name = "apiKey")
-public Response uploadChunkNoIdentifier(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-                                        @PathParam("service") String serviceString,
-                                        @PathParam("name") String name,
-                                        @FormDataParam("chunk") InputStream chunkStream,
-                                        @FormDataParam("index") int index) {
-    Security.checkApiCallAllowed(request);
-
-    try {
-		String safeService = Paths.get(serviceString).getFileName().toString();
-        String safeName = Paths.get(name).getFileName().toString();
-
-		java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName);
-		Files.createDirectories(tempDir);
-		
-        
-
-        java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index);
-        Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING);
-
-        return Response.ok("Chunk " + index + " received").build();
-    } catch (IOException e) {
-		LOGGER.error("Failed to write chunk {} for service '{}' and name '{}'", index, serviceString, name, e);
-        return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build();
-    }
-}
-
-@POST
-@Path("/{service}/{name}/finalize")
-@Produces(MediaType.TEXT_PLAIN)
-@Operation(
-    summary = "Finalize a chunked upload (no identifier) and build a raw, unsigned, ARBITRARY transaction",
-    responses = {
-        @ApiResponse(
-            description = "raw, unsigned, ARBITRARY transaction encoded in Base58",
-            content = @Content(mediaType = MediaType.TEXT_PLAIN)
-        )
-    }
-)
-@SecurityRequirement(name = "apiKey")
-public String finalizeUploadNoIdentifier(
-    @HeaderParam(Security.API_KEY_HEADER) String apiKey,
-    @PathParam("service") String serviceString,
-    @PathParam("name") String name,
-    @QueryParam("title") String title,
-    @QueryParam("description") String description,
-    @QueryParam("tags") List<String> tags,
-    @QueryParam("category") Category category,
-    @QueryParam("filename") String filename,
-    @QueryParam("fee") Long fee,
-    @QueryParam("preview") Boolean preview,
-    @QueryParam("isZip") Boolean isZip
-) {
-    Security.checkApiCallAllowed(request);
-    java.nio.file.Path tempFile = null;
-    java.nio.file.Path tempDir = null;
-	java.nio.file.Path chunkDir = null;
-    String safeService = Paths.get(serviceString).getFileName().toString();
-	String safeName = Paths.get(name).getFileName().toString();
-
-
-
-    try {
-		chunkDir = Paths.get("uploads-temp", safeService, safeName);
-
-        if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) {
-            throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload");
-        }
-
-        String safeFilename = (filename == null || filename.isBlank()) ? "qortal-" + NTP.getTime() : filename;
-        tempDir = Files.createTempDirectory("qortal-");
-        String sanitizedFilename = Paths.get(safeFilename).getFileName().toString();
-		tempFile = tempDir.resolve(sanitizedFilename);
-
-        try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
-            byte[] buffer = new byte[65536];
-            for (java.nio.file.Path chunk : Files.list(chunkDir)
-                    .filter(path -> path.getFileName().toString().startsWith("chunk_"))
-                    .sorted(Comparator.comparingInt(path -> {
-                        String name2 = path.getFileName().toString();
-                        String numberPart = name2.substring("chunk_".length());
-                        return Integer.parseInt(numberPart);
-                    })).collect(Collectors.toList())) {
-                try (InputStream in = Files.newInputStream(chunk)) {
-                    int bytesRead;
-                    while ((bytesRead = in.read(buffer)) != -1) {
-                        out.write(buffer, 0, bytesRead);
-                    }
-                }
-            }
-        }
-
-        String detectedExtension = "";
-        String uploadFilename = null;
-        boolean extensionIsValid = false;
-
-        if (filename != null && !filename.isBlank()) {
-            int lastDot = filename.lastIndexOf('.');
-            if (lastDot > 0 && lastDot < filename.length() - 1) {
-                extensionIsValid = true;
-                uploadFilename = filename;
-            }
-        }
-
-        if (!extensionIsValid) {
-            Tika tika = new Tika();
-            String mimeType = tika.detect(tempFile.toFile());
-            try {
-                MimeTypes allTypes = MimeTypes.getDefaultMimeTypes();
-                org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType);
-                detectedExtension = mime.getExtension();
-            } catch (MimeTypeException e) {
-                LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e);
-            }
-
-            if (filename != null && !filename.isBlank()) {
-                int lastDot = filename.lastIndexOf('.');
-                String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename;
-                uploadFilename = baseName + (detectedExtension != null ? detectedExtension : "");
-            } else {
-                uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : "");
-            }
-        }
-
-		Boolean isZipBoolean = false;
-
-		if (isZip != null && isZip) {
-			isZipBoolean = true;
-		}
-        
-
-        // ✅ Call upload with `null` as identifier
-        return this.upload(
-            Service.valueOf(serviceString),
-            name,
-            null, // no identifier
-            tempFile.toString(),
-            null,
-            null,
-            isZipBoolean,
-            fee,
-            uploadFilename,
-            title,
-            description,
-            tags,
-            category,
-            preview
-        );
-
-    } catch (IOException e) {
-		LOGGER.error("Failed to merge chunks for service='{}', name='{}'", serviceString, name, e);
-
-        throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage());
-    } finally {
-        if (tempDir != null) {
-            try {
-                Files.walk(tempDir)
-                    .sorted(Comparator.reverseOrder())
-                    .map(java.nio.file.Path::toFile)
-                    .forEach(File::delete);
-            } catch (IOException e) {
-                LOGGER.warn("Failed to delete temp directory: {}", tempDir, e);
-            }
-        }
-
-        try {
-            Files.walk(chunkDir)
-                .sorted(Comparator.reverseOrder())
-                .map(java.nio.file.Path::toFile)
-                .forEach(File::delete);
-        } catch (IOException e) {
-            LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e);
-        }
-    }
-}
-
-
-
-	@POST
-@Path("/{service}/{name}/{identifier}/chunk")
-@Consumes(MediaType.MULTIPART_FORM_DATA)
-@Operation(
-    summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource",
-    requestBody = @RequestBody(
-        required = true,
-        content = @Content(
-            mediaType = MediaType.MULTIPART_FORM_DATA,
-            schema = @Schema(
-                implementation = Object.class
-            )
-        )
-    ),
-    responses = {
-        @ApiResponse(
-            description = "Chunk uploaded successfully",
-            responseCode = "200"
-        ),
-        @ApiResponse(
-            description = "Error writing chunk",
-            responseCode = "500"
-        )
-    }
-)
-@SecurityRequirement(name = "apiKey")
-public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-                            @PathParam("service") String serviceString,
-                            @PathParam("name") String name,
-                            @PathParam("identifier") String identifier,
-                            @FormDataParam("chunk") InputStream chunkStream,
-                            @FormDataParam("index") int index) {
-    Security.checkApiCallAllowed(request);
-
-    try {
-        String safeService = Paths.get(serviceString).getFileName().toString();
-        String safeName = Paths.get(name).getFileName().toString();
-        String safeIdentifier = Paths.get(identifier).getFileName().toString();
-
-        java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName, safeIdentifier);
-
-        Files.createDirectories(tempDir);
-
-        java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index);
-        Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING);
-
-        return Response.ok("Chunk " + index + " received").build();
-    } catch (IOException e) {
-		LOGGER.error("Failed to write chunk {} for service='{}', name='{}', identifier='{}'", index, serviceString, name, identifier, e);
-        return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build();
-    }
-}
-
-@POST
-@Path("/{service}/{name}/{identifier}/finalize")
-@Produces(MediaType.TEXT_PLAIN)
-@Operation(
-    summary = "Finalize a chunked upload and build a raw, unsigned, ARBITRARY transaction",
-    responses = {
-        @ApiResponse(
-            description = "raw, unsigned, ARBITRARY transaction encoded in Base58",
-            content = @Content(mediaType = MediaType.TEXT_PLAIN)
-        )
-    }
-)
-@SecurityRequirement(name = "apiKey")
-public String finalizeUpload(
-    @HeaderParam(Security.API_KEY_HEADER) String apiKey,
-    @PathParam("service") String serviceString,
-    @PathParam("name") String name,
-    @PathParam("identifier") String identifier,
-    @QueryParam("title") String title,
-    @QueryParam("description") String description,
-    @QueryParam("tags") List<String> tags,
-    @QueryParam("category") Category category,
-    @QueryParam("filename") String filename,
-    @QueryParam("fee") Long fee,
-    @QueryParam("preview") Boolean preview,
-	@QueryParam("isZip") Boolean isZip
-) {
-    Security.checkApiCallAllowed(request);
-    java.nio.file.Path tempFile = null;
-    java.nio.file.Path tempDir = null;
-	java.nio.file.Path chunkDir = null;
-
-	
-	
-	
-
-    try {
-		String safeService = Paths.get(serviceString).getFileName().toString();
-	String safeName = Paths.get(name).getFileName().toString();
-	String safeIdentifier = Paths.get(identifier).getFileName().toString();
-	java.nio.file.Path baseUploadsDir = Paths.get("uploads-temp"); // relative to Qortal working dir
-	chunkDir = baseUploadsDir.resolve(safeService).resolve(safeName).resolve(safeIdentifier);
-		
-        if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) {
-            throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload");
-        }
-
-        // Step 1: Determine a safe filename for disk temp file (regardless of extension correctness)
-        String safeFilename = filename;
-        if (filename == null || filename.isBlank()) {
-			safeFilename = "qortal-" + NTP.getTime();
-		} 
-
-        tempDir = Files.createTempDirectory("qortal-");
-        String sanitizedFilename = Paths.get(safeFilename).getFileName().toString();
-		tempFile = tempDir.resolve(sanitizedFilename);
-
-
-        // Step 2: Merge chunks
-  
-        try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
-            byte[] buffer = new byte[65536];
-            for (java.nio.file.Path chunk : Files.list(chunkDir)
-                    .filter(path -> path.getFileName().toString().startsWith("chunk_"))
-                    .sorted(Comparator.comparingInt(path -> {
-                        String name2 = path.getFileName().toString();
-                        String numberPart = name2.substring("chunk_".length());
-                        return Integer.parseInt(numberPart);
-                    })).collect(Collectors.toList())) {
-                try (InputStream in = Files.newInputStream(chunk)) {
-                    int bytesRead;
-                    while ((bytesRead = in.read(buffer)) != -1) {
-                        out.write(buffer, 0, bytesRead);
-                    }
-                }
-            }
-        }
-       
-
-        // Step 3: Determine correct extension
-        String detectedExtension = "";
-        String uploadFilename  = null;
-        boolean extensionIsValid = false;
-
-        if (filename != null && !filename.isBlank()) {
-            int lastDot = filename.lastIndexOf('.');
-            if (lastDot > 0 && lastDot < filename.length() - 1) {
-                extensionIsValid = true;
-                uploadFilename = filename;
-            }
-        }
-
-        if (!extensionIsValid) {
-            Tika tika = new Tika();
-            String mimeType = tika.detect(tempFile.toFile());
-            try {
-                MimeTypes allTypes = MimeTypes.getDefaultMimeTypes();
-                org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType);
-                detectedExtension = mime.getExtension();
-            } catch (MimeTypeException e) {
-                LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e);
-            }
-
-            if (filename != null && !filename.isBlank()) {
-                int lastDot = filename.lastIndexOf('.');
-                String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename;
-                uploadFilename = baseName + (detectedExtension != null ? detectedExtension : "");
-            } else {
-                uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : "");
-            }
-        }
-
-
-		Boolean isZipBoolean = false;
-
-		if (isZip != null && isZip) {
-			isZipBoolean = true;
-		}
-        
-
-        return this.upload(
-            Service.valueOf(serviceString),
-            name,
-            identifier,
-            tempFile.toString(),
-            null,
-            null,
-            isZipBoolean,
-            fee,
-            uploadFilename,
-            title,
-            description,
-            tags,
-            category,
-            preview
-        );
-
-    } catch (IOException e) {
-		LOGGER.error("Unexpected error in finalizeUpload for service='{}', name='{}', name='{}'", serviceString, name, identifier, e);
-
-        throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage());
-    } finally {
-        if (tempDir != null) {
-            try {
-                Files.walk(tempDir)
-                    .sorted(Comparator.reverseOrder())
-                    .map(java.nio.file.Path::toFile)
-                    .forEach(File::delete);
-            } catch (IOException e) {
-                LOGGER.warn("Failed to delete temp directory: {}", tempDir, e);
-            }
-        }
-
-        try {
-            Files.walk(chunkDir)
-                .sorted(Comparator.reverseOrder())
-                .map(java.nio.file.Path::toFile)
-                .forEach(File::delete);
-        } catch (IOException e) {
-            LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e);
-        }
-    }
-}
-
-
-
-
 
 	// Upload base64-encoded data
 
@@ -1818,7 +1343,7 @@ public String finalizeUpload(
 			if (path == null) {
 				// See if we have a string instead
 				if (string != null) {
-					if (filename == null || filename.isBlank()) {
+					if (filename == null) {
 						// Use current time as filename
 						filename = String.format("qortal-%d", NTP.getTime());
 					}
@@ -1833,7 +1358,7 @@ public String finalizeUpload(
 				}
 				// ... or base64 encoded raw data
 				else if (base64 != null) {
-					if (filename == null || filename.isBlank()) {
+					if (filename == null) {
 						// Use current time as filename
 						filename = String.format("qortal-%d", NTP.getTime());
 					}
@@ -1884,7 +1409,6 @@ public String finalizeUpload(
 				);
 
 				transactionBuilder.build();
-
 				// Don't compute nonce - this is done by the client (or via POST /arbitrary/compute)
 				ArbitraryTransactionData transactionData = transactionBuilder.getArbitraryTransactionData();
 				return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData));
@@ -1900,20 +1424,22 @@ public String finalizeUpload(
 		}
 	}
 
-	private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts, boolean attachment, String attachmentFilename) {
+	private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) {
+
 		try {
 			ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
-	
+
 			int attempts = 0;
 			if (maxAttempts == null) {
 				maxAttempts = 5;
 			}
-	
+
 			// Loop until we have data
 			if (async) {
 				// Asynchronous
 				arbitraryDataReader.loadAsynchronously(false, 1);
-			} else {
+			}
+			else {
 				// Synchronous
 				while (!Controller.isStopping()) {
 					attempts++;
@@ -1923,189 +1449,88 @@ public String finalizeUpload(
 							break;
 						} catch (MissingDataException e) {
 							if (attempts > maxAttempts) {
+								// Give up after 5 attempts
 								throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later.");
 							}
 						}
 					}
+					Thread.sleep(3000L);
 				}
 			}
-	
+
 			java.nio.file.Path outputPath = arbitraryDataReader.getFilePath();
 			if (outputPath == null) {
 				// Assume the resource doesn't exist
 				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found");
 			}
-	
+
 			if (filepath == null || filepath.isEmpty()) {
 				// No file path supplied - so check if this is a single file resource
 				String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal");
 				if (files != null && files.length == 1) {
 					// This is a single file resource
 					filepath = files[0];
-				} else {
-					throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "filepath is required for resources containing more than one file");
+				}
+				else {
+					throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA,
+							"filepath is required for resources containing more than one file");
 				}
 			}
-	
+
 			java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
 			if (!Files.exists(path)) {
-				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No file exists at filepath: " + filepath);
+				String message = String.format("No file exists at filepath: %s", filepath);
+				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message);
 			}
-	
-			if (attachment) {
-				String rawFilename;
-	
-				if (attachmentFilename != null && !attachmentFilename.isEmpty()) {
-					// 1. Sanitize first
-					String safeAttachmentFilename = attachmentFilename.replaceAll("[\\\\/:*?\"<>|]", "_");
-	
-					// 2. Check for a valid extension (3–5 alphanumeric chars)
-					if (!safeAttachmentFilename.matches(".*\\.[a-zA-Z0-9]{2,5}$")) {
-						safeAttachmentFilename += ".bin";
-					}
-	
-					rawFilename = safeAttachmentFilename;
-				} else {
-					// Fallback if no filename is provided
-					String baseFilename = (identifier != null && !identifier.isEmpty())
-						? name + "-" + identifier
-						: name;
-					rawFilename = baseFilename.replaceAll("[\\\\/:*?\"<>|]", "_") + ".bin";
-				}
-	
-				// Optional: trim length
-				rawFilename = rawFilename.length() > 100 ? rawFilename.substring(0, 100) : rawFilename;
-	
-				// 3. Set Content-Disposition header
-				response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\"");
-			}
-	
-			// Determine the total size of the requested file
-			long fileSize = Files.size(path);
-			String mimeType = context.getMimeType(path.toString());
-	
-			// Attempt to read the "Range" header from the request to support partial content delivery (e.g., for video streaming or resumable downloads)
+
+			byte[] data;
+			int fileSize = (int)path.toFile().length();
+			int length = fileSize;
+
+			// Parse "Range" header
+			Integer rangeStart = null;
+			Integer rangeEnd = null;
 			String range = request.getHeader("Range");
-	
-			long rangeStart = 0;
-			long rangeEnd = fileSize - 1;
-			boolean isPartial = false;
-	
-			// If a Range header is present and no base64 encoding is requested, parse the range values
-			if (range != null && encoding == null) {
-				range = range.replace("bytes=", ""); // Remove the "bytes=" prefix
-				String[] parts = range.split("-"); // Split the range into start and end
-	
-				// Parse range start
-				if (parts.length > 0 && !parts[0].isEmpty()) {
-					rangeStart = Long.parseLong(parts[0]);
-				}
-	
-				// Parse range end, if present
-				if (parts.length > 1 && !parts[1].isEmpty()) {
-					rangeEnd = Long.parseLong(parts[1]);
-				}
-	
-				isPartial = true; // Indicate that this is a partial content request
+			if (range != null) {
+				range = range.replace("bytes=", "");
+				String[] parts = range.split("-");
+				rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null;
+				rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize;
 			}
-	
-			// Calculate how many bytes should be sent in the response
-			long contentLength = rangeEnd - rangeStart + 1;
-	
-			// Inform the client that byte ranges are supported
-			response.setHeader("Accept-Ranges", "bytes");
-	
-			if (isPartial) {
-				// If partial content was requested, return 206 Partial Content with appropriate headers
-				response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
-				response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize));
-			} else {
-				// Otherwise, return the entire file with status 200 OK
-				response.setStatus(HttpServletResponse.SC_OK);
+
+			if (rangeStart != null && rangeEnd != null) {
+				// We have a range, so update the requested length
+				length = rangeEnd - rangeStart;
 			}
-	
-			// Initialize output streams for writing the file to the response
-			OutputStream rawOut = null;
-			OutputStream base64Out = null;
-			OutputStream gzipOut = null;
-	
-			try {
-				rawOut = response.getOutputStream();
-	
-				if (encoding != null && "base64".equalsIgnoreCase(encoding)) {
-					// If base64 encoding is requested, override content type
-					response.setContentType("text/plain");
-	
-					// Check if the client accepts gzip encoding
-					String acceptEncoding = request.getHeader("Accept-Encoding");
-					boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip");
-	
-					if (wantsGzip) {
-						// Wrap output in GZIP and Base64 streams if gzip is accepted
-						response.setHeader("Content-Encoding", "gzip");
-						gzipOut = new GZIPOutputStream(rawOut);
-						base64Out = java.util.Base64.getEncoder().wrap(gzipOut);
-					} else {
-						// Wrap output in Base64 only
-						base64Out = java.util.Base64.getEncoder().wrap(rawOut);
-					}
-	
-					rawOut = base64Out; // Use the wrapped stream for writing
-				} else {
-					// For raw binary output, set the content type and length
-					response.setContentType(mimeType != null ? mimeType : "application/octet-stream");
-					response.setContentLength((int) contentLength);
-				}
-	
-				// Stream file content
-				try (InputStream inputStream = Files.newInputStream(path)) {
-					if (rangeStart > 0) {
-						inputStream.skip(rangeStart);
-					}
-	
-					byte[] buffer = new byte[65536];
-					long bytesRemaining = contentLength;
-					int bytesRead;
-	
-					while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) {
-						rawOut.write(buffer, 0, bytesRead);
-						bytesRemaining -= bytesRead;
-					}
-				}
-	
-				// Stream finished
-				if (base64Out != null) {
-					base64Out.close(); // Also flushes and closes the wrapped gzipOut
-				} else if (gzipOut != null) {
-					gzipOut.close(); // Only close gzipOut if it wasn't wrapped by base64Out
-				} else {
-					rawOut.flush(); // Flush only the base output stream if nothing was wrapped
-				}
-	
-				if (!response.isCommitted()) {
-					response.setStatus(HttpServletResponse.SC_OK);
-					response.getWriter().write(" ");
-				}
-	
-			} catch (IOException e) {
-				// Streaming errors should not rethrow — just log
-				LOGGER.warn(String.format("Streaming error for %s %s: %s", service, name, e.getMessage()));
+
+			if (length < fileSize && encoding == null) {
+				// Partial content requested, and not encoding the data
+				response.setStatus(206);
+				response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize));
+				data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length);
 			}
-	
-		} catch (IOException | ApiException | DataException e) {
-			LOGGER.warn(String.format("Unable to load %s %s: %s", service, name, e.getMessage()));
-			if (!response.isCommitted()) {
-				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage());
+			else {
+				// Full content requested (or encoded data)
+				response.setStatus(200);
+				data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory
 			}
-		} catch (NumberFormatException e) {
-			LOGGER.warn(String.format("Invalid range for %s %s: %s", service, name, e.getMessage()));
-			if (!response.isCommitted()) {
-				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage());
+
+			// Encode the data if requested
+			if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) {
+				data = Base64.encode(data);
 			}
+
+			response.addHeader("Accept-Ranges", "bytes");
+			response.setContentType(context.getMimeType(path.toString()));
+			response.setContentLength(data.length);
+			response.getOutputStream().write(data);
+
+			return response;
+		} catch (Exception e) {
+			LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage()));
+			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage());
 		}
 	}
-	
-	
 
 	private FileProperties getFileProperties(Service service, String name, String identifier) {
 		try {

src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java

@@ -502,10 +502,10 @@ public class CrossChainBitcoinResource {
 	}
 
 	@GET
-	@Path("/feerequired")
+	@Path("/feeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking BTC to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Returns Bitcoin fee per Kb.",
+			description = "Returns Bitcoin fee per Kb.",
 			responses = {
 					@ApiResponse(
 							content = @Content(
@@ -516,17 +516,17 @@ public class CrossChainBitcoinResource {
 					)
 			}
 	)
-	public String getBitcoinFeeRequired() {
+	public String getBitcoinFeeCeiling() {
 		Bitcoin bitcoin = Bitcoin.getInstance();
 
-		return String.valueOf(bitcoin.getFeeRequired());
+		return String.valueOf(bitcoin.getFeeCeiling());
 	}
 
 	@POST
-	@Path("/updatefeerequired")
+	@Path("/updatefeeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking BTC to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Sets Bitcoin fee ceiling.",
+			description = "Sets Bitcoin fee ceiling.",
 			requestBody = @RequestBody(
 					required = true,
 					content = @Content(
@@ -545,13 +545,13 @@ public class CrossChainBitcoinResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA})
-	public String setBitcoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
+	public String setBitcoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
 		Security.checkApiCallAllowed(request);
 
 		Bitcoin bitcoin = Bitcoin.getInstance();
 
 		try {
-			return CrossChainUtils.setFeeRequired(bitcoin, fee);
+			return CrossChainUtils.setFeeCeiling(bitcoin, fee);
 		}
 		catch (IllegalArgumentException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);

src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java

@@ -502,10 +502,10 @@ public class CrossChainDigibyteResource {
 	}
 
 	@GET
-	@Path("/feerequired")
+	@Path("/feeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking DGB to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Returns Digibyte fee per Kb.",
+			description = "Returns Digibyte fee per Kb.",
 			responses = {
 					@ApiResponse(
 							content = @Content(
@@ -516,17 +516,17 @@ public class CrossChainDigibyteResource {
 					)
 			}
 	)
-	public String getDigibyteFeeRequired() {
+	public String getDigibyteFeeCeiling() {
 		Digibyte digibyte = Digibyte.getInstance();
 
-		return String.valueOf(digibyte.getFeeRequired());
+		return String.valueOf(digibyte.getFeeCeiling());
 	}
 
 	@POST
-	@Path("/updatefeerequired")
+	@Path("/updatefeeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking DGB to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Sets Digibyte fee ceiling.",
+			description = "Sets Digibyte fee ceiling.",
 			requestBody = @RequestBody(
 					required = true,
 					content = @Content(
@@ -545,13 +545,13 @@ public class CrossChainDigibyteResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA})
-	public String setDigibyteFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
+	public String setDigibyteFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
 		Security.checkApiCallAllowed(request);
 
 		Digibyte digibyte = Digibyte.getInstance();
 
 		try {
-			return CrossChainUtils.setFeeRequired(digibyte, fee);
+			return CrossChainUtils.setFeeCeiling(digibyte, fee);
 		}
 		catch (IllegalArgumentException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);

src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java

@@ -502,10 +502,10 @@ public class CrossChainDogecoinResource {
 	}
 
 	@GET
-	@Path("/feerequired")
+	@Path("/feeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking DOGE to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Returns Dogecoin fee per Kb.",
+			description = "Returns Dogecoin fee per Kb.",
 			responses = {
 					@ApiResponse(
 							content = @Content(
@@ -516,17 +516,17 @@ public class CrossChainDogecoinResource {
 					)
 			}
 	)
-	public String getDogecoinFeeRequired() {
+	public String getDogecoinFeeCeiling() {
 		Dogecoin dogecoin = Dogecoin.getInstance();
 
-		return String.valueOf(dogecoin.getFeeRequired());
+		return String.valueOf(dogecoin.getFeeCeiling());
 	}
 
 	@POST
-	@Path("/updatefeerequired")
+	@Path("/updatefeeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking DOGE to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Sets Dogecoin fee ceiling.",
+			description = "Sets Dogecoin fee ceiling.",
 			requestBody = @RequestBody(
 					required = true,
 					content = @Content(
@@ -545,13 +545,13 @@ public class CrossChainDogecoinResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA})
-	public String setDogecoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
+	public String setDogecoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
 		Security.checkApiCallAllowed(request);
 
 		Dogecoin dogecoin = Dogecoin.getInstance();
 
 		try {
-			return CrossChainUtils.setFeeRequired(dogecoin, fee);
+			return CrossChainUtils.setFeeCeiling(dogecoin, fee);
 		}
 		catch (IllegalArgumentException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);

src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java

@@ -540,10 +540,10 @@ public class CrossChainLitecoinResource {
 	}
 
 	@GET
-	@Path("/feerequired")
+	@Path("/feeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking LTC to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Returns Litecoin fee per Kb.",
+			description = "Returns Litecoin fee per Kb.",
 			responses = {
 					@ApiResponse(
 							content = @Content(
@@ -554,17 +554,17 @@ public class CrossChainLitecoinResource {
 					)
 			}
 	)
-	public String getLitecoinFeeRequired() {
+	public String getLitecoinFeeCeiling() {
 		Litecoin litecoin = Litecoin.getInstance();
 
-		return String.valueOf(litecoin.getFeeRequired());
+		return String.valueOf(litecoin.getFeeCeiling());
 	}
 
 	@POST
-	@Path("/updatefeerequired")
+	@Path("/updatefeeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking LTC to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Sets Litecoin fee ceiling.",
+			description = "Sets Litecoin fee ceiling.",
 			requestBody = @RequestBody(
 					required = true,
 					content = @Content(
@@ -583,13 +583,13 @@ public class CrossChainLitecoinResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA})
-	public String setLitecoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
+	public String setLitecoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
 		Security.checkApiCallAllowed(request);
 
 		Litecoin litecoin = Litecoin.getInstance();
 
 		try {
-			return CrossChainUtils.setFeeRequired(litecoin, fee);
+			return CrossChainUtils.setFeeCeiling(litecoin, fee);
 		}
 		catch (IllegalArgumentException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);

src/main/java/org/qortal/api/resource/CrossChainPirateChainResource.java

@@ -587,10 +587,10 @@ public class CrossChainPirateChainResource {
 	}
 
 	@GET
-	@Path("/feerequired")
+	@Path("/feeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking ARRR to the trade offer creator.",
-			description = "The total fee required for unlocking ARRR to the trade offer creator.",
+			summary = "Returns PirateChain fee per Kb.",
+			description = "Returns PirateChain fee per Kb.",
 			responses = {
 					@ApiResponse(
 							content = @Content(
@@ -601,17 +601,17 @@ public class CrossChainPirateChainResource {
 					)
 			}
 	)
-	public String getPirateChainFeeRequired() {
+	public String getPirateChainFeeCeiling() {
 		PirateChain pirateChain = PirateChain.getInstance();
 
-		return String.valueOf(pirateChain.getFeeRequired());
+		return String.valueOf(pirateChain.getFeeCeiling());
 	}
 
 	@POST
-	@Path("/updatefeerequired")
+	@Path("/updatefeeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking ARRR to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Sets PirateChain fee ceiling.",
+			description = "Sets PirateChain fee ceiling.",
 			requestBody = @RequestBody(
 					required = true,
 					content = @Content(
@@ -630,13 +630,13 @@ public class CrossChainPirateChainResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA})
-	public String setPirateChainFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
+	public String setPirateChainFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
 		Security.checkApiCallAllowed(request);
 
 		PirateChain pirateChain = PirateChain.getInstance();
 
 		try {
-			return CrossChainUtils.setFeeRequired(pirateChain, fee);
+			return CrossChainUtils.setFeeCeiling(pirateChain, fee);
 		}
 		catch (IllegalArgumentException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);

src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java

@@ -502,10 +502,10 @@ public class CrossChainRavencoinResource {
 	}
 
 	@GET
-	@Path("/feerequired")
+	@Path("/feeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking RVN to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Returns Ravencoin fee per Kb.",
+			description = "Returns Ravencoin fee per Kb.",
 			responses = {
 					@ApiResponse(
 							content = @Content(
@@ -516,17 +516,17 @@ public class CrossChainRavencoinResource {
 					)
 			}
 	)
-	public String getRavencoinFeeRequired() {
+	public String getRavencoinFeeCeiling() {
 		Ravencoin ravencoin = Ravencoin.getInstance();
 
-		return String.valueOf(ravencoin.getFeeRequired());
+		return String.valueOf(ravencoin.getFeeCeiling());
 	}
 
 	@POST
-	@Path("/updatefeerequired")
+	@Path("/updatefeeceiling")
 	@Operation(
-			summary = "The total fee required for unlocking RVN to the trade offer creator.",
-			description = "This is in sats for a transaction that is approximately 300 kB in size.",
+			summary = "Sets Ravencoin fee ceiling.",
+			description = "Sets Ravencoin fee ceiling.",
 			requestBody = @RequestBody(
 					required = true,
 					content = @Content(
@@ -545,13 +545,13 @@ public class CrossChainRavencoinResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA})
-	public String setRavencoinFeeRequired(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
+	public String setRavencoinFeeCeiling(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String fee) {
 		Security.checkApiCallAllowed(request);
 
 		Ravencoin ravencoin = Ravencoin.getInstance();
 
 		try {
-			return CrossChainUtils.setFeeRequired(ravencoin, fee);
+			return CrossChainUtils.setFeeCeiling(ravencoin, fee);
 		}
 		catch (IllegalArgumentException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);

src/main/java/org/qortal/api/resource/CrossChainResource.java

@@ -10,8 +10,6 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.glassfish.jersey.media.multipart.ContentDisposition;
 import org.qortal.api.ApiError;
 import org.qortal.api.ApiErrors;
@@ -20,7 +18,6 @@ import org.qortal.api.Security;
 import org.qortal.api.model.CrossChainCancelRequest;
 import org.qortal.api.model.CrossChainTradeLedgerEntry;
 import org.qortal.api.model.CrossChainTradeSummary;
-import org.qortal.controller.ForeignFeesManager;
 import org.qortal.controller.tradebot.TradeBot;
 import org.qortal.crosschain.ACCT;
 import org.qortal.crosschain.AcctMode;
@@ -32,8 +29,6 @@ import org.qortal.data.at.ATData;
 import org.qortal.data.at.ATStateData;
 import org.qortal.data.crosschain.CrossChainTradeData;
 import org.qortal.data.crosschain.TransactionSummary;
-import org.qortal.data.crosschain.ForeignFeeDecodedData;
-import org.qortal.data.crosschain.ForeignFeeEncodedData;
 import org.qortal.data.transaction.BaseTransactionData;
 import org.qortal.data.transaction.MessageTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -69,8 +64,6 @@ import java.util.stream.Collectors;
 @Tag(name = "Cross-Chain")
 public class CrossChainResource {
 
-	private static final Logger LOGGER = LogManager.getLogger(CrossChainResource.class);
-
 	@Context
 	HttpServletRequest request;
 
@@ -367,101 +360,6 @@ public class CrossChainResource {
 		}
 	}
 
-	@POST
-	@Path("/signedfees")
-	@Operation(
-		summary = "",
-		description = "",
-		requestBody = @RequestBody(
-			required = true,
-				content = @Content(
-					mediaType = MediaType.APPLICATION_JSON,
-					array = @ArraySchema(
-						schema = @Schema(
-								implementation = ForeignFeeEncodedData.class
-						)
-					)
-				)
-		),
-		responses = {
-			@ApiResponse(
-				description = "true on success",
-				content = @Content(
-					mediaType = MediaType.TEXT_PLAIN,
-					schema = @Schema(
-							type = "boolean"
-					)
-				)
-			)
-		}
-	)
-	public String postSignedForeignFees(List<ForeignFeeEncodedData> signedFees) {
-
-		LOGGER.info("signedFees = " + signedFees);
-
-		try {
-			ForeignFeesManager.getInstance().addSignedFees(signedFees);
-
-			return "true";
-		}
-		catch( Exception e ) {
-
-			LOGGER.error(e.getMessage(), e);
-
-			return "false";
-		}
-	}
-
-	@GET
-	@Path("/unsignedfees/{address}")
-	@Operation(
-		summary = "",
-		description = "",
-		responses = {
-			@ApiResponse(
-				content = @Content(
-					array = @ArraySchema(
-						schema = @Schema(
-							implementation = ForeignFeeEncodedData.class
-						)
-					)
-				)
-			)
-		}
-	)
-	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	public List<ForeignFeeEncodedData> getUnsignedFees(@PathParam("address") String address) {
-
-		List<ForeignFeeEncodedData> unsignedFeesForAddress = ForeignFeesManager.getInstance().getUnsignedFeesForAddress(address);
-
-		LOGGER.info("address = " + address);
-		LOGGER.info("returning unsigned = " + unsignedFeesForAddress);
-		return unsignedFeesForAddress;
-	}
-
-	@GET
-	@Path("/signedfees")
-	@Operation(
-		summary = "",
-		description = "",
-		responses = {
-			@ApiResponse(
-				content = @Content(
-					array = @ArraySchema(
-						schema = @Schema(
-							implementation = ForeignFeeDecodedData.class
-						)
-					)
-				)
-			)
-		}
-	)
-	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	public List<ForeignFeeDecodedData> getSignedFees() {
-
-		return ForeignFeesManager.getInstance().getSignedFees();
-	}
-
 	/**
 	 * Decode Public Key
 	 *

src/main/java/org/qortal/api/resource/CrossChainUtils.java

@@ -12,15 +12,10 @@ import org.bouncycastle.util.Strings;
 import org.json.simple.JSONObject;
 import org.qortal.api.model.CrossChainTradeLedgerEntry;
 import org.qortal.api.model.crosschain.BitcoinyTBDRequest;
-import org.qortal.asset.Asset;
 import org.qortal.crosschain.*;
-import org.qortal.data.account.AccountBalanceData;
 import org.qortal.data.at.ATData;
 import org.qortal.data.at.ATStateData;
 import org.qortal.data.crosschain.*;
-import org.qortal.event.EventBus;
-import org.qortal.event.LockingFeeUpdateEvent;
-import org.qortal.event.RequiredFeeUpdateEvent;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.utils.Amounts;
@@ -28,11 +23,15 @@ import org.qortal.utils.BitTwiddling;
 
 import java.io.BufferedWriter;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
 import java.io.Writer;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.util.*;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 
@@ -104,13 +103,11 @@ public class CrossChainUtils {
 
         bitcoiny.setFeePerKb(Coin.valueOf(satoshis) );
 
-        EventBus.INSTANCE.notify(new LockingFeeUpdateEvent());
-
         return String.valueOf(bitcoiny.getFeePerKb().value);
     }
 
     /**
-     * Set Fee Required
+     * Set Fee Ceiling
      *
      * @param bitcoiny the blockchain support
      * @param fee the fee in satoshis
@@ -119,16 +116,14 @@ public class CrossChainUtils {
      *
      * @throws IllegalArgumentException if invalid
      */
-    public static String setFeeRequired(Bitcoiny bitcoiny, String fee)  throws IllegalArgumentException{
+    public static String setFeeCeiling(Bitcoiny bitcoiny, String fee)  throws IllegalArgumentException{
 
         long satoshis = Long.parseLong(fee);
         if( satoshis < 0 ) throw new IllegalArgumentException("can't set fee to negative number");
 
-        bitcoiny.setFeeRequired( Long.parseLong(fee));
+        bitcoiny.setFeeCeiling( Long.parseLong(fee));
 
-        EventBus.INSTANCE.notify(new RequiredFeeUpdateEvent(bitcoiny));
-
-        return String.valueOf(bitcoiny.getFeeRequired());
+        return String.valueOf(bitcoiny.getFeeCeiling());
     }
 
     /**
@@ -237,9 +232,6 @@ public class CrossChainUtils {
         return bitcoiny.getBlockchainProvider().removeServer(server);
     }
 
-    public static ChainableServer getCurrentServer( Bitcoiny bitcoiny ) {
-        return bitcoiny.getBlockchainProvider().getCurrentServer();
-    }
     /**
      * Set Current Server
      *
@@ -779,46 +771,4 @@ public class CrossChainUtils {
             entries.add(ledgerEntry);
         }
     }
-
-    public static List<CrossChainTradeData> populateTradeDataList(Repository repository, ACCT acct, List<ATData> atDataList) throws DataException {
-
-        if(atDataList.isEmpty()) return new ArrayList<>(0);
-
-        List<ATStateData> latestATStates
-            = repository.getATRepository()
-                .getLatestATStates(
-                    atDataList.stream()
-                        .map(ATData::getATAddress)
-                        .collect(Collectors.toList())
-                );
-
-        Map<String, ATStateData> atStateDataByAtAddress
-            = latestATStates.stream().collect(Collectors.toMap(ATStateData::getATAddress, Function.identity()));
-
-        Map<String, ATData> atDataByAtAddress
-                = atDataList.stream().collect(Collectors.toMap(ATData::getATAddress, Function.identity()));
-
-        Map<String, Long> balanceByAtAddress
-            = repository
-                .getAccountRepository()
-                .getBalances(new ArrayList<>(atDataByAtAddress.keySet()), Asset.QORT)
-                .stream().collect(Collectors.toMap(AccountBalanceData::getAddress, AccountBalanceData::getBalance));
-
-        List<CrossChainTradeData> crossChainTradeDataList = new ArrayList<>(latestATStates.size());
-
-        for( ATStateData atStateData : latestATStates ) {
-            ATData atData = atDataByAtAddress.get(atStateData.getATAddress());
-            crossChainTradeDataList.add(
-                acct.populateTradeData(
-                    repository,
-                    atData.getCreatorPublicKey(),
-                    atData.getCreation(),
-                    atStateData,
-                    OptionalLong.of(balanceByAtAddress.get(atStateData.getATAddress()))
-                )
-            );
-        }
-
-        return crossChainTradeDataList;
-    }
 }

src/main/java/org/qortal/api/resource/NamesResource.java

@@ -33,7 +33,6 @@ import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import java.util.List;
-import java.util.Optional;
 import java.util.stream.Collectors;
 
 @Path("/names")
@@ -105,45 +104,6 @@ public class NamesResource {
 		}
 	}
 
-	@GET
-	@Path("/primary/{address}")
-	@Operation(
-			summary = "primary name owned by address",
-			responses = {
-					@ApiResponse(
-							description = "registered primary name info",
-							content = @Content(
-									mediaType = MediaType.APPLICATION_JSON,
-									schema = @Schema(implementation = NameSummary.class)
-							)
-					)
-			}
-	)
-	@ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE, ApiError.UNAUTHORIZED})
-	public NameSummary getPrimaryNameByAddress(@PathParam("address") String address) {
-		if (!Crypto.isValidAddress(address))
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-
-			if (Settings.getInstance().isLite()) {
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
-			}
-			else {
-				Optional<String> primaryName = repository.getNameRepository().getPrimaryName(address);
-
-				if(primaryName.isPresent()) {
-					return new NameSummary(new NameData(primaryName.get(), address));
-				}
-				else {
-					return new NameSummary((new NameData(null, address)));
-				}
-			}
-		} catch (DataException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
-		}
-	}
-
 	@GET
 	@Path("/{name}")
 	@Operation(

src/main/java/org/qortal/api/restricted/resource/AdminResource.java

@@ -1092,4 +1092,25 @@ public class AdminResource {
 
 		return info;
 	}
+
+	@GET
+	@Path("/dbstates")
+	@Operation(
+			summary = "Get DB States",
+			description = "Get DB States",
+			responses = {
+					@ApiResponse(
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = DbConnectionInfo.class)))
+					)
+			}
+	)
+	public List<DbConnectionInfo> getDbConnectionsStates() {
+
+		try {
+			return Controller.REPOSITORY_FACTORY.getDbConnectionsStates();
+		} catch (Exception e) {
+			LOGGER.error(e.getMessage(), e);
+			return new ArrayList<>(0);
+		}
+	}
 }

src/main/java/org/qortal/api/restricted/resource/RenderResource.java

@@ -71,33 +71,33 @@ public class RenderResource {
     @Path("/signature/{signature}")
     @SecurityRequirement(name = "apiKey")
     public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature,
-                                                   @QueryParam("theme") String theme, @QueryParam("lang") String lang) {
+                                                   @QueryParam("theme") String theme) {
         if (!Settings.getInstance().isQDNAuthBypassEnabled())
             Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null);
 
-        return this.get(signature, ResourceIdType.SIGNATURE, null, null, "/", null, "/render/signature", true, true, theme, lang);
+        return this.get(signature, ResourceIdType.SIGNATURE, null, null, "/", null, "/render/signature", true, true, theme);
     }
 
     @GET
     @Path("/signature/{signature}/{path:.*}")
     @SecurityRequirement(name = "apiKey")
     public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath,
-                                                  @QueryParam("theme") String theme, @QueryParam("lang") String lang) {
+                                                  @QueryParam("theme") String theme) {
         if (!Settings.getInstance().isQDNAuthBypassEnabled())
             Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null);
 
-        return this.get(signature, ResourceIdType.SIGNATURE, null, null, inPath,null, "/render/signature", true, true, theme, lang);
+        return this.get(signature, ResourceIdType.SIGNATURE, null, null, inPath,null, "/render/signature", true, true, theme);
     }
 
     @GET
     @Path("/hash/{hash}")
     @SecurityRequirement(name = "apiKey")
     public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58,
-                                              @QueryParam("theme") String theme,  @QueryParam("lang") String lang) {
+                                              @QueryParam("theme") String theme) {
         if (!Settings.getInstance().isQDNAuthBypassEnabled())
             Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null);
 
-        return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, "/", secret58, "/render/hash", true, false, theme, lang);
+        return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, "/", secret58, "/render/hash", true, false, theme);
     }
 
     @GET
@@ -105,11 +105,11 @@ public class RenderResource {
     @SecurityRequirement(name = "apiKey")
     public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath,
                                              @QueryParam("secret") String secret58,
-                                             @QueryParam("theme") String theme, @QueryParam("lang") String lang) {
+                                             @QueryParam("theme") String theme) {
         if (!Settings.getInstance().isQDNAuthBypassEnabled())
             Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null);
 
-        return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, inPath, secret58, "/render/hash", true, false, theme, lang);
+        return this.get(hash58, ResourceIdType.FILE_HASH, Service.ARBITRARY_DATA, null, inPath, secret58, "/render/hash", true, false, theme);
     }
 
     @GET
@@ -119,12 +119,12 @@ public class RenderResource {
                                              @PathParam("name") String name,
                                              @PathParam("path") String inPath,
                                              @QueryParam("identifier") String identifier,
-                                             @QueryParam("theme") String theme, @QueryParam("lang") String lang) {
+                                             @QueryParam("theme") String theme) {
         if (!Settings.getInstance().isQDNAuthBypassEnabled())
             Security.requirePriorAuthorization(request, name, service, null);
 
         String prefix = String.format("/render/%s", service);
-        return this.get(name, ResourceIdType.NAME, service, identifier, inPath, null, prefix, true, true, theme, lang);
+        return this.get(name, ResourceIdType.NAME, service, identifier, inPath, null, prefix, true, true, theme);
     }
 
     @GET
@@ -133,18 +133,18 @@ public class RenderResource {
     public HttpServletResponse getIndexByName(@PathParam("service") Service service,
                                               @PathParam("name") String name,
                                               @QueryParam("identifier") String identifier,
-                                              @QueryParam("theme") String theme, @QueryParam("lang") String lang) {
+                                              @QueryParam("theme") String theme) {
         if (!Settings.getInstance().isQDNAuthBypassEnabled())
             Security.requirePriorAuthorization(request, name, service, null);
 
         String prefix = String.format("/render/%s", service);
-        return this.get(name, ResourceIdType.NAME, service, identifier, "/", null, prefix, true, true, theme, lang);
+        return this.get(name, ResourceIdType.NAME, service, identifier, "/", null, prefix, true, true, theme);
     }
 
 
 
     private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String identifier,
-                                    String inPath, String secret58, String prefix, boolean includeResourceIdInPrefix, boolean async, String theme, String lang) {
+                                    String inPath, String secret58, String prefix, boolean includeResourceIdInPrefix, boolean async, String theme) {
 
         ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, identifier, inPath,
                 secret58, prefix, includeResourceIdInPrefix, async, "render", request, response, context);
@@ -152,9 +152,6 @@ public class RenderResource {
         if (theme != null) {
             renderer.setTheme(theme);
         }
-        if (lang != null) {
-            renderer.setLang(lang);
-        }
         return renderer.render();
     }
 

src/main/java/org/qortal/api/websocket/UnsignedFeesSocket.java

@@ -1,83 +0,0 @@
-package org.qortal.api.websocket;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.eclipse.jetty.websocket.api.Session;
-import org.eclipse.jetty.websocket.api.WebSocketException;
-import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose;
-import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect;
-import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError;
-import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
-import org.eclipse.jetty.websocket.api.annotations.WebSocket;
-import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
-import org.qortal.data.crosschain.UnsignedFeeEvent;
-import org.qortal.event.Event;
-import org.qortal.event.EventBus;
-import org.qortal.event.FeeWaitingEvent;
-import org.qortal.event.Listener;
-
-import java.io.IOException;
-import java.io.StringWriter;
-
-@WebSocket
-@SuppressWarnings("serial")
-public class UnsignedFeesSocket extends ApiWebSocket implements Listener {
-
-	private static final Logger LOGGER = LogManager.getLogger(UnsignedFeesSocket.class);
-
-	@Override
-	public void configure(WebSocketServletFactory factory) {
-		LOGGER.info("configure");
-
-		factory.register(UnsignedFeesSocket.class);
-
-		EventBus.INSTANCE.addListener(this);
-	}
-
-	@Override
-	public void listen(Event event) {
-		if (!(event instanceof FeeWaitingEvent))
-			return;
-
-		for (Session session : getSessions()) {
-			FeeWaitingEvent feeWaitingEvent = (FeeWaitingEvent) event;
-			sendUnsignedFeeEvent(session, new UnsignedFeeEvent(feeWaitingEvent.isPositive(), feeWaitingEvent.getAddress()));
-		}
-	}
-
-
-	@OnWebSocketConnect
-	@Override
-	public void onWebSocketConnect(Session session) {
-		super.onWebSocketConnect(session);
-	}
-
-	@OnWebSocketClose
-	@Override
-	public void onWebSocketClose(Session session, int statusCode, String reason) {
-		super.onWebSocketClose(session, statusCode, reason);
-	}
-
-	@OnWebSocketError
-	public void onWebSocketError(Session session, Throwable throwable) {
-		/* We ignore errors for now, but method here to silence log spam */
-	}
-
-	@OnWebSocketMessage
-	public void onWebSocketMessage(Session session, String message) {
-		LOGGER.info("onWebSocketMessage: message = " + message);
-	}
-
-	private void sendUnsignedFeeEvent(Session session, UnsignedFeeEvent unsignedFeeEvent) {
-		StringWriter stringWriter = new StringWriter();
-
-		try {
-			marshall(stringWriter, unsignedFeeEvent);
-
-			session.getRemote().sendStringByFuture(stringWriter.toString());
-		} catch (IOException | WebSocketException e) {
-			// No output this time
-		}
-	}
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java

@@ -4,12 +4,9 @@ import org.qortal.repository.DataException;
 import org.qortal.utils.Base58;
 
 import java.io.IOException;
-import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
-import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.Arrays;
@@ -26,53 +23,37 @@ public class ArbitraryDataDigest {
     }
 
     public void compute() throws IOException, DataException {
-        List<Path> allPaths = Files.walk(path)
-            .filter(Files::isRegularFile)
-            .sorted()
-            .collect(Collectors.toList());
-    
+        List<Path> allPaths = Files.walk(path).filter(Files::isRegularFile).sorted().collect(Collectors.toList());
         Path basePathAbsolute = this.path.toAbsolutePath();
-    
+
         MessageDigest sha256;
         try {
             sha256 = MessageDigest.getInstance("SHA-256");
         } catch (NoSuchAlgorithmException e) {
             throw new DataException("SHA-256 hashing algorithm unavailable");
         }
-    
+
         for (Path path : allPaths) {
             // We need to work with paths relative to the base path, to ensure the same hash
             // is generated on different systems
             Path relativePath = basePathAbsolute.relativize(path.toAbsolutePath());
-    
+
             // Exclude Qortal folder since it can be different each time
             // We only care about hashing the actual user data
             if (relativePath.startsWith(".qortal/")) {
                 continue;
             }
-    
-            // Account for \ VS / : Linux VS Windows
-            String pathString = relativePath.toString();
-            if (relativePath.getFileSystem().toString().contains("Windows")) {
-                pathString = pathString.replace("\\", "/");
-            }
-    
+
             // Hash path
-            byte[] filePathBytes = pathString.getBytes(StandardCharsets.UTF_8);
+            byte[] filePathBytes = relativePath.toString().getBytes(StandardCharsets.UTF_8);
             sha256.update(filePathBytes);
-    
-            try (InputStream in = Files.newInputStream(path)) {
-                byte[] buffer = new byte[65536]; // 64 KB
-                int bytesRead;
-                while ((bytesRead = in.read(buffer)) != -1) {
-                    sha256.update(buffer, 0, bytesRead);
-                }
-            }
+
+            // Hash contents
+            byte[] fileContent = Files.readAllBytes(path);
+            sha256.update(fileContent);
         }
-    
         this.hash = sha256.digest();
     }
-    
 
     public boolean isHashValid(byte[] hash) {
         return Arrays.equals(hash, this.hash);

src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java

@@ -52,7 +52,7 @@ public class ArbitraryDataFile {
 
     private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class);
 
-    public static final long MAX_FILE_SIZE = 2L * 1024 * 1024 * 1024; // 2 GiB
+    public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB
     protected static final int MAX_CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB
     public static final int CHUNK_SIZE = 512 * 1024; // 0.5MiB
     public static int SHORT_DIGEST_LENGTH = 8;

src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java

@@ -1,7 +1,6 @@
 package org.qortal.arbitrary;
 
 import com.google.common.io.Resources;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.logging.log4j.LogManager;
@@ -16,13 +15,11 @@ import org.qortal.settings.Settings;
 import javax.servlet.ServletContext;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
-
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.URL;
-
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
@@ -40,7 +37,6 @@ public class ArbitraryDataRenderer {
     private final Service service;
     private final String identifier;
     private String theme = "light";
-    private String lang = "en"; 
     private String inPath;
     private final String secret58;
     private final String prefix;
@@ -170,16 +166,9 @@ public class ArbitraryDataRenderer {
             if (HTMLParser.isHtmlFile(filename)) {
                 // HTML file - needs to be parsed
                 byte[] data = Files.readAllBytes(filePath); // TODO: limit file size that can be read into memory
-                String encodedResourceId;
-
-                if (resourceIdType == ResourceIdType.NAME) {
-                    encodedResourceId = resourceId.replace(" ", "%20");
-                } else {
-                    encodedResourceId = resourceId;
-                }
-                HTMLParser htmlParser = new HTMLParser(encodedResourceId, inPath, prefix, includeResourceIdInPrefix, data, qdnContext, service, identifier, theme, usingCustomRouting, lang);
+                HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, includeResourceIdInPrefix, data, qdnContext, service, identifier, theme, usingCustomRouting);
                 htmlParser.addAdditionalHeaderTags();
-               response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' wss: blob:;");
+                response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' wss:;");
                 response.setContentType(context.getMimeType(filename));
                 response.setContentLength(htmlParser.getData().length);
                 response.getOutputStream().write(htmlParser.getData());
@@ -267,8 +256,5 @@ public class ArbitraryDataRenderer {
     public void setTheme(String theme) {
         this.theme = theme;
     }
-    public void setLang(String lang) {
-        this.lang = lang;
-    }    
 
 }

src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java

@@ -29,7 +29,6 @@ import org.qortal.utils.FilesystemUtils;
 import org.qortal.utils.NTP;
 
 import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
@@ -198,7 +197,7 @@ public class ArbitraryDataTransactionBuilder {
 
             // We can't use PATCH for on-chain data because this requires the .qortal directory, which can't be put on chain
             final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(this.path, false);
-            final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE);
+            final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE);
             if (shouldUseOnChainData) {
                 LOGGER.info("Data size is small enough to go on chain - using PUT");
                 return Method.PUT;
@@ -246,7 +245,7 @@ public class ArbitraryDataTransactionBuilder {
 
             // Single file resources are handled differently, especially for very small data payloads, as these go on chain
             final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(path, false);
-            final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE);
+            final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE);
 
             // Use zip compression if data isn't going on chain
             Compression compression = shouldUseOnChainData ? Compression.NONE : Compression.ZIP;

src/main/java/org/qortal/arbitrary/misc/Service.java

@@ -37,7 +37,7 @@ public enum Service {
             if (files != null && files[0] != null) {
                 final String extension = FilenameUtils.getExtension(files[0].getName()).toLowerCase();
                 // We must allow blank file extensions because these are used by data published from a plaintext or base64-encoded string
-                final List<String> allowedExtensions = Arrays.asList("qortal", "zip", "pdf", "txt", "odt", "ods", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "");
+                final List<String> allowedExtensions = Arrays.asList("zip", "pdf", "txt", "odt", "ods", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "");
                 if (extension == null || !allowedExtensions.contains(extension)) {
                     return ValidationResult.INVALID_FILE_EXTENSION;
                 }
@@ -62,17 +62,7 @@ public enum Service {
 
             // Custom validation function to require an index HTML file in the root directory
             List<String> fileNames = ArbitraryDataRenderer.indexFiles();
-            List<String> files;
-
-            // single files are paackaged differently
-            if( path.toFile().isFile() ) {
-                files = new ArrayList<>(1);
-                files.add(path.getFileName().toString());
-            }
-            else {
-                files = new ArrayList<>(Arrays.asList(path.toFile().list()));
-            }
-
+            String[] files = path.toFile().list();
             if (files != null) {
                 for (String file : files) {
                     Path fileName = Paths.get(file).getFileName();

src/main/java/org/qortal/block/Block.java

@@ -1640,8 +1640,6 @@ public class Block {
 					SelfSponsorshipAlgoV2Block.processAccountPenalties(this);
 				} else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV3Height()) {
 					SelfSponsorshipAlgoV3Block.processAccountPenalties(this);
-				} else if (this.blockData.getHeight() == BlockChain.getInstance().getMultipleNamesPerAccountHeight()) {
-					PrimaryNamesBlock.processNames(this.repository);
 				}
 			}
 		}
@@ -1723,19 +1721,11 @@ public class Block {
 			accountData.setBlocksMinted(accountData.getBlocksMinted() + 1);
 			LOGGER.trace(() -> String.format("Block minter %s up to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : "")));
 
-			int blocksMintedAdjustment
-				=
-				(this.blockData.getHeight() > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight())
-				?
-				0
-				:
-				accountData.getBlocksMintedAdjustment();
-
-			final int effectiveBlocksMinted = accountData.getBlocksMinted() + blocksMintedAdjustment + accountData.getBlocksMintedPenalty();
+			final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment() + accountData.getBlocksMintedPenalty();
 
 			for (int newLevel = maximumLevel; newLevel >= 0; --newLevel)
 				if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) {
-					if (newLevel != accountData.getLevel()) {
+					if (newLevel > accountData.getLevel()) {
 						// Account has increased in level!
 						accountData.setLevel(newLevel);
 						bumpedAccounts.put(accountData.getAddress(), newLevel);
@@ -1962,8 +1952,6 @@ public class Block {
 					SelfSponsorshipAlgoV2Block.orphanAccountPenalties(this);
 				} else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV3Height()) {
 					SelfSponsorshipAlgoV3Block.orphanAccountPenalties(this);
-				} else if (this.blockData.getHeight() == BlockChain.getInstance().getMultipleNamesPerAccountHeight()) {
-					PrimaryNamesBlock.orphanNames( this.repository );
 				}
 			}
 
@@ -2139,19 +2127,11 @@ public class Block {
 			accountData.setBlocksMinted(accountData.getBlocksMinted() - 1);
 			LOGGER.trace(() -> String.format("Block minter %s down to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : "")));
 
-			int blocksMintedAdjustment
-				=
-				(this.blockData.getHeight() -1 > BlockChain.getInstance().getMintedBlocksAdjustmentRemovalHeight())
-				?
-				0
-				:
-				accountData.getBlocksMintedAdjustment();
-
-			final int effectiveBlocksMinted = accountData.getBlocksMinted() + blocksMintedAdjustment + accountData.getBlocksMintedPenalty();
+			final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment() + accountData.getBlocksMintedPenalty();
 
 			for (int newLevel = maximumLevel; newLevel >= 0; --newLevel)
 				if (effectiveBlocksMinted >= cumulativeBlocksByLevel.get(newLevel)) {
-					if (newLevel != accountData.getLevel()) {
+					if (newLevel < accountData.getLevel()) {
 						// Account has decreased in level!
 						accountData.setLevel(newLevel);
 						repository.getAccountRepository().setLevel(accountData);

src/main/java/org/qortal/block/BlockChain.java

@@ -92,9 +92,7 @@ public class BlockChain {
 		adminsReplaceFoundersHeight,
 		nullGroupMembershipHeight,
 		ignoreLevelForRewardShareHeight,
-		adminQueryFixHeight,
-		multipleNamesPerAccountHeight,
-		mintedBlocksAdjustmentRemovalHeight
+		adminQueryFixHeight
 	}
 
 	// Custom transaction fees
@@ -114,8 +112,7 @@ public class BlockChain {
 	/** Whether to use legacy, broken RIPEMD160 implementation when converting public keys to addresses. */
 	private boolean useBrokenMD160ForAddresses = false;
 
-	/** This should get ignored and overwritten in the oneNamePerAccount(int blockchainHeight) method,
-	 * because it is based on block height, not based on the genesis block.*/
+	/** Whether only one registered name is allowed per account. */
 	private boolean oneNamePerAccount = false;
 
 	/** Checkpoints */
@@ -477,9 +474,8 @@ public class BlockChain {
 		return this.useBrokenMD160ForAddresses;
 	}
 
-	public boolean oneNamePerAccount(int blockchainHeight) {
-		// this is not set on a simple blockchain setting, it is based on a feature trigger height
-		return blockchainHeight < this.getMultipleNamesPerAccountHeight();
+	public boolean oneNamePerAccount() {
+		return this.oneNamePerAccount;
 	}
 
 	public List<Checkpoint> getCheckpoints() {
@@ -692,14 +688,6 @@ public class BlockChain {
 		return this.featureTriggers.get(FeatureTrigger.adminQueryFixHeight.name()).intValue();
 	}
 
-	public int getMultipleNamesPerAccountHeight() {
-		return this.featureTriggers.get(FeatureTrigger.multipleNamesPerAccountHeight.name()).intValue();
-	}
-
-	public int getMintedBlocksAdjustmentRemovalHeight() {
-		return this.featureTriggers.get(FeatureTrigger.mintedBlocksAdjustmentRemovalHeight.name()).intValue();
-	}
-
 	// More complex getters for aspects that change by height or timestamp
 
 	public long getRewardAtHeight(int ourHeight) {

src/main/java/org/qortal/block/PrimaryNamesBlock.java

@@ -1,47 +0,0 @@
-package org.qortal.block;
-
-import org.qortal.account.Account;
-import org.qortal.api.resource.TransactionsResource;
-import org.qortal.data.naming.NameData;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * Class PrimaryNamesBlock
- */
-public class PrimaryNamesBlock {
-
-    /**
-     * Process Primary Names
-     *
-     * @param repository
-     * @throws DataException
-     */
-    public static void processNames(Repository repository) throws DataException {
-
-        Set<String> addressesWithNames
-            = repository.getNameRepository().getAllNames().stream()
-                .map(NameData::getOwner).collect(Collectors.toSet());
-
-        // for each address with a name, set primary name to the address
-        for( String address : addressesWithNames ) {
-
-            Account account = new Account(repository, address);
-            account.resetPrimaryName(TransactionsResource.ConfirmationStatus.CONFIRMED);
-        }
-    }
-
-    /**
-     * Orphan the Primary Names Block
-     *
-     * @param repository
-     * @throws DataException
-     */
-    public static void orphanNames(Repository repository) throws DataException {
-
-        repository.getNameRepository().clearPrimaryNames();
-    }
-}

src/main/java/org/qortal/controller/RNSTransactionImporter.java

@@ -0,0 +1,460 @@
+package org.qortal.controller;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.data.block.BlockData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.GetTransactionMessage;
+import org.qortal.network.message.Message;
+import org.qortal.network.message.TransactionMessage;
+import org.qortal.network.message.TransactionSignaturesMessage;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.transaction.Transaction;
+import org.qortal.transform.TransformationException;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+
+import java.util.*;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+
+public class RNSTransactionImporter extends Thread {
+
+    private static final Logger LOGGER = LogManager.getLogger(RNSTransactionImporter.class);
+
+    private static RNSTransactionImporter instance;
+    private volatile boolean isStopping = false;
+
+    private static final int MAX_INCOMING_TRANSACTIONS = 5000;
+
+    /** Minimum time before considering an invalid unconfirmed transaction as "stale" */
+    public static final long INVALID_TRANSACTION_STALE_TIMEOUT = 30 * 60 * 1000L; // ms
+    /** Minimum frequency to re-request stale unconfirmed transactions from peers, to recheck validity */
+    public static final long INVALID_TRANSACTION_RECHECK_INTERVAL = 60 * 60 * 1000L; // ms\
+    /** Minimum frequency to re-request expired unconfirmed transactions from peers, to recheck validity
+     * This mainly exists to stop expired transactions from bloating the list */
+    public static final long EXPIRED_TRANSACTION_RECHECK_INTERVAL = 10 * 60 * 1000L; // ms
+
+
+    /** Map of incoming transaction that are in the import queue. Key is transaction data, value is whether signature has been validated. */
+    private final Map<TransactionData, Boolean> incomingTransactions = Collections.synchronizedMap(new HashMap<>());
+
+    /** Map of recent invalid unconfirmed transactions. Key is base58 transaction signature, value is do-not-request expiry timestamp. */
+    private final Map<String, Long> invalidUnconfirmedTransactions = Collections.synchronizedMap(new HashMap<>());
+
+    /** Cached list of unconfirmed transactions, used when counting per creator. This is replaced regularly */
+    public static List<TransactionData> unconfirmedTransactionsCache = null;
+
+
+    public static synchronized RNSTransactionImporter getInstance() {
+        if (instance == null) {
+            instance = new RNSTransactionImporter();
+        }
+
+        return instance;
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Transaction Importer");
+
+        try {
+            while (!Controller.isStopping()) {
+                Thread.sleep(500L);
+
+                // Process incoming transactions queue
+                validateTransactionsInQueue();
+                importTransactionsInQueue();
+
+                // Clean up invalid incoming transactions list
+                cleanupInvalidTransactionsList(NTP.getTime());
+            }
+        } catch (InterruptedException e) {
+            // Fall through to exit thread
+        }
+    }
+
+    public void shutdown() {
+        isStopping = true;
+        this.interrupt();
+    }
+
+
+    // Incoming transactions queue
+
+    private boolean incomingTransactionQueueContains(byte[] signature) {
+        synchronized (incomingTransactions) {
+            return incomingTransactions.keySet().stream().anyMatch(t -> Arrays.equals(t.getSignature(), signature));
+        }
+    }
+
+    private void removeIncomingTransaction(byte[] signature) {
+        incomingTransactions.keySet().removeIf(t -> Arrays.equals(t.getSignature(), signature));
+    }
+
+    /**
+     * Retrieve all pending unconfirmed transactions that have had their signatures validated.
+     * @return a list of TransactionData objects, with valid signatures.
+     */
+    private List<TransactionData> getCachedSigValidTransactions() {
+        synchronized (this.incomingTransactions) {
+            return this.incomingTransactions.entrySet().stream()
+                    .filter(t -> Boolean.TRUE.equals(t.getValue()))
+                    .map(Map.Entry::getKey)
+                    .collect(Collectors.toList());
+        }
+    }
+
+    /**
+     * Validate the signatures of any transactions pending import, then update their
+     * entries in the queue to mark them as valid/invalid.
+     *
+     * No database lock is required.
+     */
+    private void validateTransactionsInQueue() {
+        if (this.incomingTransactions.isEmpty()) {
+            // Nothing to do?
+            return;
+        }
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            // Take a snapshot of incomingTransactions, so we don't need to lock it while processing
+            Map<TransactionData, Boolean> incomingTransactionsCopy = Map.copyOf(this.incomingTransactions);
+
+            int unvalidatedCount = Collections.frequency(incomingTransactionsCopy.values(), Boolean.FALSE);
+            int validatedCount = 0;
+
+            if (unvalidatedCount > 0) {
+                LOGGER.debug("Validating signatures in incoming transactions queue (size {})...", unvalidatedCount);
+            }
+
+            // A list of all currently pending transactions that have valid signatures
+            List<Transaction> sigValidTransactions = new ArrayList<>();
+
+            // A list of signatures that became valid in this round
+            List<byte[]> newlyValidSignatures = new ArrayList<>();
+
+            boolean isLiteNode = Settings.getInstance().isLite();
+
+            // We need the latest block in order to check for expired transactions
+            BlockData latestBlock = Controller.getInstance().getChainTip();
+
+            // Signature validation round - does not require blockchain lock
+            for (Map.Entry<TransactionData, Boolean> transactionEntry : incomingTransactionsCopy.entrySet()) {
+                // Quick exit?
+                if (isStopping) {
+                    return;
+                }
+
+                TransactionData transactionData = transactionEntry.getKey();
+                Transaction transaction = Transaction.fromData(repository, transactionData);
+                String signature58 = Base58.encode(transactionData.getSignature());
+
+                Long now = NTP.getTime();
+                if (now == null) {
+                    return;
+                }
+
+                // Drop expired transactions before they are considered "sig valid"
+                if (latestBlock != null && transaction.getDeadline() <= latestBlock.getTimestamp()) {
+                    LOGGER.debug("Removing expired {} transaction {} from import queue", transactionData.getType().name(), signature58);
+                    removeIncomingTransaction(transactionData.getSignature());
+                    invalidUnconfirmedTransactions.put(signature58, (now + EXPIRED_TRANSACTION_RECHECK_INTERVAL));
+                    continue;
+                }
+
+                // Only validate signature if we haven't already done so
+                Boolean isSigValid = transactionEntry.getValue();
+                if (!Boolean.TRUE.equals(isSigValid)) {
+                    if (isLiteNode) {
+                        // Lite nodes can't easily validate transactions, so for now we will have to assume that everything is valid
+                        sigValidTransactions.add(transaction);
+                        newlyValidSignatures.add(transactionData.getSignature());
+                        // Add mark signature as valid if transaction still exists in import queue
+                        incomingTransactions.computeIfPresent(transactionData, (k, v) -> Boolean.TRUE);
+                        continue;
+                    }
+
+                    if (!transaction.isSignatureValid()) {
+                        LOGGER.debug("Ignoring {} transaction {} with invalid signature", transactionData.getType().name(), signature58);
+                        removeIncomingTransaction(transactionData.getSignature());
+
+                        // Also add to invalidIncomingTransactions map
+                        now = NTP.getTime();
+                        if (now != null) {
+                            Long expiry = now + INVALID_TRANSACTION_RECHECK_INTERVAL;
+                            LOGGER.trace("Adding invalid transaction {} to invalidUnconfirmedTransactions...", signature58);
+                            // Add to invalidUnconfirmedTransactions so that we don't keep requesting it
+                            invalidUnconfirmedTransactions.put(signature58, expiry);
+                        }
+
+                        // We're done with this transaction
+                        continue;
+                    }
+
+                    // Count the number that were validated in this round, for logging purposes
+                    validatedCount++;
+
+                    // Add mark signature as valid if transaction still exists in import queue
+                    incomingTransactions.computeIfPresent(transactionData, (k, v) -> Boolean.TRUE);
+
+                    // Signature validated in this round
+                    newlyValidSignatures.add(transactionData.getSignature());
+
+                } else {
+                    LOGGER.trace(() -> String.format("Transaction %s known to have valid signature", Base58.encode(transactionData.getSignature())));
+                }
+
+                // Signature valid - add to shortlist
+                sigValidTransactions.add(transaction);
+            }
+
+            if (unvalidatedCount > 0) {
+                LOGGER.debug("Finished validating signatures in incoming transactions queue (valid this round: {}, total pending import: {})...", validatedCount, sigValidTransactions.size());
+            }
+
+        } catch (DataException e) {
+            LOGGER.error("Repository issue while processing incoming transactions", e);
+        }
+    }
+
+    /**
+     * Import any transactions in the queue that have valid signatures.
+     *
+     * A database lock is required.
+     */
+    private void importTransactionsInQueue() {
+        List<TransactionData> sigValidTransactions = this.getCachedSigValidTransactions();
+        if (sigValidTransactions.isEmpty()) {
+            // Don't bother locking if there are no new transactions to process
+            return;
+        }
+
+        if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) {
+            // Prioritize syncing, and don't attempt to lock
+            return;
+        }
+
+        ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+        if (!blockchainLock.tryLock()) {
+            LOGGER.debug("Too busy to import incoming transactions queue");
+            return;
+        }
+
+        LOGGER.debug("Importing incoming transactions queue (size {})...", sigValidTransactions.size());
+
+        int processedCount = 0;
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Use a single copy of the unconfirmed transactions list for each cycle, to speed up constant lookups
+            // when counting unconfirmed transactions by creator.
+            List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions();
+            unconfirmedTransactions.removeIf(t -> t.getType() == Transaction.TransactionType.CHAT);
+            unconfirmedTransactionsCache = unconfirmedTransactions;
+
+            // A list of signatures were imported in this round
+            List<byte[]> newlyImportedSignatures = new ArrayList<>();
+
+            // Import transactions with valid signatures
+            try {
+                for (int i = 0; i < sigValidTransactions.size(); ++i) {
+                    if (isStopping) {
+                        return;
+                    }
+
+                    if (Synchronizer.getInstance().isSyncRequestPending()) {
+                        LOGGER.debug("Breaking out of transaction importing with {} remaining, because a sync request is pending", sigValidTransactions.size() - i);
+                        return;
+                    }
+
+                    TransactionData transactionData = sigValidTransactions.get(i);
+                    Transaction transaction = Transaction.fromData(repository, transactionData);
+
+                    Transaction.ValidationResult validationResult = transaction.importAsUnconfirmed();
+                    processedCount++;
+
+                    switch (validationResult) {
+                        case TRANSACTION_ALREADY_EXISTS: {
+                            LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature())));
+                            break;
+                        }
+
+                        case NO_BLOCKCHAIN_LOCK: {
+                            // Is this even possible considering we acquired blockchain lock above?
+                            LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction %s", Base58.encode(transactionData.getSignature())));
+                            break;
+                        }
+
+                        case OK: {
+                            LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature())));
+
+                            // Add to the unconfirmed transactions cache
+                            if (transactionData.getType() != Transaction.TransactionType.CHAT && unconfirmedTransactionsCache != null) {
+                                unconfirmedTransactionsCache.add(transactionData);
+                            }
+
+                            // Signature imported in this round
+                            newlyImportedSignatures.add(transactionData.getSignature());
+
+                            break;
+                        }
+
+                        // All other invalid cases:
+                        default: {
+                            final String signature58 = Base58.encode(transactionData.getSignature());
+                            LOGGER.debug(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58));
+
+                            Long now = NTP.getTime();
+                            if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) {
+                                Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL;
+
+                                if (validationResult == Transaction.ValidationResult.TIMESTAMP_TOO_OLD) {
+                                    // Use shorter recheck interval for expired transactions
+                                    expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL;
+                                }
+
+                                Long expiry = now + expiryLength;
+                                LOGGER.trace("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58);
+                                // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it
+                                invalidUnconfirmedTransactions.put(signature58, expiry);
+                            }
+                        }
+                    }
+
+                    // Transaction has been processed, even if only to reject it
+                    removeIncomingTransaction(transactionData.getSignature());
+                }
+
+                if (!newlyImportedSignatures.isEmpty()) {
+                    LOGGER.debug("Broadcasting {} newly imported signatures", newlyImportedSignatures.size());
+                    Message newTransactionSignatureMessage = new TransactionSignaturesMessage(newlyImportedSignatures);
+                    RNSNetwork.getInstance().broadcast(broadcastPeer -> newTransactionSignatureMessage);
+                }
+            } finally {
+                LOGGER.debug("Finished importing {} incoming transaction{}", processedCount, (processedCount == 1 ? "" : "s"));
+                blockchainLock.unlock();
+
+                // Clear the unconfirmed transaction cache so new data can be populated in the next cycle
+                unconfirmedTransactionsCache = null;
+            }
+        } catch (DataException e) {
+            LOGGER.error("Repository issue while importing incoming transactions", e);
+        }
+    }
+
+    private void cleanupInvalidTransactionsList(Long now) {
+        if (now == null) {
+            return;
+        }
+        // Periodically remove invalid unconfirmed transactions from the list, so that they can be fetched again
+        invalidUnconfirmedTransactions.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < now);
+    }
+
+
+    // Network handlers
+
+    public void onNetworkTransactionMessage(RNSPeer peer, Message message) {
+        TransactionMessage transactionMessage = (TransactionMessage) message;
+        TransactionData transactionData = transactionMessage.getTransactionData();
+
+        if (this.incomingTransactions.size() < MAX_INCOMING_TRANSACTIONS) {
+            synchronized (this.incomingTransactions) {
+                if (!incomingTransactionQueueContains(transactionData.getSignature())) {
+                    this.incomingTransactions.put(transactionData, Boolean.FALSE);
+                }
+            }
+        }
+    }
+
+    public void onNetworkGetTransactionMessage(RNSPeer peer, Message message) {
+        GetTransactionMessage getTransactionMessage = (GetTransactionMessage) message;
+        byte[] signature = getTransactionMessage.getSignature();
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            // Firstly check the sig-valid transactions that are currently queued for import
+            TransactionData transactionData = this.getCachedSigValidTransactions().stream()
+                    .filter(t -> Arrays.equals(signature, t.getSignature()))
+                    .findFirst().orElse(null);
+
+            if (transactionData == null) {
+                // Not found in import queue, so try the database
+                transactionData = repository.getTransactionRepository().fromSignature(signature);
+            }
+
+            if (transactionData == null) {
+                // Still not found - so we don't have this transaction
+                LOGGER.debug(() -> String.format("Ignoring GET_TRANSACTION request from peer %s for unknown transaction %s", peer, Base58.encode(signature)));
+                // Send no response at all???
+                return;
+            }
+
+            Message transactionMessage = new TransactionMessage(transactionData);
+            transactionMessage.setId(message.getId());
+            peer.sendMessage(transactionMessage);
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e);
+        } catch (TransformationException e) {
+            LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e);
+        }
+    }
+
+    public void onNetworkGetUnconfirmedTransactionsMessage(RNSPeer peer, Message message) {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            List<byte[]> signatures = Collections.emptyList();
+
+            // If we're NOT up-to-date then don't send out unconfirmed transactions
+            // as it's possible they are already included in a later block that we don't have.
+            if (Controller.getInstance().isUpToDate())
+                signatures = repository.getTransactionRepository().getUnconfirmedTransactionSignatures();
+
+            Message transactionSignaturesMessage = new TransactionSignaturesMessage(signatures);
+            peer.sendMessage(transactionSignaturesMessage);
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while sending unconfirmed transaction signatures to peer %s", peer), e);
+        }
+    }
+
+    public void onNetworkTransactionSignaturesMessage(RNSPeer peer, Message message) {
+        TransactionSignaturesMessage transactionSignaturesMessage = (TransactionSignaturesMessage) message;
+        List<byte[]> signatures = transactionSignaturesMessage.getSignatures();
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            for (byte[] signature : signatures) {
+                String signature58 = Base58.encode(signature);
+                if (invalidUnconfirmedTransactions.containsKey(signature58)) {
+                    // Previously invalid transaction - don't keep requesting it
+                    // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks
+                    continue;
+                }
+
+                // Ignore if this transaction is in the queue
+                if (incomingTransactionQueueContains(signature)) {
+                    LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peer));
+                    continue;
+                }
+
+                // Do we have it already? (Before requesting transaction data itself)
+                if (repository.getTransactionRepository().exists(signature)) {
+                    LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer));
+                    continue;
+                }
+
+                // Check isInterrupted() here and exit fast
+                if (Thread.currentThread().isInterrupted())
+                    return;
+
+                // Fetch actual transaction data from peer
+                Message getTransactionMessage = new GetTransactionMessage(signature);
+                peer.sendMessage(getTransactionMessage);
+            }
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while processing unconfirmed transactions from peer %s", peer), e);
+        }
+    }
+
+}

src/main/java/org/qortal/controller/TransactionImporter.java

@@ -2,7 +2,6 @@ package org.qortal.controller;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.qortal.controller.arbitrary.PeerMessage;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.network.Network;
@@ -21,11 +20,7 @@ import org.qortal.utils.Base58;
 import org.qortal.utils.NTP;
 
 import java.util.*;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 
 public class TransactionImporter extends Thread {
@@ -55,10 +50,6 @@ public class TransactionImporter extends Thread {
     /** Cached list of unconfirmed transactions, used when counting per creator. This is replaced regularly */
     public static List<TransactionData> unconfirmedTransactionsCache = null;
 
-    public TransactionImporter() {
-        signatureMessageScheduler.scheduleAtFixedRate(this::processNetworkTransactionSignaturesMessage, 60, 1, TimeUnit.SECONDS);
-        getTransactionMessageScheduler.scheduleAtFixedRate(this::processNetworkGetTransactionMessages, 60, 1, TimeUnit.SECONDS);
-    }
 
     public static synchronized TransactionImporter getInstance() {
         if (instance == null) {
@@ -380,104 +371,36 @@ public class TransactionImporter extends Thread {
         }
     }
 
-    // List to collect messages
-    private final List<PeerMessage> getTransactionMessageList = new ArrayList<>();
-    // Lock to synchronize access to the list
-    private final Object getTransactionMessageLock = new Object();
-
-    // Scheduled executor service to process messages every second
-    private final ScheduledExecutorService getTransactionMessageScheduler = Executors.newScheduledThreadPool(1);
-
     public void onNetworkGetTransactionMessage(Peer peer, Message message) {
+        GetTransactionMessage getTransactionMessage = (GetTransactionMessage) message;
+        byte[] signature = getTransactionMessage.getSignature();
 
-        synchronized (getTransactionMessageLock) {
-            getTransactionMessageList.add(new PeerMessage(peer, message));
-        }
-    }
-
-    private void processNetworkGetTransactionMessages() {
-
-        try {
-            List<PeerMessage> messagesToProcess;
-            synchronized (getTransactionMessageLock) {
-                messagesToProcess = new ArrayList<>(getTransactionMessageList);
-                getTransactionMessageList.clear();
-            }
-
-            if( messagesToProcess.isEmpty() ) return;
-
-            Map<String, PeerMessage> peerMessageBySignature58 = new HashMap<>(messagesToProcess.size());
-
-            for( PeerMessage peerMessage : messagesToProcess ) {
-                GetTransactionMessage getTransactionMessage = (GetTransactionMessage) peerMessage.getMessage();
-                byte[] signature = getTransactionMessage.getSignature();
-
-                peerMessageBySignature58.put(Base58.encode(signature), peerMessage);
-            }
-
+        try (final Repository repository = RepositoryManager.getRepository()) {
             // Firstly check the sig-valid transactions that are currently queued for import
-            Map<String, TransactionData> transactionsCachedBySignature58
-                = this.getCachedSigValidTransactions().stream()
-                    .collect(Collectors.toMap(t -> Base58.encode(t.getSignature()), Function.identity()));
+            TransactionData transactionData = this.getCachedSigValidTransactions().stream()
+                    .filter(t -> Arrays.equals(signature, t.getSignature()))
+                    .findFirst().orElse(null);
 
-            Map<Boolean, List<Map.Entry<String, PeerMessage>>> transactionsCachedBySignature58Partition
-                = peerMessageBySignature58.entrySet().stream()
-                    .collect(Collectors.partitioningBy(entry -> transactionsCachedBySignature58.containsKey(entry.getKey())));
-
-            List<byte[]> signaturesNeeded
-                = transactionsCachedBySignature58Partition.get(false).stream()
-                    .map(Map.Entry::getValue)
-                    .map(PeerMessage::getMessage)
-                    .map(message -> (GetTransactionMessage) message)
-                    .map(GetTransactionMessage::getSignature)
-                    .collect(Collectors.toList());
-
-            // transaction found in the import queue
-            Map<String, TransactionData> transactionsToSendBySignature58 = new HashMap<>(messagesToProcess.size());
-            for( Map.Entry<String, PeerMessage> entry : transactionsCachedBySignature58Partition.get(true)) {
-                transactionsToSendBySignature58.put(entry.getKey(), transactionsCachedBySignature58.get(entry.getKey()));
-            }
-
-            if( !signaturesNeeded.isEmpty() ) {
+            if (transactionData == null) {
                 // Not found in import queue, so try the database
-                try (final Repository repository = RepositoryManager.getRepository()) {
-                    transactionsToSendBySignature58.putAll(
-                            repository.getTransactionRepository().fromSignatures(signaturesNeeded).stream()
-                                    .collect(Collectors.toMap(transactionData -> Base58.encode(transactionData.getSignature()), Function.identity()))
-                    );
-                } catch (DataException e) {
-                    LOGGER.error(e.getMessage(), e);
-                }
+                transactionData = repository.getTransactionRepository().fromSignature(signature);
             }
 
-            for( final Map.Entry<String, TransactionData> entry : transactionsToSendBySignature58.entrySet() ) {
-
-                PeerMessage peerMessage = peerMessageBySignature58.get(entry.getKey());
-                final Message message = peerMessage.getMessage();
-                final Peer peer = peerMessage.getPeer();
-
-                Runnable sendTransactionMessageRunner = () -> sendTransactionMessage(entry.getKey(), entry.getValue(), message, peer);
-                Thread sendTransactionMessageThread = new Thread(sendTransactionMessageRunner);
-                sendTransactionMessageThread.start();
+            if (transactionData == null) {
+                // Still not found - so we don't have this transaction
+                LOGGER.debug(() -> String.format("Ignoring GET_TRANSACTION request from peer %s for unknown transaction %s", peer, Base58.encode(signature)));
+                // Send no response at all???
+                return;
             }
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(),e);
-        }
-    }
 
-    private static void sendTransactionMessage(String signature58, TransactionData data, Message message, Peer peer) {
-        try {
-            Message transactionMessage = new TransactionMessage(data);
+            Message transactionMessage = new TransactionMessage(transactionData);
             transactionMessage.setId(message.getId());
-
             if (!peer.sendMessage(transactionMessage))
                 peer.disconnect("failed to send transaction");
-        }
-        catch (TransformationException e) {
-            LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", signature58, peer), e);
-        }
-        catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e);
+        } catch (TransformationException e) {
+            LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e);
         }
     }
 
@@ -498,86 +421,44 @@ public class TransactionImporter extends Thread {
         }
     }
 
-    // List to collect messages
-    private final List<PeerMessage> signatureMessageList = new ArrayList<>();
-    // Lock to synchronize access to the list
-    private final Object signatureMessageLock = new Object();
-
-    // Scheduled executor service to process messages every second
-    private final ScheduledExecutorService signatureMessageScheduler = Executors.newScheduledThreadPool(1);
-
     public void onNetworkTransactionSignaturesMessage(Peer peer, Message message) {
-        synchronized (signatureMessageLock) {
-            signatureMessageList.add(new PeerMessage(peer, message));
-        }
-    }
+        TransactionSignaturesMessage transactionSignaturesMessage = (TransactionSignaturesMessage) message;
+        List<byte[]> signatures = transactionSignaturesMessage.getSignatures();
 
-    public void processNetworkTransactionSignaturesMessage() {
-
-        try {
-            List<PeerMessage> messagesToProcess;
-            synchronized (signatureMessageLock) {
-                messagesToProcess = new ArrayList<>(signatureMessageList);
-                signatureMessageList.clear();
-            }
-
-            Map<String, byte[]> signatureBySignature58 = new HashMap<>(messagesToProcess.size() * 10);
-            Map<String, Peer> peerBySignature58 = new HashMap<>( messagesToProcess.size() * 10 );
-
-            for( PeerMessage peerMessage : messagesToProcess ) {
-
-                TransactionSignaturesMessage transactionSignaturesMessage = (TransactionSignaturesMessage) peerMessage.getMessage();
-                List<byte[]> signatures = transactionSignaturesMessage.getSignatures();
-
-                for (byte[] signature : signatures) {
-                    String signature58 = Base58.encode(signature);
-                    if (invalidUnconfirmedTransactions.containsKey(signature58)) {
-                        // Previously invalid transaction - don't keep requesting it
-                        // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks
-                        continue;
-                    }
-
-                    // Ignore if this transaction is in the queue
-                    if (incomingTransactionQueueContains(signature)) {
-                        LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peerMessage.getPeer()));
-                        continue;
-                    }
-
-                    signatureBySignature58.put(signature58, signature);
-                    peerBySignature58.put(signature58, peerMessage.getPeer());
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            for (byte[] signature : signatures) {
+                String signature58 = Base58.encode(signature);
+                if (invalidUnconfirmedTransactions.containsKey(signature58)) {
+                    // Previously invalid transaction - don't keep requesting it
+                    // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks
+                    continue;
                 }
-            }
 
-            if( !signatureBySignature58.isEmpty() ) {
-                try (final Repository repository = RepositoryManager.getRepository()) {
-
-                    // remove signatures in db already
-                    repository.getTransactionRepository()
-                            .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream()
-                            .map(TransactionData::getSignature)
-                            .map(signature -> Base58.encode(signature))
-                            .forEach(signature58 -> signatureBySignature58.remove(signature58));
-                } catch (DataException e) {
-                    LOGGER.error(String.format("Repository issue while processing unconfirmed transactions from peer"), e);
+                // Ignore if this transaction is in the queue
+                if (incomingTransactionQueueContains(signature)) {
+                    LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peer));
+                    continue;
                 }
-            }
 
-            // Check isInterrupted() here and exit fast
-            if (Thread.currentThread().isInterrupted())
-                return;
+                // Do we have it already? (Before requesting transaction data itself)
+                if (repository.getTransactionRepository().exists(signature)) {
+                    LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer));
+                    continue;
+                }
 
-            for (Map.Entry<String, byte[]> entry : signatureBySignature58.entrySet()) {
-
-                Peer peer = peerBySignature58.get(entry.getKey());
+                // Check isInterrupted() here and exit fast
+                if (Thread.currentThread().isInterrupted())
+                    return;
 
                 // Fetch actual transaction data from peer
-                Message getTransactionMessage = new GetTransactionMessage(entry.getValue());
-                if (peer != null && !peer.sendMessage(getTransactionMessage)) {
+                Message getTransactionMessage = new GetTransactionMessage(signature);
+                if (!peer.sendMessage(getTransactionMessage)) {
                     peer.disconnect("failed to request transaction");
+                    return;
                 }
             }
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while processing unconfirmed transactions from peer %s", peer), e);
         }
     }
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java

@@ -25,10 +25,6 @@ import org.qortal.utils.NTP;
 import org.qortal.utils.Triple;
 
 import java.util.*;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
 
 import static org.qortal.controller.arbitrary.ArbitraryDataFileManager.MAX_FILE_HASH_RESPONSES;
 
@@ -77,8 +73,6 @@ public class ArbitraryDataFileListManager {
 
 
     private ArbitraryDataFileListManager() {
-        getArbitraryDataFileListMessageScheduler.scheduleAtFixedRate(this::processNetworkGetArbitraryDataFileListMessage, 60, 1, TimeUnit.SECONDS);
-        arbitraryDataFileListMessageScheduler.scheduleAtFixedRate(this::processNetworkArbitraryDataFileListMessage, 60, 1, TimeUnit.SECONDS);
     }
 
     public static ArbitraryDataFileListManager getInstance() {
@@ -124,8 +118,8 @@ public class ArbitraryDataFileListManager {
         if (timeSinceLastAttempt > 15 * 1000L) {
             // We haven't tried for at least 15 seconds
 
-            if (networkBroadcastCount < 12) {
-                // We've made less than 12 total attempts
+            if (networkBroadcastCount < 3) {
+                // We've made less than 3 total attempts
                 return true;
             }
         }
@@ -134,8 +128,8 @@ public class ArbitraryDataFileListManager {
         if (timeSinceLastAttempt > 60 * 1000L) {
             // We haven't tried for at least 1 minute
 
-            if (networkBroadcastCount < 40) {
-                // We've made less than 40 total attempts
+            if (networkBroadcastCount < 8) {
+                // We've made less than 8 total attempts
                 return true;
             }
         }
@@ -402,11 +396,11 @@ public class ArbitraryDataFileListManager {
         return true;
     }
 
-    public void deleteFileListRequestsForSignature(String signature58) {
-
+    public void deleteFileListRequestsForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
         for (Iterator<Map.Entry<Integer, Triple<String, Peer, Long>>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) {
             Map.Entry<Integer, Triple<String, Peer, Long>> entry = it.next();
-            if (entry == null || entry.getKey() == null || entry.getValue() == null) {
+            if (entry == null || entry.getKey() == null || entry.getValue() != null) {
                 continue;
             }
             if (Objects.equals(entry.getValue().getA(), signature58)) {
@@ -419,116 +413,70 @@ public class ArbitraryDataFileListManager {
 
     // Network handlers
 
-    // List to collect messages
-    private final List<PeerMessage> arbitraryDataFileListMessageList = new ArrayList<>();
-    // Lock to synchronize access to the list
-    private final Object arbitraryDataFileListMessageLock = new Object();
-
-    // Scheduled executor service to process messages every second
-    private final ScheduledExecutorService arbitraryDataFileListMessageScheduler = Executors.newScheduledThreadPool(1);
-
     public void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) {
         // Don't process if QDN is disabled
         if (!Settings.getInstance().isQdnEnabled()) {
             return;
         }
 
-        synchronized (arbitraryDataFileListMessageLock) {
-            arbitraryDataFileListMessageList.add(new PeerMessage(peer, message));
+        ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
+        LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
+
+        if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) {
+            long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime();
+            LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}",
+                    totalRequestTime, arbitraryDataFileListMessage.getRequestHops(),
+                    arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
         }
-    }
 
-    private void processNetworkArbitraryDataFileListMessage() {
+        // Do we have a pending request for this data?
+        Triple<String, Peer, Long> request = arbitraryDataFileListRequests.get(message.getId());
+        if (request == null || request.getA() == null) {
+            return;
+        }
+        boolean isRelayRequest = (request.getB() != null);
 
-        try {
-            List<PeerMessage> messagesToProcess;
-            synchronized (arbitraryDataFileListMessageLock) {
-                messagesToProcess = new ArrayList<>(arbitraryDataFileListMessageList);
-                arbitraryDataFileListMessageList.clear();
-            }
+        // Does this message's signature match what we're expecting?
+        byte[] signature = arbitraryDataFileListMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        if (!request.getA().equals(signature58)) {
+            return;
+        }
 
-            if (messagesToProcess.isEmpty()) return;
+        List<byte[]> hashes = arbitraryDataFileListMessage.getHashes();
+        if (hashes == null || hashes.isEmpty()) {
+            return;
+        }
 
-            Map<String, PeerMessage> peerMessageBySignature58 = new HashMap<>(messagesToProcess.size());
-            Map<String, byte[]> signatureBySignature58 = new HashMap<>(messagesToProcess.size());
-            Map<String, Boolean> isRelayRequestBySignature58 = new HashMap<>(messagesToProcess.size());
-            Map<String, List<byte[]>> hashesBySignature58 = new HashMap<>(messagesToProcess.size());
-            Map<String, Triple<String, Peer, Long>> requestBySignature58 = new HashMap<>(messagesToProcess.size());
+        ArbitraryTransactionData arbitraryTransactionData = null;
 
-            for (PeerMessage peerMessage : messagesToProcess) {
-                Peer peer = peerMessage.getPeer();
-                Message message = peerMessage.getMessage();
-
-                ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
-                LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
-
-                if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) {
-                    long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime();
-                    LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}",
-                            totalRequestTime, arbitraryDataFileListMessage.getRequestHops(),
-                            arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
-                }
-
-                // Do we have a pending request for this data?
-                Triple<String, Peer, Long> request = arbitraryDataFileListRequests.get(message.getId());
-                if (request == null || request.getA() == null) {
-                    continue;
-                }
-                boolean isRelayRequest = (request.getB() != null);
-
-                // Does this message's signature match what we're expecting?
-                byte[] signature = arbitraryDataFileListMessage.getSignature();
-                String signature58 = Base58.encode(signature);
-                if (!request.getA().equals(signature58)) {
-                    continue;
-                }
-
-                List<byte[]> hashes = arbitraryDataFileListMessage.getHashes();
-                if (hashes == null || hashes.isEmpty()) {
-                    continue;
-                }
-
-                peerMessageBySignature58.put(signature58, peerMessage);
-                signatureBySignature58.put(signature58, signature);
-                isRelayRequestBySignature58.put(signature58, isRelayRequest);
-                hashesBySignature58.put(signature58, hashes);
-                requestBySignature58.put(signature58, request);
-            }
-
-            if (signatureBySignature58.isEmpty()) return;
-
-            List<ArbitraryTransactionData> arbitraryTransactionDataList;
-
-            // Check transaction exists and hashes are correct
-            try (final Repository repository = RepositoryManager.getRepository()) {
-                arbitraryTransactionDataList
-                        = repository.getTransactionRepository()
-                        .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream()
-                        .filter(data -> data instanceof ArbitraryTransactionData)
-                        .map(data -> (ArbitraryTransactionData) data)
-                        .collect(Collectors.toList());
-            } catch (DataException e) {
-                LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list"), e);
+        // Check transaction exists and hashes are correct
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+            if (!(transactionData instanceof ArbitraryTransactionData))
                 return;
-            }
 
-            for (ArbitraryTransactionData arbitraryTransactionData : arbitraryTransactionDataList) {
+            arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
 
-                byte[] signature = arbitraryTransactionData.getSignature();
-                String signature58 = Base58.encode(signature);
+//          // Load data file(s)
+//          ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
+//
+//			// Check all hashes exist
+//			for (byte[] hash : hashes) {
+//				//LOGGER.debug("Received hash {}", Base58.encode(hash));
+//				if (!arbitraryDataFile.containsChunk(hash)) {
+//					// Check the hash against the complete file
+//					if (!Arrays.equals(arbitraryDataFile.getHash(), hash)) {
+//						LOGGER.info("Received non-matching chunk hash {} for signature {}. This could happen if we haven't obtained the metadata file yet.", Base58.encode(hash), signature58);
+//						return;
+//					}
+//				}
+//			}
 
-                List<byte[]> hashes = hashesBySignature58.get(signature58);
-
-                PeerMessage peerMessage = peerMessageBySignature58.get(signature58);
-                Peer peer = peerMessage.getPeer();
-                Message message = peerMessage.getMessage();
-
-                ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
-
-                Boolean isRelayRequest = isRelayRequestBySignature58.get(signature58);
-                if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) {
-                    Long now = NTP.getTime();
+            if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) {
+                Long now = NTP.getTime();
 
+                if (ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.size() < MAX_FILE_HASH_RESPONSES) {
                     // Keep track of the hashes this peer reports to have access to
                     for (byte[] hash : hashes) {
                         String hash58 = Base58.encode(hash);
@@ -539,300 +487,233 @@ public class ArbitraryDataFileListManager {
                         ArbitraryFileListResponseInfo responseInfo = new ArbitraryFileListResponseInfo(hash58, signature58,
                                 peer, now, arbitraryDataFileListMessage.getRequestTime(), requestHops);
 
-                        ArbitraryDataFileManager.getInstance().addResponse(responseInfo);
-                    }
-
-                    // Keep track of the source peer, for direct connections
-                    if (arbitraryDataFileListMessage.getPeerAddress() != null) {
-                        ArbitraryDataFileManager.getInstance().addDirectConnectionInfoIfUnique(
-                                new ArbitraryDirectConnectionInfo(signature, arbitraryDataFileListMessage.getPeerAddress(), hashes, now));
+                        ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.add(responseInfo);
                     }
                 }
 
-                // Forwarding
-                if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
+                // Keep track of the source peer, for direct connections
+                if (arbitraryDataFileListMessage.getPeerAddress() != null) {
+                    ArbitraryDataFileManager.getInstance().addDirectConnectionInfoIfUnique(
+                            new ArbitraryDirectConnectionInfo(signature, arbitraryDataFileListMessage.getPeerAddress(), hashes, now));
+                }
+            }
 
-                    boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
-                    if (!isBlocked) {
-                        Triple<String, Peer, Long> request = requestBySignature58.get(signature58);
-                        Peer requestingPeer = request.getB();
-                        if (requestingPeer != null) {
-                            Long requestTime = arbitraryDataFileListMessage.getRequestTime();
-                            Integer requestHops = arbitraryDataFileListMessage.getRequestHops();
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e);
+        }
 
-                            // Add each hash to our local mapping so we know who to ask later
-                            Long now = NTP.getTime();
-                            for (byte[] hash : hashes) {
-                                String hash58 = Base58.encode(hash);
-                                ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops);
-                                ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo);
-                            }
+        // Forwarding
+        if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
+            boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
+            if (!isBlocked) {
+                Peer requestingPeer = request.getB();
+                if (requestingPeer != null) {
+                    Long requestTime = arbitraryDataFileListMessage.getRequestTime();
+                    Integer requestHops = arbitraryDataFileListMessage.getRequestHops();
 
-                            // Bump requestHops if it exists
-                            if (requestHops != null) {
-                                requestHops++;
-                            }
+                    // Add each hash to our local mapping so we know who to ask later
+                    Long now = NTP.getTime();
+                    for (byte[] hash : hashes) {
+                        String hash58 = Base58.encode(hash);
+                        ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops);
+                        ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo);
+                    }
 
-                            ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage;
+                    // Bump requestHops if it exists
+                    if (requestHops != null) {
+                        requestHops++;
+                    }
 
-                            // Remove optional parameters if the requesting peer doesn't support it yet
-                            // A message with less statistical data is better than no message at all
-                            if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
-                                forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
-                            } else {
-                                forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops,
-                                        arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
-                            }
-                            forwardArbitraryDataFileListMessage.setId(message.getId());
+                    ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage;
 
-                            // Forward to requesting peer
-                            LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer);
-                            requestingPeer.sendMessage(forwardArbitraryDataFileListMessage);
-                        }
+                    // Remove optional parameters if the requesting peer doesn't support it yet
+                    // A message with less statistical data is better than no message at all
+                    if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
+                        forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+                    } else {
+                        forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops,
+                                arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
+                    }
+                    forwardArbitraryDataFileListMessage.setId(message.getId());
+
+                    // Forward to requesting peer
+                    LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer);
+                    if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) {
+                        requestingPeer.disconnect("failed to forward arbitrary data file list");
                     }
                 }
             }
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
         }
     }
 
-    // List to collect messages
-    private final List<PeerMessage> getArbitraryDataFileListMessageList = new ArrayList<>();
-    // Lock to synchronize access to the list
-    private final Object getArbitraryDataFileListMessageLock = new Object();
-
-    // Scheduled executor service to process messages every second
-    private final ScheduledExecutorService getArbitraryDataFileListMessageScheduler = Executors.newScheduledThreadPool(1);
-
     public void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) {
         // Don't respond if QDN is disabled
         if (!Settings.getInstance().isQdnEnabled()) {
             return;
         }
 
-        synchronized (getArbitraryDataFileListMessageLock) {
-            getArbitraryDataFileListMessageList.add(new PeerMessage(peer, message));
+        Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet();
+
+        GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message;
+        byte[] signature = getArbitraryDataFileListMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        Long now = NTP.getTime();
+        Triple<String, Peer, Long> newEntry = new Triple<>(signature58, peer, now);
+
+        // If we've seen this request recently, then ignore
+        if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) {
+            LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58);
+            return;
         }
-    }
 
-    private void processNetworkGetArbitraryDataFileListMessage() {
+        List<byte[]> requestedHashes = getArbitraryDataFileListMessage.getHashes();
+        int hashCount = requestedHashes != null ? requestedHashes.size() : 0;
+        String requestingPeer = getArbitraryDataFileListMessage.getRequestingPeer();
 
-        try {
-            List<PeerMessage> messagesToProcess;
-            synchronized (getArbitraryDataFileListMessageLock) {
-                messagesToProcess = new ArrayList<>(getArbitraryDataFileListMessageList);
-                getArbitraryDataFileListMessageList.clear();
-            }
+        if (requestingPeer != null) {
+            LOGGER.debug("Received hash list request with {} hashes from peer {} (requesting peer {}) for signature {}", hashCount, peer, requestingPeer, signature58);
+        }
+        else {
+            LOGGER.debug("Received hash list request with {} hashes from peer {} for signature {}", hashCount, peer, signature58);
+        }
 
-            if (messagesToProcess.isEmpty()) return;
+        List<byte[]> hashes = new ArrayList<>();
+        ArbitraryTransactionData transactionData = null;
+        boolean allChunksExist = false;
+        boolean hasMetadata = false;
 
-            Map<String, byte[]> signatureBySignature58 = new HashMap<>(messagesToProcess.size());
-            Map<String, List<byte[]>> requestedHashesBySignature58 = new HashMap<>(messagesToProcess.size());
-            Map<String, String> requestingPeerBySignature58 = new HashMap<>(messagesToProcess.size());
-            Map<String, Long> nowBySignature58 = new HashMap<>((messagesToProcess.size()));
-            Map<String, PeerMessage> peerMessageBySignature58 = new HashMap<>(messagesToProcess.size());
+        try (final Repository repository = RepositoryManager.getRepository()) {
 
-            for (PeerMessage messagePeer : messagesToProcess) {
-                Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet();
-
-                Message message = messagePeer.message;
-                Peer peer = messagePeer.peer;
-
-                GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message;
-                byte[] signature = getArbitraryDataFileListMessage.getSignature();
-                String signature58 = Base58.encode(signature);
-                Long now = NTP.getTime();
-                Triple<String, Peer, Long> newEntry = new Triple<>(signature58, peer, now);
-
-                // If we've seen this request recently, then ignore
-                if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) {
-                    LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58);
-                    continue;
-                }
-
-                List<byte[]> requestedHashes = getArbitraryDataFileListMessage.getHashes();
-                int hashCount = requestedHashes != null ? requestedHashes.size() : 0;
-                String requestingPeer = getArbitraryDataFileListMessage.getRequestingPeer();
-
-                if (requestingPeer != null) {
-                    LOGGER.debug("Received hash list request with {} hashes from peer {} (requesting peer {}) for signature {}", hashCount, peer, requestingPeer, signature58);
-                } else {
-                    LOGGER.debug("Received hash list request with {} hashes from peer {} for signature {}", hashCount, peer, signature58);
-                }
-
-                signatureBySignature58.put(signature58, signature);
-                requestedHashesBySignature58.put(signature58, requestedHashes);
-                requestingPeerBySignature58.put(signature58, requestingPeer);
-                nowBySignature58.put(signature58, now);
-                peerMessageBySignature58.put(signature58, messagePeer);
-            }
-
-            if (signatureBySignature58.isEmpty()) {
-                return;
-            }
-
-            List<byte[]> hashes = new ArrayList<>();
-            boolean allChunksExist = false;
-            boolean hasMetadata = false;
-
-            List<ArbitraryTransactionData> transactionDataList;
-            try (final Repository repository = RepositoryManager.getRepository()) {
-
-                // Firstly we need to lookup this file on chain to get a list of its hashes
-                transactionDataList
-                        = repository.getTransactionRepository()
-                        .fromSignatures(new ArrayList<>(signatureBySignature58.values())).stream()
-                        .filter(data -> data instanceof ArbitraryTransactionData)
-                        .map(data -> (ArbitraryTransactionData) data)
-                        .collect(Collectors.toList());
-
-            } catch (DataException e) {
-                LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer"), e);
-                return;
-            }
-
-            for (ArbitraryTransactionData transactionData : transactionDataList) {
-                byte[] signature = transactionData.getSignature();
-                String signature58 = Base58.encode(signature);
-                List<byte[]> requestedHashes = requestedHashesBySignature58.get(signature58);
+            // Firstly we need to lookup this file on chain to get a list of its hashes
+            transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature);
+            if (transactionData instanceof ArbitraryTransactionData) {
 
                 // Check if we're even allowed to serve data for this transaction
                 if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
 
-                    try {
-                        // Load file(s) and add any that exist to the list of hashes
-                        ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
+                    // Load file(s) and add any that exist to the list of hashes
+                    ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
 
-                        // If the peer didn't supply a hash list, we need to return all hashes for this transaction
-                        if (requestedHashes == null || requestedHashes.isEmpty()) {
-                            requestedHashes = new ArrayList<>();
+                    // If the peer didn't supply a hash list, we need to return all hashes for this transaction
+                    if (requestedHashes == null || requestedHashes.isEmpty()) {
+                        requestedHashes = new ArrayList<>();
 
-                            // Add the metadata file
-                            if (arbitraryDataFile.getMetadataHash() != null) {
-                                requestedHashes.add(arbitraryDataFile.getMetadataHash());
-                                hasMetadata = true;
-                            }
-
-                            // Add the chunk hashes
-                            if (!arbitraryDataFile.getChunkHashes().isEmpty()) {
-                                requestedHashes.addAll(arbitraryDataFile.getChunkHashes());
-                            }
-                            // Add complete file if there are no hashes
-                            else {
-                                requestedHashes.add(arbitraryDataFile.getHash());
-                            }
+                        // Add the metadata file
+                        if (arbitraryDataFile.getMetadataHash() != null) {
+                            requestedHashes.add(arbitraryDataFile.getMetadataHash());
+                            hasMetadata = true;
                         }
 
-
-                        // Assume all chunks exists, unless one can't be found below
-                        allChunksExist = true;
-
-                        for (byte[] requestedHash : requestedHashes) {
-                            ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature);
-                            if (chunk.exists()) {
-                                hashes.add(chunk.getHash());
-                                //LOGGER.trace("Added hash {}", chunk.getHash58());
-                            } else {
-                                LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58());
-                                allChunksExist = false;
-                            }
+                        // Add the chunk hashes
+                        if (!arbitraryDataFile.getChunkHashes().isEmpty()) {
+                            requestedHashes.addAll(arbitraryDataFile.getChunkHashes());
+                        }
+                        // Add complete file if there are no hashes
+                        else {
+                            requestedHashes.add(arbitraryDataFile.getHash());
                         }
-                    } catch (DataException e) {
-                        LOGGER.error(e.getMessage(), e);
-                    }
-                }
-
-                // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that,
-                // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam.
-                if (hasMetadata && hashes.size() == 1) {
-                    hashes.clear();
-                }
-
-                PeerMessage peerMessage = peerMessageBySignature58.get(signature58);
-                Peer peer = peerMessage.getPeer();
-                Message message = peerMessage.getMessage();
-
-                Long now = nowBySignature58.get(signature58);
-
-                // We should only respond if we have at least one hash
-                String requestingPeer = requestingPeerBySignature58.get(signature58);
-                if (!hashes.isEmpty()) {
-
-                    // Firstly we should keep track of the requesting peer, to allow for potential direct connections later
-                    ArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer);
-
-                    // We have all the chunks, so update requests map to reflect that we've sent it
-                    // There is no need to keep track of the request, as we can serve all the chunks
-                    if (allChunksExist) {
-                        Triple<String, Peer, Long> newEntry = new Triple<>(null, null, now);
-                        arbitraryDataFileListRequests.put(message.getId(), newEntry);
                     }
 
-                    String ourAddress = Network.getInstance().getOurExternalIpAddressAndPort();
-                    ArbitraryDataFileListMessage arbitraryDataFileListMessage;
-
-                    // Remove optional parameters if the requesting peer doesn't support it yet
-                    // A message with less statistical data is better than no message at all
-                    if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
-                        arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
-                    } else {
-                        arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature,
-                                hashes, NTP.getTime(), 0, ourAddress, true);
-                    }
-
-                    arbitraryDataFileListMessage.setId(message.getId());
-
-                    if (!peer.sendMessage(arbitraryDataFileListMessage)) {
-                        LOGGER.debug("Couldn't send list of hashes");
-                        continue;
-                    }
-
-                    if (allChunksExist) {
-                        // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
-                        LOGGER.debug("No need for any forwarding because file list request is fully served");
-                        continue;
-                    }
-
-                }
-
-                // We may need to forward this request on
-                boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName()));
-                if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
-                    // In relay mode - so ask our other peers if they have it
-
-
-                    GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message;
-
-                    long requestTime = getArbitraryDataFileListMessage.getRequestTime();
-                    int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1;
-                    long totalRequestTime = now - requestTime;
-
-                    if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
-                        // Relay request hasn't timed out yet, so can potentially be rebroadcast
-                        if (requestHops < RELAY_REQUEST_MAX_HOPS) {
-                            // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
-
-                            Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer);
-                            relayGetArbitraryDataFileListMessage.setId(message.getId());
-
-                            LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
-                            Network.getInstance().broadcast(
-                                    broadcastPeer ->
-                                            !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null :
-                                                    broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryDataFileListMessage
-                            );
+                    // Assume all chunks exists, unless one can't be found below
+                    allChunksExist = true;
 
+                    for (byte[] requestedHash : requestedHashes) {
+                        ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature);
+                        if (chunk.exists()) {
+                            hashes.add(chunk.getHash());
+                            //LOGGER.trace("Added hash {}", chunk.getHash58());
                         } else {
-                            // This relay request has reached the maximum number of allowed hops
+                            LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58());
+                            allChunksExist = false;
                         }
-                    } else {
-                        // This relay request has timed out
                     }
                 }
             }
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e);
+        }
+
+        // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that,
+        // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam.
+        if (hasMetadata && hashes.size() == 1) {
+            hashes.clear();
+        }
+
+        // We should only respond if we have at least one hash
+        if (!hashes.isEmpty()) {
+
+            // Firstly we should keep track of the requesting peer, to allow for potential direct connections later
+            ArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer);
+
+            // We have all the chunks, so update requests map to reflect that we've sent it
+            // There is no need to keep track of the request, as we can serve all the chunks
+            if (allChunksExist) {
+                newEntry = new Triple<>(null, null, now);
+                arbitraryDataFileListRequests.put(message.getId(), newEntry);
+            }
+
+            String ourAddress = Network.getInstance().getOurExternalIpAddressAndPort();
+            ArbitraryDataFileListMessage arbitraryDataFileListMessage;
+
+            // Remove optional parameters if the requesting peer doesn't support it yet
+            // A message with less statistical data is better than no message at all
+            if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
+                arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+            } else {
+                arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature,
+                        hashes, NTP.getTime(), 0, ourAddress, true);
+            }
+
+            arbitraryDataFileListMessage.setId(message.getId());
+
+            if (!peer.sendMessage(arbitraryDataFileListMessage)) {
+                LOGGER.debug("Couldn't send list of hashes");
+                peer.disconnect("failed to send list of hashes");
+                return;
+            }
+            LOGGER.debug("Sent list of hashes (count: {})", hashes.size());
+
+            if (allChunksExist) {
+                // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
+                LOGGER.debug("No need for any forwarding because file list request is fully served");
+                return;
+            }
+
+        }
+
+        // We may need to forward this request on
+        boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName()));
+        if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
+            // In relay mode - so ask our other peers if they have it
+
+            long requestTime = getArbitraryDataFileListMessage.getRequestTime();
+            int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1;
+            long totalRequestTime = now - requestTime;
+
+            if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
+                // Relay request hasn't timed out yet, so can potentially be rebroadcast
+                if (requestHops < RELAY_REQUEST_MAX_HOPS) {
+                    // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
+
+                    Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer);
+                    relayGetArbitraryDataFileListMessage.setId(message.getId());
+
+                    LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
+                    Network.getInstance().broadcast(
+                            broadcastPeer ->
+                                    !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null :
+                                    broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryDataFileListMessage
+                    );
+
+                }
+                else {
+                    // This relay request has reached the maximum number of allowed hops
+                }
+            }
+            else {
+                // This relay request has timed out
+            }
         }
     }
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java

@@ -1,7 +1,6 @@
 package org.qortal.controller.arbitrary;
 
 import com.google.common.net.InetAddresses;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.ArbitraryDataFile;
@@ -13,7 +12,6 @@ import org.qortal.data.network.PeerData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
-import org.qortal.network.PeerSendManagement;
 import org.qortal.network.message.*;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
@@ -25,16 +23,12 @@ import org.qortal.utils.NTP;
 
 import java.security.SecureRandom;
 import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 public class ArbitraryDataFileManager extends Thread {
 
-    public static final int SEND_TIMEOUT_MS = 500;
     private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileManager.class);
 
     private static ArbitraryDataFileManager instance;
@@ -54,7 +48,7 @@ public class ArbitraryDataFileManager extends Thread {
     /**
      * List to keep track of any arbitrary data file hash responses
      */
-    private final List<ArbitraryFileListResponseInfo> arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>());
+    public final List<ArbitraryFileListResponseInfo> arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>());
 
     /**
      * List to keep track of peers potentially available for direct connections, based on recent requests
@@ -71,9 +65,8 @@ public class ArbitraryDataFileManager extends Thread {
 
     public static int MAX_FILE_HASH_RESPONSES = 1000;
 
+
     private ArbitraryDataFileManager() {
-        this.arbitraryDataFileHashResponseScheduler.scheduleAtFixedRate( this::processResponses, 60, 1, TimeUnit.SECONDS);
-        this.arbitraryDataFileHashResponseScheduler.scheduleAtFixedRate(this::handleFileListRequestProcess, 60, 1, TimeUnit.SECONDS);
     }
 
     public static ArbitraryDataFileManager getInstance() {
@@ -83,13 +76,18 @@ public class ArbitraryDataFileManager extends Thread {
         return instance;
     }
 
-
-
     @Override
     public void run() {
         Thread.currentThread().setName("Arbitrary Data File Manager");
 
         try {
+            // Use a fixed thread pool to execute the arbitrary data file requests
+            int threadCount = 5;
+            ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount);
+            for (int i = 0; i < threadCount; i++) {
+                arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread());
+            }
+
             while (!isStopping) {
                 // Nothing to do yet
                 Thread.sleep(1000);
@@ -114,6 +112,7 @@ public class ArbitraryDataFileManager extends Thread {
 
         final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT;
         arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp);
+        arbitraryDataFileHashResponses.removeIf(entry -> entry.getTimestamp() < relayMinimumTimestamp);
 
         final long directConnectionInfoMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_DIRECT_CONNECTION_INFO_TIMEOUT;
         directConnectionInfo.removeIf(entry -> entry.getTimestamp() < directConnectionInfoMinimumTimestamp);
@@ -126,7 +125,8 @@ public class ArbitraryDataFileManager extends Thread {
 
     // Fetch data files by hash
 
-    public boolean fetchArbitraryDataFiles(Peer peer,
+    public boolean fetchArbitraryDataFiles(Repository repository,
+                                           Peer peer,
                                            byte[] signature,
                                            ArbitraryTransactionData arbitraryTransactionData,
                                            List<byte[]> hashes) throws DataException {
@@ -146,15 +146,21 @@ public class ArbitraryDataFileManager extends Thread {
                 if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
                     LOGGER.debug("Requesting data file {} from peer {}", hash58, peer);
                     Long startTime = NTP.getTime();
-                    ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, arbitraryTransactionData, signature, hash);
+                    ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, null, arbitraryTransactionData, signature, hash, null);
                     Long endTime = NTP.getTime();
                     if (receivedArbitraryDataFile != null) {
                         LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFile.getHash58(), peer, (endTime-startTime));
                         receivedAtLeastOneFile = true;
+
+                        // Remove this hash from arbitraryDataFileHashResponses now that we have received it
+                        arbitraryDataFileHashResponses.remove(hash58);
                     }
                     else {
                         LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime));
 
+                        // Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it
+                        arbitraryDataFileHashResponses.remove(hash58);
+
                         // Stop asking for files from this peer
                         break;
                     }
@@ -163,6 +169,10 @@ public class ArbitraryDataFileManager extends Thread {
                     LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer);
                 }
             }
+            else {
+                // Remove this hash from arbitraryDataFileHashResponses because we have a local copy
+                arbitraryDataFileHashResponses.remove(hash58);
+            }
         }
 
         if (receivedAtLeastOneFile) {
@@ -181,103 +191,14 @@ public class ArbitraryDataFileManager extends Thread {
         return receivedAtLeastOneFile;
     }
 
-    // Lock to synchronize access to the list
-    private final Object arbitraryDataFileHashResponseLock = new Object();
-
-    // Scheduled executor service to process messages every second
-    private final ScheduledExecutorService arbitraryDataFileHashResponseScheduler = Executors.newScheduledThreadPool(1);
-
-
-    public void addResponse( ArbitraryFileListResponseInfo responseInfo ) {
-
-        synchronized (arbitraryDataFileHashResponseLock) {
-            this.arbitraryDataFileHashResponses.add(responseInfo);
-        }
-    }
-
-    private void processResponses() {
-        try {
-            List<ArbitraryFileListResponseInfo> responsesToProcess;
-            synchronized (arbitraryDataFileHashResponseLock) {
-                responsesToProcess = new ArrayList<>(arbitraryDataFileHashResponses);
-                arbitraryDataFileHashResponses.clear();
-            }
-
-            if (responsesToProcess.isEmpty()) return;
-
-            Long now = NTP.getTime();
-
-            ArbitraryDataFileRequestThread.getInstance().processFileHashes(now, responsesToProcess, this);
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
-        }
-    }
-
-    private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, ArbitraryTransactionData arbitraryTransactionData, byte[] signature, byte[] hash) throws DataException {
+    private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer, ArbitraryTransactionData arbitraryTransactionData, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
+        ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature);
+        boolean fileAlreadyExists = existingFile.exists();
+        String hash58 = Base58.encode(hash);
         ArbitraryDataFile arbitraryDataFile;
 
-        try {
-            ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature);
-            boolean fileAlreadyExists = existingFile.exists();
-            String hash58 = Base58.encode(hash);
-
-            // Fetch the file if it doesn't exist locally
-            if (!fileAlreadyExists) {
-                LOGGER.debug(String.format("Fetching data file %.8s from peer %s", hash58, peer));
-                arbitraryDataFileRequests.put(hash58, NTP.getTime());
-                Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash);
-
-                Message response = null;
-                try {
-                    response = peer.getResponseWithTimeout(getArbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT);
-                } catch (InterruptedException e) {
-                    // Will return below due to null response
-                }
-                arbitraryDataFileRequests.remove(hash58);
-                LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58));
-
-                if (response == null) {
-                    LOGGER.debug("Received null response from peer {}", peer);
-                    return null;
-                }
-                if (response.getType() != MessageType.ARBITRARY_DATA_FILE) {
-                    LOGGER.debug("Received response with invalid type: {} from peer {}", response.getType(), peer);
-                    return null;
-                }
-
-                ArbitraryDataFileMessage peersArbitraryDataFileMessage = (ArbitraryDataFileMessage) response;
-                arbitraryDataFile = peersArbitraryDataFileMessage.getArbitraryDataFile();
-            } else {
-                LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58));
-                arbitraryDataFile = existingFile;
-            }
-
-            if (arbitraryDataFile != null) {
-
-                arbitraryDataFile.save();
-
-                // If this is a metadata file then we need to update the cache
-                if (arbitraryTransactionData != null && arbitraryTransactionData.getMetadataHash() != null) {
-                    if (Arrays.equals(arbitraryTransactionData.getMetadataHash(), hash)) {
-                        ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
-                    }
-                }
-
-                // We may need to remove the file list request, if we have all the files for this transaction
-                this.handleFileListRequests(signature);
-            }
-        } catch (DataException e) {
-            LOGGER.error(e.getMessage(), e);
-            arbitraryDataFile = null;
-        }
-
-        return arbitraryDataFile;
-    }
-
-    private void fetchFileForRelay(Peer peer, Peer requestingPeer, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
-        try {
-            String hash58 = Base58.encode(hash);
-
+        // Fetch the file if it doesn't exist locally
+        if (!fileAlreadyExists) {
             LOGGER.debug(String.format("Fetching data file %.8s from peer %s", hash58, peer));
             arbitraryDataFileRequests.put(hash58, NTP.getTime());
             Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash);
@@ -291,73 +212,73 @@ public class ArbitraryDataFileManager extends Thread {
             arbitraryDataFileRequests.remove(hash58);
             LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58));
 
+            // We may need to remove the file list request, if we have all the files for this transaction
+            this.handleFileListRequests(signature);
+
             if (response == null) {
                 LOGGER.debug("Received null response from peer {}", peer);
-                return;
+                return null;
             }
             if (response.getType() != MessageType.ARBITRARY_DATA_FILE) {
                 LOGGER.debug("Received response with invalid type: {} from peer {}", response.getType(), peer);
-                return;
+                return null;
             }
 
             ArbitraryDataFileMessage peersArbitraryDataFileMessage = (ArbitraryDataFileMessage) response;
-            ArbitraryDataFile arbitraryDataFile = peersArbitraryDataFileMessage.getArbitraryDataFile();
-
-            if (arbitraryDataFile != null) {
-
-                // We might want to forward the request to the peer that originally requested it
-                this.handleArbitraryDataFileForwarding(requestingPeer, new ArbitraryDataFileMessage(signature, arbitraryDataFile), originalMessage);
-            }
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
+            arbitraryDataFile = peersArbitraryDataFileMessage.getArbitraryDataFile();
+        } else {
+            LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58));
+            arbitraryDataFile = existingFile;
         }
+
+        if (arbitraryDataFile == null) {
+            // We don't have a file, so give up here
+            return null;
+        }
+
+        // We might want to forward the request to the peer that originally requested it
+        this.handleArbitraryDataFileForwarding(requestingPeer, new ArbitraryDataFileMessage(signature, arbitraryDataFile), originalMessage);
+
+        boolean isRelayRequest = (requestingPeer != null);
+        if (isRelayRequest) {
+            if (!fileAlreadyExists) {
+                // File didn't exist locally before the request, and it's a forwarding request, so delete it if it exists.
+                // It shouldn't exist on the filesystem yet, but leaving this here just in case.
+                arbitraryDataFile.delete(10);
+            }
+        }
+        else {
+            arbitraryDataFile.save();
+        }
+
+        // If this is a metadata file then we need to update the cache
+        if (arbitraryTransactionData != null && arbitraryTransactionData.getMetadataHash() != null) {
+            if (Arrays.equals(arbitraryTransactionData.getMetadataHash(), hash)) {
+                ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
+            }
+        }
+
+        return arbitraryDataFile;
     }
 
-    Map<String, byte[]> signatureBySignature58 = new HashMap<>();
-
-    // Lock to synchronize access to the list
-    private final Object handleFileListRequestsLock = new Object();
-
-    // Scheduled executor service to process messages every second
-    private final ScheduledExecutorService handleFileListRequestsScheduler = Executors.newScheduledThreadPool(1);
-
     private void handleFileListRequests(byte[] signature) {
-
-        synchronized (handleFileListRequestsLock) {
-            signatureBySignature58.put(Base58.encode(signature), signature);
-        }
-    }
-
-    private void handleFileListRequestProcess() {
-
-        Map<String, byte[]> signaturesToProcess;
-
-        synchronized (handleFileListRequestsLock) {
-            signaturesToProcess = new HashMap<>(signatureBySignature58);
-            signatureBySignature58.clear();
-        }
-
-        if( signaturesToProcess.isEmpty() ) return;
-
         try (final Repository repository = RepositoryManager.getRepository()) {
 
             // Fetch the transaction data
-            List<ArbitraryTransactionData> arbitraryTransactionDataList
-                = ArbitraryTransactionUtils.fetchTransactionDataList(repository, new ArrayList<>(signaturesToProcess.values()));
-
-            for( ArbitraryTransactionData arbitraryTransactionData : arbitraryTransactionDataList ) {
-                boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData);
-
-                if (completeFileExists) {
-                    String signature58 = Base58.encode(arbitraryTransactionData.getSignature());
-                    LOGGER.debug("All chunks or complete file exist for transaction {}", signature58);
-
-                    ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature58);
-                }
+            ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+            if (arbitraryTransactionData == null) {
+                return;
             }
 
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
+            boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData);
+
+            if (allChunksExist) {
+                // Update requests map to reflect that we've received all chunks
+                ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature);
+            }
+
+        } catch (DataException e) {
+            LOGGER.debug("Unable to handle file list requests: {}", e.getMessage());
         }
     }
 
@@ -374,14 +295,15 @@ public class ArbitraryDataFileManager extends Thread {
 
         LOGGER.debug("Received arbitrary data file - forwarding is needed");
 
-        try {
-            // The ID needs to match that of the original request
-            message.setId(originalMessage.getId());
+        // The ID needs to match that of the original request
+        message.setId(originalMessage.getId());
 
-            PeerSendManagement.getInstance().getOrCreateSendManager(requestingPeer).queueMessage(message, SEND_TIMEOUT_MS);
-
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
+        if (!requestingPeer.sendMessageWithTimeout(message, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT)) {
+            LOGGER.debug("Failed to forward arbitrary data file to peer {}", requestingPeer);
+            requestingPeer.disconnect("failed to forward arbitrary data file");
+        }
+        else {
+            LOGGER.debug("Forwarded arbitrary data file to peer {}", requestingPeer);
         }
     }
 
@@ -655,9 +577,13 @@ public class ArbitraryDataFileManager extends Thread {
                 LOGGER.debug("Sending file {}...", arbitraryDataFile);
                 ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, arbitraryDataFile);
                 arbitraryDataFileMessage.setId(message.getId());
-
-                PeerSendManagement.getInstance().getOrCreateSendManager(peer).queueMessage(arbitraryDataFileMessage, SEND_TIMEOUT_MS);
-
+                if (!peer.sendMessageWithTimeout(arbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT)) {
+                    LOGGER.debug("Couldn't send file {}", arbitraryDataFile);
+                    peer.disconnect("failed to send file");
+                }
+                else {
+                    LOGGER.debug("Sent file {}", arbitraryDataFile);
+                }
             }
             else if (relayInfo != null) {
                 LOGGER.debug("We have relay info for hash {}", Base58.encode(hash));
@@ -669,7 +595,7 @@ public class ArbitraryDataFileManager extends Thread {
                     LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58);
                     // No need to pass arbitraryTransactionData below because this is only used for metadata caching,
                     // and metadata isn't retained when relaying.
-                    this.fetchFileForRelay(peerToAsk, peer, signature, hash, message);
+                    this.fetchArbitraryDataFile(peerToAsk, peer, null, signature, hash, message);
                 }
                 else {
                     LOGGER.debug("Peer {} not found in relay info", peer);
@@ -691,6 +617,7 @@ public class ArbitraryDataFileManager extends Thread {
                 fileUnknownMessage.setId(message.getId());
                 if (!peer.sendMessage(fileUnknownMessage)) {
                     LOGGER.debug("Couldn't sent file-unknown response");
+                    peer.disconnect("failed to send file-unknown response");
                 }
                 else {
                     LOGGER.debug("Sent file-unknown response for file {}", arbitraryDataFile);

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java

@@ -4,186 +4,127 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.controller.Controller;
 import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo;
-import org.qortal.data.arbitrary.ArbitraryResourceData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.event.DataMonitorEvent;
+import org.qortal.event.EventBus;
 import org.qortal.network.Peer;
-import org.qortal.network.message.MessageType;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
-import org.qortal.settings.Settings;
 import org.qortal.utils.ArbitraryTransactionUtils;
 import org.qortal.utils.Base58;
 import org.qortal.utils.NTP;
-import org.qortal.utils.NamedThreadFactory;
 
-import java.net.http.HttpResponse;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
-import java.util.HashMap;
 import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Executor;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 
 import static java.lang.Thread.NORM_PRIORITY;
 
-public class ArbitraryDataFileRequestThread {
+public class ArbitraryDataFileRequestThread implements Runnable {
 
     private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class);
 
-    private static final Integer FETCHER_LIMIT_PER_PEER = Settings.getInstance().getMaxThreadsForMessageType(MessageType.GET_ARBITRARY_DATA_FILE);
-    private static final String FETCHER_THREAD_PREFIX = "Arbitrary Data Fetcher ";
+    public ArbitraryDataFileRequestThread() {
 
-    private ConcurrentHashMap<String, ExecutorService> executorByPeer = new ConcurrentHashMap<>();
-
-    private ArbitraryDataFileRequestThread() {
-        cleanupExecutorByPeerScheduler.scheduleAtFixedRate(this::cleanupExecutorsByPeer, 1, 1, TimeUnit.MINUTES);
     }
 
-    private static ArbitraryDataFileRequestThread instance = null;
-
-    public static ArbitraryDataFileRequestThread getInstance() {
-
-        if( instance == null ) {
-            instance = new ArbitraryDataFileRequestThread();
-        }
-
-        return instance;
-    }
-
-    private final ScheduledExecutorService cleanupExecutorByPeerScheduler = Executors.newScheduledThreadPool(1);
-
-    private void cleanupExecutorsByPeer() {
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data File Request Thread");
+        Thread.currentThread().setPriority(NORM_PRIORITY);
 
         try {
-            this.executorByPeer.forEach((key, value) -> {
-                if (value instanceof ThreadPoolExecutor) {
-                    ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) value;
-                    if (threadPoolExecutor.getActiveCount() == 0) {
-                        threadPoolExecutor.shutdown();
-                        if (this.executorByPeer.computeIfPresent(key, (k, v) -> null) == null) {
-                            LOGGER.trace("removed executor: peer = " + key);
-                        }
-                    }
-                } else {
-                    LOGGER.warn("casting issue in cleanup");
-                }
-            });
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
+            while (!Controller.isStopping()) {
+                Long now = NTP.getTime();
+                this.processFileHashes(now);
+            }
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
         }
     }
 
-    public void processFileHashes(Long now, List<ArbitraryFileListResponseInfo> responseInfos, ArbitraryDataFileManager arbitraryDataFileManager) {
+    private void processFileHashes(Long now) throws InterruptedException {
 		if (Controller.isStopping()) {
             return;
         }
 
-        Map<String, byte[]> signatureBySignature58 = new HashMap<>(responseInfos.size());
-        Map<String, List<ArbitraryFileListResponseInfo>> responseInfoBySignature58 = new HashMap<>();
+        ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance();
+        String signature58 = null;
+        String hash58 = null;
+        Peer peer = null;
+        boolean shouldProcess = false;
 
-        for( ArbitraryFileListResponseInfo responseInfo : responseInfos) {
+        synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) {
+            if (!arbitraryDataFileManager.arbitraryDataFileHashResponses.isEmpty()) {
 
-            if( responseInfo == null ) continue;
+                // Sort by lowest number of node hops first
+                Comparator<ArbitraryFileListResponseInfo> lowestHopsFirstComparator =
+                        Comparator.comparingInt(ArbitraryFileListResponseInfo::getRequestHops);
+                arbitraryDataFileManager.arbitraryDataFileHashResponses.sort(lowestHopsFirstComparator);
 
-            if (Controller.isStopping()) {
-                return;
+                Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.iterator();
+                while (iterator.hasNext()) {
+                    if (Controller.isStopping()) {
+                        return;
+                    }
+
+                    ArbitraryFileListResponseInfo responseInfo = (ArbitraryFileListResponseInfo) iterator.next();
+                    if (responseInfo == null) {
+                        iterator.remove();
+                        continue;
+                    }
+
+                    hash58 = responseInfo.getHash58();
+                    peer = responseInfo.getPeer();
+                    signature58 = responseInfo.getSignature58();
+                    Long timestamp = responseInfo.getTimestamp();
+
+                    if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) {
+                        // Ignore - to be deleted
+                        iterator.remove();
+                        continue;
+                    }
+
+                    // Skip if already requesting, but don't remove, as we might want to retry later
+                    if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) {
+                        // Already requesting - leave this attempt for later
+                        continue;
+                    }
+
+                    // We want to process this file
+                    shouldProcess = true;
+                    iterator.remove();
+                    break;
+                }
             }
-
-            Peer peer = responseInfo.getPeer();
-
-            // if relay timeout, then move on
-            if (now - responseInfo.getTimestamp() >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || responseInfo.getSignature58() == null || peer == null) {
-                continue;
-            }
-
-            // Skip if already requesting, but don't remove, as we might want to retry later
-            if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(responseInfo.getHash58())) {
-                // Already requesting - leave this attempt for later
-                arbitraryDataFileManager.addResponse(responseInfo); // don't remove -> adding back, beacause it was removed already above
-                continue;
-            }
-
-
-            byte[] hash = Base58.decode(responseInfo.getHash58());
-            byte[] signature = Base58.decode(responseInfo.getSignature58());
-
-            // check for null
-            if (signature == null || hash == null || peer == null) {
-                continue;
-            }
-
-            // We want to process this file, store and map data to process later
-            signatureBySignature58.put(responseInfo.getSignature58(), signature);
-            responseInfoBySignature58
-                    .computeIfAbsent(responseInfo.getSignature58(), signature58 -> new ArrayList<>())
-                    .add(responseInfo);
         }
 
-        // if there are no signatures, then there is nothing to process and nothing query the database
-        if( signatureBySignature58.isEmpty() ) return;
+        if (!shouldProcess) {
+            // Nothing to do
+            Thread.sleep(1000L);
+            return;
+        }
 
-        List<ArbitraryTransactionData> arbitraryTransactionDataList = new ArrayList<>();
+        byte[] hash = Base58.decode(hash58);
+        byte[] signature = Base58.decode(signature58);
 
         // Fetch the transaction data
         try (final Repository repository = RepositoryManager.getRepository()) {
-            arbitraryTransactionDataList.addAll(
-                ArbitraryTransactionUtils.fetchTransactionDataList(repository, new ArrayList<>(signatureBySignature58.values())));
-        } catch (DataException e) {
-            LOGGER.warn("Unable to fetch transaction data: {}", e.getMessage());
-        }
-
-        if( !arbitraryTransactionDataList.isEmpty() ) {
-            long start = System.currentTimeMillis();
-
-            for(ArbitraryTransactionData data : arbitraryTransactionDataList ) {
-                String signature58 = Base58.encode(data.getSignature());
-                for( ArbitraryFileListResponseInfo responseInfo : responseInfoBySignature58.get(signature58)) {
-                    Runnable fetcher = () -> arbitraryDataFileFetcher(arbitraryDataFileManager, responseInfo, data);
-                    this.executorByPeer
-                            .computeIfAbsent(
-                                responseInfo.getPeer().toString(),
-                                peer -> Executors.newFixedThreadPool(
-                                    FETCHER_LIMIT_PER_PEER,
-                                    new NamedThreadFactory(FETCHER_THREAD_PREFIX + responseInfo.getPeer().toString(), NORM_PRIORITY)
-                                )
-                            )
-                            .execute(fetcher);
-                }
-            }
-            long timeLapse = System.currentTimeMillis() - start;
-        }
-    }
-
-    private void arbitraryDataFileFetcher(ArbitraryDataFileManager arbitraryDataFileManager, ArbitraryFileListResponseInfo responseInfo, ArbitraryTransactionData arbitraryTransactionData)  {
-        try {
-            Long now = NTP.getTime();
-
-            if (now - responseInfo.getTimestamp() >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT ) {
-
-                Peer peer = responseInfo.getPeer();
-                String hash58 = responseInfo.getHash58();
-                String signature58 = responseInfo.getSignature58();
-                LOGGER.debug("Peer {} version {} didn't fetch data file {} for signature {} due to relay timeout.", peer, peer.getPeersVersionString(), hash58, signature58);
+            ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+            if (arbitraryTransactionData == null) {
                 return;
             }
 
-            arbitraryDataFileManager.fetchArbitraryDataFiles(
-                responseInfo.getPeer(),
-                arbitraryTransactionData.getSignature(),
-                arbitraryTransactionData,
-                Arrays.asList(Base58.decode(responseInfo.getHash58()))
-            );
+            if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) {
+                return;
+            }
+
+            LOGGER.trace("Fetching file {} from peer {} via request thread...", hash58, peer);
+            arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash));
+
         } catch (DataException e) {
-            LOGGER.warn("Unable to process file hashes: {}", e.getMessage());
+            LOGGER.debug("Unable to process file hashes: {}", e.getMessage());
         }
     }
-}
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -42,10 +42,10 @@ public class ArbitraryDataManager extends Thread {
 	private int powDifficulty = 14; // Must not be final, as unit tests need to reduce this value
 
 	/** Request timeout when transferring arbitrary data */
-	public static final long ARBITRARY_REQUEST_TIMEOUT = 24 * 1000L; // ms
+	public static final long ARBITRARY_REQUEST_TIMEOUT = 12 * 1000L; // ms
 
 	/** Maximum time to hold information about an in-progress relay */
-	public static final long ARBITRARY_RELAY_TIMEOUT = 120 * 1000L; // ms
+	public static final long ARBITRARY_RELAY_TIMEOUT = 60 * 1000L; // ms
 
 	/** Maximum time to hold direct peer connection information */
 	public static final long ARBITRARY_DIRECT_CONNECTION_INFO_TIMEOUT = 2 * 60 * 1000L; // ms

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java

@@ -47,15 +47,15 @@ public class ArbitraryDataStorageManager extends Thread {
 
     private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
 
-    /** Treat storage as full at 80% usage, to reduce risk of going over the limit.
+    /** Treat storage as full at 90% usage, to reduce risk of going over the limit.
      * This is necessary because we don't calculate total storage values before every write.
      * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit.
      * This must be lower than DELETION_THRESHOLD. */
-    private static final double STORAGE_FULL_THRESHOLD = 0.8f; // 80%
+    private static final double STORAGE_FULL_THRESHOLD = 0.90f; // 90%
 
-    /** Start deleting files once we reach 90% usage.
+    /** Start deleting files once we reach 98% usage.
      * This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */
-    public static final double DELETION_THRESHOLD = 0.9f; // 90%
+    public static final double DELETION_THRESHOLD = 0.98f; // 98%
 
     private static final long PER_NAME_STORAGE_MULTIPLIER = 4L;
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java

@@ -24,11 +24,6 @@ import org.qortal.utils.Triple;
 
 import java.io.IOException;
 import java.util.*;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-import java.util.stream.Collectors;
 
 import static org.qortal.controller.arbitrary.ArbitraryDataFileListManager.*;
 
@@ -66,7 +61,6 @@ public class ArbitraryMetadataManager {
 
 
     private ArbitraryMetadataManager() {
-        scheduler.scheduleAtFixedRate(this::processNetworkGetArbitraryMetadataMessage, 60, 1, TimeUnit.SECONDS);
     }
 
     public static ArbitraryMetadataManager getInstance() {
@@ -360,8 +354,9 @@ public class ArbitraryMetadataManager {
 
                         // Forward to requesting peer
                         LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
-                        requestingPeer.sendMessage(forwardArbitraryMetadataMessage);
-                       
+                        if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
+                            requestingPeer.disconnect("failed to forward arbitrary metadata");
+                        }
                     }
                 }
             }
@@ -376,159 +371,107 @@ public class ArbitraryMetadataManager {
         }
     }
 
-    // List to collect messages
-    private final List<PeerMessage> messageList = new ArrayList<>();
-    // Lock to synchronize access to the list
-    private final Object lock = new Object();
-
-    // Scheduled executor service to process messages every second
-    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
-
     public void onNetworkGetArbitraryMetadataMessage(Peer peer, Message message) {
-
         // Don't respond if QDN is disabled
         if (!Settings.getInstance().isQdnEnabled()) {
             return;
         }
 
-        synchronized (lock) {
-            messageList.add(new PeerMessage(peer, message));
+        Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet();
+
+        GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) message;
+        byte[] signature = getArbitraryMetadataMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        Long now = NTP.getTime();
+        Triple<String, Peer, Long> newEntry = new Triple<>(signature58, peer, now);
+
+        // If we've seen this request recently, then ignore
+        if (arbitraryMetadataRequests.putIfAbsent(message.getId(), newEntry) != null) {
+            LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peer, signature58);
+            return;
         }
-    }
 
-    private void processNetworkGetArbitraryMetadataMessage() {
+        LOGGER.debug("Received metadata request from peer {} for signature {}", peer, signature58);
 
-        try {
-            List<PeerMessage> messagesToProcess;
-            synchronized (lock) {
-                messagesToProcess = new ArrayList<>(messageList);
-                messageList.clear();
-            }
+        ArbitraryTransactionData transactionData = null;
+        ArbitraryDataFile metadataFile = null;
 
-            Map<String, byte[]> signatureBySignature58 = new HashMap<>((messagesToProcess.size()));
-            Map<String, Long> nowBySignature58 = new HashMap<>(messagesToProcess.size());
-            Map<String,PeerMessage> peerMessageBySignature58 = new HashMap<>(messagesToProcess.size());
+        try (final Repository repository = RepositoryManager.getRepository()) {
 
-            for( PeerMessage peerMessage : messagesToProcess) {
-                Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet();
+            // Firstly we need to lookup this file on chain to get its metadata hash
+            transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature);
+            if (transactionData instanceof ArbitraryTransactionData) {
 
-                GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) peerMessage.message;
-                byte[] signature = getArbitraryMetadataMessage.getSignature();
-                String signature58 = Base58.encode(signature);
-                Long now = NTP.getTime();
-                Triple<String, Peer, Long> newEntry = new Triple<>(signature58, peerMessage.peer, now);
+                // Check if we're even allowed to serve metadata for this transaction
+                if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
 
-                // If we've seen this request recently, then ignore
-                if (arbitraryMetadataRequests.putIfAbsent(peerMessage.message.getId(), newEntry) != null) {
-                    LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peerMessage.peer, signature58);
-                    continue;
+                    byte[] metadataHash = transactionData.getMetadataHash();
+                    if (metadataHash != null) {
+
+                        // Load metadata file
+                        metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+                    }
                 }
-
-                LOGGER.debug("Received metadata request from peer {} for signature {}", peerMessage.peer, signature58);
-
-                signatureBySignature58.put(signature58, signature);
-                nowBySignature58.put(signature58, now);
-                peerMessageBySignature58.put(signature58, peerMessage);
             }
 
-            if( signatureBySignature58.isEmpty() ) return;
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while fetching arbitrary metadata for peer %s", peer), e);
+        }
 
-            List<TransactionData> transactionDataList;
-            try (final Repository repository = RepositoryManager.getRepository()) {
+        // We should only respond if we have the metadata file
+        if (metadataFile != null && metadataFile.exists()) {
 
-                // Firstly we need to lookup this file on chain to get its metadata hash
-                transactionDataList = repository.getTransactionRepository().fromSignatures(new ArrayList(signatureBySignature58.values()));
-            } catch (DataException e) {
-                LOGGER.error(String.format("Repository issue while fetching arbitrary transactions"), e);
+            // We have the metadata file, so update requests map to reflect that we've sent it
+            newEntry = new Triple<>(null, null, now);
+            arbitraryMetadataRequests.put(message.getId(), newEntry);
+
+            ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, metadataFile);
+            arbitraryMetadataMessage.setId(message.getId());
+            if (!peer.sendMessage(arbitraryMetadataMessage)) {
+                LOGGER.debug("Couldn't send metadata");
+                peer.disconnect("failed to send metadata");
                 return;
             }
+            LOGGER.debug("Sent metadata");
 
-            Map<String, ArbitraryTransactionData> dataBySignature58
-                = transactionDataList.stream()
-                    .filter(data -> data instanceof ArbitraryTransactionData)
-                    .map(ArbitraryTransactionData.class::cast)
-                    .collect(Collectors.toMap(data -> Base58.encode(data.getSignature()), Function.identity()));
+            // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
+            LOGGER.debug("No need for any forwarding because metadata request is fully served");
+            return;
 
-            for(Map.Entry<String, ArbitraryTransactionData> entry : dataBySignature58.entrySet()) {
-                String signature58 = entry.getKey();
-                ArbitraryTransactionData transactionData = entry.getValue();
+        }
 
-                try {
+        // We may need to forward this request on
+        boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName()));
+        if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
+            // In relay mode - so ask our other peers if they have it
 
-                    // Check if we're even allowed to serve metadata for this transaction
-                    if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
+            long requestTime = getArbitraryMetadataMessage.getRequestTime();
+            int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1;
+            long totalRequestTime = now - requestTime;
 
-                        byte[] metadataHash = transactionData.getMetadataHash();
-                        if (metadataHash != null) {
+            if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
+                // Relay request hasn't timed out yet, so can potentially be rebroadcast
+                if (requestHops < RELAY_REQUEST_MAX_HOPS) {
+                    // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
 
-                            // Load metadata file
-                            ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, transactionData.getSignature());
-                            // We should only respond if we have the metadata file
-                            if (metadataFile != null && metadataFile.exists()) {
+                    Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops);
+                    relayGetArbitraryMetadataMessage.setId(message.getId());
 
-                                PeerMessage peerMessage = peerMessageBySignature58.get(signature58);
-                                Message message = peerMessage.message;
-                                Peer peer = peerMessage.peer;
+                    LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
+                    Network.getInstance().broadcast(
+                            broadcastPeer ->
+                                    !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null :
+                                    broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryMetadataMessage);
 
-                                // We have the metadata file, so update requests map to reflect that we've sent it
-                                Triple newEntry = new Triple<>(null, null, nowBySignature58.get(signature58));
-                                arbitraryMetadataRequests.put(message.getId(), newEntry);
-
-                                ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(entry.getValue().getSignature(), metadataFile);
-                                arbitraryMetadataMessage.setId(message.getId());
-                                if (!peer.sendMessage(arbitraryMetadataMessage)) {
-                                    LOGGER.debug("Couldn't send metadata");
-                                    continue;
-                                }
-                                LOGGER.debug("Sent metadata");
-
-                                // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
-                                LOGGER.debug("No need for any forwarding because metadata request is fully served");
-                            }
-
-                        }
-                    }
-                } catch (DataException e) {
-                    LOGGER.error(String.format("Repository issue while fetching arbitrary metadata"), e);
                 }
-
-                // We may need to forward this request on
-                boolean isBlocked = (transactionDataList == null || ListUtils.isNameBlocked(transactionData.getName()));
-                if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
-                    // In relay mode - so ask our other peers if they have it
-
-                    PeerMessage peerMessage = peerMessageBySignature58.get(signature58);
-                    GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) peerMessage.message;
-                    long requestTime = getArbitraryMetadataMessage.getRequestTime();
-                    int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1;
-                    long totalRequestTime = nowBySignature58.get(signature58) - requestTime;
-
-                    if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
-                        // Relay request hasn't timed out yet, so can potentially be rebroadcast
-                        if (requestHops < RELAY_REQUEST_MAX_HOPS) {
-                            // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
-
-                            byte[] signature = signatureBySignature58.get(signature58);
-                            Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops);
-                            relayGetArbitraryMetadataMessage.setId(getArbitraryMetadataMessage.getId());
-
-                            Peer peer = peerMessage.peer;
-                            LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
-                            Network.getInstance().broadcast(
-                                    broadcastPeer ->
-                                            !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null :
-                                                    broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryMetadataMessage);
-
-                        } else {
-                            // This relay request has reached the maximum number of allowed hops
-                        }
-                    } else {
-                        // This relay request has timed out
-                    }
+                else {
+                    // This relay request has reached the maximum number of allowed hops
                 }
             }
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
+            else {
+                // This relay request has timed out
+            }
         }
     }
+
 }

src/main/java/org/qortal/controller/arbitrary/Follower.java

@@ -1,130 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.settings.Settings;
-import org.qortal.utils.ListUtils;
-import org.qortal.utils.NamedThreadFactory;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.OptionalInt;
-import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-public class Follower {
-
-    private static final Logger LOGGER = LogManager.getLogger(Follower.class);
-
-    private ScheduledExecutorService service
-            = Executors.newScheduledThreadPool(2, new NamedThreadFactory("Follower", Thread.NORM_PRIORITY));
-
-    private Follower() {
-
-    }
-
-    private static Follower instance;
-
-    public static Follower getInstance() {
-
-        if( instance == null ) {
-            instance = new Follower();
-        }
-
-        return instance;
-    }
-
-    public void start() {
-
-        // fetch arbitrary transactions from followed names from the last 100 blocks every 2 minutes
-        service.scheduleWithFixedDelay(() -> fetch(OptionalInt.of(100)), 10, 2, TimeUnit.MINUTES);
-
-        // fetch arbitrary transaction from followed names from any block every 24 hours
-        service.scheduleWithFixedDelay(() -> fetch(OptionalInt.empty()), 4, 24, TimeUnit.HOURS);
-    }
-
-    private void fetch(OptionalInt limit) {
-
-        try {
-            // for each followed name, get arbitraty transactions, then examine those transactions before fetching
-            for (String name : ListUtils.followedNames()) {
-
-                List<ArbitraryTransactionData> transactionsInReverseOrder;
-
-                // open database to get the transactions in reverse order for the followed name
-                try (final Repository repository = RepositoryManager.getRepository()) {
-
-                    List<ArbitraryTransactionData> latestArbitraryTransactionsByName
-                            = repository.getArbitraryRepository().getLatestArbitraryTransactionsByName(name);
-
-                    if (limit.isPresent()) {
-                        final int blockHeightThreshold = repository.getBlockRepository().getBlockchainHeight() - limit.getAsInt();
-
-                        transactionsInReverseOrder
-                                = latestArbitraryTransactionsByName.stream().filter(tx -> tx.getBlockHeight() > blockHeightThreshold)
-                                .collect(Collectors.toList());
-                    } else {
-                        transactionsInReverseOrder = latestArbitraryTransactionsByName;
-                    }
-
-                } catch (Exception e) {
-                    LOGGER.error(e.getMessage(), e);
-                    transactionsInReverseOrder = new ArrayList<>(0);
-                }
-
-                // collect process transaction hashes, so we don't fetch outdated transactions
-                Set<ArbitraryTransactionDataHashWrapper> processedTransactions = new HashSet<>();
-
-                ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
-
-                // for each arbitrary transaction for the followed name process, evaluate, fetch
-                for (ArbitraryTransactionData arbitraryTransaction : transactionsInReverseOrder) {
-
-                    boolean examined = false;
-
-                    try (final Repository repository = RepositoryManager.getRepository()) {
-
-                        // if not processed
-                        if (!processedTransactions.contains(new ArbitraryTransactionDataHashWrapper(arbitraryTransaction))) {
-                            boolean isLocal = repository.getArbitraryRepository().isDataLocal(arbitraryTransaction.getSignature());
-
-                            // if not local, then continue to evaluate
-                            if (!isLocal) {
-
-                                // evaluate fetching status for this transaction on this node
-                                ArbitraryDataExamination examination = storageManager.shouldPreFetchData(repository, arbitraryTransaction);
-
-                                // if the evaluation passed, then fetch
-                                examined = examination.isPass();
-                            }
-                            // if locally stored, then nothing needs to be done
-
-                            // add to processed transactions
-                            processedTransactions.add(new ArbitraryTransactionDataHashWrapper(arbitraryTransaction));
-                        }
-                    }
-
-                    // if passed examination for fetching, then fetch
-                    if (examined) {
-                        LOGGER.info("for {} on {}, fetching {}", name, arbitraryTransaction.getService(), arbitraryTransaction.getIdentifier());
-                        boolean fetched = ArbitraryDataFileListManager.getInstance().fetchArbitraryDataFileList(arbitraryTransaction);
-
-                        LOGGER.info("fetched = " + fetched);
-                    }
-
-                    // pause a second before moving on to another transaction
-                    Thread.sleep(1000);
-                }
-            }
-        } catch (Exception e) {
-            LOGGER.error(e.getMessage(), e);
-        }
-    }
-}

src/main/java/org/qortal/controller/arbitrary/PeerMessage.java

@@ -1,22 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import org.qortal.network.Peer;
-import org.qortal.network.message.Message;
-
-public class PeerMessage {
-    Peer peer;
-    Message message;
-
-    public PeerMessage(Peer peer, Message message) {
-        this.peer = peer;
-        this.message = message;
-    }
-
-    public Peer getPeer() {
-        return peer;
-    }
-
-    public Message getMessage() {
-        return message;
-    }
-}

src/main/java/org/qortal/controller/arbitrary/RNSArbitraryDataFileListManager.java

@@ -0,0 +1,731 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.arbitrary.ArbitraryDataFileChunk;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.RNSArbitraryDirectConnectionInfo;
+import org.qortal.data.arbitrary.RNSArbitraryFileListResponseInfo;
+import org.qortal.data.arbitrary.RNSArbitraryRelayInfo;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.ArbitraryDataFileListMessage;
+import org.qortal.network.message.GetArbitraryDataFileListMessage;
+import org.qortal.network.message.Message;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+import org.qortal.utils.ListUtils;
+import org.qortal.utils.NTP;
+import org.qortal.utils.Triple;
+
+import java.util.*;
+
+import static org.qortal.controller.arbitrary.RNSArbitraryDataFileManager.MAX_FILE_HASH_RESPONSES;
+
+public class RNSArbitraryDataFileListManager {
+
+    private static final Logger LOGGER = LogManager.getLogger(RNSArbitraryDataFileListManager.class);
+
+    private static RNSArbitraryDataFileListManager instance;
+
+    private static String MIN_PEER_VERSION_FOR_FILE_LIST_STATS = "3.2.0";
+
+    /**
+     * Map of recent incoming requests for ARBITRARY transaction data file lists.
+     * <p>
+     * Key is original request's message ID<br>
+     * Value is Triple&lt;transaction signature in base58, first requesting peer, first request's timestamp&gt;
+     * <p>
+     * If peer is null then either:<br>
+     * <ul>
+     * <li>we are the original requesting peer</li>
+     * <li>we have already sent data payload to original requesting peer.</li>
+     * </ul>
+     * If signature is null then we have already received the file list and either:<br>
+     * <ul>
+     * <li>we are the original requesting peer and have processed it</li>
+     * <li>we have forwarded the file list</li>
+     * </ul>
+     */
+    public Map<Integer, Triple<String, RNSPeer, Long>> arbitraryDataFileListRequests = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of in progress arbitrary data signature requests
+     * Key: string - the signature encoded in base58
+     * Value: Triple<networkBroadcastCount, directPeerRequestCount, lastAttemptTimestamp>
+     */
+    private Map<String, Triple<Integer, Integer, Long>> arbitraryDataSignatureRequests = Collections.synchronizedMap(new HashMap<>());
+
+
+    /** Maximum number of seconds that a file list relay request is able to exist on the network */
+    public static long RELAY_REQUEST_MAX_DURATION = 5000L;
+    /** Maximum number of hops that a file list relay request is allowed to make */
+    public static int RELAY_REQUEST_MAX_HOPS = 4;
+
+    /** Minimum peer version to use relay */
+    public static String RELAY_MIN_PEER_VERSION = "3.4.0";
+
+
+    private RNSArbitraryDataFileListManager() {
+    }
+
+    public static RNSArbitraryDataFileListManager getInstance() {
+        if (instance == null)
+            instance = new RNSArbitraryDataFileListManager();
+
+        return instance;
+    }
+
+
+    public void cleanupRequestCache(Long now) {
+        if (now == null) {
+            return;
+        }
+        final long requestMinimumTimestamp = now - ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT;
+        arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < requestMinimumTimestamp);
+    }
+
+
+    // Track file list lookups by signature
+
+    private boolean shouldMakeFileListRequestForSignature(String signature58) {
+        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        // Extract the components
+        Integer networkBroadcastCount = request.getA();
+        // Integer directPeerRequestCount = request.getB();
+        Long lastAttemptTimestamp = request.getC();
+
+        if (lastAttemptTimestamp == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
+
+        // Allow a second attempt after 15 seconds, and another after 30 seconds
+        if (timeSinceLastAttempt > 15 * 1000L) {
+            // We haven't tried for at least 15 seconds
+
+            if (networkBroadcastCount < 3) {
+                // We've made less than 3 total attempts
+                return true;
+            }
+        }
+
+        // Then allow another 5 attempts, each 1 minute apart
+        if (timeSinceLastAttempt > 60 * 1000L) {
+            // We haven't tried for at least 1 minute
+
+            if (networkBroadcastCount < 8) {
+                // We've made less than 8 total attempts
+                return true;
+            }
+        }
+
+        // Then allow another 8 attempts, each 15 minutes apart
+        if (timeSinceLastAttempt > 15 * 60 * 1000L) {
+            // We haven't tried for at least 15 minutes
+
+            if (networkBroadcastCount < 16) {
+                // We've made less than 16 total attempts
+                return true;
+            }
+        }
+
+        // From then on, only try once every 6 hours, to reduce network spam
+        if (timeSinceLastAttempt > 6 * 60 * 60 * 1000L) {
+            // We haven't tried for at least 6 hours
+            return true;
+        }
+
+        return false;
+    }
+
+    private boolean shouldMakeDirectFileRequestsForSignature(String signature58) {
+        if (!Settings.getInstance().isDirectDataRetrievalEnabled()) {
+            // Direct connections are disabled in the settings
+            return false;
+        }
+
+        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        // Extract the components
+        //Integer networkBroadcastCount = request.getA();
+        Integer directPeerRequestCount = request.getB();
+        Long lastAttemptTimestamp = request.getC();
+
+        if (lastAttemptTimestamp == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        if (directPeerRequestCount == 0) {
+            // We haven't tried asking peers directly yet, so we should
+            return true;
+        }
+
+        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
+        if (timeSinceLastAttempt > 10 * 1000L) {
+            // We haven't tried for at least 10 seconds
+            if (directPeerRequestCount < 5) {
+                // We've made less than 5 total attempts
+                return true;
+            }
+        }
+
+        if (timeSinceLastAttempt > 5 * 60 * 1000L) {
+            // We haven't tried for at least 5 minutes
+            if (directPeerRequestCount < 10) {
+                // We've made less than 10 total attempts
+                return true;
+            }
+        }
+
+        if (timeSinceLastAttempt > 60 * 60 * 1000L) {
+            // We haven't tried for at least 1 hour
+            return true;
+        }
+
+        return false;
+    }
+
+    public boolean isSignatureRateLimited(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        return !this.shouldMakeFileListRequestForSignature(signature58)
+                && !this.shouldMakeDirectFileRequestsForSignature(signature58);
+    }
+
+    public long lastRequestForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return 0;
+        }
+
+        // Extract the components
+        Long lastAttemptTimestamp = request.getC();
+        if (lastAttemptTimestamp != null) {
+            return  lastAttemptTimestamp;
+        }
+        return 0;
+    }
+
+    public void addToSignatureRequests(String signature58, boolean incrementNetworkRequests, boolean incrementPeerRequests) {
+        Triple<Integer, Integer, Long> request  = arbitraryDataSignatureRequests.get(signature58);
+        Long now = NTP.getTime();
+
+        if (request == null) {
+            // No entry yet
+            Triple<Integer, Integer, Long> newRequest = new Triple<>(0, 0, now);
+            arbitraryDataSignatureRequests.put(signature58, newRequest);
+        }
+        else {
+            // There is an existing entry
+            if (incrementNetworkRequests) {
+                request.setA(request.getA() + 1);
+            }
+            if (incrementPeerRequests) {
+                request.setB(request.getB() + 1);
+            }
+            request.setC(now);
+            arbitraryDataSignatureRequests.put(signature58, request);
+        }
+    }
+
+    public void removeFromSignatureRequests(String signature58) {
+        arbitraryDataSignatureRequests.remove(signature58);
+    }
+
+
+    // Lookup file lists by signature (and optionally hashes)
+
+    public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) {
+        byte[] signature = arbitraryTransactionData.getSignature();
+        String signature58 = Base58.encode(signature);
+
+        // Require an NTP sync
+        Long now = NTP.getTime();
+        if (now == null) {
+            return false;
+        }
+
+        // If we've already tried too many times in a short space of time, make sure to give up
+        if (!this.shouldMakeFileListRequestForSignature(signature58)) {
+            // Check if we should make direct connections to peers
+            if (this.shouldMakeDirectFileRequestsForSignature(signature58)) {
+                return RNSArbitraryDataFileManager.getInstance().fetchDataFilesFromPeersForSignature(signature);
+            }
+
+            LOGGER.trace("Skipping file list request for signature {} due to rate limit", signature58);
+            return false;
+        }
+        this.addToSignatureRequests(signature58, true, false);
+
+        //List<Peer> handshakedPeers = Network.getInstance().getImmutableHandshakedPeers();
+        List<RNSPeer> handshakedPeers = RNSNetwork.getInstance().getLinkedPeers();
+        List<byte[]> missingHashes = null;
+
+        // Find hashes that we are missing
+        try {
+            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
+            missingHashes = arbitraryDataFile.missingHashes();
+        } catch (DataException e) {
+            // Leave missingHashes as null, so that all hashes are requested
+        }
+        int hashCount = missingHashes != null ? missingHashes.size() : 0;
+
+        LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size()));
+
+        //// Send our address as requestingPeer, to allow for potential direct connections with seeds/peers
+        //String requestingPeer = Network.getInstance().getOurExternalIpAddressAndPort();
+        String requestingPeer = null;
+
+        // Build request
+        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, missingHashes, now, 0, requestingPeer);
+
+        // Save our request into requests map
+        Triple<String, RNSPeer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
+
+        // Assign random ID to this message
+        int id;
+        do {
+            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
+
+            // Put queue into map (keyed by message ID) so we can poll for a response
+            // If putIfAbsent() doesn't return null, then this ID is already taken
+        } while (arbitraryDataFileListRequests.put(id, requestEntry) != null);
+        getArbitraryDataFileListMessage.setId(id);
+
+        // Broadcast request
+        RNSNetwork.getInstance().broadcast(peer -> getArbitraryDataFileListMessage);
+
+        // Poll to see if data has arrived
+        final long singleWait = 100;
+        long totalWait = 0;
+        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
+            try {
+                Thread.sleep(singleWait);
+            } catch (InterruptedException e) {
+                break;
+            }
+
+            requestEntry = arbitraryDataFileListRequests.get(id);
+            if (requestEntry == null)
+                return false;
+
+            if (requestEntry.getA() == null)
+                break;
+
+            totalWait += singleWait;
+        }
+        return true;
+    }
+
+    public boolean fetchArbitraryDataFileList(RNSPeer peer, byte[] signature) {
+        String signature58 = Base58.encode(signature);
+
+        // Require an NTP sync
+        Long now = NTP.getTime();
+        if (now == null) {
+            return false;
+        }
+
+        int hashCount = 0;
+        LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to peer %s...", signature58, hashCount, peer));
+
+        // Build request
+        // Use a time in the past, so that the recipient peer doesn't try and relay it
+        // Also, set hashes to null since it's easier to request all hashes than it is to determine which ones we need
+        // This could be optimized in the future
+        long timestamp = now - 60000L;
+        List<byte[]> hashes = null;
+        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, timestamp, 0, null);
+
+        // Save our request into requests map
+        Triple<String, RNSPeer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
+
+        // Assign random ID to this message
+        int id;
+        do {
+            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
+
+            // Put queue into map (keyed by message ID) so we can poll for a response
+            // If putIfAbsent() doesn't return null, then this ID is already taken
+        } while (arbitraryDataFileListRequests.put(id, requestEntry) != null);
+        getArbitraryDataFileListMessage.setId(id);
+
+        // Send the request
+        peer.sendMessage(getArbitraryDataFileListMessage);
+
+        // Poll to see if data has arrived
+        final long singleWait = 100;
+        long totalWait = 0;
+        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
+            try {
+                Thread.sleep(singleWait);
+            } catch (InterruptedException e) {
+                break;
+            }
+
+            requestEntry = arbitraryDataFileListRequests.get(id);
+            if (requestEntry == null)
+                return false;
+
+            if (requestEntry.getA() == null)
+                break;
+
+            totalWait += singleWait;
+        }
+        return true;
+    }
+
+    public void deleteFileListRequestsForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        for (Iterator<Map.Entry<Integer, Triple<String, RNSPeer, Long>>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) {
+            Map.Entry<Integer, Triple<String, RNSPeer, Long>> entry = it.next();
+            if (entry == null || entry.getKey() == null || entry.getValue() != null) {
+                continue;
+            }
+            if (Objects.equals(entry.getValue().getA(), signature58)) {
+                // Update requests map to reflect that we've received all chunks
+                Triple<String, RNSPeer, Long> newEntry = new Triple<>(null, null, entry.getValue().getC());
+                arbitraryDataFileListRequests.put(entry.getKey(), newEntry);
+            }
+        }
+    }
+
+    // Network handlers
+
+    public void onNetworkArbitraryDataFileListMessage(RNSPeer peer, Message message) {
+        // Don't process if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
+        LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
+
+        if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) {
+            long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime();
+            LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}",
+                    totalRequestTime, arbitraryDataFileListMessage.getRequestHops(),
+                    arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
+        }
+
+        // Do we have a pending request for this data?
+        Triple<String, RNSPeer, Long> request = arbitraryDataFileListRequests.get(message.getId());
+        if (request == null || request.getA() == null) {
+            return;
+        }
+        boolean isRelayRequest = (request.getB() != null);
+
+        // Does this message's signature match what we're expecting?
+        byte[] signature = arbitraryDataFileListMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        if (!request.getA().equals(signature58)) {
+            return;
+        }
+
+        List<byte[]> hashes = arbitraryDataFileListMessage.getHashes();
+        if (hashes == null || hashes.isEmpty()) {
+            return;
+        }
+
+        ArbitraryTransactionData arbitraryTransactionData = null;
+
+        // Check transaction exists and hashes are correct
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+            if (!(transactionData instanceof ArbitraryTransactionData))
+                return;
+
+            arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+
+//          // Load data file(s)
+//          ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
+//
+//			// Check all hashes exist
+//			for (byte[] hash : hashes) {
+//				//LOGGER.debug("Received hash {}", Base58.encode(hash));
+//				if (!arbitraryDataFile.containsChunk(hash)) {
+//					// Check the hash against the complete file
+//					if (!Arrays.equals(arbitraryDataFile.getHash(), hash)) {
+//						LOGGER.info("Received non-matching chunk hash {} for signature {}. This could happen if we haven't obtained the metadata file yet.", Base58.encode(hash), signature58);
+//						return;
+//					}
+//				}
+//			}
+
+            if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) {
+                Long now = NTP.getTime();
+
+                if (RNSArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.size() < MAX_FILE_HASH_RESPONSES) {
+                    // Keep track of the hashes this peer reports to have access to
+                    for (byte[] hash : hashes) {
+                        String hash58 = Base58.encode(hash);
+
+                        // Treat null request hops as 100, so that they are able to be sorted (and put to the end of the list)
+                        int requestHops = arbitraryDataFileListMessage.getRequestHops() != null ? arbitraryDataFileListMessage.getRequestHops() : 100;
+
+                        RNSArbitraryFileListResponseInfo responseInfo = new RNSArbitraryFileListResponseInfo(hash58, signature58,
+                                peer, now, arbitraryDataFileListMessage.getRequestTime(), requestHops);
+
+                        RNSArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.add(responseInfo);
+                    }
+                }
+
+                // Keep track of the source peer, for direct connections
+                if (arbitraryDataFileListMessage.getPeerAddress() != null) {
+                    RNSArbitraryDataFileManager.getInstance().addDirectConnectionInfoIfUnique(
+                            new RNSArbitraryDirectConnectionInfo(signature, arbitraryDataFileListMessage.getPeerAddress(), hashes, now));
+                }
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e);
+        }
+
+        // Forwarding
+        if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
+            boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
+            if (!isBlocked) {
+                RNSPeer requestingPeer = request.getB();
+                if (requestingPeer != null) {
+                    Long requestTime = arbitraryDataFileListMessage.getRequestTime();
+                    Integer requestHops = arbitraryDataFileListMessage.getRequestHops();
+
+                    // Add each hash to our local mapping so we know who to ask later
+                    Long now = NTP.getTime();
+                    for (byte[] hash : hashes) {
+                        String hash58 = Base58.encode(hash);
+                        RNSArbitraryRelayInfo relayInfo = new RNSArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops);
+                        RNSArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo);
+                    }
+
+                    // Bump requestHops if it exists
+                    if (requestHops != null) {
+                        requestHops++;
+                    }
+
+                    ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage;
+
+                    //// TODO - rework for Reticulum
+                    //// Remove optional parameters if the requesting peer doesn't support it yet
+                    //// A message with less statistical data is better than no message at all
+                    //if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
+                    //    forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+                    //} else {
+                    //    forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops,
+                    //            arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
+                    //}
+                    forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops,
+                            arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
+                    forwardArbitraryDataFileListMessage.setId(message.getId());
+
+                    // Forward to requesting peer
+                    LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer);
+                    //if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) {
+                    //    requestingPeer.disconnect("failed to forward arbitrary data file list");
+                    //}
+                    requestingPeer.sendMessage(forwardArbitraryDataFileListMessage);
+                }
+            }
+        }
+    }
+
+    public void onNetworkGetArbitraryDataFileListMessage(RNSPeer peer, Message message) {
+        // Don't respond if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet();
+
+        GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message;
+        byte[] signature = getArbitraryDataFileListMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        Long now = NTP.getTime();
+        Triple<String, RNSPeer, Long> newEntry = new Triple<>(signature58, peer, now);
+
+        // If we've seen this request recently, then ignore
+        if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) {
+            LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58);
+            return;
+        }
+
+        List<byte[]> requestedHashes = getArbitraryDataFileListMessage.getHashes();
+        int hashCount = requestedHashes != null ? requestedHashes.size() : 0;
+        String requestingPeer = getArbitraryDataFileListMessage.getRequestingPeer();
+
+        if (requestingPeer != null) {
+            LOGGER.debug("Received hash list request with {} hashes from peer {} (requesting peer {}) for signature {}", hashCount, peer, requestingPeer, signature58);
+        }
+        else {
+            LOGGER.debug("Received hash list request with {} hashes from peer {} for signature {}", hashCount, peer, signature58);
+        }
+
+        List<byte[]> hashes = new ArrayList<>();
+        ArbitraryTransactionData transactionData = null;
+        boolean allChunksExist = false;
+        boolean hasMetadata = false;
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Firstly we need to lookup this file on chain to get a list of its hashes
+            transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature);
+            if (transactionData instanceof ArbitraryTransactionData) {
+
+                // Check if we're even allowed to serve data for this transaction
+                if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
+
+                    // Load file(s) and add any that exist to the list of hashes
+                    ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
+
+                    // If the peer didn't supply a hash list, we need to return all hashes for this transaction
+                    if (requestedHashes == null || requestedHashes.isEmpty()) {
+                        requestedHashes = new ArrayList<>();
+
+                        // Add the metadata file
+                        if (arbitraryDataFile.getMetadataHash() != null) {
+                            requestedHashes.add(arbitraryDataFile.getMetadataHash());
+                            hasMetadata = true;
+                        }
+
+                        // Add the chunk hashes
+                        if (!arbitraryDataFile.getChunkHashes().isEmpty()) {
+                            requestedHashes.addAll(arbitraryDataFile.getChunkHashes());
+                        }
+                        // Add complete file if there are no hashes
+                        else {
+                            requestedHashes.add(arbitraryDataFile.getHash());
+                        }
+                    }
+
+                    // Assume all chunks exists, unless one can't be found below
+                    allChunksExist = true;
+
+                    for (byte[] requestedHash : requestedHashes) {
+                        ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature);
+                        if (chunk.exists()) {
+                            hashes.add(chunk.getHash());
+                            //LOGGER.trace("Added hash {}", chunk.getHash58());
+                        } else {
+                            LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58());
+                            allChunksExist = false;
+                        }
+                    }
+                }
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e);
+        }
+
+        // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that,
+        // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam.
+        if (hasMetadata && hashes.size() == 1) {
+            hashes.clear();
+        }
+
+        // We should only respond if we have at least one hash
+        if (!hashes.isEmpty()) {
+
+            // Firstly we should keep track of the requesting peer, to allow for potential direct connections later
+            RNSArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer);
+
+            // We have all the chunks, so update requests map to reflect that we've sent it
+            // There is no need to keep track of the request, as we can serve all the chunks
+            if (allChunksExist) {
+                newEntry = new Triple<>(null, null, now);
+                arbitraryDataFileListRequests.put(message.getId(), newEntry);
+            }
+
+            //String ourAddress = RNSNetwork.getInstance().getOurExternalIpAddressAndPort();
+            String ourAddress = RNSNetwork.getInstance().getBaseDestination().getHexHash();
+            ArbitraryDataFileListMessage arbitraryDataFileListMessage;
+
+            // TODO: rework for Reticulum
+            // Remove optional parameters if the requesting peer doesn't support it yet
+            // A message with less statistical data is better than no message at all
+            //if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
+            //    arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+            //} else {
+            //    arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature,
+            //            hashes, NTP.getTime(), 0, ourAddress, true);
+            //}
+            arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+
+            arbitraryDataFileListMessage.setId(message.getId());
+
+            //if (!peer.sendMessage(arbitraryDataFileListMessage)) {
+            //    LOGGER.debug("Couldn't send list of hashes");
+            //    peer.disconnect("failed to send list of hashes");
+            //    return;
+            //}
+            peer.sendMessage(arbitraryDataFileListMessage);
+            LOGGER.debug("Sent list of hashes (count: {})", hashes.size());
+
+            if (allChunksExist) {
+                // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
+                LOGGER.debug("No need for any forwarding because file list request is fully served");
+                return;
+            }
+
+        }
+
+        // We may need to forward this request on
+        boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName()));
+        if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
+            // In relay mode - so ask our other peers if they have it
+
+            long requestTime = getArbitraryDataFileListMessage.getRequestTime();
+            int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1;
+            long totalRequestTime = now - requestTime;
+
+            if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
+                // Relay request hasn't timed out yet, so can potentially be rebroadcast
+                if (requestHops < RELAY_REQUEST_MAX_HOPS) {
+                    // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
+
+                    Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer);
+                    relayGetArbitraryDataFileListMessage.setId(message.getId());
+
+                    LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
+                    //Network.getInstance().broadcast(
+                    //        broadcastPeer ->
+                    //                !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null :
+                    //                broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryDataFileListMessage
+                    //);
+                    RNSNetwork.getInstance().broadcast(broadcastPeer -> relayGetArbitraryDataFileListMessage);
+
+                }
+                else {
+                    // This relay request has reached the maximum number of allowed hops
+                }
+            }
+            else {
+                // This relay request has timed out
+            }
+        }
+    }
+
+}

src/main/java/org/qortal/controller/arbitrary/RNSArbitraryDataFileManager.java

@@ -0,0 +1,639 @@
+package org.qortal.controller.arbitrary;
+
+import com.google.common.net.InetAddresses;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.RNSArbitraryDirectConnectionInfo;
+import org.qortal.data.arbitrary.RNSArbitraryFileListResponseInfo;
+import org.qortal.data.arbitrary.RNSArbitraryRelayInfo;
+import org.qortal.data.network.PeerData;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.*;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+
+import java.security.SecureRandom;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
+
+public class RNSArbitraryDataFileManager extends Thread {
+
+    private static final Logger LOGGER = LogManager.getLogger(RNSArbitraryDataFileManager.class);
+
+    private static RNSArbitraryDataFileManager instance;
+    private volatile boolean isStopping = false;
+
+
+    /**
+     * Map to keep track of our in progress (outgoing) arbitrary data file requests
+     */
+    public Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of hashes that we might need to relay
+     */
+    public final List<RNSArbitraryRelayInfo> arbitraryRelayMap = Collections.synchronizedList(new ArrayList<>());
+
+    /**
+     * List to keep track of any arbitrary data file hash responses
+     */
+    public final List<RNSArbitraryFileListResponseInfo> arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>());
+
+    /**
+     * List to keep track of peers potentially available for direct connections, based on recent requests
+     */
+    private final List<RNSArbitraryDirectConnectionInfo> directConnectionInfo = Collections.synchronizedList(new ArrayList<>());
+
+    /**
+     * Map to keep track of peers requesting QDN data that we hold.
+     * Key = peer address string, value = time of last request.
+     * This allows for additional "burst" connections beyond existing limits.
+     */
+    private Map<String, Long> recentDataRequests = Collections.synchronizedMap(new HashMap<>());
+
+
+    public static int MAX_FILE_HASH_RESPONSES = 1000;
+
+
+    private RNSArbitraryDataFileManager() {
+    }
+
+    public static RNSArbitraryDataFileManager getInstance() {
+        if (instance == null)
+            instance = new RNSArbitraryDataFileManager();
+
+        return instance;
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data File Manager");
+
+        try {
+            // Use a fixed thread pool to execute the arbitrary data file requests
+            int threadCount = 5;
+            ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount);
+            for (int i = 0; i < threadCount; i++) {
+                arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread());
+            }
+
+            while (!isStopping) {
+                // Nothing to do yet
+                Thread.sleep(1000);
+            }
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
+        }
+    }
+
+    public void shutdown() {
+        isStopping = true;
+        this.interrupt();
+    }
+
+
+    public void cleanupRequestCache(Long now) {
+        if (now == null) {
+            return;
+        }
+        final long requestMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_REQUEST_TIMEOUT;
+        arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < requestMinimumTimestamp);
+
+        final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT;
+        arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp);
+        arbitraryDataFileHashResponses.removeIf(entry -> entry.getTimestamp() < relayMinimumTimestamp);
+
+        final long directConnectionInfoMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_DIRECT_CONNECTION_INFO_TIMEOUT;
+        directConnectionInfo.removeIf(entry -> entry.getTimestamp() < directConnectionInfoMinimumTimestamp);
+
+        final long recentDataRequestMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RECENT_DATA_REQUESTS_TIMEOUT;
+        recentDataRequests.entrySet().removeIf(entry -> entry.getValue() < recentDataRequestMinimumTimestamp);
+    }
+
+
+
+    // Fetch data files by hash
+
+    public boolean fetchArbitraryDataFiles(Repository repository,
+                                           RNSPeer peer,
+                                           byte[] signature,
+                                           ArbitraryTransactionData arbitraryTransactionData,
+                                           List<byte[]> hashes) throws DataException {
+
+        // Load data file(s)
+        ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
+        boolean receivedAtLeastOneFile = false;
+
+        // Now fetch actual data from this peer
+        for (byte[] hash : hashes) {
+            if (isStopping) {
+                return false;
+            }
+            String hash58 = Base58.encode(hash);
+            if (!arbitraryDataFile.chunkExists(hash)) {
+                // Only request the file if we aren't already requesting it from someone else
+                if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
+                    LOGGER.debug("Requesting data file {} from peer {}", hash58, peer);
+                    Long startTime = NTP.getTime();
+                    ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, null, arbitraryTransactionData, signature, hash, null);
+                    Long endTime = NTP.getTime();
+                    if (receivedArbitraryDataFile != null) {
+                        LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFile.getHash58(), peer, (endTime-startTime));
+                        receivedAtLeastOneFile = true;
+
+                        // Remove this hash from arbitraryDataFileHashResponses now that we have received it
+                        arbitraryDataFileHashResponses.remove(hash58);
+                    }
+                    else {
+                        LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime));
+
+                        // Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it
+                        arbitraryDataFileHashResponses.remove(hash58);
+
+                        // Stop asking for files from this peer
+                        break;
+                    }
+                }
+                else {
+                    LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer);
+                }
+            }
+            else {
+                // Remove this hash from arbitraryDataFileHashResponses because we have a local copy
+                arbitraryDataFileHashResponses.remove(hash58);
+            }
+        }
+
+        if (receivedAtLeastOneFile) {
+            // Invalidate the hosted transactions cache as we are now hosting something new
+            ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache();
+
+            // Check if we have all the files we need for this transaction
+            if (arbitraryDataFile.allFilesExist()) {
+
+                // We have all the chunks for this transaction, so we should invalidate the transaction's name's
+                // data cache so that it is rebuilt the next time we serve it
+                ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData);
+            }
+        }
+
+        return receivedAtLeastOneFile;
+    }
+
+    private ArbitraryDataFile fetchArbitraryDataFile(RNSPeer peer, RNSPeer requestingPeer, ArbitraryTransactionData arbitraryTransactionData, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
+        ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature);
+        boolean fileAlreadyExists = existingFile.exists();
+        String hash58 = Base58.encode(hash);
+        ArbitraryDataFile arbitraryDataFile;
+
+        // Fetch the file if it doesn't exist locally
+        if (!fileAlreadyExists) {
+            LOGGER.debug(String.format("Fetching data file %.8s from peer %s", hash58, peer));
+            arbitraryDataFileRequests.put(hash58, NTP.getTime());
+            Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash);
+
+            Message response = null;
+            // TODO - revisit with RNS
+            try {
+                response = peer.getResponseWithTimeout(getArbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT);
+            } catch (InterruptedException e) {
+                // Will return below due to null response
+            }
+            arbitraryDataFileRequests.remove(hash58);
+            LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58));
+
+            // We may need to remove the file list request, if we have all the files for this transaction
+            this.handleFileListRequests(signature);
+
+            if (response == null) {
+                LOGGER.debug("Received null response from peer {}", peer);
+                return null;
+            }
+            if (response.getType() != MessageType.ARBITRARY_DATA_FILE) {
+                LOGGER.debug("Received response with invalid type: {} from peer {}", response.getType(), peer);
+                return null;
+            }
+
+            ArbitraryDataFileMessage peersArbitraryDataFileMessage = (ArbitraryDataFileMessage) response;
+            arbitraryDataFile = peersArbitraryDataFileMessage.getArbitraryDataFile();
+        } else {
+            LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58));
+            arbitraryDataFile = existingFile;
+        }
+
+        if (arbitraryDataFile == null) {
+            // We don't have a file, so give up here
+            return null;
+        }
+
+        // We might want to forward the request to the peer that originally requested it
+        this.handleArbitraryDataFileForwarding(requestingPeer, new ArbitraryDataFileMessage(signature, arbitraryDataFile), originalMessage);
+
+        boolean isRelayRequest = (requestingPeer != null);
+        if (isRelayRequest) {
+            if (!fileAlreadyExists) {
+                // File didn't exist locally before the request, and it's a forwarding request, so delete it if it exists.
+                // It shouldn't exist on the filesystem yet, but leaving this here just in case.
+                arbitraryDataFile.delete(10);
+            }
+        }
+        else {
+            arbitraryDataFile.save();
+        }
+
+        // If this is a metadata file then we need to update the cache
+        if (arbitraryTransactionData != null && arbitraryTransactionData.getMetadataHash() != null) {
+            if (Arrays.equals(arbitraryTransactionData.getMetadataHash(), hash)) {
+                ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
+            }
+        }
+
+        return arbitraryDataFile;
+    }
+
+    private void handleFileListRequests(byte[] signature) {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Fetch the transaction data
+            ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+            if (arbitraryTransactionData == null) {
+                return;
+            }
+
+            boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData);
+
+            if (allChunksExist) {
+                // Update requests map to reflect that we've received all chunks
+                RNSArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature);
+            }
+
+        } catch (DataException e) {
+            LOGGER.debug("Unable to handle file list requests: {}", e.getMessage());
+        }
+    }
+
+    public void handleArbitraryDataFileForwarding(RNSPeer requestingPeer, Message message, Message originalMessage) {
+        // Return if there is no originally requesting peer to forward to
+        if (requestingPeer == null) {
+            return;
+        }
+
+        // Return if we're not in relay mode or if this request doesn't need forwarding
+        if (!Settings.getInstance().isRelayModeEnabled()) {
+            return;
+        }
+
+        LOGGER.debug("Received arbitrary data file - forwarding is needed");
+
+        // The ID needs to match that of the original request
+        message.setId(originalMessage.getId());
+
+        //if (!requestingPeer.sendMessageWithTimeout(message, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT)) {
+        //    LOGGER.debug("Failed to forward arbitrary data file to peer {}", requestingPeer);
+        //    requestingPeer.disconnect("failed to forward arbitrary data file");
+        //}
+        //else {
+        //    LOGGER.debug("Forwarded arbitrary data file to peer {}", requestingPeer);
+        //}
+        requestingPeer.sendMessageWithTimeout(message, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT);
+    }
+
+
+    // Fetch data directly from peers
+
+    private List<RNSArbitraryDirectConnectionInfo> getDirectConnectionInfoForSignature(byte[] signature) {
+        synchronized (directConnectionInfo) {
+            return directConnectionInfo.stream().filter(i -> Arrays.equals(i.getSignature(), signature)).collect(Collectors.toList());
+        }
+    }
+
+    /**
+     * Add an ArbitraryDirectConnectionInfo item, but only if one with this peer-signature combination
+     * doesn't already exist.
+     * @param connectionInfo - the direct connection info to add
+     */
+    public void addDirectConnectionInfoIfUnique(RNSArbitraryDirectConnectionInfo connectionInfo) {
+        boolean peerAlreadyExists;
+        synchronized (directConnectionInfo) {
+            peerAlreadyExists = directConnectionInfo.stream()
+                    .anyMatch(i -> Arrays.equals(i.getSignature(), connectionInfo.getSignature())
+                            && Objects.equals(i.getPeerAddress(), connectionInfo.getPeerAddress()));
+        }
+        if (!peerAlreadyExists) {
+            directConnectionInfo.add(connectionInfo);
+        }
+    }
+
+    private void removeDirectConnectionInfo(RNSArbitraryDirectConnectionInfo connectionInfo) {
+        this.directConnectionInfo.remove(connectionInfo);
+    }
+
+    public boolean fetchDataFilesFromPeersForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+
+        boolean success = false;
+
+        try {
+            while (!success) {
+                if (isStopping) {
+                    return false;
+                }
+                Thread.sleep(500L);
+
+                // Firstly fetch peers that claim to be hosting files for this signature
+                List<RNSArbitraryDirectConnectionInfo> connectionInfoList = getDirectConnectionInfoForSignature(signature);
+                if (connectionInfoList == null || connectionInfoList.isEmpty()) {
+                    LOGGER.debug("No remaining direct connection peers found for signature {}", signature58);
+                    return false;
+                }
+
+                LOGGER.debug("Attempting a direct peer connection for signature {}...", signature58);
+
+                // Peers found, so pick one with the highest number of chunks
+                Comparator<RNSArbitraryDirectConnectionInfo> highestChunkCountFirstComparator =
+                        Comparator.comparingInt(RNSArbitraryDirectConnectionInfo::getHashCount).reversed();
+                RNSArbitraryDirectConnectionInfo directConnectionInfo = connectionInfoList.stream()
+                        .sorted(highestChunkCountFirstComparator).findFirst().orElse(null);
+
+                if (directConnectionInfo == null) {
+                    return false;
+                }
+
+                // Remove from the list so that a different peer is tried next time
+                removeDirectConnectionInfo(directConnectionInfo);
+
+                //// TODO - rework this section (RNS network address?)
+                //String peerAddressString = directConnectionInfo.getPeerAddress();
+                //
+                //// Parse the peer address to find the host and port
+                //String host = null;
+                //int port = -1;
+                //String[] parts = peerAddressString.split(":");
+                //if (parts.length > 1) {
+                //    host = parts[0];
+                //    port = Integer.parseInt(parts[1]);
+                //} else {
+                //    // Assume no port included
+                //    host = peerAddressString;
+                //    // Use default listen port
+                //    port = Settings.getInstance().getDefaultListenPort();
+                //}
+                //
+                //String peerAddressStringWithPort = String.format("%s:%d", host, port);
+                //success = Network.getInstance().requestDataFromPeer(peerAddressStringWithPort, signature);
+                //
+                //int defaultPort = Settings.getInstance().getDefaultListenPort();
+                //
+                //// If unsuccessful, and using a non-standard port, try a second connection with the default listen port,
+                //// since almost all nodes use that. This is a workaround to account for any ephemeral ports that may
+                //// have made it into the dataset.
+                //if (!success) {
+                //    if (host != null && port > 0) {
+                //        if (port != defaultPort) {
+                //            String newPeerAddressString = String.format("%s:%d", host, defaultPort);
+                //            success = Network.getInstance().requestDataFromPeer(newPeerAddressString, signature);
+                //        }
+                //    }
+                //}
+                //
+                //// If _still_ unsuccessful, try matching the peer's IP address with some known peers, and then connect
+                //// to each of those in turn until one succeeds.
+                //if (!success) {
+                //    if (host != null) {
+                //        final String finalHost = host;
+                //        List<PeerData> knownPeers = Network.getInstance().getAllKnownPeers().stream()
+                //                .filter(knownPeerData -> knownPeerData.getAddress().getHost().equals(finalHost))
+                //                .collect(Collectors.toList());
+                //        // Loop through each match and attempt a connection
+                //        for (PeerData matchingPeer : knownPeers) {
+                //            String matchingPeerAddress = matchingPeer.getAddress().toString();
+                //            int matchingPeerPort = matchingPeer.getAddress().getPort();
+                //            // Make sure that it's not a port we've already tried
+                //            if (matchingPeerPort != port && matchingPeerPort != defaultPort) {
+                //                success = Network.getInstance().requestDataFromPeer(matchingPeerAddress, signature);
+                //                if (success) {
+                //                    // Successfully connected, so stop making connections
+                //                    break;
+                //                }
+                //            }
+                //        }
+                //    }
+                //}
+
+                if (success) {
+                    // We were able to connect with a peer, so track the request
+                    RNSArbitraryDataFileListManager.getInstance().addToSignatureRequests(signature58, false, true);
+                }
+
+            }
+        } catch (InterruptedException e) {
+            // Do nothing
+        }
+
+        return success;
+    }
+
+
+    // Relays
+
+    private List<RNSArbitraryRelayInfo> getRelayInfoListForHash(String hash58) {
+        synchronized (arbitraryRelayMap) {
+            return arbitraryRelayMap.stream()
+                    .filter(relayInfo -> Objects.equals(relayInfo.getHash58(), hash58))
+                    .collect(Collectors.toList());
+        }
+    }
+
+    private RNSArbitraryRelayInfo getOptimalRelayInfoEntryForHash(String hash58) {
+        LOGGER.trace("Fetching relay info for hash: {}", hash58);
+        List<RNSArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
+        if (relayInfoList != null && !relayInfoList.isEmpty()) {
+
+            // Remove any with null requestHops
+            relayInfoList.removeIf(r -> r.getRequestHops() == null);
+
+            // If list is now empty, then just return one at random
+            if (relayInfoList.isEmpty()) {
+                return this.getRandomRelayInfoEntryForHash(hash58);
+            }
+
+            // Sort by number of hops (lowest first)
+            relayInfoList.sort(Comparator.comparingInt(RNSArbitraryRelayInfo::getRequestHops));
+
+            // FUTURE: secondary sort by requestTime?
+
+            RNSArbitraryRelayInfo relayInfo = relayInfoList.get(0);
+
+            LOGGER.trace("Returning optimal relay info for hash: {} (requestHops {})", hash58, relayInfo.getRequestHops());
+            return relayInfo;
+        }
+        LOGGER.trace("No relay info exists for hash: {}", hash58);
+        return null;
+    }
+
+    private RNSArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) {
+        LOGGER.trace("Fetching random relay info for hash: {}", hash58);
+        List<RNSArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
+        if (relayInfoList != null && !relayInfoList.isEmpty()) {
+
+            // Pick random item
+            int index = new SecureRandom().nextInt(relayInfoList.size());
+            LOGGER.trace("Returning random relay info for hash: {} (index {})", hash58, index);
+            return relayInfoList.get(index);
+        }
+        LOGGER.trace("No relay info exists for hash: {}", hash58);
+        return null;
+    }
+
+    public void addToRelayMap(RNSArbitraryRelayInfo newEntry) {
+        if (newEntry == null || !newEntry.isValid()) {
+            return;
+        }
+
+        // Remove existing entry for this peer if it exists, to renew the timestamp
+        this.removeFromRelayMap(newEntry);
+
+        // Re-add
+        arbitraryRelayMap.add(newEntry);
+        LOGGER.debug("Added entry to relay map: {}", newEntry);
+    }
+
+    private void removeFromRelayMap(RNSArbitraryRelayInfo entry) {
+        arbitraryRelayMap.removeIf(relayInfo -> relayInfo.equals(entry));
+    }
+
+
+    // Peers requesting QDN data from us
+
+    /**
+     * Add an address string of a peer that is trying to request data from us.
+     * @param peerAddress
+     */
+    public void addRecentDataRequest(String peerAddress) {
+        if (peerAddress == null) {
+            return;
+        }
+
+        Long now = NTP.getTime();
+        if (now == null) {
+            return;
+        }
+
+        // Make sure to remove the port, since it isn't guaranteed to match next time
+        String[] parts = peerAddress.split(":");
+        if (parts.length == 0) {
+            return;
+        }
+        String host = parts[0];
+        if (!InetAddresses.isInetAddress(host)) {
+            // Invalid host
+            return;
+        }
+
+        this.recentDataRequests.put(host, now);
+    }
+
+    public boolean isPeerRequestingData(String peerAddressWithoutPort) {
+        return this.recentDataRequests.containsKey(peerAddressWithoutPort);
+    }
+
+    public boolean hasPendingDataRequest() {
+        return !this.recentDataRequests.isEmpty();
+    }
+
+
+    // Network handlers
+
+    public void onNetworkGetArbitraryDataFileMessage(RNSPeer peer, Message message) {
+        // Don't respond if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message;
+        byte[] hash = getArbitraryDataFileMessage.getHash();
+        String hash58 = Base58.encode(hash);
+        byte[] signature = getArbitraryDataFileMessage.getSignature();
+        Controller.getInstance().stats.getArbitraryDataFileMessageStats.requests.incrementAndGet();
+
+        LOGGER.debug("Received GetArbitraryDataFileMessage from peer {} for hash {}", peer, Base58.encode(hash));
+
+        try {
+            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
+            RNSArbitraryRelayInfo relayInfo = this.getOptimalRelayInfoEntryForHash(hash58);
+
+            if (arbitraryDataFile.exists()) {
+                LOGGER.trace("Hash {} exists", hash58);
+
+                // We can serve the file directly as we already have it
+                LOGGER.debug("Sending file {}...", arbitraryDataFile);
+                ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, arbitraryDataFile);
+                arbitraryDataFileMessage.setId(message.getId());
+                //if (!peer.sendMessageWithTimeout(arbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT)) {
+                //    LOGGER.debug("Couldn't send file {}", arbitraryDataFile);
+                //    peer.disconnect("failed to send file");
+                //}
+                //else {
+                //    LOGGER.debug("Sent file {}", arbitraryDataFile);
+                //}
+                peer.sendMessageWithTimeout(arbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT);
+            }
+            //// TODO: rework (doesn't work with Reticulum)
+            //else if (relayInfo != null) {
+            //    LOGGER.debug("We have relay info for hash {}", Base58.encode(hash));
+            //    // We need to ask this peer for the file
+            //    Peer peerToAsk = relayInfo.getPeer();
+            //    if (peerToAsk != null) {
+            //
+            //        // Forward the message to this peer
+            //        LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58);
+            //        // No need to pass arbitraryTransactionData below because this is only used for metadata caching,
+            //        // and metadata isn't retained when relaying.
+            //        this.fetchArbitraryDataFile(peerToAsk, peer, null, signature, hash, message);
+            //    }
+            //    else {
+            //        LOGGER.debug("Peer {} not found in relay info", peer);
+            //    }
+            //}
+            else {
+                LOGGER.debug("Hash {} doesn't exist and we don't have relay info", hash58);
+
+                // We don't have this file
+                Controller.getInstance().stats.getArbitraryDataFileMessageStats.unknownFiles.getAndIncrement();
+
+                // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout
+                LOGGER.debug(String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile));
+
+                //// Send generic 'unknown' message as it's very short
+                //Message fileUnknownMessage = peer.getPeersVersion() >= GenericUnknownMessage.MINIMUM_PEER_VERSION
+                //        ? new GenericUnknownMessage()
+                //        : new BlockSummariesMessage(Collections.emptyList());
+                //fileUnknownMessage.setId(message.getId());
+                //if (!peer.sendMessage(fileUnknownMessage)) {
+                //    LOGGER.debug("Couldn't sent file-unknown response");
+                //    peer.disconnect("failed to send file-unknown response");
+                //}
+                //else {
+                //    LOGGER.debug("Sent file-unknown response for file {}", arbitraryDataFile);
+                //}
+                Message fileUnknownMessage = new GenericUnknownMessage();
+                peer.sendMessage(fileUnknownMessage);
+            }
+        }
+        catch (DataException e) {
+            LOGGER.debug("Unable to handle request for arbitrary data file: {}", hash58);
+        }
+    }
+
+}

src/main/java/org/qortal/controller/arbitrary/RNSArbitraryDataFileRequestThread.java

@@ -0,0 +1,130 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.RNSArbitraryFileListResponseInfo;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.event.DataMonitorEvent;
+import org.qortal.event.EventBus;
+import org.qortal.network.RNSPeer;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Iterator;
+
+import static java.lang.Thread.NORM_PRIORITY;
+
+public class RNSArbitraryDataFileRequestThread implements Runnable {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class);
+
+    public RNSArbitraryDataFileRequestThread() {
+
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data File Request Thread");
+        Thread.currentThread().setPriority(NORM_PRIORITY);
+
+        try {
+            while (!Controller.isStopping()) {
+                Long now = NTP.getTime();
+                this.processFileHashes(now);
+            }
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
+        }
+    }
+
+    private void processFileHashes(Long now) throws InterruptedException {
+		if (Controller.isStopping()) {
+            return;
+        }
+
+        RNSArbitraryDataFileManager arbitraryDataFileManager = RNSArbitraryDataFileManager.getInstance();
+        String signature58 = null;
+        String hash58 = null;
+        RNSPeer peer = null;
+        boolean shouldProcess = false;
+
+        synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) {
+            if (!arbitraryDataFileManager.arbitraryDataFileHashResponses.isEmpty()) {
+
+                // Sort by lowest number of node hops first
+                Comparator<RNSArbitraryFileListResponseInfo> lowestHopsFirstComparator =
+                        Comparator.comparingInt(RNSArbitraryFileListResponseInfo::getRequestHops);
+                arbitraryDataFileManager.arbitraryDataFileHashResponses.sort(lowestHopsFirstComparator);
+
+                Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.iterator();
+                while (iterator.hasNext()) {
+                    if (Controller.isStopping()) {
+                        return;
+                    }
+
+                    RNSArbitraryFileListResponseInfo responseInfo = (RNSArbitraryFileListResponseInfo) iterator.next();
+                    if (responseInfo == null) {
+                        iterator.remove();
+                        continue;
+                    }
+
+                    hash58 = responseInfo.getHash58();
+                    peer = responseInfo.getPeer();
+                    signature58 = responseInfo.getSignature58();
+                    Long timestamp = responseInfo.getTimestamp();
+
+                    if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) {
+                        // Ignore - to be deleted
+                        iterator.remove();
+                        continue;
+                    }
+
+                    // Skip if already requesting, but don't remove, as we might want to retry later
+                    if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) {
+                        // Already requesting - leave this attempt for later
+                        continue;
+                    }
+
+                    // We want to process this file
+                    shouldProcess = true;
+                    iterator.remove();
+                    break;
+                }
+            }
+        }
+
+        if (!shouldProcess) {
+            // Nothing to do
+            Thread.sleep(1000L);
+            return;
+        }
+
+        byte[] hash = Base58.decode(hash58);
+        byte[] signature = Base58.decode(signature58);
+
+        // Fetch the transaction data
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+            if (arbitraryTransactionData == null) {
+                return;
+            }
+
+            if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) {
+                return;
+            }
+
+            LOGGER.trace("Fetching file {} from peer {} via request thread...", hash58, peer);
+            arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash));
+
+        } catch (DataException e) {
+            LOGGER.debug("Unable to process file hashes: {}", e.getMessage());
+        }
+    }
+}

src/main/java/org/qortal/controller/arbitrary/RNSArbitraryMetadataManager.java

@@ -0,0 +1,481 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.arbitrary.ArbitraryDataResource;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.controller.Controller;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.ArbitraryMetadataMessage;
+import org.qortal.network.message.GetArbitraryMetadataMessage;
+import org.qortal.network.message.Message;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+import org.qortal.utils.ListUtils;
+import org.qortal.utils.NTP;
+import org.qortal.utils.Triple;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.qortal.controller.arbitrary.ArbitraryDataFileListManager.*;
+
+public class RNSArbitraryMetadataManager {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryMetadataManager.class);
+
+    private static RNSArbitraryMetadataManager instance;
+
+    /**
+     * Map of recent incoming requests for ARBITRARY transaction metadata.
+     * <p>
+     * Key is original request's message ID<br>
+     * Value is Triple&lt;transaction signature in base58, first requesting peer, first request's timestamp&gt;
+     * <p>
+     * If peer is null then either:<br>
+     * <ul>
+     * <li>we are the original requesting peer</li>
+     * <li>we have already sent data payload to original requesting peer.</li>
+     * </ul>
+     * If signature is null then we have already received the file list and either:<br>
+     * <ul>
+     * <li>we are the original requesting peer and have processed it</li>
+     * <li>we have forwarded the metadata</li>
+     * </ul>
+     */
+    public Map<Integer, Triple<String, RNSPeer, Long>> arbitraryMetadataRequests = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of in progress arbitrary metadata requests
+     * Key: string - the signature encoded in base58
+     * Value: Triple<networkBroadcastCount, directPeerRequestCount, lastAttemptTimestamp>
+     */
+    private Map<String, Triple<Integer, Integer, Long>> arbitraryMetadataSignatureRequests = Collections.synchronizedMap(new HashMap<>());
+
+
+    private RNSArbitraryMetadataManager() {
+    }
+
+    public static RNSArbitraryMetadataManager getInstance() {
+        if (instance == null)
+            instance = new RNSArbitraryMetadataManager();
+
+        return instance;
+    }
+
+    public void cleanupRequestCache(Long now) {
+        if (now == null) {
+            return;
+        }
+        final long requestMinimumTimestamp = now - ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT;
+        arbitraryMetadataRequests.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < requestMinimumTimestamp);
+    }
+
+
+    public ArbitraryDataTransactionMetadata fetchMetadata(ArbitraryDataResource arbitraryDataResource, boolean useRateLimiter) {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            // Find latest transaction
+            ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
+                    .getLatestTransaction(arbitraryDataResource.getResourceId(), arbitraryDataResource.getService(),
+                            null, arbitraryDataResource.getIdentifier());
+
+            if (latestTransaction != null) {
+                byte[] signature = latestTransaction.getSignature();
+                byte[] metadataHash = latestTransaction.getMetadataHash();
+                if (metadataHash == null) {
+                    // This resource doesn't have metadata
+                    throw new IllegalArgumentException("This resource doesn't have metadata");
+                }
+
+                ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+                if (!metadataFile.exists()) {
+                    // Request from network
+                    this.fetchArbitraryMetadata(latestTransaction, useRateLimiter);
+                }
+
+                // Now check again as it may have been downloaded above
+                if (metadataFile.exists()) {
+                    // Use local copy
+                    ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
+                    try {
+                        transactionMetadata.read();
+                    } catch (DataException e) {
+                        // Invalid file, so delete it
+                        LOGGER.info("Deleting invalid metadata file due to exception: {}", e.getMessage());
+                        transactionMetadata.delete();
+                        return null;
+                    }
+                    return transactionMetadata;
+                }
+            }
+
+        } catch (DataException | IOException e) {
+            LOGGER.error("Repository issue when fetching arbitrary transaction metadata", e);
+        }
+
+        return null;
+    }
+
+
+    // Request metadata from network
+
+    public byte[] fetchArbitraryMetadata(ArbitraryTransactionData arbitraryTransactionData, boolean useRateLimiter) {
+        byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
+        if (metadataHash == null) {
+            return null;
+        }
+
+        byte[] signature = arbitraryTransactionData.getSignature();
+        String signature58 = Base58.encode(signature);
+
+        // Require an NTP sync
+        Long now = NTP.getTime();
+        if (now == null) {
+            return null;
+        }
+
+        // If we've already tried too many times in a short space of time, make sure to give up
+        if (useRateLimiter && !this.shouldMakeMetadataRequestForSignature(signature58)) {
+            LOGGER.trace("Skipping metadata request for signature {} due to rate limit", signature58);
+            return null;
+        }
+        this.addToSignatureRequests(signature58, true, false);
+
+        //List<Peer> handshakedPeers = Network.getInstance().getImmutableHandshakedPeers();
+        List<RNSPeer> handshakedPeers = RNSNetwork.getInstance().getLinkedPeers();
+        LOGGER.debug(String.format("Sending metadata request for signature %s to %d peers...", signature58, handshakedPeers.size()));
+
+        // Build request
+        Message getArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, now, 0);
+
+        // Save our request into requests map
+        Triple<String, RNSPeer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
+
+        // Assign random ID to this message
+        int id;
+        do {
+            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
+
+            // Put queue into map (keyed by message ID) so we can poll for a response
+            // If putIfAbsent() doesn't return null, then this ID is already taken
+        } while (arbitraryMetadataRequests.put(id, requestEntry) != null);
+        getArbitraryMetadataMessage.setId(id);
+
+        // Broadcast request
+        RNSNetwork.getInstance().broadcast(peer -> getArbitraryMetadataMessage);
+
+        // Poll to see if data has arrived
+        final long singleWait = 100;
+        long totalWait = 0;
+        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
+            try {
+                Thread.sleep(singleWait);
+            } catch (InterruptedException e) {
+                break;
+            }
+
+            requestEntry = arbitraryMetadataRequests.get(id);
+            if (requestEntry == null)
+                return null;
+
+            if (requestEntry.getA() == null)
+                break;
+
+            totalWait += singleWait;
+        }
+
+        try {
+            ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+            if (metadataFile.exists()) {
+                return metadataFile.getBytes();
+            }
+        } catch (DataException e) {
+            // Do nothing
+        }
+
+        return null;
+    }
+
+
+    // Track metadata lookups by signature
+
+    private boolean shouldMakeMetadataRequestForSignature(String signature58) {
+        Triple<Integer, Integer, Long> request = arbitraryMetadataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        // Extract the components
+        Integer networkBroadcastCount = request.getA();
+        // Integer directPeerRequestCount = request.getB();
+        Long lastAttemptTimestamp = request.getC();
+
+        if (lastAttemptTimestamp == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
+
+        // Allow a second attempt after 60 seconds
+        if (timeSinceLastAttempt > 60 * 1000L) {
+            // We haven't tried for at least 60 seconds
+
+            if (networkBroadcastCount < 2) {
+                // We've made less than 2 total attempts
+                return true;
+            }
+        }
+
+        // Then allow another attempt after 60 minutes
+        if (timeSinceLastAttempt > 60 * 60 * 1000L) {
+            // We haven't tried for at least 60 minutes
+
+            if (networkBroadcastCount < 3) {
+                // We've made less than 3 total attempts
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    public boolean isSignatureRateLimited(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        return !this.shouldMakeMetadataRequestForSignature(signature58);
+    }
+
+    public long lastRequestForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        Triple<Integer, Integer, Long> request = arbitraryMetadataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return 0;
+        }
+
+        // Extract the components
+        Long lastAttemptTimestamp = request.getC();
+        if (lastAttemptTimestamp != null) {
+            return  lastAttemptTimestamp;
+        }
+        return 0;
+    }
+
+    public void addToSignatureRequests(String signature58, boolean incrementNetworkRequests, boolean incrementPeerRequests) {
+        Triple<Integer, Integer, Long> request  = arbitraryMetadataSignatureRequests.get(signature58);
+        Long now = NTP.getTime();
+
+        if (request == null) {
+            // No entry yet
+            Triple<Integer, Integer, Long> newRequest = new Triple<>(0, 0, now);
+            arbitraryMetadataSignatureRequests.put(signature58, newRequest);
+        }
+        else {
+            // There is an existing entry
+            if (incrementNetworkRequests) {
+                request.setA(request.getA() + 1);
+            }
+            if (incrementPeerRequests) {
+                request.setB(request.getB() + 1);
+            }
+            request.setC(now);
+            arbitraryMetadataSignatureRequests.put(signature58, request);
+        }
+    }
+
+    public void removeFromSignatureRequests(String signature58) {
+        arbitraryMetadataSignatureRequests.remove(signature58);
+    }
+
+
+    // Network handlers
+
+    public void onNetworkArbitraryMetadataMessage(RNSPeer peer, Message message) {
+        // Don't process if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        ArbitraryMetadataMessage arbitraryMetadataMessage = (ArbitraryMetadataMessage) message;
+        LOGGER.debug("Received metadata from peer {}", peer);
+
+        // Do we have a pending request for this data?
+        Triple<String, RNSPeer, Long> request = arbitraryMetadataRequests.get(message.getId());
+        if (request == null || request.getA() == null) {
+            return;
+        }
+        boolean isRelayRequest = (request.getB() != null);
+
+        // Does this message's signature match what we're expecting?
+        byte[] signature = arbitraryMetadataMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        if (!request.getA().equals(signature58)) {
+            return;
+        }
+
+        // Update requests map to reflect that we've received this metadata
+        Triple<String, RNSPeer, Long> newEntry = new Triple<>(null, null, request.getC());
+        arbitraryMetadataRequests.put(message.getId(), newEntry);
+
+        // Get transaction info
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+            if (!(transactionData instanceof ArbitraryTransactionData)) {
+                return;
+            }
+            ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+
+            // Check if the name is blocked
+            boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
+
+            // Save if not blocked
+            ArbitraryDataFile arbitraryMetadataFile = arbitraryMetadataMessage.getArbitraryMetadataFile();
+            if (!isBlocked && arbitraryMetadataFile != null) {
+                arbitraryMetadataFile.save();
+            }
+
+            // Forwarding
+            if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
+                if (!isBlocked) {
+                    RNSPeer requestingPeer = request.getB();
+                    if (requestingPeer != null) {
+
+                        ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile());
+                        forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId());
+
+                        // Forward to requesting peer
+                        LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
+                        //if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
+                        //    requestingPeer.disconnect("failed to forward arbitrary metadata");
+                        //}
+                        requestingPeer.sendMessage(forwardArbitraryMetadataMessage);
+                    }
+                }
+            }
+
+            // Add to resource queue to update arbitrary resource caches
+            if (arbitraryTransactionData != null) {
+                ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while saving arbitrary transaction metadata from peer %s", peer), e);
+        }
+    }
+
+    public void onNetworkGetArbitraryMetadataMessage(RNSPeer peer, Message message) {
+        // Don't respond if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet();
+
+        GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) message;
+        byte[] signature = getArbitraryMetadataMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        Long now = NTP.getTime();
+        Triple<String, RNSPeer, Long> newEntry = new Triple<>(signature58, peer, now);
+
+        // If we've seen this request recently, then ignore
+        if (arbitraryMetadataRequests.putIfAbsent(message.getId(), newEntry) != null) {
+            LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peer, signature58);
+            return;
+        }
+
+        LOGGER.debug("Received metadata request from peer {} for signature {}", peer, signature58);
+
+        ArbitraryTransactionData transactionData = null;
+        ArbitraryDataFile metadataFile = null;
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Firstly we need to lookup this file on chain to get its metadata hash
+            transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature);
+            if (transactionData instanceof ArbitraryTransactionData) {
+
+                // Check if we're even allowed to serve metadata for this transaction
+                if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
+
+                    byte[] metadataHash = transactionData.getMetadataHash();
+                    if (metadataHash != null) {
+
+                        // Load metadata file
+                        metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+                    }
+                }
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while fetching arbitrary metadata for peer %s", peer), e);
+        }
+
+        // We should only respond if we have the metadata file
+        if (metadataFile != null && metadataFile.exists()) {
+
+            // We have the metadata file, so update requests map to reflect that we've sent it
+            newEntry = new Triple<>(null, null, now);
+            arbitraryMetadataRequests.put(message.getId(), newEntry);
+
+            ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, metadataFile);
+            arbitraryMetadataMessage.setId(message.getId());
+            //if (!peer.sendMessage(arbitraryMetadataMessage)) {
+            //    LOGGER.debug("Couldn't send metadata");
+            //    peer.disconnect("failed to send metadata");
+            //    return;
+            //}
+            peer.sendMessage(arbitraryMetadataMessage);
+            LOGGER.debug("Sent metadata");
+
+            // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
+            LOGGER.debug("No need for any forwarding because metadata request is fully served");
+            return;
+
+        }
+
+        // We may need to forward this request on
+        boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName()));
+        if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
+            // In relay mode - so ask our other peers if they have it
+
+            long requestTime = getArbitraryMetadataMessage.getRequestTime();
+            int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1;
+            long totalRequestTime = now - requestTime;
+
+            if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
+                // Relay request hasn't timed out yet, so can potentially be rebroadcast
+                if (requestHops < RELAY_REQUEST_MAX_HOPS) {
+                    // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
+
+                    Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops);
+                    relayGetArbitraryMetadataMessage.setId(message.getId());
+
+                    LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
+                    //Network.getInstance().broadcast(
+                    //        broadcastPeer ->
+                    //                !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null :
+                    //                broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryMetadataMessage);
+                    RNSNetwork.getInstance().broadcast(broadcastPeer -> relayGetArbitraryMetadataMessage);
+
+                }
+                else {
+                    // This relay request has reached the maximum number of allowed hops
+                }
+            }
+            else {
+                // This relay request has timed out
+            }
+        }
+    }
+
+}

src/main/java/org/qortal/controller/tradebot/RNSTradeBot.java

@@ -0,0 +1,778 @@
+package org.qortal.controller.tradebot;
+
+import com.google.common.primitives.Longs;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.bitcoinj.core.ECKey;
+import org.qortal.account.PrivateKeyAccount;
+import org.qortal.api.model.crosschain.TradeBotCreateRequest;
+import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
+import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult;
+import org.qortal.crosschain.*;
+import org.qortal.crypto.Crypto;
+import org.qortal.data.at.ATData;
+import org.qortal.data.crosschain.CrossChainTradeData;
+import org.qortal.data.crosschain.TradeBotData;
+import org.qortal.data.network.TradePresenceData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.event.Event;
+import org.qortal.event.EventBus;
+import org.qortal.event.Listener;
+import org.qortal.gui.SysTray;
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.GetTradePresencesMessage;
+import org.qortal.network.message.Message;
+import org.qortal.network.message.TradePresencesMessage;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.repository.hsqldb.HSQLDBImportExport;
+import org.qortal.settings.Settings;
+import org.qortal.transaction.Transaction;
+import org.qortal.utils.ByteArray;
+import org.qortal.utils.NTP;
+
+import java.awt.TrayIcon.MessageType;
+import java.security.SecureRandom;
+import java.util.*;
+import java.util.function.Supplier;
+
+/**
+ * Performing cross-chain trading steps on behalf of user.
+ * <p>
+ * We deal with three different independent state-spaces here:
+ * <ul>
+ * 	<li>Qortal blockchain</li>
+ * 	<li>Foreign blockchain</li>
+ * 	<li>Trade-bot entries</li>
+ * </ul>
+ */
+public class RNSTradeBot implements Listener {
+
+	private static final Logger LOGGER = LogManager.getLogger(TradeBot.class);
+	private static final Random RANDOM = new SecureRandom();
+
+	/** Maximum lifetime of trade presence timestamp. 30 mins in ms. */
+	private static final long PRESENCE_LIFETIME = 30 * 60 * 1000L;
+	/** How soon before expiry of our own trade presence timestamp that we want to trigger renewal. 5 mins in ms. */
+	private static final long EARLY_RENEWAL_PERIOD = 5 * 60 * 1000L;
+	/** Trade presence timestamps are rounded up to this nearest interval. Bigger values improve grouping of entries in [GET_]TRADE_PRESENCES network messages. 15 mins in ms. */
+	private static final long EXPIRY_ROUNDING = 15 * 60 * 1000L;
+	/** How often we want to broadcast our list of all known trade presences to peers. 5 mins in ms. */
+	private static final long PRESENCE_BROADCAST_INTERVAL = 5 * 60 * 1000L;
+
+	public interface StateNameAndValueSupplier {
+		public String getState();
+		public int getStateValue();
+	}
+
+	public static class StateChangeEvent implements Event {
+		private final TradeBotData tradeBotData;
+
+		public StateChangeEvent(TradeBotData tradeBotData) {
+			this.tradeBotData = tradeBotData;
+		}
+
+		public TradeBotData getTradeBotData() {
+			return this.tradeBotData;
+		}
+	}
+
+	public static class TradePresenceEvent implements Event {
+		private final TradePresenceData tradePresenceData;
+
+		public TradePresenceEvent(TradePresenceData tradePresenceData) {
+			this.tradePresenceData = tradePresenceData;
+		}
+
+		public TradePresenceData getTradePresenceData() {
+			return this.tradePresenceData;
+		}
+	}
+
+	private static final Map<Class<? extends ACCT>, Supplier<AcctTradeBot>> acctTradeBotSuppliers = new HashMap<>();
+	static {
+		acctTradeBotSuppliers.put(BitcoinACCTv1.class, BitcoinACCTv1TradeBot::getInstance);
+		acctTradeBotSuppliers.put(BitcoinACCTv3.class, BitcoinACCTv3TradeBot::getInstance);
+		acctTradeBotSuppliers.put(LitecoinACCTv1.class, LitecoinACCTv1TradeBot::getInstance);
+		acctTradeBotSuppliers.put(LitecoinACCTv3.class, LitecoinACCTv3TradeBot::getInstance);
+		acctTradeBotSuppliers.put(DogecoinACCTv1.class, DogecoinACCTv1TradeBot::getInstance);
+		acctTradeBotSuppliers.put(DogecoinACCTv3.class, DogecoinACCTv3TradeBot::getInstance);
+		acctTradeBotSuppliers.put(DigibyteACCTv3.class, DigibyteACCTv3TradeBot::getInstance);
+		acctTradeBotSuppliers.put(RavencoinACCTv3.class, RavencoinACCTv3TradeBot::getInstance);
+		acctTradeBotSuppliers.put(PirateChainACCTv3.class, PirateChainACCTv3TradeBot::getInstance);
+	}
+
+	private static RNSTradeBot instance;
+
+	private final Map<ByteArray, Long> ourTradePresenceTimestampsByPubkey = Collections.synchronizedMap(new HashMap<>());
+	private final List<TradePresenceData> pendingTradePresences = Collections.synchronizedList(new ArrayList<>());
+
+	private final Map<ByteArray, TradePresenceData> allTradePresencesByPubkey = Collections.synchronizedMap(new HashMap<>());
+	private Map<ByteArray, TradePresenceData> safeAllTradePresencesByPubkey = Collections.emptyMap();
+	private long nextTradePresenceBroadcastTimestamp = 0L;
+
+	private Map<String, Long> failedTrades = new HashMap<>();
+	private Map<String, Long> validTrades = new HashMap<>();
+
+	private RNSTradeBot() {
+		EventBus.INSTANCE.addListener(event -> RNSTradeBot.getInstance().listen(event));
+	}
+
+	public static synchronized RNSTradeBot getInstance() {
+		if (instance == null)
+			instance = new RNSTradeBot();
+
+		return instance;
+	}
+
+	public ACCT getAcctUsingAtData(ATData atData) {
+		byte[] codeHash = atData.getCodeHash();
+		if (codeHash == null)
+			return null;
+
+		return SupportedBlockchain.getAcctByCodeHash(codeHash);
+	}
+
+	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
+		ACCT acct = this.getAcctUsingAtData(atData);
+		if (acct == null)
+			return null;
+
+		return acct.populateTradeData(repository, atData);
+	}
+
+	/**
+	 * Creates a new trade-bot entry from the "Bob" viewpoint,
+	 * i.e. OFFERing QORT in exchange for foreign blockchain currency.
+	 * <p>
+	 * Generates:
+	 * <ul>
+	 * 	<li>new 'trade' private key</li>
+	 * 	<li>secret(s)</li>
+	 * </ul>
+	 * Derives:
+	 * <ul>
+	 * 	<li>'native' (as in Qortal) public key, public key hash, address (starting with Q)</li>
+	 * 	<li>'foreign' public key, public key hash</li>
+	 *	<li>hash(es) of secret(s)</li>
+	 * </ul>
+	 * A Qortal AT is then constructed including the following as constants in the 'data segment':
+	 * <ul>
+	 * 	<li>'native' (Qortal) 'trade' address - used to MESSAGE AT</li>
+	 * 	<li>'foreign' public key hash - used by Alice's to allow redeem of currency on foreign blockchain</li>
+	 * 	<li>hash(es) of secret(s) - used by AT (optional) and foreign blockchain as needed</li>
+	 * 	<li>QORT amount on offer by Bob</li>
+	 * 	<li>foreign currency amount expected in return by Bob (from Alice)</li>
+	 * 	<li>trading timeout, in case things go wrong and everyone needs to refund</li>
+	 * </ul>
+	 * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network.
+	 * <p>
+	 * Trade-bot will wait for Bob's AT to be deployed before taking next step.
+	 * <p>
+	 * @param repository
+	 * @param tradeBotCreateRequest
+	 * @return raw, unsigned DEPLOY_AT transaction
+	 * @throws DataException
+	 */
+	public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException {
+		// Fetch latest ACCT version for requested foreign blockchain
+		ACCT acct = tradeBotCreateRequest.foreignBlockchain.getLatestAcct();
+
+		AcctTradeBot acctTradeBot = findTradeBotForAcct(acct);
+		if (acctTradeBot == null)
+			return null;
+
+		return acctTradeBot.createTrade(repository, tradeBotCreateRequest);
+	}
+
+	/**
+	 * Creates a trade-bot entry from the 'Alice' viewpoint,
+	 * i.e. matching foreign blockchain currency to an existing QORT offer.
+	 * <p>
+	 * Requires a chosen trade offer from Bob, passed by <tt>crossChainTradeData</tt>
+	 * and access to a foreign blockchain wallet via <tt>foreignKey</tt>.
+	 * <p>
+	 * @param repository
+	 * @param crossChainTradeData chosen trade OFFER that Alice wants to match
+	 * @param foreignKey foreign blockchain wallet key
+	 * @throws DataException
+	 */
+	public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct,
+			CrossChainTradeData crossChainTradeData, String foreignKey, String receivingAddress) throws DataException {
+		AcctTradeBot acctTradeBot = findTradeBotForAcct(acct);
+		if (acctTradeBot == null) {
+			LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot for AT %s", atData.getATAddress()));
+			return ResponseResult.NETWORK_ISSUE;
+		}
+
+		// Check Alice doesn't already have an existing, on-going trade-bot entry for this AT.
+		if (repository.getCrossChainRepository().existsTradeWithAtExcludingStates(atData.getATAddress(), acctTradeBot.getEndStates()))
+			return ResponseResult.TRADE_ALREADY_EXISTS;
+
+		return acctTradeBot.startResponse(repository, atData, acct, crossChainTradeData, foreignKey, receivingAddress);
+	}
+
+	/**
+	 * Creates a trade-bot entries from the 'Alice' viewpoint,
+	 * i.e. matching foreign blockchain currency to existing QORT offers.
+	 * <p>
+	 * Requires chosen trade offers from Bob, passed by <tt>crossChainTradeData</tt>
+	 * and access to a foreign blockchain wallet via <tt>foreignKey</tt>.
+	 * <p>
+	 * @param repository
+	 * @param crossChainTradeDataList chosen trade OFFERs that Alice wants to match
+	 * @param receiveAddress Alice's Qortal address to receive her QORT
+	 * @param foreignKey foreign blockchain wallet key
+	 * @param bitcoiny
+	 * @throws DataException
+	 */
+	public ResponseResult startResponseMultiple(
+			Repository repository,
+			ACCT acct,
+			List<CrossChainTradeData> crossChainTradeDataList,
+			String receiveAddress,
+			String foreignKey,
+			Bitcoiny bitcoiny) throws DataException {
+		AcctTradeBot acctTradeBot = findTradeBotForAcct(acct);
+		if (acctTradeBot == null) {
+			LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot for %s", acct.getBlockchain()));
+			return ResponseResult.NETWORK_ISSUE;
+		}
+
+		for( CrossChainTradeData tradeData : crossChainTradeDataList) {
+			// Check Alice doesn't already have an existing, on-going trade-bot entry for this AT.
+			if (repository.getCrossChainRepository().existsTradeWithAtExcludingStates(tradeData.qortalAtAddress, acctTradeBot.getEndStates()))
+				return ResponseResult.TRADE_ALREADY_EXISTS;
+		}
+		return TradeBotUtils.startResponseMultiple(repository, acct, crossChainTradeDataList, receiveAddress, foreignKey, bitcoiny);
+	}
+
+	public boolean deleteEntry(Repository repository, byte[] tradePrivateKey) throws DataException {
+		TradeBotData tradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
+		if (tradeBotData == null)
+			// Can't delete what we don't have!
+			return false;
+
+		boolean canDelete = false;
+
+		ACCT acct = SupportedBlockchain.getAcctByName(tradeBotData.getAcctName());
+		if (acct == null)
+			// We can't/no longer support this ACCT
+			canDelete = true;
+		else {
+			AcctTradeBot acctTradeBot = findTradeBotForAcct(acct);
+			canDelete = acctTradeBot == null || acctTradeBot.canDelete(repository, tradeBotData);
+		}
+
+		if (canDelete) {
+			repository.getCrossChainRepository().delete(tradePrivateKey);
+			repository.saveChanges();
+		}
+
+		return canDelete;
+	}
+
+	@Override
+	public void listen(Event event) {
+		if (!(event instanceof Synchronizer.NewChainTipEvent))
+			return;
+
+		// Don't process trade bots or broadcast presence timestamps if our chain is more than 60 minutes old
+		final Long minLatestBlockTimestamp = NTP.getTime() - (60 * 60 * 1000L);
+		if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp))
+			return;
+
+		synchronized (this) {
+			expireOldPresenceTimestamps();
+
+			List<TradeBotData> allTradeBotData;
+
+			try (final Repository repository = RepositoryManager.getRepository()) {
+				allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
+			} catch (DataException e) {
+				LOGGER.error("Couldn't run trade bot due to repository issue", e);
+				return;
+			}
+
+			for (TradeBotData tradeBotData : allTradeBotData)
+				try (final Repository repository = RepositoryManager.getRepository()) {
+					// Find ACCT-specific trade-bot for this entry
+					ACCT acct = SupportedBlockchain.getAcctByName(tradeBotData.getAcctName());
+					if (acct == null) {
+						LOGGER.debug(() -> String.format("Couldn't find ACCT matching name %s", tradeBotData.getAcctName()));
+						continue;
+					}
+
+					AcctTradeBot acctTradeBot = findTradeBotForAcct(acct);
+					if (acctTradeBot == null) {
+						LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot matching name %s", tradeBotData.getAcctName()));
+						continue;
+					}
+
+					acctTradeBot.progress(repository, tradeBotData);
+				} catch (DataException e) {
+					LOGGER.error("Couldn't run trade bot due to repository issue", e);
+				} catch (ForeignBlockchainException e) {
+					LOGGER.warn(() -> String.format("Foreign blockchain issue processing trade-bot entry for AT %s: %s", tradeBotData.getAtAddress(), e.getMessage()));
+				}
+
+			broadcastPresenceTimestamps();
+		}
+	}
+
+	public static byte[] generateTradePrivateKey() {
+		// The private key is used for both Curve25519 and secp256k1 so needs to be valid for both.
+		// Curve25519 accepts any seed, so generate a valid secp256k1 key and use that.
+		return new ECKey().getPrivKeyBytes();
+	}
+
+	public static byte[] deriveTradeNativePublicKey(byte[] privateKey) {
+		return Crypto.toPublicKey(privateKey);
+	}
+
+	public static byte[] deriveTradeForeignPublicKey(byte[] privateKey) {
+		return ECKey.fromPrivate(privateKey).getPubKey();
+	}
+
+	/*package*/ public static byte[] generateSecret() {
+		byte[] secret = new byte[32];
+		RANDOM.nextBytes(secret);
+		return secret;
+	}
+
+	/*package*/ static void backupTradeBotData(Repository repository, List<TradeBotData> additional) {
+		// Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure
+		try {
+			LOGGER.info("About to backup trade bot data...");
+			HSQLDBImportExport.backupTradeBotStates(repository, additional);
+		} catch (DataException e) {
+			LOGGER.info(String.format("Repository issue when exporting trade bot data: %s", e.getMessage()));
+		}
+	}
+
+	/** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */
+	/*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData,
+			String newState, int newStateValue, Supplier<String> logMessageSupplier) throws DataException {
+		tradeBotData.setState(newState);
+		tradeBotData.setStateValue(newStateValue);
+		tradeBotData.setTimestamp(NTP.getTime());
+		repository.getCrossChainRepository().save(tradeBotData);
+		repository.saveChanges();
+
+		if (Settings.getInstance().isTradebotSystrayEnabled())
+			SysTray.getInstance().showMessage("Trade-Bot", String.format("%s: %s", tradeBotData.getAtAddress(), newState), MessageType.INFO);
+
+		if (logMessageSupplier != null)
+			LOGGER.info(logMessageSupplier.get());
+
+		LOGGER.debug(() -> String.format("new state for trade-bot entry based on AT %s: %s", tradeBotData.getAtAddress(), newState));
+
+		notifyStateChange(tradeBotData);
+	}
+
+	/** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */
+	/*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, StateNameAndValueSupplier newStateSupplier, Supplier<String> logMessageSupplier) throws DataException {
+		updateTradeBotState(repository, tradeBotData, newStateSupplier.getState(), newStateSupplier.getStateValue(), logMessageSupplier);
+	}
+
+	/** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */
+	/*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, Supplier<String> logMessageSupplier) throws DataException {
+		updateTradeBotState(repository, tradeBotData, tradeBotData.getState(), tradeBotData.getStateValue(), logMessageSupplier);
+	}
+
+	/*package*/ static void notifyStateChange(TradeBotData tradeBotData) {
+		StateChangeEvent stateChangeEvent = new StateChangeEvent(tradeBotData);
+		EventBus.INSTANCE.notify(stateChangeEvent);
+	}
+
+	/*package*/ static AcctTradeBot findTradeBotForAcct(ACCT acct) {
+		Supplier<AcctTradeBot> acctTradeBotSupplier = acctTradeBotSuppliers.get(acct.getClass());
+		if (acctTradeBotSupplier == null)
+			return null;
+
+		return acctTradeBotSupplier.get();
+	}
+
+	// PRESENCE-related
+
+	public Collection<TradePresenceData> getAllTradePresences() {
+		return this.safeAllTradePresencesByPubkey.values();
+	}
+
+	/** Trade presence timestamps expire in the 'future' so any that reach 'now' have expired and are removed. */
+	private void expireOldPresenceTimestamps() {
+		long now = NTP.getTime();
+
+		int allRemovedCount = 0;
+		synchronized (this.allTradePresencesByPubkey) {
+			int preRemoveCount = this.allTradePresencesByPubkey.size();
+			this.allTradePresencesByPubkey.values().removeIf(tradePresenceData -> tradePresenceData.getTimestamp() <= now);
+			allRemovedCount = this.allTradePresencesByPubkey.size() - preRemoveCount;
+		}
+
+		int ourRemovedCount = 0;
+		synchronized (this.ourTradePresenceTimestampsByPubkey) {
+			int preRemoveCount = this.ourTradePresenceTimestampsByPubkey.size();
+			this.ourTradePresenceTimestampsByPubkey.values().removeIf(timestamp -> timestamp < now);
+			ourRemovedCount = this.ourTradePresenceTimestampsByPubkey.size() - preRemoveCount;
+		}
+
+		if (allRemovedCount > 0)
+			LOGGER.debug("Removed {} expired trade presences, of which {} ours", allRemovedCount, ourRemovedCount);
+	}
+
+	/*package*/ void updatePresence(Repository repository, TradeBotData tradeBotData, CrossChainTradeData tradeData)
+			throws DataException {
+		String atAddress = tradeBotData.getAtAddress();
+
+		PrivateKeyAccount tradeNativeAccount = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
+		String signerAddress = tradeNativeAccount.getAddress();
+
+		/*
+		* There's no point in Alice trying to broadcast presence for an AT that isn't locked to her,
+		* as other peers won't be able to verify as signing public key isn't yet in the AT's data segment.
+		*/
+		if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) {
+			// Signer is neither Bob, nor trade locked to Alice
+			LOGGER.trace("Can't provide trade presence for our AT {} as it's not yet locked to Alice", atAddress);
+			return;
+		}
+
+		long now = NTP.getTime();
+		long newExpiry = generateExpiry(now);
+		ByteArray pubkeyByteArray = ByteArray.wrap(tradeNativeAccount.getPublicKey());
+
+		// If map entry's timestamp is missing, or within early renewal period, use the new expiry - otherwise use existing timestamp.
+		synchronized (this.ourTradePresenceTimestampsByPubkey) {
+			Long currentTimestamp = this.ourTradePresenceTimestampsByPubkey.get(pubkeyByteArray);
+
+			if (currentTimestamp != null && currentTimestamp - now > EARLY_RENEWAL_PERIOD) {
+				// timestamp still good
+				LOGGER.trace("Current trade presence timestamp {} still good for our trade {}", currentTimestamp, atAddress);
+				return;
+			}
+
+			this.ourTradePresenceTimestampsByPubkey.put(pubkeyByteArray, newExpiry);
+		}
+
+		// Create signature
+		byte[] signature = tradeNativeAccount.sign(Longs.toByteArray(newExpiry));
+
+		// Add new trade presence to queue to be broadcast around network
+		TradePresenceData tradePresenceData = new TradePresenceData(newExpiry, tradeNativeAccount.getPublicKey(), signature, atAddress);
+		this.pendingTradePresences.add(tradePresenceData);
+
+		this.allTradePresencesByPubkey.put(pubkeyByteArray, tradePresenceData);
+		rebuildSafeAllTradePresences();
+
+		LOGGER.trace("New trade presence timestamp {} for our trade {}", newExpiry, atAddress);
+
+		EventBus.INSTANCE.notify(new TradePresenceEvent(tradePresenceData));
+	}
+
+	private void rebuildSafeAllTradePresences() {
+		synchronized (this.allTradePresencesByPubkey) {
+			// Collect into a *new* unmodifiable map.
+			this.safeAllTradePresencesByPubkey = Map.copyOf(this.allTradePresencesByPubkey);
+		}
+	}
+
+	private void broadcastPresenceTimestamps() {
+		// If we have new trade presences that are pending broadcast, send those as a priority
+		if (!this.pendingTradePresences.isEmpty()) {
+			// Create a copy for Network to safely use in another thread
+			List<TradePresenceData> safeTradePresences;
+			synchronized (this.pendingTradePresences) {
+				safeTradePresences = List.copyOf(this.pendingTradePresences);
+				this.pendingTradePresences.clear();
+			}
+
+			LOGGER.debug("Broadcasting {} new trade presences", safeTradePresences.size());
+
+			TradePresencesMessage tradePresencesMessage = new TradePresencesMessage(safeTradePresences);
+			RNSNetwork.getInstance().broadcast(peer -> tradePresencesMessage);
+
+			return;
+		}
+
+		// As we have no new trade presences, check whether it's time to do a general broadcast
+		Long now = NTP.getTime();
+		if (now == null || now < nextTradePresenceBroadcastTimestamp)
+			return;
+
+		nextTradePresenceBroadcastTimestamp = now + PRESENCE_BROADCAST_INTERVAL;
+
+		List<TradePresenceData> safeTradePresences = List.copyOf(this.safeAllTradePresencesByPubkey.values());
+
+		LOGGER.debug("Broadcasting all {} known trade presences. Next broadcast timestamp: {}",
+				safeTradePresences.size(), nextTradePresenceBroadcastTimestamp
+		);
+
+		GetTradePresencesMessage getTradePresencesMessage = new GetTradePresencesMessage(safeTradePresences);
+		RNSNetwork.getInstance().broadcast(peer -> getTradePresencesMessage);
+	}
+
+	// Network message processing
+
+	public void onGetTradePresencesMessage(RNSPeer peer, Message message) {
+		GetTradePresencesMessage getTradePresencesMessage = (GetTradePresencesMessage) message;
+
+		List<TradePresenceData> peersTradePresences = getTradePresencesMessage.getTradePresences();
+
+		// Create mutable copy from safe snapshot
+		Map<ByteArray, TradePresenceData> entriesUnknownToPeer = new HashMap<>(this.safeAllTradePresencesByPubkey);
+		int knownCount = entriesUnknownToPeer.size();
+
+		for (TradePresenceData peersTradePresence : peersTradePresences) {
+			ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey());
+
+			TradePresenceData ourEntry = entriesUnknownToPeer.get(pubkeyByteArray);
+
+			if (ourEntry != null && ourEntry.getTimestamp() == peersTradePresence.getTimestamp())
+				entriesUnknownToPeer.remove(pubkeyByteArray);
+		}
+
+		if (entriesUnknownToPeer.isEmpty())
+			return;
+
+		LOGGER.debug("Sending {} trade presences to peer {} after excluding their {} from known {}",
+				entriesUnknownToPeer.size(), peer, peersTradePresences.size(), knownCount
+		);
+
+		// Send complement to peer
+		List<TradePresenceData> safeTradePresences = List.copyOf(entriesUnknownToPeer.values());
+		Message responseMessage = new TradePresencesMessage(safeTradePresences);
+		//if (!peer.sendMessage(responseMessage)) {
+		//	peer.disconnect("failed to send TRADE_PRESENCES response");
+		//	return;
+		//}
+		peer.sendMessage(responseMessage);
+	}
+
+	public void onTradePresencesMessage(RNSPeer peer, Message message) {
+		TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) message;
+
+		List<TradePresenceData> peersTradePresences = tradePresencesMessage.getTradePresences();
+
+		long now = NTP.getTime();
+		// Timestamps before this are too far into the past
+		long pastThreshold = now;
+		// Timestamps after this are too far into the future
+		long futureThreshold = now + PRESENCE_LIFETIME;
+
+		Map<ByteArray, Supplier<ACCT>> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap();
+
+		int newCount = 0;
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			for (TradePresenceData peersTradePresence : peersTradePresences) {
+				long timestamp = peersTradePresence.getTimestamp();
+
+				// Ignore if timestamp is out of bounds
+				if (timestamp < pastThreshold || timestamp > futureThreshold) {
+					if (timestamp < pastThreshold)
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold
+								);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold
+						);
+
+					continue;
+				}
+
+				ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey());
+
+				// Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older
+				TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray);
+				if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) {
+					if (timestamp == existingTradeData.getTimestamp())
+						LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before",
+								peersTradePresence.getAtAddress(), peer, timestamp
+						);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp()
+						);
+
+					continue;
+				}
+
+				// Check timestamp signature
+				byte[] timestampSignature = peersTradePresence.getSignature();
+				byte[] timestampBytes = Longs.toByteArray(timestamp);
+				byte[] publicKey = peersTradePresence.getPublicKey();
+				if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				ATData atData = repository.getATRepository().fromATAddress(peersTradePresence.getAtAddress());
+				if (atData == null || atData.getIsFrozen() || atData.getIsFinished()) {
+					if (atData == null)
+						LOGGER.trace("Ignoring trade presence {} from peer {} as AT doesn't exist",
+								peersTradePresence.getAtAddress(), peer
+						);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as AT is frozen or finished",
+								peersTradePresence.getAtAddress(), peer
+						);
+
+					continue;
+				}
+
+				ByteArray atCodeHash = ByteArray.wrap(atData.getCodeHash());
+				Supplier<ACCT> acctSupplier = acctSuppliersByCodeHash.get(atCodeHash);
+				if (acctSupplier == null) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as AT isn't a known ACCT?",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				CrossChainTradeData tradeData = acctSupplier.get().populateTradeData(repository, atData);
+				if (tradeData == null) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as trade data not found?",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				// Convert signer's public key to address form
+				String signerAddress = peersTradePresence.getTradeAddress();
+
+				// Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form)
+				if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				// This is new to us
+				this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence);
+				++newCount;
+
+				LOGGER.trace("Added trade presence {} from peer {} with timestamp {}",
+						peersTradePresence.getAtAddress(), peer, timestamp
+				);
+
+				EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence));
+			}
+		} catch (DataException e) {
+			LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e);
+		}
+
+		if (newCount > 0) {
+			LOGGER.debug("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size());
+			rebuildSafeAllTradePresences();
+		}
+	}
+
+	public void bridgePresence(long timestamp, byte[] publicKey, byte[] signature, String atAddress) {
+		long expiry = generateExpiry(timestamp);
+		ByteArray pubkeyByteArray = ByteArray.wrap(publicKey);
+
+		TradePresenceData fakeTradePresenceData = new TradePresenceData(expiry, publicKey, signature, atAddress);
+
+		// Only bridge if trade presence expiry timestamp is newer
+		TradePresenceData computedTradePresenceData = this.allTradePresencesByPubkey.compute(pubkeyByteArray, (k, v) ->
+				v == null || v.getTimestamp() < expiry ? fakeTradePresenceData : v
+		);
+
+		if (computedTradePresenceData == fakeTradePresenceData) {
+			LOGGER.trace("Bridged PRESENCE transaction for trade {} with timestamp {}", atAddress, expiry);
+			rebuildSafeAllTradePresences();
+
+			EventBus.INSTANCE.notify(new TradePresenceEvent(fakeTradePresenceData));
+		}
+	}
+
+	/** Decorates a CrossChainTradeData object with Alice / Bob trade-bot presence timestamp, if available. */
+	public void decorateTradeDataWithPresence(CrossChainTradeData crossChainTradeData) {
+		// Match by AT address, then check for Bob vs Alice
+		this.safeAllTradePresencesByPubkey.values().stream()
+				.filter(tradePresenceData -> tradePresenceData.getAtAddress().equals(crossChainTradeData.qortalAtAddress))
+				.forEach(tradePresenceData -> {
+					String signerAddress = tradePresenceData.getTradeAddress();
+
+					// Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form)
+					if (signerAddress.equals(crossChainTradeData.qortalCreatorTradeAddress))
+						crossChainTradeData.creatorPresenceExpiry = tradePresenceData.getTimestamp();
+					else if (signerAddress.equals(crossChainTradeData.qortalPartnerAddress))
+						crossChainTradeData.partnerPresenceExpiry = tradePresenceData.getTimestamp();
+				});
+	}
+
+	/** Removes any trades that have had multiple failures */
+	public List<CrossChainTradeData> removeFailedTrades(Repository repository, List<CrossChainTradeData> crossChainTrades) {
+		Long now = NTP.getTime();
+		if (now == null) {
+			return crossChainTrades;
+		}
+
+		List<CrossChainTradeData> updatedCrossChainTrades = new ArrayList<>(crossChainTrades);
+		int getMaxTradeOfferAttempts = Settings.getInstance().getMaxTradeOfferAttempts();
+
+		for (CrossChainTradeData crossChainTradeData : crossChainTrades) {
+			// We only care about trades in the OFFERING state
+			if (crossChainTradeData.mode != AcctMode.OFFERING) {
+				failedTrades.remove(crossChainTradeData.qortalAtAddress);
+				validTrades.remove(crossChainTradeData.qortalAtAddress);
+				continue;
+			}
+
+			// Return recently cached values if they exist
+			Long failedTimestamp = failedTrades.get(crossChainTradeData.qortalAtAddress);
+			if (failedTimestamp != null && now - failedTimestamp < 60 * 60 * 1000L) {
+				updatedCrossChainTrades.remove(crossChainTradeData);
+				//LOGGER.info("Removing cached failed trade AT {}", crossChainTradeData.qortalAtAddress);
+				continue;
+			}
+			Long validTimestamp = validTrades.get(crossChainTradeData.qortalAtAddress);
+			if (validTimestamp != null && now - validTimestamp < 60 * 60 * 1000L) {
+				//LOGGER.info("NOT removing cached valid trade AT {}", crossChainTradeData.qortalAtAddress);
+				continue;
+			}
+
+			try {
+				List<TransactionData> transactions = repository.getTransactionRepository().getUnconfirmedTransactions(Arrays.asList(Transaction.TransactionType.MESSAGE), null, null, null, null);
+
+				for (TransactionData transactionData : transactions) {
+					// Treat as failed if buy attempt was more than 60 mins ago (as it's still in the OFFERING state)
+					if (transactionData.getRecipient().equals(crossChainTradeData.qortalCreatorTradeAddress) && now - transactionData.getTimestamp() > 60*60*1000L) {
+						failedTrades.put(crossChainTradeData.qortalAtAddress, now);
+						updatedCrossChainTrades.remove(crossChainTradeData);
+					} else {
+						validTrades.put(crossChainTradeData.qortalAtAddress, now);
+					}
+				}
+
+			} catch (DataException e) {
+				LOGGER.info("Unable to determine failed state of AT {}", crossChainTradeData.qortalAtAddress);
+            }
+		}
+
+		return updatedCrossChainTrades;
+	}
+
+	public boolean isFailedTrade(Repository repository, CrossChainTradeData crossChainTradeData) {
+		List<CrossChainTradeData> results = removeFailedTrades(repository, Arrays.asList(crossChainTradeData));
+		return results.isEmpty();
+	}
+
+	private long generateExpiry(long timestamp) {
+		return ((timestamp - 1) / EXPIRY_ROUNDING) * EXPIRY_ROUNDING + PRESENCE_LIFETIME;
+	}
+
+}

src/main/java/org/qortal/controller/tradebot/TradeBot.java

@@ -8,7 +8,6 @@ import org.qortal.account.PrivateKeyAccount;
 import org.qortal.api.model.crosschain.TradeBotCreateRequest;
 import org.qortal.controller.Controller;
 import org.qortal.controller.Synchronizer;
-import org.qortal.controller.arbitrary.PeerMessage;
 import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult;
 import org.qortal.crosschain.*;
 import org.qortal.crypto.Crypto;
@@ -38,12 +37,7 @@ import org.qortal.utils.NTP;
 import java.awt.TrayIcon.MessageType;
 import java.security.SecureRandom;
 import java.util.*;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
 import java.util.function.Supplier;
-import java.util.stream.Collectors;
 
 /**
  * Performing cross-chain trading steps on behalf of user.
@@ -124,9 +118,6 @@ public class TradeBot implements Listener {
 	private Map<String, Long> validTrades = new HashMap<>();
 
 	private TradeBot() {
-
-		tradePresenceMessageScheduler.scheduleAtFixedRate( this::processTradePresencesMessages, 60, 1, TimeUnit.SECONDS);
-
 		EventBus.INSTANCE.addListener(event -> TradeBot.getInstance().listen(event));
 	}
 
@@ -560,139 +551,77 @@ public class TradeBot implements Listener {
 		}
 	}
 
-	// List to collect messages
-	private final List<PeerMessage> tradePresenceMessageList = new ArrayList<>();
-	// Lock to synchronize access to the list
-	private final Object tradePresenceMessageLock = new Object();
-
-	// Scheduled executor service to process messages every second
-	private final ScheduledExecutorService tradePresenceMessageScheduler = Executors.newScheduledThreadPool(1);
-
 	public void onTradePresencesMessage(Peer peer, Message message) {
+		TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) message;
 
-		synchronized (tradePresenceMessageLock) {
-			tradePresenceMessageList.add(new PeerMessage(peer, message));
-		}
-	}
+		List<TradePresenceData> peersTradePresences = tradePresencesMessage.getTradePresences();
 
-	public void processTradePresencesMessages() {
+		long now = NTP.getTime();
+		// Timestamps before this are too far into the past
+		long pastThreshold = now;
+		// Timestamps after this are too far into the future
+		long futureThreshold = now + PRESENCE_LIFETIME;
 
-		try {
-			List<PeerMessage> messagesToProcess;
-			synchronized (tradePresenceMessageLock) {
-				messagesToProcess = new ArrayList<>(tradePresenceMessageList);
-				tradePresenceMessageList.clear();
-			}
+		Map<ByteArray, Supplier<ACCT>> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap();
 
-			if( messagesToProcess.isEmpty() ) return;
+		int newCount = 0;
 
-			Map<Peer, List<TradePresenceData>> tradePresencesByPeer = new HashMap<>(messagesToProcess.size());
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			for (TradePresenceData peersTradePresence : peersTradePresences) {
+				long timestamp = peersTradePresence.getTimestamp();
 
-			// map all trade presences from the messages to their peer
-			for( PeerMessage peerMessage : messagesToProcess ) {
-				TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) peerMessage.getMessage();
-
-				List<TradePresenceData> peersTradePresences = tradePresencesMessage.getTradePresences();
-
-				tradePresencesByPeer.put(peerMessage.getPeer(), peersTradePresences);
-			}
-
-			long now = NTP.getTime();
-			// Timestamps before this are too far into the past
-			long pastThreshold = now;
-			// Timestamps after this are too far into the future
-			long futureThreshold = now + PRESENCE_LIFETIME;
-
-			Map<ByteArray, Supplier<ACCT>> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap();
-
-			int newCount = 0;
-
-			Map<String, List<Peer>> peersByAtAddress = new HashMap<>(tradePresencesByPeer.size());
-			Map<String, TradePresenceData> tradePresenceByAtAddress = new HashMap<>(tradePresencesByPeer.size());
-
-			// for each batch of trade presence data from a peer, validate and populate the maps declared above
-			for ( Map.Entry<Peer, List<TradePresenceData>> entry: tradePresencesByPeer.entrySet()) {
-
-				Peer peer = entry.getKey();
-
-				for( TradePresenceData peersTradePresence : entry.getValue() ) {
-					// TradePresenceData peersTradePresence
-					long timestamp = peersTradePresence.getTimestamp();
-
-					// Ignore if timestamp is out of bounds
-					if (timestamp < pastThreshold || timestamp > futureThreshold) {
-						if (timestamp < pastThreshold)
-							LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}",
-									peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold
-							);
-						else
-							LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}",
-									peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold
-							);
-
-						continue;
-					}
-
-					ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey());
-
-					// Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older
-					TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray);
-					if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) {
-						if (timestamp == existingTradeData.getTimestamp())
-							LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before",
-									peersTradePresence.getAtAddress(), peer, timestamp
-							);
-						else
-							LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}",
-									peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp()
-							);
-
-						continue;
-					}
-
-					// Check timestamp signature
-					byte[] timestampSignature = peersTradePresence.getSignature();
-					byte[] timestampBytes = Longs.toByteArray(timestamp);
-					byte[] publicKey = peersTradePresence.getPublicKey();
-					if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) {
-						LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify",
-								peersTradePresence.getAtAddress(), peer
+				// Ignore if timestamp is out of bounds
+				if (timestamp < pastThreshold || timestamp > futureThreshold) {
+					if (timestamp < pastThreshold)
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold
+								);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold
 						);
 
-						continue;
-					}
-
-					peersByAtAddress.computeIfAbsent(peersTradePresence.getAtAddress(), address -> new ArrayList<>()).add(peer);
-					tradePresenceByAtAddress.put(peersTradePresence.getAtAddress(), peersTradePresence);
+					continue;
 				}
-			}
 
-			if( tradePresenceByAtAddress.isEmpty() ) return;
+				ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey());
 
-			List<ATData> atDataList;
-			try (final Repository repository = RepositoryManager.getRepository()) {
-				atDataList = repository.getATRepository().fromATAddresses( new ArrayList<>(tradePresenceByAtAddress.keySet()) );
-			} catch (DataException e) {
-				LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e);
-				return;
-			}
-
-			Map<String, Supplier<ACCT>> supplierByAtAddress = new HashMap<>(atDataList.size());
-
-			List<ATData> validatedAtDataList = new ArrayList<>(atDataList.size());
-
-			// for each trade
-			for( ATData atData : atDataList ) {
-
-				TradePresenceData peersTradePresence = tradePresenceByAtAddress.get(atData.getATAddress());
-				if (atData == null || atData.getIsFrozen() || atData.getIsFinished()) {
-					if (atData == null)
-						LOGGER.trace("Ignoring trade presence {} from peer as AT doesn't exist",
-								peersTradePresence.getAtAddress()
+				// Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older
+				TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray);
+				if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) {
+					if (timestamp == existingTradeData.getTimestamp())
+						LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before",
+								peersTradePresence.getAtAddress(), peer, timestamp
 						);
 					else
-						LOGGER.trace("Ignoring trade presence {} from peer as AT is frozen or finished",
-								peersTradePresence.getAtAddress()
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp()
+						);
+
+					continue;
+				}
+
+				// Check timestamp signature
+				byte[] timestampSignature = peersTradePresence.getSignature();
+				byte[] timestampBytes = Longs.toByteArray(timestamp);
+				byte[] publicKey = peersTradePresence.getPublicKey();
+				if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				ATData atData = repository.getATRepository().fromATAddress(peersTradePresence.getAtAddress());
+				if (atData == null || atData.getIsFrozen() || atData.getIsFinished()) {
+					if (atData == null)
+						LOGGER.trace("Ignoring trade presence {} from peer {} as AT doesn't exist",
+								peersTradePresence.getAtAddress(), peer
+						);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as AT is frozen or finished",
+								peersTradePresence.getAtAddress(), peer
 						);
 
 					continue;
@@ -701,87 +630,51 @@ public class TradeBot implements Listener {
 				ByteArray atCodeHash = ByteArray.wrap(atData.getCodeHash());
 				Supplier<ACCT> acctSupplier = acctSuppliersByCodeHash.get(atCodeHash);
 				if (acctSupplier == null) {
-					LOGGER.trace("Ignoring trade presence {} from peer as AT isn't a known ACCT?",
-							peersTradePresence.getAtAddress()
+					LOGGER.trace("Ignoring trade presence {} from peer {} as AT isn't a known ACCT?",
+							peersTradePresence.getAtAddress(), peer
 					);
 
 					continue;
 				}
-				validatedAtDataList.add(atData);
-			}
 
-			// populated data for each trade
-			List<CrossChainTradeData> crossChainTradeDataList;
-
-			// validated trade data grouped by code (cross chain coin)
-			Map<ByteArray, List<ATData>> atDataByCodeHash
-				= validatedAtDataList.stream().collect(
-					Collectors.groupingBy(data -> ByteArray.wrap(data.getCodeHash())));
-
-			try (final Repository repository = RepositoryManager.getRepository()) {
-
-				crossChainTradeDataList = new ArrayList<>();
-
-				// for each code (cross chain coin), get each trade, then populate trade data
-				for( Map.Entry<ByteArray, List<ATData>> entry : atDataByCodeHash.entrySet() ) {
-
-					Supplier<ACCT> acctSupplier = acctSuppliersByCodeHash.get(entry.getKey());
-
-					crossChainTradeDataList.addAll(
-						acctSupplier.get().populateTradeDataList(
-								repository,
-								entry.getValue()
-						)
-						.stream().filter( data -> data != null )
-						.collect(Collectors.toList())
-					);
-				}
-			} catch (DataException e) {
-				LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e);
-				return;
-			}
-
-			// for each populated trade data, validate and fire event
-			for( CrossChainTradeData tradeData : crossChainTradeDataList ) {
-
-				List<Peer> peers = peersByAtAddress.get(tradeData.qortalAtAddress);
-
-				for( Peer peer : peers ) {
-
-					TradePresenceData peersTradePresence = tradePresenceByAtAddress.get(tradeData.qortalAtAddress);
-
-					// Convert signer's public key to address form
-					String signerAddress = peersTradePresence.getTradeAddress();
-
-					// Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form)
-					if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) {
-						LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?",
-								peersTradePresence.getAtAddress(), peer
-						);
-
-						continue;
-					}
-
-					ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey());
-
-					// This is new to us
-					this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence);
-					++newCount;
-
-					LOGGER.trace("Added trade presence {} from peer {} with timestamp {}",
-							peersTradePresence.getAtAddress(), peer, tradeData.creationTimestamp
+				CrossChainTradeData tradeData = acctSupplier.get().populateTradeData(repository, atData);
+				if (tradeData == null) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as trade data not found?",
+							peersTradePresence.getAtAddress(), peer
 					);
 
-					EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence));
+					continue;
 				}
-			}
 
-			if (newCount > 0) {
-				LOGGER.info("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size());
-				rebuildSafeAllTradePresences();
+				// Convert signer's public key to address form
+				String signerAddress = peersTradePresence.getTradeAddress();
+
+				// Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form)
+				if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				// This is new to us
+				this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence);
+				++newCount;
+
+				LOGGER.trace("Added trade presence {} from peer {} with timestamp {}",
+						peersTradePresence.getAtAddress(), peer, timestamp
+				);
+
+				EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence));
 			}
-		} catch (Exception e) {
-			LOGGER.error(e.getMessage(), e);
+		} catch (DataException e) {
+			LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e);
+		}
+
+		if (newCount > 0) {
+			LOGGER.debug("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size());
+			rebuildSafeAllTradePresences();
 		}
 	}
 

src/main/java/org/qortal/crosschain/ACCT.java

@@ -6,9 +6,6 @@ import org.qortal.data.crosschain.CrossChainTradeData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 
-import java.util.List;
-import java.util.OptionalLong;
-
 public interface ACCT {
 
 	public byte[] getCodeBytesHash();
@@ -19,12 +16,8 @@ public interface ACCT {
 
 	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException;
 
-	public List<CrossChainTradeData> populateTradeDataList(Repository respository, List<ATData> atDataList) throws DataException;
-
 	public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException;
 
-	CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException;
-
 	public byte[] buildCancelMessage(String creatorQortalAddress);
 
 	public byte[] findSecretA(Repository repository, CrossChainTradeData crossChainTradeData) throws DataException;

src/main/java/org/qortal/crosschain/Bitcoin.java

@@ -1,6 +1,5 @@
 package org.qortal.crosschain;
 
-import org.bitcoinj.core.Coin;
 import org.bitcoinj.core.Context;
 import org.bitcoinj.core.NetworkParameters;
 import org.bitcoinj.core.Transaction;
@@ -15,21 +14,15 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumMap;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
 
 public class Bitcoin extends Bitcoiny {
 
 	public static final String CURRENCY_CODE = "BTC";
 
-	// Locking fee to lock in a QORT for BTC. This is the default value that the user should reset to
-	// a value inline with the BTC fee market. This is 5 sats per kB.
-	private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(5_000); // 0.00005 BTC per 1000 bytes
+	private static final long MINIMUM_ORDER_AMOUNT = 100000; // 0.001 BTC minimum order, due to high fees
 
-	private static final long MINIMUM_ORDER_AMOUNT = 100_000; // 0.001 BTC minimum order, due to high fees
-
-	// Default value until user resets fee to compete with the current market. This is a total value for a
-	// p2sh transaction, size 300 kB, 5 sats per kB
-	private static final long NEW_FEE_AMOUNT = 1_500L;
+	// Temporary values until a dynamic fee system is written.
+	private static final long NEW_FEE_AMOUNT = 6_000L;
 
 	private static final long NON_MAINNET_FEE = 1000L; // enough for TESTNET3 and should be OK for REGTEST
 
@@ -118,7 +111,7 @@ public class Bitcoin extends Bitcoiny {
 
 			@Override
 			public long getP2shFee(Long timestamp) {
-				return this.getFeeRequired();
+				return this.getFeeCeiling();
 			}
 		},
 		TEST3 {
@@ -180,14 +173,14 @@ public class Bitcoin extends Bitcoiny {
 			}
 		};
 
-		private AtomicLong feeRequired = new AtomicLong(NEW_FEE_AMOUNT);
+		private long feeCeiling = NEW_FEE_AMOUNT;
 
-		public long getFeeRequired() {
-			return feeRequired.get();
+		public long getFeeCeiling() {
+			return feeCeiling;
 		}
 
-		public void setFeeRequired(long feeRequired) {
-			this.feeRequired.set(feeRequired);
+		public void setFeeCeiling(long feeCeiling) {
+			this.feeCeiling = feeCeiling;
 		}
 
 		public abstract NetworkParameters getParams();
@@ -203,7 +196,7 @@ public class Bitcoin extends Bitcoiny {
 	// Constructors and instance
 
 	private Bitcoin(BitcoinNet bitcoinNet, BitcoinyBlockchainProvider blockchain, Context bitcoinjContext, String currencyCode) {
-		super(blockchain, bitcoinjContext, currencyCode, DEFAULT_FEE_PER_KB);
+		super(blockchain, bitcoinjContext, currencyCode, bitcoinjContext.getFeePerKb());
 		this.bitcoinNet = bitcoinNet;
 
 		LOGGER.info(() -> String.format("Starting Bitcoin support using %s", this.bitcoinNet.name()));
@@ -249,14 +242,14 @@ public class Bitcoin extends Bitcoiny {
 	}
 
 	@Override
-	public long getFeeRequired() {
-		return this.bitcoinNet.getFeeRequired();
+	public long getFeeCeiling() {
+		return this.bitcoinNet.getFeeCeiling();
 	}
 
 	@Override
-	public void setFeeRequired(long fee) {
+	public void setFeeCeiling(long fee) {
 
-		this.bitcoinNet.setFeeRequired( fee );
+		this.bitcoinNet.setFeeCeiling( fee );
 	}
 	/**
  	* Returns bitcoinj transaction sending <tt>amount</tt> to <tt>recipient</tt> using 20 sat/byte fee.

src/main/java/org/qortal/crosschain/BitcoinACCTv1.java

@@ -4,7 +4,6 @@ import com.google.common.hash.HashCode;
 import com.google.common.primitives.Bytes;
 import org.ciyam.at.*;
 import org.qortal.account.Account;
-import org.qortal.api.resource.CrossChainUtils;
 import org.qortal.asset.Asset;
 import org.qortal.at.QortalFunctionCode;
 import org.qortal.crypto.Crypto;
@@ -20,7 +19,6 @@ import org.qortal.utils.BitTwiddling;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
-import java.util.OptionalLong;
 
 import static org.ciyam.at.OpCode.calcOffset;
 
@@ -610,14 +608,7 @@ public class BitcoinACCTv1 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
 		ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
-	}
-
-	@Override
-	public List<CrossChainTradeData> populateTradeDataList(Repository repository, List<ATData> atDataList) throws DataException {
-		List<CrossChainTradeData> crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList);
-
-		return crossChainTradeDataList;
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -626,14 +617,13 @@ public class BitcoinACCTv1 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException {
 		ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
 	 * Returns CrossChainTradeData with useful info extracted from AT.
 	 */
-	@Override
-	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException {
+	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException {
 		byte[] addressBytes = new byte[25]; // for general use
 		String atAddress = atStateData.getATAddress();
 
@@ -646,13 +636,8 @@ public class BitcoinACCTv1 implements ACCT {
 		tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey);
 		tradeData.creationTimestamp = creationTimestamp;
 
-		if(optionalBalance.isPresent()) {
-			tradeData.qortBalance = optionalBalance.getAsLong();
-		}
-		else {
-			Account atAccount = new Account(repository, atAddress);
-			tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
-		}
+		Account atAccount = new Account(repository, atAddress);
+		tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
 
 		byte[] stateData = atStateData.getStateData();
 		ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData);

src/main/java/org/qortal/crosschain/BitcoinACCTv3.java

@@ -6,7 +6,6 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.ciyam.at.*;
 import org.qortal.account.Account;
-import org.qortal.api.resource.CrossChainUtils;
 import org.qortal.asset.Asset;
 import org.qortal.at.QortalFunctionCode;
 import org.qortal.crypto.Crypto;
@@ -22,7 +21,6 @@ import org.qortal.utils.BitTwiddling;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
-import java.util.OptionalLong;
 
 import static org.ciyam.at.OpCode.calcOffset;
 
@@ -571,14 +569,7 @@ public class BitcoinACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
 		ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
-	}
-
-	@Override
-	public List<CrossChainTradeData> populateTradeDataList(Repository repository, List<ATData> atDataList) throws DataException {
-		List<CrossChainTradeData> crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList);
-
-		return crossChainTradeDataList;
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -587,14 +578,13 @@ public class BitcoinACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException {
 		ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
 	 * Returns CrossChainTradeData with useful info extracted from AT.
 	 */
-	@Override
-	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException {
+	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException {
 		byte[] addressBytes = new byte[25]; // for general use
 		String atAddress = atStateData.getATAddress();
 
@@ -607,13 +597,8 @@ public class BitcoinACCTv3 implements ACCT {
 		tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey);
 		tradeData.creationTimestamp = creationTimestamp;
 
-		if(optionalBalance.isPresent()) {
-			tradeData.qortBalance = optionalBalance.getAsLong();
-		}
-		else {
-			Account atAccount = new Account(repository, atAddress);
-			tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
-		}
+		Account atAccount = new Account(repository, atAddress);
+		tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
 
 		byte[] stateData = atStateData.getStateData();
 		ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData);

src/main/java/org/qortal/crosschain/Bitcoiny.java

@@ -8,8 +8,6 @@ import org.bitcoinj.core.*;
 import org.bitcoinj.crypto.ChildNumber;
 import org.bitcoinj.crypto.DeterministicHierarchy;
 import org.bitcoinj.crypto.DeterministicKey;
-import org.bitcoinj.crypto.HDPath;
-import org.bitcoinj.params.AbstractBitcoinNetParams;
 import org.bitcoinj.script.Script.ScriptType;
 import org.bitcoinj.script.ScriptBuilder;
 import org.bitcoinj.wallet.DeterministicKeyChain;
@@ -27,7 +25,7 @@ import java.util.*;
 import java.util.stream.Collectors;
 
 /** Bitcoin-like (Bitcoin, Litecoin, etc.) support */
-public abstract class Bitcoiny extends AbstractBitcoinNetParams implements ForeignBlockchain {
+public abstract class Bitcoiny implements ForeignBlockchain {
 
 	protected static final Logger LOGGER = LogManager.getLogger(Bitcoiny.class);
 
@@ -67,7 +65,6 @@ public abstract class Bitcoiny extends AbstractBitcoinNetParams implements Forei
 	// Constructors and instance
 
 	protected Bitcoiny(BitcoinyBlockchainProvider blockchainProvider, Context bitcoinjContext, String currencyCode, Coin feePerKb) {
-		this.genesisBlock = this.getGenesisBlock();
 		this.blockchainProvider = blockchainProvider;
 		this.bitcoinjContext = bitcoinjContext;
 		this.currencyCode = currencyCode;
@@ -77,15 +74,6 @@ public abstract class Bitcoiny extends AbstractBitcoinNetParams implements Forei
 	}
 
 	// Getters & setters
-	@Override
-	public String getPaymentProtocolId() {
-		return this.id;
-	}
-
-	@Override
-	public Block getGenesisBlock() {
-		return this.genesisBlock;
-	}
 
 	public BitcoinyBlockchainProvider getBlockchainProvider() {
 		return this.blockchainProvider;
@@ -602,27 +590,15 @@ public abstract class Bitcoiny extends AbstractBitcoinNetParams implements Forei
 
 		return new AddressInfo(
 				address.toString(),
-				toIntegerList( key.getPath() ),
+				toIntegerList( key.getPath()),
 				summingUnspentOutputs(address.toString()),
 				key.getPathAsString(),
 				transactionCount,
 				candidates.contains(address.toString()));
 	}
 
-	/**
-	 * <p>Convert BitcoinJ native type to List of Integers, BitcoinJ v16 compatible
-	 * </p>
-	 *
-	 * @param path path to deterministic key
-	 * @return Array of Ints representing the keys position in the tree
-	 * @since v4.7.2
-	 */
-	private static  List<Integer> toIntegerList(HDPath path) {
-		return path.stream().map(ChildNumber::num).collect(Collectors.toList());
-	}
-
-	// BitcoinJ v15 compatible
 	private static  List<Integer> toIntegerList(ImmutableList<ChildNumber> path) {
+
 		return path.stream().map(ChildNumber::num).collect(Collectors.toList());
 	}
 
@@ -864,9 +840,9 @@ public abstract class Bitcoiny extends AbstractBitcoinNetParams implements Forei
 		} while (true);
 	}
 
-	public abstract long getFeeRequired();
+	public abstract long getFeeCeiling();
 
-	public abstract void setFeeRequired(long fee);
+	public abstract void setFeeCeiling(long fee);
 
 	// UTXOProvider support
 

src/main/java/org/qortal/crosschain/BitcoinyTBD.java

@@ -1,6 +1,5 @@
 package org.qortal.crosschain;
 
-import org.bitcoinj.core.Block;
 import org.bitcoinj.core.Coin;
 import org.bitcoinj.core.Context;
 import org.bitcoinj.core.NetworkParameters;
@@ -90,7 +89,7 @@ public class BitcoinyTBD extends Bitcoiny {
 		NetTBD netTBD
 				= new NetTBD(
 				bitcoinyTBDRequest.getNetworkName(),
-				bitcoinyTBDRequest.getFeeRequired(),
+				bitcoinyTBDRequest.getFeeCeiling(),
 				networkParams,
 				Collections.emptyList(),
 				bitcoinyTBDRequest.getExpectedGenesisHash()
@@ -135,30 +134,18 @@ public class BitcoinyTBD extends Bitcoiny {
 	@Override
 	public long getP2shFee(Long timestamp) throws ForeignBlockchainException {
 
-		return this.netTBD.getFeeRequired();
+		return this.netTBD.getFeeCeiling();
 	}
 
 	@Override
-	public long getFeeRequired() {
+	public long getFeeCeiling() {
 
-		return this.netTBD.getFeeRequired();
+		return this.netTBD.getFeeCeiling();
 	}
 
 	@Override
-	public void setFeeRequired(long fee) {
+	public void setFeeCeiling(long fee) {
 
-		this.netTBD.setFeeRequired( fee );
-	}
-
-	@Override
-	public String getPaymentProtocolId() {
-		return params.getId();
-	}
-
-	@Override
-	public Block getGenesisBlock() {
-		if(genesisBlock == null)
-			genesisBlock = params.getGenesisBlock();
-		return this.genesisBlock;
+		this.netTBD.setFeeCeiling( fee );
 	}
 }

src/main/java/org/qortal/crosschain/DeterminedNetworkParams.java

@@ -98,10 +98,9 @@ public class DeterminedNetworkParams extends NetworkParameters implements Altcoi
 
         LOGGER.info( "Creating Genesis Block ...");
 
-        // BitcoinJ v16 has a new native method for this
         //this.genesisBlock = CoinParamsUtil.createGenesisBlockFromRequest(this, request);
 
-//        LOGGER.info("Created Genesis Block: genesisBlock = " + genesisBlock );
+        LOGGER.info("Created Genesis Block: genesisBlock = " + genesisBlock );
 
         // this is 100 for each coin from what I can tell
         this.spendableCoinbaseDepth = 100;
@@ -114,9 +113,8 @@ public class DeterminedNetworkParams extends NetworkParameters implements Altcoi
 //
 //        LOGGER.info("request = " + request);
 //
-//        checkState(genesisHash.equals(request.getExpectedGenesisHash()))
-//        alertSigningKey is removed in v16
-//        this.alertSigningKey = Hex.decode(request.getPubKey());
+//        checkState(genesisHash.equals(request.getExpectedGenesisHash()));
+        this.alertSigningKey = Hex.decode(request.getPubKey());
 
         this.majorityEnforceBlockUpgrade = request.getMajorityEnforceBlockUpgrade();
         this.majorityRejectBlockOutdated = request.getMajorityRejectBlockOutdated();
@@ -223,12 +221,6 @@ public class DeterminedNetworkParams extends NetworkParameters implements Altcoi
         }
     }
 
-    @Override
-    public Block getGenesisBlock() {
-        //ToDo: Finish
-        return null;
-    }
-
     /**
      * Get the difficulty target expected for the next block. This includes all
      * the weird cases for Litecoin such as testnet blocks which can be maximum

src/main/java/org/qortal/crosschain/Digibyte.java

@@ -14,7 +14,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumMap;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
 
 public class Digibyte extends Bitcoiny {
 
@@ -60,7 +59,7 @@ public class Digibyte extends Bitcoiny {
 
 			@Override
 			public long getP2shFee(Long timestamp) {
-				return this.getFeeRequired();
+				return this.getFeeCeiling();
 			}
 		},
 		TEST3 {
@@ -110,14 +109,14 @@ public class Digibyte extends Bitcoiny {
 			}
 		};
 
-		private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE);
+		private long feeCeiling = MAINNET_FEE;
 
-		public long getFeeRequired() {
-			return feeRequired.get();
+		public long getFeeCeiling() {
+			return feeCeiling;
 		}
 
-		public void setFeeRequired(long feeRequired) {
-			this.feeRequired.set(feeRequired);
+		public void setFeeCeiling(long feeCeiling) {
+			this.feeCeiling = feeCeiling;
 		}
 
 		public abstract NetworkParameters getParams();
@@ -179,13 +178,13 @@ public class Digibyte extends Bitcoiny {
 	}
 
 	@Override
-	public long getFeeRequired() {
-		return this.digibyteNet.getFeeRequired();
+	public long getFeeCeiling() {
+		return this.digibyteNet.getFeeCeiling();
 	}
 
 	@Override
-	public void setFeeRequired(long fee) {
+	public void setFeeCeiling(long fee) {
 
-		this.digibyteNet.setFeeRequired( fee );
+		this.digibyteNet.setFeeCeiling( fee );
 	}
 }

src/main/java/org/qortal/crosschain/DigibyteACCTv3.java

@@ -6,7 +6,6 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.ciyam.at.*;
 import org.qortal.account.Account;
-import org.qortal.api.resource.CrossChainUtils;
 import org.qortal.asset.Asset;
 import org.qortal.at.QortalFunctionCode;
 import org.qortal.crypto.Crypto;
@@ -22,7 +21,6 @@ import org.qortal.utils.BitTwiddling;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
-import java.util.OptionalLong;
 
 import static org.ciyam.at.OpCode.calcOffset;
 
@@ -571,14 +569,7 @@ public class DigibyteACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
 		ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
-	}
-
-	@Override
-	public List<CrossChainTradeData> populateTradeDataList(Repository repository, List<ATData> atDataList) throws DataException {
-		List<CrossChainTradeData> crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList);
-
-		return crossChainTradeDataList;
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -587,14 +578,13 @@ public class DigibyteACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException {
 		ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
 	 * Returns CrossChainTradeData with useful info extracted from AT.
 	 */
-	@Override
-	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException {
+	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException {
 		byte[] addressBytes = new byte[25]; // for general use
 		String atAddress = atStateData.getATAddress();
 
@@ -607,13 +597,8 @@ public class DigibyteACCTv3 implements ACCT {
 		tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey);
 		tradeData.creationTimestamp = creationTimestamp;
 
-		if(optionalBalance.isPresent()) {
-			tradeData.qortBalance = optionalBalance.getAsLong();
-		}
-		else {
-			Account atAccount = new Account(repository, atAddress);
-			tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
-		}
+		Account atAccount = new Account(repository, atAddress);
+		tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
 
 		byte[] stateData = atStateData.getStateData();
 		ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData);

src/main/java/org/qortal/crosschain/Dogecoin.java

@@ -13,7 +13,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumMap;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
 
 public class Dogecoin extends Bitcoiny {
 
@@ -61,7 +60,7 @@ public class Dogecoin extends Bitcoiny {
 
 			@Override
 			public long getP2shFee(Long timestamp) {
-				return this.getFeeRequired();
+				return this.getFeeCeiling();
 			}
 		},
 		TEST3 {
@@ -111,14 +110,14 @@ public class Dogecoin extends Bitcoiny {
 			}
 		};
 
-		private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE);
+		private long feeCeiling = MAINNET_FEE;
 
-		public long getFeeRequired() {
-			return feeRequired.get();
+		public long getFeeCeiling() {
+			return feeCeiling;
 		}
 
-		public void setFeeRequired(long feeRequired) {
-			this.feeRequired.set(feeRequired);
+		public void setFeeCeiling(long feeCeiling) {
+			this.feeCeiling = feeCeiling;
 		}
 
 		public abstract NetworkParameters getParams();
@@ -180,13 +179,13 @@ public class Dogecoin extends Bitcoiny {
 	}
 
 	@Override
-	public long getFeeRequired() {
-		return this.dogecoinNet.getFeeRequired();
+	public long getFeeCeiling() {
+		return this.dogecoinNet.getFeeCeiling();
 	}
 
 	@Override
-	public void setFeeRequired(long fee) {
+	public void setFeeCeiling(long fee) {
 
-		this.dogecoinNet.setFeeRequired( fee );
+		this.dogecoinNet.setFeeCeiling( fee );
 	}
 }

src/main/java/org/qortal/crosschain/DogecoinACCTv1.java

@@ -6,7 +6,6 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.ciyam.at.*;
 import org.qortal.account.Account;
-import org.qortal.api.resource.CrossChainUtils;
 import org.qortal.asset.Asset;
 import org.qortal.at.QortalFunctionCode;
 import org.qortal.crypto.Crypto;
@@ -22,7 +21,6 @@ import org.qortal.utils.BitTwiddling;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
-import java.util.OptionalLong;
 
 import static org.ciyam.at.OpCode.calcOffset;
 
@@ -568,14 +566,7 @@ public class DogecoinACCTv1 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
 		ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
-	}
-
-	@Override
-	public List<CrossChainTradeData> populateTradeDataList(Repository repository, List<ATData> atDataList) throws DataException {
-		List<CrossChainTradeData> crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList);
-
-		return crossChainTradeDataList;
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -584,14 +575,13 @@ public class DogecoinACCTv1 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException {
 		ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
 	 * Returns CrossChainTradeData with useful info extracted from AT.
 	 */
-	@Override
-	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException {
+	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException {
 		byte[] addressBytes = new byte[25]; // for general use
 		String atAddress = atStateData.getATAddress();
 
@@ -604,13 +594,8 @@ public class DogecoinACCTv1 implements ACCT {
 		tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey);
 		tradeData.creationTimestamp = creationTimestamp;
 
-		if(optionalBalance.isPresent()) {
-			tradeData.qortBalance = optionalBalance.getAsLong();
-		}
-		else {
-			Account atAccount = new Account(repository, atAddress);
-			tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
-		}
+		Account atAccount = new Account(repository, atAddress);
+		tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
 
 		byte[] stateData = atStateData.getStateData();
 		ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData);

src/main/java/org/qortal/crosschain/DogecoinACCTv3.java

@@ -6,7 +6,6 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.ciyam.at.*;
 import org.qortal.account.Account;
-import org.qortal.api.resource.CrossChainUtils;
 import org.qortal.asset.Asset;
 import org.qortal.at.QortalFunctionCode;
 import org.qortal.crypto.Crypto;
@@ -22,7 +21,6 @@ import org.qortal.utils.BitTwiddling;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
-import java.util.OptionalLong;
 
 import static org.ciyam.at.OpCode.calcOffset;
 
@@ -571,14 +569,7 @@ public class DogecoinACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
 		ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
-	}
-
-	@Override
-	public List<CrossChainTradeData> populateTradeDataList(Repository repository, List<ATData> atDataList) throws DataException {
-		List<CrossChainTradeData> crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList);
-
-		return crossChainTradeDataList;
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -587,14 +578,13 @@ public class DogecoinACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException {
 		ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
 	 * Returns CrossChainTradeData with useful info extracted from AT.
 	 */
-	@Override
-	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException {
+	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException {
 		byte[] addressBytes = new byte[25]; // for general use
 		String atAddress = atStateData.getATAddress();
 
@@ -607,13 +597,8 @@ public class DogecoinACCTv3 implements ACCT {
 		tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey);
 		tradeData.creationTimestamp = creationTimestamp;
 
-		if(optionalBalance.isPresent()) {
-			tradeData.qortBalance = optionalBalance.getAsLong();
-		}
-		else {
-			Account atAccount = new Account(repository, atAddress);
-			tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
-		}
+		Account atAccount = new Account(repository, atAddress);
+		tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
 
 		byte[] stateData = atStateData.getStateData();
 		ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData);

src/main/java/org/qortal/crosschain/ElectrumX.java

@@ -644,10 +644,8 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	}
 
 	/**
-	 * <p>Performs RPC call, with automatic reconnection to different server if needed.
-	 * </p>
-	 * @param method String representation of the RPC call value
-	 * @param params a list of Objects passed to the method of the Remote Server
+	 * Performs RPC call, with automatic reconnection to different server if needed.
+	 * <p>
 	 * @return "result" object from within JSON output
 	 * @throws ForeignBlockchainException if server returns error or something goes wrong
 	 */

src/main/java/org/qortal/crosschain/LegacyZcashAddress.java

@@ -184,11 +184,6 @@ public class LegacyZcashAddress extends Address {
         return p2sh ? ScriptType.P2SH : ScriptType.P2PKH;
     }
 
-    @Override
-    public int compareTo(Address address) {
-        return this.toString().compareTo(address.toString());
-    }
-
     /**
      * Given an address, examines the version byte and attempts to find a matching NetworkParameters. If you aren't sure
      * which network the address is intended for (eg, it was provided by a user), you can use this to decide if it is

src/main/java/org/qortal/crosschain/Litecoin.java

@@ -14,7 +14,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumMap;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
 
 public class Litecoin extends Bitcoiny {
 
@@ -64,7 +63,7 @@ public class Litecoin extends Bitcoiny {
 
 			@Override
 			public long getP2shFee(Long timestamp) {
-				return this.getFeeRequired();
+				return this.getFeeCeiling();
 			}
 		},
 		TEST3 {
@@ -117,14 +116,14 @@ public class Litecoin extends Bitcoiny {
 			}
 		};
 
-		private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE);
+		private long feeCeiling = MAINNET_FEE;
 
-		public long getFeeRequired() {
-			return feeRequired.get();
+		public long getFeeCeiling() {
+			return feeCeiling;
 		}
 
-		public void setFeeRequired(long feeRequired) {
-			this.feeRequired.set(feeRequired);
+		public void setFeeCeiling(long feeCeiling) {
+			this.feeCeiling = feeCeiling;
 		}
 
 		public abstract NetworkParameters getParams();
@@ -186,13 +185,13 @@ public class Litecoin extends Bitcoiny {
 	}
 
 	@Override
-	public long getFeeRequired() {
-		return this.litecoinNet.getFeeRequired();
+	public long getFeeCeiling() {
+		return this.litecoinNet.getFeeCeiling();
 	}
 
 	@Override
-	public void setFeeRequired(long fee) {
+	public void setFeeCeiling(long fee) {
 
-		this.litecoinNet.setFeeRequired( fee );
+		this.litecoinNet.setFeeCeiling( fee );
 	}
 }

src/main/java/org/qortal/crosschain/LitecoinACCTv1.java

@@ -4,7 +4,6 @@ import com.google.common.hash.HashCode;
 import com.google.common.primitives.Bytes;
 import org.ciyam.at.*;
 import org.qortal.account.Account;
-import org.qortal.api.resource.CrossChainUtils;
 import org.qortal.asset.Asset;
 import org.qortal.at.QortalFunctionCode;
 import org.qortal.crypto.Crypto;
@@ -20,7 +19,6 @@ import org.qortal.utils.BitTwiddling;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
-import java.util.OptionalLong;
 
 import static org.ciyam.at.OpCode.calcOffset;
 
@@ -561,14 +559,7 @@ public class LitecoinACCTv1 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
 		ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
-	}
-
-	@Override
-	public List<CrossChainTradeData> populateTradeDataList(Repository repository, List<ATData> atDataList) throws DataException {
-		List<CrossChainTradeData> crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList);
-
-		return crossChainTradeDataList;
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -577,14 +568,13 @@ public class LitecoinACCTv1 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException {
 		ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
 	 * Returns CrossChainTradeData with useful info extracted from AT.
 	 */
-	@Override
-	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException {
+	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException {
 		byte[] addressBytes = new byte[25]; // for general use
 		String atAddress = atStateData.getATAddress();
 
@@ -597,13 +587,8 @@ public class LitecoinACCTv1 implements ACCT {
 		tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey);
 		tradeData.creationTimestamp = creationTimestamp;
 
-		if(optionalBalance.isPresent()) {
-			tradeData.qortBalance = optionalBalance.getAsLong();
-		}
-		else {
-			Account atAccount = new Account(repository, atAddress);
-			tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
-		}
+		Account atAccount = new Account(repository, atAddress);
+		tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
 
 		byte[] stateData = atStateData.getStateData();
 		ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData);

src/main/java/org/qortal/crosschain/LitecoinACCTv3.java

@@ -4,7 +4,6 @@ import com.google.common.hash.HashCode;
 import com.google.common.primitives.Bytes;
 import org.ciyam.at.*;
 import org.qortal.account.Account;
-import org.qortal.api.resource.CrossChainUtils;
 import org.qortal.asset.Asset;
 import org.qortal.at.QortalFunctionCode;
 import org.qortal.crypto.Crypto;
@@ -20,7 +19,6 @@ import org.qortal.utils.BitTwiddling;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
-import java.util.OptionalLong;
 
 import static org.ciyam.at.OpCode.calcOffset;
 
@@ -564,14 +562,7 @@ public class LitecoinACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
 		ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
-	}
-
-	@Override
-	public List<CrossChainTradeData> populateTradeDataList(Repository repository, List<ATData> atDataList) throws DataException {
-		List<CrossChainTradeData> crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList);
-
-		return crossChainTradeDataList;
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -580,14 +571,13 @@ public class LitecoinACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException {
 		ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
 	 * Returns CrossChainTradeData with useful info extracted from AT.
 	 */
-	@Override
-	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException {
+	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException {
 		byte[] addressBytes = new byte[25]; // for general use
 		String atAddress = atStateData.getATAddress();
 
@@ -600,13 +590,8 @@ public class LitecoinACCTv3 implements ACCT {
 		tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey);
 		tradeData.creationTimestamp = creationTimestamp;
 
-		if(optionalBalance.isPresent()) {
-			tradeData.qortBalance = optionalBalance.getAsLong();
-		}
-		else {
-			Account atAccount = new Account(repository, atAddress);
-			tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
-		}
+		Account atAccount = new Account(repository, atAddress);
+		tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
 
 		byte[] stateData = atStateData.getStateData();
 		ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData);

src/main/java/org/qortal/crosschain/NetTBD.java

@@ -3,19 +3,18 @@ package org.qortal.crosschain;
 import org.bitcoinj.core.NetworkParameters;
 
 import java.util.Collection;
-import java.util.concurrent.atomic.AtomicLong;
 
 public class NetTBD {
 
     private String name;
-    private AtomicLong feeRequired;
+    private long feeCeiling;
     private NetworkParameters params;
     private Collection<ElectrumX.Server> servers;
     private String genesisHash;
 
-    public NetTBD(String name, long feeRequired, NetworkParameters params, Collection<ElectrumX.Server> servers, String genesisHash) {
+    public NetTBD(String name, long feeCeiling, NetworkParameters params, Collection<ElectrumX.Server> servers, String genesisHash) {
         this.name = name;
-        this.feeRequired = new AtomicLong(feeRequired);
+        this.feeCeiling = feeCeiling;
         this.params = params;
         this.servers = servers;
         this.genesisHash = genesisHash;
@@ -26,14 +25,14 @@ public class NetTBD {
         return this.name;
     }
 
-    public long getFeeRequired() {
+    public long getFeeCeiling() {
 
-        return feeRequired.get();
+        return feeCeiling;
     }
 
-    public void setFeeRequired(long feeRequired) {
+    public void setFeeCeiling(long feeCeiling) {
 
-        this.feeRequired.set(feeRequired);
+        this.feeCeiling = feeCeiling;
     }
 
     public NetworkParameters getParams() {

src/main/java/org/qortal/crosschain/PirateChain.java

@@ -21,7 +21,6 @@ import org.qortal.utils.BitTwiddling;
 
 import java.nio.ByteBuffer;
 import java.util.*;
-import java.util.concurrent.atomic.AtomicLong;
 
 public class PirateChain extends Bitcoiny {
 
@@ -52,7 +51,12 @@ public class PirateChain extends Bitcoiny {
 			public Collection<Server> getServers() {
 				return Arrays.asList(
 					// Servers chosen on NO BASIS WHATSOEVER from various sources!
-					new Server("lightd.pirate.black", Server.ConnectionType.SSL, 443)
+					new Server("lightd.pirate.black", Server.ConnectionType.SSL, 443),
+					new Server("wallet-arrr1.qortal.online", Server.ConnectionType.SSL, 443),
+					new Server("wallet-arrr2.qortal.online", Server.ConnectionType.SSL, 443),
+					new Server("wallet-arrr3.qortal.online", Server.ConnectionType.SSL, 443),
+					new Server("wallet-arrr4.qortal.online", Server.ConnectionType.SSL, 443),
+					new Server("wallet-arrr5.qortal.online", Server.ConnectionType.SSL, 443)
 				);
 			}
 
@@ -63,7 +67,7 @@ public class PirateChain extends Bitcoiny {
 
 			@Override
 			public long getP2shFee(Long timestamp) {
-				return this.getFeeRequired();
+				return this.getFeeCeiling();
 			}
 		},
 		TEST3 {
@@ -113,14 +117,14 @@ public class PirateChain extends Bitcoiny {
 			}
 		};
 
-		private AtomicLong feeRequired = new AtomicLong(MAINNET_FEE);
+		private long feeCeiling = MAINNET_FEE;
 
-		public long getFeeRequired() {
-			return feeRequired.get();
+		public long getFeeCeiling() {
+			return feeCeiling;
 		}
 
-		public void setFeeRequired(long feeRequired) {
-			this.feeRequired.set(feeRequired);
+		public void setFeeCeiling(long feeCeiling) {
+			this.feeCeiling = feeCeiling;
 		}
 
 		public abstract NetworkParameters getParams();
@@ -182,14 +186,14 @@ public class PirateChain extends Bitcoiny {
 	}
 
 	@Override
-	public long getFeeRequired() {
-		return this.pirateChainNet.getFeeRequired();
+	public long getFeeCeiling() {
+		return this.pirateChainNet.getFeeCeiling();
 	}
 
 	@Override
-	public void setFeeRequired(long fee) {
+	public void setFeeCeiling(long fee) {
 
-		this.pirateChainNet.setFeeRequired( fee );
+		this.pirateChainNet.setFeeCeiling( fee );
 	}
 	/**
 	 * Returns confirmed balance, based on passed payment script.

src/main/java/org/qortal/crosschain/PirateChainACCTv3.java

@@ -4,7 +4,6 @@ import com.google.common.hash.HashCode;
 import com.google.common.primitives.Bytes;
 import org.ciyam.at.*;
 import org.qortal.account.Account;
-import org.qortal.api.resource.CrossChainUtils;
 import org.qortal.asset.Asset;
 import org.qortal.at.QortalFunctionCode;
 import org.qortal.crypto.Crypto;
@@ -20,7 +19,6 @@ import org.qortal.utils.BitTwiddling;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
-import java.util.OptionalLong;
 
 import static org.ciyam.at.OpCode.calcOffset;
 
@@ -582,14 +580,7 @@ public class PirateChainACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
 		ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
-	}
-
-	@Override
-	public List<CrossChainTradeData> populateTradeDataList(Repository repository, List<ATData> atDataList) throws DataException {
-		List<CrossChainTradeData> crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList);
-
-		return crossChainTradeDataList;
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -598,14 +589,13 @@ public class PirateChainACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException {
 		ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
 	 * Returns CrossChainTradeData with useful info extracted from AT.
 	 */
-	@Override
-	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException {
+	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException {
 		byte[] addressBytes = new byte[25]; // for general use
 		String atAddress = atStateData.getATAddress();
 
@@ -618,13 +608,8 @@ public class PirateChainACCTv3 implements ACCT {
 		tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey);
 		tradeData.creationTimestamp = creationTimestamp;
 
-		if(optionalBalance.isPresent()) {
-			tradeData.qortBalance = optionalBalance.getAsLong();
-		}
-		else {
-			Account atAccount = new Account(repository, atAddress);
-			tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
-		}
+		Account atAccount = new Account(repository, atAddress);
+		tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
 
 		byte[] stateData = atStateData.getStateData();
 		ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData);

src/main/java/org/qortal/crosschain/PirateWallet.java

@@ -8,7 +8,6 @@ import org.bouncycastle.util.encoders.DecoderException;
 import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
-import org.qortal.api.resource.CrossChainUtils;
 import org.qortal.controller.PirateChainWalletController;
 import org.qortal.crypto.Crypto;
 import org.qortal.settings.Settings;
@@ -68,8 +67,8 @@ public class PirateWallet {
             }
 
             // Pick a random server
-            ChainableServer server = PirateChain.getInstance().blockchainProvider.getCurrentServer();
-            String serverUri = String.format("https://%s:%d/", server.getHostName(), server.getPort());
+            PirateLightClient.Server server = this.getRandomServer();
+            String serverUri = String.format("https://%s:%d/", server.hostname, server.port);
 
             // Pirate library uses base64 encoding
             String entropy64 = Base64.toBase64String(this.entropyBytes);

src/main/java/org/qortal/crosschain/Ravencoin.java

@@ -14,7 +14,6 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.EnumMap;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
 
 public class Ravencoin extends Bitcoiny {
 
@@ -62,7 +61,7 @@ public class Ravencoin extends Bitcoiny {
 
 			@Override
 			public long getP2shFee(Long timestamp) {
-				return this.getFeeRequired();
+				return this.getFeeCeiling();
 			}
 		},
 		TEST3 {
@@ -112,14 +111,14 @@ public class Ravencoin extends Bitcoiny {
 			}
 		};
 
-		private AtomicLong feeRequired = new AtomicLong( MAINNET_FEE );
+		private long feeCeiling = MAINNET_FEE;
 
-		public long getFeeRequired() {
-			return feeRequired.get();
+		public long getFeeCeiling() {
+			return feeCeiling;
 		}
 
-		public void setFeeRequired(long feeRequired) {
-			this.feeRequired.set(feeRequired);
+		public void setFeeCeiling(long feeCeiling) {
+			this.feeCeiling = feeCeiling;
 		}
 
 		public abstract NetworkParameters getParams();
@@ -181,13 +180,13 @@ public class Ravencoin extends Bitcoiny {
 	}
 
 	@Override
-	public long getFeeRequired() {
-		return this.ravencoinNet.getFeeRequired();
+	public long getFeeCeiling() {
+		return this.ravencoinNet.getFeeCeiling();
 	}
 
 	@Override
-	public void setFeeRequired(long fee) {
+	public void setFeeCeiling(long fee) {
 
-		this.ravencoinNet.setFeeRequired( fee );
+		this.ravencoinNet.setFeeCeiling( fee );
 	}
 }

src/main/java/org/qortal/crosschain/RavencoinACCTv3.java

@@ -6,7 +6,6 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.ciyam.at.*;
 import org.qortal.account.Account;
-import org.qortal.api.resource.CrossChainUtils;
 import org.qortal.asset.Asset;
 import org.qortal.at.QortalFunctionCode;
 import org.qortal.crypto.Crypto;
@@ -22,7 +21,6 @@ import org.qortal.utils.BitTwiddling;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.List;
-import java.util.OptionalLong;
 
 import static org.ciyam.at.OpCode.calcOffset;
 
@@ -571,14 +569,7 @@ public class RavencoinACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
 		ATStateData atStateData = repository.getATRepository().getLatestATState(atData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
-	}
-
-	@Override
-	public List<CrossChainTradeData> populateTradeDataList(Repository repository, List<ATData> atDataList) throws DataException {
-		List<CrossChainTradeData> crossChainTradeDataList = CrossChainUtils.populateTradeDataList(repository, this, atDataList);
-
-		return crossChainTradeDataList;
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
@@ -587,14 +578,13 @@ public class RavencoinACCTv3 implements ACCT {
 	@Override
 	public CrossChainTradeData populateTradeData(Repository repository, ATStateData atStateData) throws DataException {
 		ATData atData = repository.getATRepository().fromATAddress(atStateData.getATAddress());
-		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData, OptionalLong.empty());
+		return populateTradeData(repository, atData.getCreatorPublicKey(), atData.getCreation(), atStateData);
 	}
 
 	/**
 	 * Returns CrossChainTradeData with useful info extracted from AT.
 	 */
-	@Override
-	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData, OptionalLong optionalBalance) throws DataException {
+	public CrossChainTradeData populateTradeData(Repository repository, byte[] creatorPublicKey, long creationTimestamp, ATStateData atStateData) throws DataException {
 		byte[] addressBytes = new byte[25]; // for general use
 		String atAddress = atStateData.getATAddress();
 
@@ -607,13 +597,8 @@ public class RavencoinACCTv3 implements ACCT {
 		tradeData.qortalCreator = Crypto.toAddress(creatorPublicKey);
 		tradeData.creationTimestamp = creationTimestamp;
 
-		if(optionalBalance.isPresent()) {
-			tradeData.qortBalance = optionalBalance.getAsLong();
-		}
-		else {
-			Account atAccount = new Account(repository, atAddress);
-			tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
-		}
+		Account atAccount = new Account(repository, atAddress);
+		tradeData.qortBalance = atAccount.getConfirmedBalance(Asset.QORT);
 
 		byte[] stateData = atStateData.getStateData();
 		ByteBuffer dataByteBuffer = ByteBuffer.wrap(stateData);

src/main/java/org/qortal/crypto/AES.java

@@ -100,7 +100,7 @@ public class AES {
         // Prepend the output stream with the 16 byte initialization vector
         outputStream.write(iv.getIV());
 
-        byte[] buffer = new byte[65536];
+        byte[] buffer = new byte[1024];
         int bytesRead;
         while ((bytesRead = inputStream.read(buffer)) != -1) {
             byte[] output = cipher.update(buffer, 0, bytesRead);
@@ -138,7 +138,7 @@ public class AES {
         Cipher cipher = Cipher.getInstance(algorithm);
         cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
 
-        byte[] buffer = new byte[65536];
+        byte[] buffer = new byte[64];
         int bytesRead;
         while ((bytesRead = inputStream.read(buffer)) != -1) {
             byte[] output = cipher.update(buffer, 0, bytesRead);

src/main/java/org/qortal/data/arbitrary/RNSArbitraryDirectConnectionInfo.java

@@ -0,0 +1,59 @@
+package org.qortal.data.arbitrary;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class RNSArbitraryDirectConnectionInfo {
+
+    private final byte[] signature;
+    private final String peerAddress;
+    private final List<byte[]> hashes;
+    private final long timestamp;
+
+    public RNSArbitraryDirectConnectionInfo(byte[] signature, String peerAddress, List<byte[]> hashes, long timestamp) {
+        this.signature = signature;
+        this.peerAddress = peerAddress;
+        this.hashes = hashes;
+        this.timestamp = timestamp;
+    }
+
+    public byte[] getSignature() {
+        return this.signature;
+    }
+
+    public String getPeerAddress() {
+        return this.peerAddress;
+    }
+
+    public List<byte[]> getHashes() {
+        return this.hashes;
+    }
+    
+    public long getTimestamp() {
+        return this.timestamp;
+    }
+
+    public int getHashCount() {
+        if (this.hashes == null) {
+            return 0;
+        }
+        return this.hashes.size();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this)
+            return true;
+
+        if (!(other instanceof ArbitraryDirectConnectionInfo))
+            return false;
+
+        ArbitraryDirectConnectionInfo otherDirectConnectionInfo = (ArbitraryDirectConnectionInfo) other;
+
+        return Arrays.equals(this.signature, otherDirectConnectionInfo.getSignature())
+                && Objects.equals(this.peerAddress, otherDirectConnectionInfo.getPeerAddress())
+                && Objects.equals(this.hashes, otherDirectConnectionInfo.getHashes())
+                && Objects.equals(this.timestamp, otherDirectConnectionInfo.getTimestamp());
+    }
+}

src/main/java/org/qortal/data/arbitrary/RNSArbitraryFileListResponseInfo.java

@@ -0,0 +1,11 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.network.RNSPeer;
+
+public class RNSArbitraryFileListResponseInfo extends  RNSArbitraryRelayInfo {
+
+    public RNSArbitraryFileListResponseInfo(String hash58, String signature58, RNSPeer peer, Long timestamp, Long requestTime, Integer requestHops) {
+        super(hash58, signature58, peer, timestamp, requestTime, requestHops);
+    }
+
+}

src/main/java/org/qortal/data/arbitrary/RNSArbitraryRelayInfo.java

@@ -0,0 +1,73 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.network.RNSPeer;
+
+import java.util.Objects;
+
+public class RNSArbitraryRelayInfo {
+
+    private final String hash58;
+    private final String signature58;
+    private final RNSPeer peer;
+    private final Long timestamp;
+    private final Long requestTime;
+    private final Integer requestHops;
+
+    public RNSArbitraryRelayInfo(String hash58, String signature58, RNSPeer peer, Long timestamp, Long requestTime, Integer requestHops) {
+        this.hash58 = hash58;
+        this.signature58 = signature58;
+        this.peer = peer;
+        this.timestamp = timestamp;
+        this.requestTime = requestTime;
+        this.requestHops = requestHops;
+    }
+
+    public boolean isValid() {
+        return this.getHash58() != null && this.getSignature58() != null
+                && this.getPeer() != null && this.getTimestamp() != null;
+    }
+
+    public String getHash58() {
+        return this.hash58;
+    }
+
+    public String getSignature58() {
+        return signature58;
+    }
+
+    public RNSPeer getPeer() {
+        return peer;
+    }
+
+    public Long getTimestamp() {
+        return timestamp;
+    }
+
+    public Long getRequestTime() {
+        return this.requestTime;
+    }
+
+    public Integer getRequestHops() {
+        return this.requestHops;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this)
+            return true;
+
+        if (!(other instanceof RNSArbitraryRelayInfo))
+            return false;
+
+        RNSArbitraryRelayInfo otherRelayInfo = (RNSArbitraryRelayInfo) other;
+
+        return this.peer == otherRelayInfo.getPeer()
+                && Objects.equals(this.hash58, otherRelayInfo.getHash58())
+                && Objects.equals(this.signature58, otherRelayInfo.getSignature58());
+    }
+}

src/main/java/org/qortal/data/crosschain/ForeignFeeData.java

@@ -1,57 +0,0 @@
-package org.qortal.data.crosschain;
-
-import org.json.JSONObject;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
-
-// All properties to be converted to JSON via JAXB
-@XmlAccessorType(XmlAccessType.FIELD)
-public class ForeignFeeData {
-
-	private String blockchain;
-
-	@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
-	private long fee;
-
-	protected ForeignFeeData() {
-		/* JAXB */
-	}
-
-	public ForeignFeeData(String blockchain,
-                          long fee) {
-		this.blockchain = blockchain;
-		this.fee = fee;
-	}
-
-	public String getBlockchain() {
-		return this.blockchain;
-	}
-
-	public long getFee() {
-		return this.fee;
-	}
-
-	public JSONObject toJson() {
-		JSONObject jsonObject = new JSONObject();
-		jsonObject.put("blockchain", this.getBlockchain());
-	    jsonObject.put("fee", this.getFee());
-		return jsonObject;
-	}
-
-	public static ForeignFeeData fromJson(JSONObject json) {
-		return new ForeignFeeData(
-				json.isNull("blockchain") ? null : json.getString("blockchain"),
-				json.isNull("fee") ? null : json.getLong("fee")
-		);
-	}
-
-	@Override
-	public String toString() {
-		return "ForeignFeeData{" +
-				"blockchain='" + blockchain + '\'' +
-				", fee=" + fee +
-				'}';
-	}
-}

src/main/java/org/qortal/data/crosschain/ForeignFeeDecodedData.java

@@ -1,90 +0,0 @@
-package org.qortal.data.crosschain;
-
-import org.json.JSONObject;
-import org.qortal.data.account.MintingAccountData;
-import org.qortal.utils.Base58;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import java.util.Objects;
-
-// All properties to be converted to JSON via JAXB
-@XmlAccessorType(XmlAccessType.FIELD)
-public class ForeignFeeDecodedData {
-
-	protected long timestamp;
-	protected byte[] data;
-	protected String atAddress;
-	protected Integer fee;
-
-	// Constructors
-
-	// necessary for JAXB serialization
-	protected ForeignFeeDecodedData() {
-	}
-
-	public ForeignFeeDecodedData(long timestamp, byte[] data, String atAddress, Integer fee) {
-		this.timestamp = timestamp;
-		this.data = data;
-		this.atAddress = atAddress;
-		this.fee = fee;
-	}
-
-	public long getTimestamp() {
-		return this.timestamp;
-	}
-
-	public byte[] getData() {
-		return this.data;
-	}
-
-	public String getAtAddress() {
-		return atAddress;
-	}
-
-	public Integer getFee() {
-		return this.fee;
-	}
-
-	// Comparison
-
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o) return true;
-		if (o == null || getClass() != o.getClass()) return false;
-		ForeignFeeDecodedData that = (ForeignFeeDecodedData) o;
-		return timestamp == that.timestamp && Objects.equals(atAddress, that.atAddress) && Objects.equals(fee, that.fee);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hash(timestamp, atAddress, fee);
-	}
-
-	@Override
-	public String toString() {
-		return "ForeignFeeDecodedData{" +
-				"timestamp=" + timestamp +
-				", atAddress='" + atAddress + '\'' +
-				", fee=" + fee +
-				'}';
-	}
-
-	public JSONObject toJson() {
-		JSONObject jsonObject = new JSONObject();
-		jsonObject.put("data", Base58.encode(this.data));
-		jsonObject.put("atAddress", this.atAddress);
-		jsonObject.put("timestamp", this.timestamp);
-		jsonObject.put("fee", this.fee);
-		return jsonObject;
-	}
-
-	public static ForeignFeeDecodedData fromJson(JSONObject json) {
-		return new ForeignFeeDecodedData(
-				json.isNull("timestamp") ? null : json.getLong("timestamp"),
-				json.isNull("data") ? null : Base58.decode(json.getString("data")),
-				json.isNull("atAddress") ? null : json.getString("atAddress"),
-				json.isNull("fee") ? null : json.getInt("fee"));
-	}
-}

src/main/java/org/qortal/data/crosschain/ForeignFeeEncodedData.java

@@ -1,69 +0,0 @@
-package org.qortal.data.crosschain;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import java.util.Objects;
-
-// All properties to be converted to JSON via JAXB
-@XmlAccessorType(XmlAccessType.FIELD)
-public class ForeignFeeEncodedData {
-
-	protected long timestamp;
-	protected String data;
-	protected String atAddress;
-	protected Integer fee;
-
-	// Constructors
-
-	// necessary for JAXB serialization
-	protected ForeignFeeEncodedData() {
-	}
-
-	public ForeignFeeEncodedData(long timestamp, String data, String atAddress, Integer fee) {
-		this.timestamp = timestamp;
-		this.data = data;
-		this.atAddress = atAddress;
-		this.fee = fee;
-	}
-
-	public long getTimestamp() {
-		return this.timestamp;
-	}
-
-	public String getData() {
-		return this.data;
-	}
-
-	public String getAtAddress() {
-		return atAddress;
-	}
-
-	public Integer getFee() {
-		return this.fee;
-	}
-
-	// Comparison
-
-
-	@Override
-	public boolean equals(Object o) {
-		if (this == o) return true;
-		if (o == null || getClass() != o.getClass()) return false;
-		ForeignFeeEncodedData that = (ForeignFeeEncodedData) o;
-		return timestamp == that.timestamp && Objects.equals(atAddress, that.atAddress) && Objects.equals(fee, that.fee);
-	}
-
-	@Override
-	public int hashCode() {
-		return Objects.hash(timestamp, atAddress, fee);
-	}
-
-	@Override
-	public String toString() {
-		return "ForeignFeeDecodedData{" +
-				"timestamp=" + timestamp +
-				", atAddress='" + atAddress + '\'' +
-				", fee=" + fee +
-				'}';
-	}
-}

src/main/java/org/qortal/data/crosschain/UnsignedFeeEvent.java

@@ -1,29 +0,0 @@
-package org.qortal.data.crosschain;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-
-@XmlAccessorType(XmlAccessType.FIELD)
-public class UnsignedFeeEvent {
-
-    private boolean positive;
-
-    private String address;
-
-    public UnsignedFeeEvent() {
-    }
-
-    public UnsignedFeeEvent(boolean positive, String address) {
-
-        this.positive = positive;
-        this.address = address;
-    }
-
-    public boolean isPositive() {
-        return positive;
-    }
-
-    public String getAddress() {
-        return address;
-    }
-}

src/main/java/org/qortal/data/naming/NameData.java

@@ -67,11 +67,6 @@ public class NameData {
 		this(name, reducedName, owner, data, registered, null, false, null, reference, creationGroupId);
 	}
 
-	// Typically used for name summsry
-	public NameData(String name, String owner) {
-		this(name, null, owner, null, 0L, null, false, null, null, 0);
-	}
-
 	// Getters / setters
 
 	public String getName() {

src/main/java/org/qortal/data/network/RNSPeerData.java

@@ -0,0 +1,117 @@
+package org.qortal.data.network;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import org.qortal.network.PeerAddress;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlTransient;
+import static org.apache.commons.codec.binary.Hex.encodeHexString;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class RNSPeerData {
+
+	//public static final int MAX_PEER_ADDRESS_SIZE = 255;
+
+	// Properties
+
+	//// Don't expose this via JAXB - use pretty getter instead
+	//@XmlTransient
+	//@Schema(hidden = true)
+	//private PeerAddress peerAddress;
+	private byte[] peerAddress;
+
+	private Long lastAttempted;
+	private Long lastConnected;
+	private Long lastMisbehaved;
+	private Long addedWhen;
+	private String addedBy;
+
+	/** The number of consecutive times we failed to sync with this peer */
+	private int failedSyncCount = 0;
+
+	// Constructors
+
+	// necessary for JAXB serialization
+	protected RNSPeerData() {
+	}
+
+	public RNSPeerData(byte[] peerAddress, Long lastAttempted, Long lastConnected, Long lastMisbehaved, Long addedWhen, String addedBy) {
+		this.peerAddress = peerAddress;
+		this.lastAttempted = lastAttempted;
+		this.lastConnected = lastConnected;
+		this.lastMisbehaved = lastMisbehaved;
+		this.addedWhen = addedWhen;
+		this.addedBy = addedBy;
+	}
+
+	public RNSPeerData(byte[] peerAddress, Long addedWhen, String addedBy) {
+		this(peerAddress, null, null, null, addedWhen, addedBy);
+	}
+
+	public RNSPeerData(byte[] peerAddress) {
+		this(peerAddress, null, null, null, null, null);
+	}
+
+	// Getters / setters
+
+	// Don't let JAXB use this getter
+	@XmlTransient
+	@Schema(hidden = true)
+	public byte[] getAddress() {
+		return this.peerAddress;
+	}
+
+	public Long getLastAttempted() {
+		return this.lastAttempted;
+	}
+
+	public void setLastAttempted(Long lastAttempted) {
+		this.lastAttempted = lastAttempted;
+	}
+
+	public Long getLastConnected() {
+		return this.lastConnected;
+	}
+
+	public void setLastConnected(Long lastConnected) {
+		this.lastConnected = lastConnected;
+	}
+
+	public Long getLastMisbehaved() {
+		return this.lastMisbehaved;
+	}
+
+	public void setLastMisbehaved(Long lastMisbehaved) {
+		this.lastMisbehaved = lastMisbehaved;
+	}
+
+	public Long getAddedWhen() {
+		return this.addedWhen;
+	}
+
+	public String getAddedBy() {
+		return this.addedBy;
+	}
+
+	public int getFailedSyncCount() {
+		return this.failedSyncCount;
+	}
+
+	public void setFailedSyncCount(int failedSyncCount) {
+		this.failedSyncCount = failedSyncCount;
+	}
+
+	public void incrementFailedSyncCount() {
+		this.failedSyncCount++;
+	}
+
+	// Pretty peerAddress getter for JAXB
+	@XmlElement(name = "address")
+	protected String getPrettyAddress() {
+		return encodeHexString(this.peerAddress);
+	}
+
+}