Don't process file hashes if we're stopping

An alternate option would be to avoid force disconnecting while relays are in progress, but some nodes could have active relays 100% of the time and therefore would never recycle their peers. So it is simpler to just increase the average peer connection time for everyone.

.gitignore

@@ -28,3 +28,6 @@
 /WindowsInstaller/Install Files/qortal.jar
 /*.7z
 /tmp
+/data*
+/src/test/resources/arbitrary/*/.qortal/cache
+apikey.txt

Dockerfile

@@ -0,0 +1,26 @@
+FROM maven:3-openjdk-11 as builder
+
+WORKDIR /work
+COPY ./ /work/
+RUN mvn clean package
+
+###
+FROM openjdk:11
+
+RUN useradd -r -u 1000 -g users qortal && \
+    mkdir /usr/local/qortal /qortal && \
+    chown 1000:100 /qortal
+
+COPY --from=builder /work/log4j2.properties /usr/local/qortal/
+COPY --from=builder /work/target/qortal*.jar /usr/local/qortal/qortal.jar
+
+USER 1000:100
+
+EXPOSE 12391 12392
+HEALTHCHECK --start-period=5m CMD curl -sf http://127.0.0.1:12391/admin/info || exit 1
+
+WORKDIR /qortal
+VOLUME /qortal
+
+ENTRYPOINT ["java"]
+CMD ["-Djava.net.preferIPv4Stack=false", "-jar", "/usr/local/qortal/qortal.jar"]

WindowsInstaller/Install Files/log4j2.properties

@@ -61,7 +61,7 @@ appender.rolling.type = RollingFile
 appender.rolling.name = FILE
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-appender.rolling.filePattern = ${dirname:-}${filename}.%i
+appender.rolling.filePattern = ./${filename}.%i
 appender.rolling.policy.type = SizeBasedTriggeringPolicy
 appender.rolling.policy.size = 4MB
 # Set the immediate flush to true (default)

WindowsInstaller/Qortal.aip

@@ -2,7 +2,7 @@
 <DOCUMENT Type="Advanced Installer" CreateVersion="14.9" version="18.2" Modules="enterprise" RootPath="." Language="en_GB" Id="{713E21E0-28FC-422F-8A95-823D01A5F80B}">
   <COMPONENT cid="caphyon.advinst.msicomp.MsiPropsComponent">
     <ROW Property="AI_BITMAP_DISPLAY_MODE" Value="0"/>
-    <ROW Property="AI_CURRENT_YEAR" Value="2021" ValueLocId="-"/>
+    <ROW Property="AI_CURRENT_YEAR" Value="2022" ValueLocId="-"/>
     <ROW Property="AI_FINDEXE_TITLE" Value="Select the installation package for [|ProductName]" ValueLocId="AI.Property.FindExeTitle"/>
     <ROW Property="AI_PRODUCTNAME_ARP" Value="Qortal"/>
     <ROW Property="AI_RUN_AS_ADMIN" Value="0"/>
@@ -17,10 +17,10 @@
     <ROW Property="Manufacturer" Value="Qortal"/>
     <ROW Property="MsiLogging" MultiBuildValue="DefaultBuild:vp"/>
     <ROW Property="NTP_GOOD" Value="false"/>
-    <ROW Property="ProductCode" Value="1033:{51EFA0B0-C304-4043-AF6D-2C17C783A998} 1049:{C4662BB2-A247-426E-A128-B7DBD12ECE78} 2052:{1AF44520-C8AB-4261-BCFE-EEA941439718} 2057:{C096EB6A-F43F-45EE-921C-D20F9B993E80} " Type="16"/>
+    <ROW Property="ProductCode" Value="1033:{9BDE0BDF-72A2-44DA-8B55-E7C129DBE603} 1049:{F4FCC1D9-D286-4B3D-A50F-82034010A30F} 2052:{DBE9D682-F666-49BA-8B63-28C0AE06CBCA} 2057:{949F4DFE-E55C-4493-AAB6-5DB13E68C754} " Type="16"/>
     <ROW Property="ProductLanguage" Value="2057"/>
     <ROW Property="ProductName" Value="Qortal"/>
-    <ROW Property="ProductVersion" Value="2.1.3" Type="32"/>
+    <ROW Property="ProductVersion" Value="3.0.4" Type="32"/>
     <ROW Property="RECONFIG_NTP" Value="true"/>
     <ROW Property="REMOVE_BLOCKCHAIN" Value="YES" Type="4"/>
     <ROW Property="REPAIR_BLOCKCHAIN" Value="YES" Type="4"/>
@@ -212,7 +212,7 @@
     <ROW Component="ADDITIONAL_LICENSE_INFO_71" ComponentId="{12A3ADBE-BB7A-496C-8869-410681E6232F}" Directory_="jdk.zipfs_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_71" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_8" ComponentId="{D53AD95E-CF96-4999-80FC-5812277A7456}" Directory_="java.naming_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_8" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_9" ComponentId="{6B7EA9B0-5D17-47A8-B78C-FACE86D15E01}" Directory_="java.net.http_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_9" Type="0"/>
-    <ROW Component="AI_CustomARPName" ComponentId="{9A243E53-8FC9-4854-B6E2-937493305BB4}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
+    <ROW Component="AI_CustomARPName" ComponentId="{9B71A82D-8C25-40FD-806D-44BAD0B45AA2}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
     <ROW Component="AI_ExePath" ComponentId="{3644948D-AE0B-41BB-9FAF-A79E70490A08}" Directory_="APPDIR" Attributes="260" KeyPath="AI_ExePath"/>
     <ROW Component="APPDIR" ComponentId="{680DFDDE-3FB4-47A5-8FF5-934F576C6F91}" Directory_="APPDIR" Attributes="0"/>
     <ROW Component="AccessBridgeCallbacks.h" ComponentId="{288055D1-1062-47A3-AA44-5601B4E38AED}" Directory_="bridge_Dir" Attributes="0" KeyPath="AccessBridgeCallbacks.h" Type="0"/>
@@ -1173,7 +1173,7 @@
     <ROW Action="AI_STORE_LOCATION" Type="51" Source="ARPINSTALLLOCATION" Target="[APPDIR]"/>
     <ROW Action="AI_SetPermissions" Type="11265" Source="userAccounts.dll" Target="OnSetPermissions" WithoutSeq="true"/>
     <ROW Action="CustomizeLog4j2PropertiesScript" Type="3109" Target="Script Text" TargetUnformatted="var actionData = Session.Property(&quot;CustomActionData&quot;);&#13;&#10;var actionDataArray = actionData.split(&quot;|&quot;);&#13;&#10;var appDir = actionDataArray[0];&#13;&#10;var dataFolder = actionDataArray[1] + actionDataArray[2] + &quot;\\&quot;;&#13;&#10;&#13;&#10;var ForReading = 1, ForWriting = 2, ForAppending = 8;&#13;&#10;var fso = new ActiveXObject(&quot;Scripting.FileSystemObject&quot;);&#13;&#10;&#13;&#10;// Make copy&#13;&#10;fso.CopyFile(appDir + &quot;log4j2.properties&quot;, appDir + &quot;log4j2-orig.properties&quot;, true); // overwrite&#13;&#10;&#13;&#10;// Rewrite %AppDir%\log4j2.properties to update logfile storage path&#13;&#10;var fin = fso.OpenTextFile(appDir + &quot;log4j2-orig.properties&quot;, ForReading, false); // no create&#13;&#10;var fout = fso.OpenTextFile(appDir + &quot;log4j2.properties&quot;, ForWriting, true); // can create&#13;&#10;&#13;&#10;// Copy lines with rewriting where necessary&#13;&#10;while( !fin.AtEndOfStream ) {&#13;&#10;&#9;var line = fin.ReadLine();&#13;&#10;&#13;&#10;&#9;var start = line.indexOf(&quot;property.dirname&quot;);&#13;&#10;&#9;if (start &gt; 0) {&#13;&#10;&#9;&#9;// line: # property.dirname = ...appdata...&#13;&#10;&#9;&#9;// uncomment/replace this line for Windows&#13;&#10;&#9;&#9;fout.WriteLine( &quot;property.dirname = &quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) );&#13;&#10;&#9;} else {&#13;&#10;&#9;&#9;// not found - output verbatim&#13;&#10;&#9;&#9;fout.WriteLine( line );&#13;&#10;&#9;}&#13;&#10;}&#13;&#10;&#13;&#10;fin.Close();&#13;&#10;fout.Close();&#13;&#10;" AdditionalSeq="AI_DATA_SETTER_4"/>
-    <ROW Action="CustomizeSettingsJsonScript" Type="3109" Target="Script Text" TargetUnformatted="var actionData = Session.Property(&quot;CustomActionData&quot;);&#13;&#10;var actionDataArray = actionData.split(&quot;|&quot;);&#13;&#10;var appDir = actionDataArray[0];&#13;&#10;var dataFolder = actionDataArray[1] + actionDataArray[2] + &quot;\\&quot;;&#13;&#10;&#13;&#10;var ForReading = 1, ForWriting = 2, ForAppending = 8;&#13;&#10;var fso = new ActiveXObject(&quot;Scripting.FileSystemObject&quot;);&#13;&#10;&#13;&#10;// Create basic %APPDIR%\settings.json with path to real settings.json in dataFolder&#13;&#10;var fts = fso.OpenTextFile(appDir + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;fts.WriteLine( &quot;{&quot; );&#13;&#10;// We need to escape Windows path backslashes to keep JSON valid&#13;&#10;fts.WriteLine( &quot;  \&quot;userPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;\&quot;&quot; );&#13;&#10;fts.WriteLine( &quot;}&quot; );&#13;&#10;&#13;&#10;fts.Close();&#13;&#10;&#13;&#10;// Make copy&#13;&#10;fso.CopyFile(dataFolder + &quot;settings.json&quot;, dataFolder + &quot;settings-orig.json&quot;, true); // overwrite&#13;&#10;&#13;&#10;// Rewrite settings.json to update repository path&#13;&#10;var fin = fso.OpenTextFile(dataFolder + &quot;settings-orig.json&quot;, ForReading, false);&#13;&#10;var fout = fso.OpenTextFile(dataFolder + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;// First line should contain opening brace&#13;&#10;fout.WriteLine( fin.ReadLine() );&#13;&#10;&#13;&#10;// Append our entries&#13;&#10;fout.WriteLine( &quot;  \&quot;repositoryPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;db\&quot;,&quot; );&#13;&#10;&#13;&#10;// copy rest of settings&#13;&#10;while( !fin.AtEndOfStream ) {&#13;&#10;&#9;fout.WriteLine( fin.ReadLine() );&#13;&#10;}&#13;&#10;&#13;&#10;fin.Close();&#13;&#10;fout.Close();&#13;&#10;" AdditionalSeq="AI_DATA_SETTER_3"/>
+    <ROW Action="CustomizeSettingsJsonScript" Type="3109" Target="Script Text" TargetUnformatted="var actionData = Session.Property(&quot;CustomActionData&quot;);&#13;&#10;var actionDataArray = actionData.split(&quot;|&quot;);&#13;&#10;var appDir = actionDataArray[0];&#13;&#10;var dataFolder = actionDataArray[1] + actionDataArray[2] + &quot;\\&quot;;&#13;&#10;&#13;&#10;var ForReading = 1, ForWriting = 2, ForAppending = 8;&#13;&#10;var fso = new ActiveXObject(&quot;Scripting.FileSystemObject&quot;);&#13;&#10;&#13;&#10;// Create basic %APPDIR%\settings.json with path to real settings.json in dataFolder&#13;&#10;var fts = fso.OpenTextFile(appDir + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;fts.WriteLine( &quot;{&quot; );&#13;&#10;// We need to escape Windows path backslashes to keep JSON valid&#13;&#10;fts.WriteLine( &quot;  \&quot;userPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;\&quot;&quot; );&#13;&#10;fts.WriteLine( &quot;}&quot; );&#13;&#10;&#13;&#10;fts.Close();&#13;&#10;&#13;&#10;// Make copy&#13;&#10;fso.CopyFile(dataFolder + &quot;settings.json&quot;, dataFolder + &quot;settings-orig.json&quot;, true); // overwrite&#13;&#10;&#13;&#10;// Rewrite settings.json to update repository path&#13;&#10;var fin = fso.OpenTextFile(dataFolder + &quot;settings-orig.json&quot;, ForReading, false);&#13;&#10;var fout = fso.OpenTextFile(dataFolder + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;// First line should contain opening brace&#13;&#10;fout.WriteLine( fin.ReadLine() );&#13;&#10;&#13;&#10;// Append our entries&#13;&#10;fout.WriteLine( &quot;  \&quot;repositoryPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;db\&quot;,&quot; );&#13;&#10;fout.WriteLine( &quot;  \&quot;dataPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;data\&quot;,&quot; );&#13;&#10;&#13;&#10;// copy rest of settings&#13;&#10;while( !fin.AtEndOfStream ) {&#13;&#10;&#9;fout.WriteLine( fin.ReadLine() );&#13;&#10;}&#13;&#10;&#13;&#10;fin.Close();&#13;&#10;fout.Close();&#13;&#10;" AdditionalSeq="AI_DATA_SETTER_3"/>
     <ROW Action="DetectRunningProcess" Type="1" Source="aicustact.dll" Target="DetectProcess" Options="3" AdditionalSeq="AI_DATA_SETTER_8"/>
     <ROW Action="DetectW32Time" Type="1" Source="aicustact.dll" Target="DetectService" Options="3" AdditionalSeq="AI_DATA_SETTER_11"/>
     <ROW Action="NTP_config" Type="3090" Source="ntpcfg.bat"/>

log4j2.properties

@@ -61,7 +61,7 @@ appender.rolling.type = RollingFile
 appender.rolling.name = FILE
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-appender.rolling.filePattern = ${dirname:-}${filename}.%i
+appender.rolling.filePattern = ./${filename}.%i
 appender.rolling.policy.type = SizeBasedTriggeringPolicy
 appender.rolling.policy.size = 4MB
 # Set the immediate flush to true (default)

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>2.1.3</version>
+	<version>3.0.4</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>
@@ -16,18 +16,21 @@
 		<commons-text.version>1.8</commons-text.version>
 		<commons-io.version>2.6</commons-io.version>
 		<commons-compress.version>1.21</commons-compress.version>
+		<commons-lang3.version>3.12.0</commons-lang3.version>
 		<xz.version>1.9</xz.version>
 		<dagger.version>1.2.2</dagger.version>
 		<guava.version>28.1-jre</guava.version>
 		<hsqldb.version>2.5.1</hsqldb.version>
 		<jersey.version>2.29.1</jersey.version>
 		<jetty.version>9.4.29.v20200521</jetty.version>
-		<log4j.version>2.12.1</log4j.version>
+		<log4j.version>2.17.1</log4j.version>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<slf4j.version>1.7.12</slf4j.version>
 		<swagger-api.version>2.0.9</swagger-api.version>
 		<swagger-ui.version>3.23.8</swagger-ui.version>
 		<package-info-maven-plugin.version>1.1.0</package-info-maven-plugin.version>
+		<jsoup.version>1.13.1</jsoup.version>
+		<java-diff-utils.version>4.10</java-diff-utils.version>
 	</properties>
 	<build>
 		<sourceDirectory>src/main/java</sourceDirectory>
@@ -462,6 +465,11 @@
 			<artifactId>commons-compress</artifactId>
 			<version>${commons-compress.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-lang3</artifactId>
+			<version>${commons-lang3.version}</version>
+		</dependency>
 		<dependency>
 			<groupId>org.tukaani</groupId>
 			<artifactId>xz</artifactId>
@@ -667,5 +675,15 @@
 			<artifactId>bctls-jdk15on</artifactId>
 			<version>${bouncycastle.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>org.jsoup</groupId>
+			<artifactId>jsoup</artifactId>
+			<version>${jsoup.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>io.github.java-diff-utils</groupId>
+			<artifactId>java-diff-utils</artifactId>
+			<version>${java-diff-utils.version}</version>
+		</dependency>
 	</dependencies>
 </project>

src/main/java/org/qortal/ApplyUpdate.java

@@ -7,14 +7,13 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.security.Security;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
+import org.qortal.api.ApiKey;
 import org.qortal.api.ApiRequest;
 import org.qortal.controller.AutoUpdate;
 import org.qortal.settings.Settings;
@@ -70,14 +69,40 @@ public class ApplyUpdate {
 		String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
 		LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri));
 
+		// The /admin/stop endpoint requires an API key, which may or may not be already generated
+		boolean apiKeyNewlyGenerated = false;
+		ApiKey apiKey = null;
+		try {
+			apiKey = new ApiKey();
+			if (!apiKey.generated()) {
+				apiKey.generate();
+				apiKeyNewlyGenerated = true;
+				LOGGER.info("Generated API key");
+			}
+		} catch (IOException e) {
+			LOGGER.info("Error loading API key: {}", e.getMessage());
+		}
+
+		// Create GET params
+		Map<String, String> params = new HashMap<>();
+		if (apiKey != null) {
+			params.put("apiKey", apiKey.toString());
+		}
+
+		// Attempt to stop the node
 		int attempt;
 		for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) {
 			final int attemptForLogging = attempt;
 			LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS));
-			String response = ApiRequest.perform(baseUri + "admin/stop", null);
-			if (response == null)
+			String response = ApiRequest.perform(baseUri + "admin/stop", params);
+			if (response == null) {
 				// No response - consider node shut down
+				if (apiKeyNewlyGenerated) {
+					// API key was newly generated for this auto update, so we need to remove it
+					ApplyUpdate.removeGeneratedApiKey();
+				}
 				return true;
+			}
 
 			LOGGER.info(() -> String.format("Response from API: %s", response));
 
@@ -89,6 +114,11 @@ public class ApplyUpdate {
 			}
 		}
 
+		if (apiKeyNewlyGenerated) {
+			// API key was newly generated for this auto update, so we need to remove it
+			ApplyUpdate.removeGeneratedApiKey();
+		}
+
 		if (attempt == MAX_ATTEMPTS) {
 			LOGGER.error("Failed to shutdown node - giving up");
 			return false;
@@ -97,6 +127,19 @@ public class ApplyUpdate {
 		return true;
 	}
 
+	private static void removeGeneratedApiKey() {
+		try {
+			LOGGER.info("Removing newly generated API key...");
+
+			// Delete the API key since it was only generated for this auto update
+			ApiKey apiKey = new ApiKey();
+			apiKey.delete();
+
+		} catch (IOException e) {
+			LOGGER.info("Error loading or deleting API key: {}", e.getMessage());
+		}
+	}
+
 	private static void replaceJar() {
 		// Assuming current working directory contains the JAR files
 		Path realJar = Paths.get(JAR_FILENAME);

src/main/java/org/qortal/api/ApiError.java

@@ -132,7 +132,11 @@ public enum ApiError {
 	FOREIGN_BLOCKCHAIN_TOO_SOON(1203, 408),
 
 	// Trade portal
-	ORDER_SIZE_TOO_SMALL(1300, 402);
+	ORDER_SIZE_TOO_SMALL(1300, 402),
+
+	// Data
+	FILE_NOT_FOUND(1401, 404),
+	NO_REPLY(1402, 404);
 
 	private static final Map<Integer, ApiError> map = stream(ApiError.values()).collect(toMap(apiError -> apiError.code, apiError -> apiError));
 

src/main/java/org/qortal/api/ApiKey.java

@@ -0,0 +1,107 @@
+package org.qortal.api;
+
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.SecureRandom;
+
+public class ApiKey {
+
+    private String apiKey;
+
+    public ApiKey() throws IOException {
+        this.load();
+    }
+
+    public void generate() throws IOException {
+        byte[] apiKey = new byte[16];
+        new SecureRandom().nextBytes(apiKey);
+        this.apiKey = Base58.encode(apiKey);
+
+        this.save();
+    }
+
+
+    /* Filesystem */
+
+    private Path getFilePath() {
+        return Paths.get(Settings.getInstance().getApiKeyPath(), "apikey.txt");
+    }
+
+    private boolean load() throws IOException {
+        Path path = this.getFilePath();
+        File apiKeyFile = new File(path.toString());
+        if (!apiKeyFile.exists()) {
+            // Try settings - to allow legacy API keys to be supported
+            return this.loadLegacyApiKey();
+        }
+
+        try {
+            this.apiKey = new String(Files.readAllBytes(path));
+
+        } catch (IOException e) {
+            throw new IOException(String.format("Couldn't read contents from file %s", path.toString()));
+        }
+
+        return true;
+    }
+
+    private boolean loadLegacyApiKey() {
+        String legacyApiKey = Settings.getInstance().getApiKey();
+        if (legacyApiKey != null && !legacyApiKey.isEmpty()) {
+            this.apiKey = Settings.getInstance().getApiKey();
+
+            try {
+                // Save it to the apikey file
+                this.save();
+            } catch (IOException e) {
+                // Ignore failures as it will be reloaded from settings next time
+            }
+            return true;
+        }
+        return false;
+    }
+
+    public void save() throws IOException {
+        if (this.apiKey == null || this.apiKey.isEmpty()) {
+            throw new IllegalStateException("Unable to save a blank API key");
+        }
+
+        Path filePath = this.getFilePath();
+
+        BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toString()));
+        writer.write(this.apiKey);
+        writer.close();
+    }
+
+    public void delete() throws IOException {
+        this.apiKey = null;
+
+        Path filePath = this.getFilePath();
+        if (Files.exists(filePath)) {
+            Files.delete(filePath);
+        }
+    }
+
+
+    public boolean generated() {
+        return (this.apiKey != null);
+    }
+
+    public boolean exists() {
+        return this.getFilePath().toFile().exists();
+    }
+
+    @Override
+    public String toString() {
+        return this.apiKey;
+    }
+
+}

src/main/java/org/qortal/api/ApiService.java

@@ -14,8 +14,7 @@ import java.security.SecureRandom;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.checkerframework.checker.units.qual.A;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.rewrite.handler.RedirectPatternRule;
 import org.eclipse.jetty.rewrite.handler.RewriteHandler;
@@ -52,12 +51,11 @@ import org.qortal.settings.Settings;
 
 public class ApiService {
 
-	private static final Logger LOGGER = LogManager.getLogger(ApiService.class);
-
 	private static ApiService instance;
 
 	private final ResourceConfig config;
 	private Server server;
+	private ApiKey apiKey;
 
 	private ApiService() {
 		this.config = new ResourceConfig();
@@ -78,6 +76,15 @@ public class ApiService {
 		return this.config.getClasses();
 	}
 
+	public void setApiKey(ApiKey apiKey) {
+		this.apiKey = apiKey;
+	}
+
+	public ApiKey getApiKey() {
+		return this.apiKey;
+	}
+
+
 	public void start() {
 		try {
 			// Create API server
@@ -207,9 +214,6 @@ public class ApiService {
 			context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot");
 			context.addServlet(PresenceWebSocket.class, "/websockets/presence");
 
-			// Warn about API security if needed
-			this.checkApiSecurity();
-
 			// Start server
 			this.server.start();
 		} catch (Exception e) {
@@ -229,23 +233,4 @@ public class ApiService {
 		this.server = null;
 	}
 
-	private void checkApiSecurity() {
-		// Warn about API security if needed
-		boolean allConnectionsAllowed = false;
-		if (Settings.getInstance().isApiKeyDisabled()) {
-			for (String pattern : Settings.getInstance().getApiWhitelist()) {
-				if (pattern.startsWith("0.0.0.0/") || pattern.startsWith("::/") || pattern.endsWith("/0")) {
-					allConnectionsAllowed = true;
-				}
-			}
-
-			if (allConnectionsAllowed) {
-				LOGGER.warn("Warning: API key validation is currently disabled, and the API whitelist " +
-						"is allowing all connections. This can be a security risk.");
-				LOGGER.warn("To fix, set the apiKeyDisabled setting to false, or allow only specific local " +
-						"IP addresses using the apiWhitelist setting.");
-			}
-		}
-	}
-
 }

src/main/java/org/qortal/api/DomainMapService.java

@@ -0,0 +1,171 @@
+package org.qortal.api;
+
+import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.rewrite.handler.RewriteHandler;
+import org.eclipse.jetty.rewrite.handler.RewritePatternRule;
+import org.eclipse.jetty.server.*;
+import org.eclipse.jetty.server.handler.ErrorHandler;
+import org.eclipse.jetty.server.handler.InetAccessHandler;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.servlets.CrossOriginFilter;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.qortal.api.resource.AnnotationPostProcessor;
+import org.qortal.api.resource.ApiDefinition;
+import org.qortal.settings.Settings;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.KeyStore;
+import java.security.SecureRandom;
+
+public class DomainMapService {
+
+	private static DomainMapService instance;
+
+	private final ResourceConfig config;
+	private Server server;
+
+	private DomainMapService() {
+		this.config = new ResourceConfig();
+		this.config.packages("org.qortal.api.domainmap.resource");
+		this.config.register(OpenApiResource.class);
+		this.config.register(ApiDefinition.class);
+		this.config.register(AnnotationPostProcessor.class);
+	}
+
+	public static DomainMapService getInstance() {
+		if (instance == null)
+			instance = new DomainMapService();
+
+		return instance;
+	}
+
+	public Iterable<Class<?>> getResources() {
+		return this.config.getClasses();
+	}
+
+	public void start() {
+		try {
+			// Create API server
+
+			// SSL support if requested
+			String keystorePathname = Settings.getInstance().getSslKeystorePathname();
+			String keystorePassword = Settings.getInstance().getSslKeystorePassword();
+
+			if (keystorePathname != null && keystorePassword != null) {
+				// SSL version
+				if (!Files.isReadable(Path.of(keystorePathname)))
+					throw new RuntimeException("Failed to start SSL API due to broken keystore");
+
+				// BouncyCastle-specific SSLContext build
+				SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE");
+				KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
+
+				KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
+
+				try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) {
+					keyStore.load(keystoreStream, keystorePassword.toCharArray());
+				}
+
+				keyManagerFactory.init(keyStore, keystorePassword.toCharArray());
+				sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom());
+
+				SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
+				sslContextFactory.setSslContext(sslContext);
+
+				this.server = new Server();
+
+				HttpConfiguration httpConfig = new HttpConfiguration();
+				httpConfig.setSecureScheme("https");
+				httpConfig.setSecurePort(Settings.getInstance().getDomainMapPort());
+
+				SecureRequestCustomizer src = new SecureRequestCustomizer();
+				httpConfig.addCustomizer(src);
+
+				HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig);
+				SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
+
+				ServerConnector portUnifiedConnector = new ServerConnector(this.server,
+						new DetectorConnectionFactory(sslConnectionFactory),
+						httpConnectionFactory);
+				portUnifiedConnector.setHost(Settings.getInstance().getBindAddress());
+				portUnifiedConnector.setPort(Settings.getInstance().getDomainMapPort());
+
+				this.server.addConnector(portUnifiedConnector);
+			} else {
+				// Non-SSL
+				InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress());
+				InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getDomainMapPort());
+				this.server = new Server(endpoint);
+			}
+
+			// Error handler
+			ErrorHandler errorHandler = new ApiErrorHandler();
+			this.server.setErrorHandler(errorHandler);
+
+			// Request logging
+			if (Settings.getInstance().isDomainMapLoggingEnabled()) {
+				RequestLogWriter logWriter = new RequestLogWriter("domainmap-requests.log");
+				logWriter.setAppend(true);
+				logWriter.setTimeZone("UTC");
+				RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT);
+				this.server.setRequestLog(requestLog);
+			}
+
+			// Access handler (currently no whitelist is used)
+			InetAccessHandler accessHandler = new InetAccessHandler();
+			this.server.setHandler(accessHandler);
+
+			// URL rewriting
+			RewriteHandler rewriteHandler = new RewriteHandler();
+			accessHandler.setHandler(rewriteHandler);
+
+			// Context
+			ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+			context.setContextPath("/");
+			rewriteHandler.setHandler(context);
+
+			// Cross-origin resource sharing
+			FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class);
+			corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
+			corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE");
+			corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
+			context.addFilter(corsFilterHolder, "/*", null);
+
+			// API servlet
+			ServletContainer container = new ServletContainer(this.config);
+			ServletHolder apiServlet = new ServletHolder(container);
+			apiServlet.setInitOrder(1);
+			context.addServlet(apiServlet, "/*");
+
+			// Start server
+			this.server.start();
+		} catch (Exception e) {
+			// Failed to start
+			throw new RuntimeException("Failed to start API", e);
+		}
+	}
+
+	public void stop() {
+		try {
+			// Stop server
+			this.server.stop();
+		} catch (Exception e) {
+			// Failed to stop
+		}
+
+		this.server = null;
+	}
+
+}

src/main/java/org/qortal/api/GatewayService.java

@@ -0,0 +1,170 @@
+package org.qortal.api;
+
+import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
+import org.eclipse.jetty.http.HttpVersion;
+import org.eclipse.jetty.rewrite.handler.RewriteHandler;
+import org.eclipse.jetty.server.*;
+import org.eclipse.jetty.server.handler.ErrorHandler;
+import org.eclipse.jetty.server.handler.InetAccessHandler;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.servlets.CrossOriginFilter;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.qortal.api.resource.AnnotationPostProcessor;
+import org.qortal.api.resource.ApiDefinition;
+import org.qortal.settings.Settings;
+
+import javax.net.ssl.KeyManagerFactory;
+import javax.net.ssl.SSLContext;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.KeyStore;
+import java.security.SecureRandom;
+
+public class GatewayService {
+
+	private static GatewayService instance;
+
+	private final ResourceConfig config;
+	private Server server;
+
+	private GatewayService() {
+		this.config = new ResourceConfig();
+		this.config.packages("org.qortal.api.gateway.resource");
+		this.config.register(OpenApiResource.class);
+		this.config.register(ApiDefinition.class);
+		this.config.register(AnnotationPostProcessor.class);
+	}
+
+	public static GatewayService getInstance() {
+		if (instance == null)
+			instance = new GatewayService();
+
+		return instance;
+	}
+
+	public Iterable<Class<?>> getResources() {
+		return this.config.getClasses();
+	}
+
+	public void start() {
+		try {
+			// Create API server
+
+			// SSL support if requested
+			String keystorePathname = Settings.getInstance().getSslKeystorePathname();
+			String keystorePassword = Settings.getInstance().getSslKeystorePassword();
+
+			if (keystorePathname != null && keystorePassword != null) {
+				// SSL version
+				if (!Files.isReadable(Path.of(keystorePathname)))
+					throw new RuntimeException("Failed to start SSL API due to broken keystore");
+
+				// BouncyCastle-specific SSLContext build
+				SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE");
+				KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
+
+				KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
+
+				try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) {
+					keyStore.load(keystoreStream, keystorePassword.toCharArray());
+				}
+
+				keyManagerFactory.init(keyStore, keystorePassword.toCharArray());
+				sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom());
+
+				SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
+				sslContextFactory.setSslContext(sslContext);
+
+				this.server = new Server();
+
+				HttpConfiguration httpConfig = new HttpConfiguration();
+				httpConfig.setSecureScheme("https");
+				httpConfig.setSecurePort(Settings.getInstance().getGatewayPort());
+
+				SecureRequestCustomizer src = new SecureRequestCustomizer();
+				httpConfig.addCustomizer(src);
+
+				HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig);
+				SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
+
+				ServerConnector portUnifiedConnector = new ServerConnector(this.server,
+						new DetectorConnectionFactory(sslConnectionFactory),
+						httpConnectionFactory);
+				portUnifiedConnector.setHost(Settings.getInstance().getBindAddress());
+				portUnifiedConnector.setPort(Settings.getInstance().getGatewayPort());
+
+				this.server.addConnector(portUnifiedConnector);
+			} else {
+				// Non-SSL
+				InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress());
+				InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getGatewayPort());
+				this.server = new Server(endpoint);
+			}
+
+			// Error handler
+			ErrorHandler errorHandler = new ApiErrorHandler();
+			this.server.setErrorHandler(errorHandler);
+
+			// Request logging
+			if (Settings.getInstance().isGatewayLoggingEnabled()) {
+				RequestLogWriter logWriter = new RequestLogWriter("gateway-requests.log");
+				logWriter.setAppend(true);
+				logWriter.setTimeZone("UTC");
+				RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT);
+				this.server.setRequestLog(requestLog);
+			}
+
+			// Access handler (currently no whitelist is used)
+			InetAccessHandler accessHandler = new InetAccessHandler();
+			this.server.setHandler(accessHandler);
+
+			// URL rewriting
+			RewriteHandler rewriteHandler = new RewriteHandler();
+			accessHandler.setHandler(rewriteHandler);
+
+			// Context
+			ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+			context.setContextPath("/");
+			rewriteHandler.setHandler(context);
+
+			// Cross-origin resource sharing
+			FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class);
+			corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
+			corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE");
+			corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
+			context.addFilter(corsFilterHolder, "/*", null);
+
+			// API servlet
+			ServletContainer container = new ServletContainer(this.config);
+			ServletHolder apiServlet = new ServletHolder(container);
+			apiServlet.setInitOrder(1);
+			context.addServlet(apiServlet, "/*");
+
+			// Start server
+			this.server.start();
+		} catch (Exception e) {
+			// Failed to start
+			throw new RuntimeException("Failed to start API", e);
+		}
+	}
+
+	public void stop() {
+		try {
+			// Stop server
+			this.server.stop();
+		} catch (Exception e) {
+			// Failed to stop
+		}
+
+		this.server = null;
+	}
+
+}

src/main/java/org/qortal/api/HTMLParser.java

@@ -0,0 +1,46 @@
+package org.qortal.api;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
+
+public class HTMLParser {
+
+    private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
+
+    private String linkPrefix;
+    private byte[] data;
+
+    public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix, byte[] data) {
+        String inPathWithoutFilename = inPath.substring(0, inPath.lastIndexOf('/'));
+        this.linkPrefix = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
+        this.data = data;
+    }
+
+    public void addAdditionalHeaderTags() {
+        String fileContents = new String(data);
+        Document document = Jsoup.parse(fileContents);
+        String baseUrl = this.linkPrefix + "/";
+        Elements head = document.getElementsByTag("head");
+        if (!head.isEmpty()) {
+            // Add base href tag
+            String baseElement = String.format("<base href=\"%s\">", baseUrl);
+            head.get(0).prepend(baseElement);
+        }
+        String html = document.html();
+        this.data = html.getBytes();
+    }
+
+    public static boolean isHtmlFile(String path) {
+        if (path.endsWith(".html") || path.endsWith(".htm")) {
+            return true;
+        }
+        return false;
+    }
+
+    public byte[] getData() {
+        return this.data;
+    }
+}

src/main/java/org/qortal/api/Security.java

@@ -1,38 +1,111 @@
 package org.qortal.api;
 
+import org.qortal.arbitrary.ArbitraryDataResource;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.controller.arbitrary.ArbitraryDataRenderManager;
+import org.qortal.settings.Settings;
+
+import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 
 import javax.servlet.http.HttpServletRequest;
 
-import org.qortal.settings.Settings;
-
 public abstract class Security {
 
 	public static final String API_KEY_HEADER = "X-API-KEY";
 
 	public static void checkApiCallAllowed(HttpServletRequest request) {
-		// If API key checking has been disabled, we will allow the request in all cases
-		boolean isApiKeyDisabled = Settings.getInstance().isApiKeyDisabled();
-		if (isApiKeyDisabled)
-			return;
+		// We may want to allow automatic authentication for local requests, if enabled in settings
+		boolean localAuthBypassEnabled = Settings.getInstance().isLocalAuthBypassEnabled();
+		if (localAuthBypassEnabled) {
+			try {
+				InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
+				if (remoteAddr.isLoopbackAddress()) {
+					// Request originates from loopback address, so allow it
+					return;
+				}
+			} catch (UnknownHostException e) {
+				// Ignore failure, and fallback to API key authentication
+			}
+		}
 
-		String expectedApiKey = Settings.getInstance().getApiKey();
+		// Retrieve the API key
+		ApiKey apiKey = Security.getApiKey(request);
+		if (!apiKey.generated()) {
+			// Not generated an API key yet, so disallow sensitive API calls
+			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "API key not generated");
+		}
+
+		// We require an API key to be passed
 		String passedApiKey = request.getHeader(API_KEY_HEADER);
+		if (passedApiKey == null) {
+			// Try query string - this is needed to avoid a CORS preflight. See: https://stackoverflow.com/a/43881141
+			passedApiKey = request.getParameter("apiKey");
+		}
+		if (passedApiKey == null) {
+			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Missing 'X-API-KEY' header");
+		}
 
-		if ((expectedApiKey != null && !expectedApiKey.equals(passedApiKey)) ||
-				(passedApiKey != null && !passedApiKey.equals(expectedApiKey)))
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
+		// The API keys must match
+		if (!apiKey.toString().equals(passedApiKey)) {
+			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "API key invalid");
+		}
+	}
 
-		InetAddress remoteAddr;
+	public static void disallowLoopbackRequests(HttpServletRequest request) {
 		try {
-			remoteAddr = InetAddress.getByName(request.getRemoteAddr());
+			InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
+			if (remoteAddr.isLoopbackAddress()) {
+				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Local requests not allowed");
+			}
 		} catch (UnknownHostException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
 		}
+	}
 
-		if (!remoteAddr.isLoopbackAddress())
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
+	public static void disallowLoopbackRequestsIfAuthBypassEnabled(HttpServletRequest request) {
+		if (Settings.getInstance().isLocalAuthBypassEnabled()) {
+			try {
+				InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
+				if (remoteAddr.isLoopbackAddress()) {
+					throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Local requests not allowed when localAuthBypassEnabled is enabled in settings");
+				}
+			} catch (UnknownHostException e) {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
+			}
+		}
+	}
+
+	public static void requirePriorAuthorization(HttpServletRequest request, String resourceId, Service service, String identifier) {
+		ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, identifier);
+		if (!ArbitraryDataRenderManager.getInstance().isAuthorized(resource)) {
+			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Call /render/authorize first");
+		}
+	}
+
+	public static void requirePriorAuthorizationOrApiKey(HttpServletRequest request, String resourceId, Service service, String identifier) {
+		try {
+			Security.checkApiCallAllowed(request);
+
+		} catch (ApiException e) {
+			// API call wasn't allowed, but maybe it was pre-authorized
+			Security.requirePriorAuthorization(request, resourceId, service, identifier);
+		}
+	}
+
+	public static ApiKey getApiKey(HttpServletRequest request) {
+		ApiKey apiKey = ApiService.getInstance().getApiKey();
+		if (apiKey == null) {
+			try {
+				apiKey = new ApiKey();
+			} catch (IOException e) {
+				// Couldn't load API key - so we need to treat it as not generated, and therefore unauthorized
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
+			}
+			ApiService.getInstance().setApiKey(apiKey);
+		}
+		return apiKey;
 	}
 
 }

src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java

@@ -0,0 +1,58 @@
+package org.qortal.api.domainmap.resource;
+
+import io.swagger.v3.oas.annotations.tags.Tag;
+import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
+import org.qortal.arbitrary.ArbitraryDataRenderer;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.settings.Settings;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.core.Context;
+import java.util.Map;
+
+
+@Path("/")
+@Tag(name = "Gateway")
+public class DomainMapResource {
+
+    @Context HttpServletRequest request;
+    @Context HttpServletResponse response;
+    @Context ServletContext context;
+
+
+    @GET
+    public HttpServletResponse getIndexByDomainMap() {
+        return this.getDomainMap("/");
+    }
+
+    @GET
+    @Path("{path:.*}")
+    public HttpServletResponse getPathByDomainMap(@PathParam("path") String inPath) {
+        return this.getDomainMap(inPath);
+    }
+
+    private HttpServletResponse getDomainMap(String inPath) {
+        Map<String, String> domainMap = Settings.getInstance().getSimpleDomainMap();
+        if (domainMap != null && domainMap.containsKey(request.getServerName())) {
+            // Build synchronously, so that we don't need to make the summary API endpoints available over
+            // the domain map server. This means that there will be no loading screen, but this is potentially
+            // preferred in this situation anyway (e.g. to avoid confusing search engine robots).
+            return this.get(domainMap.get(request.getServerName()), ResourceIdType.NAME, Service.WEBSITE, inPath, null, "", false, false);
+        }
+        return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
+    }
+
+    private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String inPath,
+                                    String secret58, String prefix, boolean usePrefix, boolean async) {
+
+        ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, inPath,
+                secret58, prefix, usePrefix, async, request, response, context);
+        return renderer.render();
+    }
+
+}

src/main/java/org/qortal/api/gateway/resource/GatewayResource.java

@@ -0,0 +1,126 @@
+package org.qortal.api.gateway.resource;
+
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import org.qortal.api.Security;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
+import org.qortal.arbitrary.ArbitraryDataReader;
+import org.qortal.arbitrary.ArbitraryDataRenderer;
+import org.qortal.arbitrary.ArbitraryDataResource;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.data.arbitrary.ArbitraryResourceStatus;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+
+
+@Path("/")
+@Tag(name = "Gateway")
+public class GatewayResource {
+
+    @Context HttpServletRequest request;
+    @Context HttpServletResponse response;
+    @Context ServletContext context;
+
+    /**
+     * We need to allow resource status checking (and building) via the gateway, as the node's API port
+     * may not be forwarded and will almost certainly not be authenticated. Since gateways allow for
+     * all resources to be loaded except those that are blocked, there is no need for authentication.
+     */
+    @GET
+    @Path("/arbitrary/resource/status/{service}/{name}")
+    public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service,
+                                                             @PathParam("name") String name,
+                                                             @QueryParam("build") Boolean build) {
+
+        return this.getStatus(service, name, null, build);
+    }
+
+    @GET
+    @Path("/arbitrary/resource/status/{service}/{name}/{identifier}")
+    public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service,
+                                                      @PathParam("name") String name,
+                                                      @PathParam("identifier") String identifier,
+                                                      @QueryParam("build") Boolean build) {
+
+        return this.getStatus(service, name, identifier, build);
+    }
+
+    private ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) {
+
+        // If "build=true" has been specified in the query string, build the resource before returning its status
+        if (build != null && build == true) {
+            ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null);
+            try {
+                if (!reader.isBuilding()) {
+                    reader.loadSynchronously(false);
+                }
+            } catch (Exception e) {
+                // No need to handle exception, as it will be reflected in the status
+            }
+        }
+
+        ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
+        return resource.getStatus(false);
+    }
+
+
+    @GET
+    public HttpServletResponse getRoot() {
+        return ArbitraryDataRenderer.getResponse(response, 200, "");
+    }
+
+
+    @GET
+    @Path("{name}/{path:.*}")
+    @SecurityRequirement(name = "apiKey")
+    public HttpServletResponse getPathByName(@PathParam("name") String name,
+                                             @PathParam("path") String inPath) {
+        // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data
+        Security.disallowLoopbackRequests(request);
+        return this.get(name, ResourceIdType.NAME, Service.WEBSITE, inPath, null, "", true, true);
+    }
+
+    @GET
+    @Path("{name}")
+    @SecurityRequirement(name = "apiKey")
+    public HttpServletResponse getIndexByName(@PathParam("name") String name) {
+        // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data
+        Security.disallowLoopbackRequests(request);
+        return this.get(name, ResourceIdType.NAME, Service.WEBSITE, "/", null, "", true, true);
+    }
+
+
+    // Optional /site alternative for backwards support
+
+    @GET
+    @Path("/site/{name}/{path:.*}")
+    public HttpServletResponse getSitePathByName(@PathParam("name") String name,
+                                                 @PathParam("path") String inPath) {
+        // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data
+        Security.disallowLoopbackRequests(request);
+        return this.get(name, ResourceIdType.NAME, Service.WEBSITE, inPath, null, "/site", true, true);
+    }
+
+    @GET
+    @Path("/site/{name}")
+    public HttpServletResponse getSiteIndexByName(@PathParam("name") String name) {
+        // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data
+        Security.disallowLoopbackRequests(request);
+        return this.get(name, ResourceIdType.NAME, Service.WEBSITE, "/", null, "/site", true, true);
+    }
+
+    
+    private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String inPath,
+                                    String secret58, String prefix, boolean usePrefix, boolean async) {
+
+        ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, inPath,
+                secret58, prefix, usePrefix, async, request, response, context);
+        return renderer.render();
+    }
+
+}

src/main/java/org/qortal/api/model/NodeStatus.java

@@ -4,6 +4,7 @@ import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.network.Network;
 
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -22,7 +23,7 @@ public class NodeStatus {
 	public NodeStatus() {
 		this.isMintingPossible = Controller.getInstance().isMintingPossible();
 
-		this.syncPercent = Controller.getInstance().getSyncPercent();
+		this.syncPercent = Synchronizer.getInstance().getSyncPercent();
 		this.isSynchronizing = this.syncPercent != null;
 
 		this.numberOfConnections = Network.getInstance().getHandshakedPeers().size();

src/main/java/org/qortal/api/model/PeersSummary.java

@@ -0,0 +1,15 @@
+package org.qortal.api.model;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class PeersSummary {
+
+	public int inboundConnections;
+	public int outboundConnections;
+
+	public PeersSummary() {
+	}
+
+}

src/main/java/org/qortal/api/resource/AddressesResource.java

@@ -16,11 +16,7 @@ import java.util.Comparator;
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -537,7 +533,7 @@ public class AddressesResource {
 	)
 	@ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String computePublicize(String rawBytes58) {
+	public String computePublicize(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String rawBytes58) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {

src/main/java/org/qortal/api/resource/AdminResource.java

@@ -27,11 +27,7 @@ import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -39,17 +35,16 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.appender.RollingFileAppender;
+import org.checkerframework.checker.units.qual.A;
 import org.qortal.account.Account;
 import org.qortal.account.PrivateKeyAccount;
-import org.qortal.api.ApiError;
-import org.qortal.api.ApiErrors;
-import org.qortal.api.ApiExceptionFactory;
-import org.qortal.api.Security;
+import org.qortal.api.*;
 import org.qortal.api.model.ActivitySummary;
 import org.qortal.api.model.NodeInfo;
 import org.qortal.api.model.NodeStatus;
 import org.qortal.block.BlockChain;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.controller.Synchronizer.SynchronizationResult;
 import org.qortal.data.account.MintingAccountData;
 import org.qortal.data.account.RewardShareData;
@@ -80,7 +75,8 @@ public class AdminResource {
 	@Path("/unused")
 	@Parameter(in = ParameterIn.PATH, name = "assetid", description = "Asset ID, 0 is native coin", schema = @Schema(type = "integer"))
 	@Parameter(in = ParameterIn.PATH, name = "otherassetid", description = "Asset ID, 0 is native coin", schema = @Schema(type = "integer"))
-	@Parameter(in = ParameterIn.PATH, name = "address", description = "an account address", example = "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v")
+	@Parameter(in = ParameterIn.PATH, name = "address", description = "An account address", example = "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v")
+	@Parameter(in = ParameterIn.PATH, name = "path", description = "Local path to folder containing the files", schema = @Schema(type = "String", defaultValue = "/Users/user/Documents/MyStaticWebsite"))
 	@Parameter(in = ParameterIn.QUERY, name = "count", description = "Maximum number of entries to return, 0 means none", schema = @Schema(type = "integer", defaultValue = "20"))
 	@Parameter(in = ParameterIn.QUERY, name = "limit", description = "Maximum number of entries to return, 0 means unlimited", schema = @Schema(type = "integer", defaultValue = "20"))
 	@Parameter(in = ParameterIn.QUERY, name = "offset", description = "Starting entry in results, 0 is first entry", schema = @Schema(type = "integer"))
@@ -138,10 +134,7 @@ public class AdminResource {
 			)
 		}
 	)
-	@SecurityRequirement(name = "apiKey")
 	public NodeStatus status() {
-		Security.checkApiCallAllowed(request);
-
 		NodeStatus nodeStatus = new NodeStatus();
 
 		return nodeStatus;
@@ -160,7 +153,7 @@ public class AdminResource {
 		}
 	)
 	@SecurityRequirement(name = "apiKey")
-	public String shutdown() {
+	public String shutdown(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 
 		new Thread(() -> {
@@ -189,7 +182,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public ActivitySummary summary() {
+	public ActivitySummary summary(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 
 		ActivitySummary summary = new ActivitySummary();
@@ -235,7 +228,7 @@ public class AdminResource {
 		}
 	)
 	@SecurityRequirement(name = "apiKey")
-	public Controller.StatsSnapshot getEngineStats() {
+	public Controller.StatsSnapshot getEngineStats(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 
 		return Controller.getInstance().getStatsSnapshot();
@@ -253,9 +246,7 @@ public class AdminResource {
 		}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
-	@SecurityRequirement(name = "apiKey")
 	public List<MintingAccountData> getMintingAccounts() {
-		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			List<MintingAccountData> mintingAccounts = repository.getAccountRepository().getMintingAccounts();
@@ -301,7 +292,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.REPOSITORY_ISSUE, ApiError.CANNOT_MINT})
 	@SecurityRequirement(name = "apiKey")
-	public String addMintingAccount(String seed58) {
+	public String addMintingAccount(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String seed58) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -354,7 +345,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String deleteMintingAccount(String key58) {
+	public String deleteMintingAccount(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -454,7 +445,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.INVALID_HEIGHT, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String orphan(String targetHeightString) {
+	public String orphan(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String targetHeightString) {
 		Security.checkApiCallAllowed(request);
 
 		try {
@@ -513,7 +504,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String forceSync(String targetPeerAddress) {
+	public String forceSync(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String targetPeerAddress) {
 		Security.checkApiCallAllowed(request);
 
 		try {
@@ -535,7 +526,7 @@ public class AdminResource {
 			SynchronizationResult syncResult;
 			try {
 				do {
-					syncResult = Controller.getInstance().actuallySynchronize(targetPeer, true);
+					syncResult = Synchronizer.getInstance().actuallySynchronize(targetPeer, true);
 				} while (syncResult == SynchronizationResult.OK);
 			} finally {
 				blockchainLock.unlock();
@@ -559,7 +550,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String exportRepository() {
+	public String exportRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -593,7 +584,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String importRepository(String filename) {
+	public String importRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String filename) {
 		Security.checkApiCallAllowed(request);
 
 		// Hard-coded because it's too dangerous to allow user-supplied filenames in weaker security contexts
@@ -639,7 +630,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String checkpointRepository() {
+	public String checkpointRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 
 		RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
@@ -660,7 +651,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String backupRepository() {
+	public String backupRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -694,7 +685,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public void performRepositoryMaintenance() {
+	public void performRepositoryMaintenance(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -716,4 +707,58 @@ public class AdminResource {
 		}
 	}
 
+
+	@POST
+	@Path("/apikey/generate")
+	@Operation(
+			summary = "Generate an API key",
+			description = "This request is unauthenticated if no API key has been generated yet. " +
+					"If an API key already exists, it needs to be passed as a header and this endpoint " +
+					"will then generate a new key which replaces the existing one.",
+			responses = {
+					@ApiResponse(
+							description = "API key string",
+							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string"))
+					)
+			}
+	)
+	@SecurityRequirement(name = "apiKey")
+	public String generateApiKey(@HeaderParam(Security.API_KEY_HEADER) String apiKeyHeader) {
+		ApiKey apiKey = Security.getApiKey(request);
+
+		// If the API key is already generated, we need to authenticate this request
+		if (apiKey.generated() && apiKey.exists()) {
+			Security.checkApiCallAllowed(request);
+		}
+
+		// Not generated yet - so we are safe to generate one
+		// FUTURE: we may want to restrict this to local/loopback only?
+
+		try {
+			apiKey.generate();
+		} catch (IOException e) {
+			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Unable to generate API key");
+		}
+
+		return apiKey.toString();
+	}
+
+	@GET
+	@Path("/apikey/test")
+	@Operation(
+			summary = "Test an API key",
+			responses = {
+					@ApiResponse(
+							description = "true if authenticated",
+							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
+					)
+			}
+	)
+	@SecurityRequirement(name = "apiKey")
+	public String testApiKey(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+		Security.checkApiCallAllowed(request);
+
+		return "true";
+	}
+
 }

src/main/java/org/qortal/api/resource/BlocksResource.java

@@ -268,9 +268,15 @@ public class BlocksResource {
 	@ApiErrors({
 		ApiError.REPOSITORY_ISSUE
 	})
-	public BlockData getLastBlock() {
+	public BlockData getLastBlock(@QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			return repository.getBlockRepository().getLastBlock();
+			BlockData blockData = repository.getBlockRepository().getLastBlock();
+
+			if (includeOnlineSignatures == null || includeOnlineSignatures == false) {
+				blockData.setOnlineAccountsSignatures(null);
+			}
+
+			return blockData;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -548,20 +554,25 @@ public class BlocksResource {
 	@ApiErrors({
 		ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE
 	})
-	public BlockData getByTimestamp(@PathParam("timestamp") long timestamp) {
+	public BlockData getByTimestamp(@PathParam("timestamp") long timestamp,
+									@QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			BlockData blockData = null;
 
 			// Try the Blocks table
 			int height = repository.getBlockRepository().getHeightFromTimestamp(timestamp);
-			if (height > 0) {
+			if (height > 1) {
 				// Found match in Blocks table
-				return repository.getBlockRepository().fromHeight(height);
+				blockData = repository.getBlockRepository().fromHeight(height);
+				if (includeOnlineSignatures == null || includeOnlineSignatures == false) {
+					blockData.setOnlineAccountsSignatures(null);
+				}
+				return blockData;
 			}
 
 			// Not found in Blocks table, so try the archive
 			height = repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp);
-			if (height > 0) {
+			if (height > 1) {
 				// Found match in archive
 				blockData = repository.getBlockArchiveRepository().fromHeight(height);
 			}
@@ -571,6 +582,10 @@ public class BlocksResource {
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 			}
 
+			if (includeOnlineSignatures == null || includeOnlineSignatures == false) {
+				blockData.setOnlineAccountsSignatures(null);
+			}
+
 			return blockData;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);

src/main/java/org/qortal/api/resource/BootstrapResource.java

@@ -4,6 +4,7 @@ import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -43,7 +44,8 @@ public class BootstrapResource {
 			)
 		}
 	)
-	public String createBootstrap() {
+	@SecurityRequirement(name = "apiKey")
+	public String createBootstrap(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -77,7 +79,8 @@ public class BootstrapResource {
 					)
 			}
 	)
-	public boolean validateBootstrap() {
+	@SecurityRequirement(name = "apiKey")
+	public boolean validateBootstrap(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {

src/main/java/org/qortal/api/resource/ChatResource.java

@@ -13,11 +13,7 @@ import io.swagger.v3.oas.annotations.tags.Tag;
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -158,7 +154,7 @@ public class ChatResource {
 	)
 	@ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String buildChat(ChatTransactionData transactionData) {
+	public String buildChat(@HeaderParam(Security.API_KEY_HEADER) String apiKey, ChatTransactionData transactionData) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -206,7 +202,7 @@ public class ChatResource {
 	)
 	@ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String buildChat(String rawBytes58) {
+	public String buildChat(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String rawBytes58) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {

src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java

@@ -5,12 +5,14 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.util.Arrays;
 import java.util.Random;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
@@ -79,7 +81,8 @@ public class CrossChainBitcoinACCTv1Resource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_DATA, ApiError.INVALID_REFERENCE, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE})
-	public String buildTrade(CrossChainBuildRequest tradeRequest) {
+	@SecurityRequirement(name = "apiKey")
+	public String buildTrade(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainBuildRequest tradeRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] creatorPublicKey = tradeRequest.creatorPublicKey;
@@ -174,7 +177,8 @@ public class CrossChainBitcoinACCTv1Resource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	public String buildTradeMessage(CrossChainTradeRequest tradeRequest) {
+	@SecurityRequirement(name = "apiKey")
+	public String buildTradeMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainTradeRequest tradeRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] tradePublicKey = tradeRequest.tradePublicKey;
@@ -257,7 +261,8 @@ public class CrossChainBitcoinACCTv1Resource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	public String buildRedeemMessage(CrossChainDualSecretRequest secretRequest) {
+	@SecurityRequirement(name = "apiKey")
+	public String buildRedeemMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainDualSecretRequest secretRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] partnerPublicKey = secretRequest.partnerPublicKey;
@@ -360,4 +365,4 @@ public class CrossChainBitcoinACCTv1Resource {
 		}
 	}
 
-}
+}

src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java

@@ -6,11 +6,13 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
@@ -56,7 +58,8 @@ public class CrossChainBitcoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	public String getBitcoinWalletBalance(String key58) {
+	@SecurityRequirement(name = "apiKey")
+	public String getBitcoinWalletBalance(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
 		Security.checkApiCallAllowed(request);
 
 		Bitcoin bitcoin = Bitcoin.getInstance();
@@ -64,11 +67,16 @@ public class CrossChainBitcoinResource {
 		if (!bitcoin.isValidDeterministicKey(key58))
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
 
-		Long balance = bitcoin.getWalletBalance(key58);
-		if (balance == null)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
+		try {
+			Long balance = bitcoin.getWalletBalanceFromTransactions(key58);
+			if (balance == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
 
-		return balance.toString();
+			return balance.toString();
+
+		} catch (ForeignBlockchainException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
+		}
 	}
 
 	@POST
@@ -94,7 +102,8 @@ public class CrossChainBitcoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	public List<SimpleTransaction> getBitcoinWalletTransactions(String key58) {
+	@SecurityRequirement(name = "apiKey")
+	public List<SimpleTransaction> getBitcoinWalletTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
 		Security.checkApiCallAllowed(request);
 
 		Bitcoin bitcoin = Bitcoin.getInstance();
@@ -130,7 +139,8 @@ public class CrossChainBitcoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	public String sendBitcoin(BitcoinSendRequest bitcoinSendRequest) {
+	@SecurityRequirement(name = "apiKey")
+	public String sendBitcoin(@HeaderParam(Security.API_KEY_HEADER) String apiKey, BitcoinSendRequest bitcoinSendRequest) {
 		Security.checkApiCallAllowed(request);
 
 		if (bitcoinSendRequest.bitcoinAmount <= 0)
@@ -164,4 +174,4 @@ public class CrossChainBitcoinResource {
 		return spendTransaction.getTxId().toString();
 	}
 
-}
+}

src/main/java/org/qortal/api/resource/CrossChainDogecoinACCTv1Resource.java

@@ -5,6 +5,7 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.api.ApiError;
@@ -26,6 +27,7 @@ import org.qortal.transaction.Transaction.ValidationResult;
 import org.qortal.transform.Transformer;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
@@ -67,7 +69,8 @@ public class CrossChainDogecoinACCTv1Resource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	public boolean buildRedeemMessage(CrossChainSecretRequest secretRequest) {
+	@SecurityRequirement(name = "apiKey")
+	public boolean buildRedeemMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainSecretRequest secretRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] partnerPrivateKey = secretRequest.partnerPrivateKey;

src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java

@@ -6,6 +6,7 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import org.bitcoinj.core.Transaction;
 import org.qortal.api.ApiError;
@@ -18,6 +19,7 @@ import org.qortal.crosschain.Dogecoin;
 import org.qortal.crosschain.SimpleTransaction;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
@@ -54,7 +56,8 @@ public class CrossChainDogecoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	public String getDogecoinWalletBalance(String key58) {
+	@SecurityRequirement(name = "apiKey")
+	public String getDogecoinWalletBalance(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
 		Security.checkApiCallAllowed(request);
 
 		Dogecoin dogecoin = Dogecoin.getInstance();
@@ -62,11 +65,16 @@ public class CrossChainDogecoinResource {
 		if (!dogecoin.isValidDeterministicKey(key58))
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
 
-		Long balance = dogecoin.getWalletBalance(key58);
-		if (balance == null)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
+		try {
+			Long balance = dogecoin.getWalletBalanceFromTransactions(key58);
+			if (balance == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
 
-		return balance.toString();
+			return balance.toString();
+
+		} catch (ForeignBlockchainException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
+		}
 	}
 
 	@POST
@@ -92,7 +100,8 @@ public class CrossChainDogecoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	public List<SimpleTransaction> getDogecoinWalletTransactions(String key58) {
+	@SecurityRequirement(name = "apiKey")
+	public List<SimpleTransaction> getDogecoinWalletTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
 		Security.checkApiCallAllowed(request);
 
 		Dogecoin dogecoin = Dogecoin.getInstance();
@@ -128,7 +137,8 @@ public class CrossChainDogecoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	public String sendBitcoin(DogecoinSendRequest dogecoinSendRequest) {
+	@SecurityRequirement(name = "apiKey")
+	public String sendBitcoin(@HeaderParam(Security.API_KEY_HEADER) String apiKey, DogecoinSendRequest dogecoinSendRequest) {
 		Security.checkApiCallAllowed(request);
 
 		if (dogecoinSendRequest.dogecoinAmount <= 0)

src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java

@@ -4,16 +4,14 @@ import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.math.BigDecimal;
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
+import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -105,7 +103,9 @@ public class CrossChainHtlcResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
-	public CrossChainBitcoinyHTLCStatus checkHtlcStatus(@PathParam("blockchain") String blockchainName,
+	@SecurityRequirement(name = "apiKey")
+	public CrossChainBitcoinyHTLCStatus checkHtlcStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
+			@PathParam("blockchain") String blockchainName,
 			@PathParam("refundPKH") String refundPKH,
 			@PathParam("locktime") int lockTime,
 			@PathParam("redeemPKH") String redeemPKH,
@@ -188,7 +188,8 @@ public class CrossChainHtlcResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
-	public boolean redeemHtlc(@PathParam("ataddress") String atAddress) {
+	@SecurityRequirement(name = "apiKey")
+	public boolean redeemHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("ataddress") String atAddress) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -246,7 +247,8 @@ public class CrossChainHtlcResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
-	public boolean redeemAllHtlc() {
+	@SecurityRequirement(name = "apiKey")
+	public boolean redeemAllHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 		boolean success = false;
 
@@ -430,7 +432,8 @@ public class CrossChainHtlcResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
-	public boolean refundHtlc(@PathParam("ataddress") String atAddress) {
+	@SecurityRequirement(name = "apiKey")
+	public boolean refundHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("ataddress") String atAddress) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -478,7 +481,8 @@ public class CrossChainHtlcResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
-	public boolean refundAllHtlc() {
+	@SecurityRequirement(name = "apiKey")
+	public boolean refundAllHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 		boolean success = false;
 

src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java

@@ -5,6 +5,7 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.api.ApiError;
@@ -30,6 +31,7 @@ import org.qortal.utils.Base58;
 import org.qortal.utils.NTP;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
@@ -72,7 +74,8 @@ public class CrossChainLitecoinACCTv1Resource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	public boolean buildRedeemMessage(CrossChainSecretRequest secretRequest) {
+	@SecurityRequirement(name = "apiKey")
+	public boolean buildRedeemMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainSecretRequest secretRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] partnerPrivateKey = secretRequest.partnerPrivateKey;

src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java

@@ -6,11 +6,13 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
@@ -56,7 +58,8 @@ public class CrossChainLitecoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	public String getLitecoinWalletBalance(String key58) {
+	@SecurityRequirement(name = "apiKey")
+	public String getLitecoinWalletBalance(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
 		Security.checkApiCallAllowed(request);
 
 		Litecoin litecoin = Litecoin.getInstance();
@@ -64,11 +67,16 @@ public class CrossChainLitecoinResource {
 		if (!litecoin.isValidDeterministicKey(key58))
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
 
-		Long balance = litecoin.getWalletBalance(key58);
-		if (balance == null)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
+		try {
+			Long balance = litecoin.getWalletBalanceFromTransactions(key58);
+			if (balance == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
 
-		return balance.toString();
+			return balance.toString();
+
+		} catch (ForeignBlockchainException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
+		}
 	}
 
 	@POST
@@ -94,7 +102,8 @@ public class CrossChainLitecoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	public List<SimpleTransaction> getLitecoinWalletTransactions(String key58) {
+	@SecurityRequirement(name = "apiKey")
+	public List<SimpleTransaction> getLitecoinWalletTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
 		Security.checkApiCallAllowed(request);
 
 		Litecoin litecoin = Litecoin.getInstance();
@@ -130,7 +139,8 @@ public class CrossChainLitecoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	public String sendBitcoin(LitecoinSendRequest litecoinSendRequest) {
+	@SecurityRequirement(name = "apiKey")
+	public String sendBitcoin(@HeaderParam(Security.API_KEY_HEADER) String apiKey, LitecoinSendRequest litecoinSendRequest) {
 		Security.checkApiCallAllowed(request);
 
 		if (litecoinSendRequest.litecoinAmount <= 0)
@@ -164,4 +174,4 @@ public class CrossChainLitecoinResource {
 		return spendTransaction.getTxId().toString();
 	}
 
-}
+}

src/main/java/org/qortal/api/resource/CrossChainResource.java

@@ -15,11 +15,7 @@ import java.util.*;
 import java.util.function.Supplier;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -395,7 +391,7 @@ public class CrossChainResource {
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String cancelTrade(CrossChainCancelRequest cancelRequest) {
+	public String cancelTrade(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainCancelRequest cancelRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] creatorPublicKey = cancelRequest.creatorPublicKey;

src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java

@@ -7,17 +7,14 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.util.List;
 import java.util.stream.Collectors;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.QueryParam;
+import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -69,7 +66,9 @@ public class CrossChainTradeBotResource {
 		}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
+	@SecurityRequirement(name = "apiKey")
 	public List<TradeBotData> getTradeBotStates(
+			@HeaderParam(Security.API_KEY_HEADER) String apiKey,
 			@Parameter(
 					description = "Limit to specific blockchain",
 					example = "LITECOIN",
@@ -110,7 +109,8 @@ public class CrossChainTradeBotResource {
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.INSUFFICIENT_BALANCE, ApiError.REPOSITORY_ISSUE, ApiError.ORDER_SIZE_TOO_SMALL})
 	@SuppressWarnings("deprecation")
-	public String tradeBotCreator(TradeBotCreateRequest tradeBotCreateRequest) {
+	@SecurityRequirement(name = "apiKey")
+	public String tradeBotCreator(@HeaderParam(Security.API_KEY_HEADER) String apiKey, TradeBotCreateRequest tradeBotCreateRequest) {
 		Security.checkApiCallAllowed(request);
 
 		if (tradeBotCreateRequest.foreignBlockchain == null)
@@ -179,7 +179,8 @@ public class CrossChainTradeBotResource {
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE, ApiError.REPOSITORY_ISSUE})
 	@SuppressWarnings("deprecation")
-	public String tradeBotResponder(TradeBotRespondRequest tradeBotRespondRequest) {
+	@SecurityRequirement(name = "apiKey")
+	public String tradeBotResponder(@HeaderParam(Security.API_KEY_HEADER) String apiKey, TradeBotRespondRequest tradeBotRespondRequest) {
 		Security.checkApiCallAllowed(request);
 
 		final String atAddress = tradeBotRespondRequest.atAddress;
@@ -260,7 +261,8 @@ public class CrossChainTradeBotResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE})
-	public String tradeBotDelete(String tradePrivateKey58) {
+	@SecurityRequirement(name = "apiKey")
+	public String tradeBotDelete(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String tradePrivateKey58) {
 		Security.checkApiCallAllowed(request);
 
 		final byte[] tradePrivateKey;

src/main/java/org/qortal/api/resource/ListsResource.java

@@ -6,6 +6,7 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import org.qortal.api.*;
@@ -31,12 +32,10 @@ public class ListsResource {
 	HttpServletRequest request;
 
 
-	/* Address blacklist */
-
 	@POST
-	@Path("/blacklist/addresses")
+	@Path("/{listName}")
 	@Operation(
-			summary = "Add one or more QORT addresses to the local blacklist",
+			summary = "Add items to a new or existing list",
 			requestBody = @RequestBody(
 					required = true,
 					content = @Content(
@@ -48,17 +47,24 @@ public class ListsResource {
 			),
 			responses = {
 					@ApiResponse(
-							description = "Returns true if all addresses were processed, false if any couldn't be " +
+							description = "Returns true if all items were processed, false if any couldn't be " +
 									"processed, or an exception on failure. If false or an exception is returned, " +
 									"the list will not be updated, and the request will need to be re-issued.",
 							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
 					)
 			}
 	)
-	@ApiErrors({ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN, ApiError.REPOSITORY_ISSUE})
-	public String addAddressesToBlacklist(ListRequest listRequest) {
+	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
+	@SecurityRequirement(name = "apiKey")
+	public String addItemstoList(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
+								 @PathParam("listName") String listName,
+								 ListRequest listRequest) {
 		Security.checkApiCallAllowed(request);
 
+		if (listName == null) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+		}
+
 		if (listRequest == null || listRequest.items == null) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 		}
@@ -66,51 +72,33 @@ public class ListsResource {
 		int successCount = 0;
 		int errorCount = 0;
 
-		try (final Repository repository = RepositoryManager.getRepository()) {
+		for (String item : listRequest.items) {
 
-			for (String address : listRequest.items) {
-
-				if (!Crypto.isValidAddress(address)) {
-					errorCount++;
-					continue;
-				}
-
-				AccountData accountData = repository.getAccountRepository().getAccount(address);
-				// Not found?
-				if (accountData == null) {
-					errorCount++;
-					continue;
-				}
-
-				// Valid address, so go ahead and blacklist it
-				boolean success = ResourceListManager.getInstance().addToList("blacklist", "addresses", address, false);
-				if (success) {
-					successCount++;
-				}
-				else {
-					errorCount++;
-				}
+			boolean success = ResourceListManager.getInstance().addToList(listName, item, false);
+			if (success) {
+				successCount++;
+			}
+			else {
+				errorCount++;
 			}
-		} catch (DataException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
 
 		if (successCount > 0 && errorCount == 0) {
-			// All were successful, so save the blacklist
-			ResourceListManager.getInstance().saveList("blacklist", "addresses");
+			// All were successful, so save the list
+			ResourceListManager.getInstance().saveList(listName);
 			return "true";
 		}
 		else {
 			// Something went wrong, so revert
-			ResourceListManager.getInstance().revertList("blacklist", "addresses");
+			ResourceListManager.getInstance().revertList(listName);
 			return "false";
 		}
 	}
 
 	@DELETE
-	@Path("/blacklist/addresses")
+	@Path("/{listName}")
 	@Operation(
-			summary = "Remove one or more QORT addresses from the local blacklist",
+			summary = "Remove one or more items from a list",
 			requestBody = @RequestBody(
 					required = true,
 					content = @Content(
@@ -122,15 +110,18 @@ public class ListsResource {
 			),
 			responses = {
 					@ApiResponse(
-							description = "Returns true if all addresses were processed, false if any couldn't be " +
+							description = "Returns true if all items were processed, false if any couldn't be " +
 									"processed, or an exception on failure. If false or an exception is returned, " +
 									"the list will not be updated, and the request will need to be re-issued.",
 							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
 					)
 			}
 	)
-	@ApiErrors({ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN, ApiError.REPOSITORY_ISSUE})
-	public String removeAddressesFromBlacklist(ListRequest listRequest) {
+	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
+	@SecurityRequirement(name = "apiKey")
+	public String removeItemsFromList(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
+									  @PathParam("listName") String listName,
+									  ListRequest listRequest) {
 		Security.checkApiCallAllowed(request);
 
 		if (listRequest == null || listRequest.items == null) {
@@ -140,62 +131,46 @@ public class ListsResource {
 		int successCount = 0;
 		int errorCount = 0;
 
-		try (final Repository repository = RepositoryManager.getRepository()) {
+		for (String address : listRequest.items) {
 
-			for (String address : listRequest.items) {
-
-				if (!Crypto.isValidAddress(address)) {
-					errorCount++;
-					continue;
-				}
-
-				AccountData accountData = repository.getAccountRepository().getAccount(address);
-				// Not found?
-				if (accountData == null) {
-					errorCount++;
-					continue;
-				}
-
-				// Valid address, so go ahead and blacklist it
-				// Don't save as we will do this at the end of the process
-				boolean success = ResourceListManager.getInstance().removeFromList("blacklist", "addresses", address, false);
-				if (success) {
-					successCount++;
-				}
-				else {
-					errorCount++;
-				}
+			// Attempt to remove the item
+			// Don't save as we will do this at the end of the process
+			boolean success = ResourceListManager.getInstance().removeFromList(listName, address, false);
+			if (success) {
+				successCount++;
+			}
+			else {
+				errorCount++;
 			}
-		} catch (DataException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
 
 		if (successCount > 0 && errorCount == 0) {
-			// All were successful, so save the blacklist
-			ResourceListManager.getInstance().saveList("blacklist", "addresses");
+			// All were successful, so save the list
+			ResourceListManager.getInstance().saveList(listName);
 			return "true";
 		}
 		else {
 			// Something went wrong, so revert
-			ResourceListManager.getInstance().revertList("blacklist", "addresses");
+			ResourceListManager.getInstance().revertList(listName);
 			return "false";
 		}
 	}
 
 	@GET
-	@Path("/blacklist/addresses")
+	@Path("/{listName}")
 	@Operation(
-			summary = "Fetch the list of blacklisted addresses",
+			summary = "Fetch all items in a list",
 			responses = {
 					@ApiResponse(
-							description = "A JSON array of addresses",
+							description = "A JSON array of items",
 							content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = String.class)))
 					)
 			}
 	)
-	public String getAddressBlacklist() {
+	@SecurityRequirement(name = "apiKey")
+	public String getItemsInList(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("listName") String listName) {
 		Security.checkApiCallAllowed(request);
-		return ResourceListManager.getInstance().getJSONStringForList("blacklist", "addresses");
+		return ResourceListManager.getInstance().getJSONStringForList(listName);
 	}
 
 }

src/main/java/org/qortal/api/resource/PeersResource.java

@@ -16,19 +16,13 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
+import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
-import org.qortal.api.ApiError;
-import org.qortal.api.ApiErrors;
-import org.qortal.api.ApiException;
-import org.qortal.api.ApiExceptionFactory;
-import org.qortal.api.Security;
+import org.qortal.api.*;
 import org.qortal.api.model.ConnectedPeer;
+import org.qortal.api.model.PeersSummary;
 import org.qortal.controller.Controller;
 import org.qortal.controller.Synchronizer;
 import org.qortal.controller.Synchronizer.SynchronizationResult;
@@ -133,7 +127,7 @@ public class PeersResource {
 		}
 	)
 	@SecurityRequirement(name = "apiKey")
-	public ExecuteProduceConsume.StatsSnapshot getEngineStats() {
+	public ExecuteProduceConsume.StatsSnapshot getEngineStats(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
 		Security.checkApiCallAllowed(request);
 
 		return Network.getInstance().getStatsSnapshot();
@@ -171,7 +165,7 @@ public class PeersResource {
 		ApiError.INVALID_NETWORK_ADDRESS, ApiError.REPOSITORY_ISSUE
 	})
 	@SecurityRequirement(name = "apiKey")
-	public String addPeer(String address) {
+	public String addPeer(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String address) {
 		Security.checkApiCallAllowed(request);
 
 		final Long addedWhen = NTP.getTime();
@@ -226,7 +220,7 @@ public class PeersResource {
 		ApiError.INVALID_NETWORK_ADDRESS, ApiError.REPOSITORY_ISSUE
 	})
 	@SecurityRequirement(name = "apiKey")
-	public String removePeer(String address) {
+	public String removePeer(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String address) {
 		Security.checkApiCallAllowed(request);
 
 		try {
@@ -262,7 +256,7 @@ public class PeersResource {
 		ApiError.REPOSITORY_ISSUE
 	})
 	@SecurityRequirement(name = "apiKey")
-	public String removeKnownPeers(String address) {
+	public String removeKnownPeers(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String address) {
 		Security.checkApiCallAllowed(request);
 
 		try {
@@ -302,7 +296,7 @@ public class PeersResource {
 	)
 	@ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public List<BlockSummaryData> commonBlock(String targetPeerAddress) {
+	public List<BlockSummaryData> commonBlock(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String targetPeerAddress) {
 		Security.checkApiCallAllowed(request);
 
 		try {
@@ -338,4 +332,36 @@ public class PeersResource {
 		}
 	}
 
+	@GET
+	@Path("/summary")
+	@Operation(
+			summary = "Returns total inbound and outbound connections for connected peers",
+			responses = {
+					@ApiResponse(
+							content = @Content(
+									mediaType = MediaType.APPLICATION_JSON,
+									array = @ArraySchema(
+											schema = @Schema(
+													implementation = PeersSummary.class
+											)
+									)
+							)
+					)
+			}
+	)
+	public PeersSummary peersSummary() {
+		PeersSummary peersSummary = new PeersSummary();
+
+		List<Peer> connectedPeers = Network.getInstance().getConnectedPeers().stream().collect(Collectors.toList());
+		for (Peer peer : connectedPeers) {
+			if (peer.isOutbound()) {
+				peersSummary.inboundConnections++;
+			}
+			else {
+				peersSummary.outboundConnections++;
+			}
+		}
+		return peersSummary;
+	}
+
 }

src/main/java/org/qortal/api/resource/RenderResource.java

@@ -0,0 +1,200 @@
+package org.qortal.api.resource;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.*;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import java.io.*;
+import java.nio.file.Paths;
+import java.util.Map;
+
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.security.SecurityRequirement;
+import io.swagger.v3.oas.annotations.tags.Tag;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.api.ApiError;
+import org.qortal.api.ApiExceptionFactory;
+import org.qortal.api.Security;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.arbitrary.*;
+import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.controller.arbitrary.ArbitraryDataRenderManager;
+import org.qortal.data.transaction.ArbitraryTransactionData.*;
+import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
+import org.qortal.arbitrary.ArbitraryDataFile.*;
+import org.qortal.utils.Base58;
+
+
+@Path("/render")
+@Tag(name = "Render")
+public class RenderResource {
+
+    private static final Logger LOGGER = LogManager.getLogger(RenderResource.class);
+
+    @Context HttpServletRequest request;
+    @Context HttpServletResponse response;
+    @Context ServletContext context;
+
+    @POST
+    @Path("/preview")
+    @Operation(
+            summary = "Generate preview URL based on a user-supplied path and service",
+            requestBody = @RequestBody(
+                    required = true,
+                    content = @Content(
+                            mediaType = MediaType.TEXT_PLAIN,
+                            schema = @Schema(
+                                    type = "string", example = "/Users/user/Documents/MyStaticWebsite"
+                            )
+                    )
+            ),
+            responses = {
+                    @ApiResponse(
+                            description = "a temporary URL to preview the website",
+                            content = @Content(
+                                    mediaType = MediaType.TEXT_PLAIN,
+                                    schema = @Schema(
+                                            type = "string"
+                                    )
+                            )
+                    )
+            }
+    )
+    @SecurityRequirement(name = "apiKey")
+    public String preview(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String directoryPath) {
+        Security.checkApiCallAllowed(request);
+        Method method = Method.PUT;
+        Compression compression = Compression.ZIP;
+
+        ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), null, Service.WEBSITE, null, method, compression);
+        try {
+            arbitraryDataWriter.save();
+        } catch (IOException | DataException | InterruptedException | MissingDataException e) {
+            LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage());
+            throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
+        } catch (RuntimeException e) {
+            LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage());
+            throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+        }
+
+        ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile();
+        if (arbitraryDataFile != null) {
+            String digest58 = arbitraryDataFile.digest58();
+            if (digest58 != null) {
+                return "http://localhost:12393/render/hash/" + digest58 + "?secret=" + Base58.encode(arbitraryDataFile.getSecret());
+            }
+        }
+        return "Unable to generate preview URL";
+    }
+
+    @POST
+    @Path("/authorize/{resourceId}")
+    @SecurityRequirement(name = "apiKey")
+    public boolean authorizeResource(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("resourceId") String resourceId) {
+        Security.checkApiCallAllowed(request);
+        Security.disallowLoopbackRequestsIfAuthBypassEnabled(request);
+        ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, null, null);
+        ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource);
+        return true;
+    }
+
+    @POST
+    @Path("authorize/{service}/{resourceId}")
+    @SecurityRequirement(name = "apiKey")
+    public boolean authorizeResource(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
+                                     @PathParam("service") Service service,
+                                     @PathParam("resourceId") String resourceId) {
+        Security.checkApiCallAllowed(request);
+        Security.disallowLoopbackRequestsIfAuthBypassEnabled(request);
+        ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, null);
+        ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource);
+        return true;
+    }
+
+    @POST
+    @Path("authorize/{service}/{resourceId}/{identifier}")
+    @SecurityRequirement(name = "apiKey")
+    public boolean authorizeResource(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
+                                     @PathParam("service") Service service,
+                                     @PathParam("resourceId") String resourceId,
+                                     @PathParam("identifier") String identifier) {
+        Security.checkApiCallAllowed(request);
+        Security.disallowLoopbackRequestsIfAuthBypassEnabled(request);
+        ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, identifier);
+        ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource);
+        return true;
+    }
+
+    @GET
+    @Path("/signature/{signature}")
+    @SecurityRequirement(name = "apiKey")
+    public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature) {
+        Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null);
+        return this.get(signature, ResourceIdType.SIGNATURE, null, "/", null, "/render/signature", true, true);
+    }
+
+    @GET
+    @Path("/signature/{signature}/{path:.*}")
+    @SecurityRequirement(name = "apiKey")
+    public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath) {
+        Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null);
+        return this.get(signature, ResourceIdType.SIGNATURE, null, inPath,null, "/render/signature", true, true);
+    }
+
+    @GET
+    @Path("/hash/{hash}")
+    @SecurityRequirement(name = "apiKey")
+    public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58) {
+        Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null);
+        return this.get(hash58, ResourceIdType.FILE_HASH, Service.WEBSITE, "/", secret58, "/render/hash", true, false);
+    }
+
+    @GET
+    @Path("/hash/{hash}/{path:.*}")
+    @SecurityRequirement(name = "apiKey")
+    public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath,
+                                             @QueryParam("secret") String secret58) {
+        Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null);
+        return this.get(hash58, ResourceIdType.FILE_HASH, Service.WEBSITE, inPath, secret58, "/render/hash", true, false);
+    }
+
+    @GET
+    @Path("{service}/{name}/{path:.*}")
+    @SecurityRequirement(name = "apiKey")
+    public HttpServletResponse getPathByName(@PathParam("service") Service service,
+                                             @PathParam("name") String name,
+                                             @PathParam("path") String inPath) {
+        Security.requirePriorAuthorization(request, name, service, null);
+        String prefix = String.format("/render/%s", service);
+        return this.get(name, ResourceIdType.NAME, service, inPath, null, prefix, true, true);
+    }
+
+    @GET
+    @Path("{service}/{name}")
+    @SecurityRequirement(name = "apiKey")
+    public HttpServletResponse getIndexByName(@PathParam("service") Service service,
+                                              @PathParam("name") String name) {
+        Security.requirePriorAuthorization(request, name, service, null);
+        String prefix = String.format("/render/%s", service);
+        return this.get(name, ResourceIdType.NAME, service, "/", null, prefix, true, true);
+    }
+
+
+
+    private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String inPath,
+                                    String secret58, String prefix, boolean usePrefix, boolean async) {
+
+        ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, inPath,
+                secret58, prefix, usePrefix, async, request, response, context);
+        return renderer.render();
+    }
+
+}

src/main/java/org/qortal/api/resource/TransactionsResource.java

@@ -9,6 +9,8 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -44,6 +46,7 @@ import org.qortal.transform.transaction.TransactionTransformer;
 import org.qortal.utils.Base58;
 
 import com.google.common.primitives.Bytes;
+import org.qortal.utils.NTP;
 
 @Path("/transactions")
 @Tag(name = "Transactions")
@@ -348,7 +351,7 @@ public class TransactionsResource {
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(startBlock, blockLimit, txGroupId,
-					txTypes, null, address, confirmationStatus, limit, offset, reverse);
+					txTypes, null, null, address, confirmationStatus, limit, offset, reverse);
 
 			// Expand signatures to transactions
 			List<TransactionData> transactions = new ArrayList<>(signatures.size());
@@ -363,6 +366,83 @@ public class TransactionsResource {
 		}
 	}
 
+	@GET
+	@Path("/unitfee")
+	@Operation(
+			summary = "Get transaction unit fee",
+			responses = {
+					@ApiResponse(
+							content = @Content(
+									mediaType = MediaType.TEXT_PLAIN,
+									schema = @Schema(
+											type = "number"
+									)
+							)
+					)
+			}
+	)
+	@ApiErrors({
+			ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE
+	})
+	public long getTransactionUnitFee(@QueryParam("txType") TransactionType txType,
+                                      @QueryParam("timestamp") Long timestamp,
+									  @QueryParam("level") Integer accountLevel) {
+		try {
+			if (timestamp == null) {
+				timestamp = NTP.getTime();
+			}
+
+			Constructor<?> constructor = txType.constructor;
+			Transaction transaction = (Transaction) constructor.newInstance(null, null);
+			// FUTURE: add accountLevel parameter to transaction.getUnitFee() if needed
+			return transaction.getUnitFee(timestamp);
+
+		} catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e);
+		}
+	}
+
+	@POST
+	@Path("/fee")
+	@Operation(
+			summary = "Get recommended fee for supplied transaction data",
+			requestBody = @RequestBody(
+					required = true,
+					content = @Content(
+							mediaType = MediaType.TEXT_PLAIN,
+							schema = @Schema(
+									type = "string"
+							)
+					)
+			)
+	)
+	@ApiErrors({
+			ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE
+	})
+	public long getRecommendedTransactionFee(String rawInputBytes58) {
+		byte[] rawInputBytes = Base58.decode(rawInputBytes58);
+		if (rawInputBytes.length == 0)
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.JSON);
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+
+			// Append null signature on the end before transformation
+			byte[] rawBytes = Bytes.concat(rawInputBytes, new byte[TransactionTransformer.SIGNATURE_LENGTH]);
+
+			TransactionData transactionData = TransactionTransformer.fromBytes(rawBytes);
+			if (transactionData == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+
+			Transaction transaction = Transaction.fromData(repository, transactionData);
+			return transaction.calcRecommendedFee();
+
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}  catch (TransformationException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e);
+		}
+	}
+
 	@GET
 	@Path("/creator/{publickey}")
 	@Operation(
@@ -418,32 +498,83 @@ public class TransactionsResource {
 	}
 
 	@POST
-	@Path("/sign")
+	@Path("/convert")
 	@Operation(
-		summary = "Sign a raw, unsigned transaction",
-		requestBody = @RequestBody(
-			required = true,
-			content = @Content(
-				mediaType = MediaType.APPLICATION_JSON,
-				schema = @Schema(
-					implementation = SimpleTransactionSignRequest.class
-				)
-			)
-		),
-		responses = {
-			@ApiResponse(
-				description = "raw, signed transaction encoded in Base58",
-				content = @Content(
-					mediaType = MediaType.TEXT_PLAIN,
-					schema = @Schema(
-						type = "string"
+			summary = "Convert transaction bytes into bytes for signing",
+			requestBody = @RequestBody(
+					required = true,
+					content = @Content(
+							mediaType = MediaType.TEXT_PLAIN,
+							schema = @Schema(
+									type = "string",
+									description = "raw, unsigned transaction in base58 encoding",
+									example = "raw transaction base58"
+							)
 					)
-				)
-			)
-		}
+			),
+			responses = {
+					@ApiResponse(
+							description = "raw, unsigned transaction encoded in Base58, ready for signing",
+							content = @Content(
+									mediaType = MediaType.TEXT_PLAIN,
+									schema = @Schema(
+											type = "string"
+									)
+							)
+					)
+			}
 	)
 	@ApiErrors({
-		ApiError.NON_PRODUCTION, ApiError.INVALID_PRIVATE_KEY, ApiError.TRANSFORMATION_ERROR
+			ApiError.NON_PRODUCTION, ApiError.TRANSFORMATION_ERROR
+	})
+	public String convertTransactionForSigning(String rawInputBytes58) {
+		byte[] rawInputBytes = Base58.decode(rawInputBytes58);
+		if (rawInputBytes.length == 0)
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.JSON);
+
+		try {
+			// Append null signature on the end before transformation
+			byte[] rawBytes = Bytes.concat(rawInputBytes, new byte[TransactionTransformer.SIGNATURE_LENGTH]);
+
+			TransactionData transactionData = TransactionTransformer.fromBytes(rawBytes);
+			if (transactionData == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+
+			byte[] convertedBytes = TransactionTransformer.toBytesForSigning(transactionData);
+
+			return Base58.encode(convertedBytes);
+		} catch (TransformationException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e);
+		}
+	}
+
+	@POST
+	@Path("/sign")
+	@Operation(
+			summary = "Sign a raw, unsigned transaction",
+			requestBody = @RequestBody(
+					required = true,
+					content = @Content(
+							mediaType = MediaType.APPLICATION_JSON,
+							schema = @Schema(
+									implementation = SimpleTransactionSignRequest.class
+							)
+					)
+			),
+			responses = {
+					@ApiResponse(
+							description = "raw, signed transaction encoded in Base58",
+							content = @Content(
+									mediaType = MediaType.TEXT_PLAIN,
+									schema = @Schema(
+											type = "string"
+									)
+							)
+					)
+			}
+	)
+	@ApiErrors({
+			ApiError.NON_PRODUCTION, ApiError.INVALID_PRIVATE_KEY, ApiError.TRANSFORMATION_ERROR
 	})
 	public String signTransaction(SimpleTransactionSignRequest signRequest) {
 		if (Settings.getInstance().isApiRestricted())

src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java

@@ -0,0 +1,76 @@
+package org.qortal.arbitrary;
+
+import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.arbitrary.ArbitraryDataFile.*;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.repository.DataException;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+
+public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
+
+    private final Long creationTimestamp;
+    private Long buildStartTimestamp = null;
+    private Long buildEndTimestamp = null;
+    private boolean failed = false;
+
+    /* The maximum amount of time to spend on a single build */
+    // TODO: interrupt an in-progress build
+    public static long BUILD_TIMEOUT = 60*1000L; // 60 seconds
+    /* The amount of time to remember that a build has failed, to avoid retries */
+    public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes
+
+    public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
+        super(resourceId, resourceIdType, service, identifier);
+
+        this.creationTimestamp = NTP.getTime();
+    }
+
+    public void build() throws IOException, DataException, MissingDataException {
+        Long now = NTP.getTime();
+        if (now == null) {
+            throw new DataException("NTP time hasn't synced yet");
+        }
+
+        this.buildStartTimestamp = now;
+        ArbitraryDataReader arbitraryDataReader =
+                new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);
+
+        try {
+            arbitraryDataReader.loadSynchronously(true);
+        } finally {
+            this.buildEndTimestamp = NTP.getTime();
+        }
+    }
+
+    public boolean isBuilding() {
+        return this.buildStartTimestamp != null;
+    }
+
+    public boolean isQueued() {
+        return this.buildStartTimestamp == null;
+    }
+
+    public boolean hasReachedBuildTimeout(Long now) {
+        if (now == null || this.creationTimestamp == null) {
+            return true;
+        }
+        return now - this.creationTimestamp > BUILD_TIMEOUT;
+    }
+
+    public boolean hasReachedFailureTimeout(Long now) {
+        if (now == null || this.buildStartTimestamp == null) {
+            return true;
+        }
+        return now - this.buildStartTimestamp > FAILURE_TIMEOUT;
+    }
+
+    public Long getBuildStartTimestamp() {
+        return this.buildStartTimestamp;
+    }
+
+    public void setFailed(boolean failed) {
+        this.failed = failed;
+    }
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java

@@ -0,0 +1,280 @@
+package org.qortal.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.ArbitraryTransactionData.Method;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class ArbitraryDataBuilder {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuilder.class);
+
+    private final String name;
+    private final Service service;
+    private final String identifier;
+
+    private boolean canRequestMissingFiles;
+
+    private List<ArbitraryTransactionData> transactions;
+    private ArbitraryTransactionData latestPutTransaction;
+    private final List<Path> paths;
+    private byte[] latestSignature;
+    private Path finalPath;
+    private int layerCount;
+
+    public ArbitraryDataBuilder(String name, Service service, String identifier) {
+        this.name = name;
+        this.service = service;
+        this.identifier = identifier;
+        this.paths = new ArrayList<>();
+
+        // By default we can request missing files
+        // Callers can use setCanRequestMissingFiles(false) to prevent it
+        this.canRequestMissingFiles = true;
+    }
+
+    /**
+     * Process transactions, but do not build anything
+     * This is useful for checking the status of a given resource
+     *
+     * @throws DataException
+     * @throws IOException
+     * @throws MissingDataException
+     */
+    public void process() throws DataException, IOException, MissingDataException {
+        this.fetchTransactions();
+        this.validateTransactions();
+        this.processTransactions();
+        this.validatePaths();
+        this.findLatestSignature();
+    }
+
+    /**
+     * Build the latest state of a given resource
+     *
+     * @throws DataException
+     * @throws IOException
+     * @throws MissingDataException
+     */
+    public void build() throws DataException, IOException, MissingDataException {
+        this.process();
+        this.buildLatestState();
+        this.cacheLatestSignature();
+    }
+
+    private void fetchTransactions() throws DataException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Get the most recent PUT
+            ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
+                    .getLatestTransaction(this.name, this.service, Method.PUT, this.identifier);
+            if (latestPut == null) {
+                String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s",
+                        this.name, this.service, this.identifierString());
+                throw new DataException(message);
+            }
+            this.latestPutTransaction = latestPut;
+
+            // Load all transactions since the latest PUT
+            List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository()
+                    .getArbitraryTransactions(this.name, this.service, this.identifier, latestPut.getTimestamp());
+
+            this.transactions = transactionDataList;
+            this.layerCount = transactionDataList.size();
+        }
+    }
+
+    private void validateTransactions() throws DataException {
+        List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
+        ArbitraryTransactionData latestPut = this.latestPutTransaction;
+
+        if (latestPut == null) {
+            throw new DataException("Cannot PATCH without existing PUT. Deploy using PUT first.");
+        }
+        if (latestPut.getMethod() != Method.PUT) {
+            throw new DataException("Expected PUT but received PATCH");
+        }
+        if (transactionDataList.size() == 0) {
+            throw new DataException(String.format("No transactions found for name %s, service %s, " +
+                            "identifier: %s, since %d", name, service, this.identifierString(), latestPut.getTimestamp()));
+        }
+
+        // Verify that the signature of the first transaction matches the latest PUT
+        ArbitraryTransactionData firstTransaction = transactionDataList.get(0);
+        if (!Arrays.equals(firstTransaction.getSignature(), latestPut.getSignature())) {
+            throw new DataException("First transaction did not match latest PUT transaction");
+        }
+
+        // Remove the first transaction, as it should be the only PUT
+        transactionDataList.remove(0);
+
+        for (ArbitraryTransactionData transactionData : transactionDataList) {
+            if (transactionData == null) {
+                throw new DataException("Transaction not found");
+            }
+            if (transactionData.getMethod() != Method.PATCH) {
+                throw new DataException("Expected PATCH but received PUT");
+            }
+        }
+    }
+
+    private void processTransactions() throws IOException, DataException, MissingDataException {
+        List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
+
+        int count = 0;
+        for (ArbitraryTransactionData transactionData : transactionDataList) {
+            LOGGER.trace("Found arbitrary transaction {}", Base58.encode(transactionData.getSignature()));
+            count++;
+
+            // Build the data file, overwriting anything that was previously there
+            String sig58 = Base58.encode(transactionData.getSignature());
+            ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA,
+                    this.service, this.identifier);
+            arbitraryDataReader.setTransactionData(transactionData);
+            arbitraryDataReader.setCanRequestMissingFiles(this.canRequestMissingFiles);
+            boolean hasMissingData = false;
+            try {
+                arbitraryDataReader.loadSynchronously(true);
+            }
+            catch (MissingDataException e) {
+                hasMissingData = true;
+            }
+
+            // Handle missing data
+            if (hasMissingData) {
+                if (!this.canRequestMissingFiles) {
+                    throw new MissingDataException("Files are missing but were not requested.");
+                }
+                if (count == transactionDataList.size()) {
+                    // This is the final transaction in the list, so we need to fail
+                    throw new MissingDataException("Requesting missing files. Please wait and try again.");
+                }
+                // There are more transactions, so we should process them to give them the opportunity to request data
+                continue;
+            }
+
+            // By this point we should have all data needed to build the layers
+            Path path = arbitraryDataReader.getFilePath();
+            if (path == null) {
+                throw new DataException(String.format("Null path when building data from transaction %s", sig58));
+            }
+            if (!Files.exists(path)) {
+                throw new DataException(String.format("Path doesn't exist when building data from transaction %s", sig58));
+            }
+            paths.add(path);
+        }
+    }
+
+    private void findLatestSignature() throws DataException {
+        if (this.transactions.size() == 0) {
+            throw new DataException("Unable to find latest signature from empty transaction list");
+        }
+
+        // Find the latest signature
+        ArbitraryTransactionData latestTransaction = this.transactions.get(this.transactions.size() - 1);
+        if (latestTransaction == null) {
+            throw new DataException("Unable to find latest signature from null transaction");
+        }
+
+        this.latestSignature = latestTransaction.getSignature();
+    }
+
+    private void validatePaths() throws DataException {
+        if (this.paths.isEmpty()) {
+            throw new DataException("No paths available from which to build latest state");
+        }
+    }
+
+    private void buildLatestState() throws IOException, DataException {
+        if (this.paths.size() == 1) {
+            // No patching needed
+            this.finalPath = this.paths.get(0);
+            return;
+        }
+
+        Path pathBefore = this.paths.get(0);
+        boolean validateAllLayers = Settings.getInstance().shouldValidateAllDataLayers();
+
+        // Loop from the second path onwards
+        for (int i=1; i<paths.size(); i++) {
+            String identifierPrefix = this.identifier != null ? String.format("[%s]", this.identifier) : "";
+            LOGGER.debug(String.format("[%s][%s]%s Applying layer %d...", this.service, this.name, identifierPrefix, i));
+
+            // Create an instance of ArbitraryDataCombiner
+            Path pathAfter = this.paths.get(i);
+            byte[] signatureBefore = this.transactions.get(i-1).getSignature();
+            ArbitraryDataCombiner combiner = new ArbitraryDataCombiner(pathBefore, pathAfter, signatureBefore);
+
+            // We only want to validate this layer's hash if it's the final layer, or if the settings
+            // indicate that we should validate interim layers too
+            boolean isFinalLayer = (i == paths.size() - 1);
+            combiner.setShouldValidateHashes(isFinalLayer || validateAllLayers);
+
+            // Now combine this layer with the last, and set the output path to the "before" path for the next cycle
+            combiner.combine();
+            combiner.cleanup();
+            pathBefore = combiner.getFinalPath();
+        }
+        this.finalPath = pathBefore;
+    }
+
+    private void cacheLatestSignature() throws IOException, DataException {
+        byte[] latestTransactionSignature = this.transactions.get(this.transactions.size()-1).getSignature();
+        if (latestTransactionSignature == null) {
+            throw new DataException("Missing latest transaction signature");
+        }
+        Long now = NTP.getTime();
+        if (now == null) {
+            throw new DataException("NTP time not synced yet");
+        }
+
+        ArbitraryDataMetadataCache cache = new ArbitraryDataMetadataCache(this.finalPath);
+        cache.setSignature(latestTransactionSignature);
+        cache.setTimestamp(NTP.getTime());
+        cache.write();
+    }
+
+    private String identifierString() {
+        return identifier != null ? identifier : "";
+    }
+
+    public Path getFinalPath() {
+        return this.finalPath;
+    }
+
+    public byte[] getLatestSignature() {
+        return this.latestSignature;
+    }
+
+    public int getLayerCount() {
+        return this.layerCount;
+    }
+
+    /**
+     * Use the below setter to ensure that we only read existing
+     * data without requesting any missing files,
+     *
+     * @param canRequestMissingFiles
+     */
+    public void setCanRequestMissingFiles(boolean canRequestMissingFiles) {
+        this.canRequestMissingFiles = canRequestMissingFiles;
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java

@@ -0,0 +1,166 @@
+package org.qortal.arbitrary;
+
+import org.qortal.arbitrary.ArbitraryDataFile.*;
+import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.controller.arbitrary.ArbitraryDataManager;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.utils.FilesystemUtils;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+public class ArbitraryDataCache {
+
+    private final boolean overwrite;
+    private final Path filePath;
+    private final String resourceId;
+    private final ResourceIdType resourceIdType;
+    private final Service service;
+    private final String identifier;
+
+    public ArbitraryDataCache(Path filePath, boolean overwrite, String resourceId,
+                              ResourceIdType resourceIdType, Service service, String identifier) {
+        this.filePath = filePath;
+        this.overwrite = overwrite;
+        this.resourceId = resourceId;
+        this.resourceIdType = resourceIdType;
+        this.service = service;
+        this.identifier = identifier;
+    }
+
+    public boolean isCachedDataAvailable() {
+        return !this.shouldInvalidate();
+    }
+
+    public boolean shouldInvalidate() {
+        try {
+            // If the user has requested an overwrite, always invalidate the cache
+            if (this.overwrite) {
+                return true;
+            }
+
+            // Overwrite is false, but we still need to invalidate if no files exist
+            if (!Files.exists(this.filePath) || FilesystemUtils.isDirectoryEmpty(this.filePath)) {
+                return true;
+            }
+
+            // We might want to overwrite anyway, if an updated version is available
+            if (this.shouldInvalidateResource()) {
+                return true;
+            }
+
+        } catch (IOException e) {
+            // Something went wrong, so invalidate the cache just in case
+            return true;
+        }
+
+        // No need to invalidate the cache
+        return false;
+    }
+
+    private boolean shouldInvalidateResource() {
+        switch (this.resourceIdType) {
+
+            case NAME:
+                return this.shouldInvalidateName();
+
+            default:
+                // Other resource ID types remain constant, so no need to invalidate
+                return false;
+        }
+    }
+
+    private boolean shouldInvalidateName() {
+        // To avoid spamming the database too often, we shouldn't check sigs or invalidate when rate limited
+        if (this.rateLimitInEffect()) {
+            return false;
+        }
+
+        // If the state's sig doesn't match the latest transaction's sig, we need to invalidate
+        // This means that an updated layer is available
+        if (this.shouldInvalidateDueToSignatureMismatch()) {
+
+            // Add to the in-memory cache first, so that we won't check again for a while
+            ArbitraryDataManager.getInstance().addResourceToCache(this.getArbitraryDataResource());
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * rateLimitInEffect()
+     *
+     * When loading a website, we need to check the cache for every static asset loaded by the page.
+     * This would involve asking the database for the latest transaction every time.
+     * To reduce database load and page load times, we maintain an in-memory list to "rate limit" lookups.
+     * Once a resource ID is in this in-memory list, we will avoid cache invalidations until it
+     * has been present in the list for a certain amount of time.
+     * Items are automatically removed from the list when a new arbitrary transaction arrives, so this
+     * should not prevent updates from taking effect immediately.
+     *
+     * @return whether to avoid lookups for this resource due to the in-memory cache
+     */
+    private boolean rateLimitInEffect() {
+        return ArbitraryDataManager.getInstance().isResourceCached(this.getArbitraryDataResource());
+    }
+
+    private boolean shouldInvalidateDueToSignatureMismatch() {
+
+        // Fetch the latest transaction for this name and service
+        byte[] latestTransactionSig = this.fetchLatestTransactionSignature();
+
+        // Now fetch the transaction signature stored in the cache metadata
+        byte[] cachedSig = this.fetchCachedSignature();
+
+        // If either are null, we should invalidate
+        if (latestTransactionSig == null || cachedSig == null) {
+            return true;
+        }
+
+        // Check if they match
+        return !Arrays.equals(latestTransactionSig, cachedSig);
+    }
+
+    private byte[] fetchLatestTransactionSignature() {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Find latest transaction for name and service, with any method
+            ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
+                    .getLatestTransaction(this.resourceId, this.service, null, this.identifier);
+
+            if (latestTransaction != null) {
+                return latestTransaction.getSignature();
+            }
+
+        } catch (DataException e) {
+            return null;
+        }
+
+        return null;
+    }
+
+    private byte[] fetchCachedSignature() {
+        try {
+            // Fetch the transaction signature stored in the cache metadata
+            ArbitraryDataMetadataCache cache = new ArbitraryDataMetadataCache(this.filePath);
+            cache.read();
+            return cache.getSignature();
+
+        } catch (IOException | DataException e) {
+            return null;
+        }
+    }
+
+    private ArbitraryDataResource getArbitraryDataResource() {
+        // TODO: pass an ArbitraryDataResource into the constructor, rather than individual components
+        return new ArbitraryDataResource(this.resourceId, this.resourceIdType, this.service, this.identifier);
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java

@@ -0,0 +1,170 @@
+package org.qortal.arbitrary;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
+import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+import org.qortal.utils.FilesystemUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InvalidObjectException;
+import java.nio.file.DirectoryNotEmptyException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+public class ArbitraryDataCombiner {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCombiner.class);
+
+    private final Path pathBefore;
+    private final Path pathAfter;
+    private final byte[] signatureBefore;
+    private boolean shouldValidateHashes;
+    private Path finalPath;
+    private ArbitraryDataMetadataPatch metadata;
+
+    public ArbitraryDataCombiner(Path pathBefore, Path pathAfter, byte[] signatureBefore) {
+        this.pathBefore = pathBefore;
+        this.pathAfter = pathAfter;
+        this.signatureBefore = signatureBefore;
+    }
+
+    public void combine() throws IOException, DataException {
+        try {
+            this.preExecute();
+            this.readMetadata();
+            this.validatePreviousSignature();
+            this.validatePreviousHash();
+            this.process();
+            this.validateCurrentHash();
+
+        } finally {
+            this.postExecute();
+        }
+    }
+
+    public void cleanup() {
+        this.cleanupPath(this.pathBefore);
+        this.cleanupPath(this.pathAfter);
+    }
+
+    private void cleanupPath(Path path) {
+        // Delete pathBefore, if it exists in our data/temp directory
+        if (FilesystemUtils.pathInsideDataOrTempPath(path)) {
+            File directory = new File(path.toString());
+            try {
+                FileUtils.deleteDirectory(directory);
+            } catch (IOException e) {
+                // This will eventually be cleaned up by a maintenance process, so log the error and continue
+                LOGGER.debug("Unable to cleanup directory {}", directory.toString());
+            }
+        }
+
+        // Delete the parent directory of pathBefore if it is empty (and exists in our data/temp directory)
+        Path parentDirectory = path.getParent();
+        if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) {
+            try {
+                Files.deleteIfExists(parentDirectory);
+            } catch (DirectoryNotEmptyException e) {
+                // No need to log anything
+            } catch (IOException e) {
+                // This will eventually be cleaned up by a maintenance process, so log the error and continue
+                LOGGER.debug("Unable to cleanup parent directory {}", parentDirectory.toString());
+            }
+        }
+    }
+
+    private void preExecute() throws DataException {
+        if (this.pathBefore == null || this.pathAfter == null) {
+            throw new DataException("No paths available to build patch");
+        }
+        if (!Files.exists(this.pathBefore) || !Files.exists(this.pathAfter)) {
+            throw new DataException("Unable to create patch because at least one path doesn't exist");
+        }
+    }
+
+    private void postExecute() {
+
+    }
+
+    private void readMetadata() throws IOException, DataException {
+        this.metadata = new ArbitraryDataMetadataPatch(this.pathAfter);
+        this.metadata.read();
+    }
+
+    private void validatePreviousSignature() throws DataException {
+        if (this.signatureBefore == null) {
+            throw new DataException("No previous signature passed to the combiner");
+        }
+
+        byte[] previousSignature = this.metadata.getPreviousSignature();
+        if (previousSignature == null) {
+            throw new DataException("Unable to extract previous signature from patch metadata");
+        }
+
+        // Compare the signatures
+        if (!Arrays.equals(previousSignature, this.signatureBefore)) {
+            throw new DataException("Previous signatures do not match - transactions out of order?");
+        }
+    }
+
+    private void validatePreviousHash() throws IOException, DataException {
+        if (!Settings.getInstance().shouldValidateAllDataLayers()) {
+            return;
+        }
+
+        byte[] previousHash = this.metadata.getPreviousHash();
+        if (previousHash == null) {
+            throw new DataException("Unable to extract previous hash from patch metadata");
+        }
+
+        ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathBefore);
+        digest.compute();
+        boolean valid = digest.isHashValid(previousHash);
+        if (!valid) {
+            String previousHash58 = Base58.encode(previousHash);
+            throw new InvalidObjectException(String.format("Previous state hash mismatch. " +
+                    "Patch prevHash: %s, actual: %s", previousHash58, digest.getHash58()));
+        }
+    }
+
+    private void process() throws IOException, DataException {
+        ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter);
+        merge.compute();
+        this.finalPath = merge.getMergePath();
+    }
+
+    private void validateCurrentHash() throws IOException, DataException {
+        if (!this.shouldValidateHashes) {
+            return;
+        }
+
+        byte[] currentHash = this.metadata.getCurrentHash();
+        if (currentHash == null) {
+            throw new DataException("Unable to extract current hash from patch metadata");
+        }
+
+        ArbitraryDataDigest digest = new ArbitraryDataDigest(this.finalPath);
+        digest.compute();
+        boolean valid = digest.isHashValid(currentHash);
+        if (!valid) {
+            String currentHash58 = Base58.encode(currentHash);
+            throw new InvalidObjectException(String.format("Current state hash mismatch. " +
+                    "Patch curHash: %s, actual: %s", currentHash58, digest.getHash58()));
+        }
+	}
+
+    public void setShouldValidateHashes(boolean shouldValidateHashes) {
+        this.shouldValidateHashes = shouldValidateHashes;
+    }
+
+    public Path getFinalPath() {
+        return this.finalPath;
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java

@@ -0,0 +1,141 @@
+package org.qortal.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
+import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
+import org.qortal.utils.FilesystemUtils;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.UUID;
+
+public class ArbitraryDataCreatePatch {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCreatePatch.class);
+
+    private final Path pathBefore;
+    private Path pathAfter;
+    private final byte[] previousSignature;
+
+    private Path finalPath;
+    private int totalFileCount;
+    private int fileDifferencesCount;
+    private ArbitraryDataMetadataPatch metadata;
+
+    private Path workingPath;
+    private String identifier;
+
+    public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter, byte[] previousSignature) {
+        this.pathBefore = pathBefore;
+        this.pathAfter = pathAfter;
+        this.previousSignature = previousSignature;
+    }
+
+    public void create() throws DataException, IOException {
+        try {
+            this.preExecute();
+            this.copyFiles();
+            this.process();
+
+        } catch (Exception e) {
+            this.cleanupOnFailure();
+            throw e;
+
+        } finally {
+            this.postExecute();
+        }
+    }
+
+    private void preExecute() throws DataException {
+        if (this.pathBefore == null || this.pathAfter == null) {
+            throw new DataException("No paths available to build patch");
+        }
+        if (!Files.exists(this.pathBefore) || !Files.exists(this.pathAfter)) {
+            throw new DataException("Unable to create patch because at least one path doesn't exist");
+        }
+
+        this.createRandomIdentifier();
+        this.createWorkingDirectory();
+    }
+
+    private void postExecute() {
+        this.cleanupWorkingPath();
+    }
+
+    private void cleanupWorkingPath() {
+        try {
+            FilesystemUtils.safeDeleteDirectory(this.workingPath, true);
+        } catch (IOException e) {
+            LOGGER.debug("Unable to cleanup working directory");
+        }
+    }
+
+    private void cleanupOnFailure() {
+        try {
+            FilesystemUtils.safeDeleteDirectory(this.finalPath, true);
+        } catch (IOException e) {
+            LOGGER.debug("Unable to cleanup diff directory on failure");
+        }
+    }
+
+    private void createRandomIdentifier() {
+        this.identifier = UUID.randomUUID().toString();
+    }
+
+    private void createWorkingDirectory() throws DataException {
+        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
+        String baseDir = Settings.getInstance().getTempDataPath();
+        Path tempDir = Paths.get(baseDir, "patch", this.identifier);
+        try {
+            Files.createDirectories(tempDir);
+        } catch (IOException e) {
+            throw new DataException("Unable to create temp directory");
+        }
+        this.workingPath = tempDir;
+    }
+
+    private void copyFiles() throws IOException {
+        // When dealing with single files, we need to copy them to a container directory
+        // in order for the structure to align with the previous revision and therefore
+        // make comparisons possible.
+
+        if (this.pathAfter.toFile().isFile()) {
+            // Create a "data" directory within the working directory
+            Path workingDataPath = Paths.get(this.workingPath.toString(), "data");
+            Files.createDirectories(workingDataPath);
+            // Copy to temp directory
+            // Filename is currently hardcoded to "data"
+            String filename = "data"; //this.pathAfter.getFileName().toString();
+            Files.copy(this.pathAfter, Paths.get(workingDataPath.toString(), filename));
+            // Update pathAfter to point to the new path
+            this.pathAfter = workingDataPath;
+        }
+    }
+
+    private void process() throws IOException, DataException {
+
+        ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter, this.previousSignature);
+        this.finalPath = diff.getDiffPath();
+        diff.compute();
+
+        this.totalFileCount = diff.getTotalFileCount();
+        this.metadata = diff.getMetadata();
+    }
+
+    public Path getFinalPath() {
+        return this.finalPath;
+    }
+
+    public int getTotalFileCount() {
+        return this.totalFileCount;
+    }
+
+    public ArbitraryDataMetadataPatch getMetadata() {
+        return this.metadata;
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java

@@ -0,0 +1,383 @@
+package org.qortal.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.json.JSONObject;
+import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
+import org.qortal.arbitrary.patch.UnifiedDiffPatch;
+import org.qortal.crypto.Crypto;
+import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
+
+import java.io.*;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.*;
+
+
+public class ArbitraryDataDiff {
+
+    /** Only create a patch if both the before and after file sizes are within defined limit **/
+    private static final long MAX_DIFF_FILE_SIZE = 100 * 1024L; // 100kiB
+
+
+    public enum DiffType {
+        COMPLETE_FILE,
+        UNIFIED_DIFF
+    }
+
+    public static class ModifiedPath {
+        private Path path;
+        private DiffType diffType;
+
+        public ModifiedPath(Path path, DiffType diffType) {
+            this.path = path;
+            this.diffType = diffType;
+        }
+
+        public ModifiedPath(JSONObject jsonObject) {
+            String pathString = jsonObject.getString("path");
+            if (pathString != null) {
+                this.path = Paths.get(pathString);
+            }
+
+            String diffTypeString = jsonObject.getString("type");
+            if (diffTypeString != null) {
+                this.diffType = DiffType.valueOf(diffTypeString);
+            }
+        }
+
+        public Path getPath() {
+            return this.path;
+        }
+
+        public DiffType getDiffType() {
+            return this.diffType;
+        }
+
+        public String toString() {
+            return this.path.toString();
+        }
+    }
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataDiff.class);
+
+    private final Path pathBefore;
+    private final Path pathAfter;
+    private final byte[] previousSignature;
+    private byte[] previousHash;
+    private byte[] currentHash;
+    private Path diffPath;
+    private String identifier;
+
+    private final List<Path> addedPaths;
+    private final List<ModifiedPath> modifiedPaths;
+    private final List<Path> removedPaths;
+
+    private int totalFileCount;
+    private ArbitraryDataMetadataPatch metadata;
+
+    public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) throws DataException {
+        this.pathBefore = pathBefore;
+        this.pathAfter = pathAfter;
+        this.previousSignature = previousSignature;
+
+        this.addedPaths = new ArrayList<>();
+        this.modifiedPaths = new ArrayList<>();
+        this.removedPaths = new ArrayList<>();
+
+        this.createRandomIdentifier();
+        this.createOutputDirectory();
+    }
+
+    public void compute() throws IOException, DataException {
+        try {
+            this.preExecute();
+            this.hashPreviousState();
+            this.findAddedOrModifiedFiles();
+            this.findRemovedFiles();
+            this.validate();
+            this.hashCurrentState();
+            this.writeMetadata();
+
+        } finally {
+            this.postExecute();
+        }
+    }
+
+    private void preExecute() {
+        LOGGER.debug("Generating diff...");
+    }
+
+    private void postExecute() {
+
+    }
+
+    private void createRandomIdentifier() {
+        this.identifier = UUID.randomUUID().toString();
+    }
+
+    private void createOutputDirectory() throws DataException {
+        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
+        String baseDir = Settings.getInstance().getTempDataPath();
+        Path tempDir = Paths.get(baseDir, "diff", this.identifier);
+        try {
+            Files.createDirectories(tempDir);
+        } catch (IOException e) {
+            throw new DataException("Unable to create temp directory");
+        }
+        this.diffPath = tempDir;
+    }
+
+    private void hashPreviousState() throws IOException, DataException {
+        ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathBefore);
+        digest.compute();
+        this.previousHash = digest.getHash();
+    }
+
+    private void findAddedOrModifiedFiles() throws IOException {
+        try {
+            final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
+            final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
+            final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
+            final ArbitraryDataDiff diff = this;
+
+            // Check for additions or modifications
+            Files.walkFileTree(this.pathAfter, new FileVisitor<>() {
+
+                @Override
+                public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) {
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFile(Path afterPathAbsolute, BasicFileAttributes attrs) throws IOException {
+                    Path afterPathRelative = pathAfterAbsolute.relativize(afterPathAbsolute.toAbsolutePath());
+                    Path beforePathAbsolute = pathBeforeAbsolute.resolve(afterPathRelative);
+
+                    if (afterPathRelative.startsWith(".qortal")) {
+                        // Ignore the .qortal metadata folder
+                        return FileVisitResult.CONTINUE;
+                    }
+
+                    boolean wasAdded = false;
+                    boolean wasModified = false;
+
+                    if (!Files.exists(beforePathAbsolute)) {
+                        LOGGER.trace("File was added: {}", afterPathRelative.toString());
+                        diff.addedPaths.add(afterPathRelative);
+                        wasAdded = true;
+                    }
+                    else if (Files.size(afterPathAbsolute) != Files.size(beforePathAbsolute)) {
+                        // Check file size first because it's quicker
+                        LOGGER.trace("File size was modified: {}", afterPathRelative.toString());
+                        wasModified = true;
+                    }
+                    else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(afterPathAbsolute), ArbitraryDataDiff.digestFromPath(beforePathAbsolute))) {
+                        // Check hashes as a last resort
+                        LOGGER.trace("File contents were modified: {}", afterPathRelative.toString());
+                        wasModified = true;
+                    }
+
+                    if (wasAdded) {
+                        diff.copyFilePathToBaseDir(afterPathAbsolute, diffPathAbsolute, afterPathRelative);
+                    }
+                    if (wasModified) {
+                        try {
+                            diff.pathModified(beforePathAbsolute, afterPathAbsolute, afterPathRelative, diffPathAbsolute);
+                        } catch (DataException e) {
+                            // We can only throw IOExceptions because we are overriding FileVisitor.visitFile()
+                            throw new IOException(e);
+                        }
+                    }
+
+                    // Keep a tally of the total number of files to help with decision making
+                    diff.totalFileCount++;
+
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException e){
+                    LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage());
+                    // TODO: throw exception?
+                    return FileVisitResult.TERMINATE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException e) {
+                    return FileVisitResult.CONTINUE;
+                }
+
+            });
+        } catch (IOException e) {
+            LOGGER.info("IOException when walking through file tree: {}", e.getMessage());
+            throw(e);
+        }
+    }
+
+    private void findRemovedFiles() throws IOException {
+        try {
+            final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
+            final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
+            final ArbitraryDataDiff diff = this;
+
+            // Check for removals
+            Files.walkFileTree(this.pathBefore, new FileVisitor<>() {
+
+                @Override
+                public FileVisitResult preVisitDirectory(Path before, BasicFileAttributes attrs) {
+                    Path directoryPathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath());
+                    Path directoryPathAfter = pathAfterAbsolute.resolve(directoryPathBefore);
+
+                    if (directoryPathBefore.startsWith(".qortal")) {
+                        // Ignore the .qortal metadata folder
+                        return FileVisitResult.CONTINUE;
+                    }
+
+                    if (!Files.exists(directoryPathAfter)) {
+                        LOGGER.trace("Directory was removed: {}", directoryPathAfter.toString());
+                        diff.removedPaths.add(directoryPathBefore);
+                        // TODO: we might need to mark directories differently to files
+                    }
+
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFile(Path before, BasicFileAttributes attrs) {
+                    Path filePathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath());
+                    Path filePathAfter = pathAfterAbsolute.resolve(filePathBefore);
+
+                    if (filePathBefore.startsWith(".qortal")) {
+                        // Ignore the .qortal metadata folder
+                        return FileVisitResult.CONTINUE;
+                    }
+
+                    if (!Files.exists(filePathAfter)) {
+                        LOGGER.trace("File was removed: {}", filePathBefore.toString());
+                        diff.removedPaths.add(filePathBefore);
+                    }
+
+                    // Keep a tally of the total number of files to help with decision making
+                    diff.totalFileCount++;
+
+                    return FileVisitResult.CONTINUE;
+                }
+
+                @Override
+                public FileVisitResult visitFileFailed(Path file, IOException e){
+                    LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage());
+                    // TODO: throw exception?
+                    return FileVisitResult.TERMINATE;
+                }
+
+                @Override
+                public FileVisitResult postVisitDirectory(Path dir, IOException e) {
+                    return FileVisitResult.CONTINUE;
+                }
+
+            });
+        } catch (IOException e) {
+            throw new IOException(String.format("IOException when walking through file tree: %s", e.getMessage()));
+        }
+    }
+
+    private void validate() throws DataException {
+        if (this.addedPaths.isEmpty() && this.modifiedPaths.isEmpty() && this.removedPaths.isEmpty()) {
+            throw new DataException("Current state matches previous state. Nothing to do.");
+        }
+    }
+
+    private void hashCurrentState() throws IOException, DataException {
+        ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathAfter);
+        digest.compute();
+        this.currentHash = digest.getHash();
+    }
+
+    private void writeMetadata() throws IOException, DataException {
+        ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.diffPath);
+        metadata.setAddedPaths(this.addedPaths);
+        metadata.setModifiedPaths(this.modifiedPaths);
+        metadata.setRemovedPaths(this.removedPaths);
+        metadata.setPreviousSignature(this.previousSignature);
+        metadata.setPreviousHash(this.previousHash);
+        metadata.setCurrentHash(this.currentHash);
+        metadata.write();
+        this.metadata = metadata;
+    }
+
+
+    private void pathModified(Path beforePathAbsolute, Path afterPathAbsolute, Path afterPathRelative,
+                              Path destinationBasePathAbsolute) throws IOException, DataException {
+
+        Path destination = Paths.get(destinationBasePathAbsolute.toString(), afterPathRelative.toString());
+        long beforeSize = Files.size(beforePathAbsolute);
+        long afterSize = Files.size(afterPathAbsolute);
+        DiffType diffType;
+
+        if (beforeSize > MAX_DIFF_FILE_SIZE || afterSize > MAX_DIFF_FILE_SIZE) {
+            // Files are large, so don't attempt a diff
+            this.copyFilePathToBaseDir(afterPathAbsolute, destinationBasePathAbsolute, afterPathRelative);
+            diffType = DiffType.COMPLETE_FILE;
+        }
+        else {
+            // Attempt to create patch using java-diff-utils
+            UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(beforePathAbsolute, afterPathAbsolute, destination);
+            unifiedDiffPatch.create();
+            if (unifiedDiffPatch.isValid()) {
+                diffType = DiffType.UNIFIED_DIFF;
+            }
+            else {
+                // Diff failed validation, so copy the whole file instead
+                this.copyFilePathToBaseDir(afterPathAbsolute, destinationBasePathAbsolute, afterPathRelative);
+                diffType = DiffType.COMPLETE_FILE;
+            }
+        }
+
+        ModifiedPath modifiedPath = new ModifiedPath(afterPathRelative, diffType);
+        this.modifiedPaths.add(modifiedPath);
+    }
+
+    private void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
+        if (!Files.exists(source)) {
+            throw new IOException(String.format("File not found: %s", source.toString()));
+        }
+
+        // Ensure parent folders exist in the destination
+        Path dest = Paths.get(base.toString(), relativePath.toString());
+        File file = new File(dest.toString());
+        File parent = file.getParentFile();
+        if (parent != null) {
+            parent.mkdirs();
+        }
+
+        LOGGER.trace("Copying {} to {}", source, dest);
+        Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING);
+    }
+    
+
+    public Path getDiffPath() {
+        return this.diffPath;
+    }
+
+    public int getTotalFileCount() {
+        return this.totalFileCount;
+    }
+
+    public ArbitraryDataMetadataPatch getMetadata() {
+        return this.metadata;
+    }
+
+
+    // Utils
+
+    private static byte[] digestFromPath(Path path) {
+        try {
+            return Crypto.digest(path.toFile());
+        } catch (IOException e) {
+            return null;
+        }
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java

@@ -0,0 +1,73 @@
+package org.qortal.arbitrary;
+
+import org.qortal.repository.DataException;
+import org.qortal.utils.Base58;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class ArbitraryDataDigest {
+
+    private final Path path;
+    private byte[] hash;
+
+    public ArbitraryDataDigest(Path path) {
+        this.path = path;
+    }
+
+    public void compute() throws IOException, DataException {
+        List<Path> allPaths = Files.walk(path).filter(Files::isRegularFile).sorted().collect(Collectors.toList());
+        Path basePathAbsolute = this.path.toAbsolutePath();
+
+        MessageDigest sha256;
+        try {
+            sha256 = MessageDigest.getInstance("SHA-256");
+        } catch (NoSuchAlgorithmException e) {
+            throw new DataException("SHA-256 hashing algorithm unavailable");
+        }
+
+        for (Path path : allPaths) {
+            // We need to work with paths relative to the base path, to ensure the same hash
+            // is generated on different systems
+            Path relativePath = basePathAbsolute.relativize(path.toAbsolutePath());
+
+            // Exclude Qortal folder since it can be different each time
+            // We only care about hashing the actual user data
+            if (relativePath.startsWith(".qortal/")) {
+                continue;
+            }
+
+            // Hash path
+            byte[] filePathBytes = relativePath.toString().getBytes(StandardCharsets.UTF_8);
+            sha256.update(filePathBytes);
+
+            // Hash contents
+            byte[] fileContent = Files.readAllBytes(path);
+            sha256.update(fileContent);
+        }
+        this.hash = sha256.digest();
+    }
+
+    public boolean isHashValid(byte[] hash) {
+        return Arrays.equals(hash, this.hash);
+    }
+
+    public byte[] getHash() {
+        return this.hash;
+    }
+
+    public String getHash58() {
+        if (this.hash == null) {
+            return null;
+        }
+        return Base58.encode(this.hash);
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java

@@ -0,0 +1,778 @@
+package org.qortal.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.crypto.Crypto;
+import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+import org.qortal.utils.FilesystemUtils;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.util.*;
+import java.util.stream.Stream;
+
+import static java.util.Arrays.stream;
+import static java.util.stream.Collectors.toMap;
+
+
+public class ArbitraryDataFile {
+
+    // Validation results
+    public enum ValidationResult {
+        OK(1),
+        FILE_TOO_LARGE(10),
+        FILE_NOT_FOUND(11);
+
+        public final int value;
+
+        private static final Map<Integer, ArbitraryDataFile.ValidationResult> map = stream(ArbitraryDataFile.ValidationResult.values()).collect(toMap(result -> result.value, result -> result));
+
+        ValidationResult(int value) {
+            this.value = value;
+        }
+
+        public static ArbitraryDataFile.ValidationResult valueOf(int value) {
+            return map.get(value);
+        }
+    }
+
+    // Resource ID types
+    public enum ResourceIdType {
+        SIGNATURE,
+        FILE_HASH,
+        TRANSACTION_DATA,
+        NAME
+    }
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class);
+
+    public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB
+    public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB
+    public static int SHORT_DIGEST_LENGTH = 8;
+
+    protected Path filePath;
+    protected String hash58;
+    protected byte[] signature;
+    private ArrayList<ArbitraryDataFileChunk> chunks;
+    private byte[] secret;
+
+    // Metadata
+    private byte[] metadataHash;
+    private ArbitraryDataFile metadataFile;
+    private ArbitraryDataTransactionMetadata metadata;
+
+
+    public ArbitraryDataFile() {
+    }
+
+    public ArbitraryDataFile(String hash58, byte[] signature) throws DataException {
+        this.createDataDirectory();
+        this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, signature, false);
+        this.chunks = new ArrayList<>();
+        this.hash58 = hash58;
+        this.signature = signature;
+    }
+
+    public ArbitraryDataFile(byte[] fileContent, byte[] signature) throws DataException {
+        if (fileContent == null) {
+            LOGGER.error("fileContent is null");
+            return;
+        }
+
+        this.hash58 = Base58.encode(Crypto.digest(fileContent));
+        this.signature = signature;
+        LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length));
+
+        Path outputFilePath = getOutputFilePath(this.hash58, signature, true);
+        File outputFile = outputFilePath.toFile();
+        try (FileOutputStream outputStream = new FileOutputStream(outputFile)) {
+            outputStream.write(fileContent);
+            this.filePath = outputFilePath;
+            // Verify hash
+            if (!this.hash58.equals(this.digest58())) {
+                LOGGER.error("Hash {} does not match file digest {}", this.hash58, this.digest58());
+                this.delete();
+                throw new DataException("Data file digest validation failed");
+            }
+        } catch (IOException e) {
+            throw new DataException("Unable to write data to file");
+        }
+    }
+
+    public static ArbitraryDataFile fromHash58(String hash58, byte[] signature) throws DataException {
+        return new ArbitraryDataFile(hash58, signature);
+    }
+
+    public static ArbitraryDataFile fromHash(byte[] hash, byte[] signature) throws DataException {
+        return ArbitraryDataFile.fromHash58(Base58.encode(hash), signature);
+    }
+
+    public static ArbitraryDataFile fromPath(Path path, byte[] signature) {
+        if (path == null) {
+            return null;
+        }
+        File file = path.toFile();
+        if (file.exists()) {
+            try {
+                byte[] digest = Crypto.digest(file);
+                ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
+
+                // Copy file to data directory if needed
+                if (Files.exists(path) && !arbitraryDataFile.isInBaseDirectory(path)) {
+                    arbitraryDataFile.copyToDataDirectory(path, signature);
+                }
+                // Or, if it's already in the data directory, we may need to move it
+                else if (!path.equals(arbitraryDataFile.getFilePath())) {
+                    // Wrong path, so relocate (but don't cleanup, as the source folder may still be needed by the caller)
+                    Path dest = arbitraryDataFile.getFilePath();
+                    FilesystemUtils.moveFile(path, dest, false);
+                }
+                return arbitraryDataFile;
+
+            } catch (IOException | DataException e) {
+                LOGGER.error("Couldn't compute digest for ArbitraryDataFile");
+            }
+        }
+        return null;
+    }
+
+    public static ArbitraryDataFile fromFile(File file, byte[] signature) {
+        return ArbitraryDataFile.fromPath(Paths.get(file.getPath()), signature);
+    }
+
+    private boolean createDataDirectory() {
+        // Create the data directory if it doesn't exist
+        String dataPath = Settings.getInstance().getDataPath();
+        Path dataDirectory = Paths.get(dataPath);
+        try {
+            Files.createDirectories(dataDirectory);
+        } catch (IOException e) {
+            LOGGER.error("Unable to create data directory");
+            return false;
+        }
+        return true;
+    }
+
+    private Path copyToDataDirectory(Path sourcePath, byte[] signature) throws DataException {
+        if (this.hash58 == null || this.filePath == null) {
+            return null;
+        }
+        Path outputFilePath = getOutputFilePath(this.hash58, signature, true);
+        sourcePath = sourcePath.toAbsolutePath();
+        Path destPath = outputFilePath.toAbsolutePath();
+        try {
+            return Files.copy(sourcePath, destPath, StandardCopyOption.REPLACE_EXISTING);
+        } catch (IOException e) {
+            throw new DataException(String.format("Unable to copy file %s to data directory %s", sourcePath, destPath));
+        }
+    }
+
+    public static Path getOutputFilePath(String hash58, byte[] signature, boolean createDirectories) throws DataException {
+        Path directory;
+
+        if (hash58 == null) {
+            return null;
+        }
+        if (signature != null) {
+            // Key by signature
+            String signature58 = Base58.encode(signature);
+            String sig58First2Chars = signature58.substring(0, 2).toLowerCase();
+            String sig58Next2Chars = signature58.substring(2, 4).toLowerCase();
+            directory = Paths.get(Settings.getInstance().getDataPath(), sig58First2Chars, sig58Next2Chars, signature58);
+        }
+        else {
+            // Put files without signatures in a "_misc" directory, and the files will be relocated later
+            String hash58First2Chars = hash58.substring(0, 2).toLowerCase();
+            String hash58Next2Chars = hash58.substring(2, 4).toLowerCase();
+            directory = Paths.get(Settings.getInstance().getDataPath(), "_misc", hash58First2Chars, hash58Next2Chars);
+        }
+
+        if (createDirectories) {
+            try {
+                Files.createDirectories(directory);
+            } catch (IOException e) {
+                throw new DataException("Unable to create data subdirectory");
+            }
+        }
+        return Paths.get(directory.toString(), hash58);
+    }
+
+    public ValidationResult isValid() {
+        try {
+            // Ensure the file exists on disk
+            if (!Files.exists(this.filePath)) {
+                LOGGER.error("File doesn't exist at path {}", this.filePath);
+                return ValidationResult.FILE_NOT_FOUND;
+            }
+
+            // Validate the file size
+            long fileSize = Files.size(this.filePath);
+            if (fileSize > MAX_FILE_SIZE) {
+                LOGGER.error(String.format("ArbitraryDataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE));
+                return ArbitraryDataFile.ValidationResult.FILE_TOO_LARGE;
+            }
+
+        } catch (IOException e) {
+            return ValidationResult.FILE_NOT_FOUND;
+        }
+
+        return ValidationResult.OK;
+    }
+
+    public void validateFileSize(long expectedSize) throws DataException {
+        // Verify that we can determine the file's size
+        long fileSize = 0;
+        try {
+            fileSize = Files.size(this.getFilePath());
+        } catch (IOException e) {
+            throw new DataException(String.format("Couldn't get file size for transaction %s", Base58.encode(signature)));
+        }
+
+        // Ensure the file's size matches the size reported by the transaction
+        if (fileSize != expectedSize) {
+            throw new DataException(String.format("File size mismatch for transaction %s", Base58.encode(signature)));
+        }
+    }
+
+    private void addChunk(ArbitraryDataFileChunk chunk) {
+        this.chunks.add(chunk);
+    }
+
+    private void addChunkHashes(List<byte[]> chunkHashes) throws DataException {
+        if (chunkHashes == null || chunkHashes.isEmpty()) {
+            return;
+        }
+        for (byte[] chunkHash : chunkHashes) {
+            ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
+            this.addChunk(chunk);
+        }
+    }
+
+    public List<byte[]> getChunkHashes() {
+        List<byte[]> hashes = new ArrayList<>();
+        if (this.chunks == null || this.chunks.isEmpty()) {
+            return hashes;
+        }
+
+        for (ArbitraryDataFileChunk chunkData : this.chunks) {
+            hashes.add(chunkData.getHash());
+        }
+
+        return hashes;
+    }
+
+    public int split(int chunkSize) throws DataException {
+        try {
+
+            File file = this.getFile();
+            byte[] buffer = new byte[chunkSize];
+            this.chunks = new ArrayList<>();
+
+            if (file != null) {
+                try (FileInputStream fileInputStream = new FileInputStream(file);
+                     BufferedInputStream bis = new BufferedInputStream(fileInputStream)) {
+
+                    int numberOfBytes;
+                    while ((numberOfBytes = bis.read(buffer)) > 0) {
+                        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
+                            out.write(buffer, 0, numberOfBytes);
+                            out.flush();
+
+                            ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray(), this.signature);
+                            ValidationResult validationResult = chunk.isValid();
+                            if (validationResult == ValidationResult.OK) {
+                                this.chunks.add(chunk);
+                            } else {
+                                throw new DataException(String.format("Chunk %s is invalid", chunk));
+                            }
+                        }
+                    }
+                }
+            }
+        } catch (Exception e) {
+            throw new DataException("Unable to split file into chunks");
+        }
+
+        return this.chunks.size();
+    }
+
+    public boolean join() {
+        // Ensure we have chunks
+        if (this.chunks != null && this.chunks.size() > 0) {
+
+            // Create temporary path for joined file
+            // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
+            String baseDir = Settings.getInstance().getTempDataPath();
+            Path tempDir = Paths.get(baseDir, "join");
+            try {
+                Files.createDirectories(tempDir);
+            } catch (IOException e) {
+                return false;
+            }
+
+            // Join the chunks
+            Path outputPath = Paths.get(tempDir.toString(), this.chunks.get(0).digest58());
+            File outputFile = new File(outputPath.toString());
+            try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) {
+                for (ArbitraryDataFileChunk chunk : this.chunks) {
+                    File sourceFile = chunk.filePath.toFile();
+                    BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile));
+                    byte[] buffer = new byte[2048];
+                    int inSize;
+                    while ((inSize = in.read(buffer)) != -1) {
+                        out.write(buffer, 0, inSize);
+                    }
+                    in.close();
+                }
+                out.close();
+
+                // Copy temporary file to data directory
+                this.filePath = this.copyToDataDirectory(outputPath, this.signature);
+                if (FilesystemUtils.pathInsideDataOrTempPath(outputPath)) {
+                    Files.delete(outputPath);
+                }
+
+                return true;
+            } catch (FileNotFoundException e) {
+                return false;
+            } catch (IOException | DataException e) {
+                return false;
+            }
+        }
+        return false;
+    }
+
+    public boolean delete() {
+        // Delete the complete file
+        // ... but only if it's inside the Qortal data or temp directory
+        if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
+            if (Files.exists(this.filePath)) {
+                try {
+                    Files.delete(this.filePath);
+                    this.cleanupFilesystem();
+                    LOGGER.debug("Deleted file {}", this.filePath);
+                    return true;
+                } catch (IOException e) {
+                    LOGGER.warn("Couldn't delete file at path {}", this.filePath);
+                }
+            }
+        }
+        return false;
+    }
+
+    public boolean deleteAllChunks() {
+        boolean success = false;
+
+        // Delete the individual chunks
+        if (this.chunks != null && this.chunks.size() > 0) {
+            Iterator iterator = this.chunks.iterator();
+            while (iterator.hasNext()) {
+                ArbitraryDataFileChunk chunk = (ArbitraryDataFileChunk) iterator.next();
+                success = chunk.delete();
+                iterator.remove();
+            }
+        }
+        return success;
+    }
+
+    public boolean deleteMetadata() {
+        if (this.metadataFile != null && this.metadataFile.exists()) {
+            return this.metadataFile.delete();
+        }
+        return false;
+    }
+
+    public boolean deleteAll() {
+        // Delete the complete file
+        boolean fileDeleted = this.delete();
+
+        // Delete the metadata file
+        boolean metadataDeleted = this.deleteMetadata();
+
+        // Delete the individual chunks
+        boolean chunksDeleted = this.deleteAllChunks();
+
+        return fileDeleted || metadataDeleted || chunksDeleted;
+    }
+
+    protected void cleanupFilesystem() throws IOException {
+        // It is essential that use a separate path reference in this method
+        // as we don't want to modify this.filePath
+        Path path = this.filePath;
+        
+        FilesystemUtils.safeDeleteEmptyParentDirectories(path);
+    }
+
+    public byte[] getBytes() {
+        try {
+            return Files.readAllBytes(this.filePath);
+        } catch (IOException e) {
+            LOGGER.error("Unable to read bytes for file");
+            return null;
+        }
+    }
+
+
+    /* Helper methods */
+
+    private boolean isInBaseDirectory(Path filePath) {
+        Path path = filePath.toAbsolutePath();
+        String dataPath = Settings.getInstance().getDataPath();
+        String basePath = Paths.get(dataPath).toAbsolutePath().toString();
+        return path.startsWith(basePath);
+    }
+
+    public boolean exists() {
+        File file = this.filePath.toFile();
+        return file.exists();
+    }
+
+    public boolean chunkExists(byte[] hash) {
+        for (ArbitraryDataFileChunk chunk : this.chunks) {
+            if (Arrays.equals(hash, chunk.getHash())) {
+                return chunk.exists();
+            }
+        }
+        if (Arrays.equals(hash, this.metadataHash)) {
+            if (this.metadataFile != null) {
+                return this.metadataFile.exists();
+            }
+        }
+        if (Arrays.equals(this.getHash(), hash)) {
+            return this.exists();
+        }
+        return false;
+    }
+
+    public boolean allChunksExist() {
+        try {
+            if (this.metadataHash == null) {
+                // We don't have any metadata so can't check if we have the chunks
+                // Even if this transaction has no chunks, we don't have the file either (already checked above)
+                return false;
+            }
+
+            if (this.metadataFile == null) {
+                this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature);
+            }
+
+            // If the metadata file doesn't exist, we can't check if we have the chunks
+            if (!metadataFile.getFilePath().toFile().exists()) {
+                return false;
+            }
+
+            if (this.metadata == null) {
+                this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath()));
+            }
+
+            // Read the metadata
+            List<byte[]> chunks = metadata.getChunks();
+            for (byte[] chunkHash : chunks) {
+                ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
+                if (!chunk.exists()) {
+                    return false;
+                }
+            }
+
+            return true;
+
+        } catch (DataException e) {
+            // Something went wrong, so assume we don't have all the chunks
+            return false;
+        }
+    }
+
+    public boolean anyChunksExist() throws DataException {
+        try {
+            if (this.metadataHash == null) {
+                // We don't have any metadata so can't check if we have the chunks
+                // Even if this transaction has no chunks, we don't have the file either (already checked above)
+                return false;
+            }
+
+            if (this.metadataFile == null) {
+                this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature);
+            }
+
+            // If the metadata file doesn't exist, we can't check if we have any chunks
+            if (!metadataFile.getFilePath().toFile().exists()) {
+                return false;
+            }
+
+            if (this.metadata == null) {
+                this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath()));
+            }
+
+            // Read the metadata
+            List<byte[]> chunks = metadata.getChunks();
+            for (byte[] chunkHash : chunks) {
+                ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
+                if (chunk.exists()) {
+                    return true;
+                }
+            }
+
+            return false;
+
+        } catch (DataException e) {
+            // Something went wrong, so assume we don't have all the chunks
+            return false;
+        }
+    }
+
+    public boolean allFilesExist() {
+        if (this.exists()) {
+            return true;
+        }
+
+        // Complete file doesn't exist, so check the chunks
+        if (this.allChunksExist()) {
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * Retrieve a list of file hashes for this transaction that we do not hold locally
+     *
+     * @return a List of chunk hashes, or null if we are unable to determine what is missing
+     */
+    public List<byte[]> missingHashes() {
+        List<byte[]> missingHashes = new ArrayList<>();
+        try {
+            if (this.metadataHash == null) {
+                // We don't have any metadata so can't check if we have the chunks
+                // Even if this transaction has no chunks, we don't have the file either (already checked above)
+                return null;
+            }
+
+            if (this.metadataFile == null) {
+                this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature);
+            }
+
+            // If the metadata file doesn't exist, we can't check if we have the chunks
+            if (!metadataFile.getFilePath().toFile().exists()) {
+                return null;
+            }
+
+            if (this.metadata == null) {
+                this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath()));
+            }
+
+            // Read the metadata
+            List<byte[]> chunks = metadata.getChunks();
+            for (byte[] chunkHash : chunks) {
+                ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
+                if (!chunk.exists()) {
+                    missingHashes.add(chunkHash);
+                }
+            }
+
+            return missingHashes;
+
+        } catch (DataException e) {
+            // Something went wrong, so we can't make a sensible decision
+            return null;
+        }
+    }
+
+    public boolean containsChunk(byte[] hash) {
+        for (ArbitraryDataFileChunk chunk : this.chunks) {
+            if (Arrays.equals(hash, chunk.getHash())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    public long size() {
+        try {
+            return Files.size(this.filePath);
+        } catch (IOException e) {
+            return 0;
+        }
+    }
+
+    public int chunkCount() {
+        return this.chunks.size();
+    }
+
+    public List<ArbitraryDataFileChunk> getChunks() {
+        return this.chunks;
+    }
+
+    public byte[] chunkHashes() throws DataException {
+        if (this.chunks != null && this.chunks.size() > 0) {
+            // Return null if we only have one chunk, with the same hash as the parent
+            if (Arrays.equals(this.digest(), this.chunks.get(0).digest())) {
+                return null;
+            }
+
+            try {
+                ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+                for (ArbitraryDataFileChunk chunk : this.chunks) {
+                    byte[] chunkHash = chunk.digest();
+                    if (chunkHash.length != 32) {
+                        LOGGER.info("Invalid chunk hash length: {}", chunkHash.length);
+                        throw new DataException("Invalid chunk hash length");
+                    }
+                    outputStream.write(chunk.digest());
+                }
+                return outputStream.toByteArray();
+            } catch (IOException e) {
+                return null;
+            }
+        }
+        return null;
+    }
+
+    public List<byte[]> chunkHashList() {
+        List<byte[]> chunks = new ArrayList<>();
+
+        if (this.chunks != null && this.chunks.size() > 0) {
+            // Return null if we only have one chunk, with the same hash as the parent
+            if (Arrays.equals(this.digest(), this.chunks.get(0).digest())) {
+                return null;
+            }
+
+            try {
+                for (ArbitraryDataFileChunk chunk : this.chunks) {
+                    byte[] chunkHash = chunk.digest();
+                    if (chunkHash.length != 32) {
+                        LOGGER.info("Invalid chunk hash length: {}", chunkHash.length);
+                        throw new DataException("Invalid chunk hash length");
+                    }
+                    chunks.add(chunkHash);
+                }
+                return chunks;
+
+            } catch (DataException e) {
+                return null;
+            }
+        }
+        return null;
+    }
+
+    private void loadMetadata() throws DataException {
+        try {
+            this.metadata.read();
+
+        } catch (DataException | IOException e) {
+            throw new DataException(e);
+        }
+    }
+
+    private File getFile() {
+        File file = this.filePath.toFile();
+        if (file.exists()) {
+            return file;
+        }
+        return null;
+    }
+
+    public Path getFilePath() {
+        return this.filePath;
+    }
+
+    public byte[] digest() {
+        File file = this.getFile();
+        if (file != null && file.exists()) {
+            try {
+                return Crypto.digest(file);
+
+            } catch (IOException e) {
+                LOGGER.error("Couldn't compute digest for ArbitraryDataFile");
+            }
+        }
+        return null;
+    }
+
+    public String digest58() {
+        if (this.digest() != null) {
+            return Base58.encode(this.digest());
+        }
+        return null;
+    }
+
+    public String shortHash58() {
+        if (this.hash58 == null) {
+            return null;
+        }
+        return this.hash58.substring(0, Math.min(this.hash58.length(), SHORT_DIGEST_LENGTH));
+    }
+
+    public String getHash58() {
+        return this.hash58;
+    }
+
+    public byte[] getHash() {
+        return Base58.decode(this.hash58);
+    }
+
+    public String printChunks() {
+        String outputString = "";
+        if (this.chunkCount() > 0) {
+            for (ArbitraryDataFileChunk chunk : this.chunks) {
+                if (outputString.length() > 0) {
+                    outputString = outputString.concat(",");
+                }
+                outputString = outputString.concat(chunk.digest58());
+            }
+        }
+        return outputString;
+    }
+
+    public void setSecret(byte[] secret) {
+        this.secret = secret;
+    }
+
+    public byte[] getSecret() {
+        return this.secret;
+    }
+
+    public byte[] getSignature() {
+        return this.signature;
+    }
+
+    public void setMetadataFile(ArbitraryDataFile metadataFile) {
+        this.metadataFile = metadataFile;
+    }
+
+    public ArbitraryDataFile getMetadataFile() {
+        return this.metadataFile;
+    }
+
+    public void setMetadataHash(byte[] hash) throws DataException {
+        this.metadataHash = hash;
+
+        if (hash == null) {
+            return;
+        }
+        this.metadataFile = ArbitraryDataFile.fromHash(hash, this.signature);
+        if (metadataFile.exists()) {
+            this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath()));
+            this.addChunkHashes(this.metadata.getChunks());
+        }
+    }
+
+    public byte[] getMetadataHash() {
+        return this.metadataHash;
+    }
+
+    public void setMetadata(ArbitraryDataTransactionMetadata metadata) throws DataException {
+        this.metadata = metadata;
+        this.loadMetadata();
+    }
+
+    @Override
+    public String toString() {
+        return this.shortHash58();
+    }
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java

@@ -0,0 +1,54 @@
+package org.qortal.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.repository.DataException;
+import org.qortal.utils.Base58;
+
+import java.io.IOException;
+import java.nio.file.Files;
+
+
+public class ArbitraryDataFileChunk extends ArbitraryDataFile {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileChunk.class);
+
+    public ArbitraryDataFileChunk(String hash58, byte[] signature) throws DataException {
+        super(hash58, signature);
+    }
+
+    public ArbitraryDataFileChunk(byte[] fileContent, byte[] signature) throws DataException {
+        super(fileContent, signature);
+    }
+
+    public static ArbitraryDataFileChunk fromHash58(String hash58, byte[] signature) throws DataException {
+        return new ArbitraryDataFileChunk(hash58, signature);
+    }
+
+    public static ArbitraryDataFileChunk fromHash(byte[] hash, byte[] signature) throws DataException {
+        return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash), signature);
+    }
+
+    @Override
+    public ValidationResult isValid() {
+        // DataChunk validation applies here too
+        ValidationResult superclassValidationResult = super.isValid();
+        if (superclassValidationResult != ValidationResult.OK) {
+            return superclassValidationResult;
+        }
+
+        try {
+            // Validate the file size (chunks have stricter limits)
+            long fileSize = Files.size(this.filePath);
+            if (fileSize > CHUNK_SIZE) {
+                LOGGER.error(String.format("DataFileChunk is too large: %d bytes (max chunk size: %d bytes)", fileSize, CHUNK_SIZE));
+                return ValidationResult.FILE_TOO_LARGE;
+            }
+
+        } catch (IOException e) {
+            return ValidationResult.FILE_NOT_FOUND;
+        }
+
+        return ValidationResult.OK;
+    }
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java

@@ -0,0 +1,176 @@
+package org.qortal.arbitrary;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataDiff.*;
+import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
+import org.qortal.arbitrary.patch.UnifiedDiffPatch;
+import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
+import org.qortal.utils.FilesystemUtils;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.*;
+import java.util.List;
+import java.util.UUID;
+
+public class ArbitraryDataMerge {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMerge.class);
+
+    private final Path pathBefore;
+    private final Path pathAfter;
+    private Path mergePath;
+    private String identifier;
+    private ArbitraryDataMetadataPatch metadata;
+
+    public ArbitraryDataMerge(Path pathBefore, Path pathAfter) {
+        this.pathBefore = pathBefore;
+        this.pathAfter = pathAfter;
+    }
+
+    public void compute() throws IOException, DataException {
+        try {
+            this.preExecute();
+            this.copyPreviousStateToMergePath();
+            this.loadMetadata();
+            this.applyDifferences();
+            this.copyMetadata();
+
+        } finally {
+            this.postExecute();
+        }
+    }
+
+    private void preExecute() throws DataException {
+        this.createRandomIdentifier();
+        this.createOutputDirectory();
+    }
+
+    private void postExecute() {
+
+    }
+
+    private void createRandomIdentifier() {
+        this.identifier = UUID.randomUUID().toString();
+    }
+
+    private void createOutputDirectory() throws DataException {
+        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
+        String baseDir = Settings.getInstance().getTempDataPath();
+        Path tempDir = Paths.get(baseDir, "merge", this.identifier);
+        try {
+            Files.createDirectories(tempDir);
+        } catch (IOException e) {
+            throw new DataException("Unable to create temp directory");
+        }
+        this.mergePath = tempDir;
+    }
+
+    private void copyPreviousStateToMergePath() throws IOException {
+        ArbitraryDataMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get(""));
+    }
+
+    private void loadMetadata() throws IOException, DataException {
+        this.metadata = new ArbitraryDataMetadataPatch(this.pathAfter);
+        this.metadata.read();
+    }
+
+    private void applyDifferences() throws IOException, DataException {
+
+        List<Path> addedPaths = this.metadata.getAddedPaths();
+        for (Path path : addedPaths) {
+            LOGGER.trace("File was added: {}", path.toString());
+            Path filePath = Paths.get(this.pathAfter.toString(), path.toString());
+            ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path);
+        }
+
+        List<ModifiedPath> modifiedPaths = this.metadata.getModifiedPaths();
+        for (ModifiedPath modifiedPath : modifiedPaths) {
+            LOGGER.trace("File was modified: {}", modifiedPath.toString());
+            this.applyPatch(modifiedPath);
+        }
+
+        List<Path> removedPaths = this.metadata.getRemovedPaths();
+        for (Path path : removedPaths) {
+            LOGGER.trace("File was removed: {}", path.toString());
+            ArbitraryDataMerge.deletePathInBaseDir(this.mergePath, path);
+        }
+    }
+
+    private void applyPatch(ModifiedPath modifiedPath) throws IOException, DataException {
+        if (modifiedPath.getDiffType() == DiffType.UNIFIED_DIFF) {
+            // Create destination file from patch
+            UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(pathBefore, pathAfter, mergePath);
+            unifiedDiffPatch.apply(modifiedPath.getPath());
+        }
+        else if (modifiedPath.getDiffType() == DiffType.COMPLETE_FILE) {
+            // Copy complete file
+            Path filePath = Paths.get(this.pathAfter.toString(), modifiedPath.getPath().toString());
+            ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, modifiedPath.getPath());
+        }
+        else {
+            throw new DataException(String.format("Unrecognized patch diff type: %s", modifiedPath.getDiffType()));
+        }
+    }
+
+    private void copyMetadata() throws IOException {
+        Path filePath = Paths.get(this.pathAfter.toString(), ".qortal");
+        ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, Paths.get(".qortal"));
+    }
+
+
+    private static void copyPathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
+        if (!Files.exists(source)) {
+            throw new IOException(String.format("File not found: %s", source.toString()));
+        }
+
+        File sourceFile = source.toFile();
+        Path dest = Paths.get(base.toString(), relativePath.toString());
+        LOGGER.trace("Copying {} to {}", source, dest);
+
+        if (sourceFile.isFile()) {
+            Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING);
+        }
+        else if (sourceFile.isDirectory()) {
+            FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString());
+        }
+        else {
+            throw new IOException(String.format("Invalid file: %s", source.toString()));
+        }
+    }
+
+    private static void copyDirPathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
+        if (!Files.exists(source)) {
+            throw new IOException(String.format("File not found: %s", source.toString()));
+        }
+
+        Path dest = Paths.get(base.toString(), relativePath.toString());
+        LOGGER.trace("Copying {} to {}", source, dest);
+        FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString());
+    }
+
+    private static void deletePathInBaseDir(Path base, Path relativePath) throws IOException {
+        Path dest = Paths.get(base.toString(), relativePath.toString());
+        File file = new File(dest.toString());
+        if (file.exists() && file.isFile()) {
+            if (FilesystemUtils.pathInsideDataOrTempPath(dest)) {
+                LOGGER.trace("Deleting file {}", dest);
+                Files.delete(dest);
+            }
+        }
+        if (file.exists() && file.isDirectory()) {
+            if (FilesystemUtils.pathInsideDataOrTempPath(dest)) {
+                LOGGER.trace("Deleting directory {}", dest);
+                FileUtils.deleteDirectory(file);
+            }
+        }
+    }
+
+    public Path getMergePath() {
+        return this.mergePath;
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java

@@ -0,0 +1,558 @@
+package org.qortal.arbitrary;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
+import org.qortal.controller.arbitrary.ArbitraryDataManager;
+import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
+import org.qortal.crypto.AES;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.ArbitraryTransactionData.*;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.arbitrary.ArbitraryDataFile.*;
+import org.qortal.settings.Settings;
+import org.qortal.transform.Transformer;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.Base58;
+import org.qortal.utils.FilesystemUtils;
+import org.qortal.utils.ZipUtils;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+import java.io.File;
+import java.io.IOException;
+import java.io.InvalidObjectException;
+import java.nio.file.*;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+
+public class ArbitraryDataReader {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataReader.class);
+
+    private final String resourceId;
+    private final ResourceIdType resourceIdType;
+    private final Service service;
+    private final String identifier;
+    private ArbitraryTransactionData transactionData;
+    private String secret58;
+    private Path filePath;
+    private boolean canRequestMissingFiles;
+
+    // Intermediate paths
+    private final Path workingPath;
+    private final Path uncompressedPath;
+
+    // Stats (available for synchronous builds only)
+    private int layerCount;
+    private byte[] latestSignature;
+
+    public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
+        // Ensure names are always lowercase
+        if (resourceIdType == ResourceIdType.NAME) {
+            resourceId = resourceId.toLowerCase();
+        }
+
+        // If identifier is a blank string, or reserved keyword "default", treat it as null
+        if (identifier == null || identifier.equals("") || identifier.equals("default")) {
+            identifier = null;
+        }
+
+        this.resourceId = resourceId;
+        this.resourceIdType = resourceIdType;
+        this.service = service;
+        this.identifier = identifier;
+
+        this.workingPath = this.buildWorkingPath();
+        this.uncompressedPath = Paths.get(this.workingPath.toString(), "data");
+
+        // By default we can request missing files
+        // Callers can use setCanRequestMissingFiles(false) to prevent it
+        this.canRequestMissingFiles = true;
+    }
+
+    private Path buildWorkingPath() {
+        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
+        String baseDir = Settings.getInstance().getTempDataPath();
+        String identifier = this.identifier != null ?  this.identifier : "default";
+        return Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier);
+    }
+
+    public boolean isCachedDataAvailable() {
+        // If this resource is in the build queue then we shouldn't attempt to serve
+        // cached data, as it may not be fully built
+        if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(this.createQueueItem())) {
+            return false;
+        }
+
+        // Not in the build queue - so check the cache itself
+        ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false,
+                this.resourceId, this.resourceIdType, this.service, this.identifier);
+        if (cache.isCachedDataAvailable()) {
+            this.filePath = this.uncompressedPath;
+            return true;
+        }
+        return false;
+    }
+
+    public boolean isBuilding() {
+        return ArbitraryDataBuildManager.getInstance().isInBuildQueue(this.createQueueItem());
+    }
+
+    private ArbitraryDataBuildQueueItem createQueueItem() {
+        return new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service, this.identifier);
+    }
+
+    /**
+     * loadAsynchronously
+     *
+     * Attempts to load the resource asynchronously
+     * This adds the build task to a queue, and the result will be cached when complete
+     * To check the status of the build, periodically call isCachedDataAvailable()
+     * Once it returns true, you can then use getFilePath() to access the data itself.
+     *
+     * @param overwrite - set to true to force rebuild an existing cache
+     * @return true if added or already present in queue; false if not
+     */
+    public boolean loadAsynchronously(boolean overwrite) {
+        ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
+                this.resourceId, this.resourceIdType, this.service, this.identifier);
+        if (cache.isCachedDataAvailable()) {
+            // Use cached data
+            this.filePath = this.uncompressedPath;
+            return true;
+        }
+
+        return ArbitraryDataBuildManager.getInstance().addToBuildQueue(this.createQueueItem());
+    }
+
+    /**
+     * loadSynchronously
+     *
+     * Attempts to load the resource synchronously
+     * Warning: this can block for a long time when building or fetching complex data
+     * If no exception is thrown, you can then use getFilePath() to access the data immediately after returning
+     *
+     * @param overwrite - set to true to force rebuild an existing cache
+     * @throws IOException
+     * @throws DataException
+     * @throws MissingDataException
+     */
+    public void loadSynchronously(boolean overwrite) throws DataException, IOException, MissingDataException {
+        try {
+            ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
+                    this.resourceId, this.resourceIdType, this.service, this.identifier);
+            if (cache.isCachedDataAvailable()) {
+                // Use cached data
+                this.filePath = this.uncompressedPath;
+                return;
+            }
+
+            this.preExecute();
+            this.deleteExistingFiles();
+            this.fetch();
+            this.decrypt();
+            this.uncompress();
+            this.validate();
+
+        } catch (DataException e) {
+            this.deleteWorkingDirectory();
+            throw new DataException(e.getMessage());
+
+        } finally {
+            this.postExecute();
+        }
+    }
+
+    private void preExecute() throws DataException {
+        ArbitraryDataBuildManager.getInstance().setBuildInProgress(true);
+        this.checkEnabled();
+        this.createWorkingDirectory();
+        this.createUncompressedDirectory();
+    }
+
+    private void postExecute() {
+        ArbitraryDataBuildManager.getInstance().setBuildInProgress(false);
+    }
+
+    private void checkEnabled() throws DataException {
+        if (!Settings.getInstance().isQdnEnabled()) {
+            throw new DataException("QDN is disabled in settings");
+        }
+    }
+
+    private void createWorkingDirectory() throws DataException {
+        try {
+            Files.createDirectories(this.workingPath);
+        } catch (IOException e) {
+            throw new DataException("Unable to create temp directory");
+        }
+    }
+
+    /**
+     * Working directory should only be deleted on failure, since it is currently used to
+     * serve a cached version of the resource for subsequent requests.
+     * @throws IOException
+     */
+    private void deleteWorkingDirectory() throws IOException {
+        FilesystemUtils.safeDeleteDirectory(this.workingPath, true);
+    }
+
+    private void createUncompressedDirectory() throws DataException {
+        try {
+            // Create parent directory
+            Files.createDirectories(this.uncompressedPath.getParent());
+            // Ensure child directory doesn't already exist
+            FileUtils.deleteDirectory(this.uncompressedPath.toFile());
+
+        } catch (IOException e) {
+            throw new DataException("Unable to create uncompressed directory");
+        }
+    }
+
+    private void deleteExistingFiles() {
+        final Path uncompressedPath = this.uncompressedPath;
+        if (FilesystemUtils.pathInsideDataOrTempPath(uncompressedPath)) {
+            if (Files.exists(uncompressedPath)) {
+                LOGGER.trace("Attempting to delete path {}", this.uncompressedPath);
+                try {
+                    Files.walkFileTree(uncompressedPath, new SimpleFileVisitor<>() {
+
+                        @Override
+                        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                            Files.delete(file);
+                            return FileVisitResult.CONTINUE;
+                        }
+
+                        @Override
+                        public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
+                            // Don't delete the parent directory, as we want to leave an empty folder
+                            if (dir.compareTo(uncompressedPath) == 0) {
+                                return FileVisitResult.CONTINUE;
+                            }
+
+                            if (e == null) {
+                                Files.delete(dir);
+                                return FileVisitResult.CONTINUE;
+                            } else {
+                                throw e;
+                            }
+                        }
+
+                    });
+                } catch (IOException e) {
+                    LOGGER.debug("Unable to delete file or directory: {}", e.getMessage());
+                }
+            }
+        }
+    }
+
+    private void fetch() throws DataException, IOException, MissingDataException {
+        switch (resourceIdType) {
+
+            case FILE_HASH:
+                this.fetchFromFileHash();
+                break;
+
+            case NAME:
+                this.fetchFromName();
+                break;
+
+            case SIGNATURE:
+                this.fetchFromSignature();
+                break;
+
+            case TRANSACTION_DATA:
+                this.fetchFromTransactionData(this.transactionData);
+                break;
+
+            default:
+                throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType.toString()));
+        }
+    }
+
+    private void fetchFromFileHash() throws DataException {
+        // Load data file directly from the hash (without a signature)
+        ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId, null);
+        // Set filePath to the location of the ArbitraryDataFile
+        this.filePath = arbitraryDataFile.getFilePath();
+    }
+
+    private void fetchFromName() throws DataException, IOException, MissingDataException {
+        try {
+
+            // Build the existing state using past transactions
+            ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service, this.identifier);
+            builder.build();
+            Path builtPath = builder.getFinalPath();
+            if (builtPath == null) {
+                throw new DataException("Unable to build path");
+            }
+
+            // Update stats
+            this.layerCount = builder.getLayerCount();
+            this.latestSignature = builder.getLatestSignature();
+
+            // Set filePath to the builtPath
+            this.filePath = builtPath;
+
+        } catch (InvalidObjectException e) {
+            // Hash validation failed. Invalidate the cache for this name, so it can be rebuilt
+            LOGGER.info("Deleting {}", this.workingPath.toString());
+            FilesystemUtils.safeDeleteDirectory(this.workingPath, false);
+            throw(e);
+        }
+    }
+
+    private void fetchFromSignature() throws DataException, IOException, MissingDataException {
+
+        // Load the full transaction data from the database so we can access the file hashes
+        ArbitraryTransactionData transactionData;
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId));
+        }
+        if (transactionData == null) {
+            throw new DataException(String.format("Transaction data not found for signature %s", this.resourceId));
+        }
+
+        this.fetchFromTransactionData(transactionData);
+    }
+
+    private void fetchFromTransactionData(ArbitraryTransactionData transactionData) throws DataException, IOException, MissingDataException {
+        if (transactionData == null) {
+            throw new DataException(String.format("Transaction data not found for signature %s", this.resourceId));
+        }
+
+        // Load hashes
+        byte[] digest = transactionData.getData();
+        byte[] metadataHash = transactionData.getMetadataHash();
+        byte[] signature = transactionData.getSignature();
+
+        // Load secret
+        byte[] secret = transactionData.getSecret();
+        if (secret != null) {
+            this.secret58 = Base58.encode(secret);
+        }
+
+        // Load data file(s)
+        ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
+        ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
+        arbitraryDataFile.setMetadataHash(metadataHash);
+
+        if (!arbitraryDataFile.allFilesExist()) {
+            if (ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName())) {
+                throw new DataException(
+                        String.format("Unable to request missing data for file %s because the name is blocked", arbitraryDataFile));
+            }
+            else {
+                // Ask the arbitrary data manager to fetch data for this transaction
+                String message;
+                if (this.canRequestMissingFiles) {
+                    boolean requested = ArbitraryDataManager.getInstance().fetchData(transactionData);
+
+                    if (requested) {
+                        message = String.format("Requested missing data for file %s", arbitraryDataFile);
+                    } else {
+                        message = String.format("Unable to reissue request for missing file %s for signature %s due to rate limit. Please try again later.", arbitraryDataFile, Base58.encode(transactionData.getSignature()));
+                    }
+                }
+                else {
+                    message = String.format("Missing data for file %s", arbitraryDataFile);
+                }
+
+                // Throw a missing data exception, which allows subsequent layers to fetch data
+                LOGGER.trace(message);
+                throw new MissingDataException(message);
+            }
+        }
+
+        if (arbitraryDataFile.allChunksExist() && !arbitraryDataFile.exists()) {
+            // We have all the chunks but not the complete file, so join them
+            arbitraryDataFile.join();
+        }
+
+        // If the complete file still doesn't exist then something went wrong
+        if (!arbitraryDataFile.exists()) {
+            throw new IOException(String.format("File doesn't exist: %s", arbitraryDataFile));
+        }
+        // Ensure the complete hash matches the joined chunks
+        if (!Arrays.equals(arbitraryDataFile.digest(), digest)) {
+            // Delete the invalid file
+            arbitraryDataFile.delete();
+            throw new DataException("Unable to validate complete file hash");
+        }
+        // Ensure the file's size matches the size reported by the transaction (throws a DataException if not)
+        arbitraryDataFile.validateFileSize(transactionData.getSize());
+
+        // Set filePath to the location of the ArbitraryDataFile
+        this.filePath = arbitraryDataFile.getFilePath();
+    }
+
+    private void decrypt() throws DataException {
+        try {
+            // First try with explicit parameters (CBC mode with PKCS5 padding)
+            this.decryptUsingAlgo("AES/CBC/PKCS5Padding");
+
+        } catch (DataException e) {
+            // Something went wrong, so fall back to default AES params (necessary for legacy resource support)
+            this.decryptUsingAlgo("AES");
+
+            // TODO: delete files and block this resource if privateDataEnabled is false and the second attempt fails too
+        }
+    }
+
+    private void decryptUsingAlgo(String algorithm) throws DataException {
+        // Decrypt if we have the secret key.
+        byte[] secret = this.secret58 != null ? Base58.decode(this.secret58) : null;
+        if (secret != null && secret.length == Transformer.AES256_LENGTH) {
+            try {
+                Path unencryptedPath = Paths.get(this.workingPath.toString(), "zipped.zip");
+                SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, algorithm);
+                AES.decryptFile(algorithm, aesKey, this.filePath.toString(), unencryptedPath.toString());
+
+                // Replace filePath pointer with the encrypted file path
+                // Don't delete the original ArbitraryDataFile, as this is handled in the cleanup phase
+                this.filePath = unencryptedPath;
+
+            } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException
+                    | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) {
+                throw new DataException(String.format("Unable to decrypt file at path %s: %s", this.filePath, e.getMessage()));
+            }
+        } else {
+            // Assume it is unencrypted. This will be the case when we have built a custom path by combining
+            // multiple decrypted archives into a single state.
+        }
+    }
+
+    private void uncompress() throws IOException, DataException {
+        if (this.filePath == null || !Files.exists(this.filePath)) {
+            throw new DataException("Can't uncompress non-existent file path");
+        }
+        File file = new File(this.filePath.toString());
+        if (file.isDirectory()) {
+            // Already a directory - nothing to uncompress
+            // We still need to copy the directory to its final destination if it's not already there
+            this.moveFilePathToFinalDestination();
+            return;
+        }
+
+        try {
+            // Default to ZIP compression - this is needed for previews
+            Compression compression = transactionData != null ? transactionData.getCompression() : Compression.ZIP;
+
+            // Handle each type of compression
+            if (compression == Compression.ZIP) {
+                ZipUtils.unzip(this.filePath.toString(), this.uncompressedPath.getParent().toString());
+            }
+            else if (compression == Compression.NONE) {
+                Files.createDirectories(this.uncompressedPath);
+                Path finalPath = Paths.get(this.uncompressedPath.toString(), "data");
+                this.filePath.toFile().renameTo(finalPath.toFile());
+            }
+            else {
+                throw new DataException(String.format("Unrecognized compression type: %s", transactionData.getCompression()));
+            }
+        } catch (IOException e) {
+            throw new DataException(String.format("Unable to unzip file: %s", e.getMessage()));
+        }
+
+        // Replace filePath pointer with the uncompressed file path
+        if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
+            if (Files.exists(this.filePath)) {
+                Files.delete(this.filePath);
+            }
+        }
+        this.filePath = this.uncompressedPath;
+    }
+
+    private void validate() throws IOException, DataException {
+        if (this.service.isValidationRequired()) {
+            Service.ValidationResult result = this.service.validate(this.filePath);
+            if (result != Service.ValidationResult.OK) {
+                throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
+            }
+        }
+    }
+
+
+    private void moveFilePathToFinalDestination() throws IOException, DataException {
+        if (this.filePath.compareTo(this.uncompressedPath) != 0) {
+            File source = new File(this.filePath.toString());
+            File dest = new File(this.uncompressedPath.toString());
+            if (!source.exists()) {
+                throw new DataException("Source directory doesn't exist");
+            }
+            // Ensure destination directory doesn't exist
+            FileUtils.deleteDirectory(dest);
+            // Move files to destination
+            FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString());
+
+            try {
+                // Delete existing
+                if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
+                    File directory = new File(this.filePath.toString());
+                    FileUtils.deleteDirectory(directory);
+                }
+
+                // ... and its parent directory if empty
+                Path parentDirectory = this.filePath.getParent();
+                if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) {
+                    Files.deleteIfExists(parentDirectory);
+                }
+
+            } catch (DirectoryNotEmptyException e) {
+                    // No need to log anything
+            } catch (IOException e) {
+                // This will eventually be cleaned up by a maintenance process, so log the error and continue
+                LOGGER.debug("Unable to cleanup directories: {}", e.getMessage());
+            }
+
+            // Finally, update filePath to point to uncompressedPath
+            this.filePath = this.uncompressedPath;
+        }
+    }
+
+
+    public void setTransactionData(ArbitraryTransactionData transactionData) {
+        this.transactionData = transactionData;
+    }
+
+    public void setSecret58(String secret58) {
+        this.secret58 = secret58;
+    }
+
+    public Path getFilePath() {
+        return this.filePath;
+    }
+
+    public int getLayerCount() {
+        return this.layerCount;
+    }
+
+    public byte[] getLatestSignature() {
+        return this.latestSignature;
+    }
+
+    /**
+     * Use the below setter to ensure that we only read existing
+     * data without requesting any missing files,
+     *
+     * @param canRequestMissingFiles - whether or not fetching missing files is allowed
+     */
+    public void setCanRequestMissingFiles(boolean canRequestMissingFiles) {
+        this.canRequestMissingFiles = canRequestMissingFiles;
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java

@@ -0,0 +1,213 @@
+package org.qortal.arbitrary;
+
+import com.google.common.io.Resources;
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.api.HTMLParser;
+import org.qortal.arbitrary.ArbitraryDataFile.*;
+import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.controller.Controller;
+import org.qortal.settings.Settings;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ArbitraryDataRenderer {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataRenderer.class);
+
+    private final String resourceId;
+    private final ResourceIdType resourceIdType;
+    private final Service service;
+    private String inPath;
+    private final String secret58;
+    private final String prefix;
+    private final boolean usePrefix;
+    private final boolean async;
+    private final HttpServletRequest request;
+    private final HttpServletResponse response;
+    private final ServletContext context;
+
+    public ArbitraryDataRenderer(String resourceId, ResourceIdType resourceIdType, Service service, String inPath,
+                                 String secret58, String prefix, boolean usePrefix, boolean async,
+                                 HttpServletRequest request, HttpServletResponse response, ServletContext context) {
+
+        this.resourceId = resourceId;
+        this.resourceIdType = resourceIdType;
+        this.service = service;
+        this.inPath = inPath;
+        this.secret58 = secret58;
+        this.prefix = prefix;
+        this.usePrefix = usePrefix;
+        this.async = async;
+        this.request = request;
+        this.response = response;
+        this.context = context;
+    }
+
+    public HttpServletResponse render() {
+        if (!inPath.startsWith(File.separator)) {
+            inPath = File.separator + inPath;
+        }
+
+        // Don't render data if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return ArbitraryDataRenderer.getResponse(response, 500, "QDN is disabled in settings");
+        }
+
+        ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, null);
+        arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
+        try {
+            if (!arbitraryDataReader.isCachedDataAvailable()) {
+                // If async is requested, show a loading screen whilst build is in progress
+                if (async) {
+                    arbitraryDataReader.loadAsynchronously(false);
+                    return this.getLoadingResponse(service, resourceId);
+                }
+
+                // Otherwise, loop until we have data
+                int attempts = 0;
+                while (!Controller.isStopping()) {
+                    attempts++;
+                    if (!arbitraryDataReader.isBuilding()) {
+                        try {
+                            arbitraryDataReader.loadSynchronously(false);
+                            break;
+                        } catch (MissingDataException e) {
+                            if (attempts > 5) {
+                                // Give up after 5 attempts
+                                return ArbitraryDataRenderer.getResponse(response, 404, "Data unavailable. Please try again later.");
+                            }
+                        }
+                    }
+                    Thread.sleep(3000L);
+                }
+            }
+
+        } catch (Exception e) {
+            LOGGER.info(String.format("Unable to load %s %s: %s", service, resourceId, e.getMessage()));
+            return ArbitraryDataRenderer.getResponse(response, 500, "Error 500: Internal Server Error");
+        }
+
+        java.nio.file.Path path = arbitraryDataReader.getFilePath();
+        if (path == null) {
+            return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
+        }
+        String unzippedPath = path.toString();
+
+        try {
+            String filename = this.getFilename(unzippedPath, inPath);
+            String filePath = Paths.get(unzippedPath, filename).toString();
+
+            if (HTMLParser.isHtmlFile(filename)) {
+                // HTML file - needs to be parsed
+                byte[] data = Files.readAllBytes(Paths.get(filePath)); // TODO: limit file size that can be read into memory
+                HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data);
+                htmlParser.addAdditionalHeaderTags();
+                response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline'; media-src 'self' blob:");
+                response.setContentType(context.getMimeType(filename));
+                response.setContentLength(htmlParser.getData().length);
+                response.getOutputStream().write(htmlParser.getData());
+            }
+            else {
+                // Regular file - can be streamed directly
+                File file = new File(filePath);
+                FileInputStream inputStream = new FileInputStream(file);
+                response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline'; media-src 'self' blob:");
+                response.setContentType(context.getMimeType(filename));
+                int bytesRead, length = 0;
+                byte[] buffer = new byte[10240];
+                while ((bytesRead = inputStream.read(buffer)) != -1) {
+                    response.getOutputStream().write(buffer, 0, bytesRead);
+                    length += bytesRead;
+                }
+                response.setContentLength(length);
+                inputStream.close();
+            }
+            return response;
+        } catch (FileNotFoundException | NoSuchFileException e) {
+            LOGGER.info("Unable to serve file: {}", e.getMessage());
+            if (inPath.equals("/")) {
+                // Delete the unzipped folder if no index file was found
+                try {
+                    FileUtils.deleteDirectory(new File(unzippedPath));
+                } catch (IOException ioException) {
+                    LOGGER.debug("Unable to delete directory: {}", unzippedPath, e);
+                }
+            }
+        } catch (IOException e) {
+            LOGGER.info("Unable to serve file at path {}: {}", inPath, e.getMessage());
+        }
+
+        return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
+    }
+
+    private String getFilename(String directory, String userPath) {
+        if (userPath == null || userPath.endsWith("/") || userPath.equals("")) {
+            // Locate index file
+            List<String> indexFiles = ArbitraryDataRenderer.indexFiles();
+            for (String indexFile : indexFiles) {
+                Path path = Paths.get(directory, indexFile);
+                if (Files.exists(path)) {
+                    return userPath + indexFile;
+                }
+            }
+        }
+        return userPath;
+    }
+
+    private HttpServletResponse getLoadingResponse(Service service, String name) {
+        String responseString = "";
+        URL url = Resources.getResource("loading/index.html");
+        try {
+            responseString = Resources.toString(url, StandardCharsets.UTF_8);
+
+            // Replace vars
+            responseString = responseString.replace("%%SERVICE%%", service.toString());
+            responseString = responseString.replace("%%NAME%%", name);
+
+        } catch (IOException e) {
+            LOGGER.info("Unable to show loading screen: {}", e.getMessage());
+        }
+        return ArbitraryDataRenderer.getResponse(response, 503, responseString);
+    }
+
+    public static HttpServletResponse getResponse(HttpServletResponse response, int responseCode, String responseString) {
+        try {
+            byte[] responseData = responseString.getBytes();
+            response.setStatus(responseCode);
+            response.setContentLength(responseData.length);
+            response.getOutputStream().write(responseData);
+        } catch (IOException e) {
+            LOGGER.info("Error writing {} response", responseCode);
+        }
+        return response;
+    }
+
+    public static List<String> indexFiles() {
+        List<String> indexFiles = new ArrayList<>();
+        indexFiles.add("index.html");
+        indexFiles.add("index.htm");
+        indexFiles.add("default.html");
+        indexFiles.add("default.htm");
+        indexFiles.add("home.html");
+        indexFiles.add("home.htm");
+        return indexFiles;
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java

@@ -0,0 +1,352 @@
+package org.qortal.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
+import org.qortal.controller.arbitrary.ArbitraryDataManager;
+import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
+import org.qortal.data.arbitrary.ArbitraryResourceStatus;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.list.ResourceListManager;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.FilesystemUtils;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.qortal.data.arbitrary.ArbitraryResourceStatus.Status;
+
+public class ArbitraryDataResource {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataResource.class);
+
+    protected final String resourceId;
+    protected final ResourceIdType resourceIdType;
+    protected final Service service;
+    protected final String identifier;
+
+    private List<ArbitraryTransactionData> transactions;
+    private ArbitraryTransactionData latestPutTransaction;
+    private int layerCount;
+    private Integer localChunkCount = null;
+    private Integer totalChunkCount = null;
+
+    public ArbitraryDataResource(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
+        this.resourceId = resourceId.toLowerCase();
+        this.resourceIdType = resourceIdType;
+        this.service = service;
+
+        // If identifier is a blank string, or reserved keyword "default", treat it as null
+        if (identifier == null || identifier.equals("") || identifier.equals("default")) {
+            identifier = null;
+        }
+        this.identifier = identifier;
+    }
+
+    public ArbitraryResourceStatus getStatus(boolean quick) {
+        // Calculate the chunk counts
+        // Avoid this for "quick" statuses, to speed things up
+        if (!quick) {
+            this.calculateChunkCounts();
+        }
+
+        if (resourceIdType != ResourceIdType.NAME) {
+            // We only support statuses for resources with a name
+            return new ArbitraryResourceStatus(Status.UNSUPPORTED, this.localChunkCount, this.totalChunkCount);
+        }
+
+        // Check if the name is blocked
+        if (ResourceListManager.getInstance()
+                .listContains("blockedNames", this.resourceId, false)) {
+            return new ArbitraryResourceStatus(Status.BLOCKED, this.localChunkCount, this.totalChunkCount);
+        }
+
+        // Check if a build has failed
+        ArbitraryDataBuildQueueItem queueItem =
+                new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier);
+        if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) {
+            return new ArbitraryResourceStatus(Status.BUILD_FAILED, this.localChunkCount, this.totalChunkCount);
+        }
+
+        // Firstly check the cache to see if it's already built
+        ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(
+                resourceId, resourceIdType, service, identifier);
+        if (arbitraryDataReader.isCachedDataAvailable()) {
+            return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount);
+        }
+
+        // Check if we have all data locally for this resource
+        if (!this.allFilesDownloaded()) {
+            if (this.isDownloading()) {
+                return new ArbitraryResourceStatus(Status.DOWNLOADING, this.localChunkCount, this.totalChunkCount);
+            }
+            else if (this.isDataPotentiallyAvailable()) {
+                return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount);
+            }
+            return new ArbitraryResourceStatus(Status.MISSING_DATA, this.localChunkCount, this.totalChunkCount);
+        }
+
+        // Check if there's a build in progress
+        if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
+            return new ArbitraryResourceStatus(Status.BUILDING, this.localChunkCount, this.totalChunkCount);
+        }
+
+        // We have all data locally
+        return new ArbitraryResourceStatus(Status.DOWNLOADED, this.localChunkCount, this.totalChunkCount);
+    }
+
+    public boolean delete() {
+        try {
+            this.fetchTransactions();
+
+            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
+
+            for (ArbitraryTransactionData transactionData : transactionDataList) {
+                byte[] hash = transactionData.getData();
+                byte[] metadataHash = transactionData.getMetadataHash();
+                byte[] signature = transactionData.getSignature();
+                ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
+                arbitraryDataFile.setMetadataHash(metadataHash);
+
+                // Delete any chunks or complete files from each transaction
+                arbitraryDataFile.deleteAll();
+            }
+
+            // Also delete cached data for the entire resource
+            this.deleteCache();
+
+            // Invalidate the hosted transactions cache as we have removed an item
+            ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache();
+
+            return true;
+
+        } catch (DataException | IOException e) {
+            return false;
+        }
+    }
+
+    public void deleteCache() throws IOException {
+        // Don't delete anything if there's a build in progress
+        ArbitraryDataBuildQueueItem queueItem =
+                new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier);
+        if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
+            return;
+        }
+
+        String baseDir = Settings.getInstance().getTempDataPath();
+        String identifier = this.identifier != null ?  this.identifier : "default";
+        Path cachePath = Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier);
+        if (cachePath.toFile().exists()) {
+            boolean success = FilesystemUtils.safeDeleteDirectory(cachePath, true);
+            if (success) {
+                LOGGER.info("Cleared cache for resource {}", this.toString());
+            }
+        }
+    }
+
+    private boolean allFilesDownloaded() {
+        // Use chunk counts to speed things up if we can
+        if (this.localChunkCount != null && this.totalChunkCount != null &&
+                this.localChunkCount >= this.totalChunkCount) {
+            return true;
+        }
+
+        try {
+            this.fetchTransactions();
+
+            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
+
+            for (ArbitraryTransactionData transactionData : transactionDataList) {
+                if (!ArbitraryTransactionUtils.completeFileExists(transactionData) ||
+                    !ArbitraryTransactionUtils.allChunksExist(transactionData)) {
+                    return false;
+                }
+            }
+            return true;
+
+        } catch (DataException e) {
+            return false;
+        }
+    }
+
+    private void calculateChunkCounts() {
+        try {
+            this.fetchTransactions();
+
+            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
+            int localChunkCount = 0;
+            int totalChunkCount = 0;
+
+            for (ArbitraryTransactionData transactionData : transactionDataList) {
+                localChunkCount += ArbitraryTransactionUtils.ourChunkCount(transactionData);
+                totalChunkCount += ArbitraryTransactionUtils.totalChunkCount(transactionData);
+            }
+
+            this.localChunkCount = localChunkCount;
+            this.totalChunkCount = totalChunkCount;
+
+        } catch (DataException e) {}
+    }
+
+    private boolean isRateLimited() {
+        try {
+            this.fetchTransactions();
+
+            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
+
+            for (ArbitraryTransactionData transactionData : transactionDataList) {
+                if (ArbitraryDataManager.getInstance().isSignatureRateLimited(transactionData.getSignature())) {
+                    return true;
+                }
+            }
+            return true;
+
+        } catch (DataException e) {
+            return false;
+        }
+    }
+
+    /**
+     * Best guess as to whether data might be available
+     * This is only used to give an indication to the user of progress
+     * @return - whether data might be available on the network
+     */
+    private boolean isDataPotentiallyAvailable() {
+        try {
+            this.fetchTransactions();
+            Long now = NTP.getTime();
+            if (now == null) {
+                return false;
+            }
+
+            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
+
+            for (ArbitraryTransactionData transactionData : transactionDataList) {
+                long lastRequestTime = ArbitraryDataManager.getInstance().lastRequestForSignature(transactionData.getSignature());
+                // If we haven't requested yet, or requested in the last 30 seconds, there's still a
+                // chance that data is on its way but hasn't arrived yet
+                if (lastRequestTime == 0 || now - lastRequestTime < 30 * 1000L) {
+                    return true;
+                }
+            }
+            return false;
+
+        } catch (DataException e) {
+            return false;
+        }
+    }
+
+
+    /**
+     * Best guess as to whether we are currently downloading a resource
+     * This is only used to give an indication to the user of progress
+     * @return - whether we are trying to download the resource
+     */
+    private boolean isDownloading() {
+        try {
+            this.fetchTransactions();
+            Long now = NTP.getTime();
+            if (now == null) {
+                return false;
+            }
+
+            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
+
+            for (ArbitraryTransactionData transactionData : transactionDataList) {
+                long lastRequestTime = ArbitraryDataManager.getInstance().lastRequestForSignature(transactionData.getSignature());
+                // If were have requested data in the last 30 seconds, treat it as "downloading"
+                if (lastRequestTime > 0 && now - lastRequestTime < 30 * 1000L) {
+                    return true;
+                }
+            }
+
+            // FUTURE: we may want to check for file hashes (including the metadata file hash) in
+            // ArbitraryDataManager.arbitraryDataFileRequests and return true if one is found.
+
+            return false;
+
+        } catch (DataException e) {
+            return false;
+        }
+    }
+
+
+
+    private void fetchTransactions() throws DataException {
+        if (this.transactions != null && !this.transactions.isEmpty()) {
+            // Already fetched
+            return;
+        }
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Get the most recent PUT
+            ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
+                    .getLatestTransaction(this.resourceId, this.service, ArbitraryTransactionData.Method.PUT, this.identifier);
+            if (latestPut == null) {
+                String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s",
+                        this.resourceId, this.service, this.identifierString());
+                throw new DataException(message);
+            }
+            this.latestPutTransaction = latestPut;
+
+            // Load all transactions since the latest PUT
+            List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository()
+                    .getArbitraryTransactions(this.resourceId, this.service, this.identifier, latestPut.getTimestamp());
+
+            this.transactions = transactionDataList;
+            this.layerCount = transactionDataList.size();
+        }
+    }
+
+    private String resourceIdString() {
+        return resourceId != null ? resourceId : "";
+    }
+
+    private String resourceIdTypeString() {
+        return resourceIdType != null ? resourceIdType.toString() : "";
+    }
+
+    private String serviceString() {
+        return service != null ? service.toString() : "";
+    }
+
+    private String identifierString() {
+        return identifier != null ? identifier : "";
+    }
+
+    @Override
+    public String toString() {
+        return String.format("%s %s %s", this.serviceString(), this.resourceIdString(), this.identifierString());
+    }
+
+
+    /**
+     * @return unique key used to identify this resource
+     */
+    public String getUniqueKey() {
+        return String.format("%s-%s-%s", this.service, this.resourceId, this.identifier).toLowerCase();
+    }
+
+    public String getResourceId() {
+        return this.resourceId;
+    }
+
+    public Service getService() {
+        return this.service;
+    }
+
+    public String getIdentifier() {
+        return this.identifier;
+    }
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java

@@ -0,0 +1,285 @@
+package org.qortal.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
+import org.qortal.arbitrary.ArbitraryDataDiff.*;
+import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.block.BlockChain;
+import org.qortal.crypto.Crypto;
+import org.qortal.data.PaymentData;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.ArbitraryTransactionData.*;
+import org.qortal.data.transaction.BaseTransactionData;
+import org.qortal.group.Group;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.transaction.ArbitraryTransaction;
+import org.qortal.transaction.Transaction;
+import org.qortal.transform.Transformer;
+import org.qortal.utils.Base58;
+import org.qortal.utils.FilesystemUtils;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+public class ArbitraryDataTransactionBuilder {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataTransactionBuilder.class);
+
+    // Min transaction version required
+    private static final int MIN_TRANSACTION_VERSION = 5;
+
+    // Maximum number of PATCH layers allowed
+    private static final int MAX_LAYERS = 10;
+    // Maximum size difference (out of 1) allowed for PATCH transactions
+    private static final double MAX_SIZE_DIFF = 0.2f;
+    // Maximum proportion of files modified relative to total
+    private static final double MAX_FILE_DIFF = 0.5f;
+
+    private final String publicKey58;
+    private final Path path;
+    private final String name;
+    private Method method;
+    private final Service service;
+    private final String identifier;
+    private final Repository repository;
+
+    private int chunkSize = ArbitraryDataFile.CHUNK_SIZE;
+
+    private ArbitraryTransactionData arbitraryTransactionData;
+    private ArbitraryDataFile arbitraryDataFile;
+
+    public ArbitraryDataTransactionBuilder(Repository repository, String publicKey58, Path path, String name,
+                                           Method method, Service service, String identifier) {
+        this.repository = repository;
+        this.publicKey58 = publicKey58;
+        this.path = path;
+        this.name = name;
+        this.method = method;
+        this.service = service;
+
+        // If identifier is a blank string, or reserved keyword "default", treat it as null
+        if (identifier == null || identifier.equals("") || identifier.equals("default")) {
+            identifier = null;
+        }
+        this.identifier = identifier;
+    }
+
+    public void build() throws DataException {
+        try {
+            this.preExecute();
+            this.checkMethod();
+            this.createTransaction();
+        }
+        finally {
+            this.postExecute();
+        }
+    }
+
+    private void preExecute() {
+
+    }
+
+    private void postExecute() {
+
+    }
+
+    private void checkMethod() throws DataException {
+        if (this.method == null) {
+            // We need to automatically determine the method
+            this.method = this.determineMethodAutomatically();
+        }
+    }
+
+    private Method determineMethodAutomatically() throws DataException {
+        ArbitraryDataReader reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier);
+        try {
+            reader.loadSynchronously(true);
+        } catch (Exception e) {
+            // Catch all exceptions if the existing resource cannot be loaded first time
+            // In these cases it's simplest to just use a PUT transaction
+            return Method.PUT;
+        }
+
+        try {
+            // Check layer count
+            int layerCount = reader.getLayerCount();
+            if (layerCount >= MAX_LAYERS) {
+                LOGGER.info("Reached maximum layer count ({} / {}) - using PUT", layerCount, MAX_LAYERS);
+                return Method.PUT;
+            }
+
+            // Check size of differences between this layer and previous layer
+            ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(reader.getFilePath(), this.path, reader.getLatestSignature());
+            patch.create();
+            long diffSize = FilesystemUtils.getDirectorySize(patch.getFinalPath());
+            long existingStateSize = FilesystemUtils.getDirectorySize(reader.getFilePath());
+            double difference = (double) diffSize / (double) existingStateSize;
+            if (difference > MAX_SIZE_DIFF) {
+                LOGGER.info("Reached maximum difference ({} / {}) - using PUT", difference, MAX_SIZE_DIFF);
+                return Method.PUT;
+            }
+
+            // Check number of modified files
+            ArbitraryDataMetadataPatch metadata = patch.getMetadata();
+            int totalFileCount = patch.getTotalFileCount();
+            int differencesCount = metadata.getFileDifferencesCount();
+            difference = (double) differencesCount / (double) totalFileCount;
+            if (difference > MAX_FILE_DIFF) {
+                LOGGER.info("Reached maximum file differences ({} / {}) - using PUT", difference, MAX_FILE_DIFF);
+                return Method.PUT;
+            }
+
+            // Check the patch types
+            // Limit this check to single file resources only for now
+            boolean atLeastOnePatch = false;
+            if (totalFileCount == 1) {
+                for (ModifiedPath path : metadata.getModifiedPaths()) {
+                    if (path.getDiffType() != DiffType.COMPLETE_FILE) {
+                        atLeastOnePatch = true;
+                    }
+                }
+            }
+            if (!atLeastOnePatch) {
+                LOGGER.info("Patch consists of complete files only - using PUT");
+                return Method.PUT;
+            }
+
+            // State is appropriate for a PATCH transaction
+            return Method.PATCH;
+        }
+        catch (IOException | DataException e) {
+            // Handle matching states separately, as it's best to block transactions with duplicate states
+            if (e.getMessage().equals("Current state matches previous state. Nothing to do.")) {
+                throw new DataException(e.getMessage());
+            }
+            LOGGER.info("Caught exception: {}", e.getMessage());
+            LOGGER.info("Unable to load existing resource - using PUT to overwrite it.");
+            return Method.PUT;
+        }
+    }
+
+    private void createTransaction() throws DataException {
+        arbitraryDataFile = null;
+        try {
+            Long now = NTP.getTime();
+            if (now == null) {
+                throw new DataException("NTP time not synced yet");
+            }
+
+            // Ensure that this chain supports transactions necessary for complex arbitrary data
+            int transactionVersion = Transaction.getVersionByTimestamp(now);
+            if (transactionVersion < MIN_TRANSACTION_VERSION) {
+                throw new DataException("Transaction version unsupported on this blockchain.");
+            }
+
+            if (publicKey58 == null || path == null) {
+                throw new DataException("Missing public key or path");
+            }
+            byte[] creatorPublicKey = Base58.decode(publicKey58);
+            final String creatorAddress = Crypto.toAddress(creatorPublicKey);
+            byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress);
+            if (lastReference == null) {
+                // Use a random last reference on the very first transaction for an account
+                // Code copied from CrossChainResource.buildAtMessage()
+                // We already require PoW on all arbitrary transactions, so no additional logic is needed
+                Random random = new Random();
+                lastReference = new byte[Transformer.SIGNATURE_LENGTH];
+                random.nextBytes(lastReference);
+            }
+
+            Compression compression = Compression.ZIP;
+
+            // FUTURE? Use zip compression for directories, or no compression for single files
+            // Compression compression = (path.toFile().isDirectory()) ? Compression.ZIP : Compression.NONE;
+
+            ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, compression);
+            try {
+                arbitraryDataWriter.setChunkSize(this.chunkSize);
+                arbitraryDataWriter.save();
+            } catch (IOException | DataException | InterruptedException | RuntimeException | MissingDataException e) {
+                LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage());
+                throw new DataException(e.getMessage());
+            }
+
+            // Get main file
+            arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile();
+            if (arbitraryDataFile == null) {
+                throw new DataException("Arbitrary data file is null");
+            }
+
+            // Get chunks metadata file
+            ArbitraryDataFile metadataFile = arbitraryDataFile.getMetadataFile();
+            if (metadataFile == null && arbitraryDataFile.chunkCount() > 1) {
+                throw new DataException(String.format("Chunks metadata data file is null but there are %d chunks", arbitraryDataFile.chunkCount()));
+            }
+
+            String digest58 = arbitraryDataFile.digest58();
+            if (digest58 == null) {
+                LOGGER.error("Unable to calculate file digest");
+                throw new DataException("Unable to calculate file digest");
+            }
+
+            final BaseTransactionData baseTransactionData = new BaseTransactionData(now, Group.NO_GROUP,
+                    lastReference, creatorPublicKey, 0L, null);
+            final int size = (int) arbitraryDataFile.size();
+            final int version = 5;
+            final int nonce = 0;
+            byte[] secret = arbitraryDataFile.getSecret();
+            final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
+            final byte[] digest = arbitraryDataFile.digest();
+            final byte[] metadataHash = (metadataFile != null) ? metadataFile.getHash() : null;
+            final List<PaymentData> payments = new ArrayList<>();
+
+            ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
+                    version, service, nonce, size, name, identifier, method,
+                    secret, compression, digest, dataType, metadataHash, payments);
+
+            this.arbitraryTransactionData = transactionData;
+
+        } catch (DataException e) {
+            if (arbitraryDataFile != null) {
+                arbitraryDataFile.deleteAll();
+            }
+            throw(e);
+        }
+
+    }
+
+    public void computeNonce() throws DataException {
+        if (this.arbitraryTransactionData == null) {
+            throw new DataException("Arbitrary transaction data is required to compute nonce");
+        }
+
+        ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, this.arbitraryTransactionData);
+        LOGGER.info("Computing nonce...");
+        transaction.computeNonce();
+
+        Transaction.ValidationResult result = transaction.isValidUnconfirmed();
+        if (result != Transaction.ValidationResult.OK) {
+            arbitraryDataFile.deleteAll();
+            throw new DataException(String.format("Arbitrary transaction invalid: %s", result));
+        }
+        LOGGER.info("Transaction is valid");
+    }
+
+    public ArbitraryTransactionData getArbitraryTransactionData() {
+        return this.arbitraryTransactionData;
+    }
+
+    public ArbitraryDataFile getArbitraryDataFile() {
+        return this.arbitraryDataFile;
+    }
+
+    public void setChunkSize(int chunkSize) {
+        this.chunkSize = chunkSize;
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java

@@ -0,0 +1,342 @@
+package org.qortal.arbitrary;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.crypto.Crypto;
+import org.qortal.data.transaction.ArbitraryTransactionData.*;
+import org.qortal.crypto.AES;
+import org.qortal.repository.DataException;
+import org.qortal.arbitrary.ArbitraryDataFile.*;
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+import org.qortal.utils.FilesystemUtils;
+import org.qortal.utils.ZipUtils;
+
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+
+public class ArbitraryDataWriter {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataWriter.class);
+
+    private Path filePath;
+    private final String name;
+    private final Service service;
+    private final String identifier;
+    private final Method method;
+    private final Compression compression;
+
+    private int chunkSize = ArbitraryDataFile.CHUNK_SIZE;
+
+    private SecretKey aesKey;
+    private ArbitraryDataFile arbitraryDataFile;
+
+    // Intermediate paths to cleanup
+    private Path workingPath;
+    private Path compressedPath;
+    private Path encryptedPath;
+
+    public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression) {
+        this.filePath = filePath;
+        this.name = name;
+        this.service = service;
+        this.method = method;
+        this.compression = compression;
+
+        // If identifier is a blank string, or reserved keyword "default", treat it as null
+        if (identifier == null || identifier.equals("") || identifier.equals("default")) {
+            identifier = null;
+        }
+        this.identifier = identifier;
+    }
+
+    public void save() throws IOException, DataException, InterruptedException, MissingDataException {
+        try {
+            this.preExecute();
+            this.validateService();
+            this.process();
+            this.compress();
+            this.encrypt();
+            this.split();
+            this.createMetadataFile();
+            this.validate();
+
+        } finally {
+            this.postExecute();
+        }
+    }
+
+    private void preExecute() throws DataException {
+        this.checkEnabled();
+
+        // Enforce compression when uploading a directory
+        File file = new File(this.filePath.toString());
+        if (file.isDirectory() && compression == Compression.NONE) {
+            throw new DataException("Unable to upload a directory without compression");
+        }
+
+        // Create temporary working directory
+        this.createWorkingDirectory();
+    }
+
+    private void postExecute() throws IOException {
+        this.cleanupFilesystem();
+    }
+
+    private void checkEnabled() throws DataException {
+        if (!Settings.getInstance().isQdnEnabled()) {
+            throw new DataException("QDN is disabled in settings");
+        }
+    }
+
+    private void createWorkingDirectory() throws DataException {
+        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
+        String baseDir = Settings.getInstance().getTempDataPath();
+        String identifier = Base58.encode(Crypto.digest(this.filePath.toString().getBytes()));
+        Path tempDir = Paths.get(baseDir, "writer", identifier);
+        try {
+            Files.createDirectories(tempDir);
+        } catch (IOException e) {
+            throw new DataException("Unable to create temp directory");
+        }
+        this.workingPath = tempDir;
+    }
+
+    private void validateService() throws IOException, DataException {
+        if (this.service.isValidationRequired()) {
+            Service.ValidationResult result = this.service.validate(this.filePath);
+            if (result != Service.ValidationResult.OK) {
+                throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
+            }
+        }
+    }
+
+    private void process() throws DataException, IOException, MissingDataException {
+        switch (this.method) {
+
+            case PUT:
+                // Nothing to do
+                break;
+
+            case PATCH:
+                this.processPatch();
+                break;
+
+            default:
+                throw new DataException(String.format("Unknown method specified: %s", method.toString()));
+        }
+    }
+
+    private void processPatch() throws DataException, IOException, MissingDataException {
+
+        // Build the existing state using past transactions
+        ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service, this.identifier);
+        builder.build();
+        Path builtPath = builder.getFinalPath();
+
+        // Obtain the latest signature, so this can be included in the patch
+        byte[] latestSignature = builder.getLatestSignature();
+
+        // Compute a diff of the latest changes on top of the previous state
+        // Then use only the differences as our data payload
+        ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath, latestSignature);
+        patch.create();
+        this.filePath = patch.getFinalPath();
+
+        // Delete the input directory
+        if (FilesystemUtils.pathInsideDataOrTempPath(builtPath)) {
+            File directory = new File(builtPath.toString());
+            FileUtils.deleteDirectory(directory);
+        }
+
+        // Validate the patch
+        this.validatePatch();
+    }
+
+    private void validatePatch() throws DataException {
+        if (this.filePath == null) {
+            throw new DataException("Null path after creating patch");
+        }
+
+        File qortalMetadataDirectoryFile = Paths.get(this.filePath.toString(), ".qortal").toFile();
+        if (!qortalMetadataDirectoryFile.exists()) {
+            throw new DataException("Qortal metadata folder doesn't exist in patch");
+        }
+        if (!qortalMetadataDirectoryFile.isDirectory()) {
+            throw new DataException("Qortal metadata folder isn't a directory");
+        }
+
+        File qortalPatchMetadataFile = Paths.get(this.filePath.toString(), ".qortal", "patch").toFile();
+        if (!qortalPatchMetadataFile.exists()) {
+            throw new DataException("Qortal patch metadata file doesn't exist in patch");
+        }
+        if (!qortalPatchMetadataFile.isFile()) {
+            throw new DataException("Qortal patch metadata file isn't a file");
+        }
+    }
+
+    private void compress() throws InterruptedException, DataException {
+        // Compress the data if requested
+        if (this.compression != Compression.NONE) {
+            this.compressedPath = Paths.get(this.workingPath.toString(), "data.zip");
+            try {
+
+                if (this.compression == Compression.ZIP) {
+                    LOGGER.info("Compressing...");
+                    String enclosingFolderName = "data";
+                    ZipUtils.zip(this.filePath.toString(), this.compressedPath.toString(), enclosingFolderName);
+                }
+                else {
+                    throw new DataException(String.format("Unknown compression type specified: %s", compression.toString()));
+                }
+                // FUTURE: other compression types
+
+                // Delete the input directory
+                if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
+                    File directory = new File(this.filePath.toString());
+                    FileUtils.deleteDirectory(directory);
+                }
+                // Replace filePath pointer with the zipped file path
+                this.filePath = this.compressedPath;
+
+            } catch (IOException | DataException e) {
+                throw new DataException("Unable to zip directory", e);
+            }
+        }
+    }
+
+    private void encrypt() throws DataException {
+        this.encryptedPath = Paths.get(this.workingPath.toString(), "data.zip.encrypted");
+        try {
+            // Encrypt the file with AES
+            LOGGER.info("Encrypting...");
+            this.aesKey = AES.generateKey(256);
+            AES.encryptFile("AES/CBC/PKCS5Padding", this.aesKey, this.filePath.toString(), this.encryptedPath.toString());
+
+            // Delete the input file
+            if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
+                Files.delete(this.filePath);
+            }
+            // Replace filePath pointer with the encrypted file path
+            this.filePath = this.encryptedPath;
+
+        } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException
+                | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) {
+            throw new DataException(String.format("Unable to encrypt file %s: %s", this.filePath, e.getMessage()));
+        }
+    }
+
+    private void split() throws IOException, DataException {
+        // We don't have a signature yet, so use null to put the file in a generic folder
+        this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath, null);
+        if (this.arbitraryDataFile == null) {
+            throw new IOException("No file available when trying to split");
+        }
+
+        int chunkCount = this.arbitraryDataFile.split(this.chunkSize);
+        if (chunkCount > 0) {
+            LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s")));
+        }
+        else {
+            throw new DataException("Unable to split file into chunks");
+        }
+    }
+
+    private void createMetadataFile() throws IOException, DataException {
+        // If we have at least one chunk, we need to create an index file containing their hashes
+        if (this.arbitraryDataFile.chunkCount() > 1) {
+            // Create the JSON file
+            Path chunkFilePath = Paths.get(this.workingPath.toString(), "metadata.json");
+            ArbitraryDataTransactionMetadata chunkMetadata = new ArbitraryDataTransactionMetadata(chunkFilePath);
+            chunkMetadata.setChunks(this.arbitraryDataFile.chunkHashList());
+            chunkMetadata.write();
+
+            // Create an ArbitraryDataFile from the JSON file (we don't have a signature yet)
+            ArbitraryDataFile metadataFile = ArbitraryDataFile.fromPath(chunkFilePath, null);
+            this.arbitraryDataFile.setMetadataFile(metadataFile);
+        }
+    }
+
+    private void validate() throws IOException, DataException {
+        if (this.arbitraryDataFile == null) {
+            throw new DataException("No file available when validating");
+        }
+        this.arbitraryDataFile.setSecret(this.aesKey.getEncoded());
+
+        // Validate the file
+        ValidationResult validationResult = this.arbitraryDataFile.isValid();
+        if (validationResult != ValidationResult.OK) {
+            throw new DataException(String.format("File %s failed validation: %s", this.arbitraryDataFile, validationResult));
+        }
+        LOGGER.info("Whole file hash is valid: {}", this.arbitraryDataFile.digest58());
+
+        // Validate each chunk
+        for (ArbitraryDataFileChunk chunk : this.arbitraryDataFile.getChunks()) {
+            validationResult = chunk.isValid();
+            if (validationResult != ValidationResult.OK) {
+                throw new DataException(String.format("Chunk %s failed validation: %s", chunk, validationResult));
+            }
+        }
+        LOGGER.info("Chunk hashes are valid");
+
+        // Validate chunks metadata file
+        if (this.arbitraryDataFile.chunkCount() > 1) {
+            ArbitraryDataFile metadataFile = this.arbitraryDataFile.getMetadataFile();
+            if (metadataFile == null || !metadataFile.exists()) {
+                throw new DataException("No metadata file available, but there are multiple chunks");
+            }
+            // Read the file
+            ArbitraryDataTransactionMetadata metadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
+            metadata.read();
+            // Check all chunks exist
+            for (byte[] chunk : this.arbitraryDataFile.chunkHashList()) {
+                if (!metadata.containsChunk(chunk)) {
+                    throw new DataException(String.format("Missing chunk %s in metadata file", Base58.encode(chunk)));
+                }
+            }
+        }
+    }
+
+    private void cleanupFilesystem() throws IOException {
+        // Clean up
+        if (FilesystemUtils.pathInsideDataOrTempPath(this.compressedPath)) {
+            File zippedFile = new File(this.compressedPath.toString());
+            if (zippedFile.exists()) {
+                zippedFile.delete();
+            }
+        }
+        if (FilesystemUtils.pathInsideDataOrTempPath(this.encryptedPath)) {
+            File encryptedFile = new File(this.encryptedPath.toString());
+            if (encryptedFile.exists()) {
+                encryptedFile.delete();
+            }
+        }
+        if (FilesystemUtils.pathInsideDataOrTempPath(this.workingPath)) {
+            FileUtils.deleteDirectory(new File(this.workingPath.toString()));
+        }
+    }
+
+
+    public ArbitraryDataFile getArbitraryDataFile() {
+        return this.arbitraryDataFile;
+    }
+
+    public void setChunkSize(int chunkSize) {
+        this.chunkSize = chunkSize;
+    }
+
+}

src/main/java/org/qortal/arbitrary/exception/MissingDataException.java

@@ -0,0 +1,20 @@
+package org.qortal.arbitrary.exception;
+
+public class MissingDataException extends Exception {
+
+	public MissingDataException() {
+	}
+
+	public MissingDataException(String message) {
+		super(message);
+	}
+
+	public MissingDataException(String message, Throwable cause) {
+		super(message, cause);
+	}
+
+	public MissingDataException(Throwable cause) {
+		super(cause);
+	}
+
+}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java

@@ -0,0 +1,85 @@
+package org.qortal.arbitrary.metadata;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.repository.DataException;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * ArbitraryDataMetadata
+ *
+ * This is a base class to handle reading and writing JSON to the supplied filePath.
+ *
+ * It is not usable on its own; it must be subclassed, with two methods overridden:
+ *
+ * readJson() - code to unserialize the JSON file
+ * buildJson() - code to serialize the JSON file
+ *
+ */
+public class ArbitraryDataMetadata {
+
+    protected static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadata.class);
+
+    protected Path filePath;
+
+    protected String jsonString;
+
+    public ArbitraryDataMetadata(Path filePath) {
+        this.filePath = filePath;
+    }
+
+    protected void readJson() throws DataException {
+        // To be overridden
+    }
+
+    protected void buildJson() {
+        // To be overridden
+    }
+
+
+    public void read() throws IOException, DataException {
+        this.loadJson();
+        this.readJson();
+    }
+
+    public void write() throws IOException, DataException {
+        this.buildJson();
+        this.createParentDirectories();
+
+        BufferedWriter writer = new BufferedWriter(new FileWriter(this.filePath.toString()));
+        writer.write(this.jsonString);
+        writer.newLine();
+        writer.close();
+    }
+
+
+    protected void loadJson() throws IOException {
+        File metadataFile = new File(this.filePath.toString());
+        if (!metadataFile.exists()) {
+            throw new IOException(String.format("Metadata file doesn't exist: %s", this.filePath.toString()));
+        }
+
+        this.jsonString = new String(Files.readAllBytes(this.filePath));
+    }
+
+
+    protected void createParentDirectories() throws DataException {
+        try {
+            Files.createDirectories(this.filePath.getParent());
+        } catch (IOException e) {
+            throw new DataException("Unable to create parent directories");
+        }
+    }
+
+
+    public String getJsonString() {
+        return this.jsonString;
+    }
+
+}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java

@@ -0,0 +1,69 @@
+package org.qortal.arbitrary.metadata;
+
+import org.json.JSONObject;
+import org.qortal.repository.DataException;
+import org.qortal.utils.Base58;
+
+import java.nio.file.Path;
+
+public class ArbitraryDataMetadataCache extends ArbitraryDataQortalMetadata {
+
+    private byte[] signature;
+    private long timestamp;
+
+    public ArbitraryDataMetadataCache(Path filePath) {
+        super(filePath);
+
+    }
+
+    @Override
+    protected String fileName() {
+        return "cache";
+    }
+
+    @Override
+    protected void readJson() throws DataException {
+        if (this.jsonString == null) {
+            throw new DataException("Patch JSON string is null");
+        }
+
+        JSONObject cache = new JSONObject(this.jsonString);
+        if (cache.has("signature")) {
+            String sig = cache.getString("signature");
+            if (sig != null) {
+                this.signature = Base58.decode(sig);
+            }
+        }
+        if (cache.has("timestamp")) {
+            this.timestamp = cache.getLong("timestamp");
+        }
+    }
+
+    @Override
+    protected void buildJson() {
+        JSONObject patch = new JSONObject();
+        patch.put("signature", Base58.encode(this.signature));
+        patch.put("timestamp", this.timestamp);
+
+        this.jsonString = patch.toString(2);
+        LOGGER.trace("Cache metadata: {}", this.jsonString);
+    }
+
+
+    public void setSignature(byte[] signature) {
+        this.signature = signature;
+    }
+
+    public byte[] getSignature() {
+        return this.signature;
+    }
+
+    public void setTimestamp(long timestamp) {
+        this.timestamp = timestamp;
+    }
+
+    public long getTimestamp() {
+        return this.timestamp;
+    }
+
+}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java

@@ -0,0 +1,182 @@
+package org.qortal.arbitrary.metadata;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.qortal.arbitrary.ArbitraryDataDiff.*;
+import org.qortal.repository.DataException;
+import org.qortal.utils.Base58;
+
+import java.lang.reflect.Field;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+public class ArbitraryDataMetadataPatch extends ArbitraryDataQortalMetadata {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadataPatch.class);
+
+    private List<Path> addedPaths;
+    private List<ModifiedPath> modifiedPaths;
+    private List<Path> removedPaths;
+    private byte[] previousSignature;
+    private byte[] previousHash;
+    private byte[] currentHash;
+
+    public ArbitraryDataMetadataPatch(Path filePath) {
+        super(filePath);
+
+        this.addedPaths = new ArrayList<>();
+        this.modifiedPaths = new ArrayList<>();
+        this.removedPaths = new ArrayList<>();
+    }
+
+    @Override
+    protected String fileName() {
+        return "patch";
+    }
+
+    @Override
+    protected void readJson() throws DataException {
+        if (this.jsonString == null) {
+            throw new DataException("Patch JSON string is null");
+        }
+
+        JSONObject patch = new JSONObject(this.jsonString);
+        if (patch.has("prevSig")) {
+            String prevSig = patch.getString("prevSig");
+            if (prevSig != null) {
+                this.previousSignature = Base58.decode(prevSig);
+            }
+        }
+        if (patch.has("prevHash")) {
+            String prevHash = patch.getString("prevHash");
+            if (prevHash != null) {
+                this.previousHash = Base58.decode(prevHash);
+            }
+        }
+        if (patch.has("curHash")) {
+            String curHash = patch.getString("curHash");
+            if (curHash != null) {
+                this.currentHash = Base58.decode(curHash);
+            }
+        }
+        if (patch.has("added")) {
+            JSONArray added = (JSONArray) patch.get("added");
+            if (added != null) {
+                for (int i=0; i<added.length(); i++) {
+                    String pathString = added.getString(i);
+                    this.addedPaths.add(Paths.get(pathString));
+                }
+            }
+        }
+        if (patch.has("modified")) {
+            JSONArray modified = (JSONArray) patch.get("modified");
+            if (modified != null) {
+                for (int i=0; i<modified.length(); i++) {
+                    JSONObject jsonObject = modified.getJSONObject(i);
+                    ModifiedPath modifiedPath = new ModifiedPath(jsonObject);
+                    this.modifiedPaths.add(modifiedPath);
+                }
+            }
+        }
+        if (patch.has("removed")) {
+            JSONArray removed = (JSONArray) patch.get("removed");
+            if (removed != null) {
+                for (int i=0; i<removed.length(); i++) {
+                    String pathString = removed.getString(i);
+                    this.removedPaths.add(Paths.get(pathString));
+                }
+            }
+        }
+    }
+
+    @Override
+    protected void buildJson() {
+        JSONObject patch = new JSONObject();
+        // Attempt to use a LinkedHashMap so that the order of fields is maintained
+        try {
+            Field changeMap = patch.getClass().getDeclaredField("map");
+            changeMap.setAccessible(true);
+            changeMap.set(patch, new LinkedHashMap<>());
+            changeMap.setAccessible(false);
+        } catch (IllegalAccessException | NoSuchFieldException e) {
+            // Don't worry about failures as this is for optional ordering only
+        }
+
+        patch.put("prevSig", Base58.encode(this.previousSignature));
+        patch.put("prevHash", Base58.encode(this.previousHash));
+        patch.put("curHash", Base58.encode(this.currentHash));
+        patch.put("added", new JSONArray(this.addedPaths));
+        patch.put("removed", new JSONArray(this.removedPaths));
+
+        JSONArray modifiedPaths = new JSONArray();
+        for (ModifiedPath modifiedPath : this.modifiedPaths) {
+            JSONObject modifiedPathJson = new JSONObject();
+            modifiedPathJson.put("path", modifiedPath.getPath());
+            modifiedPathJson.put("type", modifiedPath.getDiffType());
+            modifiedPaths.put(modifiedPathJson);
+        }
+        patch.put("modified", modifiedPaths);
+
+        this.jsonString = patch.toString(2);
+        LOGGER.debug("Patch metadata: {}", this.jsonString);
+    }
+
+    public void setAddedPaths(List<Path> addedPaths) {
+        this.addedPaths = addedPaths;
+    }
+
+    public List<Path> getAddedPaths() {
+        return this.addedPaths;
+    }
+
+    public void setModifiedPaths(List<ModifiedPath> modifiedPaths) {
+        this.modifiedPaths = modifiedPaths;
+    }
+
+    public List<ModifiedPath> getModifiedPaths() {
+        return this.modifiedPaths;
+    }
+
+    public void setRemovedPaths(List<Path> removedPaths) {
+        this.removedPaths = removedPaths;
+    }
+
+    public List<Path> getRemovedPaths() {
+        return this.removedPaths;
+    }
+
+    public void setPreviousSignature(byte[] previousSignature) {
+        this.previousSignature = previousSignature;
+    }
+
+    public byte[] getPreviousSignature() {
+        return this.previousSignature;
+    }
+
+    public void setPreviousHash(byte[] previousHash) {
+        this.previousHash = previousHash;
+    }
+
+    public byte[] getPreviousHash() {
+        return this.previousHash;
+    }
+
+    public void setCurrentHash(byte[] currentHash) {
+        this.currentHash = currentHash;
+    }
+
+    public byte[] getCurrentHash() {
+        return this.currentHash;
+    }
+
+
+    public int getFileDifferencesCount() {
+        return this.addedPaths.size() + this.modifiedPaths.size() + this.removedPaths.size();
+    }
+
+}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java

@@ -0,0 +1,102 @@
+package org.qortal.arbitrary.metadata;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.repository.DataException;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+/**
+ * ArbitraryDataQortalMetadata
+ *
+ * This is a base class to handle reading and writing JSON to a .qortal folder
+ * within the supplied filePath. This is used when storing data against an existing
+ * arbitrary data file structure.
+ *
+ * It is not usable on its own; it must be subclassed, with three methods overridden:
+ *
+ * fileName() - the file name to use within the .qortal folder
+ * readJson() - code to unserialize the JSON file
+ * buildJson() - code to serialize the JSON file
+ *
+ */
+public class ArbitraryDataQortalMetadata extends ArbitraryDataMetadata {
+
+    protected static final Logger LOGGER = LogManager.getLogger(ArbitraryDataQortalMetadata.class);
+
+    protected Path filePath;
+    protected Path qortalDirectoryPath;
+
+    protected String jsonString;
+
+    public ArbitraryDataQortalMetadata(Path filePath) {
+        super(filePath);
+
+        this.qortalDirectoryPath = Paths.get(filePath.toString(), ".qortal");
+    }
+
+    protected String fileName() {
+        // To be overridden
+        return null;
+    }
+
+    protected void readJson() throws DataException {
+        // To be overridden
+    }
+
+    protected void buildJson() {
+        // To be overridden
+    }
+
+
+    @Override
+    public void read() throws IOException, DataException {
+        this.loadJson();
+        this.readJson();
+    }
+
+    @Override
+    public void write() throws IOException, DataException {
+        this.buildJson();
+        this.createParentDirectories();
+        this.createQortalDirectory();
+
+        Path patchPath = Paths.get(this.qortalDirectoryPath.toString(), this.fileName());
+        BufferedWriter writer = new BufferedWriter(new FileWriter(patchPath.toString()));
+        writer.write(this.jsonString);
+        writer.newLine();
+        writer.close();
+    }
+
+    @Override
+    protected void loadJson() throws IOException {
+        Path path = Paths.get(this.qortalDirectoryPath.toString(), this.fileName());
+        File patchFile = new File(path.toString());
+        if (!patchFile.exists()) {
+            throw new IOException(String.format("Patch file doesn't exist: %s", path.toString()));
+        }
+
+        this.jsonString = new String(Files.readAllBytes(path));
+    }
+
+
+    protected void createQortalDirectory() throws DataException {
+        try {
+            Files.createDirectories(this.qortalDirectoryPath);
+        } catch (IOException e) {
+            throw new DataException("Unable to create .qortal directory");
+        }
+    }
+
+
+    public String getJsonString() {
+        return this.jsonString;
+    }
+
+}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java

@@ -0,0 +1,78 @@
+package org.qortal.arbitrary.metadata;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.qortal.repository.DataException;
+import org.qortal.utils.Base58;
+
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
+
+    private List<byte[]> chunks;
+
+    public ArbitraryDataTransactionMetadata(Path filePath) {
+        super(filePath);
+
+    }
+
+    @Override
+    protected void readJson() throws DataException {
+        if (this.jsonString == null) {
+            throw new DataException("Transaction metadata JSON string is null");
+        }
+
+        List<byte[]> chunksList = new ArrayList<>();
+        JSONObject cache = new JSONObject(this.jsonString);
+        if (cache.has("chunks")) {
+            JSONArray chunks = cache.getJSONArray("chunks");
+            if (chunks != null) {
+                for (int i=0; i<chunks.length(); i++) {
+                    String chunk = chunks.getString(i);
+                    if (chunk != null) {
+                        chunksList.add(Base58.decode(chunk));
+                    }
+                }
+            }
+            this.chunks = chunksList;
+        }
+    }
+
+    @Override
+    protected void buildJson() {
+        JSONObject outer = new JSONObject();
+
+        JSONArray chunks = new JSONArray();
+        if (this.chunks != null) {
+            for (byte[] chunk : this.chunks) {
+                chunks.put(Base58.encode(chunk));
+            }
+        }
+        outer.put("chunks", chunks);
+
+        this.jsonString = outer.toString(2);
+        LOGGER.trace("Transaction metadata: {}", this.jsonString);
+    }
+
+
+    public void setChunks(List<byte[]> chunks) {
+        this.chunks = chunks;
+    }
+
+    public List<byte[]> getChunks() {
+        return this.chunks;
+    }
+
+    public boolean containsChunk(byte[] chunk) {
+        for (byte[] c : this.chunks) {
+            if (Arrays.equals(c, chunk)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

src/main/java/org/qortal/arbitrary/misc/Service.java

@@ -0,0 +1,131 @@
+package org.qortal.arbitrary.misc;
+
+import org.json.JSONObject;
+import org.qortal.arbitrary.ArbitraryDataRenderer;
+import org.qortal.transaction.Transaction;
+import org.qortal.utils.FilesystemUtils;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static java.util.Arrays.stream;
+import static java.util.stream.Collectors.toMap;
+
+public enum Service {
+    AUTO_UPDATE(1, false, null, null),
+    ARBITRARY_DATA(100, false, null, null),
+    WEBSITE(200, true, null, null) {
+        @Override
+        public ValidationResult validate(Path path) {
+            // Custom validation function to require an index HTML file in the root directory
+            List<String> fileNames = ArbitraryDataRenderer.indexFiles();
+            String[] files = path.toFile().list();
+            if (files != null) {
+                for (String file : files) {
+                    Path fileName = Paths.get(file).getFileName();
+                    if (fileName != null && fileNames.contains(fileName.toString())) {
+                        return ValidationResult.OK;
+                    }
+                }
+            }
+            return ValidationResult.MISSING_INDEX_FILE;
+        }
+    },
+    GIT_REPOSITORY(300, false, null, null),
+    IMAGE(400, true, 10*1024*1024L, null),
+    THUMBNAIL(410, true, 500*1024L, null),
+    VIDEO(500, false, null, null),
+    AUDIO(600, false, null, null),
+    BLOG(700, false, null, null),
+    BLOG_POST(777, false, null, null),
+    BLOG_COMMENT(778, false, null, null),
+    DOCUMENT(800, false, null, null),
+    LIST(900, true, null, null),
+    PLAYLIST(910, true, null, null),
+    APP(1000, false, null, null),
+    METADATA(1100, false, null, null),
+    QORTAL_METADATA(1111, true, 10*1024L, Arrays.asList("title", "description", "tags"));
+
+    public final int value;
+    private final boolean requiresValidation;
+    private final Long maxSize;
+    private final List<String> requiredKeys;
+
+    private static final Map<Integer, Service> map = stream(Service.values())
+            .collect(toMap(service -> service.value, service -> service));
+
+    Service(int value, boolean requiresValidation, Long maxSize, List<String> requiredKeys) {
+        this.value = value;
+        this.requiresValidation = requiresValidation;
+        this.maxSize = maxSize;
+        this.requiredKeys = requiredKeys;
+    }
+
+    public ValidationResult validate(Path path) throws IOException {
+        if (!this.isValidationRequired()) {
+            return ValidationResult.OK;
+        }
+
+        byte[] data = FilesystemUtils.getSingleFileContents(path);
+        long size = FilesystemUtils.getDirectorySize(path);
+
+        // Validate max size if needed
+        if (this.maxSize != null) {
+            if (size > this.maxSize) {
+                return ValidationResult.EXCEEDS_SIZE_LIMIT;
+            }
+        }
+
+        // Validate required keys if needed
+        if (this.requiredKeys != null) {
+            if (data == null) {
+                return ValidationResult.MISSING_KEYS;
+            }
+            JSONObject json = Service.toJsonObject(data);
+            for (String key : this.requiredKeys) {
+                if (!json.has(key)) {
+                    return ValidationResult.MISSING_KEYS;
+                }
+            }
+        }
+
+        // Validation passed
+        return ValidationResult.OK;
+    }
+
+    public boolean isValidationRequired() {
+        return this.requiresValidation;
+    }
+
+    public static Service valueOf(int value) {
+        return map.get(value);
+    }
+
+    public static JSONObject toJsonObject(byte[] data) {
+        String dataString = new String(data);
+        return new JSONObject(dataString);
+    }
+
+    public enum ValidationResult {
+        OK(1),
+        MISSING_KEYS(2),
+        EXCEEDS_SIZE_LIMIT(3),
+        MISSING_INDEX_FILE(4);
+
+        public final int value;
+
+        private static final Map<Integer, Transaction.ValidationResult> map = stream(Transaction.ValidationResult.values()).collect(toMap(result -> result.value, result -> result));
+
+        ValidationResult(int value) {
+            this.value = value;
+        }
+
+        public static Transaction.ValidationResult valueOf(int value) {
+            return map.get(value);
+        }
+    }
+}

src/main/java/org/qortal/arbitrary/patch/UnifiedDiffPatch.java

@@ -0,0 +1,229 @@
+package org.qortal.arbitrary.patch;
+
+import com.github.difflib.DiffUtils;
+import com.github.difflib.UnifiedDiffUtils;
+import com.github.difflib.patch.Patch;
+import com.github.difflib.patch.PatchFailedException;
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.crypto.Crypto;
+import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
+import org.qortal.utils.FilesystemUtils;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
+public class UnifiedDiffPatch {
+
+    private static final Logger LOGGER = LogManager.getLogger(UnifiedDiffPatch.class);
+
+    private final Path before;
+    private final Path after;
+    private final Path destination;
+
+    private String identifier;
+    private Path validationPath;
+
+    public UnifiedDiffPatch(Path before, Path after, Path destination) {
+        this.before = before;
+        this.after = after;
+        this.destination = destination;
+    }
+
+    /**
+     * Create a patch based on the differences in path "after"
+     * compared with base path "before", outputting the patch
+     * to the "destination" path.
+     *
+     * @throws IOException
+     */
+    public void create() throws IOException {
+        if (!Files.exists(before)) {
+            throw new IOException(String.format("File not found (before): %s", before.toString()));
+        }
+        if (!Files.exists(after)) {
+            throw new IOException(String.format("File not found (after): %s", after.toString()));
+        }
+
+        // Ensure parent folders exist in the destination
+        File file = new File(destination.toString());
+        File parent = file.getParentFile();
+        if (parent != null) {
+            parent.mkdirs();
+        }
+
+        // Delete an existing file if it exists
+        File destFile = destination.toFile();
+        if (destFile.exists() && destFile.isFile()) {
+            Files.delete(destination);
+        }
+
+        // Load the two files into memory
+        List<String> original = FileUtils.readLines(before.toFile(), StandardCharsets.UTF_8);
+        List<String> revised = FileUtils.readLines(after.toFile(), StandardCharsets.UTF_8);
+
+        // Check if the original file ends with a newline
+        boolean endsWithNewline = FilesystemUtils.fileEndsWithNewline(before);
+
+        // Generate diff information
+        Patch<String> diff = DiffUtils.diff(original, revised);
+
+        // Generate unified diff format
+        String originalFileName = before.getFileName().toString();
+        String revisedFileName = after.getFileName().toString();
+        List<String> unifiedDiff = UnifiedDiffUtils.generateUnifiedDiff(originalFileName, revisedFileName, original, diff, 0);
+
+        // Write the diff to the destination directory
+        FileWriter fileWriter = new FileWriter(destination.toString(), true);
+        BufferedWriter writer = new BufferedWriter(fileWriter);
+        for (int i=0; i<unifiedDiff.size(); i++) {
+            String line = unifiedDiff.get(i);
+            writer.append(line);
+            // Add a newline if this isn't the last line, or the original ended with a newline
+            if (i < unifiedDiff.size()-1 || endsWithNewline) {
+                writer.newLine();
+            }
+        }
+        writer.flush();
+        writer.close();
+    }
+
+    /**
+     * Validate the patch at the "destination" path to ensure
+     * it works correctly and is smaller than the original file
+     *
+     * @return true if valid, false if invalid
+     */
+    public boolean isValid() throws DataException {
+        this.createRandomIdentifier();
+        this.createTempValidationDirectory();
+
+        // Merge the patch with the original path
+        Path tempPath = Paths.get(this.validationPath.toString(), this.identifier);
+
+        try {
+            UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(before, destination, tempPath);
+            unifiedDiffPatch.apply(null);
+
+            byte[] inputDigest = Crypto.digest(after.toFile());
+            byte[] outputDigest = Crypto.digest(tempPath.toFile());
+            if (Arrays.equals(inputDigest, outputDigest)) {
+                // Patch is valid, but we might want to reject if it's larger than the original file
+                long originalSize = Files.size(after);
+                long patchSize = Files.size(destination);
+                if (patchSize < originalSize) {
+                    // Patch file is smaller than the original file size, so treat it as valid
+                    return true;
+                }
+            }
+            else {
+                LOGGER.info("Checksum mismatch when verifying patch for file {}", destination.toString());
+                return false;
+            }
+
+        }
+        catch (IOException e) {
+            LOGGER.info("Failed to compute merge for file {}: {}", destination.toString(), e.getMessage());
+        }
+        finally {
+            try {
+                Files.delete(tempPath);
+            } catch (IOException e) {
+                // Not important - will be cleaned up later
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * Apply a patch at path "after" on top of base path "before",
+     * outputting the combined results to the "destination" path.
+     * If before and after are directories, a relative path suffix
+     * can be used to specify the file within these folder structures.
+     *
+     * @param pathSuffix - a file path to append to the base paths, or null if the base paths are already files
+     * @throws IOException
+     */
+    public void apply(Path pathSuffix) throws IOException, DataException {
+        Path originalPath = this.before;
+        Path patchPath = this.after;
+        Path mergePath = this.destination;
+
+        // If a path has been supplied, we need to append it to the base paths
+        if (pathSuffix != null) {
+            originalPath = Paths.get(this.before.toString(), pathSuffix.toString());
+            patchPath = Paths.get(this.after.toString(), pathSuffix.toString());
+            mergePath = Paths.get(this.destination.toString(), pathSuffix.toString());
+        }
+
+        if (!patchPath.toFile().exists()) {
+            throw new DataException("Patch file doesn't exist, but its path was included in modifiedPaths");
+        }
+
+        // Delete an existing file, as we are starting from a duplicate of pathBefore
+        File destFile = mergePath.toFile();
+        if (destFile.exists() && destFile.isFile()) {
+            Files.delete(mergePath);
+        }
+
+        List<String> originalContents = FileUtils.readLines(originalPath.toFile(), StandardCharsets.UTF_8);
+        List<String> patchContents = FileUtils.readLines(patchPath.toFile(), StandardCharsets.UTF_8);
+
+        // Check if the patch file (and therefore the original file) ends with a newline
+        boolean endsWithNewline = FilesystemUtils.fileEndsWithNewline(patchPath);
+
+        // At first, parse the unified diff file and get the patch
+        Patch<String> patch = UnifiedDiffUtils.parseUnifiedDiff(patchContents);
+
+        // Then apply the computed patch to the given text
+        try {
+            List<String> patchedContents = DiffUtils.patch(originalContents, patch);
+
+            // Write the patched file to the merge directory
+            FileWriter fileWriter = new FileWriter(mergePath.toString(), true);
+            BufferedWriter writer = new BufferedWriter(fileWriter);
+            for (int i=0; i<patchedContents.size(); i++) {
+                String line = patchedContents.get(i);
+                writer.append(line);
+                // Add a newline if this isn't the last line, or the original ended with a newline
+                if (i < patchedContents.size()-1 || endsWithNewline) {
+                    writer.newLine();
+                }
+            }
+            writer.flush();
+            writer.close();
+
+        } catch (PatchFailedException e) {
+            throw new DataException(String.format("Failed to apply patch for path %s: %s", pathSuffix, e.getMessage()));
+        }
+    }
+
+    private void createRandomIdentifier() {
+        this.identifier = UUID.randomUUID().toString();
+    }
+
+    private void createTempValidationDirectory() throws DataException {
+        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
+        String baseDir = Settings.getInstance().getTempDataPath();
+        Path tempDir = Paths.get(baseDir, "diff", "validate");
+        try {
+            Files.createDirectories(tempDir);
+        } catch (IOException e) {
+            throw new DataException("Unable to create temp directory");
+        }
+        this.validationPath = tempDir;
+    }
+
+}

src/main/java/org/qortal/block/Block.java

@@ -476,6 +476,16 @@ public class Block {
 		return this.minter;
 	}
 
+
+	public void setRepository(Repository repository) throws DataException {
+		this.repository = repository;
+
+		for (Transaction transaction : this.getTransactions()) {
+			transaction.setRepository(repository);
+		}
+	}
+
+
 	// More information
 
 	/**
@@ -524,8 +534,10 @@ public class Block {
 		long nonAtTransactionCount = transactionsData.stream().filter(transactionData -> transactionData.getType() != TransactionType.AT).count();
 
 		// The number of non-AT transactions fetched from repository should correspond with Block's transactionCount
-		if (nonAtTransactionCount != this.blockData.getTransactionCount())
+		if (nonAtTransactionCount != this.blockData.getTransactionCount()) {
+			LOGGER.error(() -> String.format("Block's transactions from repository (%d) do not match block's transaction count (%d)", nonAtTransactionCount, this.blockData.getTransactionCount()));
 			throw new IllegalStateException("Block's transactions from repository do not match block's transaction count");
+		}
 
 		this.transactions = new ArrayList<>();
 

src/main/java/org/qortal/block/BlockChain.java

@@ -68,7 +68,8 @@ public class BlockChain {
 		atFindNextTransactionFix,
 		newBlockSigHeight,
 		shareBinFix,
-		calcChainWeightTimestamp;
+		calcChainWeightTimestamp,
+		transactionV5Timestamp;
 	}
 
 	/** Map of which blockchain features are enabled when (height/timestamp) */
@@ -141,7 +142,8 @@ public class BlockChain {
 	}
 	private List<BlockTimingByHeight> blockTimingsByHeight;
 
-	private int minAccountLevelToMint = 1;
+	private int minAccountLevelToMint;
+	private int minAccountLevelForBlockSubmissions;
 	private int minAccountLevelToRewardShare;
 	private int maxRewardSharesPerMintingAccount;
 	private int founderEffectiveMintingLevel;
@@ -344,6 +346,10 @@ public class BlockChain {
 		return this.minAccountLevelToMint;
 	}
 
+	public int getMinAccountLevelForBlockSubmissions() {
+		return this.minAccountLevelForBlockSubmissions;
+	}
+
 	public int getMinAccountLevelToRewardShare() {
 		return this.minAccountLevelToRewardShare;
 	}
@@ -386,6 +392,10 @@ public class BlockChain {
 		return this.featureTriggers.get(FeatureTrigger.calcChainWeightTimestamp.name()).longValue();
 	}
 
+	public long getTransactionV5Timestamp() {
+		return this.featureTriggers.get(FeatureTrigger.transactionV5Timestamp.name()).longValue();
+	}
+
 	// More complex getters for aspects that change by height or timestamp
 
 	public long getRewardAtHeight(int ourHeight) {

src/main/java/org/qortal/controller/ArbitraryDataManager.java

@@ -1,91 +0,0 @@
-package org.qortal.controller;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Random;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.data.transaction.TransactionData;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.transaction.ArbitraryTransaction;
-import org.qortal.transaction.Transaction.TransactionType;
-
-public class ArbitraryDataManager extends Thread {
-
-	private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataManager.class);
-	private static final List<TransactionType> ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY);
-
-	private static ArbitraryDataManager instance;
-
-	private volatile boolean isStopping = false;
-
-	private ArbitraryDataManager() {
-	}
-
-	public static ArbitraryDataManager getInstance() {
-		if (instance == null)
-			instance = new ArbitraryDataManager();
-
-		return instance;
-	}
-
-	@Override
-	public void run() {
-		Thread.currentThread().setName("Arbitrary Data Manager");
-
-		try {
-			while (!isStopping) {
-				Thread.sleep(2000);
-
-				// Any arbitrary transactions we want to fetch data for?
-				try (final Repository repository = RepositoryManager.getRepository()) {
-					List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, null, null, true);
-					if (signatures == null || signatures.isEmpty())
-						continue;
-
-					// Filter out those that already have local data
-					signatures.removeIf(signature -> hasLocalData(repository, signature));
-
-					if (signatures.isEmpty())
-						continue;
-
-					// Pick one at random
-					final int index = new Random().nextInt(signatures.size());
-					byte[] signature = signatures.get(index);
-
-					Controller.getInstance().fetchArbitraryData(signature);
-				} catch (DataException e) {
-					LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
-				}
-			}
-		} catch (InterruptedException e) {
-			// Fall-through to exit thread...
-		}
-	}
-
-	public void shutdown() {
-		isStopping = true;
-		this.interrupt();
-	}
-
-	private boolean hasLocalData(final Repository repository, final byte[] signature) {
-		try {
-			TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-			if (!(transactionData instanceof ArbitraryTransactionData))
-				return true;
-
-			ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
-
-			return arbitraryTransaction.isDataLocal();
-		} catch (DataException e) {
-			LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e);
-			return true;
-		}
-	}
-
-}

src/main/java/org/qortal/controller/BlockMinter.java

@@ -1,6 +1,8 @@
 package org.qortal.controller;
 
 import java.math.BigInteger;
+import java.text.DecimalFormat;
+import java.text.NumberFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -16,11 +18,11 @@ import org.qortal.account.PrivateKeyAccount;
 import org.qortal.block.Block;
 import org.qortal.block.Block.ValidationResult;
 import org.qortal.block.BlockChain;
-import org.qortal.data.account.AccountData;
 import org.qortal.data.account.MintingAccountData;
 import org.qortal.data.account.RewardShareData;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.block.BlockSummaryData;
+import org.qortal.data.block.CommonBlockData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
@@ -48,11 +50,6 @@ public class BlockMinter extends Thread {
 	// Recovery
 	public static final long INVALID_BLOCK_RECOVERY_TIMEOUT = 10 * 60 * 1000L; // ms
 
-	// Min account level to submit blocks
-	// This is an unvalidated version of Blockchain.minAccountLevelToMint
-	// and exists only to reduce block candidates by default.
-	private static int MIN_LEVEL_FOR_BLOCK_SUBMISSION = 3;
-
 	// Constructors
 
 	public BlockMinter() {
@@ -81,6 +78,10 @@ public class BlockMinter extends Thread {
 			BlockRepository blockRepository = repository.getBlockRepository();
 			BlockData previousBlockData = null;
 
+			// Vars to keep track of blocks that were skipped due to chain weight
+			byte[] parentSignatureForLastLowWeightBlock = null;
+			Long timeOfLastLowWeightBlock = null;
+
 			List<Block> newBlocks = new ArrayList<>();
 
 			// Flags for tracking change in whether minting is possible,
@@ -137,14 +138,13 @@ public class BlockMinter extends Thread {
 						continue;
 					}
 
-					// Optional (non-validated) prevention of block submissions below a defined level
-					AccountData accountData = repository.getAccountRepository().getAccount(mintingAccount.getAddress());
-					if (accountData != null) {
-						Integer level = accountData.getLevel();
-						if (level != null && level < MIN_LEVEL_FOR_BLOCK_SUBMISSION) {
-							madi.remove();
-							continue;
-						}
+					// Optional (non-validated) prevention of block submissions below a defined level.
+					// This is an unvalidated version of Blockchain.minAccountLevelToMint
+					// and exists only to reduce block candidates by default.
+					int level = mintingAccount.getEffectiveMintingLevel();
+					if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
+						madi.remove();
+						continue;
 					}
 				}
 
@@ -156,7 +156,7 @@ public class BlockMinter extends Thread {
 
 				// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
 				// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
-				if (Controller.getInstance().getRecoveryMode() == false)
+				if (Synchronizer.getInstance().getRecoveryMode() == false)
 					peers.removeIf(Controller.hasNoRecentBlock);
 
 				// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
@@ -181,7 +181,7 @@ public class BlockMinter extends Thread {
 
 				// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
 				if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
-					if (Controller.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false)
+					if (Synchronizer.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false)
 						continue;
 
 				// There are enough peers with a recent block and our latest block is recent
@@ -195,6 +195,9 @@ public class BlockMinter extends Thread {
 
 					// Reduce log timeout
 					logTimeout = 10 * 1000L;
+
+					// Last low weight block is no longer valid
+					parentSignatureForLastLowWeightBlock = null;
 				}
 
 				// Discard accounts we have already built blocks with
@@ -211,6 +214,14 @@ public class BlockMinter extends Thread {
 					continue;
 				}
 
+				if (parentSignatureForLastLowWeightBlock != null) {
+					// The last iteration found a higher weight block in the network, so sleep for a while
+					// to allow is to sync the higher weight chain. We are sleeping here rather than when
+					// detected as we don't want to hold the blockchain lock open.
+					LOGGER.info("Sleeping for 10 seconds...");
+					Thread.sleep(10 * 1000L);
+				}
+
 				for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
 					// First block does the AT heavy-lifting
 					if (newBlocks.isEmpty()) {
@@ -302,6 +313,41 @@ public class BlockMinter extends Thread {
 						}
 					}
 
+					try {
+						if (this.higherWeightChainExists(repository, bestWeight)) {
+
+							// Check if the base block has updated since the last time we were here
+							if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null ||
+									!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) {
+								// We've switched to a different chain, so reset the timer
+								timeOfLastLowWeightBlock = NTP.getTime();
+							}
+							parentSignatureForLastLowWeightBlock = previousBlockData.getSignature();
+
+							// If less than 30 seconds has passed since first detection the higher weight chain,
+							// we should skip our block submission to give us the opportunity to sync to the better chain
+							if (NTP.getTime() - timeOfLastLowWeightBlock < 30*1000L) {
+								LOGGER.info("Higher weight chain found in peers, so not signing a block this round");
+								LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock);
+								continue;
+							}
+							else {
+								// More than 30 seconds have passed, so we should submit our block candidate anyway.
+								LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate...");
+							}
+						}
+						else {
+							LOGGER.debug("No higher weight chain found in peers");
+						}
+					} catch (DataException e) {
+						LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
+					}
+
+					// Clear variables that track low weight blocks
+					parentSignatureForLastLowWeightBlock = null;
+					timeOfLastLowWeightBlock = null;
+
+
 					// Add unconfirmed transactions
 					addUnconfirmedTransactions(repository, newBlock);
 
@@ -469,6 +515,61 @@ public class BlockMinter extends Thread {
 		}
 	}
 
+	private BigInteger getOurChainWeightSinceBlock(Repository repository, BlockSummaryData commonBlock, List<BlockSummaryData> peerBlockSummaries) throws DataException {
+		final int commonBlockHeight = commonBlock.getHeight();
+		final byte[] commonBlockSig = commonBlock.getSignature();
+		int mutualHeight = commonBlockHeight;
+
+		// Fetch our corresponding block summaries
+		final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
+		List<BlockSummaryData> ourBlockSummaries = repository.getBlockRepository()
+				.getBlockSummaries(commonBlockHeight + 1, ourLatestBlockData.getHeight());
+		if (!ourBlockSummaries.isEmpty()) {
+			Synchronizer.getInstance().populateBlockSummariesMinterLevels(repository, ourBlockSummaries);
+		}
+
+		if (ourBlockSummaries != null && peerBlockSummaries != null) {
+			mutualHeight += Math.min(ourBlockSummaries.size(), peerBlockSummaries.size());
+		}
+		return Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight);
+	}
+
+	private boolean higherWeightChainExists(Repository repository, BigInteger blockCandidateWeight) throws DataException {
+		if (blockCandidateWeight == null) {
+			// Can't make decisions without knowing the block candidate weight
+			return false;
+		}
+		NumberFormat formatter = new DecimalFormat("0.###E0");
+
+		List<Peer> peers = Network.getInstance().getHandshakedPeers();
+		// Loop through handshaked peers and check for any new block candidates
+		for (Peer peer : peers) {
+			if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) {
+				// This peer has common block data
+				CommonBlockData commonBlockData = peer.getCommonBlockData();
+				BlockSummaryData commonBlockSummaryData = commonBlockData.getCommonBlockSummary();
+				if (commonBlockData.getChainWeight() != null) {
+					// The synchronizer has calculated this peer's chain weight
+					BigInteger ourChainWeightSinceCommonBlock = this.getOurChainWeightSinceBlock(repository, commonBlockSummaryData, commonBlockData.getBlockSummariesAfterCommonBlock());
+					BigInteger ourChainWeight = ourChainWeightSinceCommonBlock.add(blockCandidateWeight);
+					BigInteger peerChainWeight = commonBlockData.getChainWeight();
+					if (peerChainWeight.compareTo(ourChainWeight) >= 0) {
+						// This peer has a higher weight chain than ours
+						LOGGER.debug("Peer {} is on a higher weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight));
+						return true;
+
+					} else {
+						LOGGER.debug("Peer {} is on a lower weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight));
+					}
+				} else {
+					LOGGER.debug("Peer {} has no chain weight", peer);
+				}
+			} else {
+				LOGGER.debug("Peer {} has no common block data", peer);
+			}
+		}
+		return false;
+	}
 	private static void moderatedLog(Runnable logFunction) {
 		// We only log if logging at TRACE or previous log timeout has expired
 		if (!LOGGER.isTraceEnabled() && lastLogTimestamp != null && lastLogTimestamp + logTimeout > System.currentTimeMillis())

src/main/java/org/qortal/controller/Synchronizer.java

@@ -1,9 +1,11 @@
 package org.qortal.controller;
 
 import java.math.BigInteger;
+import java.security.SecureRandom;
 import java.text.DecimalFormat;
 import java.text.NumberFormat;
 import java.util.*;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
@@ -20,6 +22,7 @@ import org.qortal.data.block.CommonBlockData;
 import org.qortal.data.network.PeerChainTipData;
 import org.qortal.data.transaction.RewardShareTransactionData;
 import org.qortal.data.transaction.TransactionData;
+import org.qortal.network.Network;
 import org.qortal.network.Peer;
 import org.qortal.network.message.BlockMessage;
 import org.qortal.network.message.BlockSummariesMessage;
@@ -35,14 +38,15 @@ import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
 import org.qortal.transaction.Transaction;
 import org.qortal.utils.Base58;
+import org.qortal.utils.ByteArray;
 import org.qortal.utils.NTP;
 
-public class Synchronizer {
+public class Synchronizer extends Thread {
 
 	private static final Logger LOGGER = LogManager.getLogger(Synchronizer.class);
 
 	/** Max number of new blocks we aim to add to chain tip in each sync round */
-	private static final int SYNC_BATCH_SIZE = 200; // XXX move to Settings?
+	private static final int SYNC_BATCH_SIZE = 1000; // XXX move to Settings?
 
 	/** Initial jump back of block height when searching for common block with peer */
 	private static final int INITIAL_BLOCK_STEP = 8;
@@ -55,10 +59,31 @@ public class Synchronizer {
 	/** Maximum number of block signatures we ask from peer in one go */
 	private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings?
 
+	private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms
+
+
+	private boolean running;
+
+	/** Latest block signatures from other peers that we know are on inferior chains. */
+	List<ByteArray> inferiorChainSignatures = new ArrayList<>();
+
+	/** Recovery mode, which is used to bring back a stalled network */
+	private boolean recoveryMode = false;
+	private boolean peersAvailable = true; // peersAvailable must default to true
+	private long timePeersLastAvailable = 0;
 
 	// Keep track of the size of the last re-org, so it can be logged
 	private int lastReorgSize;
 
+	/** Synchronization object for sync variables below */
+	public final Object syncLock = new Object();
+	/** Whether we are attempting to synchronize. */
+	private volatile boolean isSynchronizing = false;
+	/** Temporary estimate of synchronization progress for SysTray use. */
+	private volatile int syncPercent = 0;
+
+	private static volatile boolean requestSync = false;
+
 	// Keep track of invalid blocks so that we don't keep trying to sync them
 	private Map<String, Long> invalidBlockSignatures = Collections.synchronizedMap(new HashMap<>());
 	public Long timeValidBlockLastReceived = null;
@@ -73,6 +98,7 @@ public class Synchronizer {
 	// Constructors
 
 	private Synchronizer() {
+		this.running = true;
 	}
 
 	public static Synchronizer getInstance() {
@@ -83,6 +109,276 @@ public class Synchronizer {
 	}
 
 
+	@Override
+	public void run() {
+		try {
+			while (running && !Controller.isStopping()) {
+				Thread.sleep(1000);
+
+				if (requestSync) {
+					requestSync = false;
+					boolean success = Synchronizer.getInstance().potentiallySynchronize();
+					if (!success) {
+						// Something went wrong, so try again next time
+						requestSync = true;
+					}
+				}
+			}
+		} catch (InterruptedException e) {
+			// Clear interrupted flag so we can shutdown trim threads
+			Thread.interrupted();
+			// Fall-through to exit
+		}
+	}
+
+	public void shutdown() {
+		this.running = false;
+		this.interrupt();
+	}
+
+
+
+	public boolean isSynchronizing() {
+		return this.isSynchronizing;
+	}
+
+	public Integer getSyncPercent() {
+		synchronized (this.syncLock) {
+			return this.isSynchronizing ? this.syncPercent : null;
+		}
+	}
+
+	public void requestSync() {
+		requestSync = true;
+	}
+
+	public boolean isSyncRequested() {
+		return requestSync;
+	}
+
+	public boolean getRecoveryMode() {
+		return this.recoveryMode;
+	}
+
+
+	public boolean potentiallySynchronize() throws InterruptedException {
+		// Already synchronizing via another thread?
+		if (this.isSynchronizing)
+			return true;
+
+		List<Peer> peers = Network.getInstance().getHandshakedPeers();
+
+		// Disregard peers that have "misbehaved" recently
+		peers.removeIf(Controller.hasMisbehaved);
+
+		// Disregard peers that only have genesis block
+		peers.removeIf(Controller.hasOnlyGenesisBlock);
+
+		// Disregard peers that don't have a recent block
+		peers.removeIf(Controller.hasNoRecentBlock);
+
+		// Disregard peers that are on an old version
+		peers.removeIf(Controller.hasOldVersion);
+
+		checkRecoveryModeForPeers(peers);
+		if (recoveryMode) {
+			peers = Network.getInstance().getHandshakedPeers();
+			peers.removeIf(Controller.hasOnlyGenesisBlock);
+			peers.removeIf(Controller.hasMisbehaved);
+			peers.removeIf(Controller.hasOldVersion);
+		}
+
+		// Check we have enough peers to potentially synchronize
+		if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
+			return true;
+
+		// Disregard peers that have no block signature or the same block signature as us
+		peers.removeIf(Controller.hasNoOrSameBlock);
+
+		// Disregard peers that are on the same block as last sync attempt and we didn't like their chain
+		peers.removeIf(Controller.hasInferiorChainTip);
+
+		final int peersBeforeComparison = peers.size();
+
+		// Request recent block summaries from the remaining peers, and locate our common block with each
+		Synchronizer.getInstance().findCommonBlocksWithPeers(peers);
+
+		// Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks
+		peers = Synchronizer.getInstance().comparePeers(peers);
+
+		// We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains
+		peers.removeIf(Controller.hasInferiorChainTip);
+
+		final int peersRemoved = peersBeforeComparison - peers.size();
+		if (peersRemoved > 0 && peers.size() > 0)
+			LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size()));
+
+		if (peers.isEmpty())
+			return true;
+
+		if (peers.size() > 1) {
+			StringBuilder finalPeersString = new StringBuilder();
+			for (Peer peer : peers)
+				finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer);
+			LOGGER.debug(String.format("Choosing random peer from: [%s]", finalPeersString.toString()));
+		}
+
+		// Pick random peer to sync with
+		int index = new SecureRandom().nextInt(peers.size());
+		Peer peer = peers.get(index);
+
+		SynchronizationResult syncResult = actuallySynchronize(peer, false);
+		if (syncResult == SynchronizationResult.NO_BLOCKCHAIN_LOCK) {
+			// No blockchain lock - force a retry by returning false
+			return false;
+		}
+
+		return true;
+	}
+
+	public SynchronizationResult actuallySynchronize(Peer peer, boolean force) throws InterruptedException {
+		boolean hasStatusChanged = false;
+		BlockData priorChainTip = Controller.getInstance().getChainTip();
+
+		synchronized (this.syncLock) {
+			this.syncPercent = (priorChainTip.getHeight() * 100) / peer.getChainTipData().getLastHeight();
+
+			// Only update SysTray if we're potentially changing height
+			if (this.syncPercent < 100) {
+				this.isSynchronizing = true;
+				hasStatusChanged = true;
+			}
+		}
+		peer.setSyncInProgress(true);
+
+		if (hasStatusChanged)
+			Controller.getInstance().updateSysTray();
+
+		try {
+			SynchronizationResult syncResult = Synchronizer.getInstance().synchronize(peer, force);
+			switch (syncResult) {
+				case GENESIS_ONLY:
+				case NO_COMMON_BLOCK:
+				case TOO_DIVERGENT:
+				case INVALID_DATA: {
+					// These are more serious results that warrant a cool-off
+					LOGGER.info(String.format("Failed to synchronize with peer %s (%s) - cooling off", peer, syncResult.name()));
+
+					// Don't use this peer again for a while
+					Network.getInstance().peerMisbehaved(peer);
+					break;
+				}
+
+				case INFERIOR_CHAIN: {
+					// Update our list of inferior chain tips
+					ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature());
+					if (!inferiorChainSignatures.contains(inferiorChainSignature))
+						inferiorChainSignatures.add(inferiorChainSignature);
+
+					// These are minor failure results so fine to try again
+					LOGGER.debug(() -> String.format("Refused to synchronize with peer %s (%s)", peer, syncResult.name()));
+
+					// Notify peer of our superior chain
+					if (!peer.sendMessage(Network.getInstance().buildHeightMessage(peer, priorChainTip)))
+						peer.disconnect("failed to notify peer of our superior chain");
+					break;
+				}
+
+				case NO_REPLY:
+				case NO_BLOCKCHAIN_LOCK:
+				case REPOSITORY_ISSUE:
+					// These are minor failure results so fine to try again
+					LOGGER.debug(() -> String.format("Failed to synchronize with peer %s (%s)", peer, syncResult.name()));
+					break;
+
+				case SHUTTING_DOWN:
+					// Just quietly exit
+					break;
+
+				case OK:
+					// fall-through...
+				case NOTHING_TO_DO: {
+					// Update our list of inferior chain tips
+					ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature());
+					if (!inferiorChainSignatures.contains(inferiorChainSignature))
+						inferiorChainSignatures.add(inferiorChainSignature);
+
+					LOGGER.debug(() -> String.format("Synchronized with peer %s (%s)", peer, syncResult.name()));
+					break;
+				}
+			}
+
+			if (!running) {
+				// We've stopped
+				return SynchronizationResult.SHUTTING_DOWN;
+			}
+
+			// Has our chain tip changed?
+			BlockData newChainTip;
+
+			try (final Repository repository = RepositoryManager.getRepository()) {
+				newChainTip = repository.getBlockRepository().getLastBlock();
+			} catch (DataException e) {
+				LOGGER.warn(String.format("Repository issue when trying to fetch post-synchronization chain tip: %s", e.getMessage()));
+				return syncResult;
+			}
+
+			if (!Arrays.equals(newChainTip.getSignature(), priorChainTip.getSignature())) {
+				// Reset our cache of inferior chains
+				inferiorChainSignatures.clear();
+
+				Network network = Network.getInstance();
+				network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip));
+			}
+
+			return syncResult;
+		} finally {
+			this.isSynchronizing = false;
+			peer.setSyncInProgress(false);
+		}
+	}
+
+	private boolean checkRecoveryModeForPeers(List<Peer> qualifiedPeers) {
+		List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
+
+		if (handshakedPeers.size() > 0) {
+			// There is at least one handshaked peer
+			if (qualifiedPeers.isEmpty()) {
+				// There are no 'qualified' peers - i.e. peers that have a recent block we can sync to
+				boolean werePeersAvailable = peersAvailable;
+				peersAvailable = false;
+
+				// If peers only just became unavailable, update our record of the time they were last available
+				if (werePeersAvailable)
+					timePeersLastAvailable = NTP.getTime();
+
+				// If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint
+				if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) {
+					if (recoveryMode == false) {
+						LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000));
+						recoveryMode = true;
+					}
+				}
+			} else {
+				// We now have at least one peer with a recent block, so we can exit recovery mode and sync normally
+				peersAvailable = true;
+				if (recoveryMode) {
+					LOGGER.info("Peers have become available again. Exiting recovery mode...");
+					recoveryMode = false;
+				}
+			}
+		}
+		return recoveryMode;
+	}
+
+	public void addInferiorChainSignature(byte[] inferiorSignature) {
+		// Update our list of inferior chain tips
+		ByteArray inferiorChainSignature = new ByteArray(inferiorSignature);
+		if (!inferiorChainSignatures.contains(inferiorChainSignature))
+			inferiorChainSignatures.add(inferiorChainSignature);
+	}
+
+
 	/**
 	 * Iterate through a list of supplied peers, and attempt to find our common block with each.
 	 * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later.
@@ -255,6 +551,8 @@ public class Synchronizer {
 				// Create a placeholder to track of common blocks that we can discard due to being inferior chains
 				int dropPeersAfterCommonBlockHeight = 0;
 
+				NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
+
 				// Remove peers with no common block data
 				Iterator iterator = peers.iterator();
 				while (iterator.hasNext()) {
@@ -275,7 +573,7 @@ public class Synchronizer {
 							// We have already determined that the correct chain diverged from a lower height. We are safe to skip these peers.
 							for (Peer peer : peersSharingCommonBlock) {
 								LOGGER.debug(String.format("Peer %s has common block at height %d but the superior chain is at height %d. Removing it from this round.", peer, commonBlockSummary.getHeight(), dropPeersAfterCommonBlockHeight));
-								Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature());
+								this.addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature());
 							}
 							continue;
 						}
@@ -377,9 +675,7 @@ public class Synchronizer {
 					if (ourBlockSummaries.size() > 0)
 						ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, maxHeightForChainWeightComparisons);
 
-					NumberFormat formatter = new DecimalFormat("0.###E0");
-					NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
-					LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", (usingSameLengthChainWeight ? minChainLength : ourBlockSummaries.size()), formatter.format(ourChainWeight)));
+					LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", (usingSameLengthChainWeight ? minChainLength : ourBlockSummaries.size()), accurateFormatter.format(ourChainWeight)));
 
 					LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature())));
 					for (Peer peer : peersSharingCommonBlock) {
@@ -401,7 +697,7 @@ public class Synchronizer {
 						LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock));
 						BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, maxHeightForChainWeightComparisons);
 						peer.getCommonBlockData().setChainWeight(peerChainWeight);
-						LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight)));
+						LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), accurateFormatter.format(peerChainWeight)));
 
 						// Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group)
 						if (ourChainWeight.compareTo(peerChainWeight) > 0) {
@@ -567,9 +863,11 @@ public class Synchronizer {
 		// Make sure we're the only thread modifying the blockchain
 		// If we're already synchronizing with another peer then this will also return fast
 		ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
-		if (!blockchainLock.tryLock())
+		if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) {
 			// Wasn't peer's fault we couldn't sync
+			LOGGER.info("Synchronizer couldn't acquire blockchain lock");
 			return SynchronizationResult.NO_BLOCKCHAIN_LOCK;
+		}
 
 		try {
 			try (final Repository repository = RepositoryManager.getRepository()) {
@@ -585,16 +883,7 @@ public class Synchronizer {
 					String syncString = String.format("Synchronizing with peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer,
 							peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(),
 							ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp());
-
-					// If our latest block is very old, we should log that we're attempting to sync with a peer
-					// Otherwise, it can appear as though nothing is happening for a while after launch
-					final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
-					if (minLatestBlockTimestamp != null && ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
-						LOGGER.info(syncString);
-					}
-					else {
-						LOGGER.debug(syncString);
-					}
+					LOGGER.info(syncString);
 
 					// Reset last re-org size as we are starting a new sync round
 					this.lastReorgSize = 0;
@@ -798,7 +1087,7 @@ public class Synchronizer {
 			return SynchronizationResult.REPOSITORY_ISSUE;
 
 		if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
-			LOGGER.info(String.format("Ditching our chain after height %d as our latest block is very old", commonBlockHeight));
+			LOGGER.info(String.format("Ditching our chain after height %d", commonBlockHeight));
 		} else {
 			// Compare chain weights
 
@@ -858,8 +1147,9 @@ public class Synchronizer {
 			BigInteger ourChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight);
 			BigInteger peerChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, peerBlockSummaries, mutualHeight);
 
-			NumberFormat formatter = new DecimalFormat("0.###E0");
-			LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", formatter.format(ourChainWeight), formatter.format(peerChainWeight)));
+			NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
+			LOGGER.debug(String.format("commonBlockHeight: %d, commonBlockSig: %.8s, ourBlockSummaries.size(): %d, peerBlockSummaries.size(): %d", commonBlockHeight, Base58.encode(commonBlockSig), ourBlockSummaries.size(), peerBlockSummaries.size()));
+			LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", accurateFormatter.format(ourChainWeight), accurateFormatter.format(peerChainWeight)));
 
 			// If our blockchain has greater weight then don't synchronize with peer
 			if (ourChainWeight.compareTo(peerChainWeight) >= 0) {
@@ -872,7 +1162,7 @@ public class Synchronizer {
 	}
 
 	private SynchronizationResult syncToPeerChain(Repository repository, BlockData commonBlockData, int ourInitialHeight,
-			Peer peer, final int peerHeight, List<BlockSummaryData> peerBlockSummaries) throws DataException, InterruptedException {
+												  Peer peer, final int peerHeight, List<BlockSummaryData> peerBlockSummaries) throws DataException, InterruptedException {
 		final int commonBlockHeight = commonBlockData.getHeight();
 		final byte[] commonBlockSig = commonBlockData.getSignature();
 		String commonBlockSig58 = Base58.encode(commonBlockSig);
@@ -902,19 +1192,19 @@ public class Synchronizer {
 			if (Controller.isStopping())
 				return SynchronizationResult.SHUTTING_DOWN;
 
-            // Ensure we don't request more than MAXIMUM_REQUEST_SIZE
-            int numberRequested = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE);
+			// Ensure we don't request more than MAXIMUM_REQUEST_SIZE
+			int numberRequested = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE);
 
-            // Do we need more signatures?
+			// Do we need more signatures?
 			if (peerBlockSignatures.isEmpty() && numberRequested > 0) {
-                LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s",
-                        numberRequested, (numberRequested != 1 ? "s" : ""), height, Base58.encode(latestPeerSignature)));
+				LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s",
+						numberRequested, (numberRequested != 1 ? "s" : ""), height, Base58.encode(latestPeerSignature)));
 
-                peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested);
+				peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested);
 
-                if (peerBlockSignatures == null || peerBlockSignatures.isEmpty()) {
-                    LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer,
-                            height, Base58.encode(latestPeerSignature)));
+				if (peerBlockSignatures == null || peerBlockSignatures.isEmpty()) {
+					LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer,
+							height, Base58.encode(latestPeerSignature)));
 
 					// Clear our cache of common block summaries for this peer, as they are likely to be invalid
 					CommonBlockData cachedCommonBlockData = peer.getCommonBlockData();
@@ -924,7 +1214,7 @@ public class Synchronizer {
                     // If we have already received newer blocks from this peer that what we have already, go ahead and apply them
                     if (peerBlocks.size() > 0) {
 						final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
-                    	final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1);
+						final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1);
 						final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
 						if (ourLatestBlockData != null && peerLatestBlock != null && minLatestBlockTimestamp != null) {
 
@@ -947,8 +1237,8 @@ public class Synchronizer {
                     return SynchronizationResult.NO_REPLY;
                 }
 
-                numberSignaturesRequired = peerHeight - height - peerBlockSignatures.size();
-                LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : "")));
+				numberSignaturesRequired = peerHeight - height - peerBlockSignatures.size();
+				LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : "")));
 			}
 
 			if (peerBlockSignatures.isEmpty()) {
@@ -1098,7 +1388,7 @@ public class Synchronizer {
 	}
 
 	private SynchronizationResult applyNewBlocks(Repository repository, BlockData commonBlockData, int ourInitialHeight,
-			Peer peer, int peerHeight, List<BlockSummaryData> peerBlockSummaries) throws InterruptedException, DataException {
+												 Peer peer, int peerHeight, List<BlockSummaryData> peerBlockSummaries) throws InterruptedException, DataException {
 		LOGGER.debug(String.format("Fetching new blocks from peer %s", peer));
 
 		final int commonBlockHeight = commonBlockData.getHeight();
@@ -1227,7 +1517,7 @@ public class Synchronizer {
 		return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates());
 	}
 
-	private void populateBlockSummariesMinterLevels(Repository repository, List<BlockSummaryData> blockSummaries) throws DataException {
+	public void populateBlockSummariesMinterLevels(Repository repository, List<BlockSummaryData> blockSummaries) throws DataException {
 		final int firstBlockHeight = blockSummaries.get(0).getHeight();
 
 		for (int i = 0; i < blockSummaries.size(); ++i) {

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java

@@ -0,0 +1,185 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
+import org.qortal.utils.NTP;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class ArbitraryDataBuildManager extends Thread {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class);
+
+    private static ArbitraryDataBuildManager instance;
+
+    private volatile boolean isStopping = false;
+    private boolean buildInProgress = false;
+
+    /**
+     * Map to keep track of arbitrary transaction resources currently being built (or queued).
+     */
+    public Map<String, ArbitraryDataBuildQueueItem> arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of failed arbitrary transaction builds.
+     */
+    public Map<String, ArbitraryDataBuildQueueItem> arbitraryDataFailedBuilds = Collections.synchronizedMap(new HashMap<>());
+
+
+    public ArbitraryDataBuildManager() {
+
+    }
+
+    @Override
+    public void run() {
+        try {
+            // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
+            // This can be expanded to have multiple threads processing the build queue when needed
+            ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
+            arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
+
+            while (!isStopping) {
+                // Nothing to do yet
+                Thread.sleep(5000);
+            }
+
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
+        }
+    }
+
+    public static ArbitraryDataBuildManager getInstance() {
+        if (instance == null)
+            instance = new ArbitraryDataBuildManager();
+
+        return instance;
+    }
+
+    public void shutdown() {
+        isStopping = true;
+        this.interrupt();
+    }
+
+
+    public void cleanupQueues(Long now) {
+        if (now == null) {
+            return;
+        }
+        arbitraryDataBuildQueue.entrySet().removeIf(entry -> entry.getValue().hasReachedBuildTimeout(now));
+        arbitraryDataFailedBuilds.entrySet().removeIf(entry -> entry.getValue().hasReachedFailureTimeout(now));
+    }
+
+    // Build queue
+
+    public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
+        String key = queueItem.getUniqueKey();
+        if (key == null) {
+            return false;
+        }
+
+        if (this.arbitraryDataBuildQueue == null) {
+            return false;
+        }
+
+        if (NTP.getTime() == null) {
+            // Can't use queues until we have synced the time
+            return false;
+        }
+
+        // Don't add builds that have failed recently
+        if (this.isInFailedBuildsList(queueItem)) {
+            return false;
+        }
+
+        if (this.arbitraryDataBuildQueue.put(key, queueItem) != null) {
+            // Already in queue
+            return true;
+        }
+
+        LOGGER.info("Added {} to build queue", queueItem);
+
+        // Added to queue
+        return true;
+    }
+
+    public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
+        String key = queueItem.getUniqueKey();
+        if (key == null) {
+            return false;
+        }
+
+        if (this.arbitraryDataBuildQueue == null) {
+            return false;
+        }
+
+        if (this.arbitraryDataBuildQueue.containsKey(key)) {
+            // Already in queue
+            return true;
+        }
+
+        // Not in queue
+        return false;
+    }
+
+
+    // Failed builds
+
+    public boolean addToFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) {
+        String key = queueItem.getUniqueKey();
+        if (key == null) {
+            return false;
+        }
+
+        if (this.arbitraryDataFailedBuilds == null) {
+            return false;
+        }
+
+        if (NTP.getTime() == null) {
+            // Can't use queues until we have synced the time
+            return false;
+        }
+
+        if (this.arbitraryDataFailedBuilds.put(key, queueItem) != null) {
+            // Already in list
+            return true;
+        }
+
+        LOGGER.info("Added {} to failed builds list", queueItem);
+
+        // Added to queue
+        return true;
+    }
+
+    public boolean isInFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) {
+        String key = queueItem.getUniqueKey();
+        if (key == null) {
+            return false;
+        }
+
+        if (this.arbitraryDataFailedBuilds == null) {
+            return false;
+        }
+
+        if (this.arbitraryDataFailedBuilds.containsKey(key)) {
+            // Already in list
+            return true;
+        }
+
+        // Not in list
+        return false;
+    }
+
+
+    public void setBuildInProgress(boolean buildInProgress) {
+        this.buildInProgress = buildInProgress;
+    }
+
+    public boolean getBuildInProgress() {
+        return this.buildInProgress;
+    }
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

@@ -0,0 +1,99 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
+import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.controller.Controller;
+import org.qortal.repository.DataException;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+import java.util.Map;
+
+
+public class ArbitraryDataBuilderThread implements Runnable {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuilderThread.class);
+
+    public ArbitraryDataBuilderThread() {
+
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data Builder Thread");
+        ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance();
+
+        while (!Controller.isStopping()) {
+            try {
+                Thread.sleep(1000);
+
+                if (buildManager.arbitraryDataBuildQueue == null) {
+                    continue;
+                }
+                if (buildManager.arbitraryDataBuildQueue.isEmpty()) {
+                    continue;
+                }
+
+                // Find resources that are queued for building
+                Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
+                        .entrySet().stream()
+                        .filter(e -> e.getValue().isQueued())
+                        .findFirst().orElse(null);
+
+                if (next == null) {
+                    continue;
+                }
+
+                Long now = NTP.getTime();
+                if (now == null) {
+                    continue;
+                }
+
+                ArbitraryDataBuildQueueItem queueItem = next.getValue();
+
+                if (queueItem == null) {
+                    this.removeFromQueue(queueItem);
+                }
+
+                // Ignore builds that have failed recently
+                if (buildManager.isInFailedBuildsList(queueItem)) {
+                    continue;
+                }
+
+
+                try {
+                    // Perform the build
+                    LOGGER.info("Building {}...", queueItem);
+                    queueItem.build();
+                    this.removeFromQueue(queueItem);
+                    LOGGER.info("Finished building {}", queueItem);
+
+                } catch (MissingDataException e) {
+                    LOGGER.info("Missing data for {}: {}", queueItem, e.getMessage());
+                    queueItem.setFailed(true);
+                    this.removeFromQueue(queueItem);
+                    // Don't add to the failed builds list, as we may want to retry sooner
+
+                } catch (IOException | DataException | RuntimeException e) {
+                    LOGGER.info("Error building {}: {}", queueItem, e.getMessage());
+                    // Something went wrong - so remove it from the queue, and add to failed builds list
+                    queueItem.setFailed(true);
+                    buildManager.addToFailedBuildsList(queueItem);
+                    this.removeFromQueue(queueItem);
+                }
+
+            } catch (InterruptedException e) {
+                // Time to exit
+            }
+        }
+    }
+
+    private void removeFromQueue(ArbitraryDataBuildQueueItem queueItem) {
+        if (queueItem == null || queueItem.getUniqueKey() == null) {
+            return;
+        }
+        ArbitraryDataBuildManager.getInstance().arbitraryDataBuildQueue.remove(queueItem.getUniqueKey());
+    }
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java

@@ -0,0 +1,576 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.transaction.Transaction;
+import org.qortal.transaction.Transaction.TransactionType;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.Base58;
+import org.qortal.utils.FilesystemUtils;
+import org.qortal.utils.NTP;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.SecureRandom;
+import java.util.*;
+
+import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD;
+
+public class ArbitraryDataCleanupManager extends Thread {
+
+	private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCleanupManager.class);
+	private static final List<TransactionType> ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY);
+
+	private static ArbitraryDataCleanupManager instance;
+
+	private volatile boolean isStopping = false;
+
+	/**
+	 * The amount of time that must pass before a file is treated as stale / not recent.
+	 * We can safely delete files created/accessed longer ago that this, if we have a means of
+	 * rebuilding them. The main purpose of this is to avoid deleting files that are currently
+	 * being used by other parts of the system.
+	 */
+	private static final long STALE_FILE_TIMEOUT = 60*60*1000L; // 1 hour
+
+	/**
+	 * The number of chunks to delete in a batch when over the capacity limit.
+	 * Storage limits are re-checked after each batch, and there could be a significant
+	 * delay between the processing of each batch as it only occurs after a complete
+	 * cleanup cycle (to allow unwanted chunks to be deleted first).
+	 */
+	private static final int CHUNK_DELETION_BATCH_SIZE = 10;
+
+
+	/*
+	TODO:
+	- Delete files from the _misc folder once they reach a certain age
+	 */
+
+
+	private ArbitraryDataCleanupManager() {
+	}
+
+	public static ArbitraryDataCleanupManager getInstance() {
+		if (instance == null)
+			instance = new ArbitraryDataCleanupManager();
+
+		return instance;
+	}
+
+	@Override
+	public void run() {
+		Thread.currentThread().setName("Arbitrary Data Cleanup Manager");
+
+		// Paginate queries when fetching arbitrary transactions
+		final int limit = 100;
+		int offset = 0;
+
+		try {
+			while (!isStopping) {
+				Thread.sleep(30000);
+
+				// Don't run if QDN is disabled
+				if (!Settings.getInstance().isQdnEnabled()) {
+					Thread.sleep(60 * 60 * 1000L);
+					continue;
+				}
+
+				Long now = NTP.getTime();
+				if (now == null) {
+					// Don't attempt to make decisions if we haven't synced our time yet
+					continue;
+				}
+
+				ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
+
+				// Wait until storage capacity has been calculated
+				if (!storageManager.isStorageCapacityCalculated()) {
+					continue;
+				}
+
+				// Periodically delete any unnecessary files from the temp directory
+				if (offset == 0 || offset % (limit * 10) == 0) {
+					this.cleanupTempDirectory(now);
+				}
+
+				// Any arbitrary transactions we want to fetch data for?
+				try (final Repository repository = RepositoryManager.getRepository()) {
+					List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
+					// LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
+					if (isStopping) {
+						return;
+					}
+
+					if (signatures == null || signatures.isEmpty()) {
+						offset = 0;
+						continue;
+					}
+					offset += limit;
+					now = NTP.getTime();
+
+					// Loop through the signatures in this batch
+					for (int i=0; i<signatures.size(); i++) {
+						if (isStopping) {
+							return;
+						}
+
+						byte[] signature = signatures.get(i);
+						if (signature == null) {
+							continue;
+						}
+
+						// Don't interfere with the filesystem whilst a build is in progress
+						if (ArbitraryDataBuildManager.getInstance().getBuildInProgress()) {
+							Thread.sleep(5000);
+						}
+
+						// Fetch the transaction data
+						ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+						if (arbitraryTransactionData == null) {
+							continue;
+						}
+
+						// Raw data doesn't have any associated files to clean up
+						if (arbitraryTransactionData.getDataType() == ArbitraryTransactionData.DataType.RAW_DATA) {
+							continue;
+						}
+
+						// Check if we have the complete file
+						boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData);
+
+						// Check if we have any of the chunks
+						boolean anyChunksExist = ArbitraryTransactionUtils.anyChunksExist(arbitraryTransactionData);
+						boolean transactionHasChunks = (arbitraryTransactionData.getMetadataHash() != null);
+
+						if (!completeFileExists && !anyChunksExist) {
+							// We don't have any files at all for this transaction - nothing to do
+							continue;
+						}
+
+						// We have at least 1 chunk or file for this transaction, so we might need to delete them...
+
+
+						// Check to see if we should be hosting data for this transaction at all
+						if (!storageManager.canStoreData(arbitraryTransactionData)) {
+							LOGGER.info("Deleting transaction {} because we can't host its data",
+									Base58.encode(arbitraryTransactionData.getSignature()));
+							ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
+							continue;
+						}
+
+						// Check to see if we have had a more recent PUT
+						boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
+						if (hasMoreRecentPutTransaction) {
+							// There is a more recent PUT transaction than the one we are currently processing.
+							// When a PUT is issued, it replaces any layers that would have been there before.
+							// Therefore any data relating to this older transaction is no longer needed.
+							LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " +
+											"Deleting all files associated with the earlier transaction.", arbitraryTransactionData.getService(),
+									arbitraryTransactionData.getName(), Base58.encode(signature)));
+
+							ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
+
+							// We should also remove peers for this transaction from the lookup table to save space
+							this.removePeersHostingTransactionData(repository, arbitraryTransactionData);
+							continue;
+						}
+
+						if (completeFileExists && !transactionHasChunks) {
+							// This file doesn't have any chunks because it is too small.
+							// We must not delete anything.
+							continue;
+						}
+
+						// Check if we have all of the chunks
+						boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData);
+
+						if (completeFileExists && allChunksExist) {
+							// We have the complete file and all the chunks, so we can delete
+							// the complete file if it has reached a certain age.
+							LOGGER.debug(String.format("Transaction %s has complete file and all chunks",
+									Base58.encode(arbitraryTransactionData.getSignature())));
+
+							ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
+							continue;
+						}
+
+						if (completeFileExists && !allChunksExist) {
+							// We have the complete file but not the chunks, so let's convert it
+							LOGGER.info(String.format("Transaction %s has complete file but no chunks",
+									Base58.encode(arbitraryTransactionData.getSignature())));
+
+							ArbitraryTransactionUtils.convertFileToChunks(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
+							continue;
+						}
+					}
+
+				} catch (DataException e) {
+					LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
+				}
+
+				try (final Repository repository = RepositoryManager.getRepository()) {
+
+					// Check if there are any hosted files that don't have matching transactions
+					this.checkForExpiredTransactions(repository);
+
+					// Delete additional data at random if we're over our storage limit
+					// Use the DELETION_THRESHOLD so that we only start deleting once the hard limit is reached
+					// This also allows some headroom between the regular threshold (90%) and the hard
+					// limit, to avoid data getting into a fetch/delete loop.
+					if (!storageManager.isStorageSpaceAvailable(DELETION_THRESHOLD)) {
+
+						// Rate limit, to avoid repeated calls to calculateDirectorySize()
+						Thread.sleep(60000);
+						// Now delete some data at random
+						this.storageLimitReached(repository);
+					}
+
+					// Delete random data associated with name if we're over our storage limit for this name
+					// Use the DELETION_THRESHOLD, for the same reasons as above
+					for (String followedName : storageManager.followedNames()) {
+						if (isStopping) {
+							return;
+						}
+						if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) {
+							this.storageLimitReachedForName(repository, followedName);
+						}
+					}
+
+				} catch (DataException e) {
+					LOGGER.error("Repository issue when cleaning up arbitrary transaction data", e);
+				}
+			}
+		} catch (InterruptedException e) {
+			// Fall-through to exit thread...
+		}
+	}
+
+	public List<Path> findPathsWithNoAssociatedTransaction(Repository repository) {
+		List<Path> pathList = new ArrayList<>();
+
+		// Find all hosted paths
+		List<Path> allPaths = ArbitraryDataStorageManager.getInstance().findAllHostedPaths();
+
+		// Loop through each path and find those without matching signatures
+		for (Path path : allPaths) {
+			if (isStopping) {
+				break;
+			}
+			try {
+				String[] contents = path.toFile().list();
+				if (contents == null || contents.length == 0) {
+					// Ignore empty directories
+					continue;
+				}
+
+				String signature58 = path.getFileName().toString();
+				byte[] signature = Base58.decode(signature58);
+				TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+				if (transactionData == null) {
+					// No transaction data, and no DataException, so we can assume that this data relates to an expired transaction
+					pathList.add(path);
+				}
+
+			} catch (DataException e) {
+				continue;
+			}
+		}
+
+		return pathList;
+	}
+
+	private void checkForExpiredTransactions(Repository repository) {
+		List<Path> expiredPaths = this.findPathsWithNoAssociatedTransaction(repository);
+		for (Path expiredPath : expiredPaths) {
+			if (isStopping) {
+				return;
+			}
+			LOGGER.info("Found path with no associated transaction: {}", expiredPath.toString());
+			this.safeDeleteDirectory(expiredPath.toFile(), "no matching transaction");
+		}
+	}
+
+	private void storageLimitReached(Repository repository) throws InterruptedException {
+		// We think that the storage limit has been reached
+
+		// Now calculate the used/total storage again, as a safety precaution
+		Long now = NTP.getTime();
+		ArbitraryDataStorageManager.getInstance().calculateDirectorySize(now);
+		if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailable(DELETION_THRESHOLD)) {
+			// We have space available, so don't delete anything
+			return;
+		}
+
+		// Delete a batch of random chunks
+		// This reduces the chance of too many nodes deleting the same chunk
+		// when they reach their storage limit
+		Path dataPath = Paths.get(Settings.getInstance().getDataPath());
+		for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
+			if (isStopping) {
+				return;
+			}
+			this.deleteRandomFile(repository, dataPath.toFile(), null);
+		}
+
+		// FUTURE: consider reducing the expiry time of the reader cache
+	}
+
+	public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException {
+		// We think that the storage limit has been reached for supplied name - but we should double check
+		if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) {
+			// We have space available for this name, so don't delete anything
+			return;
+		}
+
+		// Delete a batch of random chunks associated with this name
+		// This reduces the chance of too many nodes deleting the same chunk
+		// when they reach their storage limit
+		Path dataPath = Paths.get(Settings.getInstance().getDataPath());
+		for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
+			if (isStopping) {
+				return;
+			}
+			this.deleteRandomFile(repository, dataPath.toFile(), name);
+		}
+	}
+
+	/**
+	 * Iteratively walk through given directory and delete a single random file
+	 *
+	 * @param directory - the base directory
+	 * @return boolean - whether a file was deleted
+	 */
+	private boolean deleteRandomFile(Repository repository, File directory, String name) {
+		Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath());
+
+		// Pick a random directory
+		final File[] contentsList = directory.listFiles();
+		if (contentsList != null) {
+			SecureRandom random = new SecureRandom();
+
+			// If the directory is empty, there's nothing to do
+			if (contentsList.length == 0) {
+				return false;
+			}
+
+			File randomItem = contentsList[random.nextInt(contentsList.length)];
+
+			// Skip anything relating to the temp directory
+			if (FilesystemUtils.isChild(randomItem.toPath(), tempDataPath)) {
+				return false;
+			}
+			// Make sure it exists
+			if (!randomItem.exists()) {
+				return false;
+			}
+
+			// If it's a directory, iteratively repeat the process
+			if (randomItem.isDirectory()) {
+				return this.deleteRandomFile(repository, randomItem, name);
+			}
+
+			// If it's a file, we might be able to delete it
+			if (randomItem.isFile()) {
+
+				// If the parent directory contains an ".original" file, don't delete anything
+				// This indicates that the content was originally updated by this node and so
+				// could be the only copy that exists.
+				Path originalCopyIndicatorPath = Paths.get(randomItem.getParent(), ".original");
+				if (Files.exists(originalCopyIndicatorPath)) {
+					// This is an original seed copy and so shouldn't be deleted
+					return false;
+				}
+
+				if (name != null) {
+					// A name has been specified, so we need to make sure this file relates to
+					// the name we want to delete. The signature should be the name of parent directory.
+					try {
+						Path parentFileNamePath = randomItem.toPath().toAbsolutePath().getParent().getFileName();
+						if (parentFileNamePath != null) {
+							String signature58 = parentFileNamePath.toString();
+							byte[] signature = Base58.decode(signature58);
+							TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+							if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) {
+								// Not what we were expecting, so don't delete it
+								return false;
+							}
+							ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+							if (!Objects.equals(arbitraryTransactionData.getName(), name)) {
+								// Relates to a different name - don't delete it
+								return false;
+							}
+						}
+
+					} catch (DataException e) {
+						// Something went wrong and we weren't able to make a decision - so it's best not to delete this file
+						return false;
+					}
+				}
+
+				LOGGER.info("Deleting random file {} because we have reached max storage capacity...", randomItem.toString());
+				boolean success = randomItem.delete();
+				if (success) {
+					try {
+						FilesystemUtils.safeDeleteEmptyParentDirectories(randomItem.toPath().getParent());
+					} catch (IOException e) {
+						// Ignore cleanup failure
+					}
+				}
+				return success;
+			}
+		}
+		return false;
+	}
+
+	private void removePeersHostingTransactionData(Repository repository, ArbitraryTransactionData transactionData) {
+		byte[] signature = transactionData.getSignature();
+		try {
+			repository.getArbitraryRepository().deleteArbitraryPeersWithSignature(signature);
+			repository.saveChanges();
+		} catch (DataException e) {
+			LOGGER.debug("Unable to delete peers from lookup table for signature: {}", Base58.encode(signature));
+		}
+	}
+
+	private void cleanupTempDirectory(String folder, long now, long minAge) {
+		String baseDir = Settings.getInstance().getTempDataPath();
+		Path tempDir = Paths.get(baseDir, folder);
+		int contentsCount = 0;
+
+		// Loop through the contents and check each one
+		final File[] directories = tempDir.toFile().listFiles();
+		if (directories != null) {
+			for (final File directory : directories) {
+				if (isStopping) {
+					return;
+				}
+				contentsCount++;
+
+				// We're expecting the contents of each subfolder to be a directory
+				if (directory.isDirectory()) {
+					if (!ArbitraryTransactionUtils.isFileRecent(directory.toPath(), now, minAge)) {
+						// File isn't recent, so can be deleted
+						this.safeDeleteDirectory(directory, "not recent");
+					}
+				}
+			}
+		}
+
+		// If the directory is empty, we still need to delete its parent folder
+		if (contentsCount == 0 && tempDir.toFile().isDirectory() && tempDir.toFile().exists()) {
+			try {
+				LOGGER.debug("Parent directory {} is empty, so deleting it", tempDir);
+				FilesystemUtils.safeDeleteDirectory(tempDir, false);
+			} catch(IOException e){
+				LOGGER.info("Unable to delete parent directory: {}", tempDir);
+			}
+		}
+	}
+
+	private void cleanupReaderCache(Long now) {
+		ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
+		String baseDir = Settings.getInstance().getTempDataPath();
+		Path readerCachePath = Paths.get(baseDir, "reader");
+
+		// Clean up names
+		Path readerCacheNamesPath = Paths.get(readerCachePath.toString(), "NAME");
+
+		// Loop through the contents and check each one
+		final File[] directories = readerCacheNamesPath.toFile().listFiles();
+		if (directories != null) {
+			for (final File directory : directories) {
+				if (isStopping) {
+					return;
+				}
+
+				// Delete data relating to blocked names
+				String name = directory.getName();
+				if (name != null && storageManager.isNameBlocked(name)) {
+					this.safeDeleteDirectory(directory, "blocked name");
+				}
+
+				// Delete cached reader data that has reached its expiry
+				this.cleanupReaderCacheForName(name, now);
+			}
+		}
+	}
+
+	private void cleanupReaderCacheForName(String name, Long now) {
+		if (name == null) {
+			return;
+		}
+
+		String baseDir = Settings.getInstance().getTempDataPath();
+		Path readerNameCachePath = Paths.get(baseDir, "reader", "NAME", name);
+
+		// Loop through the contents and check each one
+		final File[] directories = readerNameCachePath.toFile().listFiles();
+		if (directories != null) {
+			for (final File directory : directories) {
+				if (isStopping) {
+					return;
+				}
+				// Each directory is a "service" type
+				String service = directory.getName();
+				this.cleanupReaderCacheForNameAndService(name, service, now);
+			}
+		}
+	}
+
+	private void cleanupReaderCacheForNameAndService(String name, String service, Long now) {
+		if (name == null || service == null) {
+			return;
+		}
+
+		Path readerNameServiceCachePath = Paths.get("reader", "NAME", name, service);
+		Long expiry = Settings.getInstance().getBuiltDataExpiryInterval();
+		this.cleanupTempDirectory(readerNameServiceCachePath.toString(), now, expiry);
+	}
+
+	private void cleanupTempDirectory(long now) {
+
+		// Use the "stale file timeout" for the intermediate directories.
+		// These aren't used for serving content - only for building it.
+		// Once the files have become stale, it's safe to delete them.
+		this.cleanupTempDirectory("diff",  now, STALE_FILE_TIMEOUT);
+		this.cleanupTempDirectory("join",  now, STALE_FILE_TIMEOUT);
+		this.cleanupTempDirectory("merge",  now, STALE_FILE_TIMEOUT);
+		this.cleanupTempDirectory("writer",  now, STALE_FILE_TIMEOUT);
+
+		// Built resources are served out of the "reader" directory so these
+		// need to be kept around for much longer.
+		// Purging currently disabled, as it's not very helpful. Will revisit
+		// once we implement local storage limits.
+		this.cleanupReaderCache(now);
+
+	}
+
+	private boolean safeDeleteDirectory(File directory, String reason) {
+		LOGGER.info("Deleting directory {} due to reason: {}", directory, reason);
+		try {
+			FilesystemUtils.safeDeleteDirectory(directory.toPath(), true);
+			return true;
+		} catch (IOException e) {
+			LOGGER.debug("Unable to delete directory: {}", directory);
+		}
+		return false;
+	}
+
+
+	public void shutdown() {
+		isStopping = true;
+		this.interrupt();
+	}
+
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java

@@ -0,0 +1,636 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.arbitrary.ArbitraryDataFileChunk;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.ArbitraryRelayInfo;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.network.Network;
+import org.qortal.network.Peer;
+import org.qortal.network.message.ArbitraryDataFileListMessage;
+import org.qortal.network.message.GetArbitraryDataFileListMessage;
+import org.qortal.network.message.Message;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+import org.qortal.utils.Triple;
+
+import java.util.*;
+
+public class ArbitraryDataFileListManager {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileListManager.class);
+
+    private static ArbitraryDataFileListManager instance;
+
+
+    /**
+     * Map of recent incoming requests for ARBITRARY transaction data file lists.
+     * <p>
+     * Key is original request's message ID<br>
+     * Value is Triple&lt;transaction signature in base58, first requesting peer, first request's timestamp&gt;
+     * <p>
+     * If peer is null then either:<br>
+     * <ul>
+     * <li>we are the original requesting peer</li>
+     * <li>we have already sent data payload to original requesting peer.</li>
+     * </ul>
+     * If signature is null then we have already received the file list and either:<br>
+     * <ul>
+     * <li>we are the original requesting peer and have processed it</li>
+     * <li>we have forwarded the file list</li>
+     * </ul>
+     */
+    public Map<Integer, Triple<String, Peer, Long>> arbitraryDataFileListRequests = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of in progress arbitrary data signature requests
+     * Key: string - the signature encoded in base58
+     * Value: Triple<networkBroadcastCount, directPeerRequestCount, lastAttemptTimestamp>
+     */
+    private Map<String, Triple<Integer, Integer, Long>> arbitraryDataSignatureRequests = Collections.synchronizedMap(new HashMap<>());
+
+
+    /** Maximum number of seconds that a file list relay request is able to exist on the network */
+    private static long RELAY_REQUEST_MAX_DURATION = 5000L;
+    /** Maximum number of hops that a file list relay request is allowed to make */
+    private static int RELAY_REQUEST_MAX_HOPS = 4;
+
+
+    private ArbitraryDataFileListManager() {
+    }
+
+    public static ArbitraryDataFileListManager getInstance() {
+        if (instance == null)
+            instance = new ArbitraryDataFileListManager();
+
+        return instance;
+    }
+
+
+    public void cleanupRequestCache(Long now) {
+        if (now == null) {
+            return;
+        }
+        final long requestMinimumTimestamp = now - ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT;
+        arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < requestMinimumTimestamp);
+    }
+
+
+    // Track file list lookups by signature
+
+    private boolean shouldMakeFileListRequestForSignature(String signature58) {
+        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        // Extract the components
+        Integer networkBroadcastCount = request.getA();
+        // Integer directPeerRequestCount = request.getB();
+        Long lastAttemptTimestamp = request.getC();
+
+        if (lastAttemptTimestamp == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
+
+        // Allow a second attempt after 15 seconds, and another after 30 seconds
+        if (timeSinceLastAttempt > 15 * 1000L) {
+            // We haven't tried for at least 15 seconds
+
+            if (networkBroadcastCount < 3) {
+                // We've made less than 3 total attempts
+                return true;
+            }
+        }
+
+        // Then allow another 5 attempts, each 5 minutes apart
+        if (timeSinceLastAttempt > 5 * 60 * 1000L) {
+            // We haven't tried for at least 5 minutes
+
+            if (networkBroadcastCount < 5) {
+                // We've made less than 5 total attempts
+                return true;
+            }
+        }
+
+        // From then on, only try once every 24 hours, to reduce network spam
+        if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) {
+            // We haven't tried for at least 24 hours
+            return true;
+        }
+
+        return false;
+    }
+
+    private boolean shouldMakeDirectFileRequestsForSignature(String signature58) {
+        if (!Settings.getInstance().isDirectDataRetrievalEnabled()) {
+            // Direct connections are disabled in the settings
+            return false;
+        }
+
+        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        // Extract the components
+        //Integer networkBroadcastCount = request.getA();
+        Integer directPeerRequestCount = request.getB();
+        Long lastAttemptTimestamp = request.getC();
+
+        if (lastAttemptTimestamp == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        if (directPeerRequestCount == 0) {
+            // We haven't tried asking peers directly yet, so we should
+            return true;
+        }
+
+        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
+        if (timeSinceLastAttempt > 10 * 1000L) {
+            // We haven't tried for at least 10 seconds
+            if (directPeerRequestCount < 5) {
+                // We've made less than 5 total attempts
+                return true;
+            }
+        }
+
+        if (timeSinceLastAttempt > 5 * 60 * 1000L) {
+            // We haven't tried for at least 5 minutes
+            if (directPeerRequestCount < 10) {
+                // We've made less than 10 total attempts
+                return true;
+            }
+        }
+
+        if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) {
+            // We haven't tried for at least 24 hours
+            return true;
+        }
+
+        return false;
+    }
+
+    public boolean isSignatureRateLimited(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        return !this.shouldMakeFileListRequestForSignature(signature58)
+                && !this.shouldMakeDirectFileRequestsForSignature(signature58);
+    }
+
+    public long lastRequestForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return 0;
+        }
+
+        // Extract the components
+        Long lastAttemptTimestamp = request.getC();
+        if (lastAttemptTimestamp != null) {
+            return  lastAttemptTimestamp;
+        }
+        return 0;
+    }
+
+    public void addToSignatureRequests(String signature58, boolean incrementNetworkRequests, boolean incrementPeerRequests) {
+        Triple<Integer, Integer, Long> request  = arbitraryDataSignatureRequests.get(signature58);
+        Long now = NTP.getTime();
+
+        if (request == null) {
+            // No entry yet
+            Triple<Integer, Integer, Long> newRequest = new Triple<>(0, 0, now);
+            arbitraryDataSignatureRequests.put(signature58, newRequest);
+        }
+        else {
+            // There is an existing entry
+            if (incrementNetworkRequests) {
+                request.setA(request.getA() + 1);
+            }
+            if (incrementPeerRequests) {
+                request.setB(request.getB() + 1);
+            }
+            request.setC(now);
+            arbitraryDataSignatureRequests.put(signature58, request);
+        }
+    }
+
+    public void removeFromSignatureRequests(String signature58) {
+        arbitraryDataSignatureRequests.remove(signature58);
+    }
+
+
+    // Lookup file lists by signature (and optionally hashes)
+
+    public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) {
+        byte[] digest = arbitraryTransactionData.getData();
+        byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
+        byte[] signature = arbitraryTransactionData.getSignature();
+        String signature58 = Base58.encode(signature);
+
+        // Require an NTP sync
+        Long now = NTP.getTime();
+        if (now == null) {
+            return false;
+        }
+
+        // If we've already tried too many times in a short space of time, make sure to give up
+        if (!this.shouldMakeFileListRequestForSignature(signature58)) {
+            // Check if we should make direct connections to peers
+            if (this.shouldMakeDirectFileRequestsForSignature(signature58)) {
+                return ArbitraryDataFileManager.getInstance().fetchDataFilesFromPeersForSignature(signature);
+            }
+
+            LOGGER.trace("Skipping file list request for signature {} due to rate limit", signature58);
+            return false;
+        }
+        this.addToSignatureRequests(signature58, true, false);
+
+        List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
+        List<byte[]> missingHashes = null;
+
+//        // TODO: uncomment after GetArbitraryDataFileListMessage updates are deployed
+//        // Find hashes that we are missing
+//        try {
+//            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
+//            arbitraryDataFile.setMetadataHash(metadataHash);
+//            missingHashes = arbitraryDataFile.missingHashes();
+//        } catch (DataException e) {
+//            // Leave missingHashes as null, so that all hashes are requested
+//        }
+//        int hashCount = missingHashes != null ? missingHashes.size() : 0;
+
+        int hashCount = 0;
+        LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size()));
+
+        // Build request
+        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, missingHashes, now, 0);
+
+        // Save our request into requests map
+        Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
+
+        // Assign random ID to this message
+        int id;
+        do {
+            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
+
+            // Put queue into map (keyed by message ID) so we can poll for a response
+            // If putIfAbsent() doesn't return null, then this ID is already taken
+        } while (arbitraryDataFileListRequests.put(id, requestEntry) != null);
+        getArbitraryDataFileListMessage.setId(id);
+
+        // Broadcast request
+        Network.getInstance().broadcast(peer -> getArbitraryDataFileListMessage);
+
+        // Poll to see if data has arrived
+        final long singleWait = 100;
+        long totalWait = 0;
+        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
+            try {
+                Thread.sleep(singleWait);
+            } catch (InterruptedException e) {
+                break;
+            }
+
+            requestEntry = arbitraryDataFileListRequests.get(id);
+            if (requestEntry == null)
+                return false;
+
+            if (requestEntry.getA() == null)
+                break;
+
+            totalWait += singleWait;
+        }
+        return true;
+    }
+
+    public boolean fetchArbitraryDataFileList(Peer peer, byte[] signature) {
+        String signature58 = Base58.encode(signature);
+
+        // Require an NTP sync
+        Long now = NTP.getTime();
+        if (now == null) {
+            return false;
+        }
+
+        int hashCount = 0;
+        LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to peer %s...", signature58, hashCount, peer));
+
+        // Build request
+        // Use a time in the past, so that the recipient peer doesn't try and relay it
+        // Also, set hashes to null since it's easier to request all hashes than it is to determine which ones we need
+        // This could be optimized in the future
+        long timestamp = now - 60000L;
+        List<byte[]> hashes = null;
+        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, timestamp, 0);
+
+        // Save our request into requests map
+        Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
+
+        // Assign random ID to this message
+        int id;
+        do {
+            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
+
+            // Put queue into map (keyed by message ID) so we can poll for a response
+            // If putIfAbsent() doesn't return null, then this ID is already taken
+        } while (arbitraryDataFileListRequests.put(id, requestEntry) != null);
+        getArbitraryDataFileListMessage.setId(id);
+
+        // Send the request
+        peer.sendMessage(getArbitraryDataFileListMessage);
+
+        // Poll to see if data has arrived
+        final long singleWait = 100;
+        long totalWait = 0;
+        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
+            try {
+                Thread.sleep(singleWait);
+            } catch (InterruptedException e) {
+                break;
+            }
+
+            requestEntry = arbitraryDataFileListRequests.get(id);
+            if (requestEntry == null)
+                return false;
+
+            if (requestEntry.getA() == null)
+                break;
+
+            totalWait += singleWait;
+        }
+        return true;
+    }
+
+    public void deleteFileListRequestsForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        for (Iterator<Map.Entry<Integer, Triple<String, Peer, Long>>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) {
+            Map.Entry<Integer, Triple<String, Peer, Long>> entry = it.next();
+            if (entry == null || entry.getKey() == null || entry.getValue() != null) {
+                continue;
+            }
+            if (Objects.equals(entry.getValue().getA(), signature58)) {
+                // Update requests map to reflect that we've received all chunks
+                Triple<String, Peer, Long> newEntry = new Triple<>(null, null, entry.getValue().getC());
+                arbitraryDataFileListRequests.put(entry.getKey(), newEntry);
+            }
+        }
+    }
+
+    // Network handlers
+
+    public void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) {
+        // Don't process if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
+        LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
+
+        // Do we have a pending request for this data?
+        Triple<String, Peer, Long> request = arbitraryDataFileListRequests.get(message.getId());
+        if (request == null || request.getA() == null) {
+            return;
+        }
+        boolean isRelayRequest = (request.getB() != null);
+
+        // Does this message's signature match what we're expecting?
+        byte[] signature = arbitraryDataFileListMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        if (!request.getA().equals(signature58)) {
+            return;
+        }
+
+        List<byte[]> hashes = arbitraryDataFileListMessage.getHashes();
+        if (hashes == null || hashes.isEmpty()) {
+            return;
+        }
+
+        ArbitraryTransactionData arbitraryTransactionData = null;
+        ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance();
+
+        // Check transaction exists and hashes are correct
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+            if (!(transactionData instanceof ArbitraryTransactionData))
+                return;
+
+            arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+
+            // Load data file(s)
+            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature);
+            arbitraryDataFile.setMetadataHash(arbitraryTransactionData.getMetadataHash());
+
+//			// Check all hashes exist
+//			for (byte[] hash : hashes) {
+//				//LOGGER.debug("Received hash {}", Base58.encode(hash));
+//				if (!arbitraryDataFile.containsChunk(hash)) {
+//					// Check the hash against the complete file
+//					if (!Arrays.equals(arbitraryDataFile.getHash(), hash)) {
+//						LOGGER.info("Received non-matching chunk hash {} for signature {}. This could happen if we haven't obtained the metadata file yet.", Base58.encode(hash), signature58);
+//						return;
+//					}
+//				}
+//			}
+
+            if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) {
+                // Keep track of the hashes this peer reports to have access to
+                Long now = NTP.getTime();
+                for (byte[] hash : hashes) {
+                    String hash58 = Base58.encode(hash);
+                    String sig58 = Base58.encode(signature);
+                    ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.put(hash58, new Triple<>(peer, sig58, now));
+                }
+
+                // Go and fetch the actual data, since this isn't a relay request
+                arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, hashes);
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e);
+        }
+
+        // Forwarding
+        if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
+            boolean isBlocked = (arbitraryTransactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(arbitraryTransactionData.getName()));
+            if (!isBlocked) {
+                Peer requestingPeer = request.getB();
+                if (requestingPeer != null) {
+                    // Add each hash to our local mapping so we know who to ask later
+                    Long now = NTP.getTime();
+                    for (byte[] hash : hashes) {
+                        String hash58 = Base58.encode(hash);
+                        ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now);
+                        ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap);
+                    }
+
+                    // Forward to requesting peer
+                    LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer);
+                    if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) {
+                        requestingPeer.disconnect("failed to forward arbitrary data file list");
+                    }
+                }
+            }
+        }
+    }
+
+    public void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) {
+        // Don't respond if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet();
+
+        GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message;
+        byte[] signature = getArbitraryDataFileListMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        List<byte[]> requestedHashes = getArbitraryDataFileListMessage.getHashes();
+        Long now = NTP.getTime();
+        Triple<String, Peer, Long> newEntry = new Triple<>(signature58, peer, now);
+
+        // If we've seen this request recently, then ignore
+        if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) {
+            LOGGER.debug("Ignoring hash list request from peer {} for signature {}", peer, signature58);
+            return;
+        }
+
+        LOGGER.debug("Received hash list request from peer {} for signature {}", peer, signature58);
+
+        List<byte[]> hashes = new ArrayList<>();
+        ArbitraryTransactionData transactionData = null;
+        boolean allChunksExist = false;
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Firstly we need to lookup this file on chain to get a list of its hashes
+            transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature);
+            if (transactionData instanceof ArbitraryTransactionData) {
+
+                // Check if we're even allowed to serve data for this transaction
+                if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
+
+                    byte[] hash = transactionData.getData();
+                    byte[] metadataHash = transactionData.getMetadataHash();
+
+                    // Load file(s) and add any that exist to the list of hashes
+                    ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
+                    arbitraryDataFile.setMetadataHash(metadataHash);
+
+                    // If the peer didn't supply a hash list, we need to return all hashes for this transaction
+                    if (requestedHashes == null || requestedHashes.isEmpty()) {
+                        requestedHashes = new ArrayList<>();
+
+                        // Add the metadata file
+                        if (arbitraryDataFile.getMetadataHash() != null) {
+                            requestedHashes.add(arbitraryDataFile.getMetadataHash());
+                        }
+
+                        // Add the chunk hashes
+                        if (arbitraryDataFile.getChunkHashes().size() > 0) {
+                            requestedHashes.addAll(arbitraryDataFile.getChunkHashes());
+                        }
+                        // Add complete file if there are no hashes
+                        else {
+                            requestedHashes.add(arbitraryDataFile.getHash());
+                        }
+                    }
+
+                    // Assume all chunks exists, unless one can't be found below
+                    allChunksExist = true;
+
+                    for (byte[] requestedHash : requestedHashes) {
+                        ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature);
+                        if (chunk.exists()) {
+                            hashes.add(chunk.getHash());
+                            //LOGGER.trace("Added hash {}", chunk.getHash58());
+                        } else {
+                            LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58());
+                            allChunksExist = false;
+                        }
+                    }
+                }
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e);
+        }
+
+        // We should only respond if we have at least one hash
+        if (hashes.size() > 0) {
+
+            // We have all the chunks, so update requests map to reflect that we've sent it
+            // There is no need to keep track of the request, as we can serve all the chunks
+            if (allChunksExist) {
+                newEntry = new Triple<>(null, null, now);
+                arbitraryDataFileListRequests.put(message.getId(), newEntry);
+            }
+
+            ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+            arbitraryDataFileListMessage.setId(message.getId());
+            if (!peer.sendMessage(arbitraryDataFileListMessage)) {
+                LOGGER.debug("Couldn't send list of hashes");
+                peer.disconnect("failed to send list of hashes");
+                return;
+            }
+            LOGGER.debug("Sent list of hashes (count: {})", hashes.size());
+
+            if (allChunksExist) {
+                // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
+                LOGGER.debug("No need for any forwarding because file list request is fully served");
+                return;
+            }
+
+        }
+
+        // We may need to forward this request on
+        boolean isBlocked = (transactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName()));
+        if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
+            // In relay mode - so ask our other peers if they have it
+
+            long requestTime = getArbitraryDataFileListMessage.getRequestTime();
+            int requestHops = getArbitraryDataFileListMessage.getRequestHops();
+            getArbitraryDataFileListMessage.setRequestHops(++requestHops);
+            long totalRequestTime = now - requestTime;
+
+            if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
+                // Relay request hasn't timed out yet, so can potentially be rebroadcast
+                if (requestHops < RELAY_REQUEST_MAX_HOPS) {
+                    // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
+
+                    LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
+                    Network.getInstance().broadcast(
+                            broadcastPeer -> broadcastPeer == peer ||
+                                    Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost())
+                                    ? null : getArbitraryDataFileListMessage);
+
+                }
+                else {
+                    // This relay request has reached the maximum number of allowed hops
+                }
+            }
+            else {
+                // This relay request has timed out
+            }
+        }
+    }
+
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java

@@ -0,0 +1,509 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.ArbitraryRelayInfo;
+import org.qortal.data.network.ArbitraryPeerData;
+import org.qortal.data.network.PeerData;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.network.Network;
+import org.qortal.network.Peer;
+import org.qortal.network.message.*;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+import org.qortal.utils.Triple;
+
+import java.security.SecureRandom;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
+
+public class ArbitraryDataFileManager extends Thread {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileManager.class);
+
+    private static ArbitraryDataFileManager instance;
+    private volatile boolean isStopping = false;
+
+
+    /**
+     * Map to keep track of our in progress (outgoing) arbitrary data file requests
+     */
+    private Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of hashes that we might need to relay
+     */
+    public List<ArbitraryRelayInfo> arbitraryRelayMap = Collections.synchronizedList(new ArrayList<>());
+
+    /**
+     * Map to keep track of any arbitrary data file hash responses
+     * Key: string - the hash encoded in base58
+     * Value: Triple<respondingPeer, signature58, timeResponded>
+     */
+    public Map<String, Triple<Peer, String, Long>> arbitraryDataFileHashResponses = Collections.synchronizedMap(new HashMap<>());
+
+
+    private ArbitraryDataFileManager() {
+    }
+
+    public static ArbitraryDataFileManager getInstance() {
+        if (instance == null)
+            instance = new ArbitraryDataFileManager();
+
+        return instance;
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data File Manager");
+
+        try {
+            // Use a fixed thread pool to execute the arbitrary data file requests
+            int threadCount = 10;
+            ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount);
+            for (int i = 0; i < threadCount; i++) {
+                arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread());
+            }
+
+            while (!isStopping) {
+                // Nothing to do yet
+                Thread.sleep(1000);
+            }
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
+        }
+    }
+
+    public void shutdown() {
+        isStopping = true;
+        this.interrupt();
+    }
+
+
+    public void cleanupRequestCache(Long now) {
+        if (now == null) {
+            return;
+        }
+        final long requestMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_REQUEST_TIMEOUT;
+        arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < requestMinimumTimestamp);
+
+        final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT;
+        arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp);
+        arbitraryDataFileHashResponses.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp);
+    }
+
+
+
+    // Fetch data files by hash
+
+    public boolean fetchArbitraryDataFiles(Repository repository,
+                                           Peer peer,
+                                           byte[] signature,
+                                           ArbitraryTransactionData arbitraryTransactionData,
+                                           List<byte[]> hashes) throws DataException {
+
+        // Load data file(s)
+        ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature);
+        byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
+        arbitraryDataFile.setMetadataHash(metadataHash);
+        boolean receivedAtLeastOneFile = false;
+
+        // Now fetch actual data from this peer
+        for (byte[] hash : hashes) {
+            if (isStopping) {
+                return false;
+            }
+            String hash58 = Base58.encode(hash);
+            if (!arbitraryDataFile.chunkExists(hash)) {
+                // Only request the file if we aren't already requesting it from someone else
+                if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
+                    LOGGER.debug("Requesting data file {} from peer {}", hash58, peer);
+                    Long startTime = NTP.getTime();
+                    ArbitraryDataFileMessage receivedArbitraryDataFileMessage = fetchArbitraryDataFile(peer, null, signature, hash, null);
+                    Long endTime = NTP.getTime();
+                    if (receivedArbitraryDataFileMessage != null) {
+                        LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFileMessage.getArbitraryDataFile().getHash58(), peer, (endTime-startTime));
+                        receivedAtLeastOneFile = true;
+
+                        // Remove this hash from arbitraryDataFileHashResponses now that we have received it
+                        arbitraryDataFileHashResponses.remove(hash58);
+                    }
+                    else {
+                        LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime));
+
+                        // Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it
+                        arbitraryDataFileHashResponses.remove(hash58);
+
+                        // Stop asking for files from this peer
+                        break;
+                    }
+                }
+                else {
+                    LOGGER.trace("Already requesting data file {} for signature {}", arbitraryDataFile, Base58.encode(signature));
+                }
+            }
+            else {
+                // Remove this hash from arbitraryDataFileHashResponses because we have a local copy
+                arbitraryDataFileHashResponses.remove(hash58);
+            }
+        }
+
+        if (receivedAtLeastOneFile) {
+            // Update our lookup table to indicate that this peer holds data for this signature
+            String peerAddress = peer.getPeerData().getAddress().toString();
+            ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(signature, peer);
+            repository.discardChanges();
+            if (arbitraryPeerData.isPeerAddressValid()) {
+                LOGGER.debug("Adding arbitrary peer: {} for signature {}", peerAddress, Base58.encode(signature));
+                repository.getArbitraryRepository().save(arbitraryPeerData);
+                repository.saveChanges();
+            }
+
+            // Invalidate the hosted transactions cache as we are now hosting something new
+            ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache();
+
+            // Check if we have all the files we need for this transaction
+            if (arbitraryDataFile.allFilesExist()) {
+
+                // We have all the chunks for this transaction, so we should invalidate the transaction's name's
+                // data cache so that it is rebuilt the next time we serve it
+                ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData);
+
+                // We may also need to broadcast to the network that we are now hosting files for this transaction,
+                // but only if these files are in accordance with our storage policy
+                if (ArbitraryDataStorageManager.getInstance().canStoreData(arbitraryTransactionData)) {
+                    // Use a null peer address to indicate our own
+                    Message newArbitrarySignatureMessage = new ArbitrarySignaturesMessage(null, 0, Arrays.asList(signature));
+                    Network.getInstance().broadcast(broadcastPeer -> newArbitrarySignatureMessage);
+                }
+            }
+
+        }
+
+        return receivedAtLeastOneFile;
+    }
+
+    private ArbitraryDataFileMessage fetchArbitraryDataFile(Peer peer, Peer requestingPeer, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
+        ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature);
+        boolean fileAlreadyExists = existingFile.exists();
+        String hash58 = Base58.encode(hash);
+        Message message = null;
+
+        // Fetch the file if it doesn't exist locally
+        if (!fileAlreadyExists) {
+            LOGGER.debug(String.format("Fetching data file %.8s from peer %s", hash58, peer));
+            arbitraryDataFileRequests.put(hash58, NTP.getTime());
+            Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash);
+
+            try {
+                message = peer.getResponseWithTimeout(getArbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT);
+            } catch (InterruptedException e) {
+                // Will return below due to null message
+            }
+            arbitraryDataFileRequests.remove(hash58);
+            LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58));
+
+            // We may need to remove the file list request, if we have all the files for this transaction
+            this.handleFileListRequests(signature);
+
+            if (message == null) {
+                LOGGER.debug("Received null message from peer {}", peer);
+                return null;
+            }
+            if (message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) {
+                LOGGER.debug("Received message with invalid type: {} from peer {}", message.getType(), peer);
+                return null;
+            }
+        }
+        else {
+            LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58));
+        }
+        ArbitraryDataFileMessage arbitraryDataFileMessage = (ArbitraryDataFileMessage) message;
+
+        // We might want to forward the request to the peer that originally requested it
+        this.handleArbitraryDataFileForwarding(requestingPeer, message, originalMessage);
+
+        boolean isRelayRequest = (requestingPeer != null);
+        if (isRelayRequest) {
+            if (!fileAlreadyExists) {
+                // File didn't exist locally before the request, and it's a forwarding request, so delete it
+                LOGGER.debug("Deleting file {} because it was needed for forwarding only", Base58.encode(hash));
+                ArbitraryDataFile dataFile = arbitraryDataFileMessage.getArbitraryDataFile();
+
+                // Keep trying to delete the data until it is deleted, or we reach 10 attempts
+                for (int i=0; i<10; i++) {
+                    if (dataFile.delete()) {
+                        break;
+                    }
+                    try {
+                        Thread.sleep(1000L);
+                    } catch (InterruptedException e) {
+                        // Fall through to exit method
+                    }
+                }
+            }
+        }
+
+        return arbitraryDataFileMessage;
+    }
+
+    private void handleFileListRequests(byte[] signature) {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Fetch the transaction data
+            ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+            if (arbitraryTransactionData == null) {
+                return;
+            }
+
+            boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData);
+
+            if (allChunksExist) {
+                // Update requests map to reflect that we've received all chunks
+                ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature);
+            }
+
+        } catch (DataException e) {
+            LOGGER.debug("Unable to handle file list requests: {}", e.getMessage());
+        }
+    }
+
+    public void handleArbitraryDataFileForwarding(Peer requestingPeer, Message message, Message originalMessage) {
+        // Return if there is no originally requesting peer to forward to
+        if (requestingPeer == null) {
+            return;
+        }
+
+        // Return if we're not in relay mode or if this request doesn't need forwarding
+        if (!Settings.getInstance().isRelayModeEnabled()) {
+            return;
+        }
+
+        LOGGER.debug("Received arbitrary data file - forwarding is needed");
+
+        // The ID needs to match that of the original request
+        message.setId(originalMessage.getId());
+
+        if (!requestingPeer.sendMessage(message)) {
+            LOGGER.debug("Failed to forward arbitrary data file to peer {}", requestingPeer);
+            requestingPeer.disconnect("failed to forward arbitrary data file");
+        }
+        else {
+            LOGGER.debug("Forwarded arbitrary data file to peer {}", requestingPeer);
+        }
+    }
+
+
+    // Fetch data directly from peers
+
+    public boolean fetchDataFilesFromPeersForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        ArbitraryDataFileListManager.getInstance().addToSignatureRequests(signature58, false, true);
+
+        // Firstly fetch peers that claim to be hosting files for this signature
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            List<ArbitraryPeerData> peers = repository.getArbitraryRepository().getArbitraryPeerDataForSignature(signature);
+            if (peers == null || peers.isEmpty()) {
+                LOGGER.debug("No peers found for signature {}", signature58);
+                return false;
+            }
+
+            LOGGER.debug("Attempting a direct peer connection for signature {}...", signature58);
+
+            // Peers found, so pick a random one and request data from it
+            int index = new SecureRandom().nextInt(peers.size());
+            ArbitraryPeerData arbitraryPeerData = peers.get(index);
+            String peerAddressString = arbitraryPeerData.getPeerAddress();
+            boolean success = Network.getInstance().requestDataFromPeer(peerAddressString, signature);
+
+            // Parse the peer address to find the host and port
+            String host = null;
+            int port = -1;
+            String[] parts = peerAddressString.split(":");
+            if (parts.length > 1) {
+                host = parts[0];
+                port = Integer.parseInt(parts[1]);
+            }
+
+            // If unsuccessful, and using a non-standard port, try a second connection with the default listen port,
+            // since almost all nodes use that. This is a workaround to account for any ephemeral ports that may
+            // have made it into the dataset.
+            if (!success) {
+                if (host != null && port > 0) {
+                    int defaultPort = Settings.getInstance().getDefaultListenPort();
+                    if (port != defaultPort) {
+                        String newPeerAddressString = String.format("%s:%d", host, defaultPort);
+                        success = Network.getInstance().requestDataFromPeer(newPeerAddressString, signature);
+                    }
+                }
+            }
+
+            // If _still_ unsuccessful, try matching the peer's IP address with some known peers, and then connect
+            // to each of those in turn until one succeeds.
+            if (!success) {
+                if (host != null) {
+                    final String finalHost = host;
+                    List<PeerData> knownPeers = Network.getInstance().getAllKnownPeers().stream()
+                            .filter(knownPeerData -> knownPeerData.getAddress().getHost().equals(finalHost))
+                            .collect(Collectors.toList());
+                    // Loop through each match and attempt a connection
+                    for (PeerData matchingPeer : knownPeers) {
+                        String matchingPeerAddress = matchingPeer.getAddress().toString();
+                        success = Network.getInstance().requestDataFromPeer(matchingPeerAddress, signature);
+                        if (success) {
+                            // Successfully connected, so stop making connections
+                            break;
+                        }
+                    }
+                }
+            }
+
+            // Keep track of the success or failure
+            arbitraryPeerData.markAsAttempted();
+            if (success) {
+                arbitraryPeerData.markAsRetrieved();
+                arbitraryPeerData.incrementSuccesses();
+            }
+            else {
+                arbitraryPeerData.incrementFailures();
+            }
+            repository.discardChanges();
+            repository.getArbitraryRepository().save(arbitraryPeerData);
+            repository.saveChanges();
+
+            return success;
+
+        } catch (DataException e) {
+            LOGGER.debug("Unable to fetch peer list from repository");
+        }
+
+        return false;
+    }
+
+
+    // Relays
+
+    private List<ArbitraryRelayInfo> getRelayInfoListForHash(String hash58) {
+        synchronized (arbitraryRelayMap) {
+            return arbitraryRelayMap.stream()
+                    .filter(relayInfo -> Objects.equals(relayInfo.getHash58(), hash58))
+                    .collect(Collectors.toList());
+        }
+    }
+
+    private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) {
+        LOGGER.info("Fetching random relay info for hash: {}", hash58);
+        List<ArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
+        if (relayInfoList != null && !relayInfoList.isEmpty()) {
+
+            // Pick random item
+            int index = new SecureRandom().nextInt(relayInfoList.size());
+            LOGGER.info("Returning random relay info for hash: {} (index {})", hash58, index);
+            return relayInfoList.get(index);
+        }
+        LOGGER.info("No relay info exists for hash: {}", hash58);
+        return null;
+    }
+
+    public void addToRelayMap(ArbitraryRelayInfo newEntry) {
+        if (newEntry == null || !newEntry.isValid()) {
+            return;
+        }
+
+        // Remove existing entry for this peer if it exists, to renew the timestamp
+        this.removeFromRelayMap(newEntry);
+
+        // Re-add
+        arbitraryRelayMap.add(newEntry);
+        LOGGER.debug("Added entry to relay map: {}", newEntry);
+    }
+
+    private void removeFromRelayMap(ArbitraryRelayInfo entry) {
+        arbitraryRelayMap.removeIf(relayInfo -> relayInfo.equals(entry));
+    }
+
+
+    // Network handlers
+
+    public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) {
+        // Don't respond if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message;
+        byte[] hash = getArbitraryDataFileMessage.getHash();
+        String hash58 = Base58.encode(hash);
+        byte[] signature = getArbitraryDataFileMessage.getSignature();
+        Controller.getInstance().stats.getArbitraryDataFileMessageStats.requests.incrementAndGet();
+
+        LOGGER.debug("Received GetArbitraryDataFileMessage from peer {} for hash {}", peer, Base58.encode(hash));
+
+        try {
+            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
+            ArbitraryRelayInfo relayInfo = this.getRandomRelayInfoEntryForHash(hash58);
+
+            if (arbitraryDataFile.exists()) {
+                LOGGER.trace("Hash {} exists", hash58);
+
+                // We can serve the file directly as we already have it
+                ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, arbitraryDataFile);
+                arbitraryDataFileMessage.setId(message.getId());
+                if (!peer.sendMessage(arbitraryDataFileMessage)) {
+                    LOGGER.debug("Couldn't sent file");
+                    peer.disconnect("failed to send file");
+                }
+                LOGGER.debug("Sent file {}", arbitraryDataFile);
+            }
+            else if (relayInfo != null) {
+                LOGGER.debug("We have relay info for hash {}", Base58.encode(hash));
+                // We need to ask this peer for the file
+                Peer peerToAsk = relayInfo.getPeer();
+                if (peerToAsk != null) {
+
+                    // Forward the message to this peer
+                    LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58);
+                    this.fetchArbitraryDataFile(peerToAsk, peer, signature, hash, message);
+                }
+                else {
+                    LOGGER.debug("Peer {} not found in relay info", peer);
+                }
+            }
+            else {
+                LOGGER.debug("Hash {} doesn't exist and we don't have relay info", hash58);
+
+                // We don't have this file
+                Controller.getInstance().stats.getArbitraryDataFileMessageStats.unknownFiles.getAndIncrement();
+
+                // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout
+                LOGGER.debug(String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile));
+
+                // We'll send empty block summaries message as it's very short
+                // TODO: use a different message type here
+                Message fileUnknownMessage = new BlockSummariesMessage(Collections.emptyList());
+                fileUnknownMessage.setId(message.getId());
+                if (!peer.sendMessage(fileUnknownMessage)) {
+                    LOGGER.debug("Couldn't sent file-unknown response");
+                    peer.disconnect("failed to send file-unknown response");
+                }
+                else {
+                    LOGGER.debug("Sent file-unknown response for file {}", arbitraryDataFile);
+                }
+            }
+        }
+        catch (DataException e) {
+            LOGGER.debug("Unable to handle request for arbitrary data file: {}", hash58);
+        }
+    }
+
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java

@@ -0,0 +1,121 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.network.Peer;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+import org.qortal.utils.Triple;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Map;
+
+public class ArbitraryDataFileRequestThread implements Runnable {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class);
+
+    public ArbitraryDataFileRequestThread() {
+
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data File Request Thread");
+
+        try {
+            while (!Controller.isStopping()) {
+                Thread.sleep(1000);
+
+                Long now = NTP.getTime();
+                this.processFileHashes(now);
+            }
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
+        }
+    }
+
+    private void processFileHashes(Long now) {
+        if (Controller.isStopping()) {
+            return;
+        }
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance();
+
+            ArbitraryTransactionData arbitraryTransactionData = null;
+            byte[] signature = null;
+            byte[] hash = null;
+            Peer peer = null;
+            boolean shouldProcess = false;
+
+            synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) {
+                Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator();
+                while (iterator.hasNext()) {
+                    if (Controller.isStopping()) {
+                        return;
+                    }
+
+                    Map.Entry entry = (Map.Entry) iterator.next();
+                    if (entry == null || entry.getKey() == null || entry.getValue() == null) {
+                        iterator.remove();
+                        continue;
+                    }
+
+                    String hash58 = (String) entry.getKey();
+                    Triple<Peer, String, Long> value = (Triple<Peer, String, Long>) entry.getValue();
+                    if (value == null) {
+                        iterator.remove();
+                        continue;
+                    }
+
+                    peer = value.getA();
+                    String signature58 = value.getB();
+                    Long timestamp = value.getC();
+
+                    if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) {
+                        // Ignore - to be deleted
+                        iterator.remove();
+                        continue;
+                    }
+
+                    hash = Base58.decode(hash58);
+                    signature = Base58.decode(signature58);
+
+                    // We want to process this file
+                    shouldProcess = true;
+                    iterator.remove();
+                    break;
+                }
+            }
+
+            if (!shouldProcess) {
+                // Nothing to do
+                return;
+            }
+
+            // Fetch the transaction data
+            arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+            if (arbitraryTransactionData == null) {
+                return;
+            }
+
+            if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) {
+                return;
+            }
+
+            String hash58 = Base58.encode(hash);
+            LOGGER.debug("Fetching file {} from peer {} via request thread...", hash58, peer);
+            arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash));
+
+        } catch (DataException e) {
+            LOGGER.debug("Unable to process file hashes: {}", e.getMessage());
+        }
+    }
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -0,0 +1,458 @@
+package org.qortal.controller.arbitrary;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.arbitrary.ArbitraryDataResource;
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.controller.Controller;
+import org.qortal.data.network.ArbitraryPeerData;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.list.ResourceListManager;
+import org.qortal.network.Network;
+import org.qortal.network.Peer;
+import org.qortal.network.message.*;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.transaction.ArbitraryTransaction;
+import org.qortal.transaction.Transaction.TransactionType;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+
+public class ArbitraryDataManager extends Thread {
+
+	private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataManager.class);
+	private static final List<TransactionType> ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY);
+
+	/** Difficulty (leading zero bits) used in arbitrary data transactions
+	 * Set here so that it can be more easily reduced when running unit tests */
+	private int powDifficulty = 14; // Must not be final, as unit tests need to reduce this value
+
+	/** Request timeout when transferring arbitrary data */
+	public static final long ARBITRARY_REQUEST_TIMEOUT = 10 * 1000L; // ms
+
+	/** Maximum time to hold information about an in-progress relay */
+	public static final long ARBITRARY_RELAY_TIMEOUT = 60 * 1000L; // ms
+
+	/** Maximum number of hops that an arbitrary signatures request is allowed to make */
+	private static int ARBITRARY_SIGNATURES_REQUEST_MAX_HOPS = 3;
+
+	private static ArbitraryDataManager instance;
+	private final Object peerDataLock = new Object();
+
+	private volatile boolean isStopping = false;
+
+	/**
+	 * Map to keep track of cached arbitrary transaction resources.
+	 * When an item is present in this list with a timestamp in the future, we won't invalidate
+	 * its cache when serving that data. This reduces the amount of database lookups that are needed.
+	 */
+	private Map<String, Long> arbitraryDataCachedResources = Collections.synchronizedMap(new HashMap<>());
+
+	/**
+	 * The amount of time to cache a data resource before it is invalidated
+	 */
+	private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes
+
+
+
+	private ArbitraryDataManager() {
+	}
+
+	public static ArbitraryDataManager getInstance() {
+		if (instance == null)
+			instance = new ArbitraryDataManager();
+
+		return instance;
+	}
+
+	@Override
+	public void run() {
+		Thread.currentThread().setName("Arbitrary Data Manager");
+
+		try {
+			while (!isStopping) {
+				Thread.sleep(2000);
+
+				// Don't run if QDN is disabled
+				if (!Settings.getInstance().isQdnEnabled()) {
+					Thread.sleep(60 * 60 * 1000L);
+					continue;
+				}
+
+				List<Peer> peers = Network.getInstance().getHandshakedPeers();
+
+				// Disregard peers that have "misbehaved" recently
+				peers.removeIf(Controller.hasMisbehaved);
+
+				// Don't fetch data if we don't have enough up-to-date peers
+				if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) {
+					continue;
+				}
+
+				// Fetch data according to storage policy
+				switch (Settings.getInstance().getStoragePolicy()) {
+					case FOLLOWED:
+					case FOLLOWED_OR_VIEWED:
+						this.processNames();
+						break;
+
+					case ALL:
+						this.processAll();
+
+					case NONE:
+					case VIEWED:
+					default:
+						// Nothing to fetch in advance
+						Thread.sleep(60000);
+						break;
+				}
+			}
+		} catch (InterruptedException e) {
+			// Fall-through to exit thread...
+		}
+	}
+
+	public void shutdown() {
+		isStopping = true;
+		this.interrupt();
+	}
+
+	private void processNames() {
+		// Fetch latest list of followed names
+		List<String> followedNames = ResourceListManager.getInstance().getStringsInList("followedNames");
+		if (followedNames == null || followedNames.isEmpty()) {
+			return;
+		}
+
+		// Loop through the names in the list and fetch transactions for each
+		for (String name : followedNames) {
+			this.fetchAndProcessTransactions(name);
+		}
+	}
+
+	private void processAll() {
+		this.fetchAndProcessTransactions(null);
+	}
+
+	private void fetchAndProcessTransactions(String name) {
+		ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
+
+		// Paginate queries when fetching arbitrary transactions
+		final int limit = 100;
+		int offset = 0;
+
+		while (!isStopping) {
+
+			// Any arbitrary transactions we want to fetch data for?
+			try (final Repository repository = RepositoryManager.getRepository()) {
+				List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true);
+				// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
+				if (signatures == null || signatures.isEmpty()) {
+					offset = 0;
+					break;
+				}
+				offset += limit;
+
+				// Loop through signatures and remove ones we don't need to process
+				Iterator iterator = signatures.iterator();
+				while (iterator.hasNext()) {
+					byte[] signature = (byte[]) iterator.next();
+
+					ArbitraryTransaction arbitraryTransaction = fetchTransaction(repository, signature);
+					if (arbitraryTransaction == null) {
+						// Best not to process this one
+						iterator.remove();
+						continue;
+					}
+					ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
+
+					// Skip transactions that we don't need to proactively store data for
+					if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) {
+						iterator.remove();
+						continue;
+					}
+
+					// Remove transactions that we already have local data for
+					if (hasLocalData(arbitraryTransaction)) {
+						iterator.remove();
+						continue;
+					}
+				}
+
+				if (signatures.isEmpty()) {
+					continue;
+				}
+
+				// Pick one at random
+				final int index = new Random().nextInt(signatures.size());
+				byte[] signature = signatures.get(index);
+
+				if (signature == null) {
+					continue;
+				}
+
+				// Check to see if we have had a more recent PUT
+				ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+				boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
+				if (hasMoreRecentPutTransaction) {
+					// There is a more recent PUT transaction than the one we are currently processing.
+					// When a PUT is issued, it replaces any layers that would have been there before.
+					// Therefore any data relating to this older transaction is no longer needed and we
+					// shouldn't fetch it from the network.
+					continue;
+				}
+
+				// Ask our connected peers if they have files for this signature
+				// This process automatically then fetches the files themselves if a peer is found
+				fetchData(arbitraryTransactionData);
+
+			} catch (DataException e) {
+				LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
+			}
+		}
+	}
+
+	private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) {
+		try {
+			TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+			if (!(transactionData instanceof ArbitraryTransactionData))
+				return null;
+
+			return new ArbitraryTransaction(repository, transactionData);
+
+		} catch (DataException e) {
+			return null;
+		}
+	}
+
+	private boolean hasLocalData(ArbitraryTransaction arbitraryTransaction) {
+		try {
+			return arbitraryTransaction.isDataLocal();
+
+		} catch (DataException e) {
+			LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e);
+			return true;
+		}
+	}
+
+
+	// Entrypoint to request new data from peers
+	public boolean fetchData(ArbitraryTransactionData arbitraryTransactionData) {
+		return ArbitraryDataFileListManager.getInstance().fetchArbitraryDataFileList(arbitraryTransactionData);
+	}
+
+
+	// Useful methods used by other parts of the app
+
+	public boolean isSignatureRateLimited(byte[] signature) {
+		return ArbitraryDataFileListManager.getInstance().isSignatureRateLimited(signature);
+	}
+
+	public long lastRequestForSignature(byte[] signature) {
+		return ArbitraryDataFileListManager.getInstance().lastRequestForSignature(signature);
+	}
+
+
+	// Arbitrary data resource cache
+
+	public void cleanupRequestCache(Long now) {
+		if (now == null) {
+			return;
+		}
+
+		// Cleanup file list request caches
+		ArbitraryDataFileListManager.getInstance().cleanupRequestCache(now);
+
+		// Cleanup file request caches
+		ArbitraryDataFileManager.getInstance().cleanupRequestCache(now);
+	}
+
+	public boolean isResourceCached(ArbitraryDataResource resource) {
+		if (resource == null) {
+			return false;
+		}
+		String key = resource.getUniqueKey();
+
+		// We don't have an entry for this resource ID, it is not cached
+		if (this.arbitraryDataCachedResources == null) {
+			return false;
+		}
+		if (!this.arbitraryDataCachedResources.containsKey(key)) {
+			return false;
+		}
+		Long timestamp = this.arbitraryDataCachedResources.get(key);
+		if (timestamp == null) {
+			return false;
+		}
+
+		// If the timestamp has reached the timeout, we should remove it from the cache
+		long now = NTP.getTime();
+		if (now > timestamp) {
+			this.arbitraryDataCachedResources.remove(key);
+			return false;
+		}
+
+		// Current time hasn't reached the timeout, so treat it as cached
+		return true;
+	}
+
+	public void addResourceToCache(ArbitraryDataResource resource) {
+		if (resource == null) {
+			return;
+		}
+		String key = resource.getUniqueKey();
+
+		// Just in case
+		if (this.arbitraryDataCachedResources == null) {
+			this.arbitraryDataCachedResources = new HashMap<>();
+		}
+
+		Long now = NTP.getTime();
+		if (now == null) {
+			return;
+		}
+
+		// Set the timestamp to now + the timeout
+		Long timestamp = NTP.getTime() + ARBITRARY_DATA_CACHE_TIMEOUT;
+		this.arbitraryDataCachedResources.put(key, timestamp);
+	}
+
+	public void invalidateCache(ArbitraryTransactionData arbitraryTransactionData) {
+		String signature58 = Base58.encode(arbitraryTransactionData.getSignature());
+
+		if (arbitraryTransactionData.getName() != null) {
+			String resourceId = arbitraryTransactionData.getName().toLowerCase();
+			Service service = arbitraryTransactionData.getService();
+			String identifier = arbitraryTransactionData.getIdentifier();
+
+			ArbitraryDataResource resource =
+					new ArbitraryDataResource(resourceId, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
+			String key = resource.getUniqueKey();
+			LOGGER.trace("Clearing cache for {}...", resource);
+
+			if (this.arbitraryDataCachedResources.containsKey(key)) {
+				this.arbitraryDataCachedResources.remove(key);
+			}
+
+			// Also remove from the failed builds queue in case it previously failed due to missing chunks
+			ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance();
+			if (buildManager.arbitraryDataFailedBuilds.containsKey(key)) {
+				buildManager.arbitraryDataFailedBuilds.remove(key);
+			}
+
+			// Remove from the signature requests list now that we have all files for this signature
+			ArbitraryDataFileListManager.getInstance().removeFromSignatureRequests(signature58);
+
+			// Delete cached files themselves
+			try {
+				resource.deleteCache();
+			} catch (IOException e) {
+				LOGGER.info("Unable to delete cache for resource {}: {}", resource, e.getMessage());
+			}
+		}
+	}
+
+
+	// Broadcast list of hosted signatures
+
+	public void broadcastHostedSignatureList() {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, null, null);
+			List<byte[]> hostedSignatures = hostedTransactions.stream().map(ArbitraryTransactionData::getSignature).collect(Collectors.toList());
+			if (!hostedSignatures.isEmpty()) {
+				// Broadcast the list, using null to represent our peer address
+				LOGGER.info("Broadcasting list of hosted signatures...");
+				Message arbitrarySignatureMessage = new ArbitrarySignaturesMessage(null, 0, hostedSignatures);
+				Network.getInstance().broadcast(broadcastPeer -> arbitrarySignatureMessage);
+			}
+		} catch (DataException e) {
+			LOGGER.error("Repository issue when fetching arbitrary transaction data for broadcast", e);
+		}
+	}
+
+
+	// Handle incoming arbitrary signatures messages
+
+	public void onNetworkArbitrarySignaturesMessage(Peer peer, Message message) {
+		// Don't process if QDN is disabled
+		if (!Settings.getInstance().isQdnEnabled()) {
+			return;
+		}
+
+		LOGGER.debug("Received arbitrary signature list from peer {}", peer);
+
+		ArbitrarySignaturesMessage arbitrarySignaturesMessage = (ArbitrarySignaturesMessage) message;
+		List<byte[]> signatures = arbitrarySignaturesMessage.getSignatures();
+
+		String peerAddress = peer.getPeerData().getAddress().toString();
+		if (arbitrarySignaturesMessage.getPeerAddress() != null && !arbitrarySignaturesMessage.getPeerAddress().isEmpty()) {
+			// This message is about a different peer than the one that sent it
+			peerAddress = arbitrarySignaturesMessage.getPeerAddress();
+		}
+
+		boolean containsNewEntry = false;
+
+		// Synchronize peer data lookups to make this process thread safe. Otherwise we could broadcast
+		// the same data multiple times, due to more than one thread processing the same message from different peers
+		synchronized (this.peerDataLock) {
+			try (final Repository repository = RepositoryManager.getRepository()) {
+				for (byte[] signature : signatures) {
+
+					// Check if a record already exists for this hash/host combination
+					// The port is not checked here - only the host/ip - in order to avoid duplicates
+					// from filling up the db due to dynamic/ephemeral ports
+					ArbitraryPeerData existingEntry = repository.getArbitraryRepository()
+							.getArbitraryPeerDataForSignatureAndHost(signature, peer.getPeerData().getAddress().getHost());
+
+					if (existingEntry == null) {
+						// We haven't got a record of this mapping yet, so add it
+						ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(signature, peerAddress);
+						repository.discardChanges();
+						if (arbitraryPeerData.isPeerAddressValid()) {
+							LOGGER.debug("Adding arbitrary peer: {} for signature {}", peerAddress, Base58.encode(signature));
+							repository.getArbitraryRepository().save(arbitraryPeerData);
+							repository.saveChanges();
+
+							// Remember that this data is new, so that it can be rebroadcast later
+							containsNewEntry = true;
+						}
+					}
+				}
+
+				// If at least one signature in this batch was new to us, we should rebroadcast the message to the
+				// network in case some peers haven't received it yet
+				if (containsNewEntry) {
+					int requestHops = arbitrarySignaturesMessage.getRequestHops();
+					arbitrarySignaturesMessage.setRequestHops(++requestHops);
+					if (requestHops < ARBITRARY_SIGNATURES_REQUEST_MAX_HOPS) {
+						LOGGER.debug("Rebroadcasting arbitrary signature list for peer {}. requestHops: {}", peerAddress, requestHops);
+						Network.getInstance().broadcast(broadcastPeer -> broadcastPeer == peer ? null : arbitrarySignaturesMessage);
+					}
+				} else {
+					// Don't rebroadcast as otherwise we could get into a loop
+				}
+
+				// If anything needed saving, it would already have called saveChanges() above
+				repository.discardChanges();
+			} catch (DataException e) {
+				LOGGER.error(String.format("Repository issue while processing arbitrary transaction signature list from peer %s", peer), e);
+			}
+		}
+	}
+
+
+	public int getPowDifficulty() {
+		return this.powDifficulty;
+	}
+
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java

@@ -0,0 +1,86 @@
+package org.qortal.controller.arbitrary;
+
+import org.qortal.arbitrary.ArbitraryDataResource;
+import org.qortal.utils.NTP;
+
+import java.util.*;
+
+public class ArbitraryDataRenderManager extends Thread {
+
+    private static ArbitraryDataRenderManager instance;
+    private volatile boolean isStopping = false;
+
+    /**
+     * Map to keep track of authorized resources for rendering.
+     * Keyed by resource ID, with the authorization time as the value.
+     */
+    private Map<String, Long> authorizedResources = Collections.synchronizedMap(new HashMap<>());
+
+    private static long AUTHORIZATION_TIMEOUT = 60 * 60 * 1000L; // 1 hour
+
+
+    public ArbitraryDataRenderManager() {
+
+    }
+
+    public static ArbitraryDataRenderManager getInstance() {
+        if (instance == null)
+            instance = new ArbitraryDataRenderManager();
+
+        return instance;
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data Manager");
+
+        try {
+            while (!isStopping) {
+                Thread.sleep(60000);
+
+                Long now = NTP.getTime();
+                this.cleanup(now);
+            }
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
+        }
+    }
+
+    public void shutdown() {
+        isStopping = true;
+        this.interrupt();
+    }
+
+    public void cleanup(Long now) {
+        if (now == null) {
+            return;
+        }
+        final long minimumTimestamp = now - AUTHORIZATION_TIMEOUT;
+        this.authorizedResources.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < minimumTimestamp);
+    }
+
+    public boolean isAuthorized(ArbitraryDataResource resource) {
+        ArbitraryDataResource broadResource = new ArbitraryDataResource(resource.getResourceId(), null, null, null);
+
+        for (String authorizedResourceKey : this.authorizedResources.keySet()) {
+            if (authorizedResourceKey != null && resource != null) {
+                // Check for exact match
+                if (Objects.equals(authorizedResourceKey, resource.getUniqueKey())) {
+                    return true;
+                }
+                // Check for a broad authorization (which applies to all services and identifiers under an authorized name)
+                if (Objects.equals(authorizedResourceKey, broadResource.getUniqueKey())) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    public void addToAuthorizedResources(ArbitraryDataResource resource) {
+        if (!this.isAuthorized(resource)) {
+            this.authorizedResources.put(resource.getUniqueKey(), NTP.getTime());
+        }
+    }
+
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java

@@ -0,0 +1,500 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.list.ResourceListManager;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.settings.Settings;
+import org.qortal.transaction.Transaction;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.Base58;
+import org.qortal.utils.FilesystemUtils;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class ArbitraryDataStorageManager extends Thread {
+
+    public enum StoragePolicy {
+        FOLLOWED_OR_VIEWED,
+        FOLLOWED,
+        VIEWED,
+        ALL,
+        NONE
+    }
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataStorageManager.class);
+
+    private static ArbitraryDataStorageManager instance;
+    private volatile boolean isStopping = false;
+
+    private Long storageCapacity = null;
+    private long totalDirectorySize = 0L;
+    private long lastDirectorySizeCheck = 0;
+
+    private List<ArbitraryTransactionData> hostedTransactions;
+
+    private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
+
+    /** Treat storage as full at 90% usage, to reduce risk of going over the limit.
+     * This is necessary because we don't calculate total storage values before every write.
+     * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit.
+     * This must be lower than DELETION_THRESHOLD. */
+    private static final double STORAGE_FULL_THRESHOLD = 0.90f; // 90%
+
+    /** Start deleting files once we reach 98% usage.
+     * This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */
+    public static final double DELETION_THRESHOLD = 0.98f; // 98%
+
+    public ArbitraryDataStorageManager() {
+    }
+
+    public static ArbitraryDataStorageManager getInstance() {
+        if (instance == null)
+            instance = new ArbitraryDataStorageManager();
+
+        return instance;
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data Storage Manager");
+        try {
+            while (!isStopping) {
+                Thread.sleep(1000);
+
+                // Don't run if QDN is disabled
+                if (!Settings.getInstance().isQdnEnabled()) {
+                    Thread.sleep(60 * 60 * 1000L);
+                    continue;
+                }
+
+                Long now = NTP.getTime();
+                if (now == null) {
+                    continue;
+                }
+
+                // Check the total directory size if we haven't in a while
+                if (this.shouldCalculateDirectorySize(now)) {
+                    this.calculateDirectorySize(now);
+                }
+
+                Thread.sleep(59000);
+            }
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
+        }
+    }
+
+    public void shutdown() {
+        isStopping = true;
+        this.interrupt();
+        instance = null;
+    }
+
+    /**
+     * Check if data relating to a transaction is allowed to
+     * exist on this node, therefore making it a mirror for this data.
+     *
+     * @param arbitraryTransactionData - the transaction
+     * @return boolean - whether to prefetch or not
+     */
+    public boolean canStoreData(ArbitraryTransactionData arbitraryTransactionData) {
+        String name = arbitraryTransactionData.getName();
+
+        // We already have RAW_DATA on chain, so we only need to store data associated with hashes
+        if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) {
+            return false;
+        }
+
+        // Don't store data unless it's an allowed type (public/private)
+        if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
+            return false;
+        }
+
+        // Don't check for storage limits here, as it can cause the cleanup manager to delete existing data
+
+        // Check if our storage policy and and lists allow us to host data for this name
+        switch (Settings.getInstance().getStoragePolicy()) {
+            case FOLLOWED_OR_VIEWED:
+            case ALL:
+            case VIEWED:
+                // If the policy includes viewed data, we can host it as long as it's not blocked
+                return !this.isNameBlocked(name);
+
+            case FOLLOWED:
+                // If the policy is for followed data only, we have to be following it
+                return this.isFollowingName(name);
+
+                // For NONE or all else, we shouldn't host this data
+            case NONE:
+            default:
+                return false;
+        }
+    }
+
+    /**
+     * Check if data relating to a transaction should be downloaded
+     * automatically, making this node a mirror for that data.
+     *
+     * @param arbitraryTransactionData - the transaction
+     * @return boolean - whether to prefetch or not
+     */
+    public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
+        String name = arbitraryTransactionData.getName();
+
+        // Only fetch data associated with hashes, as we already have RAW_DATA
+        if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) {
+            return false;
+        }
+
+        // Don't fetch anything more if we're (nearly) out of space
+        // Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
+        // avoid a fetch/delete loop
+        if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) {
+            return false;
+        }
+
+        // Don't fetch anything if we're (nearly) out of space for this name
+        // Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
+        // avoid a fetch/delete loop
+        if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
+            return false;
+        }
+
+        // Don't store data unless it's an allowed type (public/private)
+        if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
+            return false;
+        }
+
+        // Handle transactions without names differently
+        if (name == null) {
+            return this.shouldPreFetchDataWithoutName();
+        }
+
+        // Never fetch data from blocked names, even if they are followed
+        if (this.isNameBlocked(name)) {
+            return false;
+        }
+
+        switch (Settings.getInstance().getStoragePolicy()) {
+            case FOLLOWED:
+            case FOLLOWED_OR_VIEWED:
+                return this.isFollowingName(name);
+                
+            case ALL:
+                return true;
+
+            case NONE:
+            case VIEWED:
+            default:
+                return false;
+        }
+    }
+
+    /**
+     * Don't call this method directly.
+     * Use the wrapper method shouldPreFetchData() instead, as it contains
+     * additional checks.
+     *
+     * @return boolean - whether the storage policy allows for unnamed data
+     */
+    private boolean shouldPreFetchDataWithoutName() {
+        switch (Settings.getInstance().getStoragePolicy()) {
+            case ALL:
+                return true;
+
+            case NONE:
+            case VIEWED:
+            case FOLLOWED:
+            case FOLLOWED_OR_VIEWED:
+            default:
+                return false;
+        }
+    }
+
+    private boolean isDataTypeAllowed(ArbitraryTransactionData arbitraryTransactionData) {
+        byte[] secret = arbitraryTransactionData.getSecret();
+        boolean hasSecret = (secret != null && secret.length == 32);
+
+        if (!Settings.getInstance().isPrivateDataEnabled() && !hasSecret) {
+            // Private data isn't enabled so we can't store data without a valid secret
+            return false;
+        }
+        if (!Settings.getInstance().isPublicDataEnabled() && hasSecret) {
+            // Public data isn't enabled so we can't store data with a secret
+            return false;
+        }
+        return true;
+    }
+
+    public boolean isNameBlocked(String name) {
+        return ResourceListManager.getInstance().listContains("blockedNames", name, false);
+    }
+
+    private boolean isFollowingName(String name) {
+        return ResourceListManager.getInstance().listContains("followedNames", name, false);
+    }
+
+    public List<String> followedNames() {
+        return ResourceListManager.getInstance().getStringsInList("followedNames");
+    }
+
+    private int followedNamesCount() {
+        return ResourceListManager.getInstance().getItemCountForList("followedNames");
+    }
+
+
+    // Hosted data
+
+    public List<ArbitraryTransactionData> listAllHostedTransactions(Repository repository, Integer limit, Integer offset) {
+        // Load from cache if we can, to avoid disk reads
+        if (this.hostedTransactions != null) {
+            return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset);
+        }
+
+        List<ArbitraryTransactionData> arbitraryTransactionDataList = new ArrayList<>();
+
+        // Find all hosted paths
+        List<Path> allPaths = this.findAllHostedPaths();
+
+        // Loop through each path and attempt to match it to a signature
+        for (Path path : allPaths) {
+            try {
+                String[] contents = path.toFile().list();
+                if (contents == null || contents.length == 0) {
+                    // Ignore empty directories
+                    continue;
+                }
+
+                String signature58 = path.getFileName().toString();
+                byte[] signature = Base58.decode(signature58);
+                TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+                if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) {
+                    continue;
+                }
+                arbitraryTransactionDataList.add((ArbitraryTransactionData) transactionData);
+
+            } catch (DataException e) {
+                continue;
+            }
+        }
+
+        // Sort by newest first
+        arbitraryTransactionDataList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed());
+
+        // Update cache
+        this.hostedTransactions = arbitraryTransactionDataList;
+
+        return ArbitraryTransactionUtils.limitOffsetTransactions(arbitraryTransactionDataList, limit, offset);
+    }
+
+    /**
+     * Warning: this method will walk through the entire data directory
+     * Do not call it too frequently as it could create high disk load
+     * in environments with a large amount of hosted data.
+     * @return a list of paths that are being hosted
+     */
+    public List<Path> findAllHostedPaths() {
+        Path dataPath = Paths.get(Settings.getInstance().getDataPath());
+        Path tempPath = Paths.get(Settings.getInstance().getTempDataPath());
+
+        // Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length
+        // Also exclude the _temp and _misc paths if present
+        List<Path> allPaths = new ArrayList<>();
+        try {
+            allPaths = Files.walk(dataPath, 3)
+                    .filter(Files::isDirectory)
+                    .filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString())
+                            && !path.toString().contains("_misc")
+                            && path.getFileName().toString().length() > 32)
+                    .collect(Collectors.toList());
+        }
+        catch (IOException e) {
+            LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
+        }
+
+        return allPaths;
+    }
+
+    public void invalidateHostedTransactionsCache() {
+        this.hostedTransactions = null;
+    }
+
+
+    // Size limits
+
+    /**
+     * Rate limit to reduce IO load
+     */
+    public boolean shouldCalculateDirectorySize(Long now) {
+        if (now == null) {
+            return false;
+        }
+        // If storage capacity is null, we need to calculate it
+        if (this.storageCapacity == null) {
+            return true;
+        }
+        // If we haven't checked for a while, we need to check it now
+        if (now - lastDirectorySizeCheck > DIRECTORY_SIZE_CHECK_INTERVAL) {
+            return true;
+        }
+
+        // We shouldn't check this time, as we want to reduce IO load on the SSD/HDD
+        return false;
+    }
+
+    public void calculateDirectorySize(Long now) {
+        if (now == null) {
+            return;
+        }
+
+        long totalSize = 0;
+        long remainingCapacity = 0;
+
+        // Calculate remaining capacity
+        try {
+            remainingCapacity = this.getRemainingUsableStorageCapacity();
+        } catch (IOException e) {
+            LOGGER.info("Unable to calculate remaining storage capacity: {}", e.getMessage());
+            return;
+        }
+
+        // Calculate total size of data directory
+        LOGGER.trace("Calculating data directory size...");
+        Path dataDirectoryPath = Paths.get(Settings.getInstance().getDataPath());
+        if (dataDirectoryPath.toFile().exists()) {
+            totalSize += FileUtils.sizeOfDirectory(dataDirectoryPath.toFile());
+        }
+
+        // Add total size of temp directory, if it's not already inside the data directory
+        Path tempDirectoryPath = Paths.get(Settings.getInstance().getTempDataPath());
+        if (tempDirectoryPath.toFile().exists()) {
+            if (!FilesystemUtils.isChild(tempDirectoryPath, dataDirectoryPath)) {
+                LOGGER.trace("Calculating temp directory size...");
+                totalSize += FileUtils.sizeOfDirectory(dataDirectoryPath.toFile());
+            }
+        }
+
+        this.totalDirectorySize = totalSize;
+        this.lastDirectorySizeCheck = now;
+
+        // It's essential that used space (this.totalDirectorySize) is included in the storage capacity
+        LOGGER.trace("Calculating total storage capacity...");
+        long storageCapacity = remainingCapacity + this.totalDirectorySize;
+
+        // Make sure to limit the storage capacity if the user is overriding it in the settings
+        if (Settings.getInstance().getMaxStorageCapacity() != null) {
+            storageCapacity = Math.min(storageCapacity, Settings.getInstance().getMaxStorageCapacity());
+        }
+        this.storageCapacity = storageCapacity;
+
+        LOGGER.info("Total used: {} bytes, Total capacity: {} bytes", this.totalDirectorySize, this.storageCapacity);
+    }
+
+    private long getRemainingUsableStorageCapacity() throws IOException {
+        // Create data directory if it doesn't exist so that we can perform calculations on it
+        Path dataDirectoryPath = Paths.get(Settings.getInstance().getDataPath());
+        if (!dataDirectoryPath.toFile().exists()) {
+            Files.createDirectories(dataDirectoryPath);
+        }
+
+        return dataDirectoryPath.toFile().getUsableSpace();
+    }
+
+    public long getTotalDirectorySize() {
+        return this.totalDirectorySize;
+    }
+
+    public boolean isStorageSpaceAvailable(double threshold) {
+        if (!this.isStorageCapacityCalculated()) {
+            return false;
+        }
+
+        long maxStorageCapacity = (long)((double)this.storageCapacity * threshold);
+        if (this.totalDirectorySize >= maxStorageCapacity) {
+            return false;
+        }
+        return true;
+    }
+
+    public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) {
+        if (!this.isStorageSpaceAvailable(threshold)) {
+            // No storage space available at all, so no need to check this name
+            return false;
+        }
+
+        if (name == null) {
+            // This transaction doesn't have a name, so fall back to total space limitations
+            return true;
+        }
+
+        int followedNamesCount = this.followedNamesCount();
+        if (followedNamesCount == 0) {
+            // Not following any names, so we have space
+            return true;
+        }
+
+        long totalSizeForName = 0;
+        long maxStoragePerName = this.storageCapacityPerName(threshold);
+
+        // Fetch all hosted transactions
+        List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository, null, null);
+        for (ArbitraryTransactionData transactionData : hostedTransactions) {
+            String transactionName = transactionData.getName();
+            if (!Objects.equals(name, transactionName)) {
+                // Transaction relates to a different name
+                continue;
+            }
+
+            totalSizeForName += transactionData.getSize();
+        }
+
+        // Have we reached the limit for this name?
+        if (totalSizeForName > maxStoragePerName) {
+            return false;
+        }
+
+        return true;
+    }
+
+    public long storageCapacityPerName(double threshold) {
+        int followedNamesCount = this.followedNamesCount();
+        if (followedNamesCount == 0) {
+            // Not following any names, so we have the total space available
+            return this.getStorageCapacityIncludingThreshold(threshold);
+        }
+
+        double maxStorageCapacity = (double)this.storageCapacity * threshold;
+        long maxStoragePerName = (long)(maxStorageCapacity / (double)followedNamesCount);
+
+        return maxStoragePerName;
+    }
+
+    public boolean isStorageCapacityCalculated() {
+        return (this.storageCapacity != null);
+    }
+
+    public Long getStorageCapacity() {
+        return this.storageCapacity;
+    }
+
+    public Long getStorageCapacityIncludingThreshold(double threshold) {
+        if (this.storageCapacity == null) {
+            return null;
+        }
+        return (long)(this.storageCapacity * threshold);
+    }
+}

src/main/java/org/qortal/controller/repository/AtStatesPruner.java

@@ -3,6 +3,7 @@ package org.qortal.controller.repository;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.data.block.BlockData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
@@ -47,7 +48,7 @@ public class AtStatesPruner implements Runnable {
 					continue;
 
 				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Controller.getInstance().isSynchronizing())
+				if (Synchronizer.getInstance().isSynchronizing())
 					continue;
 
 				// Prune AT states for all blocks up until our latest minus pruneBlockLimit

src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java

@@ -3,6 +3,7 @@ package org.qortal.controller.repository;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.data.block.BlockData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
@@ -34,7 +35,7 @@ public class AtStatesTrimmer implements Runnable {
 					continue;
 
 				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Controller.getInstance().isSynchronizing())
+				if (Synchronizer.getInstance().isSynchronizing())
 					continue;
 
 				long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime();

src/main/java/org/qortal/controller/repository/BlockArchiver.java

@@ -3,6 +3,7 @@ package org.qortal.controller.repository;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.data.block.BlockData;
 import org.qortal.repository.*;
 import org.qortal.settings.Settings;
@@ -51,7 +52,7 @@ public class BlockArchiver implements Runnable {
 				}
 
 				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Controller.getInstance().isSynchronizing()) {
+				if (Synchronizer.getInstance().isSynchronizing()) {
 					continue;
 				}
 

src/main/java/org/qortal/controller/repository/BlockPruner.java

@@ -3,6 +3,7 @@ package org.qortal.controller.repository;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.data.block.BlockData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
@@ -51,7 +52,7 @@ public class BlockPruner implements Runnable {
 					continue;
 
 				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Controller.getInstance().isSynchronizing()) {
+				if (Synchronizer.getInstance().isSynchronizing()) {
 					continue;
 				}
 

src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java

@@ -268,49 +268,13 @@ public class NamesDatabaseIntegrityCheck {
         return registerNameTransactions;
     }
 
-    private List<UpdateNameTransactionData> fetchUpdateNameTransactions() {
-        List<UpdateNameTransactionData> updateNameTransactions = new ArrayList<>();
-
-        for (TransactionData transactionData : this.nameTransactions) {
-            if (transactionData.getType() == TransactionType.UPDATE_NAME) {
-                UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData;
-                updateNameTransactions.add(updateNameTransactionData);
-            }
-        }
-        return updateNameTransactions;
-    }
-
-    private List<SellNameTransactionData> fetchSellNameTransactions() {
-        List<SellNameTransactionData> sellNameTransactions = new ArrayList<>();
-
-        for (TransactionData transactionData : this.nameTransactions) {
-            if (transactionData.getType() == TransactionType.SELL_NAME) {
-                SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData;
-                sellNameTransactions.add(sellNameTransactionData);
-            }
-        }
-        return sellNameTransactions;
-    }
-
-    private List<BuyNameTransactionData> fetchBuyNameTransactions() {
-        List<BuyNameTransactionData> buyNameTransactions = new ArrayList<>();
-
-        for (TransactionData transactionData : this.nameTransactions) {
-            if (transactionData.getType() == TransactionType.BUY_NAME) {
-                BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData;
-                buyNameTransactions.add(buyNameTransactionData);
-            }
-        }
-        return buyNameTransactions;
-    }
-
     private void fetchAllNameTransactions(Repository repository) throws DataException {
         List<TransactionData> nameTransactions = new ArrayList<>();
 
         // Fetch all the confirmed REGISTER_NAME transaction signatures
         List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(
                 null, null, null, ALL_NAME_TX_TYPE, null, null,
-                ConfirmationStatus.CONFIRMED, null, null, false);
+                null, ConfirmationStatus.CONFIRMED, null, null, false);
 
         for (byte[] signature : signatures) {
             TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
@@ -319,41 +283,34 @@ public class NamesDatabaseIntegrityCheck {
         this.nameTransactions = nameTransactions;
     }
 
-    private List<TransactionData> fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException {
-        List<TransactionData> transactions = new ArrayList<>();
+    public List<TransactionData> fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException {
+        List<byte[]> signatures = new ArrayList<>();
         String reducedName = Unicode.sanitize(name);
 
-        // Fetch all the confirmed name-modification transactions
-        if (this.nameTransactions.isEmpty()) {
-            this.fetchAllNameTransactions(repository);
-        }
+        List<byte[]> registerNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
+                TransactionType.REGISTER_NAME, Arrays.asList("(name = ? OR reduced_name = ?)"), Arrays.asList(name, reducedName));
+        signatures.addAll(registerNameTransactions);
 
-        for (TransactionData transactionData : this.nameTransactions) {
+        List<byte[]> updateNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
+                TransactionType.UPDATE_NAME,
+                Arrays.asList("(name = ? OR new_name = ? OR (reduced_new_name != '' AND reduced_new_name = ?))"),
+                Arrays.asList(name, name, reducedName));
+        signatures.addAll(updateNameTransactions);
 
-            if ((transactionData instanceof RegisterNameTransactionData)) {
-                RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData;
-                if (Objects.equals(registerNameTransactionData.getReducedName(), reducedName)) {
-                    transactions.add(transactionData);
-                }
-            }
-            if ((transactionData instanceof UpdateNameTransactionData)) {
-                UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData;
-                if (Objects.equals(updateNameTransactionData.getName(), name) ||
-                        Objects.equals(updateNameTransactionData.getReducedNewName(), reducedName)) {
-                    transactions.add(transactionData);
-                }
-            }
-            if ((transactionData instanceof BuyNameTransactionData)) {
-                BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData;
-                if (Objects.equals(buyNameTransactionData.getName(), name)) {
-                    transactions.add(transactionData);
-                }
-            }
-            if ((transactionData instanceof SellNameTransactionData)) {
-                SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData;
-                if (Objects.equals(sellNameTransactionData.getName(), name)) {
-                    transactions.add(transactionData);
-                }
+        List<byte[]> sellNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
+                TransactionType.SELL_NAME, Arrays.asList("name = ?"), Arrays.asList(name));
+        signatures.addAll(sellNameTransactions);
+
+        List<byte[]> buyNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
+                TransactionType.BUY_NAME, Arrays.asList("name = ?"), Arrays.asList(name));
+        signatures.addAll(buyNameTransactions);
+
+        List<TransactionData> transactions = new ArrayList<>();
+        for (byte[] signature : signatures) {
+            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+            // Filter out any unconfirmed transactions
+            if (transactionData.getBlockHeight() != null && transactionData.getBlockHeight() > 0) {
+                transactions.add(transactionData);
             }
         }
         return transactions;

src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java

@@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.block.BlockChain;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.data.block.BlockData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
@@ -36,7 +37,7 @@ public class OnlineAccountsSignaturesTrimmer implements Runnable {
 					continue;
 
 				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Controller.getInstance().isSynchronizing())
+				if (Synchronizer.getInstance().isSynchronizing())
 					continue;
 
 				// Trim blockchain by removing 'old' online accounts signatures

src/main/java/org/qortal/crosschain/Bitcoiny.java

@@ -1,12 +1,6 @@
 package org.qortal.crosschain;
 
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import org.apache.logging.log4j.LogManager;
@@ -39,6 +33,7 @@ import org.qortal.utils.Amounts;
 import org.qortal.utils.BitTwiddling;
 
 import com.google.common.hash.HashCode;
+import org.qortal.utils.NTP;
 
 /** Bitcoin-like (Bitcoin, Litecoin, etc.) support */
 public abstract class Bitcoiny implements ForeignBlockchain {
@@ -53,6 +48,12 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 
 	protected final NetworkParameters params;
 
+	/** Cache recent transactions to speed up subsequent lookups */
+	protected List<SimpleTransaction> transactionsCache;
+	protected Long transactionsCacheTimestamp;
+	protected String transactionsCacheXpub;
+	protected static long TRANSACTIONS_CACHE_TIMEOUT = 2 * 60 * 1000L; // 2 minutes
+
 	/** Keys that have been previously marked as fully spent,<br>
 	 * i.e. keys with transactions but with no unspent outputs. */
 	protected final Set<ECKey> spentKeys = Collections.synchronizedSet(new HashSet<>());
@@ -337,70 +338,99 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 		return balance.value;
 	}
 
+	public Long getWalletBalanceFromTransactions(String key58) throws ForeignBlockchainException {
+		long balance = 0;
+		Comparator<SimpleTransaction> oldestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp);
+		List<SimpleTransaction> transactions = getWalletTransactions(key58).stream().sorted(oldestTimestampFirstComparator).collect(Collectors.toList());
+		for (SimpleTransaction transaction : transactions) {
+			balance += transaction.getTotalAmount();
+		}
+		return balance;
+	}
+
 	public List<SimpleTransaction> getWalletTransactions(String key58) throws ForeignBlockchainException {
-		Context.propagate(bitcoinjContext);
-
-		Wallet wallet = walletFromDeterministicKey58(key58);
-		DeterministicKeyChain keyChain = wallet.getActiveKeyChain();
-
-		keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT);
-		keyChain.maybeLookAhead();
-
-		List<DeterministicKey> keys = new ArrayList<>(keyChain.getLeafKeys());
-
-		Set<BitcoinyTransaction> walletTransactions = new HashSet<>();
-		Set<String> keySet = new HashSet<>();
-
-		// Set the number of consecutive empty batches required before giving up
-		final int numberOfAdditionalBatchesToSearch = 5;
-
-		int unusedCounter = 0;
-		int ki = 0;
-		do {
-			boolean areAllKeysUnused = true;
-
-			for (; ki < keys.size(); ++ki) {
-				DeterministicKey dKey = keys.get(ki);
-
-				// Check for transactions
-				Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH);
-				keySet.add(address.toString());
-				byte[] script = ScriptBuilder.createOutputScript(address).getProgram();
-
-				// Ask for transaction history - if it's empty then key has never been used
-				List<TransactionHash> historicTransactionHashes = this.blockchain.getAddressTransactions(script, false);
-
-				if (!historicTransactionHashes.isEmpty()) {
-					areAllKeysUnused = false;
-
-					for (TransactionHash transactionHash : historicTransactionHashes)
-						walletTransactions.add(this.getTransaction(transactionHash.txHash));
+		synchronized (this) {
+			// Serve from the cache if it's recent, and matches this xpub
+			if (Objects.equals(transactionsCacheXpub, key58)) {
+				if (transactionsCache != null && transactionsCacheTimestamp != null) {
+					Long now = NTP.getTime();
+					boolean isCacheStale = (now != null && now - transactionsCacheTimestamp >= TRANSACTIONS_CACHE_TIMEOUT);
+					if (!isCacheStale) {
+						return transactionsCache;
+					}
 				}
 			}
 
-			if (areAllKeysUnused) {
-				// No transactions
-				if (unusedCounter >= numberOfAdditionalBatchesToSearch) {
-					// ... and we've hit our search limit
-					break;
+			Context.propagate(bitcoinjContext);
+
+			Wallet wallet = walletFromDeterministicKey58(key58);
+			DeterministicKeyChain keyChain = wallet.getActiveKeyChain();
+
+			keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT);
+			keyChain.maybeLookAhead();
+
+			List<DeterministicKey> keys = new ArrayList<>(keyChain.getLeafKeys());
+
+			Set<BitcoinyTransaction> walletTransactions = new HashSet<>();
+			Set<String> keySet = new HashSet<>();
+
+			// Set the number of consecutive empty batches required before giving up
+			final int numberOfAdditionalBatchesToSearch = 5;
+
+			int unusedCounter = 0;
+			int ki = 0;
+			do {
+				boolean areAllKeysUnused = true;
+
+				for (; ki < keys.size(); ++ki) {
+					DeterministicKey dKey = keys.get(ki);
+
+					// Check for transactions
+					Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH);
+					keySet.add(address.toString());
+					byte[] script = ScriptBuilder.createOutputScript(address).getProgram();
+
+					// Ask for transaction history - if it's empty then key has never been used
+					List<TransactionHash> historicTransactionHashes = this.blockchain.getAddressTransactions(script, false);
+
+					if (!historicTransactionHashes.isEmpty()) {
+						areAllKeysUnused = false;
+
+						for (TransactionHash transactionHash : historicTransactionHashes)
+							walletTransactions.add(this.getTransaction(transactionHash.txHash));
+					}
 				}
-				// We haven't hit our search limit yet so increment the counter and keep looking
-				unusedCounter++;
-			}
-			else {
-				// Some keys in this batch were used, so reset the counter
-				unusedCounter = 0;
-			}
 
-			// Generate some more keys
-			keys.addAll(generateMoreKeys(keyChain));
+				if (areAllKeysUnused) {
+					// No transactions
+					if (unusedCounter >= numberOfAdditionalBatchesToSearch) {
+						// ... and we've hit our search limit
+						break;
+					}
+					// We haven't hit our search limit yet so increment the counter and keep looking
+					unusedCounter++;
+				} else {
+					// Some keys in this batch were used, so reset the counter
+					unusedCounter = 0;
+				}
 
-			// Process new keys
-		} while (true);
+				// Generate some more keys
+				keys.addAll(generateMoreKeys(keyChain));
 
-		Comparator<SimpleTransaction> newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed();
+				// Process new keys
+			} while (true);
 
-		return walletTransactions.stream().map(t -> convertToSimpleTransaction(t, keySet)).sorted(newestTimestampFirstComparator).collect(Collectors.toList());
+			Comparator<SimpleTransaction> newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed();
+
+			// Update cache and return
+			transactionsCacheTimestamp = NTP.getTime();
+			transactionsCacheXpub = key58;
+			transactionsCache = walletTransactions.stream()
+					.map(t -> convertToSimpleTransaction(t, keySet))
+					.sorted(newestTimestampFirstComparator).collect(Collectors.toList());
+
+			return transactionsCache;
+		}
 	}
 
 	protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set<String> keySet) {

src/main/java/org/qortal/crosschain/Dogecoin.java

@@ -19,12 +19,13 @@ public class Dogecoin extends Bitcoiny {
 
 	public static final String CURRENCY_CODE = "DOGE";
 
-	private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(500000000); // 5 DOGE per 1000 bytes
+	private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(1000000); // 0.01 DOGE per 1000 bytes
 
-	private static final long MINIMUM_ORDER_AMOUNT = 300000000L; // 3 DOGE minimum order. The RPC dust threshold is around 2 DOGE
+	private static final long MINIMUM_ORDER_AMOUNT = 100000000L; // 1 DOGE minimum order. See recommendations:
+	// https://github.com/dogecoin/dogecoin/blob/master/doc/fee-recommendation.md
 
 	// Temporary values until a dynamic fee system is written.
-	private static final long MAINNET_FEE = 110000000L;
+	private static final long MAINNET_FEE = 100000L;
 	private static final long NON_MAINNET_FEE = 10000L; // TODO: calibrate this
 
 	private static final Map<ConnectionType, Integer> DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ConnectionType.class);

src/main/java/org/qortal/crypto/AES.java

@@ -0,0 +1,205 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2017 Eugen Paraschiv
+ * Modified in 2021 by CalDescent
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+package org.qortal.crypto;
+
+import javax.crypto.Cipher;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+import javax.crypto.BadPaddingException;
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.SealedObject;
+import javax.crypto.spec.IvParameterSpec;
+import javax.crypto.spec.PBEKeySpec;
+import javax.crypto.spec.SecretKeySpec;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.KeySpec;
+import java.util.Base64;
+
+public class AES {
+
+    public static String encrypt(String algorithm, String input, SecretKey key, IvParameterSpec iv)
+            throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException,
+            InvalidKeyException, BadPaddingException, IllegalBlockSizeException {
+        Cipher cipher = Cipher.getInstance(algorithm);
+        cipher.init(Cipher.ENCRYPT_MODE, key, iv);
+        byte[] cipherText = cipher.doFinal(input.getBytes());
+        return Base64.getEncoder()
+                .encodeToString(cipherText);
+    }
+
+    public static String decrypt(String algorithm, String cipherText, SecretKey key, IvParameterSpec iv)
+            throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException,
+            InvalidKeyException, BadPaddingException, IllegalBlockSizeException {
+        Cipher cipher = Cipher.getInstance(algorithm);
+        cipher.init(Cipher.DECRYPT_MODE, key, iv);
+        byte[] plainText = cipher.doFinal(Base64.getDecoder()
+                .decode(cipherText));
+        return new String(plainText);
+    }
+
+    public static SecretKey generateKey(int n) throws NoSuchAlgorithmException {
+        KeyGenerator keyGenerator = KeyGenerator.getInstance("AES");
+        keyGenerator.init(n);
+        SecretKey key = keyGenerator.generateKey();
+        return key;
+    }
+
+    public static SecretKey getKeyFromPassword(String password, String salt)
+            throws NoSuchAlgorithmException, InvalidKeySpecException {
+        SecretKeyFactory factory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256");
+        KeySpec spec = new PBEKeySpec(password.toCharArray(), salt.getBytes(), 65536, 256);
+        SecretKey secret = new SecretKeySpec(factory.generateSecret(spec)
+                .getEncoded(), "AES");
+        return secret;
+    }
+
+    public static IvParameterSpec generateIv() {
+        byte[] iv = new byte[16];
+        new SecureRandom().nextBytes(iv);
+        return new IvParameterSpec(iv);
+    }
+
+    public static void encryptFile(String algorithm, SecretKey key,
+                                   String inputFilePath, String outputFilePath) throws IOException,
+            NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException,
+            BadPaddingException, IllegalBlockSizeException {
+
+        File inputFile = new File(inputFilePath);
+        File outputFile = new File(outputFilePath);
+
+        IvParameterSpec iv = AES.generateIv();
+        Cipher cipher = Cipher.getInstance(algorithm);
+        cipher.init(Cipher.ENCRYPT_MODE, key, iv);
+        FileInputStream inputStream = new FileInputStream(inputFile);
+        FileOutputStream outputStream = new FileOutputStream(outputFile);
+
+        // Prepend the output stream with the 16 byte initialization vector
+        outputStream.write(iv.getIV());
+
+        byte[] buffer = new byte[1024];
+        int bytesRead;
+        while ((bytesRead = inputStream.read(buffer)) != -1) {
+            byte[] output = cipher.update(buffer, 0, bytesRead);
+            if (output != null) {
+                outputStream.write(output);
+            }
+        }
+        byte[] outputBytes = cipher.doFinal();
+        if (outputBytes != null) {
+            outputStream.write(outputBytes);
+        }
+        inputStream.close();
+        outputStream.close();
+    }
+
+    public static void decryptFile(String algorithm, SecretKey key, String encryptedFilePath,
+                                   String decryptedFilePath) throws IOException, NoSuchPaddingException,
+            NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException,
+            BadPaddingException, IllegalBlockSizeException {
+
+        File encryptedFile = new File(encryptedFilePath);
+        File decryptedFile = new File(decryptedFilePath);
+
+        File parent = decryptedFile.getParentFile();
+        if (!parent.isDirectory() && !parent.mkdirs()) {
+            throw new IOException("Failed to create directory " + parent);
+        }
+
+        FileInputStream inputStream = new FileInputStream(encryptedFile);
+        FileOutputStream outputStream = new FileOutputStream(decryptedFile);
+
+        // Read the initialization vector from the first 16 bytes of the file
+        byte[] iv = new byte[16];
+        inputStream.read(iv);
+        Cipher cipher = Cipher.getInstance(algorithm);
+        cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
+
+        byte[] buffer = new byte[64];
+        int bytesRead;
+        while ((bytesRead = inputStream.read(buffer)) != -1) {
+            byte[] output = cipher.update(buffer, 0, bytesRead);
+            if (output != null) {
+                outputStream.write(output);
+            }
+        }
+        byte[] output = cipher.doFinal();
+        if (output != null) {
+            outputStream.write(output);
+        }
+        inputStream.close();
+        outputStream.close();
+    }
+
+    public static SealedObject encryptObject(String algorithm, Serializable object, SecretKey key,
+                                             IvParameterSpec iv) throws NoSuchPaddingException, NoSuchAlgorithmException,
+            InvalidAlgorithmParameterException, InvalidKeyException, IOException, IllegalBlockSizeException {
+        Cipher cipher = Cipher.getInstance(algorithm);
+        cipher.init(Cipher.ENCRYPT_MODE, key, iv);
+        SealedObject sealedObject = new SealedObject(object, cipher);
+        return sealedObject;
+    }
+
+    public static Serializable decryptObject(String algorithm, SealedObject sealedObject, SecretKey key,
+                                             IvParameterSpec iv) throws NoSuchPaddingException, NoSuchAlgorithmException,
+            InvalidAlgorithmParameterException, InvalidKeyException, ClassNotFoundException,
+            BadPaddingException, IllegalBlockSizeException, IOException {
+        Cipher cipher = Cipher.getInstance(algorithm);
+        cipher.init(Cipher.DECRYPT_MODE, key, iv);
+        Serializable unsealObject = (Serializable) sealedObject.getObject(cipher);
+        return unsealObject;
+    }
+
+    public static String encryptPasswordBased(String plainText, SecretKey key, IvParameterSpec iv)
+            throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException,
+            InvalidKeyException, BadPaddingException, IllegalBlockSizeException {
+        Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
+        cipher.init(Cipher.ENCRYPT_MODE, key, iv);
+        return Base64.getEncoder()
+                .encodeToString(cipher.doFinal(plainText.getBytes()));
+    }
+
+    public static String decryptPasswordBased(String cipherText, SecretKey key, IvParameterSpec iv)
+            throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException,
+            InvalidKeyException, BadPaddingException, IllegalBlockSizeException {
+        Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5PADDING");
+        cipher.init(Cipher.DECRYPT_MODE, key, iv);
+        return new String(cipher.doFinal(Base64.getDecoder()
+                .decode(cipherText)));
+    }
+
+}

src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java

@@ -0,0 +1,60 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.network.Peer;
+import java.util.Objects;
+
+public class ArbitraryRelayInfo {
+
+    private final String hash58;
+    private final String signature58;
+    private final Peer peer;
+    private final Long timestamp;
+
+    public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp) {
+        this.hash58 = hash58;
+        this.signature58 = signature58;
+        this.peer = peer;
+        this.timestamp = timestamp;
+    }
+
+    public boolean isValid() {
+        return this.getHash58() != null && this.getSignature58() != null
+                && this.getPeer() != null && this.getTimestamp() != null;
+    }
+
+    public String getHash58() {
+        return this.hash58;
+    }
+
+    public String getSignature58() {
+        return signature58;
+    }
+
+    public Peer getPeer() {
+        return peer;
+    }
+
+    public Long getTimestamp() {
+        return timestamp;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this)
+            return true;
+
+        if (!(other instanceof ArbitraryRelayInfo))
+            return false;
+
+        ArbitraryRelayInfo otherRelayInfo = (ArbitraryRelayInfo) other;
+
+        return this.peer == otherRelayInfo.getPeer()
+                && Objects.equals(this.hash58, otherRelayInfo.getHash58())
+                && Objects.equals(this.signature58, otherRelayInfo.getSignature58());
+    }
+}

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java

@@ -0,0 +1,37 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.arbitrary.misc.Service;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.Objects;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ArbitraryResourceInfo {
+
+	public String name;
+	public Service service;
+	public String identifier;
+	public ArbitraryResourceStatus status;
+
+	public Long size;
+
+	public ArbitraryResourceInfo() {
+	}
+
+	@Override
+	public boolean equals(Object o) {
+		if (o == this)
+			return true;
+
+		if (!(o instanceof ArbitraryResourceInfo))
+			return false;
+
+		ArbitraryResourceInfo other = (ArbitraryResourceInfo) o;
+
+		return Objects.equals(this.name, other.name) &&
+				Objects.equals(this.service, other.service) &&
+				Objects.equals(this.identifier, other.identifier);
+	}
+
+}

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java

@@ -0,0 +1,17 @@
+package org.qortal.data.arbitrary;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.ArrayList;
+import java.util.List;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ArbitraryResourceNameInfo {
+
+	public String name;
+	public List<ArbitraryResourceInfo> resources = new ArrayList<>();
+
+	public ArbitraryResourceNameInfo() {
+	}
+
+}

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java

@@ -0,0 +1,50 @@
+package org.qortal.data.arbitrary;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ArbitraryResourceStatus {
+
+    public enum Status {
+        PUBLISHED("Published", "Published but not yet downloaded"),
+        DOWNLOADING("Downloading", "Locating and downloading files..."),
+        DOWNLOADED("Downloaded", "Files downloaded"),
+        BUILDING("Building", "Building..."),
+        READY("Ready", "Ready"),
+        MISSING_DATA("Missing data", "Unable to locate all files. Please try again later"),
+        BUILD_FAILED("Build failed", "Build failed. Please try again later"),
+        UNSUPPORTED("Unsupported", "Unsupported request"),
+        BLOCKED("Blocked", "Name is blocked so content cannot be served");
+
+        private String title;
+        private String description;
+
+        Status(String title, String description) {
+            this.title = title;
+            this.description = description;
+        }
+    }
+
+    private String id;
+    private String title;
+    private String description;
+
+    private Integer localChunkCount;
+    private Integer totalChunkCount;
+
+    public ArbitraryResourceStatus() {
+    }
+
+    public ArbitraryResourceStatus(Status status, Integer localChunkCount, Integer totalChunkCount) {
+        this.id = status.toString();
+        this.title = status.title;
+        this.description = status.description;
+        this.localChunkCount = localChunkCount;
+        this.totalChunkCount = totalChunkCount;
+    }
+
+    public ArbitraryResourceStatus(Status status) {
+        this(status, null, null);
+    }
+}

src/main/java/org/qortal/data/block/BlockData.java

@@ -9,7 +9,10 @@ import javax.xml.bind.annotation.XmlAccessorType;
 import javax.xml.bind.annotation.XmlElement;
 import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
+import org.qortal.block.BlockChain;
+import org.qortal.settings.Settings;
 import org.qortal.crypto.Crypto;
+import org.qortal.utils.NTP;
 
 // All properties to be converted to JSON via JAX-RS
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -208,6 +211,13 @@ public class BlockData implements Serializable {
 		this.onlineAccountsSignatures = onlineAccountsSignatures;
 	}
 
+	public boolean isTrimmed() {
+		long onlineAccountSignaturesTrimmedTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime();
+		long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime();
+		long blockTimestamp = this.getTimestamp();
+		return blockTimestamp < onlineAccountSignaturesTrimmedTimestamp && blockTimestamp < currentTrimmableTimestamp;
+	}
+
 	// JAXB special
 
 	@XmlElement(name = "minterAddress")

src/main/java/org/qortal/data/network/ArbitraryPeerData.java

@@ -0,0 +1,112 @@
+package org.qortal.data.network;
+
+import com.google.common.net.InetAddresses;
+import org.qortal.crypto.Crypto;
+import org.qortal.network.Peer;
+import org.qortal.utils.NTP;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+public class ArbitraryPeerData {
+
+    private final byte[] hash;
+    private final String peerAddress;
+    private Integer successes;
+    private Integer failures;
+    private Long lastAttempted;
+    private Long lastRetrieved;
+
+    public ArbitraryPeerData(byte[] hash, String peerAddress, Integer successes,
+                             Integer failures, Long lastAttempted, Long lastRetrieved) {
+        this.hash = hash;
+        this.peerAddress = peerAddress;
+        this.successes = successes;
+        this.failures = failures;
+        this.lastAttempted = lastAttempted;
+        this.lastRetrieved = lastRetrieved;
+    }
+
+    public ArbitraryPeerData(byte[] signature, Peer peer) {
+        this(Crypto.digest(signature), peer.getPeerData().getAddress().toString(),
+                0, 0, 0L, 0L);
+    }
+
+    public ArbitraryPeerData(byte[] signature, String peerAddress) {
+        this(Crypto.digest(signature), peerAddress, 0, 0, 0L, 0L);
+    }
+
+    public boolean isPeerAddressValid() {
+        // Validate the peer address to prevent arbitrary values being added to the db
+        String[] parts = this.peerAddress.split(":");
+        if (parts.length != 2) {
+            // Invalid format
+            return false;
+        }
+        String host = parts[0];
+        if (!InetAddresses.isInetAddress(host)) {
+            // Invalid host
+            return false;
+        }
+        int port = Integer.valueOf(parts[1]);
+        if (port <= 0 || port > 65535) {
+            // Invalid port
+            return false;
+        }
+
+        // Make sure that it's not a local address
+        try {
+            InetAddress addr = InetAddress.getByName(host);
+            if (addr.isLoopbackAddress() || addr.isLinkLocalAddress() || addr.isSiteLocalAddress()) {
+                // Ignore local addresses
+                return false;
+            }
+        } catch (UnknownHostException e) {
+            return false;
+        }
+
+        // Valid host/port combination
+        return true;
+    }
+
+    public void incrementSuccesses() {
+        this.successes++;
+    }
+
+    public void incrementFailures() {
+        this.failures++;
+    }
+
+    public void markAsAttempted() {
+        this.lastAttempted = NTP.getTime();
+    }
+
+    public void markAsRetrieved() {
+        this.lastRetrieved = NTP.getTime();
+    }
+
+    public byte[] getHash() {
+        return this.hash;
+    }
+
+    public String getPeerAddress() {
+        return this.peerAddress;
+    }
+
+    public Integer getSuccesses() {
+        return this.successes;
+    }
+
+    public Integer getFailures() {
+        return this.failures;
+    }
+
+    public Long getLastAttempted() {
+        return this.lastAttempted;
+    }
+
+    public Long getLastRetrieved() {
+        return this.lastRetrieved;
+    }
+
+}

src/main/java/org/qortal/data/network/PeerData.java

@@ -13,6 +13,8 @@ import io.swagger.v3.oas.annotations.media.Schema;
 @XmlAccessorType(XmlAccessType.FIELD)
 public class PeerData {
 
+	public static final int MAX_PEER_ADDRESS_SIZE = 255;
+
 	// Properties
 
 	// Don't expose this via JAXB - use pretty getter instead

src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java

@@ -1,17 +1,22 @@
 package org.qortal.data.transaction;
 
 import java.util.List;
+import java.util.Map;
 
 import javax.xml.bind.Unmarshaller;
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 
 import org.eclipse.persistence.oxm.annotations.XmlDiscriminatorValue;
+import org.qortal.arbitrary.misc.Service;
 import org.qortal.data.PaymentData;
 import org.qortal.transaction.Transaction.TransactionType;
 
 import io.swagger.v3.oas.annotations.media.Schema;
 
+import static java.util.Arrays.stream;
+import static java.util.stream.Collectors.toMap;
+
 // All properties to be converted to JSON via JAXB
 @XmlAccessorType(XmlAccessType.FIELD)
 @Schema(allOf = { TransactionData.class })
@@ -25,17 +30,65 @@ public class ArbitraryTransactionData extends TransactionData {
 		DATA_HASH;
 	}
 
+	// Methods
+	public enum Method {
+		PUT(0), // A complete replacement of a resource
+		PATCH(1); // An update / partial replacement of a resource
+
+		public final int value;
+
+		private static final Map<Integer, Method> map = stream(Method.values())
+				.collect(toMap(method -> method.value, method -> method));
+
+		Method(int value) {
+			this.value = value;
+		}
+
+		public static Method valueOf(int value) {
+			return map.get(value);
+		}
+	}
+
+	// Compression types
+	public enum Compression {
+		NONE(0),
+		ZIP(1);
+
+		public final int value;
+
+		private static final Map<Integer, Compression> map = stream(Compression.values())
+				.collect(toMap(compression -> compression.value, compression -> compression));
+
+		Compression(int value) {
+			this.value = value;
+		}
+
+		public static Compression valueOf(int value) {
+			return map.get(value);
+		}
+	}
+
 	// Properties
 	private int version;
-
 	@Schema(example = "sender_public_key")
 	private byte[] senderPublicKey;
 
-	private int service;
+	private Service service;
+	private int nonce;
+	private int size;
+
+	private String name;
+	private String identifier;
+	private Method method;
+	private byte[] secret;
+	private Compression compression;
 
 	@Schema(example = "raw_data_in_base58")
 	private byte[] data;
 	private DataType dataType;
+	@Schema(example = "metadata_file_hash_in_base58")
+	private byte[] metadataHash;
+
 	private List<PaymentData> payments;
 
 	// Constructors
@@ -50,14 +103,24 @@ public class ArbitraryTransactionData extends TransactionData {
 	}
 
 	public ArbitraryTransactionData(BaseTransactionData baseTransactionData,
-			int version, int service, byte[] data, DataType dataType, List<PaymentData> payments) {
+									int version, Service service, int nonce, int size,
+									String name, String identifier, Method method, byte[] secret, Compression compression,
+									byte[] data, DataType dataType, byte[] metadataHash, List<PaymentData> payments) {
 		super(TransactionType.ARBITRARY, baseTransactionData);
 
 		this.senderPublicKey = baseTransactionData.creatorPublicKey;
 		this.version = version;
 		this.service = service;
+		this.nonce = nonce;
+		this.size = size;
+		this.name = name;
+		this.identifier = identifier;
+		this.method = method;
+		this.secret = secret;
+		this.compression = compression;
 		this.data = data;
 		this.dataType = dataType;
+		this.metadataHash = metadataHash;
 		this.payments = payments;
 	}
 
@@ -71,10 +134,42 @@ public class ArbitraryTransactionData extends TransactionData {
 		return this.version;
 	}
 
-	public int getService() {
+	public Service getService() {
 		return this.service;
 	}
 
+	public int getNonce() {
+		return this.nonce;
+	}
+
+	public void setNonce(int nonce) {
+		this.nonce = nonce;
+	}
+
+	public int getSize() {
+		return this.size;
+	}
+
+	public String getName() {
+		return this.name;
+	}
+
+	public String getIdentifier() {
+		return (this.identifier != "") ? this.identifier : null;
+	}
+
+	public Method getMethod() {
+		return this.method;
+	}
+
+	public byte[] getSecret() {
+		return this.secret;
+	}
+
+	public Compression getCompression() {
+		return this.compression;
+	}
+
 	public byte[] getData() {
 		return this.data;
 	}
@@ -91,6 +186,14 @@ public class ArbitraryTransactionData extends TransactionData {
 		this.dataType = dataType;
 	}
 
+	public byte[] getMetadataHash() {
+		return this.metadataHash;
+	}
+
+	public void setMetadataHash(byte[] metadataHash) {
+		this.metadataHash = metadataHash;
+	}
+
 	public List<PaymentData> getPayments() {
 		return this.payments;
 	}

src/main/java/org/qortal/gui/SysTray.java

@@ -290,8 +290,8 @@ public class SysTray {
 	}
 
 	public void setTrayIcon(int iconid) {
-		if (trayIcon != null) {
-			try {
+		try {
+			if (trayIcon != null) {
 				switch (iconid) {
 					case 1:
 						this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_syncing_time-alt.png"));
@@ -306,9 +306,9 @@ public class SysTray {
 						this.trayIcon.setImage(Gui.loadImage("icons/qortal_ui_tray_synced.png"));
 						break;
 				}
-			} catch (NullPointerException e) {
-				LOGGER.info("Unable to set tray icon");
 			}
+		} catch (Exception e) {
+			LOGGER.info("Unable to set tray icon: {}", e.getMessage());
 		}
 	}
 

src/main/java/org/qortal/list/ResourceList.java

@@ -13,15 +13,15 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 public class ResourceList {
 
     private static final Logger LOGGER = LogManager.getLogger(ResourceList.class);
 
-    private String category;
-    private String resourceName;
-    private List<String> list = new ArrayList<>();
+    private String name;
+    private List<String> list = Collections.synchronizedList(new ArrayList<>());
 
     /**
      * ResourceList
@@ -29,13 +29,11 @@ public class ResourceList {
      * This can be used for local blocking, or even for curating and sharing content lists
      * Lists are backed off to JSON files (in the lists folder) to ease sharing between nodes and users
      *
-     * @param category - for instance "blacklist", "whitelist", or "userlist"
-     * @param resourceName - for instance "address", "poll", or "group"
+     * @param name - the name of the list, for instance "blockedAddresses"
      * @throws IOException
      */
-    public ResourceList(String category, String resourceName) throws IOException {
-        this.category = category;
-        this.resourceName = resourceName;
+    public ResourceList(String name) throws IOException {
+        this.name = name;
         this.load();
     }
 
@@ -43,17 +41,13 @@ public class ResourceList {
     /* Filesystem */
 
     private Path getFilePath() {
-        String pathString = String.format("%s%s%s_%s.json", Settings.getInstance().getListsPath(),
-                File.separator, this.category, this.resourceName);
+        String pathString = String.format("%s.json", Paths.get(Settings.getInstance().getListsPath(), this.name));
         return Paths.get(pathString);
     }
 
     public void save() throws IOException {
-        if (this.resourceName == null) {
-            throw new IllegalStateException("Can't save list with missing resource name");
-        }
-        if (this.category == null) {
-            throw new IllegalStateException("Can't save list with missing category");
+        if (this.name == null) {
+            throw new IllegalStateException("Can't save list with missing name");
         }
         String jsonString = ResourceList.listToJSONString(this.list);
         Path filePath = this.getFilePath();
@@ -91,7 +85,7 @@ public class ResourceList {
         try {
             return this.load();
         } catch (IOException e) {
-            LOGGER.info("Unable to revert {} {}", this.resourceName, this.category);
+            LOGGER.info("Unable to revert list {}: {}", this.name, e.getMessage());
         }
         return false;
     }
@@ -103,7 +97,7 @@ public class ResourceList {
         if (resource == null || this.list == null) {
             return;
         }
-        if (!this.contains(resource)) {
+        if (!this.contains(resource, true)) {
             this.list.add(resource);
         }
     }
@@ -115,11 +109,17 @@ public class ResourceList {
         this.list.remove(resource);
     }
 
-    public boolean contains(String resource) {
+    public boolean contains(String resource, boolean caseSensitive) {
         if (resource == null || this.list == null) {
             return false;
         }
-        return this.list.contains(resource);
+
+        if (caseSensitive) {
+            return this.list.contains(resource);
+        }
+        else {
+            return this.list.stream().anyMatch(resource::equalsIgnoreCase);
+        }
     }
 
 
@@ -153,16 +153,16 @@ public class ResourceList {
         return ResourceList.listToJSONString(this.list);
     }
 
-    public String getCategory() {
-        return this.category;
+    public String getName() {
+        return this.name;
     }
 
-    public String getResourceName() {
-        return this.resourceName;
+    public List<String> getList() {
+        return this.list;
     }
 
     public String toString() {
-        return String.format("%s %s", this.category, this.resourceName);
+        return this.name;
     }
 
 }

src/main/java/org/qortal/list/ResourceListManager.java

@@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -13,7 +14,7 @@ public class ResourceListManager {
     private static final Logger LOGGER = LogManager.getLogger(ResourceListManager.class);
 
     private static ResourceListManager instance;
-    private List<ResourceList> lists = new ArrayList<>();
+    private List<ResourceList> lists = Collections.synchronizedList(new ArrayList<>());
 
 
     public ResourceListManager() {
@@ -26,10 +27,9 @@ public class ResourceListManager {
         return instance;
     }
 
-    private ResourceList getList(String category, String resourceName) {
+    private ResourceList getList(String listName) {
         for (ResourceList list : this.lists) {
-            if (Objects.equals(list.getCategory(), category) &&
-                    Objects.equals(list.getResourceName(), resourceName)) {
+            if (Objects.equals(list.getName(), listName)) {
                 return list;
             }
         }
@@ -37,19 +37,19 @@ public class ResourceListManager {
         // List doesn't exist in array yet, so create it
         // This will load any existing data from the filesystem
         try {
-            ResourceList list = new ResourceList(category, resourceName);
+            ResourceList list = new ResourceList(listName);
             this.lists.add(list);
             return list;
 
         } catch (IOException e) {
-            LOGGER.info("Unable to load or create list {} {}: {}", category, resourceName, e.getMessage());
+            LOGGER.info("Unable to load or create list {}: {}", listName, e.getMessage());
             return null;
         }
 
     }
 
-    public boolean addToList(String category, String resourceName, String item, boolean save) {
-        ResourceList list = this.getList(category, resourceName);
+    public boolean addToList(String listName, String item, boolean save) {
+        ResourceList list = this.getList(listName);
         if (list == null) {
             return false;
         }
@@ -67,8 +67,8 @@ public class ResourceListManager {
         }
     }
 
-    public boolean removeFromList(String category, String resourceName, String item, boolean save) {
-        ResourceList list = this.getList(category, resourceName);
+    public boolean removeFromList(String listName, String item, boolean save) {
+        ResourceList list = this.getList(listName);
         if (list == null) {
             return false;
         }
@@ -87,16 +87,16 @@ public class ResourceListManager {
         }
     }
 
-    public boolean listContains(String category, String resourceName, String address) {
-        ResourceList list = this.getList(category, resourceName);
+    public boolean listContains(String listName, String item, boolean caseSensitive) {
+        ResourceList list = this.getList(listName);
         if (list == null) {
             return false;
         }
-        return list.contains(address);
+        return list.contains(item, caseSensitive);
     }
 
-    public void saveList(String category, String resourceName) {
-        ResourceList list = this.getList(category, resourceName);
+    public void saveList(String listName) {
+        ResourceList list = this.getList(listName);
         if (list == null) {
             return;
         }
@@ -109,20 +109,36 @@ public class ResourceListManager {
         }
     }
 
-    public void revertList(String category, String resourceName) {
-        ResourceList list = this.getList(category, resourceName);
+    public void revertList(String listName) {
+        ResourceList list = this.getList(listName);
         if (list == null) {
             return;
         }
         list.revert();
     }
 
-    public String getJSONStringForList(String category, String resourceName) {
-        ResourceList list = this.getList(category, resourceName);
+    public String getJSONStringForList(String listName) {
+        ResourceList list = this.getList(listName);
         if (list == null) {
             return null;
         }
         return list.getJSONString();
     }
 
+    public List<String> getStringsInList(String listName) {
+        ResourceList list = this.getList(listName);
+        if (list == null) {
+            return null;
+        }
+        return list.getList();
+    }
+
+    public int getItemCountForList(String listName) {
+        ResourceList list = this.getList(listName);
+        if (list == null) {
+            return 0;
+        }
+        return list.getList().size();
+    }
+
 }

src/main/java/org/qortal/network/Handshake.java

@@ -48,6 +48,9 @@ public enum Handshake {
 				return null;
 			}
 
+			// Make a note of the senderPeerAddress, as this should be our public IP
+			Network.getInstance().ourPeerAddressUpdated(helloMessage.getSenderPeerAddress());
+
 			String versionString = helloMessage.getVersionString();
 
 			Matcher matcher = peer.VERSION_PATTERN.matcher(versionString);
@@ -87,8 +90,9 @@ public enum Handshake {
 		public void action(Peer peer) {
 			String versionString = Controller.getInstance().getVersionString();
 			long timestamp = NTP.getTime();
+			String senderPeerAddress = peer.getPeerData().getAddress().toString();
 
-			Message helloMessage = new HelloMessage(timestamp, versionString);
+			Message helloMessage = new HelloMessage(timestamp, versionString, senderPeerAddress);
 			if (!peer.sendMessage(helloMessage))
 				peer.disconnect("failed to send HELLO");
 		}

src/main/java/org/qortal/network/Network.java

@@ -6,6 +6,8 @@ import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters;
 import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters;
 import org.qortal.block.BlockChain;
 import org.qortal.controller.Controller;
+import org.qortal.controller.arbitrary.ArbitraryDataFileListManager;
+import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.network.PeerData;
@@ -15,6 +17,7 @@ import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
 import org.qortal.utils.ExecuteProduceConsume;
 import org.qortal.utils.ExecuteProduceConsume.StatsSnapshot;
 import org.qortal.utils.NTP;
@@ -114,6 +117,9 @@ public class Network {
 
     private final Lock mergePeersLock = new ReentrantLock();
 
+    private List<String> ourExternalIpAddressHistory = new ArrayList<>();
+    private String ourExternalIpAddress = null;
+
     // Constructors
 
     private Network() {
@@ -234,6 +240,76 @@ public class Network {
         }
     }
 
+    public boolean requestDataFromPeer(String peerAddressString, byte[] signature) {
+        if (peerAddressString != null) {
+            PeerAddress peerAddress = PeerAddress.fromString(peerAddressString);
+
+            // Reuse an existing PeerData instance if it's already in the known peers list
+            PeerData peerData = this.allKnownPeers.stream()
+                    .filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress))
+                    .findFirst()
+                    .orElse(null);
+
+            if (peerData == null) {
+                // Not a known peer, so we need to create one
+                Long addedWhen = NTP.getTime();
+                String addedBy = "requestDataFromPeer";
+                peerData = new PeerData(peerAddress, addedWhen, addedBy);
+            }
+
+            if (peerData == null) {
+                LOGGER.info("PeerData is null when trying to request data from peer {}", peerAddressString);
+                return false;
+            }
+
+            // Check if we're already connected to and handshaked with this peer
+            Peer connectedPeer = this.connectedPeers.stream()
+                    .filter(p -> p.getPeerData().getAddress().equals(peerAddress))
+                    .findFirst()
+                    .orElse(null);
+            boolean isConnected = (connectedPeer != null);
+
+            boolean isHandshaked = this.getHandshakedPeers().stream()
+                    .anyMatch(p -> p.getPeerData().getAddress().equals(peerAddress));
+
+            if (isConnected && isHandshaked) {
+                // Already connected
+                return this.requestDataFromConnectedPeer(connectedPeer, signature);
+            }
+            else {
+                // We need to connect to this peer before we can request data
+                try {
+                    if (!isConnected) {
+                        // Add this signature to the list of pending requests for this peer
+                        LOGGER.info("Making connection to peer {} to request files for signature {}...", peerAddressString, Base58.encode(signature));
+                        Peer peer = new Peer(peerData);
+                        peer.addPendingSignatureRequest(signature);
+                        return this.connectPeer(peer);
+                        // If connection (and handshake) is successful, data will automatically be requested
+                    }
+                    else if (!isHandshaked) {
+                        LOGGER.info("Peer {} is connected but not handshaked. Not attempting a new connection.", peerAddress);
+                        return false;
+                    }
+
+                } catch (InterruptedException e) {
+                    LOGGER.info("Interrupted when connecting to peer {}", peerAddress);
+                    return false;
+                }
+            }
+        }
+        return false;
+    }
+
+    private boolean requestDataFromConnectedPeer(Peer connectedPeer, byte[] signature) {
+        if (signature == null) {
+            // Nothing to do
+            return false;
+        }
+
+        return ArbitraryDataFileListManager.getInstance().fetchArbitraryDataFileList(connectedPeer, signature);
+    }
+
     /**
      * Returns list of connected peers that have completed handshaking.
      */
@@ -648,14 +724,14 @@ public class Network {
         }
     }
 
-    private void connectPeer(Peer newPeer) throws InterruptedException {
+    private boolean connectPeer(Peer newPeer) throws InterruptedException {
         SocketChannel socketChannel = newPeer.connect(this.channelSelector);
         if (socketChannel == null) {
-            return;
+            return false;
         }
 
         if (Thread.currentThread().isInterrupted()) {
-            return;
+            return false;
         }
 
         synchronized (this.connectedPeers) {
@@ -663,6 +739,8 @@ public class Network {
         }
 
         this.onPeerReady(newPeer);
+
+        return true;
     }
 
     private Peer getPeerFromChannel(SocketChannel socketChannel) {
@@ -913,6 +991,17 @@ public class Network {
             }
         }
 
+        // Process any pending signature requests, as this peer may have been connected for this purpose only
+        List<byte[]> pendingSignatureRequests = new ArrayList<>(peer.getPendingSignatureRequests());
+        if (pendingSignatureRequests != null && !pendingSignatureRequests.isEmpty()) {
+            for (byte[] signature : pendingSignatureRequests) {
+                this.requestDataFromConnectedPeer(peer, signature);
+                peer.removePendingSignatureRequest(signature);
+            }
+        }
+
+        // FUTURE: we may want to disconnect from this peer if we've finished requesting data from it
+
         // Start regular pings
         peer.startPings();
 
@@ -1011,6 +1100,88 @@ public class Network {
         return new GetUnconfirmedTransactionsMessage();
     }
 
+
+    // External IP / peerAddress tracking
+
+    public void ourPeerAddressUpdated(String peerAddress) {
+        if (peerAddress == null || peerAddress.isEmpty()) {
+            return;
+        }
+
+        // Validate IP address
+        String[] parts = peerAddress.split(":");
+        if (parts.length != 2) {
+            return;
+        }
+        String host = parts[0];
+        try {
+            InetAddress addr = InetAddress.getByName(host);
+            if (addr.isAnyLocalAddress() || addr.isSiteLocalAddress()) {
+                // Ignore local addresses
+                return;
+            }
+        } catch (UnknownHostException e) {
+            return;
+        }
+
+        // Add to the list
+        this.ourExternalIpAddressHistory.add(host);
+
+        // Limit to 25 entries
+        while (this.ourExternalIpAddressHistory.size() > 25) {
+            this.ourExternalIpAddressHistory.remove(0);
+        }
+
+        // Now take a copy of the IP address history so it can be safely iterated
+        // Without this, another thread could remove an element, resulting in an exception
+        List<String> ipAddressHistory = new ArrayList<>(this.ourExternalIpAddressHistory);
+
+        // If we've had 10 consecutive matching addresses, and they're different from
+        // our stored IP address value, treat it as updated.
+        int consecutiveReadingsRequired = 10;
+        int size = ipAddressHistory.size();
+        if (size < consecutiveReadingsRequired) {
+            // Need at least 10 readings
+            return;
+        }
+
+        // Count the number of consecutive IP address readings
+        String lastReading = null;
+        int consecutiveReadings = 0;
+        for (int i = size-1; i >= 0; i--) {
+            String reading = ipAddressHistory.get(i);
+            if (lastReading != null) {
+                 if (Objects.equals(reading, lastReading)) {
+                    consecutiveReadings++;
+                 }
+                 else {
+                     consecutiveReadings = 0;
+                 }
+            }
+            lastReading = reading;
+        }
+
+        if (consecutiveReadings >= consecutiveReadingsRequired) {
+            // Last 10 readings were the same - i.e. more than one peer agreed on the new IP address...
+            String ip = ipAddressHistory.get(size - 1);
+            if (ip != null && !Objects.equals(ip, "null")) {
+                if (!Objects.equals(ip, this.ourExternalIpAddress)) {
+                    // ... and the readings were different to our current recorded value, so
+                    // update our external IP address value
+                    this.ourExternalIpAddress = ip;
+                    this.onExternalIpUpdate(ip);
+                }
+            }
+        }
+    }
+
+    public void onExternalIpUpdate(String ipAddress) {
+        LOGGER.info("External IP address updated to {}", ipAddress);
+
+        ArbitraryDataManager.getInstance().broadcastHostedSignatureList();
+    }
+
+
     // Peer-management calls
 
     public void noteToSelf(Peer peer) {

src/main/java/org/qortal/network/Peer.java

@@ -47,6 +47,11 @@ public class Peer {
      */
     private static final int RESPONSE_TIMEOUT = 3000; // ms
 
+    /**
+     * Maximum time to wait for a peer to respond with blocks (ms)
+     */
+    public static final int FETCH_BLOCKS_TIMEOUT = 10000;
+
     /**
      * Interval between PING messages to a peer. (ms)
      * <p>
@@ -99,6 +104,11 @@ public class Peer {
 
     private boolean syncInProgress = false;
 
+
+    /* Pending signature requests */
+    private List<byte[]> pendingSignatureRequests = Collections.synchronizedList(new ArrayList<>());
+
+
     // Versioning
     public static final Pattern VERSION_PATTERN = Pattern.compile(Controller.VERSION_PREFIX
             + "(\\d{1,3})\\.(\\d{1,5})\\.(\\d{1,5})");
@@ -350,6 +360,34 @@ public class Peer {
         this.syncInProgress = syncInProgress;
     }
 
+
+    // Pending signature requests
+
+    public void addPendingSignatureRequest(byte[] signature) {
+        // Check if we already have this signature in the list
+        for (byte[] existingSignature : this.pendingSignatureRequests) {
+            if (Arrays.equals(existingSignature, signature )) {
+                return;
+            }
+        }
+        this.pendingSignatureRequests.add(signature);
+    }
+
+    public void removePendingSignatureRequest(byte[] signature) {
+        Iterator iterator = this.pendingSignatureRequests.iterator();
+        while (iterator.hasNext()) {
+            byte[] existingSignature = (byte[]) iterator.next();
+            if (Arrays.equals(existingSignature, signature)) {
+                iterator.remove();
+            }
+        }
+    }
+
+    public List<byte[]> getPendingSignatureRequests() {
+        return this.pendingSignatureRequests;
+    }
+
+
     @Override
     public String toString() {
         // Easier, and nicer output, than peer.getRemoteSocketAddress()
@@ -435,16 +473,18 @@ public class Peer {
                     return;
                 }
 
-                if (bytesRead > 0) {
-                    byte[] leadingBytes = new byte[Math.min(bytesRead, 8)];
-                    this.byteBuffer.asReadOnlyBuffer().position(priorPosition).get(leadingBytes);
-                    String leadingHex = HashCode.fromBytes(leadingBytes).toString();
+                if (LOGGER.isTraceEnabled()) {
+                    if (bytesRead > 0) {
+                        byte[] leadingBytes = new byte[Math.min(bytesRead, 8)];
+                        this.byteBuffer.asReadOnlyBuffer().position(priorPosition).get(leadingBytes);
+                        String leadingHex = HashCode.fromBytes(leadingBytes).toString();
 
-                    LOGGER.trace("[{}] Received {} bytes, starting {}, into byteBuffer[{}] from peer {}",
-                            this.peerConnectionId, bytesRead, leadingHex, priorPosition, this);
-                } else {
-                    LOGGER.trace("[{}] Received {} bytes into byteBuffer[{}] from peer {}", this.peerConnectionId,
-                            bytesRead, priorPosition, this);
+                        LOGGER.trace("[{}] Received {} bytes, starting {}, into byteBuffer[{}] from peer {}",
+                                this.peerConnectionId, bytesRead, leadingHex, priorPosition, this);
+                    } else {
+                        LOGGER.trace("[{}] Received {} bytes into byteBuffer[{}] from peer {}", this.peerConnectionId,
+                                bytesRead, priorPosition, this);
+                    }
                 }
                 final boolean wasByteBufferFull = !this.byteBuffer.hasRemaining();
 
@@ -544,12 +584,22 @@ public class Peer {
     }
 
     /**
-     * Attempt to send Message to peer.
+     * Attempt to send Message to peer, using default RESPONSE_TIMEOUT.
      *
      * @param message message to be sent
      * @return <code>true</code> if message successfully sent; <code>false</code> otherwise
      */
     public boolean sendMessage(Message message) {
+        return this.sendMessageWithTimeout(message, RESPONSE_TIMEOUT);
+    }
+
+    /**
+     * Attempt to send Message to peer, using custom timeout.
+     *
+     * @param message message to be sent
+     * @return <code>true</code> if message successfully sent; <code>false</code> otherwise
+     */
+    public boolean sendMessageWithTimeout(Message message, int timeout) {
         if (!this.socketChannel.isOpen()) {
             return false;
         }
@@ -563,12 +613,14 @@ public class Peer {
 
             synchronized (this.socketChannel) {
                 final long sendStart = System.currentTimeMillis();
+                long totalBytes = 0;
 
                 while (outputBuffer.hasRemaining()) {
                     int bytesWritten = this.socketChannel.write(outputBuffer);
+                    totalBytes += bytesWritten;
 
-                    LOGGER.trace("[{}] Sent {} bytes of {} message with ID {} to peer {}", this.peerConnectionId,
-                            bytesWritten, message.getType().name(), message.getId(), this);
+                    LOGGER.trace("[{}] Sent {} bytes of {} message with ID {} to peer {} ({} total)", this.peerConnectionId,
+                            bytesWritten, message.getType().name(), message.getId(), this, totalBytes);
 
                     if (bytesWritten == 0) {
                         // Underlying socket's internal buffer probably full,
@@ -583,7 +635,7 @@ public class Peer {
                          */
                         Thread.sleep(1L); //NOSONAR squid:S2276
 
-                        if (System.currentTimeMillis() - sendStart > RESPONSE_TIMEOUT) {
+                        if (System.currentTimeMillis() - sendStart > timeout) {
                             // We've taken too long to send this message
                             return false;
                         }
@@ -604,7 +656,7 @@ public class Peer {
     }
 
     /**
-     * Send message to peer and await response.
+     * Send message to peer and await response, using default RESPONSE_TIMEOUT.
      * <p>
      * Message is assigned a random ID and sent.
      * If a response with matching ID is received then it is returned to caller.
@@ -618,6 +670,24 @@ public class Peer {
      * @throws InterruptedException if interrupted while waiting
      */
     public Message getResponse(Message message) throws InterruptedException {
+        return getResponseWithTimeout(message, RESPONSE_TIMEOUT);
+    }
+
+    /**
+     * Send message to peer and await response.
+     * <p>
+     * Message is assigned a random ID and sent.
+     * If a response with matching ID is received then it is returned to caller.
+     * <p>
+     * If no response with matching ID within timeout, or some other error/exception occurs,
+     * then return <code>null</code>.<br>
+     * (Assume peer will be rapidly disconnected after this).
+     *
+     * @param message message to send
+     * @return <code>Message</code> if valid response received; <code>null</code> if not or error/exception occurs
+     * @throws InterruptedException if interrupted while waiting
+     */
+    public Message getResponseWithTimeout(Message message, int timeout) throws InterruptedException {
         BlockingQueue<Message> blockingQueue = new ArrayBlockingQueue<>(1);
 
         // Assign random ID to this message
@@ -632,13 +702,13 @@ public class Peer {
         message.setId(id);
 
         // Try to send message
-        if (!this.sendMessage(message)) {
+        if (!this.sendMessageWithTimeout(message, timeout)) {
             this.replyQueues.remove(id);
             return null;
         }
 
         try {
-            return blockingQueue.poll(RESPONSE_TIMEOUT, TimeUnit.MILLISECONDS);
+            return blockingQueue.poll(timeout, TimeUnit.MILLISECONDS);
         } finally {
             this.replyQueues.remove(id);
         }

src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java

@@ -0,0 +1,90 @@
+package org.qortal.network.message;
+
+import com.google.common.primitives.Ints;
+import org.qortal.transform.TransformationException;
+import org.qortal.transform.Transformer;
+import org.qortal.utils.Serialization;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+public class ArbitraryDataFileListMessage extends Message {
+
+	private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
+	private static final int HASH_LENGTH = Transformer.SHA256_LENGTH;
+
+	private final byte[] signature;
+	private final List<byte[]> hashes;
+
+	public ArbitraryDataFileListMessage(byte[] signature, List<byte[]> hashes) {
+		super(MessageType.ARBITRARY_DATA_FILE_LIST);
+
+		this.signature = signature;
+		this.hashes = hashes;
+	}
+
+	public ArbitraryDataFileListMessage(int id, byte[] signature, List<byte[]> hashes) {
+		super(id, MessageType.ARBITRARY_DATA_FILE_LIST);
+
+		this.signature = signature;
+		this.hashes = hashes;
+	}
+
+	public List<byte[]> getHashes() {
+		return this.hashes;
+	}
+
+	public byte[] getSignature() {
+		return this.signature;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException, TransformationException {
+		byte[] signature = new byte[SIGNATURE_LENGTH];
+		bytes.get(signature);
+
+		int count = bytes.getInt();
+
+		if (bytes.remaining() != count * HASH_LENGTH)
+			return null;
+
+		List<byte[]> hashes = new ArrayList<>();
+		for (int i = 0; i < count; ++i) {
+
+			byte[] hash = new byte[HASH_LENGTH];
+			bytes.get(hash);
+			hashes.add(hash);
+		}
+
+		return new ArbitraryDataFileListMessage(id, signature, hashes);
+	}
+
+	@Override
+	protected byte[] toData() {
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+			bytes.write(this.signature);
+
+			bytes.write(Ints.toByteArray(this.hashes.size()));
+
+			for (byte[] hash : this.hashes) {
+				bytes.write(hash);
+			}
+
+			return bytes.toByteArray();
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+	public ArbitraryDataFileListMessage cloneWithNewId(int newId) {
+		ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes);
+		clone.setId(newId);
+		return clone;
+	}
+
+}

src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java

@@ -0,0 +1,91 @@
+package org.qortal.network.message;
+
+import com.google.common.primitives.Ints;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.repository.DataException;
+import org.qortal.transform.Transformer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+public class ArbitraryDataFileMessage extends Message {
+
+	private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
+
+	private final byte[] signature;
+	private final ArbitraryDataFile arbitraryDataFile;
+
+	public ArbitraryDataFileMessage(byte[] signature, ArbitraryDataFile arbitraryDataFile) {
+		super(MessageType.ARBITRARY_DATA_FILE);
+
+		this.signature = signature;
+		this.arbitraryDataFile = arbitraryDataFile;
+	}
+
+	public ArbitraryDataFileMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) {
+		super(id, MessageType.ARBITRARY_DATA_FILE);
+
+		this.signature = signature;
+		this.arbitraryDataFile = arbitraryDataFile;
+	}
+
+	public ArbitraryDataFile getArbitraryDataFile() {
+		return this.arbitraryDataFile;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException {
+		byte[] signature = new byte[SIGNATURE_LENGTH];
+		byteBuffer.get(signature);
+
+		int dataLength = byteBuffer.getInt();
+
+		if (byteBuffer.remaining() != dataLength)
+			return null;
+
+		byte[] data = new byte[dataLength];
+		byteBuffer.get(data);
+
+		try {
+			ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data, signature);
+			return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile);
+		}
+		catch (DataException e) {
+			return null;
+		}
+	}
+
+	@Override
+	protected byte[] toData() {
+		if (this.arbitraryDataFile == null) {
+			return null;
+		}
+
+		byte[] data = this.arbitraryDataFile.getBytes();
+		if (data == null) {
+			return null;
+		}
+
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+			bytes.write(signature);
+
+			bytes.write(Ints.toByteArray(data.length));
+
+			bytes.write(data);
+
+			return bytes.toByteArray();
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+	public ArbitraryDataFileMessage cloneWithNewId(int newId) {
+		ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.signature, this.arbitraryDataFile);
+		clone.setId(newId);
+		return clone;
+	}
+
+}