Merge branch 'master' into sync-multiple-blocks

# Conflicts:
#	src/main/java/org/qortal/settings/Settings.java

.gitignore

@@ -1,5 +1,4 @@
 /db*
-/lists/
 /bin/
 /target/
 /qortal-backup/
@@ -16,8 +15,8 @@
 /settings.json
 /testnet*
 /settings*.json
-/testchain*.json
-/run-testnet*.sh
+/testchain.json
+/run-testnet.sh
 /.idea
 /qortal.iml
 .DS_Store
@@ -26,8 +25,3 @@
 /run.pid
 /run.log
 /WindowsInstaller/Install Files/qortal.jar
-/*.7z
-/tmp
-/data*
-/src/test/resources/arbitrary/*/.qortal/cache
-apikey.txt

Dockerfile

@@ -1,26 +0,0 @@
-FROM maven:3-openjdk-11 as builder
-
-WORKDIR /work
-COPY ./ /work/
-RUN mvn clean package
-
-###
-FROM openjdk:11
-
-RUN useradd -r -u 1000 -g users qortal && \
-    mkdir /usr/local/qortal /qortal && \
-    chown 1000:100 /qortal
-
-COPY --from=builder /work/log4j2.properties /usr/local/qortal/
-COPY --from=builder /work/target/qortal*.jar /usr/local/qortal/qortal.jar
-
-USER 1000:100
-
-EXPOSE 12391 12392
-HEALTHCHECK --start-period=5m CMD curl -sf http://127.0.0.1:12391/admin/info || exit 1
-
-WORKDIR /qortal
-VOLUME /qortal
-
-ENTRYPOINT ["java"]
-CMD ["-Djava.net.preferIPv4Stack=false", "-jar", "/usr/local/qortal/qortal.jar"]

WindowsInstaller/Install Files/log4j2.properties

@@ -61,7 +61,7 @@ appender.rolling.type = RollingFile
 appender.rolling.name = FILE
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-appender.rolling.filePattern = ./${filename}.%i
+appender.rolling.filePattern = ${dirname:-}${filename}.%i
 appender.rolling.policy.type = SizeBasedTriggeringPolicy
 appender.rolling.policy.size = 4MB
 # Set the immediate flush to true (default)

WindowsInstaller/Qortal.aip

@@ -2,7 +2,7 @@
 <DOCUMENT Type="Advanced Installer" CreateVersion="14.9" version="18.2" Modules="enterprise" RootPath="." Language="en_GB" Id="{713E21E0-28FC-422F-8A95-823D01A5F80B}">
   <COMPONENT cid="caphyon.advinst.msicomp.MsiPropsComponent">
     <ROW Property="AI_BITMAP_DISPLAY_MODE" Value="0"/>
-    <ROW Property="AI_CURRENT_YEAR" Value="2022" ValueLocId="-"/>
+    <ROW Property="AI_CURRENT_YEAR" Value="2021" ValueLocId="-"/>
     <ROW Property="AI_FINDEXE_TITLE" Value="Select the installation package for [|ProductName]" ValueLocId="AI.Property.FindExeTitle"/>
     <ROW Property="AI_PRODUCTNAME_ARP" Value="Qortal"/>
     <ROW Property="AI_RUN_AS_ADMIN" Value="0"/>
@@ -17,10 +17,10 @@
     <ROW Property="Manufacturer" Value="Qortal"/>
     <ROW Property="MsiLogging" MultiBuildValue="DefaultBuild:vp"/>
     <ROW Property="NTP_GOOD" Value="false"/>
-    <ROW Property="ProductCode" Value="1033:{5FC8DCC3-BF9C-4D72-8C6D-940340ACD1B8} 1049:{1DEF14AB-2397-4517-B3C8-13221B921753} 2052:{B9E3C1DF-C92D-440A-9A21-869582F8585F} 2057:{91D69E7B-CA7D-4449-8E8A-F22DCEA546FC} " Type="16"/>
+    <ROW Property="ProductCode" Value="1033:{EB5562C3-664E-4A8B-8907-6D2033B98836} 1049:{36D0E774-B970-4A13-BCC4-1BA6AB3B2633} 2052:{AF6B6B44-9404-403A-B00F-B7110C28E453} 2057:{68BB9EB8-5991-42E5-841C-E76ACE51166D} " Type="16"/>
     <ROW Property="ProductLanguage" Value="2057"/>
     <ROW Property="ProductName" Value="Qortal"/>
-    <ROW Property="ProductVersion" Value="3.1.1" Type="32"/>
+    <ROW Property="ProductVersion" Value="1.5.4" Type="32"/>
     <ROW Property="RECONFIG_NTP" Value="true"/>
     <ROW Property="REMOVE_BLOCKCHAIN" Value="YES" Type="4"/>
     <ROW Property="REPAIR_BLOCKCHAIN" Value="YES" Type="4"/>
@@ -212,7 +212,7 @@
     <ROW Component="ADDITIONAL_LICENSE_INFO_71" ComponentId="{12A3ADBE-BB7A-496C-8869-410681E6232F}" Directory_="jdk.zipfs_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_71" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_8" ComponentId="{D53AD95E-CF96-4999-80FC-5812277A7456}" Directory_="java.naming_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_8" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_9" ComponentId="{6B7EA9B0-5D17-47A8-B78C-FACE86D15E01}" Directory_="java.net.http_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_9" Type="0"/>
-    <ROW Component="AI_CustomARPName" ComponentId="{42F5EC19-E46F-4299-B9F7-6E1112F6E4FB}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
+    <ROW Component="AI_CustomARPName" ComponentId="{83DFE721-3F68-4ABE-8697-8EC3A91EEB8A}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
     <ROW Component="AI_ExePath" ComponentId="{3644948D-AE0B-41BB-9FAF-A79E70490A08}" Directory_="APPDIR" Attributes="260" KeyPath="AI_ExePath"/>
     <ROW Component="APPDIR" ComponentId="{680DFDDE-3FB4-47A5-8FF5-934F576C6F91}" Directory_="APPDIR" Attributes="0"/>
     <ROW Component="AccessBridgeCallbacks.h" ComponentId="{288055D1-1062-47A3-AA44-5601B4E38AED}" Directory_="bridge_Dir" Attributes="0" KeyPath="AccessBridgeCallbacks.h" Type="0"/>
@@ -1173,7 +1173,7 @@
     <ROW Action="AI_STORE_LOCATION" Type="51" Source="ARPINSTALLLOCATION" Target="[APPDIR]"/>
     <ROW Action="AI_SetPermissions" Type="11265" Source="userAccounts.dll" Target="OnSetPermissions" WithoutSeq="true"/>
     <ROW Action="CustomizeLog4j2PropertiesScript" Type="3109" Target="Script Text" TargetUnformatted="var actionData = Session.Property(&quot;CustomActionData&quot;);&#13;&#10;var actionDataArray = actionData.split(&quot;|&quot;);&#13;&#10;var appDir = actionDataArray[0];&#13;&#10;var dataFolder = actionDataArray[1] + actionDataArray[2] + &quot;\\&quot;;&#13;&#10;&#13;&#10;var ForReading = 1, ForWriting = 2, ForAppending = 8;&#13;&#10;var fso = new ActiveXObject(&quot;Scripting.FileSystemObject&quot;);&#13;&#10;&#13;&#10;// Make copy&#13;&#10;fso.CopyFile(appDir + &quot;log4j2.properties&quot;, appDir + &quot;log4j2-orig.properties&quot;, true); // overwrite&#13;&#10;&#13;&#10;// Rewrite %AppDir%\log4j2.properties to update logfile storage path&#13;&#10;var fin = fso.OpenTextFile(appDir + &quot;log4j2-orig.properties&quot;, ForReading, false); // no create&#13;&#10;var fout = fso.OpenTextFile(appDir + &quot;log4j2.properties&quot;, ForWriting, true); // can create&#13;&#10;&#13;&#10;// Copy lines with rewriting where necessary&#13;&#10;while( !fin.AtEndOfStream ) {&#13;&#10;&#9;var line = fin.ReadLine();&#13;&#10;&#13;&#10;&#9;var start = line.indexOf(&quot;property.dirname&quot;);&#13;&#10;&#9;if (start &gt; 0) {&#13;&#10;&#9;&#9;// line: # property.dirname = ...appdata...&#13;&#10;&#9;&#9;// uncomment/replace this line for Windows&#13;&#10;&#9;&#9;fout.WriteLine( &quot;property.dirname = &quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) );&#13;&#10;&#9;} else {&#13;&#10;&#9;&#9;// not found - output verbatim&#13;&#10;&#9;&#9;fout.WriteLine( line );&#13;&#10;&#9;}&#13;&#10;}&#13;&#10;&#13;&#10;fin.Close();&#13;&#10;fout.Close();&#13;&#10;" AdditionalSeq="AI_DATA_SETTER_4"/>
-    <ROW Action="CustomizeSettingsJsonScript" Type="3109" Target="Script Text" TargetUnformatted="var actionData = Session.Property(&quot;CustomActionData&quot;);&#13;&#10;var actionDataArray = actionData.split(&quot;|&quot;);&#13;&#10;var appDir = actionDataArray[0];&#13;&#10;var dataFolder = actionDataArray[1] + actionDataArray[2] + &quot;\\&quot;;&#13;&#10;&#13;&#10;var ForReading = 1, ForWriting = 2, ForAppending = 8;&#13;&#10;var fso = new ActiveXObject(&quot;Scripting.FileSystemObject&quot;);&#13;&#10;&#13;&#10;// Create basic %APPDIR%\settings.json with path to real settings.json in dataFolder&#13;&#10;var fts = fso.OpenTextFile(appDir + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;fts.WriteLine( &quot;{&quot; );&#13;&#10;// We need to escape Windows path backslashes to keep JSON valid&#13;&#10;fts.WriteLine( &quot;  \&quot;userPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;\&quot;&quot; );&#13;&#10;fts.WriteLine( &quot;}&quot; );&#13;&#10;&#13;&#10;fts.Close();&#13;&#10;&#13;&#10;// Make copy&#13;&#10;fso.CopyFile(dataFolder + &quot;settings.json&quot;, dataFolder + &quot;settings-orig.json&quot;, true); // overwrite&#13;&#10;&#13;&#10;// Rewrite settings.json to update repository path&#13;&#10;var fin = fso.OpenTextFile(dataFolder + &quot;settings-orig.json&quot;, ForReading, false);&#13;&#10;var fout = fso.OpenTextFile(dataFolder + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;// First line should contain opening brace&#13;&#10;fout.WriteLine( fin.ReadLine() );&#13;&#10;&#13;&#10;// Append our entries&#13;&#10;fout.WriteLine( &quot;  \&quot;repositoryPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;db\&quot;,&quot; );&#13;&#10;fout.WriteLine( &quot;  \&quot;dataPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;data\&quot;,&quot; );&#13;&#10;&#13;&#10;// copy rest of settings&#13;&#10;while( !fin.AtEndOfStream ) {&#13;&#10;&#9;fout.WriteLine( fin.ReadLine() );&#13;&#10;}&#13;&#10;&#13;&#10;fin.Close();&#13;&#10;fout.Close();&#13;&#10;" AdditionalSeq="AI_DATA_SETTER_3"/>
+    <ROW Action="CustomizeSettingsJsonScript" Type="3109" Target="Script Text" TargetUnformatted="var actionData = Session.Property(&quot;CustomActionData&quot;);&#13;&#10;var actionDataArray = actionData.split(&quot;|&quot;);&#13;&#10;var appDir = actionDataArray[0];&#13;&#10;var dataFolder = actionDataArray[1] + actionDataArray[2] + &quot;\\&quot;;&#13;&#10;&#13;&#10;var ForReading = 1, ForWriting = 2, ForAppending = 8;&#13;&#10;var fso = new ActiveXObject(&quot;Scripting.FileSystemObject&quot;);&#13;&#10;&#13;&#10;// Create basic %APPDIR%\settings.json with path to real settings.json in dataFolder&#13;&#10;var fts = fso.OpenTextFile(appDir + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;fts.WriteLine( &quot;{&quot; );&#13;&#10;// We need to escape Windows path backslashes to keep JSON valid&#13;&#10;fts.WriteLine( &quot;  \&quot;userPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;\&quot;&quot; );&#13;&#10;fts.WriteLine( &quot;}&quot; );&#13;&#10;&#13;&#10;fts.Close();&#13;&#10;&#13;&#10;// Make copy&#13;&#10;fso.CopyFile(dataFolder + &quot;settings.json&quot;, dataFolder + &quot;settings-orig.json&quot;, true); // overwrite&#13;&#10;&#13;&#10;// Rewrite settings.json to update repository path&#13;&#10;var fin = fso.OpenTextFile(dataFolder + &quot;settings-orig.json&quot;, ForReading, false);&#13;&#10;var fout = fso.OpenTextFile(dataFolder + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;// First line should contain opening brace&#13;&#10;fout.WriteLine( fin.ReadLine() );&#13;&#10;&#13;&#10;// Append our entries&#13;&#10;fout.WriteLine( &quot;  \&quot;repositoryPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;db\&quot;,&quot; );&#13;&#10;&#13;&#10;// copy rest of settings&#13;&#10;while( !fin.AtEndOfStream ) {&#13;&#10;&#9;fout.WriteLine( fin.ReadLine() );&#13;&#10;}&#13;&#10;&#13;&#10;fin.Close();&#13;&#10;fout.Close();&#13;&#10;" AdditionalSeq="AI_DATA_SETTER_3"/>
     <ROW Action="DetectRunningProcess" Type="1" Source="aicustact.dll" Target="DetectProcess" Options="3" AdditionalSeq="AI_DATA_SETTER_8"/>
     <ROW Action="DetectW32Time" Type="1" Source="aicustact.dll" Target="DetectService" Options="3" AdditionalSeq="AI_DATA_SETTER_11"/>
     <ROW Action="NTP_config" Type="3090" Source="ntpcfg.bat"/>

log4j2.properties

@@ -61,7 +61,7 @@ appender.rolling.type = RollingFile
 appender.rolling.name = FILE
 appender.rolling.layout.type = PatternLayout
 appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
-appender.rolling.filePattern = ./${filename}.%i
+appender.rolling.filePattern = ${dirname:-}${filename}.%i
 appender.rolling.policy.type = SizeBasedTriggeringPolicy
 appender.rolling.policy.size = 4MB
 # Set the immediate flush to true (default)

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>3.1.1</version>
+	<version>1.5.4</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>
@@ -14,23 +14,17 @@
 		<ciyam-at.version>1.3.8</ciyam-at.version>
 		<commons-net.version>3.6</commons-net.version>
 		<commons-text.version>1.8</commons-text.version>
-		<commons-io.version>2.6</commons-io.version>
-		<commons-compress.version>1.21</commons-compress.version>
-		<commons-lang3.version>3.12.0</commons-lang3.version>
-		<xz.version>1.9</xz.version>
 		<dagger.version>1.2.2</dagger.version>
 		<guava.version>28.1-jre</guava.version>
 		<hsqldb.version>2.5.1</hsqldb.version>
 		<jersey.version>2.29.1</jersey.version>
 		<jetty.version>9.4.29.v20200521</jetty.version>
-		<log4j.version>2.17.1</log4j.version>
+		<log4j.version>2.12.1</log4j.version>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<slf4j.version>1.7.12</slf4j.version>
 		<swagger-api.version>2.0.9</swagger-api.version>
 		<swagger-ui.version>3.23.8</swagger-ui.version>
 		<package-info-maven-plugin.version>1.1.0</package-info-maven-plugin.version>
-		<jsoup.version>1.13.1</jsoup.version>
-		<java-diff-utils.version>4.10</java-diff-utils.version>
 	</properties>
 	<build>
 		<sourceDirectory>src/main/java</sourceDirectory>
@@ -455,26 +449,6 @@
 			<artifactId>commons-text</artifactId>
 			<version>${commons-text.version}</version>
 		</dependency>
-		<dependency>
-			<groupId>commons-io</groupId>
-			<artifactId>commons-io</artifactId>
-			<version>${commons-io.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.commons</groupId>
-			<artifactId>commons-compress</artifactId>
-			<version>${commons-compress.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.commons</groupId>
-			<artifactId>commons-lang3</artifactId>
-			<version>${commons-lang3.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.tukaani</groupId>
-			<artifactId>xz</artifactId>
-			<version>${xz.version}</version>
-		</dependency>
 		<!--  For bitset/bitmap compression -->
 		<dependency>
 			<groupId>io.druid</groupId>
@@ -675,15 +649,5 @@
 			<artifactId>bctls-jdk15on</artifactId>
 			<version>${bouncycastle.version}</version>
 		</dependency>
-		<dependency>
-			<groupId>org.jsoup</groupId>
-			<artifactId>jsoup</artifactId>
-			<version>${jsoup.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>io.github.java-diff-utils</groupId>
-			<artifactId>java-diff-utils</artifactId>
-			<version>${java-diff-utils.version}</version>
-		</dependency>
 	</dependencies>
 </project>

src/main/java/org/qortal/ApplyUpdate.java

@@ -7,13 +7,14 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.security.Security;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
-import org.qortal.api.ApiKey;
 import org.qortal.api.ApiRequest;
 import org.qortal.controller.AutoUpdate;
 import org.qortal.settings.Settings;
@@ -69,40 +70,14 @@ public class ApplyUpdate {
 		String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
 		LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri));
 
-		// The /admin/stop endpoint requires an API key, which may or may not be already generated
-		boolean apiKeyNewlyGenerated = false;
-		ApiKey apiKey = null;
-		try {
-			apiKey = new ApiKey();
-			if (!apiKey.generated()) {
-				apiKey.generate();
-				apiKeyNewlyGenerated = true;
-				LOGGER.info("Generated API key");
-			}
-		} catch (IOException e) {
-			LOGGER.info("Error loading API key: {}", e.getMessage());
-		}
-
-		// Create GET params
-		Map<String, String> params = new HashMap<>();
-		if (apiKey != null) {
-			params.put("apiKey", apiKey.toString());
-		}
-
-		// Attempt to stop the node
 		int attempt;
 		for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) {
 			final int attemptForLogging = attempt;
 			LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS));
-			String response = ApiRequest.perform(baseUri + "admin/stop", params);
-			if (response == null) {
+			String response = ApiRequest.perform(baseUri + "admin/stop", null);
+			if (response == null)
 				// No response - consider node shut down
-				if (apiKeyNewlyGenerated) {
-					// API key was newly generated for this auto update, so we need to remove it
-					ApplyUpdate.removeGeneratedApiKey();
-				}
 				return true;
-			}
 
 			LOGGER.info(() -> String.format("Response from API: %s", response));
 
@@ -114,11 +89,6 @@ public class ApplyUpdate {
 			}
 		}
 
-		if (apiKeyNewlyGenerated) {
-			// API key was newly generated for this auto update, so we need to remove it
-			ApplyUpdate.removeGeneratedApiKey();
-		}
-
 		if (attempt == MAX_ATTEMPTS) {
 			LOGGER.error("Failed to shutdown node - giving up");
 			return false;
@@ -127,19 +97,6 @@ public class ApplyUpdate {
 		return true;
 	}
 
-	private static void removeGeneratedApiKey() {
-		try {
-			LOGGER.info("Removing newly generated API key...");
-
-			// Delete the API key since it was only generated for this auto update
-			ApiKey apiKey = new ApiKey();
-			apiKey.delete();
-
-		} catch (IOException e) {
-			LOGGER.info("Error loading or deleting API key: {}", e.getMessage());
-		}
-	}
-
 	private static void replaceJar() {
 		// Assuming current working directory contains the JAR files
 		Path realJar = Paths.get(JAR_FILENAME);

src/main/java/org/qortal/RepositoryMaintenance.java

@@ -1,7 +1,6 @@
 package org.qortal;
 
 import java.security.Security;
-import java.util.concurrent.TimeoutException;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -58,10 +57,10 @@ public class RepositoryMaintenance {
 
 		LOGGER.info("Starting repository periodic maintenance. This can take a while...");
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			repository.performPeriodicMaintenance(null);
+			repository.performPeriodicMaintenance();
 
 			LOGGER.info("Repository periodic maintenance completed");
-		} catch (DataException | TimeoutException e) {
+		} catch (DataException e) {
 			LOGGER.error("Repository periodic maintenance failed", e);
 		}
 

src/main/java/org/qortal/account/Account.java

@@ -272,7 +272,7 @@ public class Account {
 	/**
 	 * Returns 'effective' minting level, or zero if reward-share does not exist.
 	 * <p>
-	 * this is being used on src/main/java/org/qortal/api/resource/AddressesResource.java to fulfil the online accounts api call
+	 * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
 	 * 
 	 * @param repository
 	 * @param rewardSharePublicKey
@@ -288,26 +288,5 @@ public class Account {
 		Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
 		return rewardShareMinter.getEffectiveMintingLevel();
 	}
-	/**
-	 * Returns 'effective' minting level, with a fix for the zero level.
-	 * <p>
-	 * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
-	 *
-	 * @param repository
-	 * @param rewardSharePublicKey
-	 * @return 0+
-	 * @throws DataException
-	 */
-	public static int getRewardShareEffectiveMintingLevelIncludingLevelZero(Repository repository, byte[] rewardSharePublicKey) throws DataException {
-		// Find actual minter and get their effective minting level
-		RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey);
-		if (rewardShareData == null)
-			return 0;
 
-		else if(!rewardShareData.getMinter().equals(rewardShareData.getRecipient()))//the minter is different than the recipient this means sponsorship
-			return 0;
-
-		Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
-		return rewardShareMinter.getEffectiveMintingLevel();
-	}
 }

src/main/java/org/qortal/api/ApiError.java

@@ -129,14 +129,7 @@ public enum ApiError {
 	// Foreign blockchain
 	FOREIGN_BLOCKCHAIN_NETWORK_ISSUE(1201, 500),
 	FOREIGN_BLOCKCHAIN_BALANCE_ISSUE(1202, 402),
-	FOREIGN_BLOCKCHAIN_TOO_SOON(1203, 408),
-
-	// Trade portal
-	ORDER_SIZE_TOO_SMALL(1300, 402),
-
-	// Data
-	FILE_NOT_FOUND(1401, 404),
-	NO_REPLY(1402, 404);
+	FOREIGN_BLOCKCHAIN_TOO_SOON(1203, 408);
 
 	private static final Map<Integer, ApiError> map = stream(ApiError.values()).collect(toMap(apiError -> apiError.code, apiError -> apiError));
 
@@ -164,4 +157,4 @@ public enum ApiError {
 		return this.status;
 	}
 
-}
+}

src/main/java/org/qortal/api/ApiExceptionFactory.java

@@ -16,8 +16,4 @@ public enum ApiExceptionFactory {
 		return createException(request, apiError, null);
 	}
 
-	public ApiException createCustomException(HttpServletRequest request, ApiError apiError, String message) {
-		return new ApiException(apiError.getStatus(), apiError.getCode(), message, null);
-	}
-
 }

src/main/java/org/qortal/api/ApiKey.java

@@ -1,107 +0,0 @@
-package org.qortal.api;
-
-import org.qortal.settings.Settings;
-import org.qortal.utils.Base58;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.SecureRandom;
-
-public class ApiKey {
-
-    private String apiKey;
-
-    public ApiKey() throws IOException {
-        this.load();
-    }
-
-    public void generate() throws IOException {
-        byte[] apiKey = new byte[16];
-        new SecureRandom().nextBytes(apiKey);
-        this.apiKey = Base58.encode(apiKey);
-
-        this.save();
-    }
-
-
-    /* Filesystem */
-
-    private Path getFilePath() {
-        return Paths.get(Settings.getInstance().getApiKeyPath(), "apikey.txt");
-    }
-
-    private boolean load() throws IOException {
-        Path path = this.getFilePath();
-        File apiKeyFile = new File(path.toString());
-        if (!apiKeyFile.exists()) {
-            // Try settings - to allow legacy API keys to be supported
-            return this.loadLegacyApiKey();
-        }
-
-        try {
-            this.apiKey = new String(Files.readAllBytes(path));
-
-        } catch (IOException e) {
-            throw new IOException(String.format("Couldn't read contents from file %s", path.toString()));
-        }
-
-        return true;
-    }
-
-    private boolean loadLegacyApiKey() {
-        String legacyApiKey = Settings.getInstance().getApiKey();
-        if (legacyApiKey != null && !legacyApiKey.isEmpty()) {
-            this.apiKey = Settings.getInstance().getApiKey();
-
-            try {
-                // Save it to the apikey file
-                this.save();
-            } catch (IOException e) {
-                // Ignore failures as it will be reloaded from settings next time
-            }
-            return true;
-        }
-        return false;
-    }
-
-    public void save() throws IOException {
-        if (this.apiKey == null || this.apiKey.isEmpty()) {
-            throw new IllegalStateException("Unable to save a blank API key");
-        }
-
-        Path filePath = this.getFilePath();
-
-        BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toString()));
-        writer.write(this.apiKey);
-        writer.close();
-    }
-
-    public void delete() throws IOException {
-        this.apiKey = null;
-
-        Path filePath = this.getFilePath();
-        if (Files.exists(filePath)) {
-            Files.delete(filePath);
-        }
-    }
-
-
-    public boolean generated() {
-        return (this.apiKey != null);
-    }
-
-    public boolean exists() {
-        return this.getFilePath().toFile().exists();
-    }
-
-    @Override
-    public String toString() {
-        return this.apiKey;
-    }
-
-}

src/main/java/org/qortal/api/ApiService.java

@@ -14,7 +14,6 @@ import java.security.SecureRandom;
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.SSLContext;
 
-import org.checkerframework.checker.units.qual.A;
 import org.eclipse.jetty.http.HttpVersion;
 import org.eclipse.jetty.rewrite.handler.RedirectPatternRule;
 import org.eclipse.jetty.rewrite.handler.RewriteHandler;
@@ -55,7 +54,6 @@ public class ApiService {
 
 	private final ResourceConfig config;
 	private Server server;
-	private ApiKey apiKey;
 
 	private ApiService() {
 		this.config = new ResourceConfig();
@@ -76,15 +74,6 @@ public class ApiService {
 		return this.config.getClasses();
 	}
 
-	public void setApiKey(ApiKey apiKey) {
-		this.apiKey = apiKey;
-	}
-
-	public ApiKey getApiKey() {
-		return this.apiKey;
-	}
-
-
 	public void start() {
 		try {
 			// Create API server

src/main/java/org/qortal/api/DomainMapService.java

@@ -1,171 +0,0 @@
-package org.qortal.api;
-
-import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
-import org.eclipse.jetty.http.HttpVersion;
-import org.eclipse.jetty.rewrite.handler.RewriteHandler;
-import org.eclipse.jetty.rewrite.handler.RewritePatternRule;
-import org.eclipse.jetty.server.*;
-import org.eclipse.jetty.server.handler.ErrorHandler;
-import org.eclipse.jetty.server.handler.InetAccessHandler;
-import org.eclipse.jetty.servlet.FilterHolder;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.eclipse.jetty.servlets.CrossOriginFilter;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.glassfish.jersey.server.ResourceConfig;
-import org.glassfish.jersey.servlet.ServletContainer;
-import org.qortal.api.resource.AnnotationPostProcessor;
-import org.qortal.api.resource.ApiDefinition;
-import org.qortal.settings.Settings;
-
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import java.io.InputStream;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.KeyStore;
-import java.security.SecureRandom;
-
-public class DomainMapService {
-
-	private static DomainMapService instance;
-
-	private final ResourceConfig config;
-	private Server server;
-
-	private DomainMapService() {
-		this.config = new ResourceConfig();
-		this.config.packages("org.qortal.api.domainmap.resource");
-		this.config.register(OpenApiResource.class);
-		this.config.register(ApiDefinition.class);
-		this.config.register(AnnotationPostProcessor.class);
-	}
-
-	public static DomainMapService getInstance() {
-		if (instance == null)
-			instance = new DomainMapService();
-
-		return instance;
-	}
-
-	public Iterable<Class<?>> getResources() {
-		return this.config.getClasses();
-	}
-
-	public void start() {
-		try {
-			// Create API server
-
-			// SSL support if requested
-			String keystorePathname = Settings.getInstance().getSslKeystorePathname();
-			String keystorePassword = Settings.getInstance().getSslKeystorePassword();
-
-			if (keystorePathname != null && keystorePassword != null) {
-				// SSL version
-				if (!Files.isReadable(Path.of(keystorePathname)))
-					throw new RuntimeException("Failed to start SSL API due to broken keystore");
-
-				// BouncyCastle-specific SSLContext build
-				SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE");
-				KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
-
-				KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
-
-				try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) {
-					keyStore.load(keystoreStream, keystorePassword.toCharArray());
-				}
-
-				keyManagerFactory.init(keyStore, keystorePassword.toCharArray());
-				sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom());
-
-				SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
-				sslContextFactory.setSslContext(sslContext);
-
-				this.server = new Server();
-
-				HttpConfiguration httpConfig = new HttpConfiguration();
-				httpConfig.setSecureScheme("https");
-				httpConfig.setSecurePort(Settings.getInstance().getDomainMapPort());
-
-				SecureRequestCustomizer src = new SecureRequestCustomizer();
-				httpConfig.addCustomizer(src);
-
-				HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig);
-				SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
-
-				ServerConnector portUnifiedConnector = new ServerConnector(this.server,
-						new DetectorConnectionFactory(sslConnectionFactory),
-						httpConnectionFactory);
-				portUnifiedConnector.setHost(Settings.getInstance().getBindAddress());
-				portUnifiedConnector.setPort(Settings.getInstance().getDomainMapPort());
-
-				this.server.addConnector(portUnifiedConnector);
-			} else {
-				// Non-SSL
-				InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress());
-				InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getDomainMapPort());
-				this.server = new Server(endpoint);
-			}
-
-			// Error handler
-			ErrorHandler errorHandler = new ApiErrorHandler();
-			this.server.setErrorHandler(errorHandler);
-
-			// Request logging
-			if (Settings.getInstance().isDomainMapLoggingEnabled()) {
-				RequestLogWriter logWriter = new RequestLogWriter("domainmap-requests.log");
-				logWriter.setAppend(true);
-				logWriter.setTimeZone("UTC");
-				RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT);
-				this.server.setRequestLog(requestLog);
-			}
-
-			// Access handler (currently no whitelist is used)
-			InetAccessHandler accessHandler = new InetAccessHandler();
-			this.server.setHandler(accessHandler);
-
-			// URL rewriting
-			RewriteHandler rewriteHandler = new RewriteHandler();
-			accessHandler.setHandler(rewriteHandler);
-
-			// Context
-			ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
-			context.setContextPath("/");
-			rewriteHandler.setHandler(context);
-
-			// Cross-origin resource sharing
-			FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class);
-			corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
-			corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE");
-			corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
-			context.addFilter(corsFilterHolder, "/*", null);
-
-			// API servlet
-			ServletContainer container = new ServletContainer(this.config);
-			ServletHolder apiServlet = new ServletHolder(container);
-			apiServlet.setInitOrder(1);
-			context.addServlet(apiServlet, "/*");
-
-			// Start server
-			this.server.start();
-		} catch (Exception e) {
-			// Failed to start
-			throw new RuntimeException("Failed to start API", e);
-		}
-	}
-
-	public void stop() {
-		try {
-			// Stop server
-			this.server.stop();
-		} catch (Exception e) {
-			// Failed to stop
-		}
-
-		this.server = null;
-	}
-
-}

src/main/java/org/qortal/api/GatewayService.java

@@ -1,170 +0,0 @@
-package org.qortal.api;
-
-import io.swagger.v3.jaxrs2.integration.resources.OpenApiResource;
-import org.eclipse.jetty.http.HttpVersion;
-import org.eclipse.jetty.rewrite.handler.RewriteHandler;
-import org.eclipse.jetty.server.*;
-import org.eclipse.jetty.server.handler.ErrorHandler;
-import org.eclipse.jetty.server.handler.InetAccessHandler;
-import org.eclipse.jetty.servlet.FilterHolder;
-import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
-import org.eclipse.jetty.servlets.CrossOriginFilter;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
-import org.glassfish.jersey.server.ResourceConfig;
-import org.glassfish.jersey.servlet.ServletContainer;
-import org.qortal.api.resource.AnnotationPostProcessor;
-import org.qortal.api.resource.ApiDefinition;
-import org.qortal.settings.Settings;
-
-import javax.net.ssl.KeyManagerFactory;
-import javax.net.ssl.SSLContext;
-import java.io.InputStream;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.KeyStore;
-import java.security.SecureRandom;
-
-public class GatewayService {
-
-	private static GatewayService instance;
-
-	private final ResourceConfig config;
-	private Server server;
-
-	private GatewayService() {
-		this.config = new ResourceConfig();
-		this.config.packages("org.qortal.api.gateway.resource");
-		this.config.register(OpenApiResource.class);
-		this.config.register(ApiDefinition.class);
-		this.config.register(AnnotationPostProcessor.class);
-	}
-
-	public static GatewayService getInstance() {
-		if (instance == null)
-			instance = new GatewayService();
-
-		return instance;
-	}
-
-	public Iterable<Class<?>> getResources() {
-		return this.config.getClasses();
-	}
-
-	public void start() {
-		try {
-			// Create API server
-
-			// SSL support if requested
-			String keystorePathname = Settings.getInstance().getSslKeystorePathname();
-			String keystorePassword = Settings.getInstance().getSslKeystorePassword();
-
-			if (keystorePathname != null && keystorePassword != null) {
-				// SSL version
-				if (!Files.isReadable(Path.of(keystorePathname)))
-					throw new RuntimeException("Failed to start SSL API due to broken keystore");
-
-				// BouncyCastle-specific SSLContext build
-				SSLContext sslContext = SSLContext.getInstance("TLS", "BCJSSE");
-				KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("PKIX", "BCJSSE");
-
-				KeyStore keyStore = KeyStore.getInstance(KeyStore.getDefaultType(), "BC");
-
-				try (InputStream keystoreStream = Files.newInputStream(Paths.get(keystorePathname))) {
-					keyStore.load(keystoreStream, keystorePassword.toCharArray());
-				}
-
-				keyManagerFactory.init(keyStore, keystorePassword.toCharArray());
-				sslContext.init(keyManagerFactory.getKeyManagers(), null, new SecureRandom());
-
-				SslContextFactory.Server sslContextFactory = new SslContextFactory.Server();
-				sslContextFactory.setSslContext(sslContext);
-
-				this.server = new Server();
-
-				HttpConfiguration httpConfig = new HttpConfiguration();
-				httpConfig.setSecureScheme("https");
-				httpConfig.setSecurePort(Settings.getInstance().getGatewayPort());
-
-				SecureRequestCustomizer src = new SecureRequestCustomizer();
-				httpConfig.addCustomizer(src);
-
-				HttpConnectionFactory httpConnectionFactory = new HttpConnectionFactory(httpConfig);
-				SslConnectionFactory sslConnectionFactory = new SslConnectionFactory(sslContextFactory, HttpVersion.HTTP_1_1.asString());
-
-				ServerConnector portUnifiedConnector = new ServerConnector(this.server,
-						new DetectorConnectionFactory(sslConnectionFactory),
-						httpConnectionFactory);
-				portUnifiedConnector.setHost(Settings.getInstance().getBindAddress());
-				portUnifiedConnector.setPort(Settings.getInstance().getGatewayPort());
-
-				this.server.addConnector(portUnifiedConnector);
-			} else {
-				// Non-SSL
-				InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress());
-				InetSocketAddress endpoint = new InetSocketAddress(bindAddr, Settings.getInstance().getGatewayPort());
-				this.server = new Server(endpoint);
-			}
-
-			// Error handler
-			ErrorHandler errorHandler = new ApiErrorHandler();
-			this.server.setErrorHandler(errorHandler);
-
-			// Request logging
-			if (Settings.getInstance().isGatewayLoggingEnabled()) {
-				RequestLogWriter logWriter = new RequestLogWriter("gateway-requests.log");
-				logWriter.setAppend(true);
-				logWriter.setTimeZone("UTC");
-				RequestLog requestLog = new CustomRequestLog(logWriter, CustomRequestLog.EXTENDED_NCSA_FORMAT);
-				this.server.setRequestLog(requestLog);
-			}
-
-			// Access handler (currently no whitelist is used)
-			InetAccessHandler accessHandler = new InetAccessHandler();
-			this.server.setHandler(accessHandler);
-
-			// URL rewriting
-			RewriteHandler rewriteHandler = new RewriteHandler();
-			accessHandler.setHandler(rewriteHandler);
-
-			// Context
-			ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
-			context.setContextPath("/");
-			rewriteHandler.setHandler(context);
-
-			// Cross-origin resource sharing
-			FilterHolder corsFilterHolder = new FilterHolder(CrossOriginFilter.class);
-			corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_ORIGINS_PARAM, "*");
-			corsFilterHolder.setInitParameter(CrossOriginFilter.ALLOWED_METHODS_PARAM, "GET, POST, DELETE");
-			corsFilterHolder.setInitParameter(CrossOriginFilter.CHAIN_PREFLIGHT_PARAM, "false");
-			context.addFilter(corsFilterHolder, "/*", null);
-
-			// API servlet
-			ServletContainer container = new ServletContainer(this.config);
-			ServletHolder apiServlet = new ServletHolder(container);
-			apiServlet.setInitOrder(1);
-			context.addServlet(apiServlet, "/*");
-
-			// Start server
-			this.server.start();
-		} catch (Exception e) {
-			// Failed to start
-			throw new RuntimeException("Failed to start API", e);
-		}
-	}
-
-	public void stop() {
-		try {
-			// Stop server
-			this.server.stop();
-		} catch (Exception e) {
-			// Failed to stop
-		}
-
-		this.server = null;
-	}
-
-}

src/main/java/org/qortal/api/HTMLParser.java

@@ -1,46 +0,0 @@
-package org.qortal.api;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.select.Elements;
-
-public class HTMLParser {
-
-    private static final Logger LOGGER = LogManager.getLogger(HTMLParser.class);
-
-    private String linkPrefix;
-    private byte[] data;
-
-    public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix, byte[] data) {
-        String inPathWithoutFilename = inPath.substring(0, inPath.lastIndexOf('/'));
-        this.linkPrefix = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
-        this.data = data;
-    }
-
-    public void addAdditionalHeaderTags() {
-        String fileContents = new String(data);
-        Document document = Jsoup.parse(fileContents);
-        String baseUrl = this.linkPrefix + "/";
-        Elements head = document.getElementsByTag("head");
-        if (!head.isEmpty()) {
-            // Add base href tag
-            String baseElement = String.format("<base href=\"%s\">", baseUrl);
-            head.get(0).prepend(baseElement);
-        }
-        String html = document.html();
-        this.data = html.getBytes();
-    }
-
-    public static boolean isHtmlFile(String path) {
-        if (path.endsWith(".html") || path.endsWith(".htm")) {
-            return true;
-        }
-        return false;
-    }
-
-    public byte[] getData() {
-        return this.data;
-    }
-}

src/main/java/org/qortal/api/Security.java

@@ -1,111 +1,33 @@
 package org.qortal.api;
 
-import org.qortal.arbitrary.ArbitraryDataResource;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.controller.arbitrary.ArbitraryDataRenderManager;
-import org.qortal.settings.Settings;
-
-import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 
 import javax.servlet.http.HttpServletRequest;
 
+import org.qortal.settings.Settings;
+
 public abstract class Security {
 
 	public static final String API_KEY_HEADER = "X-API-KEY";
 
 	public static void checkApiCallAllowed(HttpServletRequest request) {
-		// We may want to allow automatic authentication for local requests, if enabled in settings
-		boolean localAuthBypassEnabled = Settings.getInstance().isLocalAuthBypassEnabled();
-		if (localAuthBypassEnabled) {
-			try {
-				InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
-				if (remoteAddr.isLoopbackAddress()) {
-					// Request originates from loopback address, so allow it
-					return;
-				}
-			} catch (UnknownHostException e) {
-				// Ignore failure, and fallback to API key authentication
-			}
-		}
-
-		// Retrieve the API key
-		ApiKey apiKey = Security.getApiKey(request);
-		if (!apiKey.generated()) {
-			// Not generated an API key yet, so disallow sensitive API calls
-			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "API key not generated");
-		}
-
-		// We require an API key to be passed
+		String expectedApiKey = Settings.getInstance().getApiKey();
 		String passedApiKey = request.getHeader(API_KEY_HEADER);
-		if (passedApiKey == null) {
-			// Try query string - this is needed to avoid a CORS preflight. See: https://stackoverflow.com/a/43881141
-			passedApiKey = request.getParameter("apiKey");
-		}
-		if (passedApiKey == null) {
-			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Missing 'X-API-KEY' header");
-		}
 
-		// The API keys must match
-		if (!apiKey.toString().equals(passedApiKey)) {
-			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "API key invalid");
-		}
-	}
+		if ((expectedApiKey != null && !expectedApiKey.equals(passedApiKey)) ||
+				(passedApiKey != null && !passedApiKey.equals(expectedApiKey)))
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
 
-	public static void disallowLoopbackRequests(HttpServletRequest request) {
+		InetAddress remoteAddr;
 		try {
-			InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
-			if (remoteAddr.isLoopbackAddress()) {
-				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Local requests not allowed");
-			}
+			remoteAddr = InetAddress.getByName(request.getRemoteAddr());
 		} catch (UnknownHostException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
 		}
-	}
 
-	public static void disallowLoopbackRequestsIfAuthBypassEnabled(HttpServletRequest request) {
-		if (Settings.getInstance().isLocalAuthBypassEnabled()) {
-			try {
-				InetAddress remoteAddr = InetAddress.getByName(request.getRemoteAddr());
-				if (remoteAddr.isLoopbackAddress()) {
-					throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Local requests not allowed when localAuthBypassEnabled is enabled in settings");
-				}
-			} catch (UnknownHostException e) {
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
-			}
-		}
-	}
-
-	public static void requirePriorAuthorization(HttpServletRequest request, String resourceId, Service service, String identifier) {
-		ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, identifier);
-		if (!ArbitraryDataRenderManager.getInstance().isAuthorized(resource)) {
-			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Call /render/authorize first");
-		}
-	}
-
-	public static void requirePriorAuthorizationOrApiKey(HttpServletRequest request, String resourceId, Service service, String identifier) {
-		try {
-			Security.checkApiCallAllowed(request);
-
-		} catch (ApiException e) {
-			// API call wasn't allowed, but maybe it was pre-authorized
-			Security.requirePriorAuthorization(request, resourceId, service, identifier);
-		}
-	}
-
-	public static ApiKey getApiKey(HttpServletRequest request) {
-		ApiKey apiKey = ApiService.getInstance().getApiKey();
-		if (apiKey == null) {
-			try {
-				apiKey = new ApiKey();
-			} catch (IOException e) {
-				// Couldn't load API key - so we need to treat it as not generated, and therefore unauthorized
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
-			}
-			ApiService.getInstance().setApiKey(apiKey);
-		}
-		return apiKey;
+		if (!remoteAddr.isLoopbackAddress())
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.UNAUTHORIZED);
 	}
 
 }

src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java

@@ -1,58 +0,0 @@
-package org.qortal.api.domainmap.resource;
-
-import io.swagger.v3.oas.annotations.tags.Tag;
-import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
-import org.qortal.arbitrary.ArbitraryDataRenderer;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.settings.Settings;
-
-import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.core.Context;
-import java.util.Map;
-
-
-@Path("/")
-@Tag(name = "Gateway")
-public class DomainMapResource {
-
-    @Context HttpServletRequest request;
-    @Context HttpServletResponse response;
-    @Context ServletContext context;
-
-
-    @GET
-    public HttpServletResponse getIndexByDomainMap() {
-        return this.getDomainMap("/");
-    }
-
-    @GET
-    @Path("{path:.*}")
-    public HttpServletResponse getPathByDomainMap(@PathParam("path") String inPath) {
-        return this.getDomainMap(inPath);
-    }
-
-    private HttpServletResponse getDomainMap(String inPath) {
-        Map<String, String> domainMap = Settings.getInstance().getSimpleDomainMap();
-        if (domainMap != null && domainMap.containsKey(request.getServerName())) {
-            // Build synchronously, so that we don't need to make the summary API endpoints available over
-            // the domain map server. This means that there will be no loading screen, but this is potentially
-            // preferred in this situation anyway (e.g. to avoid confusing search engine robots).
-            return this.get(domainMap.get(request.getServerName()), ResourceIdType.NAME, Service.WEBSITE, inPath, null, "", false, false);
-        }
-        return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
-    }
-
-    private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String inPath,
-                                    String secret58, String prefix, boolean usePrefix, boolean async) {
-
-        ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, inPath,
-                secret58, prefix, usePrefix, async, request, response, context);
-        return renderer.render();
-    }
-
-}

src/main/java/org/qortal/api/gateway/resource/GatewayResource.java

@@ -1,126 +0,0 @@
-package org.qortal.api.gateway.resource;
-
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
-import io.swagger.v3.oas.annotations.tags.Tag;
-import org.qortal.api.Security;
-import org.qortal.arbitrary.ArbitraryDataFile;
-import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
-import org.qortal.arbitrary.ArbitraryDataReader;
-import org.qortal.arbitrary.ArbitraryDataRenderer;
-import org.qortal.arbitrary.ArbitraryDataResource;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.data.arbitrary.ArbitraryResourceStatus;
-
-import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-
-
-@Path("/")
-@Tag(name = "Gateway")
-public class GatewayResource {
-
-    @Context HttpServletRequest request;
-    @Context HttpServletResponse response;
-    @Context ServletContext context;
-
-    /**
-     * We need to allow resource status checking (and building) via the gateway, as the node's API port
-     * may not be forwarded and will almost certainly not be authenticated. Since gateways allow for
-     * all resources to be loaded except those that are blocked, there is no need for authentication.
-     */
-    @GET
-    @Path("/arbitrary/resource/status/{service}/{name}")
-    public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service,
-                                                             @PathParam("name") String name,
-                                                             @QueryParam("build") Boolean build) {
-
-        return this.getStatus(service, name, null, build);
-    }
-
-    @GET
-    @Path("/arbitrary/resource/status/{service}/{name}/{identifier}")
-    public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service,
-                                                      @PathParam("name") String name,
-                                                      @PathParam("identifier") String identifier,
-                                                      @QueryParam("build") Boolean build) {
-
-        return this.getStatus(service, name, identifier, build);
-    }
-
-    private ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) {
-
-        // If "build=true" has been specified in the query string, build the resource before returning its status
-        if (build != null && build == true) {
-            ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null);
-            try {
-                if (!reader.isBuilding()) {
-                    reader.loadSynchronously(false);
-                }
-            } catch (Exception e) {
-                // No need to handle exception, as it will be reflected in the status
-            }
-        }
-
-        ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
-        return resource.getStatus(false);
-    }
-
-
-    @GET
-    public HttpServletResponse getRoot() {
-        return ArbitraryDataRenderer.getResponse(response, 200, "");
-    }
-
-
-    @GET
-    @Path("{name}/{path:.*}")
-    @SecurityRequirement(name = "apiKey")
-    public HttpServletResponse getPathByName(@PathParam("name") String name,
-                                             @PathParam("path") String inPath) {
-        // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data
-        Security.disallowLoopbackRequests(request);
-        return this.get(name, ResourceIdType.NAME, Service.WEBSITE, inPath, null, "", true, true);
-    }
-
-    @GET
-    @Path("{name}")
-    @SecurityRequirement(name = "apiKey")
-    public HttpServletResponse getIndexByName(@PathParam("name") String name) {
-        // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data
-        Security.disallowLoopbackRequests(request);
-        return this.get(name, ResourceIdType.NAME, Service.WEBSITE, "/", null, "", true, true);
-    }
-
-
-    // Optional /site alternative for backwards support
-
-    @GET
-    @Path("/site/{name}/{path:.*}")
-    public HttpServletResponse getSitePathByName(@PathParam("name") String name,
-                                                 @PathParam("path") String inPath) {
-        // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data
-        Security.disallowLoopbackRequests(request);
-        return this.get(name, ResourceIdType.NAME, Service.WEBSITE, inPath, null, "/site", true, true);
-    }
-
-    @GET
-    @Path("/site/{name}")
-    public HttpServletResponse getSiteIndexByName(@PathParam("name") String name) {
-        // Block requests from localhost, to prevent websites/apps from running javascript that fetches unvetted data
-        Security.disallowLoopbackRequests(request);
-        return this.get(name, ResourceIdType.NAME, Service.WEBSITE, "/", null, "/site", true, true);
-    }
-
-    
-    private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String inPath,
-                                    String secret58, String prefix, boolean usePrefix, boolean async) {
-
-        ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, inPath,
-                secret58, prefix, usePrefix, async, request, response, context);
-        return renderer.render();
-    }
-
-}

src/main/java/org/qortal/api/model/CrossChainTradeSummary.java

@@ -25,12 +25,6 @@ public class CrossChainTradeSummary {
 	@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
 	private long foreignAmount;
 
-	private String atAddress;
-
-	private String sellerAddress;
-
-	private String buyerReceivingAddress;
-
 	protected CrossChainTradeSummary() {
 		/* For JAXB */
 	}
@@ -40,9 +34,6 @@ public class CrossChainTradeSummary {
 		this.qortAmount = crossChainTradeData.qortAmount;
 		this.foreignAmount = crossChainTradeData.expectedForeignAmount;
 		this.btcAmount = this.foreignAmount;
-		this.sellerAddress = crossChainTradeData.qortalCreator;
-		this.buyerReceivingAddress = crossChainTradeData.qortalPartnerReceivingAddress;
-		this.atAddress = crossChainTradeData.qortalAtAddress;
 	}
 
 	public long getTradeTimestamp() {
@@ -57,11 +48,7 @@ public class CrossChainTradeSummary {
 		return this.btcAmount;
 	}
 
-	public long getForeignAmount() { return this.foreignAmount; }
-
-	public String getAtAddress() { return this.atAddress; }
-
-	public String getSellerAddress() { return this.sellerAddress; }
-
-	public String getBuyerReceivingAddressAddress() { return this.buyerReceivingAddress; }
+	public long getForeignAmount() {
+		return this.foreignAmount;
+	}
 }

src/main/java/org/qortal/api/model/ListRequest.java

@@ -1,18 +0,0 @@
-package org.qortal.api.model;
-
-import io.swagger.v3.oas.annotations.media.Schema;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import java.util.List;
-
-@XmlAccessorType(XmlAccessType.FIELD)
-public class ListRequest {
-
-	@Schema(description = "A list of items")
-	public List<String> items;
-
-	public ListRequest() {
-	}
-
-}

src/main/java/org/qortal/api/model/NodeStatus.java

@@ -4,7 +4,6 @@ import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 
 import org.qortal.controller.Controller;
-import org.qortal.controller.Synchronizer;
 import org.qortal.network.Network;
 
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -23,7 +22,7 @@ public class NodeStatus {
 	public NodeStatus() {
 		this.isMintingPossible = Controller.getInstance().isMintingPossible();
 
-		this.syncPercent = Synchronizer.getInstance().getSyncPercent();
+		this.syncPercent = Controller.getInstance().getSyncPercent();
 		this.isSynchronizing = this.syncPercent != null;
 
 		this.numberOfConnections = Network.getInstance().getHandshakedPeers().size();

src/main/java/org/qortal/api/model/PeersSummary.java

@@ -1,15 +0,0 @@
-package org.qortal.api.model;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-
-@XmlAccessorType(XmlAccessType.FIELD)
-public class PeersSummary {
-
-	public int inboundConnections;
-	public int outboundConnections;
-
-	public PeersSummary() {
-	}
-
-}

src/main/java/org/qortal/api/model/crosschain/DogecoinSendRequest.java

@@ -1,29 +0,0 @@
-package org.qortal.api.model.crosschain;
-
-import io.swagger.v3.oas.annotations.media.Schema;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
-
-@XmlAccessorType(XmlAccessType.FIELD)
-public class DogecoinSendRequest {
-
-	@Schema(description = "Dogecoin BIP32 extended private key", example = "tprv___________________________________________________________________________________________________________")
-	public String xprv58;
-
-	@Schema(description = "Recipient's Dogecoin address ('legacy' P2PKH only)", example = "DoGecoinEaterAddressDontSendhLfzKD")
-	public String receivingAddress;
-
-	@Schema(description = "Amount of DOGE to send", type = "number")
-	@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
-	public long dogecoinAmount;
-
-	@Schema(description = "Transaction fee per byte (optional). Default is 0.00000100 DOGE (100 sats) per byte", example = "0.00000100", type = "number")
-	@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
-	public Long feePerByte;
-
-	public DogecoinSendRequest() {
-	}
-
-}

src/main/java/org/qortal/api/resource/AddressesResource.java

@@ -12,11 +12,14 @@ import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.math.BigDecimal;
 import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -35,7 +38,6 @@ import org.qortal.crypto.Crypto;
 import org.qortal.data.account.AccountData;
 import org.qortal.data.account.RewardShareData;
 import org.qortal.data.network.OnlineAccountData;
-import org.qortal.data.network.OnlineAccountLevel;
 import org.qortal.data.transaction.PublicizeTransactionData;
 import org.qortal.data.transaction.RewardShareTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -178,66 +180,6 @@ public class AddressesResource {
 		}
 	}
 
-	@GET
-	@Path("/online/levels")
-	@Operation(
-			summary = "Return currently 'online' accounts counts, grouped by level",
-			responses = {
-					@ApiResponse(
-							description = "online accounts",
-							content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = ApiOnlineAccount.class)))
-					)
-			}
-	)
-	@ApiErrors({ApiError.PUBLIC_KEY_NOT_FOUND, ApiError.REPOSITORY_ISSUE})
-	public List<OnlineAccountLevel> getOnlineAccountsByLevel() {
-		List<OnlineAccountData> onlineAccounts = Controller.getInstance().getOnlineAccounts();
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			List<OnlineAccountLevel> onlineAccountLevels = new ArrayList<>();
-
-			for (OnlineAccountData onlineAccountData : onlineAccounts) {
-				try {
-					final int minterLevel = Account.getRewardShareEffectiveMintingLevelIncludingLevelZero(repository, onlineAccountData.getPublicKey());
-
-					OnlineAccountLevel onlineAccountLevel = onlineAccountLevels.stream()
-							.filter(a -> a.getLevel() == minterLevel)
-							.findFirst().orElse(null);
-
-					// Note: I don't think we can use the level as the List index here because there will be gaps.
-					// So we are forced to manually look up the existing item each time.
-					// There's probably a nice shorthand java way of doing this, but this approach gets the same result.
-
-					if (onlineAccountLevel == null) {
-						// No entry exists for this level yet, so create one
-						onlineAccountLevel = new OnlineAccountLevel(minterLevel, 1);
-						onlineAccountLevels.add(onlineAccountLevel);
-					}
-					else {
-						// Already exists - so increment the count
-						int existingCount = onlineAccountLevel.getCount();
-						onlineAccountLevel.setCount(++existingCount);
-
-						// Then replace the existing item
-						int index = onlineAccountLevels.indexOf(onlineAccountLevel);
-						onlineAccountLevels.set(index, onlineAccountLevel);
-					}
-
-				} catch (DataException e) {
-					continue;
-				}
-			}
-
-			// Sort by level
-			onlineAccountLevels.sort(Comparator.comparingInt(OnlineAccountLevel::getLevel));
-
-			return onlineAccountLevels;
-
-		} catch (DataException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
-		}
-	}
-
 	@GET
 	@Path("/balance/{address}")
 	@Operation(
@@ -533,7 +475,7 @@ public class AddressesResource {
 	)
 	@ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String computePublicize(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String rawBytes58) {
+	public String computePublicize(String rawBytes58) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {

src/main/java/org/qortal/api/resource/AdminResource.java

@@ -22,29 +22,32 @@ import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
 import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.appender.RollingFileAppender;
-import org.checkerframework.checker.units.qual.A;
 import org.qortal.account.Account;
 import org.qortal.account.PrivateKeyAccount;
-import org.qortal.api.*;
+import org.qortal.api.ApiError;
+import org.qortal.api.ApiErrors;
+import org.qortal.api.ApiExceptionFactory;
+import org.qortal.api.Security;
 import org.qortal.api.model.ActivitySummary;
 import org.qortal.api.model.NodeInfo;
 import org.qortal.api.model.NodeStatus;
 import org.qortal.block.BlockChain;
 import org.qortal.controller.Controller;
-import org.qortal.controller.Synchronizer;
 import org.qortal.controller.Synchronizer.SynchronizationResult;
 import org.qortal.data.account.MintingAccountData;
 import org.qortal.data.account.RewardShareData;
@@ -64,8 +67,6 @@ import com.google.common.collect.Lists;
 @Tag(name = "Admin")
 public class AdminResource {
 
-	private static final Logger LOGGER = LogManager.getLogger(AdminResource.class);
-
 	private static final int MAX_LOG_LINES = 500;
 
 	@Context
@@ -75,8 +76,7 @@ public class AdminResource {
 	@Path("/unused")
 	@Parameter(in = ParameterIn.PATH, name = "assetid", description = "Asset ID, 0 is native coin", schema = @Schema(type = "integer"))
 	@Parameter(in = ParameterIn.PATH, name = "otherassetid", description = "Asset ID, 0 is native coin", schema = @Schema(type = "integer"))
-	@Parameter(in = ParameterIn.PATH, name = "address", description = "An account address", example = "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v")
-	@Parameter(in = ParameterIn.PATH, name = "path", description = "Local path to folder containing the files", schema = @Schema(type = "String", defaultValue = "/Users/user/Documents/MyStaticWebsite"))
+	@Parameter(in = ParameterIn.PATH, name = "address", description = "an account address", example = "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v")
 	@Parameter(in = ParameterIn.QUERY, name = "count", description = "Maximum number of entries to return, 0 means none", schema = @Schema(type = "integer", defaultValue = "20"))
 	@Parameter(in = ParameterIn.QUERY, name = "limit", description = "Maximum number of entries to return, 0 means unlimited", schema = @Schema(type = "integer", defaultValue = "20"))
 	@Parameter(in = ParameterIn.QUERY, name = "offset", description = "Starting entry in results, 0 is first entry", schema = @Schema(type = "integer"))
@@ -134,7 +134,10 @@ public class AdminResource {
 			)
 		}
 	)
+	@SecurityRequirement(name = "apiKey")
 	public NodeStatus status() {
+		Security.checkApiCallAllowed(request);
+
 		NodeStatus nodeStatus = new NodeStatus();
 
 		return nodeStatus;
@@ -153,7 +156,7 @@ public class AdminResource {
 		}
 	)
 	@SecurityRequirement(name = "apiKey")
-	public String shutdown(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+	public String shutdown() {
 		Security.checkApiCallAllowed(request);
 
 		new Thread(() -> {
@@ -182,7 +185,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public ActivitySummary summary(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+	public ActivitySummary summary() {
 		Security.checkApiCallAllowed(request);
 
 		ActivitySummary summary = new ActivitySummary();
@@ -228,7 +231,7 @@ public class AdminResource {
 		}
 	)
 	@SecurityRequirement(name = "apiKey")
-	public Controller.StatsSnapshot getEngineStats(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+	public Controller.StatsSnapshot getEngineStats() {
 		Security.checkApiCallAllowed(request);
 
 		return Controller.getInstance().getStatsSnapshot();
@@ -246,7 +249,9 @@ public class AdminResource {
 		}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
+	@SecurityRequirement(name = "apiKey")
 	public List<MintingAccountData> getMintingAccounts() {
+		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			List<MintingAccountData> mintingAccounts = repository.getAccountRepository().getMintingAccounts();
@@ -292,7 +297,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.REPOSITORY_ISSUE, ApiError.CANNOT_MINT})
 	@SecurityRequirement(name = "apiKey")
-	public String addMintingAccount(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String seed58) {
+	public String addMintingAccount(String seed58) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -345,7 +350,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String deleteMintingAccount(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
+	public String deleteMintingAccount(String key58) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -445,7 +450,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.INVALID_HEIGHT, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String orphan(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String targetHeightString) {
+	public String orphan(String targetHeightString) {
 		Security.checkApiCallAllowed(request);
 
 		try {
@@ -454,23 +459,6 @@ public class AdminResource {
 			if (targetHeight <= 0 || targetHeight > Controller.getInstance().getChainHeight())
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT);
 
-			// Make sure we're not orphaning as far back as the archived blocks
-			// FUTURE: we could support this by first importing earlier blocks from the archive
-			if (Settings.getInstance().isTopOnly() ||
-				Settings.getInstance().isArchiveEnabled()) {
-
-				try (final Repository repository = RepositoryManager.getRepository()) {
-					// Find the first unarchived block
-					int oldestBlock = repository.getBlockArchiveRepository().getBlockArchiveHeight();
-					// Add some extra blocks just in case we're currently archiving/pruning
-					oldestBlock += 100;
-					if (targetHeight <= oldestBlock) {
-						LOGGER.info("Unable to orphan beyond block {} because it is archived", oldestBlock);
-						throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT);
-					}
-				}
-			}
-
 			if (BlockChain.orphan(targetHeight))
 				return "true";
 			else
@@ -504,7 +492,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String forceSync(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String targetPeerAddress) {
+	public String forceSync(String targetPeerAddress) {
 		Security.checkApiCallAllowed(request);
 
 		try {
@@ -526,7 +514,7 @@ public class AdminResource {
 			SynchronizationResult syncResult;
 			try {
 				do {
-					syncResult = Synchronizer.getInstance().actuallySynchronize(targetPeer, true);
+					syncResult = Controller.getInstance().actuallySynchronize(targetPeer, true);
 				} while (syncResult == SynchronizationResult.OK);
 			} finally {
 				blockchainLock.unlock();
@@ -550,7 +538,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String exportRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+	public String exportRepository() {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -565,13 +553,13 @@ public class AdminResource {
 	@Path("/repository/data")
 	@Operation(
 		summary = "Import data into repository.",
-		description = "Imports data from file on local machine. Filename is forced to 'qortal-backup/TradeBotStates.json' if apiKey is not set.",
+		description = "Imports data from file on local machine. Filename is forced to 'import.json' if apiKey is not set.",
 		requestBody = @RequestBody(
 			required = true,
 			content = @Content(
 				mediaType = MediaType.TEXT_PLAIN,
 				schema = @Schema(
-					type = "string", example = "qortal-backup/TradeBotStates.json"
+					type = "string", example = "MintingAccounts.script"
 				)
 			)
 		),
@@ -584,12 +572,12 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String importRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String filename) {
+	public String importRepository(String filename) {
 		Security.checkApiCallAllowed(request);
 
 		// Hard-coded because it's too dangerous to allow user-supplied filenames in weaker security contexts
 		if (Settings.getInstance().getApiKey() == null)
-			filename = "qortal-backup/TradeBotStates.json";
+			filename = "import.json";
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
@@ -601,10 +589,6 @@ public class AdminResource {
 				repository.saveChanges();
 
 				return "true";
-
-			} catch (IOException e) {
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e);
-
 			} finally {
 				blockchainLock.unlock();
 			}
@@ -630,7 +614,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String checkpointRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+	public String checkpointRepository() {
 		Security.checkApiCallAllowed(request);
 
 		RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
@@ -651,7 +635,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String backupRepository(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+	public String backupRepository() {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -660,16 +644,14 @@ public class AdminResource {
 			blockchainLock.lockInterruptibly();
 
 			try {
-				// Timeout if the database isn't ready for backing up after 60 seconds
-				long timeout = 60 * 1000L;
-				repository.backup(true, "backup", timeout);
+				repository.backup(true);
 				repository.saveChanges();
 
 				return "true";
 			} finally {
 				blockchainLock.unlock();
 			}
-		} catch (InterruptedException | TimeoutException e) {
+		} catch (InterruptedException e) {
 			// We couldn't lock blockchain to perform backup
 			return "false";
 		} catch (DataException e) {
@@ -685,7 +667,7 @@ public class AdminResource {
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public void performRepositoryMaintenance(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+	public void performRepositoryMaintenance() {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -694,71 +676,15 @@ public class AdminResource {
 			blockchainLock.lockInterruptibly();
 
 			try {
-				// Timeout if the database isn't ready to start after 60 seconds
-				long timeout = 60 * 1000L;
-				repository.performPeriodicMaintenance(timeout);
+				repository.performPeriodicMaintenance();
 			} finally {
 				blockchainLock.unlock();
 			}
 		} catch (InterruptedException e) {
 			// No big deal
-		} catch (DataException | TimeoutException e) {
+		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
 	}
 
-
-	@POST
-	@Path("/apikey/generate")
-	@Operation(
-			summary = "Generate an API key",
-			description = "This request is unauthenticated if no API key has been generated yet. " +
-					"If an API key already exists, it needs to be passed as a header and this endpoint " +
-					"will then generate a new key which replaces the existing one.",
-			responses = {
-					@ApiResponse(
-							description = "API key string",
-							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string"))
-					)
-			}
-	)
-	@SecurityRequirement(name = "apiKey")
-	public String generateApiKey(@HeaderParam(Security.API_KEY_HEADER) String apiKeyHeader) {
-		ApiKey apiKey = Security.getApiKey(request);
-
-		// If the API key is already generated, we need to authenticate this request
-		if (apiKey.generated() && apiKey.exists()) {
-			Security.checkApiCallAllowed(request);
-		}
-
-		// Not generated yet - so we are safe to generate one
-		// FUTURE: we may want to restrict this to local/loopback only?
-
-		try {
-			apiKey.generate();
-		} catch (IOException e) {
-			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.UNAUTHORIZED, "Unable to generate API key");
-		}
-
-		return apiKey.toString();
-	}
-
-	@GET
-	@Path("/apikey/test")
-	@Operation(
-			summary = "Test an API key",
-			responses = {
-					@ApiResponse(
-							description = "true if authenticated",
-							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
-					)
-			}
-	)
-	@SecurityRequirement(name = "apiKey")
-	public String testApiKey(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
-		Security.checkApiCallAllowed(request);
-
-		return "true";
-	}
-
 }

src/main/java/org/qortal/api/resource/BlocksResource.java

@@ -15,8 +15,6 @@ import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.math.RoundingMode;
 import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
@@ -35,13 +33,11 @@ import org.qortal.api.ApiExceptionFactory;
 import org.qortal.api.model.BlockMintingInfo;
 import org.qortal.api.model.BlockSignerSummary;
 import org.qortal.block.Block;
-import org.qortal.controller.Controller;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.account.AccountData;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.block.BlockSummaryData;
 import org.qortal.data.transaction.TransactionData;
-import org.qortal.repository.BlockArchiveReader;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
@@ -75,8 +71,7 @@ public class BlocksResource {
 	@ApiErrors({
 		ApiError.INVALID_SIGNATURE, ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE
 	})
-	public BlockData getBlock(@PathParam("signature") String signature58,
-							  @QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) {
+	public BlockData getBlock(@PathParam("signature") String signature58) {
 		// Decode signature
 		byte[] signature;
 		try {
@@ -86,25 +81,11 @@ public class BlocksResource {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-		    // Check the database first
 			BlockData blockData = repository.getBlockRepository().fromSignature(signature);
-			if (blockData != null) {
-				if (includeOnlineSignatures == null || includeOnlineSignatures == false) {
-					blockData.setOnlineAccountsSignatures(null);
-				}
-				return blockData;
-			}
+			if (blockData == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 
-            // Not found, so try the block archive
-			blockData = repository.getBlockArchiveRepository().fromSignature(signature);
-			if (blockData != null) {
-				if (includeOnlineSignatures == null || includeOnlineSignatures == false) {
-					blockData.setOnlineAccountsSignatures(null);
-				}
-				return blockData;
-			}
-
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+			return blockData;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -135,24 +116,16 @@ public class BlocksResource {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-
-            // Check the database first
 			BlockData blockData = repository.getBlockRepository().fromSignature(signature);
-			if (blockData != null) {
-                Block block = new Block(repository, blockData);
-                ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-                bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
-                bytes.write(BlockTransformer.toBytes(block));
-                return Base58.encode(bytes.toByteArray());
-            }
+			if (blockData == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 
-            // Not found, so try the block archive
-            byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, false, repository);
-            if (bytes != null) {
-                return Base58.encode(bytes);
-            }
+			Block block = new Block(repository, blockData);
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+			bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
+			bytes.write(BlockTransformer.toBytes(block));
+			return Base58.encode(bytes.toByteArray());
 
-            throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 		} catch (TransformationException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e);
 		} catch (DataException | IOException e) {
@@ -197,12 +170,8 @@ public class BlocksResource {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-		    // Check if the block exists in either the database or archive
-			if (repository.getBlockRepository().getHeightFromSignature(signature) == 0 &&
-					repository.getBlockArchiveRepository().getHeightFromSignature(signature) == 0) {
-				// Not found in either the database or archive
+			if (repository.getBlockRepository().getHeightFromSignature(signature) == 0)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
-            }
 
 			return repository.getBlockRepository().getTransactionsFromSignature(signature, limit, offset, reverse);
 		} catch (DataException e) {
@@ -231,19 +200,7 @@ public class BlocksResource {
 	})
 	public BlockData getFirstBlock() {
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			// Check the database first
-			BlockData blockData = repository.getBlockRepository().fromHeight(1);
-			if (blockData != null) {
-				return blockData;
-			}
-
-			// Try the archive
-			blockData = repository.getBlockArchiveRepository().fromHeight(1);
-			if (blockData != null) {
-				return blockData;
-			}
-
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+			return repository.getBlockRepository().fromHeight(1);
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -268,15 +225,9 @@ public class BlocksResource {
 	@ApiErrors({
 		ApiError.REPOSITORY_ISSUE
 	})
-	public BlockData getLastBlock(@QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) {
+	public BlockData getLastBlock() {
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			BlockData blockData = repository.getBlockRepository().getLastBlock();
-
-			if (includeOnlineSignatures == null || includeOnlineSignatures == false) {
-				blockData.setOnlineAccountsSignatures(null);
-			}
-
-			return blockData;
+			return repository.getBlockRepository().getLastBlock();
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -311,28 +262,17 @@ public class BlocksResource {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			BlockData childBlockData = null;
-
-			// Check if block exists in database
 			BlockData blockData = repository.getBlockRepository().fromSignature(signature);
-			if (blockData != null) {
-				return repository.getBlockRepository().fromReference(signature);
-			}
 
-			// Not found, so try the archive
-			// This also checks that the parent block exists
-			// It will return null if either the parent or child don't exit
-			childBlockData = repository.getBlockArchiveRepository().fromReference(signature);
+			// Check block exists
+			if (blockData == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+
+			BlockData childBlockData = repository.getBlockRepository().fromReference(signature);
 
 			// Check child block exists
-			if (childBlockData == null) {
+			if (childBlockData == null)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
-			}
-
-			// Check child block's reference matches the supplied signature
-			if (!Arrays.equals(childBlockData.getReference(), signature)) {
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
-			}
 
 			return childBlockData;
 		} catch (DataException e) {
@@ -398,20 +338,13 @@ public class BlocksResource {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			// Firstly check the database
 			BlockData blockData = repository.getBlockRepository().fromSignature(signature);
-			if (blockData != null) {
-				return blockData.getHeight();
-			}
 
-			// Not found, so try the archive
-			blockData = repository.getBlockArchiveRepository().fromSignature(signature);
-			if (blockData != null) {
-				return blockData.getHeight();
-			}
-
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+			// Check block exists
+			if (blockData == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 
+			return blockData.getHeight();
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -436,29 +369,13 @@ public class BlocksResource {
 	@ApiErrors({
 		ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE
 	})
-	public BlockData getByHeight(@PathParam("height") int height,
-								 @QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) {
+	public BlockData getByHeight(@PathParam("height") int height) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			// Firstly check the database
 			BlockData blockData = repository.getBlockRepository().fromHeight(height);
-			if (blockData != null) {
-				if (includeOnlineSignatures == null || includeOnlineSignatures == false) {
-					blockData.setOnlineAccountsSignatures(null);
-				}
-				return blockData;
-			}
-
-			// Not found, so try the archive
-			blockData = repository.getBlockArchiveRepository().fromHeight(height);
-			if (blockData != null) {
-				if (includeOnlineSignatures == null || includeOnlineSignatures == false) {
-					blockData.setOnlineAccountsSignatures(null);
-				}
-				return blockData;
-			}
-
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+			if (blockData == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 
+			return blockData;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -485,31 +402,12 @@ public class BlocksResource {
 	})
 	public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			// Try the database
 			BlockData blockData = repository.getBlockRepository().fromHeight(height);
-			if (blockData == null) {
-
-				// Not found, so try the archive
-				blockData = repository.getBlockArchiveRepository().fromHeight(height);
-				if (blockData == null) {
-
-					// Still not found
-					throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
-				}
-			}
+			if (blockData == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 
 			Block block = new Block(repository, blockData);
 			BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference());
-			if (parentBlockData == null) {
-				// Parent block not found - try the archive
-				parentBlockData = repository.getBlockArchiveRepository().fromSignature(blockData.getReference());
-				if (parentBlockData == null) {
-
-					// Still not found
-					throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
-				}
-			}
-
 			int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
 			if (minterLevel == 0)
 				// This may be unavailable when requesting a trimmed block
@@ -554,37 +452,15 @@ public class BlocksResource {
 	@ApiErrors({
 		ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE
 	})
-	public BlockData getByTimestamp(@PathParam("timestamp") long timestamp,
-									@QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) {
+	public BlockData getByTimestamp(@PathParam("timestamp") long timestamp) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			BlockData blockData = null;
-
-			// Try the Blocks table
 			int height = repository.getBlockRepository().getHeightFromTimestamp(timestamp);
-			if (height > 1) {
-				// Found match in Blocks table
-				blockData = repository.getBlockRepository().fromHeight(height);
-				if (includeOnlineSignatures == null || includeOnlineSignatures == false) {
-					blockData.setOnlineAccountsSignatures(null);
-				}
-				return blockData;
-			}
-
-			// Not found in Blocks table, so try the archive
-			height = repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp);
-			if (height > 1) {
-				// Found match in archive
-				blockData = repository.getBlockArchiveRepository().fromHeight(height);
-			}
-
-			// Ensure block exists
-			if (blockData == null) {
+			if (height == 0)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
-			}
 
-			if (includeOnlineSignatures == null || includeOnlineSignatures == false) {
-				blockData.setOnlineAccountsSignatures(null);
-			}
+			BlockData blockData = repository.getBlockRepository().fromHeight(height);
+			if (blockData == null)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 
 			return blockData;
 		} catch (DataException e) {
@@ -621,14 +497,9 @@ public class BlocksResource {
 
 			for (/* count already set */; count > 0; --count, ++height) {
 				BlockData blockData = repository.getBlockRepository().fromHeight(height);
-				if (blockData == null) {
-					// Not found - try the archive
-					blockData = repository.getBlockArchiveRepository().fromHeight(height);
-					if (blockData == null) {
-						// Run out of blocks!
-						break;
-					}
-				}
+				if (blockData == null)
+					// Run out of blocks!
+					break;
 
 				blocks.add(blockData);
 			}
@@ -673,29 +544,7 @@ public class BlocksResource {
 			if (accountData == null || accountData.getPublicKey() == null)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.PUBLIC_KEY_NOT_FOUND);
 
-
-			List<BlockSummaryData> summaries = repository.getBlockRepository()
-					.getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse);
-
-			// Add any from the archive
-			List<BlockSummaryData> archivedSummaries = repository.getBlockArchiveRepository()
-					.getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse);
-			if (archivedSummaries != null && !archivedSummaries.isEmpty()) {
-				summaries.addAll(archivedSummaries);
-			}
-			else {
-				summaries = archivedSummaries;
-			}
-
-			// Sort the results (because they may have been obtained from two places)
-			if (reverse != null && reverse) {
-				summaries.sort((s1, s2) -> Integer.valueOf(s2.getHeight()).compareTo(Integer.valueOf(s1.getHeight())));
-			}
-			else {
-				summaries.sort(Comparator.comparing(s -> Integer.valueOf(s.getHeight())));
-			}
-
-			return summaries;
+			return repository.getBlockRepository().getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse);
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -731,8 +580,7 @@ public class BlocksResource {
 				if (!Crypto.isValidAddress(address))
 					throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
 
-			// This method pulls data from both Blocks and BlockArchive, so no need to query serparately
-			return repository.getBlockArchiveRepository().getBlockSigners(addresses, limit, offset, reverse);
+			return repository.getBlockRepository().getBlockSigners(addresses, limit, offset, reverse);
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -772,76 +620,7 @@ public class BlocksResource {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-
-			/*
-			 * start	end		count		result
-			 * 10		40		null		blocks 10 to 39 (excludes end block, ignore count)
-			 *
-			 * null		null	null		blocks 1 to 50 (assume count=50, maybe start=1)
-			 * 30		null	null		blocks 30 to 79 (assume count=50)
-			 * 30		null	10			blocks 30 to 39
-			 *
-			 * null		null	50			last 50 blocks? so if max(blocks.height) is 200, then blocks 151 to 200
-			 * null		200		null		blocks 150 to 199 (excludes end block, assume count=50)
-			 * null		200		10			blocks 190 to 199 (excludes end block)
-			 */
-
-			List<BlockSummaryData> blockSummaries = new ArrayList<>();
-
-			// Use the latest X blocks if only a count is specified
-			if (startHeight == null && endHeight == null && count != null) {
-				BlockData chainTip = repository.getBlockRepository().getLastBlock();
-				startHeight = chainTip.getHeight() - count;
-				endHeight = chainTip.getHeight();
-			}
-
-			// ... otherwise default the start height to 1
-			if (startHeight == null && endHeight == null) {
-				startHeight = 1;
-			}
-
-			// Default the count to 50
-			if (count == null) {
-				count = 50;
-			}
-
-			// If both a start and end height exist, ignore the count
-			if (startHeight != null && endHeight != null) {
-				if (startHeight > 0 && endHeight > 0) {
-					count = Integer.MAX_VALUE;
-				}
-			}
-
-			// Derive start height from end height if missing
-			if (startHeight == null || startHeight == 0) {
-				if (endHeight != null && endHeight > 0) {
-					if (count != null) {
-						startHeight = endHeight - count;
-					}
-				}
-			}
-
-			for (/* count already set */; count > 0; --count, ++startHeight) {
-				if (endHeight != null && startHeight >= endHeight) {
-					break;
-				}
-				BlockData blockData = repository.getBlockRepository().fromHeight(startHeight);
-				if (blockData == null) {
-					// Not found - try the archive
-					blockData = repository.getBlockArchiveRepository().fromHeight(startHeight);
-					if (blockData == null) {
-						// Run out of blocks!
-						break;
-					}
-				}
-
-				if (blockData != null) {
-					BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
-					blockSummaries.add(blockSummaryData);
-				}
-			}
-
-			return blockSummaries;
+			return repository.getBlockRepository().getBlockSummaries(startHeight, endHeight, count);
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}

src/main/java/org/qortal/api/resource/BootstrapResource.java

@@ -1,95 +0,0 @@
-package org.qortal.api.resource;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.media.Content;
-import io.swagger.v3.oas.annotations.media.Schema;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
-import io.swagger.v3.oas.annotations.tags.Tag;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.api.ApiError;
-import org.qortal.api.ApiExceptionFactory;
-import org.qortal.api.Security;
-import org.qortal.repository.Bootstrap;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import java.io.IOException;
-
-
-@Path("/bootstrap")
-@Tag(name = "Bootstrap")
-public class BootstrapResource {
-
-	private static final Logger LOGGER = LogManager.getLogger(BootstrapResource.class);
-
-	@Context
-	HttpServletRequest request;
-
-	@POST
-	@Path("/create")
-	@Operation(
-		summary = "Create bootstrap",
-		description = "Builds a bootstrap file for distribution",
-		responses = {
-			@ApiResponse(
-				description = "path to file on success, an exception on failure",
-				content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string"))
-			)
-		}
-	)
-	@SecurityRequirement(name = "apiKey")
-	public String createBootstrap(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
-		Security.checkApiCallAllowed(request);
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-
-			Bootstrap bootstrap = new Bootstrap(repository);
-			try {
-				bootstrap.checkRepositoryState();
-			} catch (DataException e) {
-				LOGGER.info("Not ready to create bootstrap: {}", e.getMessage());
-				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
-			}
-			bootstrap.validateBlockchain();
-			return bootstrap.create();
-
-		} catch (DataException | InterruptedException | IOException e) {
-			LOGGER.info("Unable to create bootstrap", e);
-			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
-		}
-	}
-
-	@GET
-	@Path("/validate")
-	@Operation(
-			summary = "Validate blockchain",
-			description = "Useful to check database integrity prior to creating or after installing a bootstrap. " +
-					"This process is intensive and can take over an hour to run.",
-			responses = {
-					@ApiResponse(
-							description = "true if valid, false if invalid",
-							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
-					)
-			}
-	)
-	@SecurityRequirement(name = "apiKey")
-	public boolean validateBootstrap(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
-		Security.checkApiCallAllowed(request);
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-
-			Bootstrap bootstrap = new Bootstrap(repository);
-			return bootstrap.validateCompleteBlockchain();
-
-		} catch (DataException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
-		}
-	}
-}

src/main/java/org/qortal/api/resource/ChatResource.java

@@ -13,7 +13,11 @@ import io.swagger.v3.oas.annotations.tags.Tag;
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -154,7 +158,7 @@ public class ChatResource {
 	)
 	@ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String buildChat(@HeaderParam(Security.API_KEY_HEADER) String apiKey, ChatTransactionData transactionData) {
+	public String buildChat(ChatTransactionData transactionData) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -202,7 +206,7 @@ public class ChatResource {
 	)
 	@ApiErrors({ApiError.TRANSACTION_INVALID, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String buildChat(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String rawBytes58) {
+	public String buildChat(String rawBytes58) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {

src/main/java/org/qortal/api/resource/CrossChainBitcoinACCTv1Resource.java

@@ -5,14 +5,12 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.util.Arrays;
 import java.util.Random;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
@@ -81,8 +79,7 @@ public class CrossChainBitcoinACCTv1Resource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_DATA, ApiError.INVALID_REFERENCE, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String buildTrade(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainBuildRequest tradeRequest) {
+	public String buildTrade(CrossChainBuildRequest tradeRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] creatorPublicKey = tradeRequest.creatorPublicKey;
@@ -177,8 +174,7 @@ public class CrossChainBitcoinACCTv1Resource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String buildTradeMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainTradeRequest tradeRequest) {
+	public String buildTradeMessage(CrossChainTradeRequest tradeRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] tradePublicKey = tradeRequest.tradePublicKey;
@@ -261,8 +257,7 @@ public class CrossChainBitcoinACCTv1Resource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String buildRedeemMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainDualSecretRequest secretRequest) {
+	public String buildRedeemMessage(CrossChainDualSecretRequest secretRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] partnerPublicKey = secretRequest.partnerPublicKey;
@@ -365,4 +360,4 @@ public class CrossChainBitcoinACCTv1Resource {
 		}
 	}
 
-}
+}

src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java

@@ -6,13 +6,11 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
@@ -58,8 +56,7 @@ public class CrossChainBitcoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String getBitcoinWalletBalance(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
+	public String getBitcoinWalletBalance(String key58) {
 		Security.checkApiCallAllowed(request);
 
 		Bitcoin bitcoin = Bitcoin.getInstance();
@@ -67,16 +64,11 @@ public class CrossChainBitcoinResource {
 		if (!bitcoin.isValidDeterministicKey(key58))
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
 
-		try {
-			Long balance = bitcoin.getWalletBalanceFromTransactions(key58);
-			if (balance == null)
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
-
-			return balance.toString();
-
-		} catch (ForeignBlockchainException e) {
+		Long balance = bitcoin.getWalletBalance(key58);
+		if (balance == null)
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
-		}
+
+		return balance.toString();
 	}
 
 	@POST
@@ -102,8 +94,7 @@ public class CrossChainBitcoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public List<SimpleTransaction> getBitcoinWalletTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
+	public List<SimpleTransaction> getBitcoinWalletTransactions(String key58) {
 		Security.checkApiCallAllowed(request);
 
 		Bitcoin bitcoin = Bitcoin.getInstance();
@@ -139,8 +130,7 @@ public class CrossChainBitcoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String sendBitcoin(@HeaderParam(Security.API_KEY_HEADER) String apiKey, BitcoinSendRequest bitcoinSendRequest) {
+	public String sendBitcoin(BitcoinSendRequest bitcoinSendRequest) {
 		Security.checkApiCallAllowed(request);
 
 		if (bitcoinSendRequest.bitcoinAmount <= 0)
@@ -174,4 +164,4 @@ public class CrossChainBitcoinResource {
 		return spendTransaction.getTxId().toString();
 	}
 
-}
+}

src/main/java/org/qortal/api/resource/CrossChainDogecoinACCTv1Resource.java

@@ -1,143 +0,0 @@
-package org.qortal.api.resource;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.media.Content;
-import io.swagger.v3.oas.annotations.media.Schema;
-import io.swagger.v3.oas.annotations.parameters.RequestBody;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
-import io.swagger.v3.oas.annotations.tags.Tag;
-import org.qortal.account.PrivateKeyAccount;
-import org.qortal.api.ApiError;
-import org.qortal.api.ApiErrors;
-import org.qortal.api.ApiExceptionFactory;
-import org.qortal.api.Security;
-import org.qortal.api.model.CrossChainSecretRequest;
-import org.qortal.crosschain.AcctMode;
-import org.qortal.crosschain.DogecoinACCTv1;
-import org.qortal.crypto.Crypto;
-import org.qortal.data.at.ATData;
-import org.qortal.data.crosschain.CrossChainTradeData;
-import org.qortal.group.Group;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.transaction.MessageTransaction;
-import org.qortal.transaction.Transaction.ValidationResult;
-import org.qortal.transform.Transformer;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.HeaderParam;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import java.util.Arrays;
-
-@Path("/crosschain/DogecoinACCTv1")
-@Tag(name = "Cross-Chain (DogecoinACCTv1)")
-public class CrossChainDogecoinACCTv1Resource {
-
-	@Context
-	HttpServletRequest request;
-
-	@POST
-	@Path("/redeemmessage")
-	@Operation(
-		summary = "Signs and broadcasts a 'redeem' MESSAGE transaction that sends secrets to AT, releasing funds to partner",
-		description = "Specify address of cross-chain AT that needs to be messaged, Alice's trade private key, the 32-byte secret,<br>"
-			+ "and an address for receiving QORT from AT. All of these can be found in Alice's trade bot data.<br>"
-			+ "AT needs to be in 'trade' mode. Messages sent to an AT in any other mode will be ignored, but still cost fees to send!<br>"
-			+ "You need to use the private key that the AT considers the trade 'partner' otherwise the MESSAGE transaction will be invalid.",
-		requestBody = @RequestBody(
-			required = true,
-			content = @Content(
-				mediaType = MediaType.APPLICATION_JSON,
-				schema = @Schema(
-					implementation = CrossChainSecretRequest.class
-				)
-			)
-		),
-		responses = {
-			@ApiResponse(
-				content = @Content(
-					schema = @Schema(
-						type = "string"
-					)
-				)
-			)
-		}
-	)
-	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public boolean buildRedeemMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainSecretRequest secretRequest) {
-		Security.checkApiCallAllowed(request);
-
-		byte[] partnerPrivateKey = secretRequest.partnerPrivateKey;
-
-		if (partnerPrivateKey == null || partnerPrivateKey.length != Transformer.PRIVATE_KEY_LENGTH)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
-
-		if (secretRequest.atAddress == null || !Crypto.isValidAtAddress(secretRequest.atAddress))
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
-
-		if (secretRequest.secret == null || secretRequest.secret.length != DogecoinACCTv1.SECRET_LENGTH)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
-
-		if (secretRequest.receivingAddress == null || !Crypto.isValidAddress(secretRequest.receivingAddress))
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			ATData atData = fetchAtDataWithChecking(repository, secretRequest.atAddress);
-			CrossChainTradeData crossChainTradeData = DogecoinACCTv1.getInstance().populateTradeData(repository, atData);
-
-			if (crossChainTradeData.mode != AcctMode.TRADING)
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-
-			byte[] partnerPublicKey = new PrivateKeyAccount(null, partnerPrivateKey).getPublicKey();
-			String partnerAddress = Crypto.toAddress(partnerPublicKey);
-
-			// MESSAGE must come from address that AT considers trade partner
-			if (!crossChainTradeData.qortalPartnerAddress.equals(partnerAddress))
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
-
-			// Good to make MESSAGE
-
-			byte[] messageData = DogecoinACCTv1.buildRedeemMessage(secretRequest.secret, secretRequest.receivingAddress);
-
-			PrivateKeyAccount sender = new PrivateKeyAccount(repository, partnerPrivateKey);
-			MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, secretRequest.atAddress, messageData, false, false);
-
-			messageTransaction.computeNonce();
-			messageTransaction.sign(sender);
-
-			// reset repository state to prevent deadlock
-			repository.discardChanges();
-			ValidationResult result = messageTransaction.importAsUnconfirmed();
-
-			if (result != ValidationResult.OK)
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSACTION_INVALID);
-
-			return true;
-		} catch (DataException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
-		}
-	}
-
-	private ATData fetchAtDataWithChecking(Repository repository, String atAddress) throws DataException {
-		ATData atData = repository.getATRepository().fromATAddress(atAddress);
-		if (atData == null)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN);
-
-		// Must be correct AT - check functionality using code hash
-		if (!Arrays.equals(atData.getCodeHash(), DogecoinACCTv1.CODE_BYTES_HASH))
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-
-		// No point sending message to AT that's finished
-		if (atData.getIsFinished())
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-
-		return atData;
-	}
-
-}

src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java

@@ -1,175 +0,0 @@
-package org.qortal.api.resource;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.media.ArraySchema;
-import io.swagger.v3.oas.annotations.media.Content;
-import io.swagger.v3.oas.annotations.media.Schema;
-import io.swagger.v3.oas.annotations.parameters.RequestBody;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
-import io.swagger.v3.oas.annotations.tags.Tag;
-import org.bitcoinj.core.Transaction;
-import org.qortal.api.ApiError;
-import org.qortal.api.ApiErrors;
-import org.qortal.api.ApiExceptionFactory;
-import org.qortal.api.Security;
-import org.qortal.api.model.crosschain.DogecoinSendRequest;
-import org.qortal.crosschain.ForeignBlockchainException;
-import org.qortal.crosschain.Dogecoin;
-import org.qortal.crosschain.SimpleTransaction;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.HeaderParam;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import java.util.List;
-
-@Path("/crosschain/doge")
-@Tag(name = "Cross-Chain (Dogecoin)")
-public class CrossChainDogecoinResource {
-
-	@Context
-	HttpServletRequest request;
-
-	@POST
-	@Path("/walletbalance")
-	@Operation(
-		summary = "Returns DOGE balance for hierarchical, deterministic BIP32 wallet",
-		description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet",
-		requestBody = @RequestBody(
-			required = true,
-			content = @Content(
-				mediaType = MediaType.TEXT_PLAIN,
-				schema = @Schema(
-					type = "string",
-					description = "BIP32 'm' private/public key in base58",
-					example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc"
-				)
-			)
-		),
-		responses = {
-			@ApiResponse(
-				content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "balance (satoshis)"))
-			)
-		}
-	)
-	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String getDogecoinWalletBalance(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
-		Security.checkApiCallAllowed(request);
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		if (!dogecoin.isValidDeterministicKey(key58))
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
-
-		try {
-			Long balance = dogecoin.getWalletBalanceFromTransactions(key58);
-			if (balance == null)
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
-
-			return balance.toString();
-
-		} catch (ForeignBlockchainException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
-		}
-	}
-
-	@POST
-	@Path("/wallettransactions")
-	@Operation(
-		summary = "Returns transactions for hierarchical, deterministic BIP32 wallet",
-		description = "Supply BIP32 'm' private/public key in base58, starting with 'xprv'/'xpub' for mainnet, 'tprv'/'tpub' for testnet",
-		requestBody = @RequestBody(
-			required = true,
-			content = @Content(
-				mediaType = MediaType.TEXT_PLAIN,
-				schema = @Schema(
-					type = "string",
-					description = "BIP32 'm' private/public key in base58",
-					example = "tpubD6NzVbkrYhZ4XTPc4btCZ6SMgn8CxmWkj6VBVZ1tfcJfMq4UwAjZbG8U74gGSypL9XBYk2R2BLbDBe8pcEyBKM1edsGQEPKXNbEskZozeZc"
-				)
-			)
-		),
-		responses = {
-			@ApiResponse(
-				content = @Content(array = @ArraySchema( schema = @Schema( implementation = SimpleTransaction.class ) ) )
-			)
-		}
-	)
-	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public List<SimpleTransaction> getDogecoinWalletTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
-		Security.checkApiCallAllowed(request);
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		if (!dogecoin.isValidDeterministicKey(key58))
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
-
-		try {
-			return dogecoin.getWalletTransactions(key58);
-		} catch (ForeignBlockchainException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
-		}
-	}
-
-	@POST
-	@Path("/send")
-	@Operation(
-		summary = "Sends DOGE from hierarchical, deterministic BIP32 wallet to specific address",
-		description = "Currently only supports 'legacy' P2PKH Dogecoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
-		requestBody = @RequestBody(
-			required = true,
-			content = @Content(
-				mediaType = MediaType.APPLICATION_JSON,
-				schema = @Schema(
-					implementation = DogecoinSendRequest.class
-				)
-			)
-		),
-		responses = {
-			@ApiResponse(
-				content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", description = "transaction hash"))
-			)
-		}
-	)
-	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String sendBitcoin(@HeaderParam(Security.API_KEY_HEADER) String apiKey, DogecoinSendRequest dogecoinSendRequest) {
-		Security.checkApiCallAllowed(request);
-
-		if (dogecoinSendRequest.dogecoinAmount <= 0)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-
-		if (dogecoinSendRequest.feePerByte != null && dogecoinSendRequest.feePerByte <= 0)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		if (!dogecoin.isValidAddress(dogecoinSendRequest.receivingAddress))
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
-
-		if (!dogecoin.isValidDeterministicKey(dogecoinSendRequest.xprv58))
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
-
-		Transaction spendTransaction = dogecoin.buildSpend(dogecoinSendRequest.xprv58,
-				dogecoinSendRequest.receivingAddress,
-				dogecoinSendRequest.dogecoinAmount,
-				dogecoinSendRequest.feePerByte);
-
-		if (spendTransaction == null)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE);
-
-		try {
-			dogecoin.broadcastTransaction(spendTransaction);
-		} catch (ForeignBlockchainException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
-		}
-
-		return spendTransaction.getTxId().toString();
-	}
-
-}

src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java

@@ -4,14 +4,15 @@ import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.math.BigDecimal;
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -45,7 +46,7 @@ public class CrossChainHtlcResource {
 	@Path("/address/{blockchain}/{refundPKH}/{locktime}/{redeemPKH}/{hashOfSecret}")
 	@Operation(
 		summary = "Returns HTLC address based on trade info",
-		description = "Public key hashes (PKH) and hash of secret should be 20 bytes (base58 encoded). Locktime is seconds since epoch.",
+		description = "Blockchain can be BITCOIN or LITECOIN. Public key hashes (PKH) and hash of secret should be 20 bytes (base58 encoded). Locktime is seconds since epoch.",
 		responses = {
 			@ApiResponse(
 				content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string"))
@@ -95,7 +96,7 @@ public class CrossChainHtlcResource {
 	@Path("/status/{blockchain}/{refundPKH}/{locktime}/{redeemPKH}/{hashOfSecret}")
 	@Operation(
 		summary = "Checks HTLC status",
-		description = "Public key hashes (PKH) and hash of secret should be 20 bytes (base58 encoded). Locktime is seconds since epoch.",
+		description = "Blockchain can be BITCOIN or LITECOIN. Public key hashes (PKH) and hash of secret should be 20 bytes (base58 encoded). Locktime is seconds since epoch.",
 		responses = {
 			@ApiResponse(
 				content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = CrossChainBitcoinyHTLCStatus.class))
@@ -103,9 +104,7 @@ public class CrossChainHtlcResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
-	@SecurityRequirement(name = "apiKey")
-	public CrossChainBitcoinyHTLCStatus checkHtlcStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-			@PathParam("blockchain") String blockchainName,
+	public CrossChainBitcoinyHTLCStatus checkHtlcStatus(@PathParam("blockchain") String blockchainName,
 			@PathParam("refundPKH") String refundPKH,
 			@PathParam("locktime") int lockTime,
 			@PathParam("redeemPKH") String redeemPKH,
@@ -174,11 +173,56 @@ public class CrossChainHtlcResource {
 		}
 	}
 
-	@POST
-	@Path("/redeem/{ataddress}")
+	@GET
+	@Path("/redeem/LITECOIN/{ataddress}/{tradePrivateKey}/{secret}/{receivingAddress}")
+	@Operation(
+			summary = "Redeems HTLC associated with supplied AT, using private key, secret, and receiving address",
+			description = "Secret and private key should be 32 bytes (base58 encoded). Receiving address must be a valid LTC P2PKH address.<br>" +
+					"The secret can be found in Alice's trade bot data or in the message to Bob's AT.<br>" +
+					"The trade private key and receiving address can be found in Bob's trade bot data.",
+			responses = {
+					@ApiResponse(
+							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
+					)
+			}
+	)
+	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
+	public boolean redeemHtlc(@PathParam("ataddress") String atAddress,
+							  @PathParam("tradePrivateKey") String tradePrivateKey,
+							  @PathParam("secret") String secret,
+							  @PathParam("receivingAddress") String receivingAddress) {
+		Security.checkApiCallAllowed(request);
+
+		// base58 decode the trade private key
+		byte[] decodedTradePrivateKey = null;
+		if (tradePrivateKey != null)
+			decodedTradePrivateKey = Base58.decode(tradePrivateKey);
+
+		// base58 decode the secret
+		byte[] decodedSecret = null;
+		if (secret != null)
+			decodedSecret = Base58.decode(secret);
+
+		// Convert supplied Litecoin receiving address into public key hash (we only support P2PKH at this time)
+		Address litecoinReceivingAddress;
+		try {
+			litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), receivingAddress);
+		} catch (AddressFormatException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+		}
+		if (litecoinReceivingAddress.getOutputScriptType() != Script.ScriptType.P2PKH)
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash();
+
+		return this.doRedeemHtlc(atAddress, decodedTradePrivateKey, decodedSecret, litecoinReceivingAccountInfo);
+	}
+
+	@GET
+	@Path("/redeem/LITECOIN/{ataddress}")
 	@Operation(
 			summary = "Redeems HTLC associated with supplied AT",
-			description = "To be used by a QORT seller (Bob) who needs to redeem LTC/DOGE/etc proceeds that are stuck in a P2SH.<br>" +
+			description = "To be used by a QORT seller (Bob) who needs to redeem LTC proceeds that are stuck in a P2SH.<br>" +
 					"This requires Bob's trade bot data to be present in the database for this AT.<br>" +
 					"It will fail if the buyer has yet to redeem the QORT held in the AT.",
 			responses = {
@@ -188,8 +232,7 @@ public class CrossChainHtlcResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
-	@SecurityRequirement(name = "apiKey")
-	public boolean redeemHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("ataddress") String atAddress) {
+	public boolean redeemHtlc(@PathParam("ataddress") String atAddress) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -206,7 +249,7 @@ public class CrossChainHtlcResource {
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
 			// Attempt to find secret from the buyer's message to AT
-			byte[] decodedSecret = acct.findSecretA(repository, crossChainTradeData);
+			byte[] decodedSecret = LitecoinACCTv1.findSecretA(repository, crossChainTradeData);
 			if (decodedSecret == null) {
 				LOGGER.info(() -> String.format("Unable to find secret-A from redeem message to AT %s", atAddress));
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
@@ -220,24 +263,24 @@ public class CrossChainHtlcResource {
 			if (tradeBotData != null)
 				decodedPrivateKey = tradeBotData.getTradePrivateKey();
 
-			// Search for the foreign blockchain receiving address in the tradebot data
-			byte[] foreignBlockchainReceivingAccountInfo = null;
+			// Search for the litecoin receiving address in the tradebot data
+			byte[] litecoinReceivingAccountInfo = null;
 			if (tradeBotData != null)
 				// Use receiving address PKH from tradebot data
-				foreignBlockchainReceivingAccountInfo = tradeBotData.getReceivingAccountInfo();
+				litecoinReceivingAccountInfo = tradeBotData.getReceivingAccountInfo();
 
-			return this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, foreignBlockchainReceivingAccountInfo);
+			return this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, litecoinReceivingAccountInfo);
 
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
 	}
 
-	@POST
-	@Path("/redeemAll")
+	@GET
+	@Path("/redeemAll/LITECOIN")
 	@Operation(
 			summary = "Redeems HTLC for all applicable ATs in tradebot data",
-			description = "To be used by a QORT seller (Bob) who needs to redeem LTC/DOGE/etc proceeds that are stuck in P2SH transactions.<br>" +
+			description = "To be used by a QORT seller (Bob) who needs to redeem LTC proceeds that are stuck in P2SH transactions.<br>" +
 					"This requires Bob's trade bot data to be present in the database for any ATs that need redeeming.<br>" +
 					"Returns true if at least one trade is redeemed. More detail is available in the log.txt.* file.",
 			responses = {
@@ -247,8 +290,7 @@ public class CrossChainHtlcResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
-	@SecurityRequirement(name = "apiKey")
-	public boolean redeemAllHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+	public boolean redeemAllHtlc() {
 		Security.checkApiCallAllowed(request);
 		boolean success = false;
 
@@ -291,7 +333,7 @@ public class CrossChainHtlcResource {
 				}
 
 				// Attempt to find secret from the buyer's message to AT
-				byte[] decodedSecret = acct.findSecretA(repository, crossChainTradeData);
+				byte[] decodedSecret = LitecoinACCTv1.findSecretA(repository, crossChainTradeData);
 				if (decodedSecret == null) {
 					LOGGER.info("Unable to find secret-A from redeem message to AT {}", atAddress);
 					continue;
@@ -300,12 +342,12 @@ public class CrossChainHtlcResource {
 				// Search for the tradePrivateKey in the tradebot data
 				byte[] decodedPrivateKey = tradeBotData.getTradePrivateKey();
 
-				// Search for the foreign blockchain receiving address PKH in the tradebot data
-				byte[] foreignBlockchainReceivingAccountInfo = tradeBotData.getReceivingAccountInfo();
+				// Search for the litecoin receiving address PKH in the tradebot data
+				byte[] litecoinReceivingAccountInfo = tradeBotData.getReceivingAccountInfo();
 
 				try {
 					LOGGER.info("Attempting to redeem P2SH balance associated with AT {}...", atAddress);
-					boolean redeemed = this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, foreignBlockchainReceivingAccountInfo);
+					boolean redeemed = this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, litecoinReceivingAccountInfo);
 					if (redeemed) {
 						LOGGER.info("Redeemed P2SH balance associated with AT {}", atAddress);
 						success = true;
@@ -325,10 +367,8 @@ public class CrossChainHtlcResource {
 		return success;
 	}
 
-	private boolean doRedeemHtlc(String atAddress, byte[] decodedTradePrivateKey, byte[] decodedSecret,
-								 byte[] foreignBlockchainReceivingAccountInfo) {
+	private boolean doRedeemHtlc(String atAddress, byte[] decodedTradePrivateKey, byte[] decodedSecret, byte[] litecoinReceivingAccountInfo) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
-
 			ATData atData = repository.getATRepository().fromATAddress(atAddress);
 			if (atData == null)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN);
@@ -350,34 +390,30 @@ public class CrossChainHtlcResource {
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
 			// Validate receiving address
-			if (foreignBlockchainReceivingAccountInfo == null || foreignBlockchainReceivingAccountInfo.length != 20)
+			if (litecoinReceivingAccountInfo == null || litecoinReceivingAccountInfo.length != 20)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
-			// Make sure the receiving address isn't a QORT address, given that we can share the same field for both QORT and foreign blockchains
-			if (Crypto.isValidAddress(foreignBlockchainReceivingAccountInfo))
-				if (Base58.encode(foreignBlockchainReceivingAccountInfo).startsWith("Q"))
-					// This is likely a QORT address, not a foreign blockchain
+			// Make sure the receiving address isn't a QORT address, given that we can share the same field for both QORT and LTC
+			if (Crypto.isValidAddress(litecoinReceivingAccountInfo))
+				if (Base58.encode(litecoinReceivingAccountInfo).startsWith("Q"))
+					// This is likely a QORT address, not an LTC
 					throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
 
 			// Use secret-A to redeem P2SH-A
 
-			Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain();
-			if (bitcoiny.getClass() == Bitcoin.class) {
-				LOGGER.info("Redeeming a Bitcoin HTLC is not yet supported");
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-			}
+			Litecoin litecoin = Litecoin.getInstance();
 
 			int lockTime = crossChainTradeData.lockTimeA;
 			byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTime, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA);
-			String p2shAddressA = bitcoiny.deriveP2shAddress(redeemScriptA);
+			String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA);
 			LOGGER.info(String.format("Redeeming P2SH address: %s", p2shAddressA));
 
 			// Fee for redeem/refund is subtracted from P2SH-A balance.
 			long feeTimestamp = calcFeeTimestamp(lockTime, crossChainTradeData.tradeTimeout);
-			long p2shFee = bitcoiny.getP2shFee(feeTimestamp);
+			long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp);
 			long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-			BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoiny.getBlockchainProvider(), p2shAddressA, minimumAmountA);
+			BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
 
 			switch (htlcStatusA) {
 				case UNFUNDED:
@@ -398,14 +434,13 @@ public class CrossChainHtlcResource {
 				case FUNDED: {
 					Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount);
 					ECKey redeemKey = ECKey.fromPrivate(decodedTradePrivateKey);
-					List<TransactionOutput> fundingOutputs = bitcoiny.getUnspentOutputs(p2shAddressA);
+					List<TransactionOutput> fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA);
 
-					Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(bitcoiny.getNetworkParameters(), redeemAmount, redeemKey,
-							fundingOutputs, redeemScriptA, decodedSecret, foreignBlockchainReceivingAccountInfo);
+					Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(litecoin.getNetworkParameters(), redeemAmount, redeemKey,
+							fundingOutputs, redeemScriptA, decodedSecret, litecoinReceivingAccountInfo);
 
-					bitcoiny.broadcastTransaction(p2shRedeemTransaction);
-					LOGGER.info(String.format("P2SH address %s redeemed!", p2shAddressA));
-					return true;
+					litecoin.broadcastTransaction(p2shRedeemTransaction);
+					return true; // TODO: validate?
 				}
 			}
 
@@ -418,11 +453,11 @@ public class CrossChainHtlcResource {
 		return false;
 	}
 
-	@POST
-	@Path("/refund/{ataddress}")
+	@GET
+	@Path("/refund/LITECOIN/{ataddress}")
 	@Operation(
 			summary = "Refunds HTLC associated with supplied AT",
-			description = "To be used by a QORT buyer (Alice) who needs to refund their LTC/DOGE/etc that is stuck in a P2SH.<br>" +
+			description = "To be used by a QORT buyer (Alice) who needs to refund their LTC that is stuck in a P2SH.<br>" +
 					"This requires Alice's trade bot data to be present in the database for this AT.<br>" +
 					"It will fail if it's already redeemed by the seller, or if the lockTime (60 minutes) hasn't passed yet.",
 			responses = {
@@ -432,8 +467,7 @@ public class CrossChainHtlcResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
-	@SecurityRequirement(name = "apiKey")
-	public boolean refundHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("ataddress") String atAddress) {
+	public boolean refundHtlc(@PathParam("ataddress") String atAddress) {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -445,17 +479,9 @@ public class CrossChainHtlcResource {
 			if (tradeBotData.getForeignKey() == null)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
-			ATData atData = repository.getATRepository().fromATAddress(atAddress);
-			if (atData == null)
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN);
-
-			ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash());
-			if (acct == null)
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-
-			// Determine foreign blockchain receive address for refund
-			Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain();
-			String receiveAddress = bitcoiny.getUnusedReceiveAddress(tradeBotData.getForeignKey());
+			// Determine LTC receive address for refund
+			Litecoin litecoin = Litecoin.getInstance();
+			String receiveAddress = litecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey());
 
 			return this.doRefundHtlc(atAddress, receiveAddress);
 
@@ -466,12 +492,11 @@ public class CrossChainHtlcResource {
 		}
 	}
 
-
-	@POST
-	@Path("/refundAll")
+	@GET
+	@Path("/refund/LITECOIN/{ataddress}/{receivingAddress}")
 	@Operation(
-			summary = "Refunds HTLC for all applicable ATs in tradebot data",
-			description = "To be used by a QORT buyer (Alice) who needs to refund their LTC/DOGE/etc proceeds that are stuck in P2SH transactions.<br>" +
+			summary = "Refunds HTLC associated with supplied AT, to the specified LTC receiving address",
+			description = "To be used by a QORT buyer (Alice) who needs to refund their LTC that is stuck in a P2SH.<br>" +
 					"This requires Alice's trade bot data to be present in the database for this AT.<br>" +
 					"It will fail if it's already redeemed by the seller, or if the lockTime (60 minutes) hasn't passed yet.",
 			responses = {
@@ -481,84 +506,15 @@ public class CrossChainHtlcResource {
 			}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
-	@SecurityRequirement(name = "apiKey")
-	public boolean refundAllHtlc(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+	public boolean refundHtlc(@PathParam("ataddress") String atAddress,
+							  @PathParam("receivingAddress") String receivingAddress) {
 		Security.checkApiCallAllowed(request);
-		boolean success = false;
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
-
-			for (TradeBotData tradeBotData : allTradeBotData) {
-				String atAddress = tradeBotData.getAtAddress();
-				if (atAddress == null) {
-					LOGGER.info("Missing AT address in tradebot data", atAddress);
-					continue;
-				}
-
-				String tradeState = tradeBotData.getState();
-				if (tradeState == null) {
-					LOGGER.info("Missing trade state for AT {}", atAddress);
-					continue;
-				}
-
-				if (tradeState.startsWith("BOB")) {
-					LOGGER.info("AT {} isn't refundable because it is a sell order", atAddress);
-					continue;
-				}
-
-				ATData atData = repository.getATRepository().fromATAddress(atAddress);
-				if (atData == null) {
-					LOGGER.info("Couldn't find AT with address {}", atAddress);
-					continue;
-				}
-
-				ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash());
-				if (acct == null) {
-					continue;
-				}
-
-				CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData);
-				if (crossChainTradeData == null) {
-					LOGGER.info("Couldn't find crosschain trade data for AT {}", atAddress);
-					continue;
-				}
-
-				if (tradeBotData.getForeignKey() == null) {
-					LOGGER.info("Couldn't find foreign key for AT {}", atAddress);
-					continue;
-				}
-
-				try {
-					// Determine foreign blockchain receive address for refund
-					Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain();
-					String receivingAddress = bitcoiny.getUnusedReceiveAddress(tradeBotData.getForeignKey());
-
-					LOGGER.info("Attempting to refund P2SH balance associated with AT {}...", atAddress);
-					boolean refunded = this.doRefundHtlc(atAddress, receivingAddress);
-					if (refunded) {
-						LOGGER.info("Refunded P2SH balance associated with AT {}", atAddress);
-						success = true;
-					}
-					else {
-						LOGGER.info("Couldn't refund P2SH balance associated with AT {}. Already redeemed?", atAddress);
-					}
-				} catch (ApiException | ForeignBlockchainException e) {
-					LOGGER.info("Couldn't refund P2SH balance associated with AT {}. Missing data?", atAddress);
-				}
-			}
-
-		} catch (DataException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
-		}
-
-		return success;
+		return this.doRefundHtlc(atAddress, receivingAddress);
 	}
 
 
 	private boolean doRefundHtlc(String atAddress, String receiveAddress) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
-
 			ATData atData = repository.getATRepository().fromATAddress(atAddress);
 			if (atData == null)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ADDRESS_UNKNOWN);
@@ -571,23 +527,11 @@ public class CrossChainHtlcResource {
 			if (crossChainTradeData == null)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
-			// If the AT is "finished" then it will have a zero balance
-			// In these cases we should avoid HTLC refunds if tbe QORT haven't been returned to the seller
-			if (atData.getIsFinished() && crossChainTradeData.mode != AcctMode.REFUNDED && crossChainTradeData.mode != AcctMode.CANCELLED) {
-				LOGGER.info(String.format("Skipping AT %s because the QORT has already been redemed", atAddress));
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-			}
-
 			List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
 			TradeBotData tradeBotData = allTradeBotData.stream().filter(tradeBotDataItem -> tradeBotDataItem.getAtAddress().equals(atAddress)).findFirst().orElse(null);
 			if (tradeBotData == null)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
-			Bitcoiny bitcoiny = (Bitcoiny) acct.getBlockchain();
-			if (bitcoiny.getClass() == Bitcoin.class) {
-				LOGGER.info("Refunding a Bitcoin HTLC is not yet supported");
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-			}
 
 			int lockTime = tradeBotData.getLockTimeA();
 
@@ -595,20 +539,22 @@ public class CrossChainHtlcResource {
 			if (NTP.getTime() <= lockTime * 1000L)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON);
 
+			Litecoin litecoin = Litecoin.getInstance();
+
 			// We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113)
-			int medianBlockTime = bitcoiny.getMedianBlockTime();
+			int medianBlockTime = litecoin.getMedianBlockTime();
 			if (medianBlockTime <= lockTime)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_TOO_SOON);
 
 			byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTime, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret());
-			String p2shAddressA = bitcoiny.deriveP2shAddress(redeemScriptA);
+			String p2shAddressA = litecoin.deriveP2shAddress(redeemScriptA);
 			LOGGER.info(String.format("Refunding P2SH address: %s", p2shAddressA));
 
 			// Fee for redeem/refund is subtracted from P2SH-A balance.
 			long feeTimestamp = calcFeeTimestamp(lockTime, crossChainTradeData.tradeTimeout);
-			long p2shFee = bitcoiny.getP2shFee(feeTimestamp);
+			long p2shFee = Litecoin.getInstance().getP2shFee(feeTimestamp);
 			long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-			BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(bitcoiny.getBlockchainProvider(), p2shAddressA, minimumAmountA);
+			BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
 
 			switch (htlcStatusA) {
 				case UNFUNDED:
@@ -626,18 +572,18 @@ public class CrossChainHtlcResource {
 				case FUNDED:{
 					Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount);
 					ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey());
-					List<TransactionOutput> fundingOutputs = bitcoiny.getUnspentOutputs(p2shAddressA);
+					List<TransactionOutput> fundingOutputs = litecoin.getUnspentOutputs(p2shAddressA);
 
-					// Validate the destination foreign blockchain address
-					Address receiving = Address.fromString(bitcoiny.getNetworkParameters(), receiveAddress);
+					// Validate the destination LTC address
+					Address receiving = Address.fromString(litecoin.getNetworkParameters(), receiveAddress);
 					if (receiving.getOutputScriptType() != Script.ScriptType.P2PKH)
 						throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
-					Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(bitcoiny.getNetworkParameters(), refundAmount, refundKey,
+					Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(litecoin.getNetworkParameters(), refundAmount, refundKey,
 							fundingOutputs, redeemScriptA, lockTime, receiving.getHash());
 
-					bitcoiny.broadcastTransaction(p2shRefundTransaction);
-					return true;
+					litecoin.broadcastTransaction(p2shRefundTransaction);
+					return true; // TODO: validate?
 				}
 			}
 

src/main/java/org/qortal/api/resource/CrossChainLitecoinACCTv1Resource.java

@@ -5,7 +5,6 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.api.ApiError;
@@ -31,7 +30,6 @@ import org.qortal.utils.Base58;
 import org.qortal.utils.NTP;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
@@ -74,8 +72,7 @@ public class CrossChainLitecoinACCTv1Resource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_DATA, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public boolean buildRedeemMessage(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainSecretRequest secretRequest) {
+	public boolean buildRedeemMessage(CrossChainSecretRequest secretRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] partnerPrivateKey = secretRequest.partnerPrivateKey;

src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java

@@ -6,13 +6,11 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.HeaderParam;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
@@ -58,8 +56,7 @@ public class CrossChainLitecoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String getLitecoinWalletBalance(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
+	public String getLitecoinWalletBalance(String key58) {
 		Security.checkApiCallAllowed(request);
 
 		Litecoin litecoin = Litecoin.getInstance();
@@ -67,16 +64,11 @@ public class CrossChainLitecoinResource {
 		if (!litecoin.isValidDeterministicKey(key58))
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
 
-		try {
-			Long balance = litecoin.getWalletBalanceFromTransactions(key58);
-			if (balance == null)
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
-
-			return balance.toString();
-
-		} catch (ForeignBlockchainException e) {
+		Long balance = litecoin.getWalletBalance(key58);
+		if (balance == null)
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
-		}
+
+		return balance.toString();
 	}
 
 	@POST
@@ -102,8 +94,7 @@ public class CrossChainLitecoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public List<SimpleTransaction> getLitecoinWalletTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String key58) {
+	public List<SimpleTransaction> getLitecoinWalletTransactions(String key58) {
 		Security.checkApiCallAllowed(request);
 
 		Litecoin litecoin = Litecoin.getInstance();
@@ -139,8 +130,7 @@ public class CrossChainLitecoinResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String sendBitcoin(@HeaderParam(Security.API_KEY_HEADER) String apiKey, LitecoinSendRequest litecoinSendRequest) {
+	public String sendBitcoin(LitecoinSendRequest litecoinSendRequest) {
 		Security.checkApiCallAllowed(request);
 
 		if (litecoinSendRequest.litecoinAmount <= 0)
@@ -174,4 +164,4 @@ public class CrossChainLitecoinResource {
 		return spendTransaction.getTxId().toString();
 	}
 
-}
+}

src/main/java/org/qortal/api/resource/CrossChainResource.java

@@ -1,6 +1,5 @@
 package org.qortal.api.resource;
 
-import com.google.common.primitives.Longs;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.ArraySchema;
@@ -11,11 +10,20 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
 import java.util.function.Supplier;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -87,7 +95,7 @@ public class CrossChainResource {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
 		final boolean isExecutable = true;
-		List<CrossChainTradeData> crossChainTrades = new ArrayList<>();
+		List<CrossChainTradeData> crossChainTradesData = new ArrayList<>();
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			Map<ByteArray, Supplier<ACCT>> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain);
@@ -100,27 +108,11 @@ public class CrossChainResource {
 
 				for (ATData atData : atsData) {
 					CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData);
-					if (crossChainTradeData.mode == AcctMode.OFFERING) {
-						crossChainTrades.add(crossChainTradeData);
-					}
+					crossChainTradesData.add(crossChainTradeData);
 				}
 			}
 
-			// Sort the trades by timestamp
-			if (reverse != null && reverse) {
-				crossChainTrades.sort((a, b) -> Longs.compare(b.creationTimestamp, a.creationTimestamp));
-			}
-			else {
-				crossChainTrades.sort((a, b) -> Longs.compare(a.creationTimestamp, b.creationTimestamp));
-			}
-
-			if (limit != null && limit > 0) {
-				// Make sure to not return more than the limit
-				int upperLimit = Math.min(limit, crossChainTrades.size());
-				crossChainTrades = crossChainTrades.subList(0, upperLimit);
-			}
-
-			return crossChainTrades;
+			return crossChainTradesData;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -203,11 +195,6 @@ public class CrossChainResource {
 
 			if (minimumTimestamp != null) {
 				minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(minimumTimestamp);
-				// If not found in the block repository it will return either 0 or 1
-				if (minimumFinalHeight == 0 || minimumFinalHeight == 1) {
-					// Try the archive
-					minimumFinalHeight = repository.getBlockArchiveRepository().getHeightFromTimestamp(minimumTimestamp);
-				}
 
 				if (minimumFinalHeight == 0)
 					// We don't have any blocks since minimumTimestamp, let alone trades, so nothing to return
@@ -235,30 +222,12 @@ public class CrossChainResource {
 
 					// We also need block timestamp for use as trade timestamp
 					long timestamp = repository.getBlockRepository().getTimestampFromHeight(atState.getHeight());
-					if (timestamp == 0) {
-						// Try the archive
-						timestamp = repository.getBlockArchiveRepository().getTimestampFromHeight(atState.getHeight());
-					}
 
 					CrossChainTradeSummary crossChainTradeSummary = new CrossChainTradeSummary(crossChainTradeData, timestamp);
 					crossChainTrades.add(crossChainTradeSummary);
 				}
 			}
 
-			// Sort the trades by timestamp
-			if (reverse != null && reverse) {
-				crossChainTrades.sort((a, b) -> Longs.compare(b.getTradeTimestamp(), a.getTradeTimestamp()));
-			}
-			else {
-				crossChainTrades.sort((a, b) -> Longs.compare(a.getTradeTimestamp(), b.getTradeTimestamp()));
-			}
-
-			if (limit != null && limit > 0) {
-				// Make sure to not return more than the limit
-				int upperLimit = Math.min(limit, crossChainTrades.size());
-				crossChainTrades = crossChainTrades.subList(0, upperLimit);
-			}
-
 			return crossChainTrades;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
@@ -291,12 +260,7 @@ public class CrossChainResource {
 					description = "Maximum number of trades to include in price calculation",
 					example = "10",
 					schema = @Schema(type = "integer", defaultValue = "10")
-			) @QueryParam("maxtrades") Integer maxtrades,
-			@Parameter(
-					description = "Display price in terms of foreign currency per unit QORT",
-					example = "false",
-					schema = @Schema(type = "boolean", defaultValue = "false")
-			) @QueryParam("inverse") Boolean inverse) {
+			) @QueryParam("maxtrades") Integer maxtrades) {
 		// foreignBlockchain is required
 		if (foreignBlockchain == null)
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
@@ -306,7 +270,6 @@ public class CrossChainResource {
 		int maximumCount = maxtrades != null ? maxtrades : 10;
 		long minimumPeriod = 4 * 60 * 60 * 1000L; // ms
 		Boolean isFinished = Boolean.TRUE;
-		boolean useInversePrice = (inverse != null && inverse == true);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			Map<ByteArray, Supplier<ACCT>> acctsByCodeHash = SupportedBlockchain.getFilteredAcctMap(foreignBlockchain);
@@ -314,9 +277,6 @@ public class CrossChainResource {
 			long totalForeign = 0;
 			long totalQort = 0;
 
-			Map<Long, CrossChainTradeData> reverseSortedTradeData = new TreeMap<>(Collections.reverseOrder());
-
-			// Collect recent AT states for each ACCT version
 			for (Map.Entry<ByteArray, Supplier<ACCT>> acctInfo : acctsByCodeHash.entrySet()) {
 				byte[] codeHash = acctInfo.getKey().value;
 				ACCT acct = acctInfo.getValue().get();
@@ -325,38 +285,13 @@ public class CrossChainResource {
 						isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumCount, maximumCount, minimumPeriod);
 
 				for (ATStateData atState : atStates) {
-					// We also need block timestamp for use as trade timestamp
-					long timestamp = repository.getBlockRepository().getTimestampFromHeight(atState.getHeight());
-					if (timestamp == 0) {
-						// Try the archive
-						timestamp = repository.getBlockArchiveRepository().getTimestampFromHeight(atState.getHeight());
-					}
-
 					CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState);
-					reverseSortedTradeData.put(timestamp, crossChainTradeData);
+					totalForeign += crossChainTradeData.expectedForeignAmount;
+					totalQort += crossChainTradeData.qortAmount;
 				}
 			}
 
-			// Loop through the sorted map and calculate the average price
-			// Also remove elements beyond the maxtrades limit
-			Set set = reverseSortedTradeData.entrySet();
-			Iterator i = set.iterator();
-			int index = 0;
-			while (i.hasNext()) {
-				Map.Entry tradeDataMap = (Map.Entry)i.next();
-				CrossChainTradeData crossChainTradeData = (CrossChainTradeData) tradeDataMap.getValue();
-
-				if (maxtrades != null && index >= maxtrades) {
-					// We've reached the limit
-					break;
-				}
-
-				totalForeign += crossChainTradeData.expectedForeignAmount;
-				totalQort += crossChainTradeData.qortAmount;
-				index++;
-			}
-
-			return useInversePrice ? Amounts.scaledDivide(totalForeign, totalQort) : Amounts.scaledDivide(totalQort, totalForeign);
+			return Amounts.scaledDivide(totalQort, totalForeign);
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -391,7 +326,7 @@ public class CrossChainResource {
 	)
 	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public String cancelTrade(@HeaderParam(Security.API_KEY_HEADER) String apiKey, CrossChainCancelRequest cancelRequest) {
+	public String cancelTrade(CrossChainCancelRequest cancelRequest) {
 		Security.checkApiCallAllowed(request);
 
 		byte[] creatorPublicKey = cancelRequest.creatorPublicKey;
@@ -486,4 +421,4 @@ public class CrossChainResource {
 		}
 	}
 
-}
+}

src/main/java/org/qortal/api/resource/CrossChainTradeBotResource.java

@@ -7,14 +7,17 @@ import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
 import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.util.List;
 import java.util.stream.Collectors;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
@@ -27,7 +30,6 @@ import org.qortal.api.Security;
 import org.qortal.api.model.crosschain.TradeBotCreateRequest;
 import org.qortal.api.model.crosschain.TradeBotRespondRequest;
 import org.qortal.asset.Asset;
-import org.qortal.controller.Controller;
 import org.qortal.controller.tradebot.AcctTradeBot;
 import org.qortal.controller.tradebot.TradeBot;
 import org.qortal.crosschain.ForeignBlockchain;
@@ -66,9 +68,7 @@ public class CrossChainTradeBotResource {
 		}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
-	@SecurityRequirement(name = "apiKey")
 	public List<TradeBotData> getTradeBotStates(
-			@HeaderParam(Security.API_KEY_HEADER) String apiKey,
 			@Parameter(
 					description = "Limit to specific blockchain",
 					example = "LITECOIN",
@@ -107,10 +107,9 @@ public class CrossChainTradeBotResource {
 			)
 		}
 	)
-	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.INSUFFICIENT_BALANCE, ApiError.REPOSITORY_ISSUE, ApiError.ORDER_SIZE_TOO_SMALL})
+	@ApiErrors({ApiError.INVALID_PUBLIC_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.INSUFFICIENT_BALANCE, ApiError.REPOSITORY_ISSUE})
 	@SuppressWarnings("deprecation")
-	@SecurityRequirement(name = "apiKey")
-	public String tradeBotCreator(@HeaderParam(Security.API_KEY_HEADER) String apiKey, TradeBotCreateRequest tradeBotCreateRequest) {
+	public String tradeBotCreator(TradeBotCreateRequest tradeBotCreateRequest) {
 		Security.checkApiCallAllowed(request);
 
 		if (tradeBotCreateRequest.foreignBlockchain == null)
@@ -129,16 +128,10 @@ public class CrossChainTradeBotResource {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
 		if (tradeBotCreateRequest.foreignAmount == null || tradeBotCreateRequest.foreignAmount <= 0)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ORDER_SIZE_TOO_SMALL);
-
-		if (tradeBotCreateRequest.foreignAmount < foreignBlockchain.getMinimumOrderAmount())
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ORDER_SIZE_TOO_SMALL);
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
 		if (tradeBotCreateRequest.qortAmount <= 0 || tradeBotCreateRequest.fundingQortAmount <= 0)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.ORDER_SIZE_TOO_SMALL);
-
-		if (!Controller.getInstance().isUpToDate())
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC);
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			// Do some simple checking first
@@ -179,8 +172,7 @@ public class CrossChainTradeBotResource {
 	)
 	@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.INVALID_ADDRESS, ApiError.INVALID_CRITERIA, ApiError.FOREIGN_BLOCKCHAIN_BALANCE_ISSUE, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE, ApiError.REPOSITORY_ISSUE})
 	@SuppressWarnings("deprecation")
-	@SecurityRequirement(name = "apiKey")
-	public String tradeBotResponder(@HeaderParam(Security.API_KEY_HEADER) String apiKey, TradeBotRespondRequest tradeBotRespondRequest) {
+	public String tradeBotResponder(TradeBotRespondRequest tradeBotRespondRequest) {
 		Security.checkApiCallAllowed(request);
 
 		final String atAddress = tradeBotRespondRequest.atAddress;
@@ -198,9 +190,6 @@ public class CrossChainTradeBotResource {
 		if (tradeBotRespondRequest.receivingAddress == null || !Crypto.isValidAddress(tradeBotRespondRequest.receivingAddress))
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
 
-		if (!Controller.getInstance().isUpToDate())
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC);
-
 		// Extract data from cross-chain trading AT
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			ATData atData = fetchAtDataWithChecking(repository, atAddress);
@@ -261,8 +250,7 @@ public class CrossChainTradeBotResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String tradeBotDelete(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String tradePrivateKey58) {
+	public String tradeBotDelete(String tradePrivateKey58) {
 		Security.checkApiCallAllowed(request);
 
 		final byte[] tradePrivateKey;
@@ -295,4 +283,4 @@ public class CrossChainTradeBotResource {
 		return atData;
 	}
 
-}
+}

src/main/java/org/qortal/api/resource/ListsResource.java

@@ -1,176 +0,0 @@
-package org.qortal.api.resource;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.media.ArraySchema;
-import io.swagger.v3.oas.annotations.media.Content;
-import io.swagger.v3.oas.annotations.media.Schema;
-import io.swagger.v3.oas.annotations.parameters.RequestBody;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
-import io.swagger.v3.oas.annotations.tags.Tag;
-
-import org.qortal.api.*;
-import org.qortal.api.model.ListRequest;
-import org.qortal.crypto.Crypto;
-import org.qortal.data.account.AccountData;
-import org.qortal.list.ResourceListManager;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-
-import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-
-
-@Path("/lists")
-@Tag(name = "Lists")
-public class ListsResource {
-
-	@Context
-	HttpServletRequest request;
-
-
-	@POST
-	@Path("/{listName}")
-	@Operation(
-			summary = "Add items to a new or existing list",
-			requestBody = @RequestBody(
-					required = true,
-					content = @Content(
-							mediaType = MediaType.APPLICATION_JSON,
-							schema = @Schema(
-									implementation = ListRequest.class
-							)
-					)
-			),
-			responses = {
-					@ApiResponse(
-							description = "Returns true if all items were processed, false if any couldn't be " +
-									"processed, or an exception on failure. If false or an exception is returned, " +
-									"the list will not be updated, and the request will need to be re-issued.",
-							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
-					)
-			}
-	)
-	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String addItemstoList(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-								 @PathParam("listName") String listName,
-								 ListRequest listRequest) {
-		Security.checkApiCallAllowed(request);
-
-		if (listName == null) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-		}
-
-		if (listRequest == null || listRequest.items == null) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-		}
-
-		int successCount = 0;
-		int errorCount = 0;
-
-		for (String item : listRequest.items) {
-
-			boolean success = ResourceListManager.getInstance().addToList(listName, item, false);
-			if (success) {
-				successCount++;
-			}
-			else {
-				errorCount++;
-			}
-		}
-
-		if (successCount > 0 && errorCount == 0) {
-			// All were successful, so save the list
-			ResourceListManager.getInstance().saveList(listName);
-			return "true";
-		}
-		else {
-			// Something went wrong, so revert
-			ResourceListManager.getInstance().revertList(listName);
-			return "false";
-		}
-	}
-
-	@DELETE
-	@Path("/{listName}")
-	@Operation(
-			summary = "Remove one or more items from a list",
-			requestBody = @RequestBody(
-					required = true,
-					content = @Content(
-							mediaType = MediaType.APPLICATION_JSON,
-							schema = @Schema(
-									implementation = ListRequest.class
-							)
-					)
-			),
-			responses = {
-					@ApiResponse(
-							description = "Returns true if all items were processed, false if any couldn't be " +
-									"processed, or an exception on failure. If false or an exception is returned, " +
-									"the list will not be updated, and the request will need to be re-issued.",
-							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
-					)
-			}
-	)
-	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
-	@SecurityRequirement(name = "apiKey")
-	public String removeItemsFromList(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-									  @PathParam("listName") String listName,
-									  ListRequest listRequest) {
-		Security.checkApiCallAllowed(request);
-
-		if (listRequest == null || listRequest.items == null) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
-		}
-
-		int successCount = 0;
-		int errorCount = 0;
-
-		for (String address : listRequest.items) {
-
-			// Attempt to remove the item
-			// Don't save as we will do this at the end of the process
-			boolean success = ResourceListManager.getInstance().removeFromList(listName, address, false);
-			if (success) {
-				successCount++;
-			}
-			else {
-				errorCount++;
-			}
-		}
-
-		if (successCount > 0 && errorCount == 0) {
-			// All were successful, so save the list
-			ResourceListManager.getInstance().saveList(listName);
-			return "true";
-		}
-		else {
-			// Something went wrong, so revert
-			ResourceListManager.getInstance().revertList(listName);
-			return "false";
-		}
-	}
-
-	@GET
-	@Path("/{listName}")
-	@Operation(
-			summary = "Fetch all items in a list",
-			responses = {
-					@ApiResponse(
-							description = "A JSON array of items",
-							content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = String.class)))
-					)
-			}
-	)
-	@SecurityRequirement(name = "apiKey")
-	public String getItemsInList(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("listName") String listName) {
-		Security.checkApiCallAllowed(request);
-		return ResourceListManager.getInstance().getJSONStringForList(listName);
-	}
-
-}

src/main/java/org/qortal/api/resource/PeersResource.java

@@ -16,13 +16,19 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import javax.servlet.http.HttpServletRequest;
-import javax.ws.rs.*;
+import javax.ws.rs.DELETE;
+import javax.ws.rs.GET;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
-import org.qortal.api.*;
+import org.qortal.api.ApiError;
+import org.qortal.api.ApiErrors;
+import org.qortal.api.ApiException;
+import org.qortal.api.ApiExceptionFactory;
+import org.qortal.api.Security;
 import org.qortal.api.model.ConnectedPeer;
-import org.qortal.api.model.PeersSummary;
 import org.qortal.controller.Controller;
 import org.qortal.controller.Synchronizer;
 import org.qortal.controller.Synchronizer.SynchronizationResult;
@@ -127,7 +133,7 @@ public class PeersResource {
 		}
 	)
 	@SecurityRequirement(name = "apiKey")
-	public ExecuteProduceConsume.StatsSnapshot getEngineStats(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+	public ExecuteProduceConsume.StatsSnapshot getEngineStats() {
 		Security.checkApiCallAllowed(request);
 
 		return Network.getInstance().getStatsSnapshot();
@@ -165,7 +171,7 @@ public class PeersResource {
 		ApiError.INVALID_NETWORK_ADDRESS, ApiError.REPOSITORY_ISSUE
 	})
 	@SecurityRequirement(name = "apiKey")
-	public String addPeer(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String address) {
+	public String addPeer(String address) {
 		Security.checkApiCallAllowed(request);
 
 		final Long addedWhen = NTP.getTime();
@@ -220,7 +226,7 @@ public class PeersResource {
 		ApiError.INVALID_NETWORK_ADDRESS, ApiError.REPOSITORY_ISSUE
 	})
 	@SecurityRequirement(name = "apiKey")
-	public String removePeer(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String address) {
+	public String removePeer(String address) {
 		Security.checkApiCallAllowed(request);
 
 		try {
@@ -256,7 +262,7 @@ public class PeersResource {
 		ApiError.REPOSITORY_ISSUE
 	})
 	@SecurityRequirement(name = "apiKey")
-	public String removeKnownPeers(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String address) {
+	public String removeKnownPeers(String address) {
 		Security.checkApiCallAllowed(request);
 
 		try {
@@ -296,7 +302,7 @@ public class PeersResource {
 	)
 	@ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")
-	public List<BlockSummaryData> commonBlock(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String targetPeerAddress) {
+	public List<BlockSummaryData> commonBlock(String targetPeerAddress) {
 		Security.checkApiCallAllowed(request);
 
 		try {
@@ -332,36 +338,4 @@ public class PeersResource {
 		}
 	}
 
-	@GET
-	@Path("/summary")
-	@Operation(
-			summary = "Returns total inbound and outbound connections for connected peers",
-			responses = {
-					@ApiResponse(
-							content = @Content(
-									mediaType = MediaType.APPLICATION_JSON,
-									array = @ArraySchema(
-											schema = @Schema(
-													implementation = PeersSummary.class
-											)
-									)
-							)
-					)
-			}
-	)
-	public PeersSummary peersSummary() {
-		PeersSummary peersSummary = new PeersSummary();
-
-		List<Peer> connectedPeers = Network.getInstance().getConnectedPeers().stream().collect(Collectors.toList());
-		for (Peer peer : connectedPeers) {
-			if (peer.isOutbound()) {
-				peersSummary.inboundConnections++;
-			}
-			else {
-				peersSummary.outboundConnections++;
-			}
-		}
-		return peersSummary;
-	}
-
 }

src/main/java/org/qortal/api/resource/RenderResource.java

@@ -1,200 +0,0 @@
-package org.qortal.api.resource;
-
-import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import java.io.*;
-import java.nio.file.Paths;
-import java.util.Map;
-
-import io.swagger.v3.oas.annotations.Operation;
-import io.swagger.v3.oas.annotations.media.Content;
-import io.swagger.v3.oas.annotations.media.Schema;
-import io.swagger.v3.oas.annotations.parameters.RequestBody;
-import io.swagger.v3.oas.annotations.responses.ApiResponse;
-import io.swagger.v3.oas.annotations.security.SecurityRequirement;
-import io.swagger.v3.oas.annotations.tags.Tag;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.api.ApiError;
-import org.qortal.api.ApiExceptionFactory;
-import org.qortal.api.Security;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.arbitrary.*;
-import org.qortal.arbitrary.exception.MissingDataException;
-import org.qortal.controller.arbitrary.ArbitraryDataRenderManager;
-import org.qortal.data.transaction.ArbitraryTransactionData.*;
-import org.qortal.repository.DataException;
-import org.qortal.settings.Settings;
-import org.qortal.arbitrary.ArbitraryDataFile.*;
-import org.qortal.utils.Base58;
-
-
-@Path("/render")
-@Tag(name = "Render")
-public class RenderResource {
-
-    private static final Logger LOGGER = LogManager.getLogger(RenderResource.class);
-
-    @Context HttpServletRequest request;
-    @Context HttpServletResponse response;
-    @Context ServletContext context;
-
-    @POST
-    @Path("/preview")
-    @Operation(
-            summary = "Generate preview URL based on a user-supplied path and service",
-            requestBody = @RequestBody(
-                    required = true,
-                    content = @Content(
-                            mediaType = MediaType.TEXT_PLAIN,
-                            schema = @Schema(
-                                    type = "string", example = "/Users/user/Documents/MyStaticWebsite"
-                            )
-                    )
-            ),
-            responses = {
-                    @ApiResponse(
-                            description = "a temporary URL to preview the website",
-                            content = @Content(
-                                    mediaType = MediaType.TEXT_PLAIN,
-                                    schema = @Schema(
-                                            type = "string"
-                                    )
-                            )
-                    )
-            }
-    )
-    @SecurityRequirement(name = "apiKey")
-    public String preview(@HeaderParam(Security.API_KEY_HEADER) String apiKey, String directoryPath) {
-        Security.checkApiCallAllowed(request);
-        Method method = Method.PUT;
-        Compression compression = Compression.ZIP;
-
-        ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), null, Service.WEBSITE, null, method, compression);
-        try {
-            arbitraryDataWriter.save();
-        } catch (IOException | DataException | InterruptedException | MissingDataException e) {
-            LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage());
-            throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
-        } catch (RuntimeException e) {
-            LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage());
-            throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
-        }
-
-        ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile();
-        if (arbitraryDataFile != null) {
-            String digest58 = arbitraryDataFile.digest58();
-            if (digest58 != null) {
-                return "http://localhost:12393/render/hash/" + digest58 + "?secret=" + Base58.encode(arbitraryDataFile.getSecret());
-            }
-        }
-        return "Unable to generate preview URL";
-    }
-
-    @POST
-    @Path("/authorize/{resourceId}")
-    @SecurityRequirement(name = "apiKey")
-    public boolean authorizeResource(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @PathParam("resourceId") String resourceId) {
-        Security.checkApiCallAllowed(request);
-        Security.disallowLoopbackRequestsIfAuthBypassEnabled(request);
-        ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, null, null);
-        ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource);
-        return true;
-    }
-
-    @POST
-    @Path("authorize/{service}/{resourceId}")
-    @SecurityRequirement(name = "apiKey")
-    public boolean authorizeResource(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-                                     @PathParam("service") Service service,
-                                     @PathParam("resourceId") String resourceId) {
-        Security.checkApiCallAllowed(request);
-        Security.disallowLoopbackRequestsIfAuthBypassEnabled(request);
-        ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, null);
-        ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource);
-        return true;
-    }
-
-    @POST
-    @Path("authorize/{service}/{resourceId}/{identifier}")
-    @SecurityRequirement(name = "apiKey")
-    public boolean authorizeResource(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-                                     @PathParam("service") Service service,
-                                     @PathParam("resourceId") String resourceId,
-                                     @PathParam("identifier") String identifier) {
-        Security.checkApiCallAllowed(request);
-        Security.disallowLoopbackRequestsIfAuthBypassEnabled(request);
-        ArbitraryDataResource resource = new ArbitraryDataResource(resourceId, null, service, identifier);
-        ArbitraryDataRenderManager.getInstance().addToAuthorizedResources(resource);
-        return true;
-    }
-
-    @GET
-    @Path("/signature/{signature}")
-    @SecurityRequirement(name = "apiKey")
-    public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature) {
-        Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null);
-        return this.get(signature, ResourceIdType.SIGNATURE, null, "/", null, "/render/signature", true, true);
-    }
-
-    @GET
-    @Path("/signature/{signature}/{path:.*}")
-    @SecurityRequirement(name = "apiKey")
-    public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath) {
-        Security.requirePriorAuthorization(request, signature, Service.WEBSITE, null);
-        return this.get(signature, ResourceIdType.SIGNATURE, null, inPath,null, "/render/signature", true, true);
-    }
-
-    @GET
-    @Path("/hash/{hash}")
-    @SecurityRequirement(name = "apiKey")
-    public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58) {
-        Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null);
-        return this.get(hash58, ResourceIdType.FILE_HASH, Service.WEBSITE, "/", secret58, "/render/hash", true, false);
-    }
-
-    @GET
-    @Path("/hash/{hash}/{path:.*}")
-    @SecurityRequirement(name = "apiKey")
-    public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath,
-                                             @QueryParam("secret") String secret58) {
-        Security.requirePriorAuthorization(request, hash58, Service.WEBSITE, null);
-        return this.get(hash58, ResourceIdType.FILE_HASH, Service.WEBSITE, inPath, secret58, "/render/hash", true, false);
-    }
-
-    @GET
-    @Path("{service}/{name}/{path:.*}")
-    @SecurityRequirement(name = "apiKey")
-    public HttpServletResponse getPathByName(@PathParam("service") Service service,
-                                             @PathParam("name") String name,
-                                             @PathParam("path") String inPath) {
-        Security.requirePriorAuthorization(request, name, service, null);
-        String prefix = String.format("/render/%s", service);
-        return this.get(name, ResourceIdType.NAME, service, inPath, null, prefix, true, true);
-    }
-
-    @GET
-    @Path("{service}/{name}")
-    @SecurityRequirement(name = "apiKey")
-    public HttpServletResponse getIndexByName(@PathParam("service") Service service,
-                                              @PathParam("name") String name) {
-        Security.requirePriorAuthorization(request, name, service, null);
-        String prefix = String.format("/render/%s", service);
-        return this.get(name, ResourceIdType.NAME, service, "/", null, prefix, true, true);
-    }
-
-
-
-    private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, Service service, String inPath,
-                                    String secret58, String prefix, boolean usePrefix, boolean async) {
-
-        ArbitraryDataRenderer renderer = new ArbitraryDataRenderer(resourceId, resourceIdType, service, inPath,
-                secret58, prefix, usePrefix, async, request, response, context);
-        return renderer.render();
-    }
-
-}

src/main/java/org/qortal/api/resource/TransactionsResource.java

@@ -9,8 +9,6 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
@@ -46,7 +44,6 @@ import org.qortal.transform.transaction.TransactionTransformer;
 import org.qortal.utils.Base58;
 
 import com.google.common.primitives.Bytes;
-import org.qortal.utils.NTP;
 
 @Path("/transactions")
 @Tag(name = "Transactions")
@@ -351,7 +348,7 @@ public class TransactionsResource {
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(startBlock, blockLimit, txGroupId,
-					txTypes, null, null, address, confirmationStatus, limit, offset, reverse);
+					txTypes, null, address, confirmationStatus, limit, offset, reverse);
 
 			// Expand signatures to transactions
 			List<TransactionData> transactions = new ArrayList<>(signatures.size());
@@ -366,83 +363,6 @@ public class TransactionsResource {
 		}
 	}
 
-	@GET
-	@Path("/unitfee")
-	@Operation(
-			summary = "Get transaction unit fee",
-			responses = {
-					@ApiResponse(
-							content = @Content(
-									mediaType = MediaType.TEXT_PLAIN,
-									schema = @Schema(
-											type = "number"
-									)
-							)
-					)
-			}
-	)
-	@ApiErrors({
-			ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE
-	})
-	public long getTransactionUnitFee(@QueryParam("txType") TransactionType txType,
-                                      @QueryParam("timestamp") Long timestamp,
-									  @QueryParam("level") Integer accountLevel) {
-		try {
-			if (timestamp == null) {
-				timestamp = NTP.getTime();
-			}
-
-			Constructor<?> constructor = txType.constructor;
-			Transaction transaction = (Transaction) constructor.newInstance(null, null);
-			// FUTURE: add accountLevel parameter to transaction.getUnitFee() if needed
-			return transaction.getUnitFee(timestamp);
-
-		} catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e);
-		}
-	}
-
-	@POST
-	@Path("/fee")
-	@Operation(
-			summary = "Get recommended fee for supplied transaction data",
-			requestBody = @RequestBody(
-					required = true,
-					content = @Content(
-							mediaType = MediaType.TEXT_PLAIN,
-							schema = @Schema(
-									type = "string"
-							)
-					)
-			)
-	)
-	@ApiErrors({
-			ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE
-	})
-	public long getRecommendedTransactionFee(String rawInputBytes58) {
-		byte[] rawInputBytes = Base58.decode(rawInputBytes58);
-		if (rawInputBytes.length == 0)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.JSON);
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-
-			// Append null signature on the end before transformation
-			byte[] rawBytes = Bytes.concat(rawInputBytes, new byte[TransactionTransformer.SIGNATURE_LENGTH]);
-
-			TransactionData transactionData = TransactionTransformer.fromBytes(rawBytes);
-			if (transactionData == null)
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
-
-			Transaction transaction = Transaction.fromData(repository, transactionData);
-			return transaction.calcRecommendedFee();
-
-		} catch (DataException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
-		}  catch (TransformationException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e);
-		}
-	}
-
 	@GET
 	@Path("/creator/{publickey}")
 	@Operation(
@@ -497,84 +417,33 @@ public class TransactionsResource {
 		}
 	}
 
-	@POST
-	@Path("/convert")
-	@Operation(
-			summary = "Convert transaction bytes into bytes for signing",
-			requestBody = @RequestBody(
-					required = true,
-					content = @Content(
-							mediaType = MediaType.TEXT_PLAIN,
-							schema = @Schema(
-									type = "string",
-									description = "raw, unsigned transaction in base58 encoding",
-									example = "raw transaction base58"
-							)
-					)
-			),
-			responses = {
-					@ApiResponse(
-							description = "raw, unsigned transaction encoded in Base58, ready for signing",
-							content = @Content(
-									mediaType = MediaType.TEXT_PLAIN,
-									schema = @Schema(
-											type = "string"
-									)
-							)
-					)
-			}
-	)
-	@ApiErrors({
-			ApiError.NON_PRODUCTION, ApiError.TRANSFORMATION_ERROR
-	})
-	public String convertTransactionForSigning(String rawInputBytes58) {
-		byte[] rawInputBytes = Base58.decode(rawInputBytes58);
-		if (rawInputBytes.length == 0)
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.JSON);
-
-		try {
-			// Append null signature on the end before transformation
-			byte[] rawBytes = Bytes.concat(rawInputBytes, new byte[TransactionTransformer.SIGNATURE_LENGTH]);
-
-			TransactionData transactionData = TransactionTransformer.fromBytes(rawBytes);
-			if (transactionData == null)
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
-
-			byte[] convertedBytes = TransactionTransformer.toBytesForSigning(transactionData);
-
-			return Base58.encode(convertedBytes);
-		} catch (TransformationException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e);
-		}
-	}
-
 	@POST
 	@Path("/sign")
 	@Operation(
-			summary = "Sign a raw, unsigned transaction",
-			requestBody = @RequestBody(
-					required = true,
-					content = @Content(
-							mediaType = MediaType.APPLICATION_JSON,
-							schema = @Schema(
-									implementation = SimpleTransactionSignRequest.class
-							)
+		summary = "Sign a raw, unsigned transaction",
+		requestBody = @RequestBody(
+			required = true,
+			content = @Content(
+				mediaType = MediaType.APPLICATION_JSON,
+				schema = @Schema(
+					implementation = SimpleTransactionSignRequest.class
+				)
+			)
+		),
+		responses = {
+			@ApiResponse(
+				description = "raw, signed transaction encoded in Base58",
+				content = @Content(
+					mediaType = MediaType.TEXT_PLAIN,
+					schema = @Schema(
+						type = "string"
 					)
-			),
-			responses = {
-					@ApiResponse(
-							description = "raw, signed transaction encoded in Base58",
-							content = @Content(
-									mediaType = MediaType.TEXT_PLAIN,
-									schema = @Schema(
-											type = "string"
-									)
-							)
-					)
-			}
+				)
+			)
+		}
 	)
 	@ApiErrors({
-			ApiError.NON_PRODUCTION, ApiError.INVALID_PRIVATE_KEY, ApiError.TRANSFORMATION_ERROR
+		ApiError.NON_PRODUCTION, ApiError.INVALID_PRIVATE_KEY, ApiError.TRANSFORMATION_ERROR
 	})
 	public String signTransaction(SimpleTransactionSignRequest signRequest) {
 		if (Settings.getInstance().isApiRestricted())

src/main/java/org/qortal/api/resource/UtilsResource.java

@@ -33,6 +33,7 @@ import org.qortal.transaction.Transaction.TransactionType;
 import org.qortal.transform.Transformer;
 import org.qortal.transform.transaction.TransactionTransformer;
 import org.qortal.transform.transaction.TransactionTransformer.Transformation;
+import org.qortal.utils.BIP39;
 import org.qortal.utils.Base58;
 
 import com.google.common.hash.HashCode;
@@ -194,6 +195,123 @@ public class UtilsResource {
 		return Base58.encode(random);
 	}
 
+	@GET
+	@Path("/mnemonic")
+	@Operation(
+		summary = "Generate 12-word BIP39 mnemonic",
+		description = "Optionally pass 16-byte, base58-encoded entropy or entropy will be internally generated.<br>"
+				+ "Example entropy input: YcVfxkQb6JRzqk5kF2tNLv",
+		responses = {
+			@ApiResponse(
+				description = "mnemonic",
+				content = @Content(
+					mediaType = MediaType.TEXT_PLAIN,
+					schema = @Schema(
+						type = "string"
+					)
+				)
+			)
+		}
+	)
+	@ApiErrors({ApiError.NON_PRODUCTION, ApiError.INVALID_DATA})
+	public String getMnemonic(@QueryParam("entropy") String suppliedEntropy) {
+		if (Settings.getInstance().isApiRestricted())
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION);
+
+		/*
+		 * BIP39 word lists have 2048 entries so can be represented by 11 bits.
+		 * UUID (128bits) and another 4 bits gives 132 bits.
+		 * 132 bits, divided by 11, gives 12 words.
+		 */
+		byte[] entropy;
+		if (suppliedEntropy != null) {
+			// Use caller-supplied entropy input
+			try {
+				entropy = Base58.decode(suppliedEntropy);
+			} catch (NumberFormatException e) {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+			}
+
+			// Must be 16-bytes
+			if (entropy.length != 16)
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+		} else {
+			// Generate entropy internally
+			UUID uuid = UUID.randomUUID();
+
+			byte[] uuidMSB = Longs.toByteArray(uuid.getMostSignificantBits());
+			byte[] uuidLSB = Longs.toByteArray(uuid.getLeastSignificantBits());
+			entropy = Bytes.concat(uuidMSB, uuidLSB);
+		}
+
+		// Use SHA256 to generate more bits
+		byte[] hash = Crypto.digest(entropy);
+
+		// Append first 4 bits from hash to end. (Actually 8 bits but we only use 4).
+		byte checksum = (byte) (hash[0] & 0xf0);
+		entropy = Bytes.concat(entropy, new byte[] {
+			checksum
+		});
+
+		return BIP39.encode(entropy, "en");
+	}
+
+	@POST
+	@Path("/mnemonic")
+	@Operation(
+		summary = "Calculate binary entropy from 12-word BIP39 mnemonic",
+		description = "Returns the base58-encoded binary form, or \"false\" if mnemonic is invalid.",
+		requestBody = @RequestBody(
+			required = true,
+			content = @Content(
+				mediaType = MediaType.TEXT_PLAIN,
+				schema = @Schema(
+					type = "string"
+				)
+			)
+		),
+		responses = {
+			@ApiResponse(
+				description = "entropy in base58",
+				content = @Content(
+					mediaType = MediaType.TEXT_PLAIN,
+					schema = @Schema(
+						type = "string"
+					)
+				)
+			)
+		}
+	)
+	@ApiErrors({ApiError.NON_PRODUCTION})
+	public String fromMnemonic(String mnemonic) {
+		if (Settings.getInstance().isApiRestricted())
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION);
+
+		if (mnemonic.isEmpty())
+			return "false";
+
+		// Strip leading/trailing whitespace if any
+		mnemonic = mnemonic.trim();
+
+		String[] phraseWords = mnemonic.split(" ");
+		if (phraseWords.length != 12)
+			return "false";
+
+		// Convert BIP39 mnemonic to binary
+		byte[] binary = BIP39.decode(phraseWords, "en");
+		if (binary == null)
+			return "false";
+
+		byte[] entropy = Arrays.copyOf(binary, 16); // 132 bits is 16.5 bytes, but we're discarding checksum nybble
+
+		byte checksumNybble = (byte) (binary[16] & 0xf0);
+		byte[] checksum = Crypto.digest(entropy);
+		if (checksumNybble != (byte) (checksum[0] & 0xf0))
+			return "false";
+
+		return Base58.encode(entropy);
+	}
+
 	@POST
 	@Path("/privatekey")
 	@Operation(

src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java

@@ -1,101 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.qortal.arbitrary.exception.MissingDataException;
-import org.qortal.arbitrary.ArbitraryDataFile.*;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.repository.DataException;
-import org.qortal.utils.NTP;
-
-import java.io.IOException;
-
-public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
-
-    private final Long creationTimestamp;
-    private Long buildStartTimestamp = null;
-    private Long buildEndTimestamp = null;
-    private Integer priority = 0;
-    private boolean failed = false;
-
-    private static int HIGH_PRIORITY_THRESHOLD = 5;
-
-    /* The maximum amount of time to spend on a single build */
-    // TODO: interrupt an in-progress build
-    public static long BUILD_TIMEOUT = 60*1000L; // 60 seconds
-    /* The amount of time to remember that a build has failed, to avoid retries */
-    public static long FAILURE_TIMEOUT = 5*60*1000L; // 5 minutes
-
-    public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
-        super(resourceId, resourceIdType, service, identifier);
-
-        this.creationTimestamp = NTP.getTime();
-    }
-
-    public void prepareForBuild() {
-        this.buildStartTimestamp = NTP.getTime();
-    }
-
-    public void build() throws IOException, DataException, MissingDataException {
-        Long now = NTP.getTime();
-        if (now == null) {
-            this.buildStartTimestamp = null;
-            throw new DataException("NTP time hasn't synced yet");
-        }
-
-        if (this.buildStartTimestamp == null) {
-            this.buildStartTimestamp = now;
-        }
-        ArbitraryDataReader arbitraryDataReader =
-                new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);
-
-        try {
-            arbitraryDataReader.loadSynchronously(true);
-        } finally {
-            this.buildEndTimestamp = NTP.getTime();
-        }
-    }
-
-    public boolean isBuilding() {
-        return this.buildStartTimestamp != null;
-    }
-
-    public boolean isQueued() {
-        return this.buildStartTimestamp == null;
-    }
-
-    public boolean hasReachedBuildTimeout(Long now) {
-        if (now == null || this.creationTimestamp == null) {
-            return true;
-        }
-        return now - this.creationTimestamp > BUILD_TIMEOUT;
-    }
-
-    public boolean hasReachedFailureTimeout(Long now) {
-        if (now == null || this.buildStartTimestamp == null) {
-            return true;
-        }
-        return now - this.buildStartTimestamp > FAILURE_TIMEOUT;
-    }
-
-    public Long getBuildStartTimestamp() {
-        return this.buildStartTimestamp;
-    }
-
-    public Integer getPriority() {
-        if (this.priority != null) {
-            return this.priority;
-        }
-        return 0;
-    }
-
-    public void setPriority(Integer priority) {
-        this.priority = priority;
-    }
-
-    public boolean isHighPriority() {
-        return this.priority >= HIGH_PRIORITY_THRESHOLD;
-    }
-
-    public void setFailed(boolean failed) {
-        this.failed = failed;
-    }
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java

@@ -1,280 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.exception.MissingDataException;
-import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.data.transaction.ArbitraryTransactionData.Method;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
-import org.qortal.settings.Settings;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class ArbitraryDataBuilder {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuilder.class);
-
-    private final String name;
-    private final Service service;
-    private final String identifier;
-
-    private boolean canRequestMissingFiles;
-
-    private List<ArbitraryTransactionData> transactions;
-    private ArbitraryTransactionData latestPutTransaction;
-    private final List<Path> paths;
-    private byte[] latestSignature;
-    private Path finalPath;
-    private int layerCount;
-
-    public ArbitraryDataBuilder(String name, Service service, String identifier) {
-        this.name = name;
-        this.service = service;
-        this.identifier = identifier;
-        this.paths = new ArrayList<>();
-
-        // By default we can request missing files
-        // Callers can use setCanRequestMissingFiles(false) to prevent it
-        this.canRequestMissingFiles = true;
-    }
-
-    /**
-     * Process transactions, but do not build anything
-     * This is useful for checking the status of a given resource
-     *
-     * @throws DataException
-     * @throws IOException
-     * @throws MissingDataException
-     */
-    public void process() throws DataException, IOException, MissingDataException {
-        this.fetchTransactions();
-        this.validateTransactions();
-        this.processTransactions();
-        this.validatePaths();
-        this.findLatestSignature();
-    }
-
-    /**
-     * Build the latest state of a given resource
-     *
-     * @throws DataException
-     * @throws IOException
-     * @throws MissingDataException
-     */
-    public void build() throws DataException, IOException, MissingDataException {
-        this.process();
-        this.buildLatestState();
-        this.cacheLatestSignature();
-    }
-
-    private void fetchTransactions() throws DataException {
-        try (final Repository repository = RepositoryManager.getRepository()) {
-
-            // Get the most recent PUT
-            ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
-                    .getLatestTransaction(this.name, this.service, Method.PUT, this.identifier);
-            if (latestPut == null) {
-                String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s",
-                        this.name, this.service, this.identifierString());
-                throw new DataException(message);
-            }
-            this.latestPutTransaction = latestPut;
-
-            // Load all transactions since the latest PUT
-            List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository()
-                    .getArbitraryTransactions(this.name, this.service, this.identifier, latestPut.getTimestamp());
-
-            this.transactions = transactionDataList;
-            this.layerCount = transactionDataList.size();
-        }
-    }
-
-    private void validateTransactions() throws DataException {
-        List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
-        ArbitraryTransactionData latestPut = this.latestPutTransaction;
-
-        if (latestPut == null) {
-            throw new DataException("Cannot PATCH without existing PUT. Deploy using PUT first.");
-        }
-        if (latestPut.getMethod() != Method.PUT) {
-            throw new DataException("Expected PUT but received PATCH");
-        }
-        if (transactionDataList.size() == 0) {
-            throw new DataException(String.format("No transactions found for name %s, service %s, " +
-                            "identifier: %s, since %d", name, service, this.identifierString(), latestPut.getTimestamp()));
-        }
-
-        // Verify that the signature of the first transaction matches the latest PUT
-        ArbitraryTransactionData firstTransaction = transactionDataList.get(0);
-        if (!Arrays.equals(firstTransaction.getSignature(), latestPut.getSignature())) {
-            throw new DataException("First transaction did not match latest PUT transaction");
-        }
-
-        // Remove the first transaction, as it should be the only PUT
-        transactionDataList.remove(0);
-
-        for (ArbitraryTransactionData transactionData : transactionDataList) {
-            if (transactionData == null) {
-                throw new DataException("Transaction not found");
-            }
-            if (transactionData.getMethod() != Method.PATCH) {
-                throw new DataException("Expected PATCH but received PUT");
-            }
-        }
-    }
-
-    private void processTransactions() throws IOException, DataException, MissingDataException {
-        List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
-
-        int count = 0;
-        for (ArbitraryTransactionData transactionData : transactionDataList) {
-            LOGGER.trace("Found arbitrary transaction {}", Base58.encode(transactionData.getSignature()));
-            count++;
-
-            // Build the data file, overwriting anything that was previously there
-            String sig58 = Base58.encode(transactionData.getSignature());
-            ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA,
-                    this.service, this.identifier);
-            arbitraryDataReader.setTransactionData(transactionData);
-            arbitraryDataReader.setCanRequestMissingFiles(this.canRequestMissingFiles);
-            boolean hasMissingData = false;
-            try {
-                arbitraryDataReader.loadSynchronously(true);
-            }
-            catch (MissingDataException e) {
-                hasMissingData = true;
-            }
-
-            // Handle missing data
-            if (hasMissingData) {
-                if (!this.canRequestMissingFiles) {
-                    throw new MissingDataException("Files are missing but were not requested.");
-                }
-                if (count == transactionDataList.size()) {
-                    // This is the final transaction in the list, so we need to fail
-                    throw new MissingDataException("Requesting missing files. Please wait and try again.");
-                }
-                // There are more transactions, so we should process them to give them the opportunity to request data
-                continue;
-            }
-
-            // By this point we should have all data needed to build the layers
-            Path path = arbitraryDataReader.getFilePath();
-            if (path == null) {
-                throw new DataException(String.format("Null path when building data from transaction %s", sig58));
-            }
-            if (!Files.exists(path)) {
-                throw new DataException(String.format("Path doesn't exist when building data from transaction %s", sig58));
-            }
-            paths.add(path);
-        }
-    }
-
-    private void findLatestSignature() throws DataException {
-        if (this.transactions.size() == 0) {
-            throw new DataException("Unable to find latest signature from empty transaction list");
-        }
-
-        // Find the latest signature
-        ArbitraryTransactionData latestTransaction = this.transactions.get(this.transactions.size() - 1);
-        if (latestTransaction == null) {
-            throw new DataException("Unable to find latest signature from null transaction");
-        }
-
-        this.latestSignature = latestTransaction.getSignature();
-    }
-
-    private void validatePaths() throws DataException {
-        if (this.paths.isEmpty()) {
-            throw new DataException("No paths available from which to build latest state");
-        }
-    }
-
-    private void buildLatestState() throws IOException, DataException {
-        if (this.paths.size() == 1) {
-            // No patching needed
-            this.finalPath = this.paths.get(0);
-            return;
-        }
-
-        Path pathBefore = this.paths.get(0);
-        boolean validateAllLayers = Settings.getInstance().shouldValidateAllDataLayers();
-
-        // Loop from the second path onwards
-        for (int i=1; i<paths.size(); i++) {
-            String identifierPrefix = this.identifier != null ? String.format("[%s]", this.identifier) : "";
-            LOGGER.debug(String.format("[%s][%s]%s Applying layer %d...", this.service, this.name, identifierPrefix, i));
-
-            // Create an instance of ArbitraryDataCombiner
-            Path pathAfter = this.paths.get(i);
-            byte[] signatureBefore = this.transactions.get(i-1).getSignature();
-            ArbitraryDataCombiner combiner = new ArbitraryDataCombiner(pathBefore, pathAfter, signatureBefore);
-
-            // We only want to validate this layer's hash if it's the final layer, or if the settings
-            // indicate that we should validate interim layers too
-            boolean isFinalLayer = (i == paths.size() - 1);
-            combiner.setShouldValidateHashes(isFinalLayer || validateAllLayers);
-
-            // Now combine this layer with the last, and set the output path to the "before" path for the next cycle
-            combiner.combine();
-            combiner.cleanup();
-            pathBefore = combiner.getFinalPath();
-        }
-        this.finalPath = pathBefore;
-    }
-
-    private void cacheLatestSignature() throws IOException, DataException {
-        byte[] latestTransactionSignature = this.transactions.get(this.transactions.size()-1).getSignature();
-        if (latestTransactionSignature == null) {
-            throw new DataException("Missing latest transaction signature");
-        }
-        Long now = NTP.getTime();
-        if (now == null) {
-            throw new DataException("NTP time not synced yet");
-        }
-
-        ArbitraryDataMetadataCache cache = new ArbitraryDataMetadataCache(this.finalPath);
-        cache.setSignature(latestTransactionSignature);
-        cache.setTimestamp(NTP.getTime());
-        cache.write();
-    }
-
-    private String identifierString() {
-        return identifier != null ? identifier : "";
-    }
-
-    public Path getFinalPath() {
-        return this.finalPath;
-    }
-
-    public byte[] getLatestSignature() {
-        return this.latestSignature;
-    }
-
-    public int getLayerCount() {
-        return this.layerCount;
-    }
-
-    /**
-     * Use the below setter to ensure that we only read existing
-     * data without requesting any missing files,
-     *
-     * @param canRequestMissingFiles
-     */
-    public void setCanRequestMissingFiles(boolean canRequestMissingFiles) {
-        this.canRequestMissingFiles = canRequestMissingFiles;
-    }
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java

@@ -1,162 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.qortal.arbitrary.ArbitraryDataFile.*;
-import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.controller.arbitrary.ArbitraryDataManager;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.utils.FilesystemUtils;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-
-public class ArbitraryDataCache {
-
-    private final boolean overwrite;
-    private final Path filePath;
-    private final String resourceId;
-    private final ResourceIdType resourceIdType;
-    private final Service service;
-    private final String identifier;
-
-    public ArbitraryDataCache(Path filePath, boolean overwrite, String resourceId,
-                              ResourceIdType resourceIdType, Service service, String identifier) {
-        this.filePath = filePath;
-        this.overwrite = overwrite;
-        this.resourceId = resourceId;
-        this.resourceIdType = resourceIdType;
-        this.service = service;
-        this.identifier = identifier;
-    }
-
-    public boolean isCachedDataAvailable() {
-        return !this.shouldInvalidate();
-    }
-
-    public boolean shouldInvalidate() {
-        try {
-            // If the user has requested an overwrite, always invalidate the cache
-            if (this.overwrite) {
-                return true;
-            }
-
-            // Overwrite is false, but we still need to invalidate if no files exist
-            if (!Files.exists(this.filePath) || FilesystemUtils.isDirectoryEmpty(this.filePath)) {
-                return true;
-            }
-
-            // We might want to overwrite anyway, if an updated version is available
-            if (this.shouldInvalidateResource()) {
-                return true;
-            }
-
-        } catch (IOException e) {
-            // Something went wrong, so invalidate the cache just in case
-            return true;
-        }
-
-        // No need to invalidate the cache
-        // Remember that it's up to date, so that we won't check again for a while
-        ArbitraryDataManager.getInstance().addResourceToCache(this.getArbitraryDataResource());
-
-        return false;
-    }
-
-    private boolean shouldInvalidateResource() {
-        switch (this.resourceIdType) {
-
-            case NAME:
-                return this.shouldInvalidateName();
-
-            default:
-                // Other resource ID types remain constant, so no need to invalidate
-                return false;
-        }
-    }
-
-    private boolean shouldInvalidateName() {
-        // To avoid spamming the database too often, we shouldn't check sigs or invalidate when rate limited
-        if (this.rateLimitInEffect()) {
-            return false;
-        }
-
-        // If the state's sig doesn't match the latest transaction's sig, we need to invalidate
-        // This means that an updated layer is available
-        return this.shouldInvalidateDueToSignatureMismatch();
-    }
-
-    /**
-     * rateLimitInEffect()
-     *
-     * When loading a website, we need to check the cache for every static asset loaded by the page.
-     * This would involve asking the database for the latest transaction every time.
-     * To reduce database load and page load times, we maintain an in-memory list to "rate limit" lookups.
-     * Once a resource ID is in this in-memory list, we will avoid cache invalidations until it
-     * has been present in the list for a certain amount of time.
-     * Items are automatically removed from the list when a new arbitrary transaction arrives, so this
-     * should not prevent updates from taking effect immediately.
-     *
-     * @return whether to avoid lookups for this resource due to the in-memory cache
-     */
-    private boolean rateLimitInEffect() {
-        return ArbitraryDataManager.getInstance().isResourceCached(this.getArbitraryDataResource());
-    }
-
-    private boolean shouldInvalidateDueToSignatureMismatch() {
-
-        // Fetch the latest transaction for this name and service
-        byte[] latestTransactionSig = this.fetchLatestTransactionSignature();
-
-        // Now fetch the transaction signature stored in the cache metadata
-        byte[] cachedSig = this.fetchCachedSignature();
-
-        // If either are null, we should invalidate
-        if (latestTransactionSig == null || cachedSig == null) {
-            return true;
-        }
-
-        // Check if they match
-        return !Arrays.equals(latestTransactionSig, cachedSig);
-    }
-
-    private byte[] fetchLatestTransactionSignature() {
-        try (final Repository repository = RepositoryManager.getRepository()) {
-
-            // Find latest transaction for name and service, with any method
-            ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
-                    .getLatestTransaction(this.resourceId, this.service, null, this.identifier);
-
-            if (latestTransaction != null) {
-                return latestTransaction.getSignature();
-            }
-
-        } catch (DataException e) {
-            return null;
-        }
-
-        return null;
-    }
-
-    private byte[] fetchCachedSignature() {
-        try {
-            // Fetch the transaction signature stored in the cache metadata
-            ArbitraryDataMetadataCache cache = new ArbitraryDataMetadataCache(this.filePath);
-            cache.read();
-            return cache.getSignature();
-
-        } catch (IOException | DataException e) {
-            return null;
-        }
-    }
-
-    private ArbitraryDataResource getArbitraryDataResource() {
-        // TODO: pass an ArbitraryDataResource into the constructor, rather than individual components
-        return new ArbitraryDataResource(this.resourceId, this.resourceIdType, this.service, this.identifier);
-    }
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java

@@ -1,170 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
-import org.qortal.repository.DataException;
-import org.qortal.settings.Settings;
-import org.qortal.utils.Base58;
-import org.qortal.utils.FilesystemUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InvalidObjectException;
-import java.nio.file.DirectoryNotEmptyException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Arrays;
-
-public class ArbitraryDataCombiner {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCombiner.class);
-
-    private final Path pathBefore;
-    private final Path pathAfter;
-    private final byte[] signatureBefore;
-    private boolean shouldValidateHashes;
-    private Path finalPath;
-    private ArbitraryDataMetadataPatch metadata;
-
-    public ArbitraryDataCombiner(Path pathBefore, Path pathAfter, byte[] signatureBefore) {
-        this.pathBefore = pathBefore;
-        this.pathAfter = pathAfter;
-        this.signatureBefore = signatureBefore;
-    }
-
-    public void combine() throws IOException, DataException {
-        try {
-            this.preExecute();
-            this.readMetadata();
-            this.validatePreviousSignature();
-            this.validatePreviousHash();
-            this.process();
-            this.validateCurrentHash();
-
-        } finally {
-            this.postExecute();
-        }
-    }
-
-    public void cleanup() {
-        this.cleanupPath(this.pathBefore);
-        this.cleanupPath(this.pathAfter);
-    }
-
-    private void cleanupPath(Path path) {
-        // Delete pathBefore, if it exists in our data/temp directory
-        if (FilesystemUtils.pathInsideDataOrTempPath(path)) {
-            File directory = new File(path.toString());
-            try {
-                FileUtils.deleteDirectory(directory);
-            } catch (IOException e) {
-                // This will eventually be cleaned up by a maintenance process, so log the error and continue
-                LOGGER.debug("Unable to cleanup directory {}", directory.toString());
-            }
-        }
-
-        // Delete the parent directory of pathBefore if it is empty (and exists in our data/temp directory)
-        Path parentDirectory = path.getParent();
-        if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) {
-            try {
-                Files.deleteIfExists(parentDirectory);
-            } catch (DirectoryNotEmptyException e) {
-                // No need to log anything
-            } catch (IOException e) {
-                // This will eventually be cleaned up by a maintenance process, so log the error and continue
-                LOGGER.debug("Unable to cleanup parent directory {}", parentDirectory.toString());
-            }
-        }
-    }
-
-    private void preExecute() throws DataException {
-        if (this.pathBefore == null || this.pathAfter == null) {
-            throw new DataException("No paths available to build patch");
-        }
-        if (!Files.exists(this.pathBefore) || !Files.exists(this.pathAfter)) {
-            throw new DataException("Unable to create patch because at least one path doesn't exist");
-        }
-    }
-
-    private void postExecute() {
-
-    }
-
-    private void readMetadata() throws IOException, DataException {
-        this.metadata = new ArbitraryDataMetadataPatch(this.pathAfter);
-        this.metadata.read();
-    }
-
-    private void validatePreviousSignature() throws DataException {
-        if (this.signatureBefore == null) {
-            throw new DataException("No previous signature passed to the combiner");
-        }
-
-        byte[] previousSignature = this.metadata.getPreviousSignature();
-        if (previousSignature == null) {
-            throw new DataException("Unable to extract previous signature from patch metadata");
-        }
-
-        // Compare the signatures
-        if (!Arrays.equals(previousSignature, this.signatureBefore)) {
-            throw new DataException("Previous signatures do not match - transactions out of order?");
-        }
-    }
-
-    private void validatePreviousHash() throws IOException, DataException {
-        if (!Settings.getInstance().shouldValidateAllDataLayers()) {
-            return;
-        }
-
-        byte[] previousHash = this.metadata.getPreviousHash();
-        if (previousHash == null) {
-            throw new DataException("Unable to extract previous hash from patch metadata");
-        }
-
-        ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathBefore);
-        digest.compute();
-        boolean valid = digest.isHashValid(previousHash);
-        if (!valid) {
-            String previousHash58 = Base58.encode(previousHash);
-            throw new InvalidObjectException(String.format("Previous state hash mismatch. " +
-                    "Patch prevHash: %s, actual: %s", previousHash58, digest.getHash58()));
-        }
-    }
-
-    private void process() throws IOException, DataException {
-        ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter);
-        merge.compute();
-        this.finalPath = merge.getMergePath();
-    }
-
-    private void validateCurrentHash() throws IOException, DataException {
-        if (!this.shouldValidateHashes) {
-            return;
-        }
-
-        byte[] currentHash = this.metadata.getCurrentHash();
-        if (currentHash == null) {
-            throw new DataException("Unable to extract current hash from patch metadata");
-        }
-
-        ArbitraryDataDigest digest = new ArbitraryDataDigest(this.finalPath);
-        digest.compute();
-        boolean valid = digest.isHashValid(currentHash);
-        if (!valid) {
-            String currentHash58 = Base58.encode(currentHash);
-            throw new InvalidObjectException(String.format("Current state hash mismatch. " +
-                    "Patch curHash: %s, actual: %s", currentHash58, digest.getHash58()));
-        }
-	}
-
-    public void setShouldValidateHashes(boolean shouldValidateHashes) {
-        this.shouldValidateHashes = shouldValidateHashes;
-    }
-
-    public Path getFinalPath() {
-        return this.finalPath;
-    }
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java

@@ -1,141 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
-import org.qortal.repository.DataException;
-import org.qortal.settings.Settings;
-import org.qortal.utils.FilesystemUtils;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.UUID;
-
-public class ArbitraryDataCreatePatch {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCreatePatch.class);
-
-    private final Path pathBefore;
-    private Path pathAfter;
-    private final byte[] previousSignature;
-
-    private Path finalPath;
-    private int totalFileCount;
-    private int fileDifferencesCount;
-    private ArbitraryDataMetadataPatch metadata;
-
-    private Path workingPath;
-    private String identifier;
-
-    public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter, byte[] previousSignature) {
-        this.pathBefore = pathBefore;
-        this.pathAfter = pathAfter;
-        this.previousSignature = previousSignature;
-    }
-
-    public void create() throws DataException, IOException {
-        try {
-            this.preExecute();
-            this.copyFiles();
-            this.process();
-
-        } catch (Exception e) {
-            this.cleanupOnFailure();
-            throw e;
-
-        } finally {
-            this.postExecute();
-        }
-    }
-
-    private void preExecute() throws DataException {
-        if (this.pathBefore == null || this.pathAfter == null) {
-            throw new DataException("No paths available to build patch");
-        }
-        if (!Files.exists(this.pathBefore) || !Files.exists(this.pathAfter)) {
-            throw new DataException("Unable to create patch because at least one path doesn't exist");
-        }
-
-        this.createRandomIdentifier();
-        this.createWorkingDirectory();
-    }
-
-    private void postExecute() {
-        this.cleanupWorkingPath();
-    }
-
-    private void cleanupWorkingPath() {
-        try {
-            FilesystemUtils.safeDeleteDirectory(this.workingPath, true);
-        } catch (IOException e) {
-            LOGGER.debug("Unable to cleanup working directory");
-        }
-    }
-
-    private void cleanupOnFailure() {
-        try {
-            FilesystemUtils.safeDeleteDirectory(this.finalPath, true);
-        } catch (IOException e) {
-            LOGGER.debug("Unable to cleanup diff directory on failure");
-        }
-    }
-
-    private void createRandomIdentifier() {
-        this.identifier = UUID.randomUUID().toString();
-    }
-
-    private void createWorkingDirectory() throws DataException {
-        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
-        String baseDir = Settings.getInstance().getTempDataPath();
-        Path tempDir = Paths.get(baseDir, "patch", this.identifier);
-        try {
-            Files.createDirectories(tempDir);
-        } catch (IOException e) {
-            throw new DataException("Unable to create temp directory");
-        }
-        this.workingPath = tempDir;
-    }
-
-    private void copyFiles() throws IOException {
-        // When dealing with single files, we need to copy them to a container directory
-        // in order for the structure to align with the previous revision and therefore
-        // make comparisons possible.
-
-        if (this.pathAfter.toFile().isFile()) {
-            // Create a "data" directory within the working directory
-            Path workingDataPath = Paths.get(this.workingPath.toString(), "data");
-            Files.createDirectories(workingDataPath);
-            // Copy to temp directory
-            // Filename is currently hardcoded to "data"
-            String filename = "data"; //this.pathAfter.getFileName().toString();
-            Files.copy(this.pathAfter, Paths.get(workingDataPath.toString(), filename));
-            // Update pathAfter to point to the new path
-            this.pathAfter = workingDataPath;
-        }
-    }
-
-    private void process() throws IOException, DataException {
-
-        ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter, this.previousSignature);
-        this.finalPath = diff.getDiffPath();
-        diff.compute();
-
-        this.totalFileCount = diff.getTotalFileCount();
-        this.metadata = diff.getMetadata();
-    }
-
-    public Path getFinalPath() {
-        return this.finalPath;
-    }
-
-    public int getTotalFileCount() {
-        return this.totalFileCount;
-    }
-
-    public ArbitraryDataMetadataPatch getMetadata() {
-        return this.metadata;
-    }
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java

@@ -1,383 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.json.JSONObject;
-import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
-import org.qortal.arbitrary.patch.UnifiedDiffPatch;
-import org.qortal.crypto.Crypto;
-import org.qortal.repository.DataException;
-import org.qortal.settings.Settings;
-
-import java.io.*;
-import java.nio.file.*;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.*;
-
-
-public class ArbitraryDataDiff {
-
-    /** Only create a patch if both the before and after file sizes are within defined limit **/
-    private static final long MAX_DIFF_FILE_SIZE = 100 * 1024L; // 100kiB
-
-
-    public enum DiffType {
-        COMPLETE_FILE,
-        UNIFIED_DIFF
-    }
-
-    public static class ModifiedPath {
-        private Path path;
-        private DiffType diffType;
-
-        public ModifiedPath(Path path, DiffType diffType) {
-            this.path = path;
-            this.diffType = diffType;
-        }
-
-        public ModifiedPath(JSONObject jsonObject) {
-            String pathString = jsonObject.getString("path");
-            if (pathString != null) {
-                this.path = Paths.get(pathString);
-            }
-
-            String diffTypeString = jsonObject.getString("type");
-            if (diffTypeString != null) {
-                this.diffType = DiffType.valueOf(diffTypeString);
-            }
-        }
-
-        public Path getPath() {
-            return this.path;
-        }
-
-        public DiffType getDiffType() {
-            return this.diffType;
-        }
-
-        public String toString() {
-            return this.path.toString();
-        }
-    }
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataDiff.class);
-
-    private final Path pathBefore;
-    private final Path pathAfter;
-    private final byte[] previousSignature;
-    private byte[] previousHash;
-    private byte[] currentHash;
-    private Path diffPath;
-    private String identifier;
-
-    private final List<Path> addedPaths;
-    private final List<ModifiedPath> modifiedPaths;
-    private final List<Path> removedPaths;
-
-    private int totalFileCount;
-    private ArbitraryDataMetadataPatch metadata;
-
-    public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) throws DataException {
-        this.pathBefore = pathBefore;
-        this.pathAfter = pathAfter;
-        this.previousSignature = previousSignature;
-
-        this.addedPaths = new ArrayList<>();
-        this.modifiedPaths = new ArrayList<>();
-        this.removedPaths = new ArrayList<>();
-
-        this.createRandomIdentifier();
-        this.createOutputDirectory();
-    }
-
-    public void compute() throws IOException, DataException {
-        try {
-            this.preExecute();
-            this.hashPreviousState();
-            this.findAddedOrModifiedFiles();
-            this.findRemovedFiles();
-            this.validate();
-            this.hashCurrentState();
-            this.writeMetadata();
-
-        } finally {
-            this.postExecute();
-        }
-    }
-
-    private void preExecute() {
-        LOGGER.debug("Generating diff...");
-    }
-
-    private void postExecute() {
-
-    }
-
-    private void createRandomIdentifier() {
-        this.identifier = UUID.randomUUID().toString();
-    }
-
-    private void createOutputDirectory() throws DataException {
-        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
-        String baseDir = Settings.getInstance().getTempDataPath();
-        Path tempDir = Paths.get(baseDir, "diff", this.identifier);
-        try {
-            Files.createDirectories(tempDir);
-        } catch (IOException e) {
-            throw new DataException("Unable to create temp directory");
-        }
-        this.diffPath = tempDir;
-    }
-
-    private void hashPreviousState() throws IOException, DataException {
-        ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathBefore);
-        digest.compute();
-        this.previousHash = digest.getHash();
-    }
-
-    private void findAddedOrModifiedFiles() throws IOException {
-        try {
-            final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
-            final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
-            final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
-            final ArbitraryDataDiff diff = this;
-
-            // Check for additions or modifications
-            Files.walkFileTree(this.pathAfter, new FileVisitor<>() {
-
-                @Override
-                public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) {
-                    return FileVisitResult.CONTINUE;
-                }
-
-                @Override
-                public FileVisitResult visitFile(Path afterPathAbsolute, BasicFileAttributes attrs) throws IOException {
-                    Path afterPathRelative = pathAfterAbsolute.relativize(afterPathAbsolute.toAbsolutePath());
-                    Path beforePathAbsolute = pathBeforeAbsolute.resolve(afterPathRelative);
-
-                    if (afterPathRelative.startsWith(".qortal")) {
-                        // Ignore the .qortal metadata folder
-                        return FileVisitResult.CONTINUE;
-                    }
-
-                    boolean wasAdded = false;
-                    boolean wasModified = false;
-
-                    if (!Files.exists(beforePathAbsolute)) {
-                        LOGGER.trace("File was added: {}", afterPathRelative.toString());
-                        diff.addedPaths.add(afterPathRelative);
-                        wasAdded = true;
-                    }
-                    else if (Files.size(afterPathAbsolute) != Files.size(beforePathAbsolute)) {
-                        // Check file size first because it's quicker
-                        LOGGER.trace("File size was modified: {}", afterPathRelative.toString());
-                        wasModified = true;
-                    }
-                    else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(afterPathAbsolute), ArbitraryDataDiff.digestFromPath(beforePathAbsolute))) {
-                        // Check hashes as a last resort
-                        LOGGER.trace("File contents were modified: {}", afterPathRelative.toString());
-                        wasModified = true;
-                    }
-
-                    if (wasAdded) {
-                        diff.copyFilePathToBaseDir(afterPathAbsolute, diffPathAbsolute, afterPathRelative);
-                    }
-                    if (wasModified) {
-                        try {
-                            diff.pathModified(beforePathAbsolute, afterPathAbsolute, afterPathRelative, diffPathAbsolute);
-                        } catch (DataException e) {
-                            // We can only throw IOExceptions because we are overriding FileVisitor.visitFile()
-                            throw new IOException(e);
-                        }
-                    }
-
-                    // Keep a tally of the total number of files to help with decision making
-                    diff.totalFileCount++;
-
-                    return FileVisitResult.CONTINUE;
-                }
-
-                @Override
-                public FileVisitResult visitFileFailed(Path file, IOException e){
-                    LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage());
-                    // TODO: throw exception?
-                    return FileVisitResult.TERMINATE;
-                }
-
-                @Override
-                public FileVisitResult postVisitDirectory(Path dir, IOException e) {
-                    return FileVisitResult.CONTINUE;
-                }
-
-            });
-        } catch (IOException e) {
-            LOGGER.info("IOException when walking through file tree: {}", e.getMessage());
-            throw(e);
-        }
-    }
-
-    private void findRemovedFiles() throws IOException {
-        try {
-            final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
-            final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
-            final ArbitraryDataDiff diff = this;
-
-            // Check for removals
-            Files.walkFileTree(this.pathBefore, new FileVisitor<>() {
-
-                @Override
-                public FileVisitResult preVisitDirectory(Path before, BasicFileAttributes attrs) {
-                    Path directoryPathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath());
-                    Path directoryPathAfter = pathAfterAbsolute.resolve(directoryPathBefore);
-
-                    if (directoryPathBefore.startsWith(".qortal")) {
-                        // Ignore the .qortal metadata folder
-                        return FileVisitResult.CONTINUE;
-                    }
-
-                    if (!Files.exists(directoryPathAfter)) {
-                        LOGGER.trace("Directory was removed: {}", directoryPathAfter.toString());
-                        diff.removedPaths.add(directoryPathBefore);
-                        // TODO: we might need to mark directories differently to files
-                    }
-
-                    return FileVisitResult.CONTINUE;
-                }
-
-                @Override
-                public FileVisitResult visitFile(Path before, BasicFileAttributes attrs) {
-                    Path filePathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath());
-                    Path filePathAfter = pathAfterAbsolute.resolve(filePathBefore);
-
-                    if (filePathBefore.startsWith(".qortal")) {
-                        // Ignore the .qortal metadata folder
-                        return FileVisitResult.CONTINUE;
-                    }
-
-                    if (!Files.exists(filePathAfter)) {
-                        LOGGER.trace("File was removed: {}", filePathBefore.toString());
-                        diff.removedPaths.add(filePathBefore);
-                    }
-
-                    // Keep a tally of the total number of files to help with decision making
-                    diff.totalFileCount++;
-
-                    return FileVisitResult.CONTINUE;
-                }
-
-                @Override
-                public FileVisitResult visitFileFailed(Path file, IOException e){
-                    LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage());
-                    // TODO: throw exception?
-                    return FileVisitResult.TERMINATE;
-                }
-
-                @Override
-                public FileVisitResult postVisitDirectory(Path dir, IOException e) {
-                    return FileVisitResult.CONTINUE;
-                }
-
-            });
-        } catch (IOException e) {
-            throw new IOException(String.format("IOException when walking through file tree: %s", e.getMessage()));
-        }
-    }
-
-    private void validate() throws DataException {
-        if (this.addedPaths.isEmpty() && this.modifiedPaths.isEmpty() && this.removedPaths.isEmpty()) {
-            throw new DataException("Current state matches previous state. Nothing to do.");
-        }
-    }
-
-    private void hashCurrentState() throws IOException, DataException {
-        ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathAfter);
-        digest.compute();
-        this.currentHash = digest.getHash();
-    }
-
-    private void writeMetadata() throws IOException, DataException {
-        ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.diffPath);
-        metadata.setAddedPaths(this.addedPaths);
-        metadata.setModifiedPaths(this.modifiedPaths);
-        metadata.setRemovedPaths(this.removedPaths);
-        metadata.setPreviousSignature(this.previousSignature);
-        metadata.setPreviousHash(this.previousHash);
-        metadata.setCurrentHash(this.currentHash);
-        metadata.write();
-        this.metadata = metadata;
-    }
-
-
-    private void pathModified(Path beforePathAbsolute, Path afterPathAbsolute, Path afterPathRelative,
-                              Path destinationBasePathAbsolute) throws IOException, DataException {
-
-        Path destination = Paths.get(destinationBasePathAbsolute.toString(), afterPathRelative.toString());
-        long beforeSize = Files.size(beforePathAbsolute);
-        long afterSize = Files.size(afterPathAbsolute);
-        DiffType diffType;
-
-        if (beforeSize > MAX_DIFF_FILE_SIZE || afterSize > MAX_DIFF_FILE_SIZE) {
-            // Files are large, so don't attempt a diff
-            this.copyFilePathToBaseDir(afterPathAbsolute, destinationBasePathAbsolute, afterPathRelative);
-            diffType = DiffType.COMPLETE_FILE;
-        }
-        else {
-            // Attempt to create patch using java-diff-utils
-            UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(beforePathAbsolute, afterPathAbsolute, destination);
-            unifiedDiffPatch.create();
-            if (unifiedDiffPatch.isValid()) {
-                diffType = DiffType.UNIFIED_DIFF;
-            }
-            else {
-                // Diff failed validation, so copy the whole file instead
-                this.copyFilePathToBaseDir(afterPathAbsolute, destinationBasePathAbsolute, afterPathRelative);
-                diffType = DiffType.COMPLETE_FILE;
-            }
-        }
-
-        ModifiedPath modifiedPath = new ModifiedPath(afterPathRelative, diffType);
-        this.modifiedPaths.add(modifiedPath);
-    }
-
-    private void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
-        if (!Files.exists(source)) {
-            throw new IOException(String.format("File not found: %s", source.toString()));
-        }
-
-        // Ensure parent folders exist in the destination
-        Path dest = Paths.get(base.toString(), relativePath.toString());
-        File file = new File(dest.toString());
-        File parent = file.getParentFile();
-        if (parent != null) {
-            parent.mkdirs();
-        }
-
-        LOGGER.trace("Copying {} to {}", source, dest);
-        Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING);
-    }
-    
-
-    public Path getDiffPath() {
-        return this.diffPath;
-    }
-
-    public int getTotalFileCount() {
-        return this.totalFileCount;
-    }
-
-    public ArbitraryDataMetadataPatch getMetadata() {
-        return this.metadata;
-    }
-
-
-    // Utils
-
-    private static byte[] digestFromPath(Path path) {
-        try {
-            return Crypto.digest(path.toFile());
-        } catch (IOException e) {
-            return null;
-        }
-    }
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java

@@ -1,73 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.qortal.repository.DataException;
-import org.qortal.utils.Base58;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.stream.Collectors;
-
-public class ArbitraryDataDigest {
-
-    private final Path path;
-    private byte[] hash;
-
-    public ArbitraryDataDigest(Path path) {
-        this.path = path;
-    }
-
-    public void compute() throws IOException, DataException {
-        List<Path> allPaths = Files.walk(path).filter(Files::isRegularFile).sorted().collect(Collectors.toList());
-        Path basePathAbsolute = this.path.toAbsolutePath();
-
-        MessageDigest sha256;
-        try {
-            sha256 = MessageDigest.getInstance("SHA-256");
-        } catch (NoSuchAlgorithmException e) {
-            throw new DataException("SHA-256 hashing algorithm unavailable");
-        }
-
-        for (Path path : allPaths) {
-            // We need to work with paths relative to the base path, to ensure the same hash
-            // is generated on different systems
-            Path relativePath = basePathAbsolute.relativize(path.toAbsolutePath());
-
-            // Exclude Qortal folder since it can be different each time
-            // We only care about hashing the actual user data
-            if (relativePath.startsWith(".qortal/")) {
-                continue;
-            }
-
-            // Hash path
-            byte[] filePathBytes = relativePath.toString().getBytes(StandardCharsets.UTF_8);
-            sha256.update(filePathBytes);
-
-            // Hash contents
-            byte[] fileContent = Files.readAllBytes(path);
-            sha256.update(fileContent);
-        }
-        this.hash = sha256.digest();
-    }
-
-    public boolean isHashValid(byte[] hash) {
-        return Arrays.equals(hash, this.hash);
-    }
-
-    public byte[] getHash() {
-        return this.hash;
-    }
-
-    public String getHash58() {
-        if (this.hash == null) {
-            return null;
-        }
-        return Base58.encode(this.hash);
-    }
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java

@@ -1,778 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
-import org.qortal.crypto.Crypto;
-import org.qortal.repository.DataException;
-import org.qortal.settings.Settings;
-import org.qortal.utils.Base58;
-import org.qortal.utils.FilesystemUtils;
-
-import java.io.*;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-import java.util.*;
-import java.util.stream.Stream;
-
-import static java.util.Arrays.stream;
-import static java.util.stream.Collectors.toMap;
-
-
-public class ArbitraryDataFile {
-
-    // Validation results
-    public enum ValidationResult {
-        OK(1),
-        FILE_TOO_LARGE(10),
-        FILE_NOT_FOUND(11);
-
-        public final int value;
-
-        private static final Map<Integer, ArbitraryDataFile.ValidationResult> map = stream(ArbitraryDataFile.ValidationResult.values()).collect(toMap(result -> result.value, result -> result));
-
-        ValidationResult(int value) {
-            this.value = value;
-        }
-
-        public static ArbitraryDataFile.ValidationResult valueOf(int value) {
-            return map.get(value);
-        }
-    }
-
-    // Resource ID types
-    public enum ResourceIdType {
-        SIGNATURE,
-        FILE_HASH,
-        TRANSACTION_DATA,
-        NAME
-    }
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class);
-
-    public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB
-    public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB
-    public static int SHORT_DIGEST_LENGTH = 8;
-
-    protected Path filePath;
-    protected String hash58;
-    protected byte[] signature;
-    private ArrayList<ArbitraryDataFileChunk> chunks;
-    private byte[] secret;
-
-    // Metadata
-    private byte[] metadataHash;
-    private ArbitraryDataFile metadataFile;
-    private ArbitraryDataTransactionMetadata metadata;
-
-
-    public ArbitraryDataFile() {
-    }
-
-    public ArbitraryDataFile(String hash58, byte[] signature) throws DataException {
-        this.createDataDirectory();
-        this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, signature, false);
-        this.chunks = new ArrayList<>();
-        this.hash58 = hash58;
-        this.signature = signature;
-    }
-
-    public ArbitraryDataFile(byte[] fileContent, byte[] signature) throws DataException {
-        if (fileContent == null) {
-            LOGGER.error("fileContent is null");
-            return;
-        }
-
-        this.hash58 = Base58.encode(Crypto.digest(fileContent));
-        this.signature = signature;
-        LOGGER.trace(String.format("File digest: %s, size: %d bytes", this.hash58, fileContent.length));
-
-        Path outputFilePath = getOutputFilePath(this.hash58, signature, true);
-        File outputFile = outputFilePath.toFile();
-        try (FileOutputStream outputStream = new FileOutputStream(outputFile)) {
-            outputStream.write(fileContent);
-            this.filePath = outputFilePath;
-            // Verify hash
-            if (!this.hash58.equals(this.digest58())) {
-                LOGGER.error("Hash {} does not match file digest {}", this.hash58, this.digest58());
-                this.delete();
-                throw new DataException("Data file digest validation failed");
-            }
-        } catch (IOException e) {
-            throw new DataException("Unable to write data to file");
-        }
-    }
-
-    public static ArbitraryDataFile fromHash58(String hash58, byte[] signature) throws DataException {
-        return new ArbitraryDataFile(hash58, signature);
-    }
-
-    public static ArbitraryDataFile fromHash(byte[] hash, byte[] signature) throws DataException {
-        return ArbitraryDataFile.fromHash58(Base58.encode(hash), signature);
-    }
-
-    public static ArbitraryDataFile fromPath(Path path, byte[] signature) {
-        if (path == null) {
-            return null;
-        }
-        File file = path.toFile();
-        if (file.exists()) {
-            try {
-                byte[] digest = Crypto.digest(file);
-                ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
-
-                // Copy file to data directory if needed
-                if (Files.exists(path) && !arbitraryDataFile.isInBaseDirectory(path)) {
-                    arbitraryDataFile.copyToDataDirectory(path, signature);
-                }
-                // Or, if it's already in the data directory, we may need to move it
-                else if (!path.equals(arbitraryDataFile.getFilePath())) {
-                    // Wrong path, so relocate (but don't cleanup, as the source folder may still be needed by the caller)
-                    Path dest = arbitraryDataFile.getFilePath();
-                    FilesystemUtils.moveFile(path, dest, false);
-                }
-                return arbitraryDataFile;
-
-            } catch (IOException | DataException e) {
-                LOGGER.error("Couldn't compute digest for ArbitraryDataFile");
-            }
-        }
-        return null;
-    }
-
-    public static ArbitraryDataFile fromFile(File file, byte[] signature) {
-        return ArbitraryDataFile.fromPath(Paths.get(file.getPath()), signature);
-    }
-
-    private boolean createDataDirectory() {
-        // Create the data directory if it doesn't exist
-        String dataPath = Settings.getInstance().getDataPath();
-        Path dataDirectory = Paths.get(dataPath);
-        try {
-            Files.createDirectories(dataDirectory);
-        } catch (IOException e) {
-            LOGGER.error("Unable to create data directory");
-            return false;
-        }
-        return true;
-    }
-
-    private Path copyToDataDirectory(Path sourcePath, byte[] signature) throws DataException {
-        if (this.hash58 == null || this.filePath == null) {
-            return null;
-        }
-        Path outputFilePath = getOutputFilePath(this.hash58, signature, true);
-        sourcePath = sourcePath.toAbsolutePath();
-        Path destPath = outputFilePath.toAbsolutePath();
-        try {
-            return Files.copy(sourcePath, destPath, StandardCopyOption.REPLACE_EXISTING);
-        } catch (IOException e) {
-            throw new DataException(String.format("Unable to copy file %s to data directory %s", sourcePath, destPath));
-        }
-    }
-
-    public static Path getOutputFilePath(String hash58, byte[] signature, boolean createDirectories) throws DataException {
-        Path directory;
-
-        if (hash58 == null) {
-            return null;
-        }
-        if (signature != null) {
-            // Key by signature
-            String signature58 = Base58.encode(signature);
-            String sig58First2Chars = signature58.substring(0, 2).toLowerCase();
-            String sig58Next2Chars = signature58.substring(2, 4).toLowerCase();
-            directory = Paths.get(Settings.getInstance().getDataPath(), sig58First2Chars, sig58Next2Chars, signature58);
-        }
-        else {
-            // Put files without signatures in a "_misc" directory, and the files will be relocated later
-            String hash58First2Chars = hash58.substring(0, 2).toLowerCase();
-            String hash58Next2Chars = hash58.substring(2, 4).toLowerCase();
-            directory = Paths.get(Settings.getInstance().getDataPath(), "_misc", hash58First2Chars, hash58Next2Chars);
-        }
-
-        if (createDirectories) {
-            try {
-                Files.createDirectories(directory);
-            } catch (IOException e) {
-                throw new DataException("Unable to create data subdirectory");
-            }
-        }
-        return Paths.get(directory.toString(), hash58);
-    }
-
-    public ValidationResult isValid() {
-        try {
-            // Ensure the file exists on disk
-            if (!Files.exists(this.filePath)) {
-                LOGGER.error("File doesn't exist at path {}", this.filePath);
-                return ValidationResult.FILE_NOT_FOUND;
-            }
-
-            // Validate the file size
-            long fileSize = Files.size(this.filePath);
-            if (fileSize > MAX_FILE_SIZE) {
-                LOGGER.error(String.format("ArbitraryDataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE));
-                return ArbitraryDataFile.ValidationResult.FILE_TOO_LARGE;
-            }
-
-        } catch (IOException e) {
-            return ValidationResult.FILE_NOT_FOUND;
-        }
-
-        return ValidationResult.OK;
-    }
-
-    public void validateFileSize(long expectedSize) throws DataException {
-        // Verify that we can determine the file's size
-        long fileSize = 0;
-        try {
-            fileSize = Files.size(this.getFilePath());
-        } catch (IOException e) {
-            throw new DataException(String.format("Couldn't get file size for transaction %s", Base58.encode(signature)));
-        }
-
-        // Ensure the file's size matches the size reported by the transaction
-        if (fileSize != expectedSize) {
-            throw new DataException(String.format("File size mismatch for transaction %s", Base58.encode(signature)));
-        }
-    }
-
-    private void addChunk(ArbitraryDataFileChunk chunk) {
-        this.chunks.add(chunk);
-    }
-
-    private void addChunkHashes(List<byte[]> chunkHashes) throws DataException {
-        if (chunkHashes == null || chunkHashes.isEmpty()) {
-            return;
-        }
-        for (byte[] chunkHash : chunkHashes) {
-            ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
-            this.addChunk(chunk);
-        }
-    }
-
-    public List<byte[]> getChunkHashes() {
-        List<byte[]> hashes = new ArrayList<>();
-        if (this.chunks == null || this.chunks.isEmpty()) {
-            return hashes;
-        }
-
-        for (ArbitraryDataFileChunk chunkData : this.chunks) {
-            hashes.add(chunkData.getHash());
-        }
-
-        return hashes;
-    }
-
-    public int split(int chunkSize) throws DataException {
-        try {
-
-            File file = this.getFile();
-            byte[] buffer = new byte[chunkSize];
-            this.chunks = new ArrayList<>();
-
-            if (file != null) {
-                try (FileInputStream fileInputStream = new FileInputStream(file);
-                     BufferedInputStream bis = new BufferedInputStream(fileInputStream)) {
-
-                    int numberOfBytes;
-                    while ((numberOfBytes = bis.read(buffer)) > 0) {
-                        try (ByteArrayOutputStream out = new ByteArrayOutputStream()) {
-                            out.write(buffer, 0, numberOfBytes);
-                            out.flush();
-
-                            ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray(), this.signature);
-                            ValidationResult validationResult = chunk.isValid();
-                            if (validationResult == ValidationResult.OK) {
-                                this.chunks.add(chunk);
-                            } else {
-                                throw new DataException(String.format("Chunk %s is invalid", chunk));
-                            }
-                        }
-                    }
-                }
-            }
-        } catch (Exception e) {
-            throw new DataException("Unable to split file into chunks");
-        }
-
-        return this.chunks.size();
-    }
-
-    public boolean join() {
-        // Ensure we have chunks
-        if (this.chunks != null && this.chunks.size() > 0) {
-
-            // Create temporary path for joined file
-            // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
-            String baseDir = Settings.getInstance().getTempDataPath();
-            Path tempDir = Paths.get(baseDir, "join");
-            try {
-                Files.createDirectories(tempDir);
-            } catch (IOException e) {
-                return false;
-            }
-
-            // Join the chunks
-            Path outputPath = Paths.get(tempDir.toString(), this.chunks.get(0).digest58());
-            File outputFile = new File(outputPath.toString());
-            try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) {
-                for (ArbitraryDataFileChunk chunk : this.chunks) {
-                    File sourceFile = chunk.filePath.toFile();
-                    BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile));
-                    byte[] buffer = new byte[2048];
-                    int inSize;
-                    while ((inSize = in.read(buffer)) != -1) {
-                        out.write(buffer, 0, inSize);
-                    }
-                    in.close();
-                }
-                out.close();
-
-                // Copy temporary file to data directory
-                this.filePath = this.copyToDataDirectory(outputPath, this.signature);
-                if (FilesystemUtils.pathInsideDataOrTempPath(outputPath)) {
-                    Files.delete(outputPath);
-                }
-
-                return true;
-            } catch (FileNotFoundException e) {
-                return false;
-            } catch (IOException | DataException e) {
-                return false;
-            }
-        }
-        return false;
-    }
-
-    public boolean delete() {
-        // Delete the complete file
-        // ... but only if it's inside the Qortal data or temp directory
-        if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
-            if (Files.exists(this.filePath)) {
-                try {
-                    Files.delete(this.filePath);
-                    this.cleanupFilesystem();
-                    LOGGER.debug("Deleted file {}", this.filePath);
-                    return true;
-                } catch (IOException e) {
-                    LOGGER.warn("Couldn't delete file at path {}", this.filePath);
-                }
-            }
-        }
-        return false;
-    }
-
-    public boolean deleteAllChunks() {
-        boolean success = false;
-
-        // Delete the individual chunks
-        if (this.chunks != null && this.chunks.size() > 0) {
-            Iterator iterator = this.chunks.iterator();
-            while (iterator.hasNext()) {
-                ArbitraryDataFileChunk chunk = (ArbitraryDataFileChunk) iterator.next();
-                success = chunk.delete();
-                iterator.remove();
-            }
-        }
-        return success;
-    }
-
-    public boolean deleteMetadata() {
-        if (this.metadataFile != null && this.metadataFile.exists()) {
-            return this.metadataFile.delete();
-        }
-        return false;
-    }
-
-    public boolean deleteAll() {
-        // Delete the complete file
-        boolean fileDeleted = this.delete();
-
-        // Delete the metadata file
-        boolean metadataDeleted = this.deleteMetadata();
-
-        // Delete the individual chunks
-        boolean chunksDeleted = this.deleteAllChunks();
-
-        return fileDeleted || metadataDeleted || chunksDeleted;
-    }
-
-    protected void cleanupFilesystem() throws IOException {
-        // It is essential that use a separate path reference in this method
-        // as we don't want to modify this.filePath
-        Path path = this.filePath;
-        
-        FilesystemUtils.safeDeleteEmptyParentDirectories(path);
-    }
-
-    public byte[] getBytes() {
-        try {
-            return Files.readAllBytes(this.filePath);
-        } catch (IOException e) {
-            LOGGER.error("Unable to read bytes for file");
-            return null;
-        }
-    }
-
-
-    /* Helper methods */
-
-    private boolean isInBaseDirectory(Path filePath) {
-        Path path = filePath.toAbsolutePath();
-        String dataPath = Settings.getInstance().getDataPath();
-        String basePath = Paths.get(dataPath).toAbsolutePath().toString();
-        return path.startsWith(basePath);
-    }
-
-    public boolean exists() {
-        File file = this.filePath.toFile();
-        return file.exists();
-    }
-
-    public boolean chunkExists(byte[] hash) {
-        for (ArbitraryDataFileChunk chunk : this.chunks) {
-            if (Arrays.equals(hash, chunk.getHash())) {
-                return chunk.exists();
-            }
-        }
-        if (Arrays.equals(hash, this.metadataHash)) {
-            if (this.metadataFile != null) {
-                return this.metadataFile.exists();
-            }
-        }
-        if (Arrays.equals(this.getHash(), hash)) {
-            return this.exists();
-        }
-        return false;
-    }
-
-    public boolean allChunksExist() {
-        try {
-            if (this.metadataHash == null) {
-                // We don't have any metadata so can't check if we have the chunks
-                // Even if this transaction has no chunks, we don't have the file either (already checked above)
-                return false;
-            }
-
-            if (this.metadataFile == null) {
-                this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature);
-            }
-
-            // If the metadata file doesn't exist, we can't check if we have the chunks
-            if (!metadataFile.getFilePath().toFile().exists()) {
-                return false;
-            }
-
-            if (this.metadata == null) {
-                this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath()));
-            }
-
-            // Read the metadata
-            List<byte[]> chunks = metadata.getChunks();
-            for (byte[] chunkHash : chunks) {
-                ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
-                if (!chunk.exists()) {
-                    return false;
-                }
-            }
-
-            return true;
-
-        } catch (DataException e) {
-            // Something went wrong, so assume we don't have all the chunks
-            return false;
-        }
-    }
-
-    public boolean anyChunksExist() throws DataException {
-        try {
-            if (this.metadataHash == null) {
-                // We don't have any metadata so can't check if we have the chunks
-                // Even if this transaction has no chunks, we don't have the file either (already checked above)
-                return false;
-            }
-
-            if (this.metadataFile == null) {
-                this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature);
-            }
-
-            // If the metadata file doesn't exist, we can't check if we have any chunks
-            if (!metadataFile.getFilePath().toFile().exists()) {
-                return false;
-            }
-
-            if (this.metadata == null) {
-                this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath()));
-            }
-
-            // Read the metadata
-            List<byte[]> chunks = metadata.getChunks();
-            for (byte[] chunkHash : chunks) {
-                ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
-                if (chunk.exists()) {
-                    return true;
-                }
-            }
-
-            return false;
-
-        } catch (DataException e) {
-            // Something went wrong, so assume we don't have all the chunks
-            return false;
-        }
-    }
-
-    public boolean allFilesExist() {
-        if (this.exists()) {
-            return true;
-        }
-
-        // Complete file doesn't exist, so check the chunks
-        if (this.allChunksExist()) {
-            return true;
-        }
-
-        return false;
-    }
-
-    /**
-     * Retrieve a list of file hashes for this transaction that we do not hold locally
-     *
-     * @return a List of chunk hashes, or null if we are unable to determine what is missing
-     */
-    public List<byte[]> missingHashes() {
-        List<byte[]> missingHashes = new ArrayList<>();
-        try {
-            if (this.metadataHash == null) {
-                // We don't have any metadata so can't check if we have the chunks
-                // Even if this transaction has no chunks, we don't have the file either (already checked above)
-                return null;
-            }
-
-            if (this.metadataFile == null) {
-                this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature);
-            }
-
-            // If the metadata file doesn't exist, we can't check if we have the chunks
-            if (!metadataFile.getFilePath().toFile().exists()) {
-                return null;
-            }
-
-            if (this.metadata == null) {
-                this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath()));
-            }
-
-            // Read the metadata
-            List<byte[]> chunks = metadata.getChunks();
-            for (byte[] chunkHash : chunks) {
-                ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
-                if (!chunk.exists()) {
-                    missingHashes.add(chunkHash);
-                }
-            }
-
-            return missingHashes;
-
-        } catch (DataException e) {
-            // Something went wrong, so we can't make a sensible decision
-            return null;
-        }
-    }
-
-    public boolean containsChunk(byte[] hash) {
-        for (ArbitraryDataFileChunk chunk : this.chunks) {
-            if (Arrays.equals(hash, chunk.getHash())) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    public long size() {
-        try {
-            return Files.size(this.filePath);
-        } catch (IOException e) {
-            return 0;
-        }
-    }
-
-    public int chunkCount() {
-        return this.chunks.size();
-    }
-
-    public List<ArbitraryDataFileChunk> getChunks() {
-        return this.chunks;
-    }
-
-    public byte[] chunkHashes() throws DataException {
-        if (this.chunks != null && this.chunks.size() > 0) {
-            // Return null if we only have one chunk, with the same hash as the parent
-            if (Arrays.equals(this.digest(), this.chunks.get(0).digest())) {
-                return null;
-            }
-
-            try {
-                ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-                for (ArbitraryDataFileChunk chunk : this.chunks) {
-                    byte[] chunkHash = chunk.digest();
-                    if (chunkHash.length != 32) {
-                        LOGGER.info("Invalid chunk hash length: {}", chunkHash.length);
-                        throw new DataException("Invalid chunk hash length");
-                    }
-                    outputStream.write(chunk.digest());
-                }
-                return outputStream.toByteArray();
-            } catch (IOException e) {
-                return null;
-            }
-        }
-        return null;
-    }
-
-    public List<byte[]> chunkHashList() {
-        List<byte[]> chunks = new ArrayList<>();
-
-        if (this.chunks != null && this.chunks.size() > 0) {
-            // Return null if we only have one chunk, with the same hash as the parent
-            if (Arrays.equals(this.digest(), this.chunks.get(0).digest())) {
-                return null;
-            }
-
-            try {
-                for (ArbitraryDataFileChunk chunk : this.chunks) {
-                    byte[] chunkHash = chunk.digest();
-                    if (chunkHash.length != 32) {
-                        LOGGER.info("Invalid chunk hash length: {}", chunkHash.length);
-                        throw new DataException("Invalid chunk hash length");
-                    }
-                    chunks.add(chunkHash);
-                }
-                return chunks;
-
-            } catch (DataException e) {
-                return null;
-            }
-        }
-        return null;
-    }
-
-    private void loadMetadata() throws DataException {
-        try {
-            this.metadata.read();
-
-        } catch (DataException | IOException e) {
-            throw new DataException(e);
-        }
-    }
-
-    private File getFile() {
-        File file = this.filePath.toFile();
-        if (file.exists()) {
-            return file;
-        }
-        return null;
-    }
-
-    public Path getFilePath() {
-        return this.filePath;
-    }
-
-    public byte[] digest() {
-        File file = this.getFile();
-        if (file != null && file.exists()) {
-            try {
-                return Crypto.digest(file);
-
-            } catch (IOException e) {
-                LOGGER.error("Couldn't compute digest for ArbitraryDataFile");
-            }
-        }
-        return null;
-    }
-
-    public String digest58() {
-        if (this.digest() != null) {
-            return Base58.encode(this.digest());
-        }
-        return null;
-    }
-
-    public String shortHash58() {
-        if (this.hash58 == null) {
-            return null;
-        }
-        return this.hash58.substring(0, Math.min(this.hash58.length(), SHORT_DIGEST_LENGTH));
-    }
-
-    public String getHash58() {
-        return this.hash58;
-    }
-
-    public byte[] getHash() {
-        return Base58.decode(this.hash58);
-    }
-
-    public String printChunks() {
-        String outputString = "";
-        if (this.chunkCount() > 0) {
-            for (ArbitraryDataFileChunk chunk : this.chunks) {
-                if (outputString.length() > 0) {
-                    outputString = outputString.concat(",");
-                }
-                outputString = outputString.concat(chunk.digest58());
-            }
-        }
-        return outputString;
-    }
-
-    public void setSecret(byte[] secret) {
-        this.secret = secret;
-    }
-
-    public byte[] getSecret() {
-        return this.secret;
-    }
-
-    public byte[] getSignature() {
-        return this.signature;
-    }
-
-    public void setMetadataFile(ArbitraryDataFile metadataFile) {
-        this.metadataFile = metadataFile;
-    }
-
-    public ArbitraryDataFile getMetadataFile() {
-        return this.metadataFile;
-    }
-
-    public void setMetadataHash(byte[] hash) throws DataException {
-        this.metadataHash = hash;
-
-        if (hash == null) {
-            return;
-        }
-        this.metadataFile = ArbitraryDataFile.fromHash(hash, this.signature);
-        if (metadataFile.exists()) {
-            this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath()));
-            this.addChunkHashes(this.metadata.getChunks());
-        }
-    }
-
-    public byte[] getMetadataHash() {
-        return this.metadataHash;
-    }
-
-    public void setMetadata(ArbitraryDataTransactionMetadata metadata) throws DataException {
-        this.metadata = metadata;
-        this.loadMetadata();
-    }
-
-    @Override
-    public String toString() {
-        return this.shortHash58();
-    }
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataFileChunk.java

@@ -1,54 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.repository.DataException;
-import org.qortal.utils.Base58;
-
-import java.io.IOException;
-import java.nio.file.Files;
-
-
-public class ArbitraryDataFileChunk extends ArbitraryDataFile {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileChunk.class);
-
-    public ArbitraryDataFileChunk(String hash58, byte[] signature) throws DataException {
-        super(hash58, signature);
-    }
-
-    public ArbitraryDataFileChunk(byte[] fileContent, byte[] signature) throws DataException {
-        super(fileContent, signature);
-    }
-
-    public static ArbitraryDataFileChunk fromHash58(String hash58, byte[] signature) throws DataException {
-        return new ArbitraryDataFileChunk(hash58, signature);
-    }
-
-    public static ArbitraryDataFileChunk fromHash(byte[] hash, byte[] signature) throws DataException {
-        return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash), signature);
-    }
-
-    @Override
-    public ValidationResult isValid() {
-        // DataChunk validation applies here too
-        ValidationResult superclassValidationResult = super.isValid();
-        if (superclassValidationResult != ValidationResult.OK) {
-            return superclassValidationResult;
-        }
-
-        try {
-            // Validate the file size (chunks have stricter limits)
-            long fileSize = Files.size(this.filePath);
-            if (fileSize > CHUNK_SIZE) {
-                LOGGER.error(String.format("DataFileChunk is too large: %d bytes (max chunk size: %d bytes)", fileSize, CHUNK_SIZE));
-                return ValidationResult.FILE_TOO_LARGE;
-            }
-
-        } catch (IOException e) {
-            return ValidationResult.FILE_NOT_FOUND;
-        }
-
-        return ValidationResult.OK;
-    }
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java

@@ -1,176 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.ArbitraryDataDiff.*;
-import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
-import org.qortal.arbitrary.patch.UnifiedDiffPatch;
-import org.qortal.repository.DataException;
-import org.qortal.settings.Settings;
-import org.qortal.utils.FilesystemUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.*;
-import java.util.List;
-import java.util.UUID;
-
-public class ArbitraryDataMerge {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMerge.class);
-
-    private final Path pathBefore;
-    private final Path pathAfter;
-    private Path mergePath;
-    private String identifier;
-    private ArbitraryDataMetadataPatch metadata;
-
-    public ArbitraryDataMerge(Path pathBefore, Path pathAfter) {
-        this.pathBefore = pathBefore;
-        this.pathAfter = pathAfter;
-    }
-
-    public void compute() throws IOException, DataException {
-        try {
-            this.preExecute();
-            this.copyPreviousStateToMergePath();
-            this.loadMetadata();
-            this.applyDifferences();
-            this.copyMetadata();
-
-        } finally {
-            this.postExecute();
-        }
-    }
-
-    private void preExecute() throws DataException {
-        this.createRandomIdentifier();
-        this.createOutputDirectory();
-    }
-
-    private void postExecute() {
-
-    }
-
-    private void createRandomIdentifier() {
-        this.identifier = UUID.randomUUID().toString();
-    }
-
-    private void createOutputDirectory() throws DataException {
-        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
-        String baseDir = Settings.getInstance().getTempDataPath();
-        Path tempDir = Paths.get(baseDir, "merge", this.identifier);
-        try {
-            Files.createDirectories(tempDir);
-        } catch (IOException e) {
-            throw new DataException("Unable to create temp directory");
-        }
-        this.mergePath = tempDir;
-    }
-
-    private void copyPreviousStateToMergePath() throws IOException {
-        ArbitraryDataMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get(""));
-    }
-
-    private void loadMetadata() throws IOException, DataException {
-        this.metadata = new ArbitraryDataMetadataPatch(this.pathAfter);
-        this.metadata.read();
-    }
-
-    private void applyDifferences() throws IOException, DataException {
-
-        List<Path> addedPaths = this.metadata.getAddedPaths();
-        for (Path path : addedPaths) {
-            LOGGER.trace("File was added: {}", path.toString());
-            Path filePath = Paths.get(this.pathAfter.toString(), path.toString());
-            ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path);
-        }
-
-        List<ModifiedPath> modifiedPaths = this.metadata.getModifiedPaths();
-        for (ModifiedPath modifiedPath : modifiedPaths) {
-            LOGGER.trace("File was modified: {}", modifiedPath.toString());
-            this.applyPatch(modifiedPath);
-        }
-
-        List<Path> removedPaths = this.metadata.getRemovedPaths();
-        for (Path path : removedPaths) {
-            LOGGER.trace("File was removed: {}", path.toString());
-            ArbitraryDataMerge.deletePathInBaseDir(this.mergePath, path);
-        }
-    }
-
-    private void applyPatch(ModifiedPath modifiedPath) throws IOException, DataException {
-        if (modifiedPath.getDiffType() == DiffType.UNIFIED_DIFF) {
-            // Create destination file from patch
-            UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(pathBefore, pathAfter, mergePath);
-            unifiedDiffPatch.apply(modifiedPath.getPath());
-        }
-        else if (modifiedPath.getDiffType() == DiffType.COMPLETE_FILE) {
-            // Copy complete file
-            Path filePath = Paths.get(this.pathAfter.toString(), modifiedPath.getPath().toString());
-            ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, modifiedPath.getPath());
-        }
-        else {
-            throw new DataException(String.format("Unrecognized patch diff type: %s", modifiedPath.getDiffType()));
-        }
-    }
-
-    private void copyMetadata() throws IOException {
-        Path filePath = Paths.get(this.pathAfter.toString(), ".qortal");
-        ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, Paths.get(".qortal"));
-    }
-
-
-    private static void copyPathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
-        if (!Files.exists(source)) {
-            throw new IOException(String.format("File not found: %s", source.toString()));
-        }
-
-        File sourceFile = source.toFile();
-        Path dest = Paths.get(base.toString(), relativePath.toString());
-        LOGGER.trace("Copying {} to {}", source, dest);
-
-        if (sourceFile.isFile()) {
-            Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING);
-        }
-        else if (sourceFile.isDirectory()) {
-            FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString());
-        }
-        else {
-            throw new IOException(String.format("Invalid file: %s", source.toString()));
-        }
-    }
-
-    private static void copyDirPathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
-        if (!Files.exists(source)) {
-            throw new IOException(String.format("File not found: %s", source.toString()));
-        }
-
-        Path dest = Paths.get(base.toString(), relativePath.toString());
-        LOGGER.trace("Copying {} to {}", source, dest);
-        FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString());
-    }
-
-    private static void deletePathInBaseDir(Path base, Path relativePath) throws IOException {
-        Path dest = Paths.get(base.toString(), relativePath.toString());
-        File file = new File(dest.toString());
-        if (file.exists() && file.isFile()) {
-            if (FilesystemUtils.pathInsideDataOrTempPath(dest)) {
-                LOGGER.trace("Deleting file {}", dest);
-                Files.delete(dest);
-            }
-        }
-        if (file.exists() && file.isDirectory()) {
-            if (FilesystemUtils.pathInsideDataOrTempPath(dest)) {
-                LOGGER.trace("Deleting directory {}", dest);
-                FileUtils.deleteDirectory(file);
-            }
-        }
-    }
-
-    public Path getMergePath() {
-        return this.mergePath;
-    }
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java

@@ -1,566 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import org.qortal.arbitrary.exception.MissingDataException;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
-import org.qortal.controller.arbitrary.ArbitraryDataManager;
-import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
-import org.qortal.crypto.AES;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.data.transaction.ArbitraryTransactionData.*;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.arbitrary.ArbitraryDataFile.*;
-import org.qortal.settings.Settings;
-import org.qortal.transform.Transformer;
-import org.qortal.utils.ArbitraryTransactionUtils;
-import org.qortal.utils.Base58;
-import org.qortal.utils.FilesystemUtils;
-import org.qortal.utils.ZipUtils;
-
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.SecretKey;
-import javax.crypto.spec.SecretKeySpec;
-import java.io.File;
-import java.io.IOException;
-import java.io.InvalidObjectException;
-import java.nio.file.*;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.security.InvalidAlgorithmParameterException;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-import java.util.Arrays;
-
-public class ArbitraryDataReader {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataReader.class);
-
-    private final String resourceId;
-    private final ResourceIdType resourceIdType;
-    private final Service service;
-    private final String identifier;
-    private ArbitraryTransactionData transactionData;
-    private String secret58;
-    private Path filePath;
-    private boolean canRequestMissingFiles;
-
-    // Intermediate paths
-    private final Path workingPath;
-    private final Path uncompressedPath;
-
-    // Stats (available for synchronous builds only)
-    private int layerCount;
-    private byte[] latestSignature;
-
-    public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
-        // Ensure names are always lowercase
-        if (resourceIdType == ResourceIdType.NAME) {
-            resourceId = resourceId.toLowerCase();
-        }
-
-        // If identifier is a blank string, or reserved keyword "default", treat it as null
-        if (identifier == null || identifier.equals("") || identifier.equals("default")) {
-            identifier = null;
-        }
-
-        this.resourceId = resourceId;
-        this.resourceIdType = resourceIdType;
-        this.service = service;
-        this.identifier = identifier;
-
-        this.workingPath = this.buildWorkingPath();
-        this.uncompressedPath = Paths.get(this.workingPath.toString(), "data");
-
-        // By default we can request missing files
-        // Callers can use setCanRequestMissingFiles(false) to prevent it
-        this.canRequestMissingFiles = true;
-    }
-
-    private Path buildWorkingPath() {
-        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
-        String baseDir = Settings.getInstance().getTempDataPath();
-        String identifier = this.identifier != null ?  this.identifier : "default";
-        return Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier);
-    }
-
-    public boolean isCachedDataAvailable() {
-        // If this resource is in the build queue then we shouldn't attempt to serve
-        // cached data, as it may not be fully built
-        if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(this.createQueueItem())) {
-            return false;
-        }
-
-        // Not in the build queue - so check the cache itself
-        ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false,
-                this.resourceId, this.resourceIdType, this.service, this.identifier);
-        if (cache.isCachedDataAvailable()) {
-            this.filePath = this.uncompressedPath;
-            return true;
-        }
-        return false;
-    }
-
-    public boolean isBuilding() {
-        return ArbitraryDataBuildManager.getInstance().isInBuildQueue(this.createQueueItem());
-    }
-
-    private ArbitraryDataBuildQueueItem createQueueItem() {
-        return new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service, this.identifier);
-    }
-
-    /**
-     * loadAsynchronously
-     *
-     * Attempts to load the resource asynchronously
-     * This adds the build task to a queue, and the result will be cached when complete
-     * To check the status of the build, periodically call isCachedDataAvailable()
-     * Once it returns true, you can then use getFilePath() to access the data itself.
-     *
-     * @param overwrite - set to true to force rebuild an existing cache
-     * @return true if added or already present in queue; false if not
-     */
-    public boolean loadAsynchronously(boolean overwrite, int priority) {
-        ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
-                this.resourceId, this.resourceIdType, this.service, this.identifier);
-        if (cache.isCachedDataAvailable()) {
-            // Use cached data
-            this.filePath = this.uncompressedPath;
-            return true;
-        }
-
-        ArbitraryDataBuildQueueItem item = this.createQueueItem();
-        item.setPriority(priority);
-        return ArbitraryDataBuildManager.getInstance().addToBuildQueue(item);
-    }
-
-    /**
-     * loadSynchronously
-     *
-     * Attempts to load the resource synchronously
-     * Warning: this can block for a long time when building or fetching complex data
-     * If no exception is thrown, you can then use getFilePath() to access the data immediately after returning
-     *
-     * @param overwrite - set to true to force rebuild an existing cache
-     * @throws IOException
-     * @throws DataException
-     * @throws MissingDataException
-     */
-    public void loadSynchronously(boolean overwrite) throws DataException, IOException, MissingDataException {
-        try {
-            ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
-                    this.resourceId, this.resourceIdType, this.service, this.identifier);
-            if (cache.isCachedDataAvailable()) {
-                // Use cached data
-                this.filePath = this.uncompressedPath;
-                return;
-            }
-
-            this.preExecute();
-            this.deleteExistingFiles();
-            this.fetch();
-            this.decrypt();
-            this.uncompress();
-            this.validate();
-
-        } catch (DataException e) {
-            this.deleteWorkingDirectory();
-            throw new DataException(e.getMessage());
-
-        } finally {
-            this.postExecute();
-        }
-    }
-
-    private void preExecute() throws DataException {
-        ArbitraryDataBuildManager.getInstance().setBuildInProgress(true);
-        this.checkEnabled();
-        this.createWorkingDirectory();
-        this.createUncompressedDirectory();
-    }
-
-    private void postExecute() {
-        ArbitraryDataBuildManager.getInstance().setBuildInProgress(false);
-    }
-
-    private void checkEnabled() throws DataException {
-        if (!Settings.getInstance().isQdnEnabled()) {
-            throw new DataException("QDN is disabled in settings");
-        }
-    }
-
-    private void createWorkingDirectory() throws DataException {
-        try {
-            Files.createDirectories(this.workingPath);
-        } catch (IOException e) {
-            throw new DataException("Unable to create temp directory");
-        }
-    }
-
-    /**
-     * Working directory should only be deleted on failure, since it is currently used to
-     * serve a cached version of the resource for subsequent requests.
-     * @throws IOException
-     */
-    private void deleteWorkingDirectory() throws IOException {
-        FilesystemUtils.safeDeleteDirectory(this.workingPath, true);
-    }
-
-    private void createUncompressedDirectory() throws DataException {
-        try {
-            // Create parent directory
-            Files.createDirectories(this.uncompressedPath.getParent());
-            // Ensure child directory doesn't already exist
-            FileUtils.deleteDirectory(this.uncompressedPath.toFile());
-
-        } catch (IOException e) {
-            throw new DataException("Unable to create uncompressed directory");
-        }
-    }
-
-    private void deleteExistingFiles() {
-        final Path uncompressedPath = this.uncompressedPath;
-        if (FilesystemUtils.pathInsideDataOrTempPath(uncompressedPath)) {
-            if (Files.exists(uncompressedPath)) {
-                LOGGER.trace("Attempting to delete path {}", this.uncompressedPath);
-                try {
-                    Files.walkFileTree(uncompressedPath, new SimpleFileVisitor<>() {
-
-                        @Override
-                        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                            Files.delete(file);
-                            return FileVisitResult.CONTINUE;
-                        }
-
-                        @Override
-                        public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
-                            // Don't delete the parent directory, as we want to leave an empty folder
-                            if (dir.compareTo(uncompressedPath) == 0) {
-                                return FileVisitResult.CONTINUE;
-                            }
-
-                            if (e == null) {
-                                Files.delete(dir);
-                                return FileVisitResult.CONTINUE;
-                            } else {
-                                throw e;
-                            }
-                        }
-
-                    });
-                } catch (IOException e) {
-                    LOGGER.debug("Unable to delete file or directory: {}", e.getMessage());
-                }
-            }
-        }
-    }
-
-    private void fetch() throws DataException, IOException, MissingDataException {
-        switch (resourceIdType) {
-
-            case FILE_HASH:
-                this.fetchFromFileHash();
-                break;
-
-            case NAME:
-                this.fetchFromName();
-                break;
-
-            case SIGNATURE:
-                this.fetchFromSignature();
-                break;
-
-            case TRANSACTION_DATA:
-                this.fetchFromTransactionData(this.transactionData);
-                break;
-
-            default:
-                throw new DataException(String.format("Unknown resource ID type specified: %s", resourceIdType.toString()));
-        }
-    }
-
-    private void fetchFromFileHash() throws DataException {
-        // Load data file directly from the hash (without a signature)
-        ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId, null);
-        // Set filePath to the location of the ArbitraryDataFile
-        this.filePath = arbitraryDataFile.getFilePath();
-    }
-
-    private void fetchFromName() throws DataException, IOException, MissingDataException {
-        try {
-
-            // Build the existing state using past transactions
-            ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.resourceId, this.service, this.identifier);
-            builder.build();
-            Path builtPath = builder.getFinalPath();
-            if (builtPath == null) {
-                throw new DataException("Unable to build path");
-            }
-
-            // Update stats
-            this.layerCount = builder.getLayerCount();
-            this.latestSignature = builder.getLatestSignature();
-
-            // Set filePath to the builtPath
-            this.filePath = builtPath;
-
-        } catch (InvalidObjectException e) {
-            // Hash validation failed. Invalidate the cache for this name, so it can be rebuilt
-            LOGGER.info("Deleting {}", this.workingPath.toString());
-            FilesystemUtils.safeDeleteDirectory(this.workingPath, false);
-            throw(e);
-        }
-    }
-
-    private void fetchFromSignature() throws DataException, IOException, MissingDataException {
-
-        // Load the full transaction data from the database so we can access the file hashes
-        ArbitraryTransactionData transactionData;
-        try (final Repository repository = RepositoryManager.getRepository()) {
-            transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId));
-        }
-        if (transactionData == null) {
-            throw new DataException(String.format("Transaction data not found for signature %s", this.resourceId));
-        }
-
-        this.fetchFromTransactionData(transactionData);
-    }
-
-    private void fetchFromTransactionData(ArbitraryTransactionData transactionData) throws DataException, IOException, MissingDataException {
-        if (transactionData == null) {
-            throw new DataException(String.format("Transaction data not found for signature %s", this.resourceId));
-        }
-
-        // Load hashes
-        byte[] digest = transactionData.getData();
-        byte[] metadataHash = transactionData.getMetadataHash();
-        byte[] signature = transactionData.getSignature();
-
-        // Load secret
-        byte[] secret = transactionData.getSecret();
-        if (secret != null) {
-            this.secret58 = Base58.encode(secret);
-        }
-
-        // Load data file(s)
-        ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
-        ArbitraryTransactionUtils.checkAndRelocateMiscFiles(transactionData);
-        arbitraryDataFile.setMetadataHash(metadataHash);
-
-        if (!arbitraryDataFile.allFilesExist()) {
-            if (ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName())) {
-                throw new DataException(
-                        String.format("Unable to request missing data for file %s because the name is blocked", arbitraryDataFile));
-            }
-            else {
-                // Ask the arbitrary data manager to fetch data for this transaction
-                String message;
-                if (this.canRequestMissingFiles) {
-                    boolean requested = ArbitraryDataManager.getInstance().fetchData(transactionData);
-
-                    if (requested) {
-                        message = String.format("Requested missing data for file %s", arbitraryDataFile);
-                    } else {
-                        message = String.format("Unable to reissue request for missing file %s for signature %s due to rate limit. Please try again later.", arbitraryDataFile, Base58.encode(transactionData.getSignature()));
-                    }
-                }
-                else {
-                    message = String.format("Missing data for file %s", arbitraryDataFile);
-                }
-
-                // Throw a missing data exception, which allows subsequent layers to fetch data
-                LOGGER.trace(message);
-                throw new MissingDataException(message);
-            }
-        }
-
-        if (arbitraryDataFile.allChunksExist() && !arbitraryDataFile.exists()) {
-            // We have all the chunks but not the complete file, so join them
-            arbitraryDataFile.join();
-        }
-
-        // If the complete file still doesn't exist then something went wrong
-        if (!arbitraryDataFile.exists()) {
-            throw new IOException(String.format("File doesn't exist: %s", arbitraryDataFile));
-        }
-        // Ensure the complete hash matches the joined chunks
-        if (!Arrays.equals(arbitraryDataFile.digest(), digest)) {
-            // Delete the invalid file
-            arbitraryDataFile.delete();
-            throw new DataException("Unable to validate complete file hash");
-        }
-        // Ensure the file's size matches the size reported by the transaction (throws a DataException if not)
-        arbitraryDataFile.validateFileSize(transactionData.getSize());
-
-        // Set filePath to the location of the ArbitraryDataFile
-        this.filePath = arbitraryDataFile.getFilePath();
-    }
-
-    private void decrypt() throws DataException {
-        try {
-            // First try with explicit parameters (CBC mode with PKCS5 padding)
-            this.decryptUsingAlgo("AES/CBC/PKCS5Padding");
-
-        } catch (DataException e) {
-            // Something went wrong, so fall back to default AES params (necessary for legacy resource support)
-            this.decryptUsingAlgo("AES");
-
-            // TODO: delete files and block this resource if privateDataEnabled is false and the second attempt fails too
-        }
-    }
-
-    private void decryptUsingAlgo(String algorithm) throws DataException {
-        // Decrypt if we have the secret key.
-        byte[] secret = this.secret58 != null ? Base58.decode(this.secret58) : null;
-        if (secret != null && secret.length == Transformer.AES256_LENGTH) {
-            try {
-                Path unencryptedPath = Paths.get(this.workingPath.toString(), "zipped.zip");
-                SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, algorithm);
-                AES.decryptFile(algorithm, aesKey, this.filePath.toString(), unencryptedPath.toString());
-
-                // Replace filePath pointer with the encrypted file path
-                // Don't delete the original ArbitraryDataFile, as this is handled in the cleanup phase
-                this.filePath = unencryptedPath;
-
-            } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException
-                    | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) {
-                throw new DataException(String.format("Unable to decrypt file at path %s: %s", this.filePath, e.getMessage()));
-            }
-        } else {
-            // Assume it is unencrypted. This will be the case when we have built a custom path by combining
-            // multiple decrypted archives into a single state.
-        }
-    }
-
-    private void uncompress() throws IOException, DataException {
-        if (this.filePath == null || !Files.exists(this.filePath)) {
-            throw new DataException("Can't uncompress non-existent file path");
-        }
-        File file = new File(this.filePath.toString());
-        if (file.isDirectory()) {
-            // Already a directory - nothing to uncompress
-            // We still need to copy the directory to its final destination if it's not already there
-            this.moveFilePathToFinalDestination();
-            return;
-        }
-
-        try {
-            // Default to ZIP compression - this is needed for previews
-            Compression compression = transactionData != null ? transactionData.getCompression() : Compression.ZIP;
-
-            // Handle each type of compression
-            if (compression == Compression.ZIP) {
-                ZipUtils.unzip(this.filePath.toString(), this.uncompressedPath.getParent().toString());
-            }
-            else if (compression == Compression.NONE) {
-                Files.createDirectories(this.uncompressedPath);
-                Path finalPath = Paths.get(this.uncompressedPath.toString(), "data");
-                this.filePath.toFile().renameTo(finalPath.toFile());
-            }
-            else {
-                throw new DataException(String.format("Unrecognized compression type: %s", transactionData.getCompression()));
-            }
-        } catch (IOException e) {
-            throw new DataException(String.format("Unable to unzip file: %s", e.getMessage()));
-        }
-
-        if (!this.uncompressedPath.toFile().exists()) {
-            throw new DataException(String.format("Unable to unzip file: %s", this.filePath));
-        }
-
-        // Delete original compressed file
-        if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
-            if (Files.exists(this.filePath)) {
-                Files.delete(this.filePath);
-            }
-        }
-
-        // Replace filePath pointer with the uncompressed file path
-        this.filePath = this.uncompressedPath;
-    }
-
-    private void validate() throws IOException, DataException {
-        if (this.service.isValidationRequired()) {
-            Service.ValidationResult result = this.service.validate(this.filePath);
-            if (result != Service.ValidationResult.OK) {
-                throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
-            }
-        }
-    }
-
-
-    private void moveFilePathToFinalDestination() throws IOException, DataException {
-        if (this.filePath.compareTo(this.uncompressedPath) != 0) {
-            File source = new File(this.filePath.toString());
-            File dest = new File(this.uncompressedPath.toString());
-            if (!source.exists()) {
-                throw new DataException("Source directory doesn't exist");
-            }
-            // Ensure destination directory doesn't exist
-            FileUtils.deleteDirectory(dest);
-            // Move files to destination
-            FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString());
-
-            try {
-                // Delete existing
-                if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
-                    File directory = new File(this.filePath.toString());
-                    FileUtils.deleteDirectory(directory);
-                }
-
-                // ... and its parent directory if empty
-                Path parentDirectory = this.filePath.getParent();
-                if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) {
-                    Files.deleteIfExists(parentDirectory);
-                }
-
-            } catch (DirectoryNotEmptyException e) {
-                    // No need to log anything
-            } catch (IOException e) {
-                // This will eventually be cleaned up by a maintenance process, so log the error and continue
-                LOGGER.debug("Unable to cleanup directories: {}", e.getMessage());
-            }
-
-            // Finally, update filePath to point to uncompressedPath
-            this.filePath = this.uncompressedPath;
-        }
-    }
-
-
-    public void setTransactionData(ArbitraryTransactionData transactionData) {
-        this.transactionData = transactionData;
-    }
-
-    public void setSecret58(String secret58) {
-        this.secret58 = secret58;
-    }
-
-    public Path getFilePath() {
-        return this.filePath;
-    }
-
-    public int getLayerCount() {
-        return this.layerCount;
-    }
-
-    public byte[] getLatestSignature() {
-        return this.latestSignature;
-    }
-
-    /**
-     * Use the below setter to ensure that we only read existing
-     * data without requesting any missing files,
-     *
-     * @param canRequestMissingFiles - whether or not fetching missing files is allowed
-     */
-    public void setCanRequestMissingFiles(boolean canRequestMissingFiles) {
-        this.canRequestMissingFiles = canRequestMissingFiles;
-    }
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java

@@ -1,213 +0,0 @@
-package org.qortal.arbitrary;
-
-import com.google.common.io.Resources;
-import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.api.HTMLParser;
-import org.qortal.arbitrary.ArbitraryDataFile.*;
-import org.qortal.arbitrary.exception.MissingDataException;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.controller.Controller;
-import org.qortal.settings.Settings;
-
-import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.URL;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.NoSuchFileException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
-public class ArbitraryDataRenderer {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataRenderer.class);
-
-    private final String resourceId;
-    private final ResourceIdType resourceIdType;
-    private final Service service;
-    private String inPath;
-    private final String secret58;
-    private final String prefix;
-    private final boolean usePrefix;
-    private final boolean async;
-    private final HttpServletRequest request;
-    private final HttpServletResponse response;
-    private final ServletContext context;
-
-    public ArbitraryDataRenderer(String resourceId, ResourceIdType resourceIdType, Service service, String inPath,
-                                 String secret58, String prefix, boolean usePrefix, boolean async,
-                                 HttpServletRequest request, HttpServletResponse response, ServletContext context) {
-
-        this.resourceId = resourceId;
-        this.resourceIdType = resourceIdType;
-        this.service = service;
-        this.inPath = inPath;
-        this.secret58 = secret58;
-        this.prefix = prefix;
-        this.usePrefix = usePrefix;
-        this.async = async;
-        this.request = request;
-        this.response = response;
-        this.context = context;
-    }
-
-    public HttpServletResponse render() {
-        if (!inPath.startsWith(File.separator)) {
-            inPath = File.separator + inPath;
-        }
-
-        // Don't render data if QDN is disabled
-        if (!Settings.getInstance().isQdnEnabled()) {
-            return ArbitraryDataRenderer.getResponse(response, 500, "QDN is disabled in settings");
-        }
-
-        ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, null);
-        arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
-        try {
-            if (!arbitraryDataReader.isCachedDataAvailable()) {
-                // If async is requested, show a loading screen whilst build is in progress
-                if (async) {
-                    arbitraryDataReader.loadAsynchronously(false, 10);
-                    return this.getLoadingResponse(service, resourceId);
-                }
-
-                // Otherwise, loop until we have data
-                int attempts = 0;
-                while (!Controller.isStopping()) {
-                    attempts++;
-                    if (!arbitraryDataReader.isBuilding()) {
-                        try {
-                            arbitraryDataReader.loadSynchronously(false);
-                            break;
-                        } catch (MissingDataException e) {
-                            if (attempts > 5) {
-                                // Give up after 5 attempts
-                                return ArbitraryDataRenderer.getResponse(response, 404, "Data unavailable. Please try again later.");
-                            }
-                        }
-                    }
-                    Thread.sleep(3000L);
-                }
-            }
-
-        } catch (Exception e) {
-            LOGGER.info(String.format("Unable to load %s %s: %s", service, resourceId, e.getMessage()));
-            return ArbitraryDataRenderer.getResponse(response, 500, "Error 500: Internal Server Error");
-        }
-
-        java.nio.file.Path path = arbitraryDataReader.getFilePath();
-        if (path == null) {
-            return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
-        }
-        String unzippedPath = path.toString();
-
-        try {
-            String filename = this.getFilename(unzippedPath, inPath);
-            String filePath = Paths.get(unzippedPath, filename).toString();
-
-            if (HTMLParser.isHtmlFile(filename)) {
-                // HTML file - needs to be parsed
-                byte[] data = Files.readAllBytes(Paths.get(filePath)); // TODO: limit file size that can be read into memory
-                HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix, data);
-                htmlParser.addAdditionalHeaderTags();
-                response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline'; media-src 'self' blob:");
-                response.setContentType(context.getMimeType(filename));
-                response.setContentLength(htmlParser.getData().length);
-                response.getOutputStream().write(htmlParser.getData());
-            }
-            else {
-                // Regular file - can be streamed directly
-                File file = new File(filePath);
-                FileInputStream inputStream = new FileInputStream(file);
-                response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline'; media-src 'self' blob:");
-                response.setContentType(context.getMimeType(filename));
-                int bytesRead, length = 0;
-                byte[] buffer = new byte[10240];
-                while ((bytesRead = inputStream.read(buffer)) != -1) {
-                    response.getOutputStream().write(buffer, 0, bytesRead);
-                    length += bytesRead;
-                }
-                response.setContentLength(length);
-                inputStream.close();
-            }
-            return response;
-        } catch (FileNotFoundException | NoSuchFileException e) {
-            LOGGER.info("Unable to serve file: {}", e.getMessage());
-            if (inPath.equals("/")) {
-                // Delete the unzipped folder if no index file was found
-                try {
-                    FileUtils.deleteDirectory(new File(unzippedPath));
-                } catch (IOException ioException) {
-                    LOGGER.debug("Unable to delete directory: {}", unzippedPath, e);
-                }
-            }
-        } catch (IOException e) {
-            LOGGER.info("Unable to serve file at path {}: {}", inPath, e.getMessage());
-        }
-
-        return ArbitraryDataRenderer.getResponse(response, 404, "Error 404: File Not Found");
-    }
-
-    private String getFilename(String directory, String userPath) {
-        if (userPath == null || userPath.endsWith("/") || userPath.equals("")) {
-            // Locate index file
-            List<String> indexFiles = ArbitraryDataRenderer.indexFiles();
-            for (String indexFile : indexFiles) {
-                Path path = Paths.get(directory, indexFile);
-                if (Files.exists(path)) {
-                    return userPath + indexFile;
-                }
-            }
-        }
-        return userPath;
-    }
-
-    private HttpServletResponse getLoadingResponse(Service service, String name) {
-        String responseString = "";
-        URL url = Resources.getResource("loading/index.html");
-        try {
-            responseString = Resources.toString(url, StandardCharsets.UTF_8);
-
-            // Replace vars
-            responseString = responseString.replace("%%SERVICE%%", service.toString());
-            responseString = responseString.replace("%%NAME%%", name);
-
-        } catch (IOException e) {
-            LOGGER.info("Unable to show loading screen: {}", e.getMessage());
-        }
-        return ArbitraryDataRenderer.getResponse(response, 503, responseString);
-    }
-
-    public static HttpServletResponse getResponse(HttpServletResponse response, int responseCode, String responseString) {
-        try {
-            byte[] responseData = responseString.getBytes();
-            response.setStatus(responseCode);
-            response.setContentLength(responseData.length);
-            response.getOutputStream().write(responseData);
-        } catch (IOException e) {
-            LOGGER.info("Error writing {} response", responseCode);
-        }
-        return response;
-    }
-
-    public static List<String> indexFiles() {
-        List<String> indexFiles = new ArrayList<>();
-        indexFiles.add("index.html");
-        indexFiles.add("index.htm");
-        indexFiles.add("default.html");
-        indexFiles.add("default.htm");
-        indexFiles.add("home.html");
-        indexFiles.add("home.htm");
-        return indexFiles;
-    }
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java

@@ -1,352 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
-import org.qortal.controller.arbitrary.ArbitraryDataManager;
-import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
-import org.qortal.data.arbitrary.ArbitraryResourceStatus;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.list.ResourceListManager;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.settings.Settings;
-import org.qortal.utils.ArbitraryTransactionUtils;
-import org.qortal.utils.FilesystemUtils;
-import org.qortal.utils.NTP;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.qortal.data.arbitrary.ArbitraryResourceStatus.Status;
-
-public class ArbitraryDataResource {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataResource.class);
-
-    protected final String resourceId;
-    protected final ResourceIdType resourceIdType;
-    protected final Service service;
-    protected final String identifier;
-
-    private List<ArbitraryTransactionData> transactions;
-    private ArbitraryTransactionData latestPutTransaction;
-    private int layerCount;
-    private Integer localChunkCount = null;
-    private Integer totalChunkCount = null;
-
-    public ArbitraryDataResource(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
-        this.resourceId = resourceId.toLowerCase();
-        this.resourceIdType = resourceIdType;
-        this.service = service;
-
-        // If identifier is a blank string, or reserved keyword "default", treat it as null
-        if (identifier == null || identifier.equals("") || identifier.equals("default")) {
-            identifier = null;
-        }
-        this.identifier = identifier;
-    }
-
-    public ArbitraryResourceStatus getStatus(boolean quick) {
-        // Calculate the chunk counts
-        // Avoid this for "quick" statuses, to speed things up
-        if (!quick) {
-            this.calculateChunkCounts();
-        }
-
-        if (resourceIdType != ResourceIdType.NAME) {
-            // We only support statuses for resources with a name
-            return new ArbitraryResourceStatus(Status.UNSUPPORTED, this.localChunkCount, this.totalChunkCount);
-        }
-
-        // Check if the name is blocked
-        if (ResourceListManager.getInstance()
-                .listContains("blockedNames", this.resourceId, false)) {
-            return new ArbitraryResourceStatus(Status.BLOCKED, this.localChunkCount, this.totalChunkCount);
-        }
-
-        // Check if a build has failed
-        ArbitraryDataBuildQueueItem queueItem =
-                new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier);
-        if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) {
-            return new ArbitraryResourceStatus(Status.BUILD_FAILED, this.localChunkCount, this.totalChunkCount);
-        }
-
-        // Firstly check the cache to see if it's already built
-        ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(
-                resourceId, resourceIdType, service, identifier);
-        if (arbitraryDataReader.isCachedDataAvailable()) {
-            return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount);
-        }
-
-        // Check if we have all data locally for this resource
-        if (!this.allFilesDownloaded()) {
-            if (this.isDownloading()) {
-                return new ArbitraryResourceStatus(Status.DOWNLOADING, this.localChunkCount, this.totalChunkCount);
-            }
-            else if (this.isDataPotentiallyAvailable()) {
-                return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount);
-            }
-            return new ArbitraryResourceStatus(Status.MISSING_DATA, this.localChunkCount, this.totalChunkCount);
-        }
-
-        // Check if there's a build in progress
-        if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
-            return new ArbitraryResourceStatus(Status.BUILDING, this.localChunkCount, this.totalChunkCount);
-        }
-
-        // We have all data locally
-        return new ArbitraryResourceStatus(Status.DOWNLOADED, this.localChunkCount, this.totalChunkCount);
-    }
-
-    public boolean delete() {
-        try {
-            this.fetchTransactions();
-
-            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
-
-            for (ArbitraryTransactionData transactionData : transactionDataList) {
-                byte[] hash = transactionData.getData();
-                byte[] metadataHash = transactionData.getMetadataHash();
-                byte[] signature = transactionData.getSignature();
-                ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
-                arbitraryDataFile.setMetadataHash(metadataHash);
-
-                // Delete any chunks or complete files from each transaction
-                arbitraryDataFile.deleteAll();
-            }
-
-            // Also delete cached data for the entire resource
-            this.deleteCache();
-
-            // Invalidate the hosted transactions cache as we have removed an item
-            ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache();
-
-            return true;
-
-        } catch (DataException | IOException e) {
-            return false;
-        }
-    }
-
-    public void deleteCache() throws IOException {
-        // Don't delete anything if there's a build in progress
-        ArbitraryDataBuildQueueItem queueItem =
-                new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier);
-        if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
-            return;
-        }
-
-        String baseDir = Settings.getInstance().getTempDataPath();
-        String identifier = this.identifier != null ?  this.identifier : "default";
-        Path cachePath = Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier);
-        if (cachePath.toFile().exists()) {
-            boolean success = FilesystemUtils.safeDeleteDirectory(cachePath, true);
-            if (success) {
-                LOGGER.info("Cleared cache for resource {}", this.toString());
-            }
-        }
-    }
-
-    private boolean allFilesDownloaded() {
-        // Use chunk counts to speed things up if we can
-        if (this.localChunkCount != null && this.totalChunkCount != null &&
-                this.localChunkCount >= this.totalChunkCount) {
-            return true;
-        }
-
-        try {
-            this.fetchTransactions();
-
-            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
-
-            for (ArbitraryTransactionData transactionData : transactionDataList) {
-                if (!ArbitraryTransactionUtils.completeFileExists(transactionData) ||
-                    !ArbitraryTransactionUtils.allChunksExist(transactionData)) {
-                    return false;
-                }
-            }
-            return true;
-
-        } catch (DataException e) {
-            return false;
-        }
-    }
-
-    private void calculateChunkCounts() {
-        try {
-            this.fetchTransactions();
-
-            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
-            int localChunkCount = 0;
-            int totalChunkCount = 0;
-
-            for (ArbitraryTransactionData transactionData : transactionDataList) {
-                localChunkCount += ArbitraryTransactionUtils.ourChunkCount(transactionData);
-                totalChunkCount += ArbitraryTransactionUtils.totalChunkCount(transactionData);
-            }
-
-            this.localChunkCount = localChunkCount;
-            this.totalChunkCount = totalChunkCount;
-
-        } catch (DataException e) {}
-    }
-
-    private boolean isRateLimited() {
-        try {
-            this.fetchTransactions();
-
-            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
-
-            for (ArbitraryTransactionData transactionData : transactionDataList) {
-                if (ArbitraryDataManager.getInstance().isSignatureRateLimited(transactionData.getSignature())) {
-                    return true;
-                }
-            }
-            return true;
-
-        } catch (DataException e) {
-            return false;
-        }
-    }
-
-    /**
-     * Best guess as to whether data might be available
-     * This is only used to give an indication to the user of progress
-     * @return - whether data might be available on the network
-     */
-    private boolean isDataPotentiallyAvailable() {
-        try {
-            this.fetchTransactions();
-            Long now = NTP.getTime();
-            if (now == null) {
-                return false;
-            }
-
-            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
-
-            for (ArbitraryTransactionData transactionData : transactionDataList) {
-                long lastRequestTime = ArbitraryDataManager.getInstance().lastRequestForSignature(transactionData.getSignature());
-                // If we haven't requested yet, or requested in the last 30 seconds, there's still a
-                // chance that data is on its way but hasn't arrived yet
-                if (lastRequestTime == 0 || now - lastRequestTime < 30 * 1000L) {
-                    return true;
-                }
-            }
-            return false;
-
-        } catch (DataException e) {
-            return false;
-        }
-    }
-
-
-    /**
-     * Best guess as to whether we are currently downloading a resource
-     * This is only used to give an indication to the user of progress
-     * @return - whether we are trying to download the resource
-     */
-    private boolean isDownloading() {
-        try {
-            this.fetchTransactions();
-            Long now = NTP.getTime();
-            if (now == null) {
-                return false;
-            }
-
-            List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
-
-            for (ArbitraryTransactionData transactionData : transactionDataList) {
-                long lastRequestTime = ArbitraryDataManager.getInstance().lastRequestForSignature(transactionData.getSignature());
-                // If were have requested data in the last 30 seconds, treat it as "downloading"
-                if (lastRequestTime > 0 && now - lastRequestTime < 30 * 1000L) {
-                    return true;
-                }
-            }
-
-            // FUTURE: we may want to check for file hashes (including the metadata file hash) in
-            // ArbitraryDataManager.arbitraryDataFileRequests and return true if one is found.
-
-            return false;
-
-        } catch (DataException e) {
-            return false;
-        }
-    }
-
-
-
-    private void fetchTransactions() throws DataException {
-        if (this.transactions != null && !this.transactions.isEmpty()) {
-            // Already fetched
-            return;
-        }
-
-        try (final Repository repository = RepositoryManager.getRepository()) {
-
-            // Get the most recent PUT
-            ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
-                    .getLatestTransaction(this.resourceId, this.service, ArbitraryTransactionData.Method.PUT, this.identifier);
-            if (latestPut == null) {
-                String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s",
-                        this.resourceId, this.service, this.identifierString());
-                throw new DataException(message);
-            }
-            this.latestPutTransaction = latestPut;
-
-            // Load all transactions since the latest PUT
-            List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository()
-                    .getArbitraryTransactions(this.resourceId, this.service, this.identifier, latestPut.getTimestamp());
-
-            this.transactions = transactionDataList;
-            this.layerCount = transactionDataList.size();
-        }
-    }
-
-    private String resourceIdString() {
-        return resourceId != null ? resourceId : "";
-    }
-
-    private String resourceIdTypeString() {
-        return resourceIdType != null ? resourceIdType.toString() : "";
-    }
-
-    private String serviceString() {
-        return service != null ? service.toString() : "";
-    }
-
-    private String identifierString() {
-        return identifier != null ? identifier : "";
-    }
-
-    @Override
-    public String toString() {
-        return String.format("%s %s %s", this.serviceString(), this.resourceIdString(), this.identifierString());
-    }
-
-
-    /**
-     * @return unique key used to identify this resource
-     */
-    public String getUniqueKey() {
-        return String.format("%s-%s-%s", this.service, this.resourceId, this.identifier).toLowerCase();
-    }
-
-    public String getResourceId() {
-        return this.resourceId;
-    }
-
-    public Service getService() {
-        return this.service;
-    }
-
-    public String getIdentifier() {
-        return this.identifier;
-    }
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java

@@ -1,285 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.exception.MissingDataException;
-import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
-import org.qortal.arbitrary.ArbitraryDataDiff.*;
-import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.block.BlockChain;
-import org.qortal.crypto.Crypto;
-import org.qortal.data.PaymentData;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.data.transaction.ArbitraryTransactionData.*;
-import org.qortal.data.transaction.BaseTransactionData;
-import org.qortal.group.Group;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.transaction.ArbitraryTransaction;
-import org.qortal.transaction.Transaction;
-import org.qortal.transform.Transformer;
-import org.qortal.utils.Base58;
-import org.qortal.utils.FilesystemUtils;
-import org.qortal.utils.NTP;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Random;
-
-public class ArbitraryDataTransactionBuilder {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataTransactionBuilder.class);
-
-    // Min transaction version required
-    private static final int MIN_TRANSACTION_VERSION = 5;
-
-    // Maximum number of PATCH layers allowed
-    private static final int MAX_LAYERS = 10;
-    // Maximum size difference (out of 1) allowed for PATCH transactions
-    private static final double MAX_SIZE_DIFF = 0.2f;
-    // Maximum proportion of files modified relative to total
-    private static final double MAX_FILE_DIFF = 0.5f;
-
-    private final String publicKey58;
-    private final Path path;
-    private final String name;
-    private Method method;
-    private final Service service;
-    private final String identifier;
-    private final Repository repository;
-
-    private int chunkSize = ArbitraryDataFile.CHUNK_SIZE;
-
-    private ArbitraryTransactionData arbitraryTransactionData;
-    private ArbitraryDataFile arbitraryDataFile;
-
-    public ArbitraryDataTransactionBuilder(Repository repository, String publicKey58, Path path, String name,
-                                           Method method, Service service, String identifier) {
-        this.repository = repository;
-        this.publicKey58 = publicKey58;
-        this.path = path;
-        this.name = name;
-        this.method = method;
-        this.service = service;
-
-        // If identifier is a blank string, or reserved keyword "default", treat it as null
-        if (identifier == null || identifier.equals("") || identifier.equals("default")) {
-            identifier = null;
-        }
-        this.identifier = identifier;
-    }
-
-    public void build() throws DataException {
-        try {
-            this.preExecute();
-            this.checkMethod();
-            this.createTransaction();
-        }
-        finally {
-            this.postExecute();
-        }
-    }
-
-    private void preExecute() {
-
-    }
-
-    private void postExecute() {
-
-    }
-
-    private void checkMethod() throws DataException {
-        if (this.method == null) {
-            // We need to automatically determine the method
-            this.method = this.determineMethodAutomatically();
-        }
-    }
-
-    private Method determineMethodAutomatically() throws DataException {
-        ArbitraryDataReader reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier);
-        try {
-            reader.loadSynchronously(true);
-        } catch (Exception e) {
-            // Catch all exceptions if the existing resource cannot be loaded first time
-            // In these cases it's simplest to just use a PUT transaction
-            return Method.PUT;
-        }
-
-        try {
-            // Check layer count
-            int layerCount = reader.getLayerCount();
-            if (layerCount >= MAX_LAYERS) {
-                LOGGER.info("Reached maximum layer count ({} / {}) - using PUT", layerCount, MAX_LAYERS);
-                return Method.PUT;
-            }
-
-            // Check size of differences between this layer and previous layer
-            ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(reader.getFilePath(), this.path, reader.getLatestSignature());
-            patch.create();
-            long diffSize = FilesystemUtils.getDirectorySize(patch.getFinalPath());
-            long existingStateSize = FilesystemUtils.getDirectorySize(reader.getFilePath());
-            double difference = (double) diffSize / (double) existingStateSize;
-            if (difference > MAX_SIZE_DIFF) {
-                LOGGER.info("Reached maximum difference ({} / {}) - using PUT", difference, MAX_SIZE_DIFF);
-                return Method.PUT;
-            }
-
-            // Check number of modified files
-            ArbitraryDataMetadataPatch metadata = patch.getMetadata();
-            int totalFileCount = patch.getTotalFileCount();
-            int differencesCount = metadata.getFileDifferencesCount();
-            difference = (double) differencesCount / (double) totalFileCount;
-            if (difference > MAX_FILE_DIFF) {
-                LOGGER.info("Reached maximum file differences ({} / {}) - using PUT", difference, MAX_FILE_DIFF);
-                return Method.PUT;
-            }
-
-            // Check the patch types
-            // Limit this check to single file resources only for now
-            boolean atLeastOnePatch = false;
-            if (totalFileCount == 1) {
-                for (ModifiedPath path : metadata.getModifiedPaths()) {
-                    if (path.getDiffType() != DiffType.COMPLETE_FILE) {
-                        atLeastOnePatch = true;
-                    }
-                }
-            }
-            if (!atLeastOnePatch) {
-                LOGGER.info("Patch consists of complete files only - using PUT");
-                return Method.PUT;
-            }
-
-            // State is appropriate for a PATCH transaction
-            return Method.PATCH;
-        }
-        catch (IOException | DataException e) {
-            // Handle matching states separately, as it's best to block transactions with duplicate states
-            if (e.getMessage().equals("Current state matches previous state. Nothing to do.")) {
-                throw new DataException(e.getMessage());
-            }
-            LOGGER.info("Caught exception: {}", e.getMessage());
-            LOGGER.info("Unable to load existing resource - using PUT to overwrite it.");
-            return Method.PUT;
-        }
-    }
-
-    private void createTransaction() throws DataException {
-        arbitraryDataFile = null;
-        try {
-            Long now = NTP.getTime();
-            if (now == null) {
-                throw new DataException("NTP time not synced yet");
-            }
-
-            // Ensure that this chain supports transactions necessary for complex arbitrary data
-            int transactionVersion = Transaction.getVersionByTimestamp(now);
-            if (transactionVersion < MIN_TRANSACTION_VERSION) {
-                throw new DataException("Transaction version unsupported on this blockchain.");
-            }
-
-            if (publicKey58 == null || path == null) {
-                throw new DataException("Missing public key or path");
-            }
-            byte[] creatorPublicKey = Base58.decode(publicKey58);
-            final String creatorAddress = Crypto.toAddress(creatorPublicKey);
-            byte[] lastReference = repository.getAccountRepository().getLastReference(creatorAddress);
-            if (lastReference == null) {
-                // Use a random last reference on the very first transaction for an account
-                // Code copied from CrossChainResource.buildAtMessage()
-                // We already require PoW on all arbitrary transactions, so no additional logic is needed
-                Random random = new Random();
-                lastReference = new byte[Transformer.SIGNATURE_LENGTH];
-                random.nextBytes(lastReference);
-            }
-
-            Compression compression = Compression.ZIP;
-
-            // FUTURE? Use zip compression for directories, or no compression for single files
-            // Compression compression = (path.toFile().isDirectory()) ? Compression.ZIP : Compression.NONE;
-
-            ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, compression);
-            try {
-                arbitraryDataWriter.setChunkSize(this.chunkSize);
-                arbitraryDataWriter.save();
-            } catch (IOException | DataException | InterruptedException | RuntimeException | MissingDataException e) {
-                LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage());
-                throw new DataException(e.getMessage());
-            }
-
-            // Get main file
-            arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile();
-            if (arbitraryDataFile == null) {
-                throw new DataException("Arbitrary data file is null");
-            }
-
-            // Get chunks metadata file
-            ArbitraryDataFile metadataFile = arbitraryDataFile.getMetadataFile();
-            if (metadataFile == null && arbitraryDataFile.chunkCount() > 1) {
-                throw new DataException(String.format("Chunks metadata data file is null but there are %d chunks", arbitraryDataFile.chunkCount()));
-            }
-
-            String digest58 = arbitraryDataFile.digest58();
-            if (digest58 == null) {
-                LOGGER.error("Unable to calculate file digest");
-                throw new DataException("Unable to calculate file digest");
-            }
-
-            final BaseTransactionData baseTransactionData = new BaseTransactionData(now, Group.NO_GROUP,
-                    lastReference, creatorPublicKey, 0L, null);
-            final int size = (int) arbitraryDataFile.size();
-            final int version = 5;
-            final int nonce = 0;
-            byte[] secret = arbitraryDataFile.getSecret();
-            final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
-            final byte[] digest = arbitraryDataFile.digest();
-            final byte[] metadataHash = (metadataFile != null) ? metadataFile.getHash() : null;
-            final List<PaymentData> payments = new ArrayList<>();
-
-            ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
-                    version, service, nonce, size, name, identifier, method,
-                    secret, compression, digest, dataType, metadataHash, payments);
-
-            this.arbitraryTransactionData = transactionData;
-
-        } catch (DataException e) {
-            if (arbitraryDataFile != null) {
-                arbitraryDataFile.deleteAll();
-            }
-            throw(e);
-        }
-
-    }
-
-    public void computeNonce() throws DataException {
-        if (this.arbitraryTransactionData == null) {
-            throw new DataException("Arbitrary transaction data is required to compute nonce");
-        }
-
-        ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, this.arbitraryTransactionData);
-        LOGGER.info("Computing nonce...");
-        transaction.computeNonce();
-
-        Transaction.ValidationResult result = transaction.isValidUnconfirmed();
-        if (result != Transaction.ValidationResult.OK) {
-            arbitraryDataFile.deleteAll();
-            throw new DataException(String.format("Arbitrary transaction invalid: %s", result));
-        }
-        LOGGER.info("Transaction is valid");
-    }
-
-    public ArbitraryTransactionData getArbitraryTransactionData() {
-        return this.arbitraryTransactionData;
-    }
-
-    public ArbitraryDataFile getArbitraryDataFile() {
-        return this.arbitraryDataFile;
-    }
-
-    public void setChunkSize(int chunkSize) {
-        this.chunkSize = chunkSize;
-    }
-
-}

src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java

@@ -1,342 +0,0 @@
-package org.qortal.arbitrary;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.exception.MissingDataException;
-import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.crypto.Crypto;
-import org.qortal.data.transaction.ArbitraryTransactionData.*;
-import org.qortal.crypto.AES;
-import org.qortal.repository.DataException;
-import org.qortal.arbitrary.ArbitraryDataFile.*;
-import org.qortal.settings.Settings;
-import org.qortal.utils.Base58;
-import org.qortal.utils.FilesystemUtils;
-import org.qortal.utils.ZipUtils;
-
-import javax.crypto.BadPaddingException;
-import javax.crypto.IllegalBlockSizeException;
-import javax.crypto.NoSuchPaddingException;
-import javax.crypto.SecretKey;
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.InvalidAlgorithmParameterException;
-import java.security.InvalidKeyException;
-import java.security.NoSuchAlgorithmException;
-
-public class ArbitraryDataWriter {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataWriter.class);
-
-    private Path filePath;
-    private final String name;
-    private final Service service;
-    private final String identifier;
-    private final Method method;
-    private final Compression compression;
-
-    private int chunkSize = ArbitraryDataFile.CHUNK_SIZE;
-
-    private SecretKey aesKey;
-    private ArbitraryDataFile arbitraryDataFile;
-
-    // Intermediate paths to cleanup
-    private Path workingPath;
-    private Path compressedPath;
-    private Path encryptedPath;
-
-    public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression) {
-        this.filePath = filePath;
-        this.name = name;
-        this.service = service;
-        this.method = method;
-        this.compression = compression;
-
-        // If identifier is a blank string, or reserved keyword "default", treat it as null
-        if (identifier == null || identifier.equals("") || identifier.equals("default")) {
-            identifier = null;
-        }
-        this.identifier = identifier;
-    }
-
-    public void save() throws IOException, DataException, InterruptedException, MissingDataException {
-        try {
-            this.preExecute();
-            this.validateService();
-            this.process();
-            this.compress();
-            this.encrypt();
-            this.split();
-            this.createMetadataFile();
-            this.validate();
-
-        } finally {
-            this.postExecute();
-        }
-    }
-
-    private void preExecute() throws DataException {
-        this.checkEnabled();
-
-        // Enforce compression when uploading a directory
-        File file = new File(this.filePath.toString());
-        if (file.isDirectory() && compression == Compression.NONE) {
-            throw new DataException("Unable to upload a directory without compression");
-        }
-
-        // Create temporary working directory
-        this.createWorkingDirectory();
-    }
-
-    private void postExecute() throws IOException {
-        this.cleanupFilesystem();
-    }
-
-    private void checkEnabled() throws DataException {
-        if (!Settings.getInstance().isQdnEnabled()) {
-            throw new DataException("QDN is disabled in settings");
-        }
-    }
-
-    private void createWorkingDirectory() throws DataException {
-        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
-        String baseDir = Settings.getInstance().getTempDataPath();
-        String identifier = Base58.encode(Crypto.digest(this.filePath.toString().getBytes()));
-        Path tempDir = Paths.get(baseDir, "writer", identifier);
-        try {
-            Files.createDirectories(tempDir);
-        } catch (IOException e) {
-            throw new DataException("Unable to create temp directory");
-        }
-        this.workingPath = tempDir;
-    }
-
-    private void validateService() throws IOException, DataException {
-        if (this.service.isValidationRequired()) {
-            Service.ValidationResult result = this.service.validate(this.filePath);
-            if (result != Service.ValidationResult.OK) {
-                throw new DataException(String.format("Validation of %s failed: %s", this.service, result.toString()));
-            }
-        }
-    }
-
-    private void process() throws DataException, IOException, MissingDataException {
-        switch (this.method) {
-
-            case PUT:
-                // Nothing to do
-                break;
-
-            case PATCH:
-                this.processPatch();
-                break;
-
-            default:
-                throw new DataException(String.format("Unknown method specified: %s", method.toString()));
-        }
-    }
-
-    private void processPatch() throws DataException, IOException, MissingDataException {
-
-        // Build the existing state using past transactions
-        ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service, this.identifier);
-        builder.build();
-        Path builtPath = builder.getFinalPath();
-
-        // Obtain the latest signature, so this can be included in the patch
-        byte[] latestSignature = builder.getLatestSignature();
-
-        // Compute a diff of the latest changes on top of the previous state
-        // Then use only the differences as our data payload
-        ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath, latestSignature);
-        patch.create();
-        this.filePath = patch.getFinalPath();
-
-        // Delete the input directory
-        if (FilesystemUtils.pathInsideDataOrTempPath(builtPath)) {
-            File directory = new File(builtPath.toString());
-            FileUtils.deleteDirectory(directory);
-        }
-
-        // Validate the patch
-        this.validatePatch();
-    }
-
-    private void validatePatch() throws DataException {
-        if (this.filePath == null) {
-            throw new DataException("Null path after creating patch");
-        }
-
-        File qortalMetadataDirectoryFile = Paths.get(this.filePath.toString(), ".qortal").toFile();
-        if (!qortalMetadataDirectoryFile.exists()) {
-            throw new DataException("Qortal metadata folder doesn't exist in patch");
-        }
-        if (!qortalMetadataDirectoryFile.isDirectory()) {
-            throw new DataException("Qortal metadata folder isn't a directory");
-        }
-
-        File qortalPatchMetadataFile = Paths.get(this.filePath.toString(), ".qortal", "patch").toFile();
-        if (!qortalPatchMetadataFile.exists()) {
-            throw new DataException("Qortal patch metadata file doesn't exist in patch");
-        }
-        if (!qortalPatchMetadataFile.isFile()) {
-            throw new DataException("Qortal patch metadata file isn't a file");
-        }
-    }
-
-    private void compress() throws InterruptedException, DataException {
-        // Compress the data if requested
-        if (this.compression != Compression.NONE) {
-            this.compressedPath = Paths.get(this.workingPath.toString(), "data.zip");
-            try {
-
-                if (this.compression == Compression.ZIP) {
-                    LOGGER.info("Compressing...");
-                    String enclosingFolderName = "data";
-                    ZipUtils.zip(this.filePath.toString(), this.compressedPath.toString(), enclosingFolderName);
-                }
-                else {
-                    throw new DataException(String.format("Unknown compression type specified: %s", compression.toString()));
-                }
-                // FUTURE: other compression types
-
-                // Delete the input directory
-                if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
-                    File directory = new File(this.filePath.toString());
-                    FileUtils.deleteDirectory(directory);
-                }
-                // Replace filePath pointer with the zipped file path
-                this.filePath = this.compressedPath;
-
-            } catch (IOException | DataException e) {
-                throw new DataException("Unable to zip directory", e);
-            }
-        }
-    }
-
-    private void encrypt() throws DataException {
-        this.encryptedPath = Paths.get(this.workingPath.toString(), "data.zip.encrypted");
-        try {
-            // Encrypt the file with AES
-            LOGGER.info("Encrypting...");
-            this.aesKey = AES.generateKey(256);
-            AES.encryptFile("AES/CBC/PKCS5Padding", this.aesKey, this.filePath.toString(), this.encryptedPath.toString());
-
-            // Delete the input file
-            if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
-                Files.delete(this.filePath);
-            }
-            // Replace filePath pointer with the encrypted file path
-            this.filePath = this.encryptedPath;
-
-        } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException
-                | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) {
-            throw new DataException(String.format("Unable to encrypt file %s: %s", this.filePath, e.getMessage()));
-        }
-    }
-
-    private void split() throws IOException, DataException {
-        // We don't have a signature yet, so use null to put the file in a generic folder
-        this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath, null);
-        if (this.arbitraryDataFile == null) {
-            throw new IOException("No file available when trying to split");
-        }
-
-        int chunkCount = this.arbitraryDataFile.split(this.chunkSize);
-        if (chunkCount > 0) {
-            LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s")));
-        }
-        else {
-            throw new DataException("Unable to split file into chunks");
-        }
-    }
-
-    private void createMetadataFile() throws IOException, DataException {
-        // If we have at least one chunk, we need to create an index file containing their hashes
-        if (this.arbitraryDataFile.chunkCount() > 1) {
-            // Create the JSON file
-            Path chunkFilePath = Paths.get(this.workingPath.toString(), "metadata.json");
-            ArbitraryDataTransactionMetadata chunkMetadata = new ArbitraryDataTransactionMetadata(chunkFilePath);
-            chunkMetadata.setChunks(this.arbitraryDataFile.chunkHashList());
-            chunkMetadata.write();
-
-            // Create an ArbitraryDataFile from the JSON file (we don't have a signature yet)
-            ArbitraryDataFile metadataFile = ArbitraryDataFile.fromPath(chunkFilePath, null);
-            this.arbitraryDataFile.setMetadataFile(metadataFile);
-        }
-    }
-
-    private void validate() throws IOException, DataException {
-        if (this.arbitraryDataFile == null) {
-            throw new DataException("No file available when validating");
-        }
-        this.arbitraryDataFile.setSecret(this.aesKey.getEncoded());
-
-        // Validate the file
-        ValidationResult validationResult = this.arbitraryDataFile.isValid();
-        if (validationResult != ValidationResult.OK) {
-            throw new DataException(String.format("File %s failed validation: %s", this.arbitraryDataFile, validationResult));
-        }
-        LOGGER.info("Whole file hash is valid: {}", this.arbitraryDataFile.digest58());
-
-        // Validate each chunk
-        for (ArbitraryDataFileChunk chunk : this.arbitraryDataFile.getChunks()) {
-            validationResult = chunk.isValid();
-            if (validationResult != ValidationResult.OK) {
-                throw new DataException(String.format("Chunk %s failed validation: %s", chunk, validationResult));
-            }
-        }
-        LOGGER.info("Chunk hashes are valid");
-
-        // Validate chunks metadata file
-        if (this.arbitraryDataFile.chunkCount() > 1) {
-            ArbitraryDataFile metadataFile = this.arbitraryDataFile.getMetadataFile();
-            if (metadataFile == null || !metadataFile.exists()) {
-                throw new DataException("No metadata file available, but there are multiple chunks");
-            }
-            // Read the file
-            ArbitraryDataTransactionMetadata metadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
-            metadata.read();
-            // Check all chunks exist
-            for (byte[] chunk : this.arbitraryDataFile.chunkHashList()) {
-                if (!metadata.containsChunk(chunk)) {
-                    throw new DataException(String.format("Missing chunk %s in metadata file", Base58.encode(chunk)));
-                }
-            }
-        }
-    }
-
-    private void cleanupFilesystem() throws IOException {
-        // Clean up
-        if (FilesystemUtils.pathInsideDataOrTempPath(this.compressedPath)) {
-            File zippedFile = new File(this.compressedPath.toString());
-            if (zippedFile.exists()) {
-                zippedFile.delete();
-            }
-        }
-        if (FilesystemUtils.pathInsideDataOrTempPath(this.encryptedPath)) {
-            File encryptedFile = new File(this.encryptedPath.toString());
-            if (encryptedFile.exists()) {
-                encryptedFile.delete();
-            }
-        }
-        if (FilesystemUtils.pathInsideDataOrTempPath(this.workingPath)) {
-            FileUtils.deleteDirectory(new File(this.workingPath.toString()));
-        }
-    }
-
-
-    public ArbitraryDataFile getArbitraryDataFile() {
-        return this.arbitraryDataFile;
-    }
-
-    public void setChunkSize(int chunkSize) {
-        this.chunkSize = chunkSize;
-    }
-
-}

src/main/java/org/qortal/arbitrary/exception/MissingDataException.java

@@ -1,20 +0,0 @@
-package org.qortal.arbitrary.exception;
-
-public class MissingDataException extends Exception {
-
-	public MissingDataException() {
-	}
-
-	public MissingDataException(String message) {
-		super(message);
-	}
-
-	public MissingDataException(String message, Throwable cause) {
-		super(message, cause);
-	}
-
-	public MissingDataException(Throwable cause) {
-		super(cause);
-	}
-
-}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadata.java

@@ -1,85 +0,0 @@
-package org.qortal.arbitrary.metadata;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.repository.DataException;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-
-/**
- * ArbitraryDataMetadata
- *
- * This is a base class to handle reading and writing JSON to the supplied filePath.
- *
- * It is not usable on its own; it must be subclassed, with two methods overridden:
- *
- * readJson() - code to unserialize the JSON file
- * buildJson() - code to serialize the JSON file
- *
- */
-public class ArbitraryDataMetadata {
-
-    protected static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadata.class);
-
-    protected Path filePath;
-
-    protected String jsonString;
-
-    public ArbitraryDataMetadata(Path filePath) {
-        this.filePath = filePath;
-    }
-
-    protected void readJson() throws DataException {
-        // To be overridden
-    }
-
-    protected void buildJson() {
-        // To be overridden
-    }
-
-
-    public void read() throws IOException, DataException {
-        this.loadJson();
-        this.readJson();
-    }
-
-    public void write() throws IOException, DataException {
-        this.buildJson();
-        this.createParentDirectories();
-
-        BufferedWriter writer = new BufferedWriter(new FileWriter(this.filePath.toString()));
-        writer.write(this.jsonString);
-        writer.newLine();
-        writer.close();
-    }
-
-
-    protected void loadJson() throws IOException {
-        File metadataFile = new File(this.filePath.toString());
-        if (!metadataFile.exists()) {
-            throw new IOException(String.format("Metadata file doesn't exist: %s", this.filePath.toString()));
-        }
-
-        this.jsonString = new String(Files.readAllBytes(this.filePath));
-    }
-
-
-    protected void createParentDirectories() throws DataException {
-        try {
-            Files.createDirectories(this.filePath.getParent());
-        } catch (IOException e) {
-            throw new DataException("Unable to create parent directories");
-        }
-    }
-
-
-    public String getJsonString() {
-        return this.jsonString;
-    }
-
-}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java

@@ -1,69 +0,0 @@
-package org.qortal.arbitrary.metadata;
-
-import org.json.JSONObject;
-import org.qortal.repository.DataException;
-import org.qortal.utils.Base58;
-
-import java.nio.file.Path;
-
-public class ArbitraryDataMetadataCache extends ArbitraryDataQortalMetadata {
-
-    private byte[] signature;
-    private long timestamp;
-
-    public ArbitraryDataMetadataCache(Path filePath) {
-        super(filePath);
-
-    }
-
-    @Override
-    protected String fileName() {
-        return "cache";
-    }
-
-    @Override
-    protected void readJson() throws DataException {
-        if (this.jsonString == null) {
-            throw new DataException("Patch JSON string is null");
-        }
-
-        JSONObject cache = new JSONObject(this.jsonString);
-        if (cache.has("signature")) {
-            String sig = cache.getString("signature");
-            if (sig != null) {
-                this.signature = Base58.decode(sig);
-            }
-        }
-        if (cache.has("timestamp")) {
-            this.timestamp = cache.getLong("timestamp");
-        }
-    }
-
-    @Override
-    protected void buildJson() {
-        JSONObject patch = new JSONObject();
-        patch.put("signature", Base58.encode(this.signature));
-        patch.put("timestamp", this.timestamp);
-
-        this.jsonString = patch.toString(2);
-        LOGGER.trace("Cache metadata: {}", this.jsonString);
-    }
-
-
-    public void setSignature(byte[] signature) {
-        this.signature = signature;
-    }
-
-    public byte[] getSignature() {
-        return this.signature;
-    }
-
-    public void setTimestamp(long timestamp) {
-        this.timestamp = timestamp;
-    }
-
-    public long getTimestamp() {
-        return this.timestamp;
-    }
-
-}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java

@@ -1,182 +0,0 @@
-package org.qortal.arbitrary.metadata;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.json.JSONArray;
-import org.json.JSONObject;
-import org.qortal.arbitrary.ArbitraryDataDiff.*;
-import org.qortal.repository.DataException;
-import org.qortal.utils.Base58;
-
-import java.lang.reflect.Field;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-
-public class ArbitraryDataMetadataPatch extends ArbitraryDataQortalMetadata {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadataPatch.class);
-
-    private List<Path> addedPaths;
-    private List<ModifiedPath> modifiedPaths;
-    private List<Path> removedPaths;
-    private byte[] previousSignature;
-    private byte[] previousHash;
-    private byte[] currentHash;
-
-    public ArbitraryDataMetadataPatch(Path filePath) {
-        super(filePath);
-
-        this.addedPaths = new ArrayList<>();
-        this.modifiedPaths = new ArrayList<>();
-        this.removedPaths = new ArrayList<>();
-    }
-
-    @Override
-    protected String fileName() {
-        return "patch";
-    }
-
-    @Override
-    protected void readJson() throws DataException {
-        if (this.jsonString == null) {
-            throw new DataException("Patch JSON string is null");
-        }
-
-        JSONObject patch = new JSONObject(this.jsonString);
-        if (patch.has("prevSig")) {
-            String prevSig = patch.getString("prevSig");
-            if (prevSig != null) {
-                this.previousSignature = Base58.decode(prevSig);
-            }
-        }
-        if (patch.has("prevHash")) {
-            String prevHash = patch.getString("prevHash");
-            if (prevHash != null) {
-                this.previousHash = Base58.decode(prevHash);
-            }
-        }
-        if (patch.has("curHash")) {
-            String curHash = patch.getString("curHash");
-            if (curHash != null) {
-                this.currentHash = Base58.decode(curHash);
-            }
-        }
-        if (patch.has("added")) {
-            JSONArray added = (JSONArray) patch.get("added");
-            if (added != null) {
-                for (int i=0; i<added.length(); i++) {
-                    String pathString = added.getString(i);
-                    this.addedPaths.add(Paths.get(pathString));
-                }
-            }
-        }
-        if (patch.has("modified")) {
-            JSONArray modified = (JSONArray) patch.get("modified");
-            if (modified != null) {
-                for (int i=0; i<modified.length(); i++) {
-                    JSONObject jsonObject = modified.getJSONObject(i);
-                    ModifiedPath modifiedPath = new ModifiedPath(jsonObject);
-                    this.modifiedPaths.add(modifiedPath);
-                }
-            }
-        }
-        if (patch.has("removed")) {
-            JSONArray removed = (JSONArray) patch.get("removed");
-            if (removed != null) {
-                for (int i=0; i<removed.length(); i++) {
-                    String pathString = removed.getString(i);
-                    this.removedPaths.add(Paths.get(pathString));
-                }
-            }
-        }
-    }
-
-    @Override
-    protected void buildJson() {
-        JSONObject patch = new JSONObject();
-        // Attempt to use a LinkedHashMap so that the order of fields is maintained
-        try {
-            Field changeMap = patch.getClass().getDeclaredField("map");
-            changeMap.setAccessible(true);
-            changeMap.set(patch, new LinkedHashMap<>());
-            changeMap.setAccessible(false);
-        } catch (IllegalAccessException | NoSuchFieldException e) {
-            // Don't worry about failures as this is for optional ordering only
-        }
-
-        patch.put("prevSig", Base58.encode(this.previousSignature));
-        patch.put("prevHash", Base58.encode(this.previousHash));
-        patch.put("curHash", Base58.encode(this.currentHash));
-        patch.put("added", new JSONArray(this.addedPaths));
-        patch.put("removed", new JSONArray(this.removedPaths));
-
-        JSONArray modifiedPaths = new JSONArray();
-        for (ModifiedPath modifiedPath : this.modifiedPaths) {
-            JSONObject modifiedPathJson = new JSONObject();
-            modifiedPathJson.put("path", modifiedPath.getPath());
-            modifiedPathJson.put("type", modifiedPath.getDiffType());
-            modifiedPaths.put(modifiedPathJson);
-        }
-        patch.put("modified", modifiedPaths);
-
-        this.jsonString = patch.toString(2);
-        LOGGER.debug("Patch metadata: {}", this.jsonString);
-    }
-
-    public void setAddedPaths(List<Path> addedPaths) {
-        this.addedPaths = addedPaths;
-    }
-
-    public List<Path> getAddedPaths() {
-        return this.addedPaths;
-    }
-
-    public void setModifiedPaths(List<ModifiedPath> modifiedPaths) {
-        this.modifiedPaths = modifiedPaths;
-    }
-
-    public List<ModifiedPath> getModifiedPaths() {
-        return this.modifiedPaths;
-    }
-
-    public void setRemovedPaths(List<Path> removedPaths) {
-        this.removedPaths = removedPaths;
-    }
-
-    public List<Path> getRemovedPaths() {
-        return this.removedPaths;
-    }
-
-    public void setPreviousSignature(byte[] previousSignature) {
-        this.previousSignature = previousSignature;
-    }
-
-    public byte[] getPreviousSignature() {
-        return this.previousSignature;
-    }
-
-    public void setPreviousHash(byte[] previousHash) {
-        this.previousHash = previousHash;
-    }
-
-    public byte[] getPreviousHash() {
-        return this.previousHash;
-    }
-
-    public void setCurrentHash(byte[] currentHash) {
-        this.currentHash = currentHash;
-    }
-
-    public byte[] getCurrentHash() {
-        return this.currentHash;
-    }
-
-
-    public int getFileDifferencesCount() {
-        return this.addedPaths.size() + this.modifiedPaths.size() + this.removedPaths.size();
-    }
-
-}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataQortalMetadata.java

@@ -1,102 +0,0 @@
-package org.qortal.arbitrary.metadata;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.repository.DataException;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * ArbitraryDataQortalMetadata
- *
- * This is a base class to handle reading and writing JSON to a .qortal folder
- * within the supplied filePath. This is used when storing data against an existing
- * arbitrary data file structure.
- *
- * It is not usable on its own; it must be subclassed, with three methods overridden:
- *
- * fileName() - the file name to use within the .qortal folder
- * readJson() - code to unserialize the JSON file
- * buildJson() - code to serialize the JSON file
- *
- */
-public class ArbitraryDataQortalMetadata extends ArbitraryDataMetadata {
-
-    protected static final Logger LOGGER = LogManager.getLogger(ArbitraryDataQortalMetadata.class);
-
-    protected Path filePath;
-    protected Path qortalDirectoryPath;
-
-    protected String jsonString;
-
-    public ArbitraryDataQortalMetadata(Path filePath) {
-        super(filePath);
-
-        this.qortalDirectoryPath = Paths.get(filePath.toString(), ".qortal");
-    }
-
-    protected String fileName() {
-        // To be overridden
-        return null;
-    }
-
-    protected void readJson() throws DataException {
-        // To be overridden
-    }
-
-    protected void buildJson() {
-        // To be overridden
-    }
-
-
-    @Override
-    public void read() throws IOException, DataException {
-        this.loadJson();
-        this.readJson();
-    }
-
-    @Override
-    public void write() throws IOException, DataException {
-        this.buildJson();
-        this.createParentDirectories();
-        this.createQortalDirectory();
-
-        Path patchPath = Paths.get(this.qortalDirectoryPath.toString(), this.fileName());
-        BufferedWriter writer = new BufferedWriter(new FileWriter(patchPath.toString()));
-        writer.write(this.jsonString);
-        writer.newLine();
-        writer.close();
-    }
-
-    @Override
-    protected void loadJson() throws IOException {
-        Path path = Paths.get(this.qortalDirectoryPath.toString(), this.fileName());
-        File patchFile = new File(path.toString());
-        if (!patchFile.exists()) {
-            throw new IOException(String.format("Patch file doesn't exist: %s", path.toString()));
-        }
-
-        this.jsonString = new String(Files.readAllBytes(path));
-    }
-
-
-    protected void createQortalDirectory() throws DataException {
-        try {
-            Files.createDirectories(this.qortalDirectoryPath);
-        } catch (IOException e) {
-            throw new DataException("Unable to create .qortal directory");
-        }
-    }
-
-
-    public String getJsonString() {
-        return this.jsonString;
-    }
-
-}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java

@@ -1,78 +0,0 @@
-package org.qortal.arbitrary.metadata;
-
-import org.json.JSONArray;
-import org.json.JSONObject;
-import org.qortal.repository.DataException;
-import org.qortal.utils.Base58;
-
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
-
-    private List<byte[]> chunks;
-
-    public ArbitraryDataTransactionMetadata(Path filePath) {
-        super(filePath);
-
-    }
-
-    @Override
-    protected void readJson() throws DataException {
-        if (this.jsonString == null) {
-            throw new DataException("Transaction metadata JSON string is null");
-        }
-
-        List<byte[]> chunksList = new ArrayList<>();
-        JSONObject cache = new JSONObject(this.jsonString);
-        if (cache.has("chunks")) {
-            JSONArray chunks = cache.getJSONArray("chunks");
-            if (chunks != null) {
-                for (int i=0; i<chunks.length(); i++) {
-                    String chunk = chunks.getString(i);
-                    if (chunk != null) {
-                        chunksList.add(Base58.decode(chunk));
-                    }
-                }
-            }
-            this.chunks = chunksList;
-        }
-    }
-
-    @Override
-    protected void buildJson() {
-        JSONObject outer = new JSONObject();
-
-        JSONArray chunks = new JSONArray();
-        if (this.chunks != null) {
-            for (byte[] chunk : this.chunks) {
-                chunks.put(Base58.encode(chunk));
-            }
-        }
-        outer.put("chunks", chunks);
-
-        this.jsonString = outer.toString(2);
-        LOGGER.trace("Transaction metadata: {}", this.jsonString);
-    }
-
-
-    public void setChunks(List<byte[]> chunks) {
-        this.chunks = chunks;
-    }
-
-    public List<byte[]> getChunks() {
-        return this.chunks;
-    }
-
-    public boolean containsChunk(byte[] chunk) {
-        for (byte[] c : this.chunks) {
-            if (Arrays.equals(c, chunk)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-}

src/main/java/org/qortal/arbitrary/misc/Service.java

@@ -1,131 +0,0 @@
-package org.qortal.arbitrary.misc;
-
-import org.json.JSONObject;
-import org.qortal.arbitrary.ArbitraryDataRenderer;
-import org.qortal.transaction.Transaction;
-import org.qortal.utils.FilesystemUtils;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static java.util.Arrays.stream;
-import static java.util.stream.Collectors.toMap;
-
-public enum Service {
-    AUTO_UPDATE(1, false, null, null),
-    ARBITRARY_DATA(100, false, null, null),
-    WEBSITE(200, true, null, null) {
-        @Override
-        public ValidationResult validate(Path path) {
-            // Custom validation function to require an index HTML file in the root directory
-            List<String> fileNames = ArbitraryDataRenderer.indexFiles();
-            String[] files = path.toFile().list();
-            if (files != null) {
-                for (String file : files) {
-                    Path fileName = Paths.get(file).getFileName();
-                    if (fileName != null && fileNames.contains(fileName.toString())) {
-                        return ValidationResult.OK;
-                    }
-                }
-            }
-            return ValidationResult.MISSING_INDEX_FILE;
-        }
-    },
-    GIT_REPOSITORY(300, false, null, null),
-    IMAGE(400, true, 10*1024*1024L, null),
-    THUMBNAIL(410, true, 500*1024L, null),
-    VIDEO(500, false, null, null),
-    AUDIO(600, false, null, null),
-    BLOG(700, false, null, null),
-    BLOG_POST(777, false, null, null),
-    BLOG_COMMENT(778, false, null, null),
-    DOCUMENT(800, false, null, null),
-    LIST(900, true, null, null),
-    PLAYLIST(910, true, null, null),
-    APP(1000, false, null, null),
-    METADATA(1100, false, null, null),
-    QORTAL_METADATA(1111, true, 10*1024L, Arrays.asList("title", "description", "tags"));
-
-    public final int value;
-    private final boolean requiresValidation;
-    private final Long maxSize;
-    private final List<String> requiredKeys;
-
-    private static final Map<Integer, Service> map = stream(Service.values())
-            .collect(toMap(service -> service.value, service -> service));
-
-    Service(int value, boolean requiresValidation, Long maxSize, List<String> requiredKeys) {
-        this.value = value;
-        this.requiresValidation = requiresValidation;
-        this.maxSize = maxSize;
-        this.requiredKeys = requiredKeys;
-    }
-
-    public ValidationResult validate(Path path) throws IOException {
-        if (!this.isValidationRequired()) {
-            return ValidationResult.OK;
-        }
-
-        byte[] data = FilesystemUtils.getSingleFileContents(path);
-        long size = FilesystemUtils.getDirectorySize(path);
-
-        // Validate max size if needed
-        if (this.maxSize != null) {
-            if (size > this.maxSize) {
-                return ValidationResult.EXCEEDS_SIZE_LIMIT;
-            }
-        }
-
-        // Validate required keys if needed
-        if (this.requiredKeys != null) {
-            if (data == null) {
-                return ValidationResult.MISSING_KEYS;
-            }
-            JSONObject json = Service.toJsonObject(data);
-            for (String key : this.requiredKeys) {
-                if (!json.has(key)) {
-                    return ValidationResult.MISSING_KEYS;
-                }
-            }
-        }
-
-        // Validation passed
-        return ValidationResult.OK;
-    }
-
-    public boolean isValidationRequired() {
-        return this.requiresValidation;
-    }
-
-    public static Service valueOf(int value) {
-        return map.get(value);
-    }
-
-    public static JSONObject toJsonObject(byte[] data) {
-        String dataString = new String(data);
-        return new JSONObject(dataString);
-    }
-
-    public enum ValidationResult {
-        OK(1),
-        MISSING_KEYS(2),
-        EXCEEDS_SIZE_LIMIT(3),
-        MISSING_INDEX_FILE(4);
-
-        public final int value;
-
-        private static final Map<Integer, Transaction.ValidationResult> map = stream(Transaction.ValidationResult.values()).collect(toMap(result -> result.value, result -> result));
-
-        ValidationResult(int value) {
-            this.value = value;
-        }
-
-        public static Transaction.ValidationResult valueOf(int value) {
-            return map.get(value);
-        }
-    }
-}

src/main/java/org/qortal/arbitrary/patch/UnifiedDiffPatch.java

@@ -1,229 +0,0 @@
-package org.qortal.arbitrary.patch;
-
-import com.github.difflib.DiffUtils;
-import com.github.difflib.UnifiedDiffUtils;
-import com.github.difflib.patch.Patch;
-import com.github.difflib.patch.PatchFailedException;
-import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.crypto.Crypto;
-import org.qortal.repository.DataException;
-import org.qortal.settings.Settings;
-import org.qortal.utils.FilesystemUtils;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
-
-public class UnifiedDiffPatch {
-
-    private static final Logger LOGGER = LogManager.getLogger(UnifiedDiffPatch.class);
-
-    private final Path before;
-    private final Path after;
-    private final Path destination;
-
-    private String identifier;
-    private Path validationPath;
-
-    public UnifiedDiffPatch(Path before, Path after, Path destination) {
-        this.before = before;
-        this.after = after;
-        this.destination = destination;
-    }
-
-    /**
-     * Create a patch based on the differences in path "after"
-     * compared with base path "before", outputting the patch
-     * to the "destination" path.
-     *
-     * @throws IOException
-     */
-    public void create() throws IOException {
-        if (!Files.exists(before)) {
-            throw new IOException(String.format("File not found (before): %s", before.toString()));
-        }
-        if (!Files.exists(after)) {
-            throw new IOException(String.format("File not found (after): %s", after.toString()));
-        }
-
-        // Ensure parent folders exist in the destination
-        File file = new File(destination.toString());
-        File parent = file.getParentFile();
-        if (parent != null) {
-            parent.mkdirs();
-        }
-
-        // Delete an existing file if it exists
-        File destFile = destination.toFile();
-        if (destFile.exists() && destFile.isFile()) {
-            Files.delete(destination);
-        }
-
-        // Load the two files into memory
-        List<String> original = FileUtils.readLines(before.toFile(), StandardCharsets.UTF_8);
-        List<String> revised = FileUtils.readLines(after.toFile(), StandardCharsets.UTF_8);
-
-        // Check if the original file ends with a newline
-        boolean endsWithNewline = FilesystemUtils.fileEndsWithNewline(before);
-
-        // Generate diff information
-        Patch<String> diff = DiffUtils.diff(original, revised);
-
-        // Generate unified diff format
-        String originalFileName = before.getFileName().toString();
-        String revisedFileName = after.getFileName().toString();
-        List<String> unifiedDiff = UnifiedDiffUtils.generateUnifiedDiff(originalFileName, revisedFileName, original, diff, 0);
-
-        // Write the diff to the destination directory
-        FileWriter fileWriter = new FileWriter(destination.toString(), true);
-        BufferedWriter writer = new BufferedWriter(fileWriter);
-        for (int i=0; i<unifiedDiff.size(); i++) {
-            String line = unifiedDiff.get(i);
-            writer.append(line);
-            // Add a newline if this isn't the last line, or the original ended with a newline
-            if (i < unifiedDiff.size()-1 || endsWithNewline) {
-                writer.newLine();
-            }
-        }
-        writer.flush();
-        writer.close();
-    }
-
-    /**
-     * Validate the patch at the "destination" path to ensure
-     * it works correctly and is smaller than the original file
-     *
-     * @return true if valid, false if invalid
-     */
-    public boolean isValid() throws DataException {
-        this.createRandomIdentifier();
-        this.createTempValidationDirectory();
-
-        // Merge the patch with the original path
-        Path tempPath = Paths.get(this.validationPath.toString(), this.identifier);
-
-        try {
-            UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(before, destination, tempPath);
-            unifiedDiffPatch.apply(null);
-
-            byte[] inputDigest = Crypto.digest(after.toFile());
-            byte[] outputDigest = Crypto.digest(tempPath.toFile());
-            if (Arrays.equals(inputDigest, outputDigest)) {
-                // Patch is valid, but we might want to reject if it's larger than the original file
-                long originalSize = Files.size(after);
-                long patchSize = Files.size(destination);
-                if (patchSize < originalSize) {
-                    // Patch file is smaller than the original file size, so treat it as valid
-                    return true;
-                }
-            }
-            else {
-                LOGGER.info("Checksum mismatch when verifying patch for file {}", destination.toString());
-                return false;
-            }
-
-        }
-        catch (IOException e) {
-            LOGGER.info("Failed to compute merge for file {}: {}", destination.toString(), e.getMessage());
-        }
-        finally {
-            try {
-                Files.delete(tempPath);
-            } catch (IOException e) {
-                // Not important - will be cleaned up later
-            }
-        }
-
-        return false;
-    }
-
-    /**
-     * Apply a patch at path "after" on top of base path "before",
-     * outputting the combined results to the "destination" path.
-     * If before and after are directories, a relative path suffix
-     * can be used to specify the file within these folder structures.
-     *
-     * @param pathSuffix - a file path to append to the base paths, or null if the base paths are already files
-     * @throws IOException
-     */
-    public void apply(Path pathSuffix) throws IOException, DataException {
-        Path originalPath = this.before;
-        Path patchPath = this.after;
-        Path mergePath = this.destination;
-
-        // If a path has been supplied, we need to append it to the base paths
-        if (pathSuffix != null) {
-            originalPath = Paths.get(this.before.toString(), pathSuffix.toString());
-            patchPath = Paths.get(this.after.toString(), pathSuffix.toString());
-            mergePath = Paths.get(this.destination.toString(), pathSuffix.toString());
-        }
-
-        if (!patchPath.toFile().exists()) {
-            throw new DataException("Patch file doesn't exist, but its path was included in modifiedPaths");
-        }
-
-        // Delete an existing file, as we are starting from a duplicate of pathBefore
-        File destFile = mergePath.toFile();
-        if (destFile.exists() && destFile.isFile()) {
-            Files.delete(mergePath);
-        }
-
-        List<String> originalContents = FileUtils.readLines(originalPath.toFile(), StandardCharsets.UTF_8);
-        List<String> patchContents = FileUtils.readLines(patchPath.toFile(), StandardCharsets.UTF_8);
-
-        // Check if the patch file (and therefore the original file) ends with a newline
-        boolean endsWithNewline = FilesystemUtils.fileEndsWithNewline(patchPath);
-
-        // At first, parse the unified diff file and get the patch
-        Patch<String> patch = UnifiedDiffUtils.parseUnifiedDiff(patchContents);
-
-        // Then apply the computed patch to the given text
-        try {
-            List<String> patchedContents = DiffUtils.patch(originalContents, patch);
-
-            // Write the patched file to the merge directory
-            FileWriter fileWriter = new FileWriter(mergePath.toString(), true);
-            BufferedWriter writer = new BufferedWriter(fileWriter);
-            for (int i=0; i<patchedContents.size(); i++) {
-                String line = patchedContents.get(i);
-                writer.append(line);
-                // Add a newline if this isn't the last line, or the original ended with a newline
-                if (i < patchedContents.size()-1 || endsWithNewline) {
-                    writer.newLine();
-                }
-            }
-            writer.flush();
-            writer.close();
-
-        } catch (PatchFailedException e) {
-            throw new DataException(String.format("Failed to apply patch for path %s: %s", pathSuffix, e.getMessage()));
-        }
-    }
-
-    private void createRandomIdentifier() {
-        this.identifier = UUID.randomUUID().toString();
-    }
-
-    private void createTempValidationDirectory() throws DataException {
-        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
-        String baseDir = Settings.getInstance().getTempDataPath();
-        Path tempDir = Paths.get(baseDir, "diff", "validate");
-        try {
-            Files.createDirectories(tempDir);
-        } catch (IOException e) {
-            throw new DataException("Unable to create temp directory");
-        }
-        this.validationPath = tempDir;
-    }
-
-}

src/main/java/org/qortal/at/AT.java

@@ -1,7 +1,5 @@
 package org.qortal.at;
 
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 
 import org.ciyam.at.MachineState;
@@ -58,12 +56,12 @@ public class AT {
 
 		this.atData = new ATData(atAddress, creatorPublicKey, creation, machineState.version, assetId, codeBytes, codeHash,
 				machineState.isSleeping(), machineState.getSleepUntilHeight(), machineState.isFinished(), machineState.hadFatalError(),
-				machineState.isFrozen(), machineState.getFrozenBalance(), null);
+				machineState.isFrozen(), machineState.getFrozenBalance());
 
 		byte[] stateData = machineState.toBytes();
 		byte[] stateHash = Crypto.digest(stateData);
 
-		this.atStateData = new ATStateData(atAddress, height, stateData, stateHash, 0L, true, null);
+		this.atStateData = new ATStateData(atAddress, height, stateData, stateHash, 0L, true);
 	}
 
 	// Getters / setters
@@ -86,28 +84,13 @@ public class AT {
 		this.repository.getATRepository().delete(this.atData.getATAddress());
 	}
 
-	/**
-	 * Potentially execute AT.
-	 * <p>
-	 * Note that sleep-until-message support might set/reset
-	 * sleep-related flags/values.
-	 * <p>
-	 * {@link #getATStateData()} will return null if nothing happened.
-	 * <p>
-	 * @param blockHeight
-	 * @param blockTimestamp
-	 * @return AT-generated transactions, possibly empty
-	 * @throws DataException
-	 */
 	public List<AtTransaction> run(int blockHeight, long blockTimestamp) throws DataException {
 		String atAddress = this.atData.getATAddress();
 
 		QortalATAPI api = new QortalATAPI(repository, this.atData, blockTimestamp);
 		QortalAtLoggerFactory loggerFactory = QortalAtLoggerFactory.getInstance();
 
-		if (!api.willExecute(blockHeight))
-			// this.atStateData will be null
-			return Collections.emptyList();
+		byte[] codeBytes = this.atData.getCodeBytes();
 
 		// Fetch latest ATStateData for this AT
 		ATStateData latestAtStateData = this.repository.getATRepository().getLatestATState(atAddress);
@@ -117,10 +100,8 @@ public class AT {
 			throw new IllegalStateException("No previous AT state data found");
 
 		// [Re]create AT machine state using AT state data or from scratch as applicable
-		byte[] codeBytes = this.atData.getCodeBytes();
 		MachineState state = MachineState.fromBytes(api, loggerFactory, latestAtStateData.getStateData(), codeBytes);
 		try {
-			api.preExecute(state);
 			state.execute();
 		} catch (Exception e) {
 			throw new DataException(String.format("Uncaught exception while running AT '%s'", atAddress), e);
@@ -128,18 +109,9 @@ public class AT {
 
 		byte[] stateData = state.toBytes();
 		byte[] stateHash = Crypto.digest(stateData);
-
-		// Nothing happened?
-		if (state.getSteps() == 0 && Arrays.equals(stateHash, latestAtStateData.getStateHash()))
-			// We currently want to execute frozen ATs, to maintain backwards support.
-			if (state.isFrozen() == false)
-				// this.atStateData will be null
-				return Collections.emptyList();
-
 		long atFees = api.calcFinalFees(state);
-		Long sleepUntilMessageTimestamp = this.atData.getSleepUntilMessageTimestamp();
 
-		this.atStateData = new ATStateData(atAddress, blockHeight, stateData, stateHash, atFees, false, sleepUntilMessageTimestamp);
+		this.atStateData = new ATStateData(atAddress, blockHeight, stateData, stateHash, atFees, false);
 
 		return api.getTransactions();
 	}
@@ -158,10 +130,6 @@ public class AT {
 		this.atData.setHadFatalError(state.hadFatalError());
 		this.atData.setIsFrozen(state.isFrozen());
 		this.atData.setFrozenBalance(state.getFrozenBalance());
-
-		// Special sleep-until-message support
-		this.atData.setSleepUntilMessageTimestamp(this.atStateData.getSleepUntilMessageTimestamp());
-
 		this.repository.getATRepository().save(this.atData);
 	}
 
@@ -189,10 +157,6 @@ public class AT {
 		this.atData.setHadFatalError(state.hadFatalError());
 		this.atData.setIsFrozen(state.isFrozen());
 		this.atData.setFrozenBalance(state.getFrozenBalance());
-
-		// Special sleep-until-message support
-		this.atData.setSleepUntilMessageTimestamp(previousStateData.getSleepUntilMessageTimestamp());
-
 		this.repository.getATRepository().save(this.atData);
 	}
 

src/main/java/org/qortal/at/QortalATAPI.java

@@ -32,7 +32,6 @@ import org.qortal.group.Group;
 import org.qortal.repository.ATRepository;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
-import org.qortal.repository.ATRepository.NextTransactionInfo;
 import org.qortal.transaction.AtTransaction;
 import org.qortal.transaction.Transaction.TransactionType;
 import org.qortal.utils.Base58;
@@ -75,45 +74,8 @@ public class QortalATAPI extends API {
 		return this.transactions;
 	}
 
-	public boolean willExecute(int blockHeight) throws DataException {
-		// Sleep-until-message/height checking
-		Long sleepUntilMessageTimestamp = this.atData.getSleepUntilMessageTimestamp();
-
-		if (sleepUntilMessageTimestamp != null) {
-			// Quicker to check height, if sleep-until-height also active
-			Integer sleepUntilHeight = this.atData.getSleepUntilHeight();
-
-			boolean wakeDueToHeight = sleepUntilHeight != null && sleepUntilHeight != 0 && blockHeight >= sleepUntilHeight;
-
-			boolean wakeDueToMessage = false;
-			if (!wakeDueToHeight) {
-				// No avoiding asking repository
-				Timestamp previousTxTimestamp = new Timestamp(sleepUntilMessageTimestamp);
-				NextTransactionInfo nextTransactionInfo = this.repository.getATRepository().findNextTransaction(this.atData.getATAddress(),
-						previousTxTimestamp.blockHeight,
-						previousTxTimestamp.transactionSequence);
-
-				wakeDueToMessage = nextTransactionInfo != null;
-			}
-
-			// Can we skip?
-			if (!wakeDueToHeight && !wakeDueToMessage)
-				return false;
-		}
-
-		return true;
-	}
-
-	public void preExecute(MachineState state) {
-		// Sleep-until-message/height checking
-		Long sleepUntilMessageTimestamp = this.atData.getSleepUntilMessageTimestamp();
-
-		if (sleepUntilMessageTimestamp != null) {
-			// We've passed checks, so clear sleep-related flags/values
-			this.setIsSleeping(state, false);
-			this.setSleepUntilHeight(state, 0);
-			this.atData.setSleepUntilMessageTimestamp(null);
-		}
+	public long calcFinalFees(MachineState state) {
+		return state.getSteps() * this.ciyamAtSettings.feePerStep;
 	}
 
 	// Inherited methods from CIYAM AT API
@@ -450,10 +412,6 @@ public class QortalATAPI extends API {
 
 	// Utility methods
 
-	public long calcFinalFees(MachineState state) {
-		return state.getSteps() * this.ciyamAtSettings.feePerStep;
-	}
-
 	/** Returns partial transaction signature, used to verify we're operating on the same transaction and not naively using block height & sequence. */
 	public static byte[] partialSignature(byte[] fullSignature) {
 		return Arrays.copyOfRange(fullSignature, 8, 32);
@@ -502,15 +460,6 @@ public class QortalATAPI extends API {
 		}
 	}
 
-	/*package*/ void sleepUntilMessageOrHeight(MachineState state, long txTimestamp, Long sleepUntilHeight) {
-		this.setIsSleeping(state, true);
-
-		this.atData.setSleepUntilMessageTimestamp(txTimestamp);
-
-		if (sleepUntilHeight != null)
-			this.setSleepUntilHeight(state, sleepUntilHeight.intValue());
-	}
-
 	/** Returns AT's account */
 	/* package */ Account getATAccount() {
 		return new Account(this.repository, this.atData.getATAddress());

src/main/java/org/qortal/at/QortalFunctionCode.java

@@ -84,43 +84,6 @@ public enum QortalFunctionCode {
 			api.setB(state, bBytes);
 		}
 	},
-	/**
-	 * Sleep AT until a new message arrives after 'tx-timestamp'.<br>
-	 * <tt>0x0503 tx-timestamp</tt>
-	 */
-	SLEEP_UNTIL_MESSAGE(0x0503, 1, false) {
-		@Override
-		protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException {
-			if (functionData.value1 <= 0)
-				return;
-
-			long txTimestamp = functionData.value1;
-
-			QortalATAPI api = (QortalATAPI) state.getAPI();
-			api.sleepUntilMessageOrHeight(state, txTimestamp, null);
-		}
-	},
-	/**
-	 * Sleep AT until a new message arrives, after 'tx-timestamp', or height reached.<br>
-	 * <tt>0x0504 tx-timestamp height</tt>
-	 */
-	SLEEP_UNTIL_MESSAGE_OR_HEIGHT(0x0504, 2, false) {
-		@Override
-		protected void postCheckExecute(FunctionData functionData, MachineState state, short rawFunctionCode) throws ExecutionException {
-			if (functionData.value1 <= 0)
-				return;
-
-			long txTimestamp = functionData.value1;
-
-			if (functionData.value2 <= 0)
-				return;
-
-			long sleepUntilHeight = functionData.value2;
-
-			QortalATAPI api = (QortalATAPI) state.getAPI();
-			api.sleepUntilMessageOrHeight(state, txTimestamp, sleepUntilHeight);
-		}
-	},
 	/**
 	 * Convert address in B to 20-byte value in LSB of B1, and all of B2 & B3.<br>
 	 * <tt>0x0510</tt>

src/main/java/org/qortal/block/Block.java

@@ -1104,14 +1104,9 @@ public class Block {
 			// Create repository savepoint here so we can rollback to it after testing transactions
 			repository.setSavepoint();
 
-			if (this.blockData.getHeight() == 212937) {
+			if (this.blockData.getHeight() == 212937)
 				// Apply fix for block 212937 but fix will be rolled back before we exit method
 				Block212937.processFix(this);
-			}
-			else if (InvalidNameRegistrationBlocks.isAffectedBlock(this.blockData.getHeight())) {
-				// Apply fix for affected name registration blocks, but fix will be rolled back before we exit method
-				InvalidNameRegistrationBlocks.processFix(this);
-			}
 
 			for (Transaction transaction : this.getTransactions()) {
 				TransactionData transactionData = transaction.getTransactionData();
@@ -1150,7 +1145,7 @@ public class Block {
 				// Check transaction can even be processed
 				validationResult = transaction.isProcessable();
 				if (validationResult != Transaction.ValidationResult.OK) {
-					LOGGER.info(String.format("Error during transaction validation, tx %s: %s", Base58.encode(transactionData.getSignature()), validationResult.name()));
+					LOGGER.debug(String.format("Error during transaction validation, tx %s: %s", Base58.encode(transactionData.getSignature()), validationResult.name()));
 					return ValidationResult.TRANSACTION_INVALID;
 				}
 
@@ -1264,13 +1259,12 @@ public class Block {
 		for (ATData atData : executableATs) {
 			AT at = new AT(this.repository, atData);
 			List<AtTransaction> atTransactions = at.run(this.blockData.getHeight(), this.blockData.getTimestamp());
-			ATStateData atStateData = at.getATStateData();
-			// Didn't execute? (e.g. sleeping)
-			if (atStateData == null)
-				continue;
 
 			allAtTransactions.addAll(atTransactions);
+
+			ATStateData atStateData = at.getATStateData();
 			this.ourAtStates.add(atStateData);
+
 			this.ourAtFees += atStateData.getFees();
 		}
 
@@ -1299,21 +1293,6 @@ public class Block {
 		return mintingAccount.canMint();
 	}
 
-	/**
-	 * Pre-process block, and its transactions.
-	 * This allows for any database integrity checks prior to validation.
-	 * This is called before isValid() and process()
-	 *
-	 * @throws DataException
-	 */
-	public void preProcess() throws DataException {
-		List<Transaction> blocksTransactions = this.getTransactions();
-
-		for (Transaction transaction : blocksTransactions) {
-			transaction.preProcess();
-		}
-	}
-
 	/**
 	 * Process block, and its transactions, adding them to the blockchain.
 	 * 

src/main/java/org/qortal/block/BlockChain.java

@@ -4,7 +4,10 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.InputStream;
 import java.sql.SQLException;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.locks.ReentrantLock;
 
 import javax.xml.bind.JAXBContext;
@@ -24,9 +27,11 @@ import org.eclipse.persistence.jaxb.UnmarshallerProperties;
 import org.qortal.controller.Controller;
 import org.qortal.data.block.BlockData;
 import org.qortal.network.Network;
-import org.qortal.repository.*;
+import org.qortal.repository.BlockRepository;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
-import org.qortal.utils.Base58;
 import org.qortal.utils.StringLongMapXmlAdapter;
 
 /**
@@ -68,15 +73,9 @@ public class BlockChain {
 		atFindNextTransactionFix,
 		newBlockSigHeight,
 		shareBinFix,
-		calcChainWeightTimestamp,
-		transactionV5Timestamp;
+		calcChainWeightTimestamp;
 	}
 
-	// Custom transaction fees
-	@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
-	private long nameRegistrationUnitFee;
-	private long nameRegistrationUnitFeeTimestamp;
-
 	/** Map of which blockchain features are enabled when (height/timestamp) */
 	@XmlJavaTypeAdapter(StringLongMapXmlAdapter.class)
 	private Map<String, Long> featureTriggers;
@@ -147,8 +146,7 @@ public class BlockChain {
 	}
 	private List<BlockTimingByHeight> blockTimingsByHeight;
 
-	private int minAccountLevelToMint;
-	private int minAccountLevelForBlockSubmissions;
+	private int minAccountLevelToMint = 1;
 	private int minAccountLevelToRewardShare;
 	private int maxRewardSharesPerMintingAccount;
 	private int founderEffectiveMintingLevel;
@@ -306,16 +304,6 @@ public class BlockChain {
 		return this.maxBlockSize;
 	}
 
-	// Custom transaction fees
-	public long getNameRegistrationUnitFee() {
-		return this.nameRegistrationUnitFee;
-	}
-
-	public long getNameRegistrationUnitFeeTimestamp() {
-		// FUTURE: we could use a separate structure to indicate fee adjustments for different transaction types
-		return this.nameRegistrationUnitFeeTimestamp;
-	}
-
 	/** Returns true if approval-needing transaction types require a txGroupId other than NO_GROUP. */
 	public boolean getRequireGroupForApproval() {
 		return this.requireGroupForApproval;
@@ -361,10 +349,6 @@ public class BlockChain {
 		return this.minAccountLevelToMint;
 	}
 
-	public int getMinAccountLevelForBlockSubmissions() {
-		return this.minAccountLevelForBlockSubmissions;
-	}
-
 	public int getMinAccountLevelToRewardShare() {
 		return this.minAccountLevelToRewardShare;
 	}
@@ -407,10 +391,6 @@ public class BlockChain {
 		return this.featureTriggers.get(FeatureTrigger.calcChainWeightTimestamp.name()).longValue();
 	}
 
-	public long getTransactionV5Timestamp() {
-		return this.featureTriggers.get(FeatureTrigger.transactionV5Timestamp.name()).longValue();
-	}
-
 	// More complex getters for aspects that change by height or timestamp
 
 	public long getRewardAtHeight(int ourHeight) {
@@ -526,105 +506,29 @@ public class BlockChain {
 	 * @throws SQLException
 	 */
 	public static void validate() throws DataException {
+		// Check first block is Genesis Block
+		if (!isGenesisBlockValid())
+			rebuildBlockchain();
 
-		boolean isTopOnly = Settings.getInstance().isTopOnly();
-		boolean archiveEnabled = Settings.getInstance().isArchiveEnabled();
-		boolean canBootstrap = Settings.getInstance().getBootstrap();
-		boolean needsArchiveRebuild = false;
-		BlockData chainTip;
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			chainTip = repository.getBlockRepository().getLastBlock();
-
-			// Ensure archive is (at least partially) intact, and force a bootstrap if it isn't
-			if (!isTopOnly && archiveEnabled && canBootstrap) {
-				needsArchiveRebuild = (repository.getBlockArchiveRepository().fromHeight(2) == null);
-				if (needsArchiveRebuild) {
-					LOGGER.info("Couldn't retrieve block 2 from archive. Bootstrapping...");
-
-					// If there are minting accounts, make sure to back them up
-					// Don't backup if there are no minting accounts, as this can cause problems
-					if (!repository.getAccountRepository().getMintingAccounts().isEmpty()) {
-						Controller.getInstance().exportRepositoryData();
-					}
-				}
-			}
-		}
-
-		boolean hasBlocks = (chainTip != null && chainTip.getHeight() > 1);
-
-		if (isTopOnly && hasBlocks) {
-			// Top-only mode is enabled and we have blocks, so it's possible that the genesis block has been pruned
-			// It's best not to validate it, and there's no real need to
-		} else {
-			// Check first block is Genesis Block
-			if (!isGenesisBlockValid() || needsArchiveRebuild) {
-				try {
-					rebuildBlockchain();
-
-				} catch (InterruptedException e) {
-					throw new DataException(String.format("Interrupted when trying to rebuild blockchain: %s", e.getMessage()));
-				}
-			}
-		}
-
-		// We need to create a new connection, as the previous repository and its connections may be been
-		// closed by rebuildBlockchain() if a bootstrap was applied
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			repository.checkConsistency();
 
-			// Set the number of blocks to validate based on the pruned state of the chain
-			// If pruned, subtract an extra 10 to allow room for error
-			int blocksToValidate = (isTopOnly || archiveEnabled) ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440;
+			int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - 1440, 1);
 
-			int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - blocksToValidate, 1);
 			BlockData detachedBlockData = repository.getBlockRepository().getDetachedBlockSignature(startHeight);
 
 			if (detachedBlockData != null) {
-				LOGGER.error(String.format("Block %d's reference does not match any block's signature",
-						detachedBlockData.getHeight()));
-				LOGGER.error(String.format("Your chain may be invalid and you should consider bootstrapping" +
-						" or re-syncing from genesis."));
-			}
-		}
-	}
+				LOGGER.error(String.format("Block %d's reference does not match any block's signature", detachedBlockData.getHeight()));
 
-	/**
-	 * More thorough blockchain validation method. Useful for validating bootstraps.
-	 * A DataException is thrown if anything is invalid.
-	 *
-	 * @throws DataException
-	 */
-	public static void validateAllBlocks() throws DataException {
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			BlockData chainTip = repository.getBlockRepository().getLastBlock();
-			final int chainTipHeight = chainTip.getHeight();
-			final int oldestBlock = 1; // TODO: increase if in pruning mode
-			byte[] lastReference = null;
-
-			for (int height = chainTipHeight; height > oldestBlock; height--) {
-				BlockData blockData = repository.getBlockRepository().fromHeight(height);
-				if (blockData == null) {
-					blockData = repository.getBlockArchiveRepository().fromHeight(height);
+				// Wait for blockchain lock (whereas orphan() only tries to get lock)
+				ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+				blockchainLock.lock();
+				try {
+					LOGGER.info(String.format("Orphaning back to block %d", detachedBlockData.getHeight() - 1));
+					orphan(detachedBlockData.getHeight() - 1);
+				} finally {
+					blockchainLock.unlock();
 				}
-
-				if (blockData == null) {
-					String error = String.format("Missing block at height %d", height);
-					LOGGER.error(error);
-					throw new DataException(error);
-				}
-
-				if (height != chainTipHeight) {
-					// Check reference
-					if (!Arrays.equals(blockData.getSignature(), lastReference)) {
-						String error = String.format("Invalid reference for block at height %d: %s (should be %s)",
-								height, Base58.encode(blockData.getReference()), Base58.encode(lastReference));
-						LOGGER.error(error);
-						throw new DataException(error);
-					}
-				}
-
-				lastReference = blockData.getReference();
 			}
 		}
 	}
@@ -647,15 +551,7 @@ public class BlockChain {
 		}
 	}
 
-	private static void rebuildBlockchain() throws DataException, InterruptedException {
-		boolean shouldBootstrap = Settings.getInstance().getBootstrap();
-		if (shouldBootstrap) {
-			// Settings indicate that we should apply a bootstrap rather than rebuilding and syncing from genesis
-			Bootstrap bootstrap = new Bootstrap();
-			bootstrap.startImport();
-			return;
-		}
-
+	private static void rebuildBlockchain() throws DataException {
 		// (Re)build repository
 		if (!RepositoryManager.wasPristineAtOpen())
 			RepositoryManager.rebuild();

src/main/java/org/qortal/block/InvalidNameRegistrationBlocks.java

@@ -1,114 +0,0 @@
-package org.qortal.block;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.naming.Name;
-import org.qortal.repository.DataException;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Invalid Name Registration Blocks
- * <p>
- * A node minted a version of block 535658 that contained one transaction:
- * a REGISTER_NAME transaction that attempted to register a name that was already registered.
- * <p>
- * This invalid transaction made block 535658 (rightly) invalid to several nodes,
- * which refused to use that block.
- * However, it seems there were no other nodes minting an alternative, valid block at that time
- * and so the chain stalled for several nodes in the network.
- * <p>
- * Additionally, the invalid block 535658 affected all new installations, regardless of whether
- * they synchronized from scratch (block 1) or used an 'official release' bootstrap.
- * <p>
- * The diagnosis found the following:
- * - The original problem occurred in block 535205 where for some unknown reason many nodes didn't
- *   add the name from a REGISTER_NAME transaction to their Names table.
- * - As a result, those nodes had a corrupt db, because they weren't holding a record of the name.
- * - This invalid db then caused them to treat a candidate for block 535658 as valid when it
- *   should have been invalid.
- * - As such, the chain continued on with a technically invalid block in it, for a subset of the network
- * <p>
- * As with block 212937, there were three options, but the only feasible one was to apply edits to block
- * 535658 to make it valid. There were several cross-chain trades completed after this block, so doing
- * any kind of rollback was out of the question.
- * <p>
- * To complicate things further, a custom data field was used for the first REGISTER_NAME transaction,
- * and the default data field was used for the second. So it was important that all nodes ended up with
- * the exact same data regardless of how they arrived there.
- * <p>
- * The invalid block 535658 signature is: <tt>3oiuDhok...NdXvCLEV</tt>.
- * <p>
- * The invalid transaction in block 212937 is:
- * <p>
- * <code><pre>
-	 {
-		 "type": "REGISTER_NAME",
-		 "timestamp": 1630739437517,
-		 "reference": "4peRechwSPxP6UkRj9Y8ox9YxkWb34sWk5zyMc1WyMxEsACxD4Gmm7LZVsQ6Skpze8QCSBMZasvEZg6RgdqkyADW",
-		 "fee": "0.00100000",
-		 "signature": "2t1CryCog8KPDBarzY5fDCKu499nfnUcGrz4Lz4w5wNb5nWqm7y126P48dChYY7huhufcBV3RJPkgKP4Ywxc1gXx",
-		 "txGroupId": 0,
-		 "blockHeight": 535658,
-		 "approvalStatus": "NOT_REQUIRED",
-		 "creatorAddress": "Qbx9ojxv7XNi1xDMWzzw7xDvd1zYW6SKFB",
-		 "registrantPublicKey": "HJqGEf6cW695Xun4ydhkB2excGFwsDxznhNCRHZStyyx",
-		 "name": "Qplay",
-		 "data": "Registered Name on the Qortal Chain"
-	 }
-   </pre></code>
- * <p>
- * Account <tt>Qbx9ojxv7XNi1xDMWzzw7xDvd1zYW6SKFB</tt> attempted to register the name <tt>Qplay</tt>
- * when they had already registered it 12 hours before in block <tt>535205</tt>.
- * <p>
- * However, on the broken DB nodes, their Names table was missing a record for the `Qplay` name
- * which was sufficient to make the transaction valid.
- *
- * This problem then occurred two more times, in blocks 536140 and 541334
- * To reduce duplication, I have combined all three block fixes into a single class
- *
- */
-public final class InvalidNameRegistrationBlocks {
-
-	private static final Logger LOGGER = LogManager.getLogger(InvalidNameRegistrationBlocks.class);
-
-	public static Map<Integer, String> invalidBlocksNamesMap = new HashMap<Integer, String>()
-	{
-		{
-			put(535658, "Qplay");
-			put(536140, "Qweb");
-			put(541334, "Qithub");
-		}
-	};
-
-	private InvalidNameRegistrationBlocks() {
-		/* Do not instantiate */
-	}
-
-	public static boolean isAffectedBlock(int height) {
-		return (invalidBlocksNamesMap.containsKey(height));
-	}
-
-	public static void processFix(Block block) throws DataException {
-		Integer blockHeight = block.getBlockData().getHeight();
-		String invalidName = invalidBlocksNamesMap.get(blockHeight);
-		if (invalidName == null) {
-			throw new DataException(String.format("Unable to lookup invalid name for block height %d", blockHeight));
-		}
-
-		// Unregister the existing name record if it exists
-		// This ensures that the duplicate name is considered valid, and therefore
-		// the second (i.e. duplicate) REGISTER_NAME transaction data is applied.
-		// Both were issued by the same user account, so there is no conflict.
-		Name name = new Name(block.repository, invalidName);
-		name.unregister();
-
-		LOGGER.debug("Applied name registration patch for block {}", blockHeight);
-	}
-
-	// Note:
-	// There is no need to write an orphanFix() method, as we do not have
-	// the necessary ATStatesData to orphan back this far anyway
-
-}

src/main/java/org/qortal/controller/ArbitraryDataManager.java

@@ -0,0 +1,91 @@
+package org.qortal.controller;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.transaction.ArbitraryTransaction;
+import org.qortal.transaction.Transaction.TransactionType;
+
+public class ArbitraryDataManager extends Thread {
+
+	private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataManager.class);
+	private static final List<TransactionType> ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY);
+
+	private static ArbitraryDataManager instance;
+
+	private volatile boolean isStopping = false;
+
+	private ArbitraryDataManager() {
+	}
+
+	public static ArbitraryDataManager getInstance() {
+		if (instance == null)
+			instance = new ArbitraryDataManager();
+
+		return instance;
+	}
+
+	@Override
+	public void run() {
+		Thread.currentThread().setName("Arbitrary Data Manager");
+
+		try {
+			while (!isStopping) {
+				Thread.sleep(2000);
+
+				// Any arbitrary transactions we want to fetch data for?
+				try (final Repository repository = RepositoryManager.getRepository()) {
+					List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, null, null, true);
+					if (signatures == null || signatures.isEmpty())
+						continue;
+
+					// Filter out those that already have local data
+					signatures.removeIf(signature -> hasLocalData(repository, signature));
+
+					if (signatures.isEmpty())
+						continue;
+
+					// Pick one at random
+					final int index = new Random().nextInt(signatures.size());
+					byte[] signature = signatures.get(index);
+
+					Controller.getInstance().fetchArbitraryData(signature);
+				} catch (DataException e) {
+					LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
+				}
+			}
+		} catch (InterruptedException e) {
+			// Fall-through to exit thread...
+		}
+	}
+
+	public void shutdown() {
+		isStopping = true;
+		this.interrupt();
+	}
+
+	private boolean hasLocalData(final Repository repository, final byte[] signature) {
+		try {
+			TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+			if (!(transactionData instanceof ArbitraryTransactionData))
+				return true;
+
+			ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
+
+			return arbitraryTransaction.isDataLocal();
+		} catch (DataException e) {
+			LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e);
+			return true;
+		}
+	}
+
+}

src/main/java/org/qortal/controller/AtStatesTrimmer.java

@@ -1,9 +1,7 @@
-package org.qortal.controller.repository;
+package org.qortal.controller;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.qortal.controller.Controller;
-import org.qortal.controller.Synchronizer;
 import org.qortal.data.block.BlockData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
@@ -22,8 +20,8 @@ public class AtStatesTrimmer implements Runnable {
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			int trimStartHeight = repository.getATRepository().getAtTrimHeight();
 
-			repository.discardChanges();
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().prepareForAtStateTrimming();
+			repository.saveChanges();
 
 			while (!Controller.isStopping()) {
 				repository.discardChanges();
@@ -35,7 +33,7 @@ public class AtStatesTrimmer implements Runnable {
 					continue;
 
 				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Synchronizer.getInstance().isSynchronizing())
+				if (Controller.getInstance().isSynchronizing())
 					continue;
 
 				long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime();
@@ -64,7 +62,7 @@ public class AtStatesTrimmer implements Runnable {
 					if (upperTrimmableHeight > upperBatchHeight) {
 						trimStartHeight = upperBatchHeight;
 						repository.getATRepository().setAtTrimHeight(trimStartHeight);
-						repository.getATRepository().rebuildLatestAtStates();
+						repository.getATRepository().prepareForAtStateTrimming();
 						repository.saveChanges();
 
 						final int finalTrimStartHeight = trimStartHeight;

src/main/java/org/qortal/controller/AutoUpdate.java

@@ -14,7 +14,6 @@ import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.concurrent.TimeoutException;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -216,17 +215,8 @@ public class AutoUpdate extends Thread {
 		}
 
 		// Give repository a chance to backup in case things go badly wrong (if enabled)
-		if (Settings.getInstance().getRepositoryBackupInterval() > 0) {
-			try {
-				// Timeout if the database isn't ready for backing up after 60 seconds
-				long timeout = 60 * 1000L;
-				RepositoryManager.backup(true, "backup", timeout);
-
-			} catch (TimeoutException e) {
-				LOGGER.info("Attempt to backup repository failed due to timeout: {}", e.getMessage());
-				// Continue with the auto update anyway...
-			}
-		}
+		if (Settings.getInstance().getRepositoryBackupInterval() > 0)
+			RepositoryManager.backup(true);
 
 		// Call ApplyUpdate to end this process (unlocking current JAR so it can be replaced)
 		String javaHome = System.getProperty("java.home");

src/main/java/org/qortal/controller/BlockMinter.java

@@ -1,8 +1,6 @@
 package org.qortal.controller;
 
 import java.math.BigInteger;
-import java.text.DecimalFormat;
-import java.text.NumberFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -22,7 +20,6 @@ import org.qortal.data.account.MintingAccountData;
 import org.qortal.data.account.RewardShareData;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.block.BlockSummaryData;
-import org.qortal.data.block.CommonBlockData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
@@ -47,9 +44,6 @@ public class BlockMinter extends Thread {
 	private static Long lastLogTimestamp;
 	private static Long logTimeout;
 
-	// Recovery
-	public static final long INVALID_BLOCK_RECOVERY_TIMEOUT = 10 * 60 * 1000L; // ms
-
 	// Constructors
 
 	public BlockMinter() {
@@ -78,10 +72,6 @@ public class BlockMinter extends Thread {
 			BlockRepository blockRepository = repository.getBlockRepository();
 			BlockData previousBlockData = null;
 
-			// Vars to keep track of blocks that were skipped due to chain weight
-			byte[] parentSignatureForLastLowWeightBlock = null;
-			Long timeOfLastLowWeightBlock = null;
-
 			List<Block> newBlocks = new ArrayList<>();
 
 			// Flags for tracking change in whether minting is possible,
@@ -137,15 +127,6 @@ public class BlockMinter extends Thread {
 						madi.remove();
 						continue;
 					}
-
-					// Optional (non-validated) prevention of block submissions below a defined level.
-					// This is an unvalidated version of Blockchain.minAccountLevelToMint
-					// and exists only to reduce block candidates by default.
-					int level = mintingAccount.getEffectiveMintingLevel();
-					if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
-						madi.remove();
-						continue;
-					}
 				}
 
 				List<Peer> peers = Network.getInstance().getHandshakedPeers();
@@ -156,32 +137,16 @@ public class BlockMinter extends Thread {
 
 				// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
 				// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
-				if (Synchronizer.getInstance().getRecoveryMode() == false)
+				if (Controller.getInstance().getRecoveryMode() == false)
 					peers.removeIf(Controller.hasNoRecentBlock);
 
 				// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
 				if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
 					continue;
 
-				// If we are stuck on an invalid block, we should allow an alternative to be minted
-				boolean recoverInvalidBlock = false;
-				if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) {
-					// We've had at least one invalid block
-					long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived;
-					long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived;
-					if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) {
-						if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) {
-							// Last valid block was more than 10 mins ago, but we've had an invalid block since then
-							// Assume that the chain has stalled because there is no alternative valid candidate
-							// Enter recovery mode to allow alternative, valid candidates to be minted
-							recoverInvalidBlock = true;
-						}
-					}
-				}
-
 				// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
 				if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
-					if (Synchronizer.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false)
+					if (Controller.getInstance().getRecoveryMode() == false)
 						continue;
 
 				// There are enough peers with a recent block and our latest block is recent
@@ -195,9 +160,6 @@ public class BlockMinter extends Thread {
 
 					// Reduce log timeout
 					logTimeout = 10 * 1000L;
-
-					// Last low weight block is no longer valid
-					parentSignatureForLastLowWeightBlock = null;
 				}
 
 				// Discard accounts we have already built blocks with
@@ -214,14 +176,6 @@ public class BlockMinter extends Thread {
 					continue;
 				}
 
-				if (parentSignatureForLastLowWeightBlock != null) {
-					// The last iteration found a higher weight block in the network, so sleep for a while
-					// to allow is to sync the higher weight chain. We are sleeping here rather than when
-					// detected as we don't want to hold the blockchain lock open.
-					LOGGER.info("Sleeping for 10 seconds...");
-					Thread.sleep(10 * 1000L);
-				}
-
 				for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
 					// First block does the AT heavy-lifting
 					if (newBlocks.isEmpty()) {
@@ -276,8 +230,6 @@ public class BlockMinter extends Thread {
 						if (testBlock.isTimestampValid() != ValidationResult.OK)
 							continue;
 
-						testBlock.preProcess();
-
 						// Is new block valid yet? (Before adding unconfirmed transactions)
 						ValidationResult result = testBlock.isValid();
 						if (result != ValidationResult.OK) {
@@ -313,44 +265,6 @@ public class BlockMinter extends Thread {
 						}
 					}
 
-					try {
-						if (this.higherWeightChainExists(repository, bestWeight)) {
-
-							// Check if the base block has updated since the last time we were here
-							if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null ||
-									!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) {
-								// We've switched to a different chain, so reset the timer
-								timeOfLastLowWeightBlock = NTP.getTime();
-							}
-							parentSignatureForLastLowWeightBlock = previousBlockData.getSignature();
-
-							// If less than 30 seconds has passed since first detection the higher weight chain,
-							// we should skip our block submission to give us the opportunity to sync to the better chain
-							if (NTP.getTime() - timeOfLastLowWeightBlock < 30*1000L) {
-								LOGGER.info("Higher weight chain found in peers, so not signing a block this round");
-								LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock);
-								continue;
-							}
-							else {
-								// More than 30 seconds have passed, so we should submit our block candidate anyway.
-								LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate...");
-							}
-						}
-						else {
-							LOGGER.debug("No higher weight chain found in peers");
-						}
-					} catch (DataException e) {
-						LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
-					}
-
-					// Discard any uncommitted changes as a result of the higher weight chain detection
-					repository.discardChanges();
-
-					// Clear variables that track low weight blocks
-					parentSignatureForLastLowWeightBlock = null;
-					timeOfLastLowWeightBlock = null;
-
-
 					// Add unconfirmed transactions
 					addUnconfirmedTransactions(repository, newBlock);
 
@@ -507,8 +421,7 @@ public class BlockMinter extends Thread {
 
 			// Add to blockchain
 			newBlock.process();
-			LOGGER.info(String.format("Minted new test block: %d sig: %.8s",
-					newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature())));
+			LOGGER.info(String.format("Minted new test block: %d", newBlock.getBlockData().getHeight()));
 
 			repository.saveChanges();
 
@@ -518,61 +431,6 @@ public class BlockMinter extends Thread {
 		}
 	}
 
-	private BigInteger getOurChainWeightSinceBlock(Repository repository, BlockSummaryData commonBlock, List<BlockSummaryData> peerBlockSummaries) throws DataException {
-		final int commonBlockHeight = commonBlock.getHeight();
-		final byte[] commonBlockSig = commonBlock.getSignature();
-		int mutualHeight = commonBlockHeight;
-
-		// Fetch our corresponding block summaries
-		final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
-		List<BlockSummaryData> ourBlockSummaries = repository.getBlockRepository()
-				.getBlockSummaries(commonBlockHeight + 1, ourLatestBlockData.getHeight());
-		if (!ourBlockSummaries.isEmpty()) {
-			Synchronizer.getInstance().populateBlockSummariesMinterLevels(repository, ourBlockSummaries);
-		}
-
-		if (ourBlockSummaries != null && peerBlockSummaries != null) {
-			mutualHeight += Math.min(ourBlockSummaries.size(), peerBlockSummaries.size());
-		}
-		return Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight);
-	}
-
-	private boolean higherWeightChainExists(Repository repository, BigInteger blockCandidateWeight) throws DataException {
-		if (blockCandidateWeight == null) {
-			// Can't make decisions without knowing the block candidate weight
-			return false;
-		}
-		NumberFormat formatter = new DecimalFormat("0.###E0");
-
-		List<Peer> peers = Network.getInstance().getHandshakedPeers();
-		// Loop through handshaked peers and check for any new block candidates
-		for (Peer peer : peers) {
-			if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) {
-				// This peer has common block data
-				CommonBlockData commonBlockData = peer.getCommonBlockData();
-				BlockSummaryData commonBlockSummaryData = commonBlockData.getCommonBlockSummary();
-				if (commonBlockData.getChainWeight() != null) {
-					// The synchronizer has calculated this peer's chain weight
-					BigInteger ourChainWeightSinceCommonBlock = this.getOurChainWeightSinceBlock(repository, commonBlockSummaryData, commonBlockData.getBlockSummariesAfterCommonBlock());
-					BigInteger ourChainWeight = ourChainWeightSinceCommonBlock.add(blockCandidateWeight);
-					BigInteger peerChainWeight = commonBlockData.getChainWeight();
-					if (peerChainWeight.compareTo(ourChainWeight) >= 0) {
-						// This peer has a higher weight chain than ours
-						LOGGER.debug("Peer {} is on a higher weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight));
-						return true;
-
-					} else {
-						LOGGER.debug("Peer {} is on a lower weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight));
-					}
-				} else {
-					LOGGER.debug("Peer {} has no chain weight", peer);
-				}
-			} else {
-				LOGGER.debug("Peer {} has no common block data", peer);
-			}
-		}
-		return false;
-	}
 	private static void moderatedLog(Runnable logFunction) {
 		// We only log if logging at TRACE or previous log timeout has expired
 		if (!LOGGER.isTraceEnabled() && lastLogTimestamp != null && lastLogTimestamp + logTimeout > System.currentTimeMillis())

src/main/java/org/qortal/controller/OnlineAccountsSignaturesTrimmer.java

@@ -1,10 +1,8 @@
-package org.qortal.controller.repository;
+package org.qortal.controller;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.block.BlockChain;
-import org.qortal.controller.Controller;
-import org.qortal.controller.Synchronizer;
 import org.qortal.data.block.BlockData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
@@ -37,7 +35,7 @@ public class OnlineAccountsSignaturesTrimmer implements Runnable {
 					continue;
 
 				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Synchronizer.getInstance().isSynchronizing())
+				if (Controller.getInstance().isSynchronizing())
 					continue;
 
 				// Trim blockchain by removing 'old' online accounts signatures

src/main/java/org/qortal/controller/Synchronizer.java

@@ -1,13 +1,14 @@
 package org.qortal.controller;
 
 import java.math.BigInteger;
-import java.security.SecureRandom;
 import java.text.DecimalFormat;
 import java.text.NumberFormat;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
+import java.util.Iterator;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -22,11 +23,12 @@ import org.qortal.data.block.CommonBlockData;
 import org.qortal.data.network.PeerChainTipData;
 import org.qortal.data.transaction.RewardShareTransactionData;
 import org.qortal.data.transaction.TransactionData;
-import org.qortal.network.Network;
 import org.qortal.network.Peer;
 import org.qortal.network.message.BlockMessage;
+import org.qortal.network.message.BlocksMessage;
 import org.qortal.network.message.BlockSummariesMessage;
 import org.qortal.network.message.GetBlockMessage;
+import org.qortal.network.message.GetBlocksMessage;
 import org.qortal.network.message.GetBlockSummariesMessage;
 import org.qortal.network.message.GetSignaturesV2Message;
 import org.qortal.network.message.Message;
@@ -38,10 +40,11 @@ import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
 import org.qortal.transaction.Transaction;
 import org.qortal.utils.Base58;
-import org.qortal.utils.ByteArray;
 import org.qortal.utils.NTP;
 
-public class Synchronizer extends Thread {
+import static org.qortal.network.Peer.FETCH_BLOCKS_TIMEOUT;
+
+public class Synchronizer {
 
 	private static final Logger LOGGER = LogManager.getLogger(Synchronizer.class);
 
@@ -59,37 +62,15 @@ public class Synchronizer extends Thread {
 	/** Maximum number of block signatures we ask from peer in one go */
 	private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings?
 
-	private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms
+	/* Minimum peer version that supports syncing multiple blocks at once via GetBlocksMessage */
+	private static final long PEER_VERSION_160 = 0x0100060000L;
 
 
-	private boolean running;
 
-	/** Latest block signatures from other peers that we know are on inferior chains. */
-	List<ByteArray> inferiorChainSignatures = new ArrayList<>();
-
-	/** Recovery mode, which is used to bring back a stalled network */
-	private boolean recoveryMode = false;
-	private boolean peersAvailable = true; // peersAvailable must default to true
-	private long timePeersLastAvailable = 0;
 
 	// Keep track of the size of the last re-org, so it can be logged
 	private int lastReorgSize;
 
-	/** Synchronization object for sync variables below */
-	public final Object syncLock = new Object();
-	/** Whether we are attempting to synchronize. */
-	private volatile boolean isSynchronizing = false;
-	/** Temporary estimate of synchronization progress for SysTray use. */
-	private volatile int syncPercent = 0;
-
-	private static volatile boolean requestSync = false;
-	private boolean syncRequestPending = false;
-
-	// Keep track of invalid blocks so that we don't keep trying to sync them
-	private Map<String, Long> invalidBlockSignatures = Collections.synchronizedMap(new HashMap<>());
-	public Long timeValidBlockLastReceived = null;
-	public Long timeInvalidBlockLastReceived = null;
-
 	private static Synchronizer instance;
 
 	public enum SynchronizationResult {
@@ -99,7 +80,6 @@ public class Synchronizer extends Thread {
 	// Constructors
 
 	private Synchronizer() {
-		this.running = true;
 	}
 
 	public static Synchronizer getInstance() {
@@ -110,284 +90,6 @@ public class Synchronizer extends Thread {
 	}
 
 
-	@Override
-	public void run() {
-		Thread.currentThread().setName("Synchronizer");
-
-		try {
-			while (running && !Controller.isStopping()) {
-				Thread.sleep(1000);
-
-				if (requestSync) {
-					requestSync = false;
-					boolean success = Synchronizer.getInstance().potentiallySynchronize();
-					if (!success) {
-						// Something went wrong, so try again next time
-						requestSync = true;
-					}
-					// Remember that we have a pending sync request if this attempt failed
-					syncRequestPending = !success;
-				}
-			}
-		} catch (InterruptedException e) {
-			// Clear interrupted flag so we can shutdown trim threads
-			Thread.interrupted();
-			// Fall-through to exit
-		}
-	}
-
-	public void shutdown() {
-		this.running = false;
-		this.interrupt();
-	}
-
-
-
-	public boolean isSynchronizing() {
-		return this.isSynchronizing;
-	}
-
-	public boolean isSyncRequestPending() {
-		return this.syncRequestPending;
-	}
-
-	public Integer getSyncPercent() {
-		synchronized (this.syncLock) {
-			return this.isSynchronizing ? this.syncPercent : null;
-		}
-	}
-
-	public void requestSync() {
-		requestSync = true;
-	}
-
-	public boolean isSyncRequested() {
-		return requestSync;
-	}
-
-	public boolean getRecoveryMode() {
-		return this.recoveryMode;
-	}
-
-
-	public boolean potentiallySynchronize() throws InterruptedException {
-		// Already synchronizing via another thread?
-		if (this.isSynchronizing)
-			return true;
-
-		List<Peer> peers = Network.getInstance().getHandshakedPeers();
-
-		// Disregard peers that have "misbehaved" recently
-		peers.removeIf(Controller.hasMisbehaved);
-
-		// Disregard peers that only have genesis block
-		peers.removeIf(Controller.hasOnlyGenesisBlock);
-
-		// Disregard peers that don't have a recent block
-		peers.removeIf(Controller.hasNoRecentBlock);
-
-		// Disregard peers that are on an old version
-		peers.removeIf(Controller.hasOldVersion);
-
-		checkRecoveryModeForPeers(peers);
-		if (recoveryMode) {
-			peers = Network.getInstance().getHandshakedPeers();
-			peers.removeIf(Controller.hasOnlyGenesisBlock);
-			peers.removeIf(Controller.hasMisbehaved);
-			peers.removeIf(Controller.hasOldVersion);
-		}
-
-		// Check we have enough peers to potentially synchronize
-		if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
-			return true;
-
-		// Disregard peers that have no block signature or the same block signature as us
-		peers.removeIf(Controller.hasNoOrSameBlock);
-
-		// Disregard peers that are on the same block as last sync attempt and we didn't like their chain
-		peers.removeIf(Controller.hasInferiorChainTip);
-
-		final int peersBeforeComparison = peers.size();
-
-		// Request recent block summaries from the remaining peers, and locate our common block with each
-		Synchronizer.getInstance().findCommonBlocksWithPeers(peers);
-
-		// Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks
-		peers = Synchronizer.getInstance().comparePeers(peers);
-
-		// We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains
-		peers.removeIf(Controller.hasInferiorChainTip);
-
-		final int peersRemoved = peersBeforeComparison - peers.size();
-		if (peersRemoved > 0 && peers.size() > 0)
-			LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size()));
-
-		if (peers.isEmpty())
-			return true;
-
-		if (peers.size() > 1) {
-			StringBuilder finalPeersString = new StringBuilder();
-			for (Peer peer : peers)
-				finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer);
-			LOGGER.debug(String.format("Choosing random peer from: [%s]", finalPeersString.toString()));
-		}
-
-		// Pick random peer to sync with
-		int index = new SecureRandom().nextInt(peers.size());
-		Peer peer = peers.get(index);
-
-		SynchronizationResult syncResult = actuallySynchronize(peer, false);
-		if (syncResult == SynchronizationResult.NO_BLOCKCHAIN_LOCK) {
-			// No blockchain lock - force a retry by returning false
-			return false;
-		}
-
-		return true;
-	}
-
-	public SynchronizationResult actuallySynchronize(Peer peer, boolean force) throws InterruptedException {
-		boolean hasStatusChanged = false;
-		BlockData priorChainTip = Controller.getInstance().getChainTip();
-
-		synchronized (this.syncLock) {
-			this.syncPercent = (priorChainTip.getHeight() * 100) / peer.getChainTipData().getLastHeight();
-
-			// Only update SysTray if we're potentially changing height
-			if (this.syncPercent < 100) {
-				this.isSynchronizing = true;
-				hasStatusChanged = true;
-			}
-		}
-		peer.setSyncInProgress(true);
-
-		if (hasStatusChanged)
-			Controller.getInstance().updateSysTray();
-
-		try {
-			SynchronizationResult syncResult = Synchronizer.getInstance().synchronize(peer, force);
-			switch (syncResult) {
-				case GENESIS_ONLY:
-				case NO_COMMON_BLOCK:
-				case TOO_DIVERGENT:
-				case INVALID_DATA: {
-					// These are more serious results that warrant a cool-off
-					LOGGER.info(String.format("Failed to synchronize with peer %s (%s) - cooling off", peer, syncResult.name()));
-
-					// Don't use this peer again for a while
-					Network.getInstance().peerMisbehaved(peer);
-					break;
-				}
-
-				case INFERIOR_CHAIN: {
-					// Update our list of inferior chain tips
-					ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature());
-					if (!inferiorChainSignatures.contains(inferiorChainSignature))
-						inferiorChainSignatures.add(inferiorChainSignature);
-
-					// These are minor failure results so fine to try again
-					LOGGER.debug(() -> String.format("Refused to synchronize with peer %s (%s)", peer, syncResult.name()));
-
-					// Notify peer of our superior chain
-					if (!peer.sendMessage(Network.getInstance().buildHeightMessage(peer, priorChainTip)))
-						peer.disconnect("failed to notify peer of our superior chain");
-					break;
-				}
-
-				case NO_REPLY:
-				case NO_BLOCKCHAIN_LOCK:
-				case REPOSITORY_ISSUE:
-					// These are minor failure results so fine to try again
-					LOGGER.debug(() -> String.format("Failed to synchronize with peer %s (%s)", peer, syncResult.name()));
-					break;
-
-				case SHUTTING_DOWN:
-					// Just quietly exit
-					break;
-
-				case OK:
-					// fall-through...
-				case NOTHING_TO_DO: {
-					// Update our list of inferior chain tips
-					ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature());
-					if (!inferiorChainSignatures.contains(inferiorChainSignature))
-						inferiorChainSignatures.add(inferiorChainSignature);
-
-					LOGGER.debug(() -> String.format("Synchronized with peer %s (%s)", peer, syncResult.name()));
-					break;
-				}
-			}
-
-			if (!running) {
-				// We've stopped
-				return SynchronizationResult.SHUTTING_DOWN;
-			}
-
-			// Has our chain tip changed?
-			BlockData newChainTip;
-
-			try (final Repository repository = RepositoryManager.getRepository()) {
-				newChainTip = repository.getBlockRepository().getLastBlock();
-			} catch (DataException e) {
-				LOGGER.warn(String.format("Repository issue when trying to fetch post-synchronization chain tip: %s", e.getMessage()));
-				return syncResult;
-			}
-
-			if (!Arrays.equals(newChainTip.getSignature(), priorChainTip.getSignature())) {
-				// Reset our cache of inferior chains
-				inferiorChainSignatures.clear();
-
-				Network network = Network.getInstance();
-				network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip));
-			}
-
-			return syncResult;
-		} finally {
-			this.isSynchronizing = false;
-			peer.setSyncInProgress(false);
-		}
-	}
-
-	private boolean checkRecoveryModeForPeers(List<Peer> qualifiedPeers) {
-		List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
-
-		if (handshakedPeers.size() > 0) {
-			// There is at least one handshaked peer
-			if (qualifiedPeers.isEmpty()) {
-				// There are no 'qualified' peers - i.e. peers that have a recent block we can sync to
-				boolean werePeersAvailable = peersAvailable;
-				peersAvailable = false;
-
-				// If peers only just became unavailable, update our record of the time they were last available
-				if (werePeersAvailable)
-					timePeersLastAvailable = NTP.getTime();
-
-				// If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint
-				if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) {
-					if (recoveryMode == false) {
-						LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000));
-						recoveryMode = true;
-					}
-				}
-			} else {
-				// We now have at least one peer with a recent block, so we can exit recovery mode and sync normally
-				peersAvailable = true;
-				if (recoveryMode) {
-					LOGGER.info("Peers have become available again. Exiting recovery mode...");
-					recoveryMode = false;
-				}
-			}
-		}
-		return recoveryMode;
-	}
-
-	public void addInferiorChainSignature(byte[] inferiorSignature) {
-		// Update our list of inferior chain tips
-		ByteArray inferiorChainSignature = new ByteArray(inferiorSignature);
-		if (!inferiorChainSignatures.contains(inferiorChainSignature))
-			inferiorChainSignatures.add(inferiorChainSignature);
-	}
-
-
 	/**
 	 * Iterate through a list of supplied peers, and attempt to find our common block with each.
 	 * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later.
@@ -560,8 +262,6 @@ public class Synchronizer extends Thread {
 				// Create a placeholder to track of common blocks that we can discard due to being inferior chains
 				int dropPeersAfterCommonBlockHeight = 0;
 
-				NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
-
 				// Remove peers with no common block data
 				Iterator iterator = peers.iterator();
 				while (iterator.hasNext()) {
@@ -582,7 +282,7 @@ public class Synchronizer extends Thread {
 							// We have already determined that the correct chain diverged from a lower height. We are safe to skip these peers.
 							for (Peer peer : peersSharingCommonBlock) {
 								LOGGER.debug(String.format("Peer %s has common block at height %d but the superior chain is at height %d. Removing it from this round.", peer, commonBlockSummary.getHeight(), dropPeersAfterCommonBlockHeight));
-								this.addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature());
+								Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature());
 							}
 							continue;
 						}
@@ -646,12 +346,6 @@ public class Synchronizer extends Thread {
 							}
 						}
 
-						// Ignore this peer if it holds an invalid block
-						if (this.containsInvalidBlockSummary(peer.getCommonBlockData().getBlockSummariesAfterCommonBlock())) {
-							LOGGER.debug("Ignoring peer %s because it holds an invalid block", peer);
-							peers.remove(peer);
-						}
-
 						// Reduce minChainLength if needed. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength
 						List <BlockSummaryData> peerBlockSummaries = peer.getCommonBlockData().getBlockSummariesAfterCommonBlock();
 						if (peerBlockSummaries != null && peerBlockSummaries.size() > 0)
@@ -684,7 +378,9 @@ public class Synchronizer extends Thread {
 					if (ourBlockSummaries.size() > 0)
 						ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, maxHeightForChainWeightComparisons);
 
-					LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", (usingSameLengthChainWeight ? minChainLength : ourBlockSummaries.size()), accurateFormatter.format(ourChainWeight)));
+					NumberFormat formatter = new DecimalFormat("0.###E0");
+					NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
+					LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", (usingSameLengthChainWeight ? minChainLength : ourBlockSummaries.size()), formatter.format(ourChainWeight)));
 
 					LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature())));
 					for (Peer peer : peersSharingCommonBlock) {
@@ -706,7 +402,7 @@ public class Synchronizer extends Thread {
 						LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock));
 						BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, maxHeightForChainWeightComparisons);
 						peer.getCommonBlockData().setChainWeight(peerChainWeight);
-						LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), accurateFormatter.format(peerChainWeight)));
+						LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight)));
 
 						// Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group)
 						if (ourChainWeight.compareTo(peerChainWeight) > 0) {
@@ -793,71 +489,6 @@ public class Synchronizer extends Thread {
 	}
 
 
-
-	/* Invalid block signature tracking */
-
-	private void addInvalidBlockSignature(byte[] signature) {
-		Long now = NTP.getTime();
-		if (now == null) {
-			return;
-		}
-
-		// Add or update existing entry
-		String sig58 = Base58.encode(signature);
-		invalidBlockSignatures.put(sig58, now);
-	}
-	private void deleteOlderInvalidSignatures(Long now) {
-		if (now == null) {
-			return;
-		}
-
-		// Delete signatures with older timestamps
-		Iterator it = invalidBlockSignatures.entrySet().iterator();
-		while (it.hasNext()) {
-			Map.Entry pair = (Map.Entry)it.next();
-			Long lastSeen = (Long) pair.getValue();
-
-			// Remove signature if we haven't seen it for more than 1 hour
-			if (now - lastSeen > 60 * 60 * 1000L) {
-				it.remove();
-			}
-		}
-	}
-	private boolean containsInvalidBlockSummary(List<BlockSummaryData> blockSummaries) {
-		if (blockSummaries == null || invalidBlockSignatures == null) {
-			return false;
-		}
-
-		// Loop through our known invalid blocks and check each one against supplied block summaries
-		for (String invalidSignature58 : invalidBlockSignatures.keySet()) {
-			byte[] invalidSignature = Base58.decode(invalidSignature58);
-			for (BlockSummaryData blockSummary : blockSummaries) {
-				byte[] signature = blockSummary.getSignature();
-				if (Arrays.equals(signature, invalidSignature)) {
-					return true;
-				}
-			}
-		}
-		return false;
-	}
-	private boolean containsInvalidBlockSignature(List<byte[]> blockSignatures) {
-		if (blockSignatures == null || invalidBlockSignatures == null) {
-			return false;
-		}
-
-		// Loop through our known invalid blocks and check each one against supplied block signatures
-		for (String invalidSignature58 : invalidBlockSignatures.keySet()) {
-			byte[] invalidSignature = Base58.decode(invalidSignature58);
-			for (byte[] signature : blockSignatures) {
-				if (Arrays.equals(signature, invalidSignature)) {
-					return true;
-				}
-			}
-		}
-		return false;
-	}
-
-
 	/**
 	 * Attempt to synchronize blockchain with peer.
 	 * <p>
@@ -872,11 +503,9 @@ public class Synchronizer extends Thread {
 		// Make sure we're the only thread modifying the blockchain
 		// If we're already synchronizing with another peer then this will also return fast
 		ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
-		if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) {
+		if (!blockchainLock.tryLock())
 			// Wasn't peer's fault we couldn't sync
-			LOGGER.info("Synchronizer couldn't acquire blockchain lock");
 			return SynchronizationResult.NO_BLOCKCHAIN_LOCK;
-		}
 
 		try {
 			try (final Repository repository = RepositoryManager.getRepository()) {
@@ -892,20 +521,20 @@ public class Synchronizer extends Thread {
 					String syncString = String.format("Synchronizing with peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer,
 							peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(),
 							ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp());
-					LOGGER.info(syncString);
+
+					// If our latest block is very old, we should log that we're attempting to sync with a peer
+					// Otherwise, it can appear as though nothing is happening for a while after launch
+					final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
+					if (minLatestBlockTimestamp != null && ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
+						LOGGER.info(syncString);
+					}
+					else {
+						LOGGER.debug(syncString);
+					}
 
 					// Reset last re-org size as we are starting a new sync round
 					this.lastReorgSize = 0;
 
-					// Set the initial value of timeValidBlockLastReceived if it's null
-					Long now = NTP.getTime();
-					if (this.timeValidBlockLastReceived == null) {
-						this.timeValidBlockLastReceived = now;
-					}
-
-					// Delete invalid signatures with older timestamps
-					this.deleteOlderInvalidSignatures(now);
-
 					List<BlockSummaryData> peerBlockSummaries = new ArrayList<>();
 					SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, force, peerBlockSummaries, true);
 					if (findCommonBlockResult != SynchronizationResult.OK) {
@@ -1096,7 +725,7 @@ public class Synchronizer extends Thread {
 			return SynchronizationResult.REPOSITORY_ISSUE;
 
 		if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
-			LOGGER.info(String.format("Ditching our chain after height %d", commonBlockHeight));
+			LOGGER.info(String.format("Ditching our chain after height %d as our latest block is very old", commonBlockHeight));
 		} else {
 			// Compare chain weights
 
@@ -1156,9 +785,8 @@ public class Synchronizer extends Thread {
 			BigInteger ourChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight);
 			BigInteger peerChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, peerBlockSummaries, mutualHeight);
 
-			NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
-			LOGGER.debug(String.format("commonBlockHeight: %d, commonBlockSig: %.8s, ourBlockSummaries.size(): %d, peerBlockSummaries.size(): %d", commonBlockHeight, Base58.encode(commonBlockSig), ourBlockSummaries.size(), peerBlockSummaries.size()));
-			LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", accurateFormatter.format(ourChainWeight), accurateFormatter.format(peerChainWeight)));
+			NumberFormat formatter = new DecimalFormat("0.###E0");
+			LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", formatter.format(ourChainWeight), formatter.format(peerChainWeight)));
 
 			// If our blockchain has greater weight then don't synchronize with peer
 			if (ourChainWeight.compareTo(peerChainWeight) >= 0) {
@@ -1255,12 +883,6 @@ public class Synchronizer extends Thread {
 				break;
 			}
 
-			// Catch a block with an invalid signature before orphaning, so that we retain our existing valid candidate
-			if (this.containsInvalidBlockSignature(peerBlockSignatures)) {
-				LOGGER.info(String.format("Peer %s sent invalid block signature: %.8s", peer, Base58.encode(latestPeerSignature)));
-				return SynchronizationResult.INVALID_DATA;
-			}
-
 			byte[] nextPeerSignature = peerBlockSignatures.get(0);
 			int nextHeight = height + 1;
 
@@ -1363,20 +985,13 @@ public class Synchronizer extends Thread {
 			if (Controller.isStopping())
 				return SynchronizationResult.SHUTTING_DOWN;
 
-			newBlock.preProcess();
-
 			ValidationResult blockResult = newBlock.isValid();
 			if (blockResult != ValidationResult.OK) {
 				LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer,
 						newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getSignature()), blockResult.name()));
-				this.addInvalidBlockSignature(newBlock.getSignature());
-				this.timeInvalidBlockLastReceived = NTP.getTime();
 				return SynchronizationResult.INVALID_DATA;
 			}
 
-			// Block is valid
-			this.timeValidBlockLastReceived = NTP.getTime();
-
 			// Save transactions attached to this block
 			for (Transaction transaction : newBlock.getTransactions()) {
 				TransactionData transactionData = transaction.getTransactionData();
@@ -1398,7 +1013,107 @@ public class Synchronizer extends Thread {
 
 	private SynchronizationResult applyNewBlocks(Repository repository, BlockData commonBlockData, int ourInitialHeight,
 												 Peer peer, int peerHeight, List<BlockSummaryData> peerBlockSummaries) throws InterruptedException, DataException {
-		LOGGER.debug(String.format("Fetching new blocks from peer %s", peer));
+
+		final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
+		if (Settings.getInstance().isFastSyncEnabled() && peer.getPeersVersion() >= PEER_VERSION_160 && ourLatestBlockData.isTrimmed())
+			// This peer supports syncing multiple blocks at once via GetBlocksMessage, and it is enabled in the settings
+			return this.applyNewBlocksUsingFastSync(repository, commonBlockData, ourInitialHeight, peer, peerHeight, peerBlockSummaries);
+		else
+			// Older peer version, or fast sync is disabled in the settings - use slow sync
+			return this.applyNewBlocksUsingSlowSync(repository, commonBlockData, ourInitialHeight, peer, peerHeight, peerBlockSummaries);
+
+	}
+
+	private SynchronizationResult applyNewBlocksUsingFastSync(Repository repository, BlockData commonBlockData, int ourInitialHeight,
+												 Peer peer, int peerHeight, List<BlockSummaryData> peerBlockSummaries) throws InterruptedException, DataException {
+		LOGGER.debug(String.format("Fetching new blocks from peer %s using fast sync", peer));
+
+		final int commonBlockHeight = commonBlockData.getHeight();
+		final byte[] commonBlockSig = commonBlockData.getSignature();
+		byte[] latestPeerSignature = commonBlockSig;
+
+		int ourHeight = ourInitialHeight;
+
+		// Fetch, and apply, blocks from peer
+		int maxBatchHeight = commonBlockHeight + SYNC_BATCH_SIZE;
+
+		// Ensure that we don't request more blocks than specified in the settings
+		int maxBlocksPerRequest = Settings.getInstance().getMaxBlocksPerRequest();
+
+		while (ourHeight < peerHeight && ourHeight < maxBatchHeight) {
+			if (Controller.isStopping())
+				return SynchronizationResult.SHUTTING_DOWN;
+
+			int numberRequested = Math.min(maxBatchHeight - ourHeight, maxBlocksPerRequest);
+
+			LOGGER.trace(String.format("Fetching %d blocks after height %d, sig %.8s from %s", numberRequested, ourHeight, Base58.encode(latestPeerSignature), peer));
+			List<Block> blocks = this.fetchBlocks(repository, peer, latestPeerSignature, numberRequested);
+			if (blocks == null || blocks.isEmpty()) {
+				LOGGER.info(String.format("Peer %s failed to respond with more blocks after height %d, sig %.8s", peer,
+						ourHeight, Base58.encode(latestPeerSignature)));
+				return SynchronizationResult.NO_REPLY;
+			}
+			LOGGER.trace(String.format("Received %d blocks after height %d, sig %.8s from %s", blocks.size(), ourHeight, Base58.encode(latestPeerSignature), peer));
+
+			for (Block newBlock : blocks) {
+				++ourHeight;
+
+				if (Controller.isStopping())
+					return SynchronizationResult.SHUTTING_DOWN;
+
+				if (newBlock == null) {
+					LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer,
+							ourHeight, Base58.encode(latestPeerSignature)));
+					return SynchronizationResult.NO_REPLY;
+				}
+
+				if (!newBlock.isSignatureValid()) {
+					LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer,
+							ourHeight, Base58.encode(latestPeerSignature)));
+					return SynchronizationResult.INVALID_DATA;
+				}
+
+				// Set the repository, because we couldn't do that when originally constructing the Block
+				newBlock.setRepository(repository);
+
+				// Transactions are transmitted without approval status so determine that now
+				for (Transaction transaction : newBlock.getTransactions()) {
+					transaction.setInitialApprovalStatus();
+				}
+
+				ValidationResult blockResult = newBlock.isValid();
+				if (blockResult != ValidationResult.OK) {
+					LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer,
+							ourHeight, Base58.encode(latestPeerSignature), blockResult.name()));
+					return SynchronizationResult.INVALID_DATA;
+				}
+
+				// Save transactions attached to this block
+				for (Transaction transaction : newBlock.getTransactions()) {
+					TransactionData transactionData = transaction.getTransactionData();
+					repository.getTransactionRepository().save(transactionData);
+				}
+
+				newBlock.process();
+
+				LOGGER.trace(String.format("Processed block height %d, sig %.8s", newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature())));
+
+				repository.saveChanges();
+
+				Controller.getInstance().onNewBlock(newBlock.getBlockData());
+
+				// Update latestPeerSignature so that subsequent batches start requesting from the correct block
+				latestPeerSignature = newBlock.getSignature();
+			}
+
+		}
+
+		return SynchronizationResult.OK;
+	}
+
+	private SynchronizationResult applyNewBlocksUsingSlowSync(Repository repository, BlockData commonBlockData, int ourInitialHeight,
+			Peer peer, int peerHeight, List<BlockSummaryData> peerBlockSummaries) throws InterruptedException, DataException {
+		LOGGER.debug(String.format("Fetching new blocks from peer %s using slow sync", peer));
 
 		final int commonBlockHeight = commonBlockData.getHeight();
 		final byte[] commonBlockSig = commonBlockData.getSignature();
@@ -1458,20 +1173,13 @@ public class Synchronizer extends Thread {
 			for (Transaction transaction : newBlock.getTransactions())
 				transaction.setInitialApprovalStatus();
 
-			newBlock.preProcess();
-
 			ValidationResult blockResult = newBlock.isValid();
 			if (blockResult != ValidationResult.OK) {
 				LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer,
 						ourHeight, Base58.encode(latestPeerSignature), blockResult.name()));
-				this.addInvalidBlockSignature(newBlock.getSignature());
-				this.timeInvalidBlockLastReceived = NTP.getTime();
 				return SynchronizationResult.INVALID_DATA;
 			}
 
-			// Block is valid
-			this.timeValidBlockLastReceived = NTP.getTime();
-
 			// Save transactions attached to this block
 			for (Transaction transaction : newBlock.getTransactions()) {
 				TransactionData transactionData = transaction.getTransactionData();
@@ -1526,7 +1234,23 @@ public class Synchronizer extends Thread {
 		return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates());
 	}
 
-	public void populateBlockSummariesMinterLevels(Repository repository, List<BlockSummaryData> blockSummaries) throws DataException {
+	private List<Block> fetchBlocks(Repository repository, Peer peer, byte[] parentSignature, int numberRequested) throws InterruptedException {
+		Message getBlocksMessage = new GetBlocksMessage(parentSignature, numberRequested);
+
+		Message message = peer.getResponseWithTimeout(getBlocksMessage, FETCH_BLOCKS_TIMEOUT);
+		if (message == null || message.getType() != MessageType.BLOCKS) {
+			return null;
+		}
+
+		BlocksMessage blocksMessage = (BlocksMessage) message;
+		if (blocksMessage == null || blocksMessage.getBlocks() == null) {
+			return null;
+		}
+
+		return blocksMessage.getBlocks();
+	}
+
+	private void populateBlockSummariesMinterLevels(Repository repository, List<BlockSummaryData> blockSummaries) throws DataException {
 		final int firstBlockHeight = blockSummaries.get(0).getHeight();
 
 		for (int i = 0; i < blockSummaries.size(); ++i) {

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java

@@ -1,203 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
-import org.qortal.utils.NTP;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-public class ArbitraryDataBuildManager extends Thread {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class);
-
-    private static ArbitraryDataBuildManager instance;
-
-    private volatile boolean isStopping = false;
-    private boolean buildInProgress = false;
-
-    /**
-     * Map to keep track of arbitrary transaction resources currently being built (or queued).
-     */
-    public Map<String, ArbitraryDataBuildQueueItem> arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>());
-
-    /**
-     * Map to keep track of failed arbitrary transaction builds.
-     */
-    public Map<String, ArbitraryDataBuildQueueItem> arbitraryDataFailedBuilds = Collections.synchronizedMap(new HashMap<>());
-
-
-    public ArbitraryDataBuildManager() {
-
-    }
-
-    @Override
-    public void run() {
-        Thread.currentThread().setName("Arbitrary Data Build Manager");
-
-        try {
-            // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
-            // This can be expanded to have multiple threads processing the build queue when needed
-            int threadCount = 5;
-            ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(threadCount);
-            for (int i = 0; i < threadCount; i++) {
-                arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
-            }
-
-            while (!isStopping) {
-                // Nothing to do yet
-                Thread.sleep(5000);
-            }
-
-        } catch (InterruptedException e) {
-            // Fall-through to exit thread...
-        }
-    }
-
-    public static ArbitraryDataBuildManager getInstance() {
-        if (instance == null)
-            instance = new ArbitraryDataBuildManager();
-
-        return instance;
-    }
-
-    public void shutdown() {
-        isStopping = true;
-        this.interrupt();
-    }
-
-
-    public void cleanupQueues(Long now) {
-        if (now == null) {
-            return;
-        }
-        arbitraryDataBuildQueue.entrySet().removeIf(entry -> entry.getValue().hasReachedBuildTimeout(now));
-        arbitraryDataFailedBuilds.entrySet().removeIf(entry -> entry.getValue().hasReachedFailureTimeout(now));
-    }
-
-    // Build queue
-
-    public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
-        String key = queueItem.getUniqueKey();
-        if (key == null) {
-            return false;
-        }
-
-        if (this.arbitraryDataBuildQueue == null) {
-            return false;
-        }
-
-        if (NTP.getTime() == null) {
-            // Can't use queues until we have synced the time
-            return false;
-        }
-
-        // Don't add builds that have failed recently
-        if (this.isInFailedBuildsList(queueItem)) {
-            return false;
-        }
-
-        if (this.arbitraryDataBuildQueue.put(key, queueItem) != null) {
-            // Already in queue
-            return true;
-        }
-
-        log(queueItem, String.format("Added %s to build queue", queueItem));
-
-        // Added to queue
-        return true;
-    }
-
-    public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
-        String key = queueItem.getUniqueKey();
-        if (key == null) {
-            return false;
-        }
-
-        if (this.arbitraryDataBuildQueue == null) {
-            return false;
-        }
-
-        if (this.arbitraryDataBuildQueue.containsKey(key)) {
-            // Already in queue
-            return true;
-        }
-
-        // Not in queue
-        return false;
-    }
-
-
-    // Failed builds
-
-    public boolean addToFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) {
-        String key = queueItem.getUniqueKey();
-        if (key == null) {
-            return false;
-        }
-
-        if (this.arbitraryDataFailedBuilds == null) {
-            return false;
-        }
-
-        if (NTP.getTime() == null) {
-            // Can't use queues until we have synced the time
-            return false;
-        }
-
-        if (this.arbitraryDataFailedBuilds.put(key, queueItem) != null) {
-            // Already in list
-            return true;
-        }
-
-        log(queueItem, String.format("Added %s to failed builds list", queueItem));
-
-        // Added to queue
-        return true;
-    }
-
-    public boolean isInFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) {
-        String key = queueItem.getUniqueKey();
-        if (key == null) {
-            return false;
-        }
-
-        if (this.arbitraryDataFailedBuilds == null) {
-            return false;
-        }
-
-        if (this.arbitraryDataFailedBuilds.containsKey(key)) {
-            // Already in list
-            return true;
-        }
-
-        // Not in list
-        return false;
-    }
-
-
-    public void setBuildInProgress(boolean buildInProgress) {
-        this.buildInProgress = buildInProgress;
-    }
-
-    public boolean getBuildInProgress() {
-        return this.buildInProgress;
-    }
-
-    private void log(ArbitraryDataBuildQueueItem queueItem, String message) {
-        if (queueItem == null) {
-            return;
-        }
-
-        if (queueItem.isHighPriority()) {
-            LOGGER.info(message);
-        }
-        else {
-            LOGGER.debug(message);
-        }
-    }
-}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

@@ -1,122 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
-import org.qortal.arbitrary.exception.MissingDataException;
-import org.qortal.controller.Controller;
-import org.qortal.repository.DataException;
-import org.qortal.utils.NTP;
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.Map;
-
-
-public class ArbitraryDataBuilderThread implements Runnable {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuilderThread.class);
-
-    public ArbitraryDataBuilderThread() {
-
-    }
-
-    @Override
-    public void run() {
-        Thread.currentThread().setName("Arbitrary Data Builder Thread");
-        ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance();
-
-        while (!Controller.isStopping()) {
-            try {
-                Thread.sleep(100);
-
-                if (buildManager.arbitraryDataBuildQueue == null) {
-                    continue;
-                }
-                if (buildManager.arbitraryDataBuildQueue.isEmpty()) {
-                    continue;
-                }
-
-                Long now = NTP.getTime();
-                if (now == null) {
-                    continue;
-                }
-
-                ArbitraryDataBuildQueueItem queueItem = null;
-
-                // Find resources that are queued for building (sorted by highest priority first)
-                synchronized (buildManager.arbitraryDataBuildQueue) {
-                    Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
-                            .entrySet().stream()
-                            .filter(e -> e.getValue().isQueued())
-                            .sorted(Comparator.comparing(item -> item.getValue().getPriority()))
-                            .reduce((first, second) -> second).orElse(null);
-
-                    if (next == null) {
-                        continue;
-                    }
-
-                    queueItem = next.getValue();
-
-                    if (queueItem == null) {
-                        this.removeFromQueue(queueItem);
-                        continue;
-                    }
-
-                    // Ignore builds that have failed recently
-                    if (buildManager.isInFailedBuildsList(queueItem)) {
-                        this.removeFromQueue(queueItem);
-                        continue;
-                    }
-
-                    // Set the start timestamp, to prevent other threads from building it at the same time
-                    queueItem.prepareForBuild();
-                }
-
-                try {
-                    // Perform the build
-                    log(queueItem, String.format("Building %s... priority: %d", queueItem, queueItem.getPriority()));
-                    queueItem.build();
-                    this.removeFromQueue(queueItem);
-                    log(queueItem, String.format("Finished building %s", queueItem));
-
-                } catch (MissingDataException e) {
-                    log(queueItem, String.format("Missing data for %s: %s", queueItem, e.getMessage()));
-                    queueItem.setFailed(true);
-                    this.removeFromQueue(queueItem);
-                    // Don't add to the failed builds list, as we may want to retry sooner
-
-                } catch (IOException | DataException | RuntimeException e) {
-                    log(queueItem, String.format("Error building %s: %s", queueItem, e.getMessage()));
-                    // Something went wrong - so remove it from the queue, and add to failed builds list
-                    queueItem.setFailed(true);
-                    buildManager.addToFailedBuildsList(queueItem);
-                    this.removeFromQueue(queueItem);
-                }
-
-            } catch (InterruptedException e) {
-                // Time to exit
-            }
-        }
-    }
-
-    private void removeFromQueue(ArbitraryDataBuildQueueItem queueItem) {
-        if (queueItem == null || queueItem.getUniqueKey() == null) {
-            return;
-        }
-        ArbitraryDataBuildManager.getInstance().arbitraryDataBuildQueue.remove(queueItem.getUniqueKey());
-    }
-
-    private void log(ArbitraryDataBuildQueueItem queueItem, String message) {
-        if (queueItem == null) {
-            return;
-        }
-
-        if (queueItem.isHighPriority()) {
-            LOGGER.info(message);
-        }
-        else {
-            LOGGER.debug(message);
-        }
-    }
-}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java

@@ -1,576 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.data.transaction.TransactionData;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.settings.Settings;
-import org.qortal.transaction.Transaction;
-import org.qortal.transaction.Transaction.TransactionType;
-import org.qortal.utils.ArbitraryTransactionUtils;
-import org.qortal.utils.Base58;
-import org.qortal.utils.FilesystemUtils;
-import org.qortal.utils.NTP;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.SecureRandom;
-import java.util.*;
-
-import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD;
-
-public class ArbitraryDataCleanupManager extends Thread {
-
-	private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCleanupManager.class);
-	private static final List<TransactionType> ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY);
-
-	private static ArbitraryDataCleanupManager instance;
-
-	private volatile boolean isStopping = false;
-
-	/**
-	 * The amount of time that must pass before a file is treated as stale / not recent.
-	 * We can safely delete files created/accessed longer ago that this, if we have a means of
-	 * rebuilding them. The main purpose of this is to avoid deleting files that are currently
-	 * being used by other parts of the system.
-	 */
-	private static final long STALE_FILE_TIMEOUT = 60*60*1000L; // 1 hour
-
-	/**
-	 * The number of chunks to delete in a batch when over the capacity limit.
-	 * Storage limits are re-checked after each batch, and there could be a significant
-	 * delay between the processing of each batch as it only occurs after a complete
-	 * cleanup cycle (to allow unwanted chunks to be deleted first).
-	 */
-	private static final int CHUNK_DELETION_BATCH_SIZE = 10;
-
-
-	/*
-	TODO:
-	- Delete files from the _misc folder once they reach a certain age
-	 */
-
-
-	private ArbitraryDataCleanupManager() {
-	}
-
-	public static ArbitraryDataCleanupManager getInstance() {
-		if (instance == null)
-			instance = new ArbitraryDataCleanupManager();
-
-		return instance;
-	}
-
-	@Override
-	public void run() {
-		Thread.currentThread().setName("Arbitrary Data Cleanup Manager");
-
-		// Paginate queries when fetching arbitrary transactions
-		final int limit = 100;
-		int offset = 0;
-
-		try {
-			while (!isStopping) {
-				Thread.sleep(30000);
-
-				// Don't run if QDN is disabled
-				if (!Settings.getInstance().isQdnEnabled()) {
-					Thread.sleep(60 * 60 * 1000L);
-					continue;
-				}
-
-				Long now = NTP.getTime();
-				if (now == null) {
-					// Don't attempt to make decisions if we haven't synced our time yet
-					continue;
-				}
-
-				ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
-
-				// Wait until storage capacity has been calculated
-				if (!storageManager.isStorageCapacityCalculated()) {
-					continue;
-				}
-
-				// Periodically delete any unnecessary files from the temp directory
-				if (offset == 0 || offset % (limit * 10) == 0) {
-					this.cleanupTempDirectory(now);
-				}
-
-				// Any arbitrary transactions we want to fetch data for?
-				try (final Repository repository = RepositoryManager.getRepository()) {
-					List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
-					// LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
-					if (isStopping) {
-						return;
-					}
-
-					if (signatures == null || signatures.isEmpty()) {
-						offset = 0;
-						continue;
-					}
-					offset += limit;
-					now = NTP.getTime();
-
-					// Loop through the signatures in this batch
-					for (int i=0; i<signatures.size(); i++) {
-						if (isStopping) {
-							return;
-						}
-
-						byte[] signature = signatures.get(i);
-						if (signature == null) {
-							continue;
-						}
-
-						// Don't interfere with the filesystem whilst a build is in progress
-						if (ArbitraryDataBuildManager.getInstance().getBuildInProgress()) {
-							Thread.sleep(5000);
-						}
-
-						// Fetch the transaction data
-						ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
-						if (arbitraryTransactionData == null) {
-							continue;
-						}
-
-						// Raw data doesn't have any associated files to clean up
-						if (arbitraryTransactionData.getDataType() == ArbitraryTransactionData.DataType.RAW_DATA) {
-							continue;
-						}
-
-						// Check if we have the complete file
-						boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData);
-
-						// Check if we have any of the chunks
-						boolean anyChunksExist = ArbitraryTransactionUtils.anyChunksExist(arbitraryTransactionData);
-						boolean transactionHasChunks = (arbitraryTransactionData.getMetadataHash() != null);
-
-						if (!completeFileExists && !anyChunksExist) {
-							// We don't have any files at all for this transaction - nothing to do
-							continue;
-						}
-
-						// We have at least 1 chunk or file for this transaction, so we might need to delete them...
-
-
-						// Check to see if we should be hosting data for this transaction at all
-						if (!storageManager.canStoreData(arbitraryTransactionData)) {
-							LOGGER.info("Deleting transaction {} because we can't host its data",
-									Base58.encode(arbitraryTransactionData.getSignature()));
-							ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
-							continue;
-						}
-
-						// Check to see if we have had a more recent PUT
-						boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
-						if (hasMoreRecentPutTransaction) {
-							// There is a more recent PUT transaction than the one we are currently processing.
-							// When a PUT is issued, it replaces any layers that would have been there before.
-							// Therefore any data relating to this older transaction is no longer needed.
-							LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " +
-											"Deleting all files associated with the earlier transaction.", arbitraryTransactionData.getService(),
-									arbitraryTransactionData.getName(), Base58.encode(signature)));
-
-							ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
-
-							// We should also remove peers for this transaction from the lookup table to save space
-							this.removePeersHostingTransactionData(repository, arbitraryTransactionData);
-							continue;
-						}
-
-						if (completeFileExists && !transactionHasChunks) {
-							// This file doesn't have any chunks because it is too small.
-							// We must not delete anything.
-							continue;
-						}
-
-						// Check if we have all of the chunks
-						boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData);
-
-						if (completeFileExists && allChunksExist) {
-							// We have the complete file and all the chunks, so we can delete
-							// the complete file if it has reached a certain age.
-							LOGGER.debug(String.format("Transaction %s has complete file and all chunks",
-									Base58.encode(arbitraryTransactionData.getSignature())));
-
-							ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
-							continue;
-						}
-
-						if (completeFileExists && !allChunksExist) {
-							// We have the complete file but not the chunks, so let's convert it
-							LOGGER.info(String.format("Transaction %s has complete file but no chunks",
-									Base58.encode(arbitraryTransactionData.getSignature())));
-
-							ArbitraryTransactionUtils.convertFileToChunks(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
-							continue;
-						}
-					}
-
-				} catch (DataException e) {
-					LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
-				}
-
-				try (final Repository repository = RepositoryManager.getRepository()) {
-
-					// Check if there are any hosted files that don't have matching transactions
-					this.checkForExpiredTransactions(repository);
-
-					// Delete additional data at random if we're over our storage limit
-					// Use the DELETION_THRESHOLD so that we only start deleting once the hard limit is reached
-					// This also allows some headroom between the regular threshold (90%) and the hard
-					// limit, to avoid data getting into a fetch/delete loop.
-					if (!storageManager.isStorageSpaceAvailable(DELETION_THRESHOLD)) {
-
-						// Rate limit, to avoid repeated calls to calculateDirectorySize()
-						Thread.sleep(60000);
-						// Now delete some data at random
-						this.storageLimitReached(repository);
-					}
-
-					// Delete random data associated with name if we're over our storage limit for this name
-					// Use the DELETION_THRESHOLD, for the same reasons as above
-					for (String followedName : storageManager.followedNames()) {
-						if (isStopping) {
-							return;
-						}
-						if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) {
-							this.storageLimitReachedForName(repository, followedName);
-						}
-					}
-
-				} catch (DataException e) {
-					LOGGER.error("Repository issue when cleaning up arbitrary transaction data", e);
-				}
-			}
-		} catch (InterruptedException e) {
-			// Fall-through to exit thread...
-		}
-	}
-
-	public List<Path> findPathsWithNoAssociatedTransaction(Repository repository) {
-		List<Path> pathList = new ArrayList<>();
-
-		// Find all hosted paths
-		List<Path> allPaths = ArbitraryDataStorageManager.getInstance().findAllHostedPaths();
-
-		// Loop through each path and find those without matching signatures
-		for (Path path : allPaths) {
-			if (isStopping) {
-				break;
-			}
-			try {
-				String[] contents = path.toFile().list();
-				if (contents == null || contents.length == 0) {
-					// Ignore empty directories
-					continue;
-				}
-
-				String signature58 = path.getFileName().toString();
-				byte[] signature = Base58.decode(signature58);
-				TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-				if (transactionData == null) {
-					// No transaction data, and no DataException, so we can assume that this data relates to an expired transaction
-					pathList.add(path);
-				}
-
-			} catch (DataException e) {
-				continue;
-			}
-		}
-
-		return pathList;
-	}
-
-	private void checkForExpiredTransactions(Repository repository) {
-		List<Path> expiredPaths = this.findPathsWithNoAssociatedTransaction(repository);
-		for (Path expiredPath : expiredPaths) {
-			if (isStopping) {
-				return;
-			}
-			LOGGER.info("Found path with no associated transaction: {}", expiredPath.toString());
-			this.safeDeleteDirectory(expiredPath.toFile(), "no matching transaction");
-		}
-	}
-
-	private void storageLimitReached(Repository repository) throws InterruptedException {
-		// We think that the storage limit has been reached
-
-		// Now calculate the used/total storage again, as a safety precaution
-		Long now = NTP.getTime();
-		ArbitraryDataStorageManager.getInstance().calculateDirectorySize(now);
-		if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailable(DELETION_THRESHOLD)) {
-			// We have space available, so don't delete anything
-			return;
-		}
-
-		// Delete a batch of random chunks
-		// This reduces the chance of too many nodes deleting the same chunk
-		// when they reach their storage limit
-		Path dataPath = Paths.get(Settings.getInstance().getDataPath());
-		for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
-			if (isStopping) {
-				return;
-			}
-			this.deleteRandomFile(repository, dataPath.toFile(), null);
-		}
-
-		// FUTURE: consider reducing the expiry time of the reader cache
-	}
-
-	public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException {
-		// We think that the storage limit has been reached for supplied name - but we should double check
-		if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) {
-			// We have space available for this name, so don't delete anything
-			return;
-		}
-
-		// Delete a batch of random chunks associated with this name
-		// This reduces the chance of too many nodes deleting the same chunk
-		// when they reach their storage limit
-		Path dataPath = Paths.get(Settings.getInstance().getDataPath());
-		for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
-			if (isStopping) {
-				return;
-			}
-			this.deleteRandomFile(repository, dataPath.toFile(), name);
-		}
-	}
-
-	/**
-	 * Iteratively walk through given directory and delete a single random file
-	 *
-	 * @param directory - the base directory
-	 * @return boolean - whether a file was deleted
-	 */
-	private boolean deleteRandomFile(Repository repository, File directory, String name) {
-		Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath());
-
-		// Pick a random directory
-		final File[] contentsList = directory.listFiles();
-		if (contentsList != null) {
-			SecureRandom random = new SecureRandom();
-
-			// If the directory is empty, there's nothing to do
-			if (contentsList.length == 0) {
-				return false;
-			}
-
-			File randomItem = contentsList[random.nextInt(contentsList.length)];
-
-			// Skip anything relating to the temp directory
-			if (FilesystemUtils.isChild(randomItem.toPath(), tempDataPath)) {
-				return false;
-			}
-			// Make sure it exists
-			if (!randomItem.exists()) {
-				return false;
-			}
-
-			// If it's a directory, iteratively repeat the process
-			if (randomItem.isDirectory()) {
-				return this.deleteRandomFile(repository, randomItem, name);
-			}
-
-			// If it's a file, we might be able to delete it
-			if (randomItem.isFile()) {
-
-				// If the parent directory contains an ".original" file, don't delete anything
-				// This indicates that the content was originally updated by this node and so
-				// could be the only copy that exists.
-				Path originalCopyIndicatorPath = Paths.get(randomItem.getParent(), ".original");
-				if (Files.exists(originalCopyIndicatorPath)) {
-					// This is an original seed copy and so shouldn't be deleted
-					return false;
-				}
-
-				if (name != null) {
-					// A name has been specified, so we need to make sure this file relates to
-					// the name we want to delete. The signature should be the name of parent directory.
-					try {
-						Path parentFileNamePath = randomItem.toPath().toAbsolutePath().getParent().getFileName();
-						if (parentFileNamePath != null) {
-							String signature58 = parentFileNamePath.toString();
-							byte[] signature = Base58.decode(signature58);
-							TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-							if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) {
-								// Not what we were expecting, so don't delete it
-								return false;
-							}
-							ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
-							if (!Objects.equals(arbitraryTransactionData.getName(), name)) {
-								// Relates to a different name - don't delete it
-								return false;
-							}
-						}
-
-					} catch (DataException e) {
-						// Something went wrong and we weren't able to make a decision - so it's best not to delete this file
-						return false;
-					}
-				}
-
-				LOGGER.info("Deleting random file {} because we have reached max storage capacity...", randomItem.toString());
-				boolean success = randomItem.delete();
-				if (success) {
-					try {
-						FilesystemUtils.safeDeleteEmptyParentDirectories(randomItem.toPath().getParent());
-					} catch (IOException e) {
-						// Ignore cleanup failure
-					}
-				}
-				return success;
-			}
-		}
-		return false;
-	}
-
-	private void removePeersHostingTransactionData(Repository repository, ArbitraryTransactionData transactionData) {
-		byte[] signature = transactionData.getSignature();
-		try {
-			repository.getArbitraryRepository().deleteArbitraryPeersWithSignature(signature);
-			repository.saveChanges();
-		} catch (DataException e) {
-			LOGGER.debug("Unable to delete peers from lookup table for signature: {}", Base58.encode(signature));
-		}
-	}
-
-	private void cleanupTempDirectory(String folder, long now, long minAge) {
-		String baseDir = Settings.getInstance().getTempDataPath();
-		Path tempDir = Paths.get(baseDir, folder);
-		int contentsCount = 0;
-
-		// Loop through the contents and check each one
-		final File[] directories = tempDir.toFile().listFiles();
-		if (directories != null) {
-			for (final File directory : directories) {
-				if (isStopping) {
-					return;
-				}
-				contentsCount++;
-
-				// We're expecting the contents of each subfolder to be a directory
-				if (directory.isDirectory()) {
-					if (!ArbitraryTransactionUtils.isFileRecent(directory.toPath(), now, minAge)) {
-						// File isn't recent, so can be deleted
-						this.safeDeleteDirectory(directory, "not recent");
-					}
-				}
-			}
-		}
-
-		// If the directory is empty, we still need to delete its parent folder
-		if (contentsCount == 0 && tempDir.toFile().isDirectory() && tempDir.toFile().exists()) {
-			try {
-				LOGGER.debug("Parent directory {} is empty, so deleting it", tempDir);
-				FilesystemUtils.safeDeleteDirectory(tempDir, false);
-			} catch(IOException e){
-				LOGGER.info("Unable to delete parent directory: {}", tempDir);
-			}
-		}
-	}
-
-	private void cleanupReaderCache(Long now) {
-		ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
-		String baseDir = Settings.getInstance().getTempDataPath();
-		Path readerCachePath = Paths.get(baseDir, "reader");
-
-		// Clean up names
-		Path readerCacheNamesPath = Paths.get(readerCachePath.toString(), "NAME");
-
-		// Loop through the contents and check each one
-		final File[] directories = readerCacheNamesPath.toFile().listFiles();
-		if (directories != null) {
-			for (final File directory : directories) {
-				if (isStopping) {
-					return;
-				}
-
-				// Delete data relating to blocked names
-				String name = directory.getName();
-				if (name != null && storageManager.isNameBlocked(name)) {
-					this.safeDeleteDirectory(directory, "blocked name");
-				}
-
-				// Delete cached reader data that has reached its expiry
-				this.cleanupReaderCacheForName(name, now);
-			}
-		}
-	}
-
-	private void cleanupReaderCacheForName(String name, Long now) {
-		if (name == null) {
-			return;
-		}
-
-		String baseDir = Settings.getInstance().getTempDataPath();
-		Path readerNameCachePath = Paths.get(baseDir, "reader", "NAME", name);
-
-		// Loop through the contents and check each one
-		final File[] directories = readerNameCachePath.toFile().listFiles();
-		if (directories != null) {
-			for (final File directory : directories) {
-				if (isStopping) {
-					return;
-				}
-				// Each directory is a "service" type
-				String service = directory.getName();
-				this.cleanupReaderCacheForNameAndService(name, service, now);
-			}
-		}
-	}
-
-	private void cleanupReaderCacheForNameAndService(String name, String service, Long now) {
-		if (name == null || service == null) {
-			return;
-		}
-
-		Path readerNameServiceCachePath = Paths.get("reader", "NAME", name, service);
-		Long expiry = Settings.getInstance().getBuiltDataExpiryInterval();
-		this.cleanupTempDirectory(readerNameServiceCachePath.toString(), now, expiry);
-	}
-
-	private void cleanupTempDirectory(long now) {
-
-		// Use the "stale file timeout" for the intermediate directories.
-		// These aren't used for serving content - only for building it.
-		// Once the files have become stale, it's safe to delete them.
-		this.cleanupTempDirectory("diff",  now, STALE_FILE_TIMEOUT);
-		this.cleanupTempDirectory("join",  now, STALE_FILE_TIMEOUT);
-		this.cleanupTempDirectory("merge",  now, STALE_FILE_TIMEOUT);
-		this.cleanupTempDirectory("writer",  now, STALE_FILE_TIMEOUT);
-
-		// Built resources are served out of the "reader" directory so these
-		// need to be kept around for much longer.
-		// Purging currently disabled, as it's not very helpful. Will revisit
-		// once we implement local storage limits.
-		this.cleanupReaderCache(now);
-
-	}
-
-	private boolean safeDeleteDirectory(File directory, String reason) {
-		LOGGER.info("Deleting directory {} due to reason: {}", directory, reason);
-		try {
-			FilesystemUtils.safeDeleteDirectory(directory.toPath(), true);
-			return true;
-		} catch (IOException e) {
-			LOGGER.debug("Unable to delete directory: {}", directory);
-		}
-		return false;
-	}
-
-
-	public void shutdown() {
-		isStopping = true;
-		this.interrupt();
-	}
-
-}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java

@@ -1,634 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.ArbitraryDataFile;
-import org.qortal.arbitrary.ArbitraryDataFileChunk;
-import org.qortal.controller.Controller;
-import org.qortal.data.arbitrary.ArbitraryRelayInfo;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.data.transaction.TransactionData;
-import org.qortal.network.Network;
-import org.qortal.network.Peer;
-import org.qortal.network.message.ArbitraryDataFileListMessage;
-import org.qortal.network.message.GetArbitraryDataFileListMessage;
-import org.qortal.network.message.Message;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.settings.Settings;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-import org.qortal.utils.Triple;
-
-import java.util.*;
-
-public class ArbitraryDataFileListManager {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileListManager.class);
-
-    private static ArbitraryDataFileListManager instance;
-
-
-    /**
-     * Map of recent incoming requests for ARBITRARY transaction data file lists.
-     * <p>
-     * Key is original request's message ID<br>
-     * Value is Triple&lt;transaction signature in base58, first requesting peer, first request's timestamp&gt;
-     * <p>
-     * If peer is null then either:<br>
-     * <ul>
-     * <li>we are the original requesting peer</li>
-     * <li>we have already sent data payload to original requesting peer.</li>
-     * </ul>
-     * If signature is null then we have already received the file list and either:<br>
-     * <ul>
-     * <li>we are the original requesting peer and have processed it</li>
-     * <li>we have forwarded the file list</li>
-     * </ul>
-     */
-    public Map<Integer, Triple<String, Peer, Long>> arbitraryDataFileListRequests = Collections.synchronizedMap(new HashMap<>());
-
-    /**
-     * Map to keep track of in progress arbitrary data signature requests
-     * Key: string - the signature encoded in base58
-     * Value: Triple<networkBroadcastCount, directPeerRequestCount, lastAttemptTimestamp>
-     */
-    private Map<String, Triple<Integer, Integer, Long>> arbitraryDataSignatureRequests = Collections.synchronizedMap(new HashMap<>());
-
-
-    /** Maximum number of seconds that a file list relay request is able to exist on the network */
-    private static long RELAY_REQUEST_MAX_DURATION = 5000L;
-    /** Maximum number of hops that a file list relay request is allowed to make */
-    private static int RELAY_REQUEST_MAX_HOPS = 4;
-
-
-    private ArbitraryDataFileListManager() {
-    }
-
-    public static ArbitraryDataFileListManager getInstance() {
-        if (instance == null)
-            instance = new ArbitraryDataFileListManager();
-
-        return instance;
-    }
-
-
-    public void cleanupRequestCache(Long now) {
-        if (now == null) {
-            return;
-        }
-        final long requestMinimumTimestamp = now - ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT;
-        arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < requestMinimumTimestamp);
-    }
-
-
-    // Track file list lookups by signature
-
-    private boolean shouldMakeFileListRequestForSignature(String signature58) {
-        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
-
-        if (request == null) {
-            // Not attempted yet
-            return true;
-        }
-
-        // Extract the components
-        Integer networkBroadcastCount = request.getA();
-        // Integer directPeerRequestCount = request.getB();
-        Long lastAttemptTimestamp = request.getC();
-
-        if (lastAttemptTimestamp == null) {
-            // Not attempted yet
-            return true;
-        }
-
-        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
-
-        // Allow a second attempt after 15 seconds, and another after 30 seconds
-        if (timeSinceLastAttempt > 15 * 1000L) {
-            // We haven't tried for at least 15 seconds
-
-            if (networkBroadcastCount < 3) {
-                // We've made less than 3 total attempts
-                return true;
-            }
-        }
-
-        // Then allow another 5 attempts, each 5 minutes apart
-        if (timeSinceLastAttempt > 5 * 60 * 1000L) {
-            // We haven't tried for at least 5 minutes
-
-            if (networkBroadcastCount < 5) {
-                // We've made less than 5 total attempts
-                return true;
-            }
-        }
-
-        // From then on, only try once every 24 hours, to reduce network spam
-        if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) {
-            // We haven't tried for at least 24 hours
-            return true;
-        }
-
-        return false;
-    }
-
-    private boolean shouldMakeDirectFileRequestsForSignature(String signature58) {
-        if (!Settings.getInstance().isDirectDataRetrievalEnabled()) {
-            // Direct connections are disabled in the settings
-            return false;
-        }
-
-        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
-
-        if (request == null) {
-            // Not attempted yet
-            return true;
-        }
-
-        // Extract the components
-        //Integer networkBroadcastCount = request.getA();
-        Integer directPeerRequestCount = request.getB();
-        Long lastAttemptTimestamp = request.getC();
-
-        if (lastAttemptTimestamp == null) {
-            // Not attempted yet
-            return true;
-        }
-
-        if (directPeerRequestCount == 0) {
-            // We haven't tried asking peers directly yet, so we should
-            return true;
-        }
-
-        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
-        if (timeSinceLastAttempt > 10 * 1000L) {
-            // We haven't tried for at least 10 seconds
-            if (directPeerRequestCount < 5) {
-                // We've made less than 5 total attempts
-                return true;
-            }
-        }
-
-        if (timeSinceLastAttempt > 5 * 60 * 1000L) {
-            // We haven't tried for at least 5 minutes
-            if (directPeerRequestCount < 10) {
-                // We've made less than 10 total attempts
-                return true;
-            }
-        }
-
-        if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) {
-            // We haven't tried for at least 24 hours
-            return true;
-        }
-
-        return false;
-    }
-
-    public boolean isSignatureRateLimited(byte[] signature) {
-        String signature58 = Base58.encode(signature);
-        return !this.shouldMakeFileListRequestForSignature(signature58)
-                && !this.shouldMakeDirectFileRequestsForSignature(signature58);
-    }
-
-    public long lastRequestForSignature(byte[] signature) {
-        String signature58 = Base58.encode(signature);
-        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
-
-        if (request == null) {
-            // Not attempted yet
-            return 0;
-        }
-
-        // Extract the components
-        Long lastAttemptTimestamp = request.getC();
-        if (lastAttemptTimestamp != null) {
-            return  lastAttemptTimestamp;
-        }
-        return 0;
-    }
-
-    public void addToSignatureRequests(String signature58, boolean incrementNetworkRequests, boolean incrementPeerRequests) {
-        Triple<Integer, Integer, Long> request  = arbitraryDataSignatureRequests.get(signature58);
-        Long now = NTP.getTime();
-
-        if (request == null) {
-            // No entry yet
-            Triple<Integer, Integer, Long> newRequest = new Triple<>(0, 0, now);
-            arbitraryDataSignatureRequests.put(signature58, newRequest);
-        }
-        else {
-            // There is an existing entry
-            if (incrementNetworkRequests) {
-                request.setA(request.getA() + 1);
-            }
-            if (incrementPeerRequests) {
-                request.setB(request.getB() + 1);
-            }
-            request.setC(now);
-            arbitraryDataSignatureRequests.put(signature58, request);
-        }
-    }
-
-    public void removeFromSignatureRequests(String signature58) {
-        arbitraryDataSignatureRequests.remove(signature58);
-    }
-
-
-    // Lookup file lists by signature (and optionally hashes)
-
-    public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) {
-        byte[] digest = arbitraryTransactionData.getData();
-        byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
-        byte[] signature = arbitraryTransactionData.getSignature();
-        String signature58 = Base58.encode(signature);
-
-        // Require an NTP sync
-        Long now = NTP.getTime();
-        if (now == null) {
-            return false;
-        }
-
-        // If we've already tried too many times in a short space of time, make sure to give up
-        if (!this.shouldMakeFileListRequestForSignature(signature58)) {
-            // Check if we should make direct connections to peers
-            if (this.shouldMakeDirectFileRequestsForSignature(signature58)) {
-                return ArbitraryDataFileManager.getInstance().fetchDataFilesFromPeersForSignature(signature);
-            }
-
-            LOGGER.trace("Skipping file list request for signature {} due to rate limit", signature58);
-            return false;
-        }
-        this.addToSignatureRequests(signature58, true, false);
-
-        List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
-        List<byte[]> missingHashes = null;
-
-        // Find hashes that we are missing
-        try {
-            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
-            arbitraryDataFile.setMetadataHash(metadataHash);
-            missingHashes = arbitraryDataFile.missingHashes();
-        } catch (DataException e) {
-            // Leave missingHashes as null, so that all hashes are requested
-        }
-        int hashCount = missingHashes != null ? missingHashes.size() : 0;
-
-        LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size()));
-
-        // Build request
-        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, missingHashes, now, 0);
-
-        // Save our request into requests map
-        Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
-
-        // Assign random ID to this message
-        int id;
-        do {
-            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
-
-            // Put queue into map (keyed by message ID) so we can poll for a response
-            // If putIfAbsent() doesn't return null, then this ID is already taken
-        } while (arbitraryDataFileListRequests.put(id, requestEntry) != null);
-        getArbitraryDataFileListMessage.setId(id);
-
-        // Broadcast request
-        Network.getInstance().broadcast(peer -> getArbitraryDataFileListMessage);
-
-        // Poll to see if data has arrived
-        final long singleWait = 100;
-        long totalWait = 0;
-        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
-            try {
-                Thread.sleep(singleWait);
-            } catch (InterruptedException e) {
-                break;
-            }
-
-            requestEntry = arbitraryDataFileListRequests.get(id);
-            if (requestEntry == null)
-                return false;
-
-            if (requestEntry.getA() == null)
-                break;
-
-            totalWait += singleWait;
-        }
-        return true;
-    }
-
-    public boolean fetchArbitraryDataFileList(Peer peer, byte[] signature) {
-        String signature58 = Base58.encode(signature);
-
-        // Require an NTP sync
-        Long now = NTP.getTime();
-        if (now == null) {
-            return false;
-        }
-
-        int hashCount = 0;
-        LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to peer %s...", signature58, hashCount, peer));
-
-        // Build request
-        // Use a time in the past, so that the recipient peer doesn't try and relay it
-        // Also, set hashes to null since it's easier to request all hashes than it is to determine which ones we need
-        // This could be optimized in the future
-        long timestamp = now - 60000L;
-        List<byte[]> hashes = null;
-        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, timestamp, 0);
-
-        // Save our request into requests map
-        Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
-
-        // Assign random ID to this message
-        int id;
-        do {
-            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
-
-            // Put queue into map (keyed by message ID) so we can poll for a response
-            // If putIfAbsent() doesn't return null, then this ID is already taken
-        } while (arbitraryDataFileListRequests.put(id, requestEntry) != null);
-        getArbitraryDataFileListMessage.setId(id);
-
-        // Send the request
-        peer.sendMessage(getArbitraryDataFileListMessage);
-
-        // Poll to see if data has arrived
-        final long singleWait = 100;
-        long totalWait = 0;
-        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
-            try {
-                Thread.sleep(singleWait);
-            } catch (InterruptedException e) {
-                break;
-            }
-
-            requestEntry = arbitraryDataFileListRequests.get(id);
-            if (requestEntry == null)
-                return false;
-
-            if (requestEntry.getA() == null)
-                break;
-
-            totalWait += singleWait;
-        }
-        return true;
-    }
-
-    public void deleteFileListRequestsForSignature(byte[] signature) {
-        String signature58 = Base58.encode(signature);
-        for (Iterator<Map.Entry<Integer, Triple<String, Peer, Long>>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) {
-            Map.Entry<Integer, Triple<String, Peer, Long>> entry = it.next();
-            if (entry == null || entry.getKey() == null || entry.getValue() != null) {
-                continue;
-            }
-            if (Objects.equals(entry.getValue().getA(), signature58)) {
-                // Update requests map to reflect that we've received all chunks
-                Triple<String, Peer, Long> newEntry = new Triple<>(null, null, entry.getValue().getC());
-                arbitraryDataFileListRequests.put(entry.getKey(), newEntry);
-            }
-        }
-    }
-
-    // Network handlers
-
-    public void onNetworkArbitraryDataFileListMessage(Peer peer, Message message) {
-        // Don't process if QDN is disabled
-        if (!Settings.getInstance().isQdnEnabled()) {
-            return;
-        }
-
-        ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
-        LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
-
-        // Do we have a pending request for this data?
-        Triple<String, Peer, Long> request = arbitraryDataFileListRequests.get(message.getId());
-        if (request == null || request.getA() == null) {
-            return;
-        }
-        boolean isRelayRequest = (request.getB() != null);
-
-        // Does this message's signature match what we're expecting?
-        byte[] signature = arbitraryDataFileListMessage.getSignature();
-        String signature58 = Base58.encode(signature);
-        if (!request.getA().equals(signature58)) {
-            return;
-        }
-
-        List<byte[]> hashes = arbitraryDataFileListMessage.getHashes();
-        if (hashes == null || hashes.isEmpty()) {
-            return;
-        }
-
-        ArbitraryTransactionData arbitraryTransactionData = null;
-        ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance();
-
-        // Check transaction exists and hashes are correct
-        try (final Repository repository = RepositoryManager.getRepository()) {
-            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-            if (!(transactionData instanceof ArbitraryTransactionData))
-                return;
-
-            arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
-
-            // Load data file(s)
-            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature);
-            arbitraryDataFile.setMetadataHash(arbitraryTransactionData.getMetadataHash());
-
-//			// Check all hashes exist
-//			for (byte[] hash : hashes) {
-//				//LOGGER.debug("Received hash {}", Base58.encode(hash));
-//				if (!arbitraryDataFile.containsChunk(hash)) {
-//					// Check the hash against the complete file
-//					if (!Arrays.equals(arbitraryDataFile.getHash(), hash)) {
-//						LOGGER.info("Received non-matching chunk hash {} for signature {}. This could happen if we haven't obtained the metadata file yet.", Base58.encode(hash), signature58);
-//						return;
-//					}
-//				}
-//			}
-
-            if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) {
-                // Keep track of the hashes this peer reports to have access to
-                Long now = NTP.getTime();
-                for (byte[] hash : hashes) {
-                    String hash58 = Base58.encode(hash);
-                    String sig58 = Base58.encode(signature);
-                    ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.put(hash58, new Triple<>(peer, sig58, now));
-                }
-
-                // Go and fetch the actual data, since this isn't a relay request
-                arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, hashes);
-            }
-
-        } catch (DataException e) {
-            LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e);
-        }
-
-        // Forwarding
-        if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
-            boolean isBlocked = (arbitraryTransactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(arbitraryTransactionData.getName()));
-            if (!isBlocked) {
-                Peer requestingPeer = request.getB();
-                if (requestingPeer != null) {
-                    // Add each hash to our local mapping so we know who to ask later
-                    Long now = NTP.getTime();
-                    for (byte[] hash : hashes) {
-                        String hash58 = Base58.encode(hash);
-                        ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now);
-                        ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap);
-                    }
-
-                    // Forward to requesting peer
-                    LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer);
-                    if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) {
-                        requestingPeer.disconnect("failed to forward arbitrary data file list");
-                    }
-                }
-            }
-        }
-    }
-
-    public void onNetworkGetArbitraryDataFileListMessage(Peer peer, Message message) {
-        // Don't respond if QDN is disabled
-        if (!Settings.getInstance().isQdnEnabled()) {
-            return;
-        }
-
-        Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet();
-
-        GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message;
-        byte[] signature = getArbitraryDataFileListMessage.getSignature();
-        String signature58 = Base58.encode(signature);
-        List<byte[]> requestedHashes = getArbitraryDataFileListMessage.getHashes();
-        Long now = NTP.getTime();
-        Triple<String, Peer, Long> newEntry = new Triple<>(signature58, peer, now);
-
-        // If we've seen this request recently, then ignore
-        if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) {
-            LOGGER.debug("Ignoring hash list request from peer {} for signature {}", peer, signature58);
-            return;
-        }
-
-        LOGGER.debug("Received hash list request from peer {} for signature {}", peer, signature58);
-
-        List<byte[]> hashes = new ArrayList<>();
-        ArbitraryTransactionData transactionData = null;
-        boolean allChunksExist = false;
-
-        try (final Repository repository = RepositoryManager.getRepository()) {
-
-            // Firstly we need to lookup this file on chain to get a list of its hashes
-            transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature);
-            if (transactionData instanceof ArbitraryTransactionData) {
-
-                // Check if we're even allowed to serve data for this transaction
-                if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
-
-                    byte[] hash = transactionData.getData();
-                    byte[] metadataHash = transactionData.getMetadataHash();
-
-                    // Load file(s) and add any that exist to the list of hashes
-                    ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
-                    arbitraryDataFile.setMetadataHash(metadataHash);
-
-                    // If the peer didn't supply a hash list, we need to return all hashes for this transaction
-                    if (requestedHashes == null || requestedHashes.isEmpty()) {
-                        requestedHashes = new ArrayList<>();
-
-                        // Add the metadata file
-                        if (arbitraryDataFile.getMetadataHash() != null) {
-                            requestedHashes.add(arbitraryDataFile.getMetadataHash());
-                        }
-
-                        // Add the chunk hashes
-                        if (arbitraryDataFile.getChunkHashes().size() > 0) {
-                            requestedHashes.addAll(arbitraryDataFile.getChunkHashes());
-                        }
-                        // Add complete file if there are no hashes
-                        else {
-                            requestedHashes.add(arbitraryDataFile.getHash());
-                        }
-                    }
-
-                    // Assume all chunks exists, unless one can't be found below
-                    allChunksExist = true;
-
-                    for (byte[] requestedHash : requestedHashes) {
-                        ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature);
-                        if (chunk.exists()) {
-                            hashes.add(chunk.getHash());
-                            //LOGGER.trace("Added hash {}", chunk.getHash58());
-                        } else {
-                            LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58());
-                            allChunksExist = false;
-                        }
-                    }
-                }
-            }
-
-        } catch (DataException e) {
-            LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e);
-        }
-
-        // We should only respond if we have at least one hash
-        if (hashes.size() > 0) {
-
-            // We have all the chunks, so update requests map to reflect that we've sent it
-            // There is no need to keep track of the request, as we can serve all the chunks
-            if (allChunksExist) {
-                newEntry = new Triple<>(null, null, now);
-                arbitraryDataFileListRequests.put(message.getId(), newEntry);
-            }
-
-            ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
-            arbitraryDataFileListMessage.setId(message.getId());
-            if (!peer.sendMessage(arbitraryDataFileListMessage)) {
-                LOGGER.debug("Couldn't send list of hashes");
-                peer.disconnect("failed to send list of hashes");
-                return;
-            }
-            LOGGER.debug("Sent list of hashes (count: {})", hashes.size());
-
-            if (allChunksExist) {
-                // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
-                LOGGER.debug("No need for any forwarding because file list request is fully served");
-                return;
-            }
-
-        }
-
-        // We may need to forward this request on
-        boolean isBlocked = (transactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName()));
-        if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
-            // In relay mode - so ask our other peers if they have it
-
-            long requestTime = getArbitraryDataFileListMessage.getRequestTime();
-            int requestHops = getArbitraryDataFileListMessage.getRequestHops();
-            getArbitraryDataFileListMessage.setRequestHops(++requestHops);
-            long totalRequestTime = now - requestTime;
-
-            if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
-                // Relay request hasn't timed out yet, so can potentially be rebroadcast
-                if (requestHops < RELAY_REQUEST_MAX_HOPS) {
-                    // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
-
-                    LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
-                    Network.getInstance().broadcast(
-                            broadcastPeer -> broadcastPeer == peer ||
-                                    Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost())
-                                    ? null : getArbitraryDataFileListMessage);
-
-                }
-                else {
-                    // This relay request has reached the maximum number of allowed hops
-                }
-            }
-            else {
-                // This relay request has timed out
-            }
-        }
-    }
-
-}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java

@@ -1,509 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.arbitrary.ArbitraryDataFile;
-import org.qortal.controller.Controller;
-import org.qortal.data.arbitrary.ArbitraryRelayInfo;
-import org.qortal.data.network.ArbitraryPeerData;
-import org.qortal.data.network.PeerData;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.network.Network;
-import org.qortal.network.Peer;
-import org.qortal.network.message.*;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.settings.Settings;
-import org.qortal.utils.ArbitraryTransactionUtils;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-import org.qortal.utils.Triple;
-
-import java.security.SecureRandom;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.stream.Collectors;
-
-public class ArbitraryDataFileManager extends Thread {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileManager.class);
-
-    private static ArbitraryDataFileManager instance;
-    private volatile boolean isStopping = false;
-
-
-    /**
-     * Map to keep track of our in progress (outgoing) arbitrary data file requests
-     */
-    private Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
-
-    /**
-     * Map to keep track of hashes that we might need to relay
-     */
-    public List<ArbitraryRelayInfo> arbitraryRelayMap = Collections.synchronizedList(new ArrayList<>());
-
-    /**
-     * Map to keep track of any arbitrary data file hash responses
-     * Key: string - the hash encoded in base58
-     * Value: Triple<respondingPeer, signature58, timeResponded>
-     */
-    public Map<String, Triple<Peer, String, Long>> arbitraryDataFileHashResponses = Collections.synchronizedMap(new HashMap<>());
-
-
-    private ArbitraryDataFileManager() {
-    }
-
-    public static ArbitraryDataFileManager getInstance() {
-        if (instance == null)
-            instance = new ArbitraryDataFileManager();
-
-        return instance;
-    }
-
-    @Override
-    public void run() {
-        Thread.currentThread().setName("Arbitrary Data File Manager");
-
-        try {
-            // Use a fixed thread pool to execute the arbitrary data file requests
-            int threadCount = 10;
-            ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount);
-            for (int i = 0; i < threadCount; i++) {
-                arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread());
-            }
-
-            while (!isStopping) {
-                // Nothing to do yet
-                Thread.sleep(1000);
-            }
-        } catch (InterruptedException e) {
-            // Fall-through to exit thread...
-        }
-    }
-
-    public void shutdown() {
-        isStopping = true;
-        this.interrupt();
-    }
-
-
-    public void cleanupRequestCache(Long now) {
-        if (now == null) {
-            return;
-        }
-        final long requestMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_REQUEST_TIMEOUT;
-        arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < requestMinimumTimestamp);
-
-        final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT;
-        arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp);
-        arbitraryDataFileHashResponses.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp);
-    }
-
-
-
-    // Fetch data files by hash
-
-    public boolean fetchArbitraryDataFiles(Repository repository,
-                                           Peer peer,
-                                           byte[] signature,
-                                           ArbitraryTransactionData arbitraryTransactionData,
-                                           List<byte[]> hashes) throws DataException {
-
-        // Load data file(s)
-        ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature);
-        byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
-        arbitraryDataFile.setMetadataHash(metadataHash);
-        boolean receivedAtLeastOneFile = false;
-
-        // Now fetch actual data from this peer
-        for (byte[] hash : hashes) {
-            if (isStopping) {
-                return false;
-            }
-            String hash58 = Base58.encode(hash);
-            if (!arbitraryDataFile.chunkExists(hash)) {
-                // Only request the file if we aren't already requesting it from someone else
-                if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
-                    LOGGER.debug("Requesting data file {} from peer {}", hash58, peer);
-                    Long startTime = NTP.getTime();
-                    ArbitraryDataFileMessage receivedArbitraryDataFileMessage = fetchArbitraryDataFile(peer, null, signature, hash, null);
-                    Long endTime = NTP.getTime();
-                    if (receivedArbitraryDataFileMessage != null) {
-                        LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFileMessage.getArbitraryDataFile().getHash58(), peer, (endTime-startTime));
-                        receivedAtLeastOneFile = true;
-
-                        // Remove this hash from arbitraryDataFileHashResponses now that we have received it
-                        arbitraryDataFileHashResponses.remove(hash58);
-                    }
-                    else {
-                        LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime));
-
-                        // Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it
-                        arbitraryDataFileHashResponses.remove(hash58);
-
-                        // Stop asking for files from this peer
-                        break;
-                    }
-                }
-                else {
-                    LOGGER.trace("Already requesting data file {} for signature {}", arbitraryDataFile, Base58.encode(signature));
-                }
-            }
-            else {
-                // Remove this hash from arbitraryDataFileHashResponses because we have a local copy
-                arbitraryDataFileHashResponses.remove(hash58);
-            }
-        }
-
-        if (receivedAtLeastOneFile) {
-            // Update our lookup table to indicate that this peer holds data for this signature
-            String peerAddress = peer.getPeerData().getAddress().toString();
-            ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(signature, peer);
-            repository.discardChanges();
-            if (arbitraryPeerData.isPeerAddressValid()) {
-                LOGGER.debug("Adding arbitrary peer: {} for signature {}", peerAddress, Base58.encode(signature));
-                repository.getArbitraryRepository().save(arbitraryPeerData);
-                repository.saveChanges();
-            }
-
-            // Invalidate the hosted transactions cache as we are now hosting something new
-            ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache();
-
-            // Check if we have all the files we need for this transaction
-            if (arbitraryDataFile.allFilesExist()) {
-
-                // We have all the chunks for this transaction, so we should invalidate the transaction's name's
-                // data cache so that it is rebuilt the next time we serve it
-                ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData);
-
-                // We may also need to broadcast to the network that we are now hosting files for this transaction,
-                // but only if these files are in accordance with our storage policy
-                if (ArbitraryDataStorageManager.getInstance().canStoreData(arbitraryTransactionData)) {
-                    // Use a null peer address to indicate our own
-                    Message newArbitrarySignatureMessage = new ArbitrarySignaturesMessage(null, 0, Arrays.asList(signature));
-                    Network.getInstance().broadcast(broadcastPeer -> newArbitrarySignatureMessage);
-                }
-            }
-
-        }
-
-        return receivedAtLeastOneFile;
-    }
-
-    private ArbitraryDataFileMessage fetchArbitraryDataFile(Peer peer, Peer requestingPeer, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
-        ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature);
-        boolean fileAlreadyExists = existingFile.exists();
-        String hash58 = Base58.encode(hash);
-        Message message = null;
-
-        // Fetch the file if it doesn't exist locally
-        if (!fileAlreadyExists) {
-            LOGGER.debug(String.format("Fetching data file %.8s from peer %s", hash58, peer));
-            arbitraryDataFileRequests.put(hash58, NTP.getTime());
-            Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash);
-
-            try {
-                message = peer.getResponseWithTimeout(getArbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT);
-            } catch (InterruptedException e) {
-                // Will return below due to null message
-            }
-            arbitraryDataFileRequests.remove(hash58);
-            LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58));
-
-            // We may need to remove the file list request, if we have all the files for this transaction
-            this.handleFileListRequests(signature);
-
-            if (message == null) {
-                LOGGER.debug("Received null message from peer {}", peer);
-                return null;
-            }
-            if (message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) {
-                LOGGER.debug("Received message with invalid type: {} from peer {}", message.getType(), peer);
-                return null;
-            }
-        }
-        else {
-            LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58));
-        }
-        ArbitraryDataFileMessage arbitraryDataFileMessage = (ArbitraryDataFileMessage) message;
-
-        // We might want to forward the request to the peer that originally requested it
-        this.handleArbitraryDataFileForwarding(requestingPeer, message, originalMessage);
-
-        boolean isRelayRequest = (requestingPeer != null);
-        if (isRelayRequest) {
-            if (!fileAlreadyExists) {
-                // File didn't exist locally before the request, and it's a forwarding request, so delete it
-                LOGGER.debug("Deleting file {} because it was needed for forwarding only", Base58.encode(hash));
-                ArbitraryDataFile dataFile = arbitraryDataFileMessage.getArbitraryDataFile();
-
-                // Keep trying to delete the data until it is deleted, or we reach 10 attempts
-                for (int i=0; i<10; i++) {
-                    if (dataFile.delete()) {
-                        break;
-                    }
-                    try {
-                        Thread.sleep(1000L);
-                    } catch (InterruptedException e) {
-                        // Fall through to exit method
-                    }
-                }
-            }
-        }
-
-        return arbitraryDataFileMessage;
-    }
-
-    private void handleFileListRequests(byte[] signature) {
-        try (final Repository repository = RepositoryManager.getRepository()) {
-
-            // Fetch the transaction data
-            ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
-            if (arbitraryTransactionData == null) {
-                return;
-            }
-
-            boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData);
-
-            if (allChunksExist) {
-                // Update requests map to reflect that we've received all chunks
-                ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature);
-            }
-
-        } catch (DataException e) {
-            LOGGER.debug("Unable to handle file list requests: {}", e.getMessage());
-        }
-    }
-
-    public void handleArbitraryDataFileForwarding(Peer requestingPeer, Message message, Message originalMessage) {
-        // Return if there is no originally requesting peer to forward to
-        if (requestingPeer == null) {
-            return;
-        }
-
-        // Return if we're not in relay mode or if this request doesn't need forwarding
-        if (!Settings.getInstance().isRelayModeEnabled()) {
-            return;
-        }
-
-        LOGGER.debug("Received arbitrary data file - forwarding is needed");
-
-        // The ID needs to match that of the original request
-        message.setId(originalMessage.getId());
-
-        if (!requestingPeer.sendMessage(message)) {
-            LOGGER.debug("Failed to forward arbitrary data file to peer {}", requestingPeer);
-            requestingPeer.disconnect("failed to forward arbitrary data file");
-        }
-        else {
-            LOGGER.debug("Forwarded arbitrary data file to peer {}", requestingPeer);
-        }
-    }
-
-
-    // Fetch data directly from peers
-
-    public boolean fetchDataFilesFromPeersForSignature(byte[] signature) {
-        String signature58 = Base58.encode(signature);
-        ArbitraryDataFileListManager.getInstance().addToSignatureRequests(signature58, false, true);
-
-        // Firstly fetch peers that claim to be hosting files for this signature
-        try (final Repository repository = RepositoryManager.getRepository()) {
-
-            List<ArbitraryPeerData> peers = repository.getArbitraryRepository().getArbitraryPeerDataForSignature(signature);
-            if (peers == null || peers.isEmpty()) {
-                LOGGER.debug("No peers found for signature {}", signature58);
-                return false;
-            }
-
-            LOGGER.debug("Attempting a direct peer connection for signature {}...", signature58);
-
-            // Peers found, so pick a random one and request data from it
-            int index = new SecureRandom().nextInt(peers.size());
-            ArbitraryPeerData arbitraryPeerData = peers.get(index);
-            String peerAddressString = arbitraryPeerData.getPeerAddress();
-            boolean success = Network.getInstance().requestDataFromPeer(peerAddressString, signature);
-
-            // Parse the peer address to find the host and port
-            String host = null;
-            int port = -1;
-            String[] parts = peerAddressString.split(":");
-            if (parts.length > 1) {
-                host = parts[0];
-                port = Integer.parseInt(parts[1]);
-            }
-
-            // If unsuccessful, and using a non-standard port, try a second connection with the default listen port,
-            // since almost all nodes use that. This is a workaround to account for any ephemeral ports that may
-            // have made it into the dataset.
-            if (!success) {
-                if (host != null && port > 0) {
-                    int defaultPort = Settings.getInstance().getDefaultListenPort();
-                    if (port != defaultPort) {
-                        String newPeerAddressString = String.format("%s:%d", host, defaultPort);
-                        success = Network.getInstance().requestDataFromPeer(newPeerAddressString, signature);
-                    }
-                }
-            }
-
-            // If _still_ unsuccessful, try matching the peer's IP address with some known peers, and then connect
-            // to each of those in turn until one succeeds.
-            if (!success) {
-                if (host != null) {
-                    final String finalHost = host;
-                    List<PeerData> knownPeers = Network.getInstance().getAllKnownPeers().stream()
-                            .filter(knownPeerData -> knownPeerData.getAddress().getHost().equals(finalHost))
-                            .collect(Collectors.toList());
-                    // Loop through each match and attempt a connection
-                    for (PeerData matchingPeer : knownPeers) {
-                        String matchingPeerAddress = matchingPeer.getAddress().toString();
-                        success = Network.getInstance().requestDataFromPeer(matchingPeerAddress, signature);
-                        if (success) {
-                            // Successfully connected, so stop making connections
-                            break;
-                        }
-                    }
-                }
-            }
-
-            // Keep track of the success or failure
-            arbitraryPeerData.markAsAttempted();
-            if (success) {
-                arbitraryPeerData.markAsRetrieved();
-                arbitraryPeerData.incrementSuccesses();
-            }
-            else {
-                arbitraryPeerData.incrementFailures();
-            }
-            repository.discardChanges();
-            repository.getArbitraryRepository().save(arbitraryPeerData);
-            repository.saveChanges();
-
-            return success;
-
-        } catch (DataException e) {
-            LOGGER.debug("Unable to fetch peer list from repository");
-        }
-
-        return false;
-    }
-
-
-    // Relays
-
-    private List<ArbitraryRelayInfo> getRelayInfoListForHash(String hash58) {
-        synchronized (arbitraryRelayMap) {
-            return arbitraryRelayMap.stream()
-                    .filter(relayInfo -> Objects.equals(relayInfo.getHash58(), hash58))
-                    .collect(Collectors.toList());
-        }
-    }
-
-    private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) {
-        LOGGER.trace("Fetching random relay info for hash: {}", hash58);
-        List<ArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
-        if (relayInfoList != null && !relayInfoList.isEmpty()) {
-
-            // Pick random item
-            int index = new SecureRandom().nextInt(relayInfoList.size());
-            LOGGER.trace("Returning random relay info for hash: {} (index {})", hash58, index);
-            return relayInfoList.get(index);
-        }
-        LOGGER.trace("No relay info exists for hash: {}", hash58);
-        return null;
-    }
-
-    public void addToRelayMap(ArbitraryRelayInfo newEntry) {
-        if (newEntry == null || !newEntry.isValid()) {
-            return;
-        }
-
-        // Remove existing entry for this peer if it exists, to renew the timestamp
-        this.removeFromRelayMap(newEntry);
-
-        // Re-add
-        arbitraryRelayMap.add(newEntry);
-        LOGGER.debug("Added entry to relay map: {}", newEntry);
-    }
-
-    private void removeFromRelayMap(ArbitraryRelayInfo entry) {
-        arbitraryRelayMap.removeIf(relayInfo -> relayInfo.equals(entry));
-    }
-
-
-    // Network handlers
-
-    public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) {
-        // Don't respond if QDN is disabled
-        if (!Settings.getInstance().isQdnEnabled()) {
-            return;
-        }
-
-        GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message;
-        byte[] hash = getArbitraryDataFileMessage.getHash();
-        String hash58 = Base58.encode(hash);
-        byte[] signature = getArbitraryDataFileMessage.getSignature();
-        Controller.getInstance().stats.getArbitraryDataFileMessageStats.requests.incrementAndGet();
-
-        LOGGER.debug("Received GetArbitraryDataFileMessage from peer {} for hash {}", peer, Base58.encode(hash));
-
-        try {
-            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
-            ArbitraryRelayInfo relayInfo = this.getRandomRelayInfoEntryForHash(hash58);
-
-            if (arbitraryDataFile.exists()) {
-                LOGGER.trace("Hash {} exists", hash58);
-
-                // We can serve the file directly as we already have it
-                ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, arbitraryDataFile);
-                arbitraryDataFileMessage.setId(message.getId());
-                if (!peer.sendMessage(arbitraryDataFileMessage)) {
-                    LOGGER.debug("Couldn't sent file");
-                    peer.disconnect("failed to send file");
-                }
-                LOGGER.debug("Sent file {}", arbitraryDataFile);
-            }
-            else if (relayInfo != null) {
-                LOGGER.debug("We have relay info for hash {}", Base58.encode(hash));
-                // We need to ask this peer for the file
-                Peer peerToAsk = relayInfo.getPeer();
-                if (peerToAsk != null) {
-
-                    // Forward the message to this peer
-                    LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58);
-                    this.fetchArbitraryDataFile(peerToAsk, peer, signature, hash, message);
-                }
-                else {
-                    LOGGER.debug("Peer {} not found in relay info", peer);
-                }
-            }
-            else {
-                LOGGER.debug("Hash {} doesn't exist and we don't have relay info", hash58);
-
-                // We don't have this file
-                Controller.getInstance().stats.getArbitraryDataFileMessageStats.unknownFiles.getAndIncrement();
-
-                // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout
-                LOGGER.debug(String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile));
-
-                // We'll send empty block summaries message as it's very short
-                // TODO: use a different message type here
-                Message fileUnknownMessage = new BlockSummariesMessage(Collections.emptyList());
-                fileUnknownMessage.setId(message.getId());
-                if (!peer.sendMessage(fileUnknownMessage)) {
-                    LOGGER.debug("Couldn't sent file-unknown response");
-                    peer.disconnect("failed to send file-unknown response");
-                }
-                else {
-                    LOGGER.debug("Sent file-unknown response for file {}", arbitraryDataFile);
-                }
-            }
-        }
-        catch (DataException e) {
-            LOGGER.debug("Unable to handle request for arbitrary data file: {}", hash58);
-        }
-    }
-
-}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java

@@ -1,117 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.controller.Controller;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.network.Peer;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.utils.ArbitraryTransactionUtils;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-import org.qortal.utils.Triple;
-
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.Map;
-
-public class ArbitraryDataFileRequestThread implements Runnable {
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class);
-
-    public ArbitraryDataFileRequestThread() {
-
-    }
-
-    @Override
-    public void run() {
-        Thread.currentThread().setName("Arbitrary Data File Request Thread");
-
-        try {
-            while (!Controller.isStopping()) {
-                Long now = NTP.getTime();
-                this.processFileHashes(now);
-            }
-        } catch (InterruptedException e) {
-            // Fall-through to exit thread...
-        }
-    }
-
-    private void processFileHashes(Long now) throws InterruptedException {
-		if (Controller.isStopping()) {
-            return;
-        }
-
-        ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance();
-        String signature58 = null;
-        String hash58 = null;
-        Peer peer = null;
-        boolean shouldProcess = false;
-
-        synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) {
-            Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator();
-            while (iterator.hasNext()) {
-                if (Controller.isStopping()) {
-                    return;
-                }
-
-                Map.Entry entry = (Map.Entry) iterator.next();
-                if (entry == null || entry.getKey() == null || entry.getValue() == null) {
-                    iterator.remove();
-                    continue;
-                }
-
-                hash58 = (String) entry.getKey();
-                Triple<Peer, String, Long> value = (Triple<Peer, String, Long>) entry.getValue();
-                if (value == null) {
-                    iterator.remove();
-                    continue;
-                }
-
-                peer = value.getA();
-                signature58 = value.getB();
-                Long timestamp = value.getC();
-
-                if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) {
-                    // Ignore - to be deleted
-                    iterator.remove();
-                    continue;
-                }
-
-                // We want to process this file
-                shouldProcess = true;
-                iterator.remove();
-                break;
-            }
-        }
-
-        if (!shouldProcess) {
-            // Nothing to do
-            Thread.sleep(1000L);
-            return;
-        }
-
-        byte[] hash = Base58.decode(hash58);
-        byte[] signature = Base58.decode(signature58);
-
-        // Fetch the transaction data
-        try (final Repository repository = RepositoryManager.getRepository()) {
-            ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
-            if (arbitraryTransactionData == null) {
-                return;
-            }
-
-            if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) {
-                return;
-            }
-
-            LOGGER.debug("Fetching file {} from peer {} via request thread...", hash58, peer);
-            arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash));
-
-        } catch (DataException e) {
-            LOGGER.debug("Unable to process file hashes: {}", e.getMessage());
-        }
-    }
-}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -1,461 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
-import org.qortal.arbitrary.ArbitraryDataFile;
-import org.qortal.arbitrary.ArbitraryDataResource;
-import org.qortal.arbitrary.misc.Service;
-import org.qortal.controller.Controller;
-import org.qortal.data.network.ArbitraryPeerData;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.data.transaction.TransactionData;
-import org.qortal.list.ResourceListManager;
-import org.qortal.network.Network;
-import org.qortal.network.Peer;
-import org.qortal.network.message.*;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.settings.Settings;
-import org.qortal.transaction.ArbitraryTransaction;
-import org.qortal.transaction.Transaction.TransactionType;
-import org.qortal.utils.ArbitraryTransactionUtils;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-
-public class ArbitraryDataManager extends Thread {
-
-	private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataManager.class);
-	private static final List<TransactionType> ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY);
-
-	/** Difficulty (leading zero bits) used in arbitrary data transactions
-	 * Set here so that it can be more easily reduced when running unit tests */
-	private int powDifficulty = 14; // Must not be final, as unit tests need to reduce this value
-
-	/** Request timeout when transferring arbitrary data */
-	public static final long ARBITRARY_REQUEST_TIMEOUT = 12 * 1000L; // ms
-
-	/** Maximum time to hold information about an in-progress relay */
-	public static final long ARBITRARY_RELAY_TIMEOUT = 60 * 1000L; // ms
-
-	/** Maximum number of hops that an arbitrary signatures request is allowed to make */
-	private static int ARBITRARY_SIGNATURES_REQUEST_MAX_HOPS = 3;
-
-	private static ArbitraryDataManager instance;
-	private final Object peerDataLock = new Object();
-
-	private volatile boolean isStopping = false;
-
-	/**
-	 * Map to keep track of cached arbitrary transaction resources.
-	 * When an item is present in this list with a timestamp in the future, we won't invalidate
-	 * its cache when serving that data. This reduces the amount of database lookups that are needed.
-	 */
-	private Map<String, Long> arbitraryDataCachedResources = Collections.synchronizedMap(new HashMap<>());
-
-	/**
-	 * The amount of time to cache a data resource before it is invalidated
-	 */
-	private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes
-
-
-
-	private ArbitraryDataManager() {
-	}
-
-	public static ArbitraryDataManager getInstance() {
-		if (instance == null)
-			instance = new ArbitraryDataManager();
-
-		return instance;
-	}
-
-	@Override
-	public void run() {
-		Thread.currentThread().setName("Arbitrary Data Manager");
-
-		try {
-			// Wait for node to finish starting up and making connections
-			Thread.sleep(2 * 60 * 1000L);
-
-			while (!isStopping) {
-				Thread.sleep(2000);
-
-				// Don't run if QDN is disabled
-				if (!Settings.getInstance().isQdnEnabled()) {
-					Thread.sleep(60 * 60 * 1000L);
-					continue;
-				}
-
-				List<Peer> peers = Network.getInstance().getHandshakedPeers();
-
-				// Disregard peers that have "misbehaved" recently
-				peers.removeIf(Controller.hasMisbehaved);
-
-				// Don't fetch data if we don't have enough up-to-date peers
-				if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) {
-					continue;
-				}
-
-				// Fetch data according to storage policy
-				switch (Settings.getInstance().getStoragePolicy()) {
-					case FOLLOWED:
-					case FOLLOWED_OR_VIEWED:
-						this.processNames();
-						break;
-
-					case ALL:
-						this.processAll();
-
-					case NONE:
-					case VIEWED:
-					default:
-						// Nothing to fetch in advance
-						Thread.sleep(60000);
-						break;
-				}
-			}
-		} catch (InterruptedException e) {
-			// Fall-through to exit thread...
-		}
-	}
-
-	public void shutdown() {
-		isStopping = true;
-		this.interrupt();
-	}
-
-	private void processNames() {
-		// Fetch latest list of followed names
-		List<String> followedNames = ResourceListManager.getInstance().getStringsInList("followedNames");
-		if (followedNames == null || followedNames.isEmpty()) {
-			return;
-		}
-
-		// Loop through the names in the list and fetch transactions for each
-		for (String name : followedNames) {
-			this.fetchAndProcessTransactions(name);
-		}
-	}
-
-	private void processAll() {
-		this.fetchAndProcessTransactions(null);
-	}
-
-	private void fetchAndProcessTransactions(String name) {
-		ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
-
-		// Paginate queries when fetching arbitrary transactions
-		final int limit = 100;
-		int offset = 0;
-
-		while (!isStopping) {
-
-			// Any arbitrary transactions we want to fetch data for?
-			try (final Repository repository = RepositoryManager.getRepository()) {
-				List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true);
-				// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
-				if (signatures == null || signatures.isEmpty()) {
-					offset = 0;
-					break;
-				}
-				offset += limit;
-
-				// Loop through signatures and remove ones we don't need to process
-				Iterator iterator = signatures.iterator();
-				while (iterator.hasNext()) {
-					byte[] signature = (byte[]) iterator.next();
-
-					ArbitraryTransaction arbitraryTransaction = fetchTransaction(repository, signature);
-					if (arbitraryTransaction == null) {
-						// Best not to process this one
-						iterator.remove();
-						continue;
-					}
-					ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
-
-					// Skip transactions that we don't need to proactively store data for
-					if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) {
-						iterator.remove();
-						continue;
-					}
-
-					// Remove transactions that we already have local data for
-					if (hasLocalData(arbitraryTransaction)) {
-						iterator.remove();
-						continue;
-					}
-				}
-
-				if (signatures.isEmpty()) {
-					continue;
-				}
-
-				// Pick one at random
-				final int index = new Random().nextInt(signatures.size());
-				byte[] signature = signatures.get(index);
-
-				if (signature == null) {
-					continue;
-				}
-
-				// Check to see if we have had a more recent PUT
-				ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
-				boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
-				if (hasMoreRecentPutTransaction) {
-					// There is a more recent PUT transaction than the one we are currently processing.
-					// When a PUT is issued, it replaces any layers that would have been there before.
-					// Therefore any data relating to this older transaction is no longer needed and we
-					// shouldn't fetch it from the network.
-					continue;
-				}
-
-				// Ask our connected peers if they have files for this signature
-				// This process automatically then fetches the files themselves if a peer is found
-				fetchData(arbitraryTransactionData);
-
-			} catch (DataException e) {
-				LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
-			}
-		}
-	}
-
-	private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) {
-		try {
-			TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-			if (!(transactionData instanceof ArbitraryTransactionData))
-				return null;
-
-			return new ArbitraryTransaction(repository, transactionData);
-
-		} catch (DataException e) {
-			return null;
-		}
-	}
-
-	private boolean hasLocalData(ArbitraryTransaction arbitraryTransaction) {
-		try {
-			return arbitraryTransaction.isDataLocal();
-
-		} catch (DataException e) {
-			LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e);
-			return true;
-		}
-	}
-
-
-	// Entrypoint to request new data from peers
-	public boolean fetchData(ArbitraryTransactionData arbitraryTransactionData) {
-		return ArbitraryDataFileListManager.getInstance().fetchArbitraryDataFileList(arbitraryTransactionData);
-	}
-
-
-	// Useful methods used by other parts of the app
-
-	public boolean isSignatureRateLimited(byte[] signature) {
-		return ArbitraryDataFileListManager.getInstance().isSignatureRateLimited(signature);
-	}
-
-	public long lastRequestForSignature(byte[] signature) {
-		return ArbitraryDataFileListManager.getInstance().lastRequestForSignature(signature);
-	}
-
-
-	// Arbitrary data resource cache
-
-	public void cleanupRequestCache(Long now) {
-		if (now == null) {
-			return;
-		}
-
-		// Cleanup file list request caches
-		ArbitraryDataFileListManager.getInstance().cleanupRequestCache(now);
-
-		// Cleanup file request caches
-		ArbitraryDataFileManager.getInstance().cleanupRequestCache(now);
-	}
-
-	public boolean isResourceCached(ArbitraryDataResource resource) {
-		if (resource == null) {
-			return false;
-		}
-		String key = resource.getUniqueKey();
-
-		// We don't have an entry for this resource ID, it is not cached
-		if (this.arbitraryDataCachedResources == null) {
-			return false;
-		}
-		if (!this.arbitraryDataCachedResources.containsKey(key)) {
-			return false;
-		}
-		Long timestamp = this.arbitraryDataCachedResources.get(key);
-		if (timestamp == null) {
-			return false;
-		}
-
-		// If the timestamp has reached the timeout, we should remove it from the cache
-		long now = NTP.getTime();
-		if (now > timestamp) {
-			this.arbitraryDataCachedResources.remove(key);
-			return false;
-		}
-
-		// Current time hasn't reached the timeout, so treat it as cached
-		return true;
-	}
-
-	public void addResourceToCache(ArbitraryDataResource resource) {
-		if (resource == null) {
-			return;
-		}
-		String key = resource.getUniqueKey();
-
-		// Just in case
-		if (this.arbitraryDataCachedResources == null) {
-			this.arbitraryDataCachedResources = new HashMap<>();
-		}
-
-		Long now = NTP.getTime();
-		if (now == null) {
-			return;
-		}
-
-		// Set the timestamp to now + the timeout
-		Long timestamp = NTP.getTime() + ARBITRARY_DATA_CACHE_TIMEOUT;
-		this.arbitraryDataCachedResources.put(key, timestamp);
-	}
-
-	public void invalidateCache(ArbitraryTransactionData arbitraryTransactionData) {
-		String signature58 = Base58.encode(arbitraryTransactionData.getSignature());
-
-		if (arbitraryTransactionData.getName() != null) {
-			String resourceId = arbitraryTransactionData.getName().toLowerCase();
-			Service service = arbitraryTransactionData.getService();
-			String identifier = arbitraryTransactionData.getIdentifier();
-
-			ArbitraryDataResource resource =
-					new ArbitraryDataResource(resourceId, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
-			String key = resource.getUniqueKey();
-			LOGGER.trace("Clearing cache for {}...", resource);
-
-			if (this.arbitraryDataCachedResources.containsKey(key)) {
-				this.arbitraryDataCachedResources.remove(key);
-			}
-
-			// Also remove from the failed builds queue in case it previously failed due to missing chunks
-			ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance();
-			if (buildManager.arbitraryDataFailedBuilds.containsKey(key)) {
-				buildManager.arbitraryDataFailedBuilds.remove(key);
-			}
-
-			// Remove from the signature requests list now that we have all files for this signature
-			ArbitraryDataFileListManager.getInstance().removeFromSignatureRequests(signature58);
-
-			// Delete cached files themselves
-			try {
-				resource.deleteCache();
-			} catch (IOException e) {
-				LOGGER.info("Unable to delete cache for resource {}: {}", resource, e.getMessage());
-			}
-		}
-	}
-
-
-	// Broadcast list of hosted signatures
-
-	public void broadcastHostedSignatureList() {
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, null, null);
-			List<byte[]> hostedSignatures = hostedTransactions.stream().map(ArbitraryTransactionData::getSignature).collect(Collectors.toList());
-			if (!hostedSignatures.isEmpty()) {
-				// Broadcast the list, using null to represent our peer address
-				LOGGER.info("Broadcasting list of hosted signatures...");
-				Message arbitrarySignatureMessage = new ArbitrarySignaturesMessage(null, 0, hostedSignatures);
-				Network.getInstance().broadcast(broadcastPeer -> arbitrarySignatureMessage);
-			}
-		} catch (DataException e) {
-			LOGGER.error("Repository issue when fetching arbitrary transaction data for broadcast", e);
-		}
-	}
-
-
-	// Handle incoming arbitrary signatures messages
-
-	public void onNetworkArbitrarySignaturesMessage(Peer peer, Message message) {
-		// Don't process if QDN is disabled
-		if (!Settings.getInstance().isQdnEnabled()) {
-			return;
-		}
-
-		LOGGER.debug("Received arbitrary signature list from peer {}", peer);
-
-		ArbitrarySignaturesMessage arbitrarySignaturesMessage = (ArbitrarySignaturesMessage) message;
-		List<byte[]> signatures = arbitrarySignaturesMessage.getSignatures();
-
-		String peerAddress = peer.getPeerData().getAddress().toString();
-		if (arbitrarySignaturesMessage.getPeerAddress() != null && !arbitrarySignaturesMessage.getPeerAddress().isEmpty()) {
-			// This message is about a different peer than the one that sent it
-			peerAddress = arbitrarySignaturesMessage.getPeerAddress();
-		}
-
-		boolean containsNewEntry = false;
-
-		// Synchronize peer data lookups to make this process thread safe. Otherwise we could broadcast
-		// the same data multiple times, due to more than one thread processing the same message from different peers
-		synchronized (this.peerDataLock) {
-			try (final Repository repository = RepositoryManager.getRepository()) {
-				for (byte[] signature : signatures) {
-
-					// Check if a record already exists for this hash/host combination
-					// The port is not checked here - only the host/ip - in order to avoid duplicates
-					// from filling up the db due to dynamic/ephemeral ports
-					ArbitraryPeerData existingEntry = repository.getArbitraryRepository()
-							.getArbitraryPeerDataForSignatureAndHost(signature, peer.getPeerData().getAddress().getHost());
-
-					if (existingEntry == null) {
-						// We haven't got a record of this mapping yet, so add it
-						ArbitraryPeerData arbitraryPeerData = new ArbitraryPeerData(signature, peerAddress);
-						repository.discardChanges();
-						if (arbitraryPeerData.isPeerAddressValid()) {
-							LOGGER.debug("Adding arbitrary peer: {} for signature {}", peerAddress, Base58.encode(signature));
-							repository.getArbitraryRepository().save(arbitraryPeerData);
-							repository.saveChanges();
-
-							// Remember that this data is new, so that it can be rebroadcast later
-							containsNewEntry = true;
-						}
-					}
-				}
-
-				// If at least one signature in this batch was new to us, we should rebroadcast the message to the
-				// network in case some peers haven't received it yet
-				if (containsNewEntry) {
-					int requestHops = arbitrarySignaturesMessage.getRequestHops();
-					arbitrarySignaturesMessage.setRequestHops(++requestHops);
-					if (requestHops < ARBITRARY_SIGNATURES_REQUEST_MAX_HOPS) {
-						LOGGER.debug("Rebroadcasting arbitrary signature list for peer {}. requestHops: {}", peerAddress, requestHops);
-						Network.getInstance().broadcast(broadcastPeer -> broadcastPeer == peer ? null : arbitrarySignaturesMessage);
-					}
-				} else {
-					// Don't rebroadcast as otherwise we could get into a loop
-				}
-
-				// If anything needed saving, it would already have called saveChanges() above
-				repository.discardChanges();
-			} catch (DataException e) {
-				LOGGER.error(String.format("Repository issue while processing arbitrary transaction signature list from peer %s", peer), e);
-			}
-		}
-	}
-
-
-	public int getPowDifficulty() {
-		return this.powDifficulty;
-	}
-
-}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java

@@ -1,86 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import org.qortal.arbitrary.ArbitraryDataResource;
-import org.qortal.utils.NTP;
-
-import java.util.*;
-
-public class ArbitraryDataRenderManager extends Thread {
-
-    private static ArbitraryDataRenderManager instance;
-    private volatile boolean isStopping = false;
-
-    /**
-     * Map to keep track of authorized resources for rendering.
-     * Keyed by resource ID, with the authorization time as the value.
-     */
-    private Map<String, Long> authorizedResources = Collections.synchronizedMap(new HashMap<>());
-
-    private static long AUTHORIZATION_TIMEOUT = 60 * 60 * 1000L; // 1 hour
-
-
-    public ArbitraryDataRenderManager() {
-
-    }
-
-    public static ArbitraryDataRenderManager getInstance() {
-        if (instance == null)
-            instance = new ArbitraryDataRenderManager();
-
-        return instance;
-    }
-
-    @Override
-    public void run() {
-        Thread.currentThread().setName("Arbitrary Data Render Manager");
-
-        try {
-            while (!isStopping) {
-                Thread.sleep(60000);
-
-                Long now = NTP.getTime();
-                this.cleanup(now);
-            }
-        } catch (InterruptedException e) {
-            // Fall-through to exit thread...
-        }
-    }
-
-    public void shutdown() {
-        isStopping = true;
-        this.interrupt();
-    }
-
-    public void cleanup(Long now) {
-        if (now == null) {
-            return;
-        }
-        final long minimumTimestamp = now - AUTHORIZATION_TIMEOUT;
-        this.authorizedResources.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < minimumTimestamp);
-    }
-
-    public boolean isAuthorized(ArbitraryDataResource resource) {
-        ArbitraryDataResource broadResource = new ArbitraryDataResource(resource.getResourceId(), null, null, null);
-
-        for (String authorizedResourceKey : this.authorizedResources.keySet()) {
-            if (authorizedResourceKey != null && resource != null) {
-                // Check for exact match
-                if (Objects.equals(authorizedResourceKey, resource.getUniqueKey())) {
-                    return true;
-                }
-                // Check for a broad authorization (which applies to all services and identifiers under an authorized name)
-                if (Objects.equals(authorizedResourceKey, broadResource.getUniqueKey())) {
-                    return true;
-                }
-            }
-        }
-        return false;
-    }
-
-    public void addToAuthorizedResources(ArbitraryDataResource resource) {
-        if (!this.isAuthorized(resource)) {
-            this.authorizedResources.put(resource.getUniqueKey(), NTP.getTime());
-        }
-    }
-
-}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java

@@ -1,500 +0,0 @@
-package org.qortal.controller.arbitrary;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.data.transaction.ArbitraryTransactionData;
-import org.qortal.data.transaction.TransactionData;
-import org.qortal.list.ResourceListManager;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.settings.Settings;
-import org.qortal.transaction.Transaction;
-import org.qortal.utils.ArbitraryTransactionUtils;
-import org.qortal.utils.Base58;
-import org.qortal.utils.FilesystemUtils;
-import org.qortal.utils.NTP;
-
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
-public class ArbitraryDataStorageManager extends Thread {
-
-    public enum StoragePolicy {
-        FOLLOWED_OR_VIEWED,
-        FOLLOWED,
-        VIEWED,
-        ALL,
-        NONE
-    }
-
-    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataStorageManager.class);
-
-    private static ArbitraryDataStorageManager instance;
-    private volatile boolean isStopping = false;
-
-    private Long storageCapacity = null;
-    private long totalDirectorySize = 0L;
-    private long lastDirectorySizeCheck = 0;
-
-    private List<ArbitraryTransactionData> hostedTransactions;
-
-    private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
-
-    /** Treat storage as full at 90% usage, to reduce risk of going over the limit.
-     * This is necessary because we don't calculate total storage values before every write.
-     * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit.
-     * This must be lower than DELETION_THRESHOLD. */
-    private static final double STORAGE_FULL_THRESHOLD = 0.90f; // 90%
-
-    /** Start deleting files once we reach 98% usage.
-     * This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */
-    public static final double DELETION_THRESHOLD = 0.98f; // 98%
-
-    public ArbitraryDataStorageManager() {
-    }
-
-    public static ArbitraryDataStorageManager getInstance() {
-        if (instance == null)
-            instance = new ArbitraryDataStorageManager();
-
-        return instance;
-    }
-
-    @Override
-    public void run() {
-        Thread.currentThread().setName("Arbitrary Data Storage Manager");
-        try {
-            while (!isStopping) {
-                Thread.sleep(1000);
-
-                // Don't run if QDN is disabled
-                if (!Settings.getInstance().isQdnEnabled()) {
-                    Thread.sleep(60 * 60 * 1000L);
-                    continue;
-                }
-
-                Long now = NTP.getTime();
-                if (now == null) {
-                    continue;
-                }
-
-                // Check the total directory size if we haven't in a while
-                if (this.shouldCalculateDirectorySize(now)) {
-                    this.calculateDirectorySize(now);
-                }
-
-                Thread.sleep(59000);
-            }
-        } catch (InterruptedException e) {
-            // Fall-through to exit thread...
-        }
-    }
-
-    public void shutdown() {
-        isStopping = true;
-        this.interrupt();
-        instance = null;
-    }
-
-    /**
-     * Check if data relating to a transaction is allowed to
-     * exist on this node, therefore making it a mirror for this data.
-     *
-     * @param arbitraryTransactionData - the transaction
-     * @return boolean - whether to prefetch or not
-     */
-    public boolean canStoreData(ArbitraryTransactionData arbitraryTransactionData) {
-        String name = arbitraryTransactionData.getName();
-
-        // We already have RAW_DATA on chain, so we only need to store data associated with hashes
-        if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) {
-            return false;
-        }
-
-        // Don't store data unless it's an allowed type (public/private)
-        if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
-            return false;
-        }
-
-        // Don't check for storage limits here, as it can cause the cleanup manager to delete existing data
-
-        // Check if our storage policy and and lists allow us to host data for this name
-        switch (Settings.getInstance().getStoragePolicy()) {
-            case FOLLOWED_OR_VIEWED:
-            case ALL:
-            case VIEWED:
-                // If the policy includes viewed data, we can host it as long as it's not blocked
-                return !this.isNameBlocked(name);
-
-            case FOLLOWED:
-                // If the policy is for followed data only, we have to be following it
-                return this.isFollowingName(name);
-
-                // For NONE or all else, we shouldn't host this data
-            case NONE:
-            default:
-                return false;
-        }
-    }
-
-    /**
-     * Check if data relating to a transaction should be downloaded
-     * automatically, making this node a mirror for that data.
-     *
-     * @param arbitraryTransactionData - the transaction
-     * @return boolean - whether to prefetch or not
-     */
-    public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
-        String name = arbitraryTransactionData.getName();
-
-        // Only fetch data associated with hashes, as we already have RAW_DATA
-        if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) {
-            return false;
-        }
-
-        // Don't fetch anything more if we're (nearly) out of space
-        // Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
-        // avoid a fetch/delete loop
-        if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) {
-            return false;
-        }
-
-        // Don't fetch anything if we're (nearly) out of space for this name
-        // Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
-        // avoid a fetch/delete loop
-        if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
-            return false;
-        }
-
-        // Don't store data unless it's an allowed type (public/private)
-        if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
-            return false;
-        }
-
-        // Handle transactions without names differently
-        if (name == null) {
-            return this.shouldPreFetchDataWithoutName();
-        }
-
-        // Never fetch data from blocked names, even if they are followed
-        if (this.isNameBlocked(name)) {
-            return false;
-        }
-
-        switch (Settings.getInstance().getStoragePolicy()) {
-            case FOLLOWED:
-            case FOLLOWED_OR_VIEWED:
-                return this.isFollowingName(name);
-                
-            case ALL:
-                return true;
-
-            case NONE:
-            case VIEWED:
-            default:
-                return false;
-        }
-    }
-
-    /**
-     * Don't call this method directly.
-     * Use the wrapper method shouldPreFetchData() instead, as it contains
-     * additional checks.
-     *
-     * @return boolean - whether the storage policy allows for unnamed data
-     */
-    private boolean shouldPreFetchDataWithoutName() {
-        switch (Settings.getInstance().getStoragePolicy()) {
-            case ALL:
-                return true;
-
-            case NONE:
-            case VIEWED:
-            case FOLLOWED:
-            case FOLLOWED_OR_VIEWED:
-            default:
-                return false;
-        }
-    }
-
-    private boolean isDataTypeAllowed(ArbitraryTransactionData arbitraryTransactionData) {
-        byte[] secret = arbitraryTransactionData.getSecret();
-        boolean hasSecret = (secret != null && secret.length == 32);
-
-        if (!Settings.getInstance().isPrivateDataEnabled() && !hasSecret) {
-            // Private data isn't enabled so we can't store data without a valid secret
-            return false;
-        }
-        if (!Settings.getInstance().isPublicDataEnabled() && hasSecret) {
-            // Public data isn't enabled so we can't store data with a secret
-            return false;
-        }
-        return true;
-    }
-
-    public boolean isNameBlocked(String name) {
-        return ResourceListManager.getInstance().listContains("blockedNames", name, false);
-    }
-
-    private boolean isFollowingName(String name) {
-        return ResourceListManager.getInstance().listContains("followedNames", name, false);
-    }
-
-    public List<String> followedNames() {
-        return ResourceListManager.getInstance().getStringsInList("followedNames");
-    }
-
-    private int followedNamesCount() {
-        return ResourceListManager.getInstance().getItemCountForList("followedNames");
-    }
-
-
-    // Hosted data
-
-    public List<ArbitraryTransactionData> listAllHostedTransactions(Repository repository, Integer limit, Integer offset) {
-        // Load from cache if we can, to avoid disk reads
-        if (this.hostedTransactions != null) {
-            return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset);
-        }
-
-        List<ArbitraryTransactionData> arbitraryTransactionDataList = new ArrayList<>();
-
-        // Find all hosted paths
-        List<Path> allPaths = this.findAllHostedPaths();
-
-        // Loop through each path and attempt to match it to a signature
-        for (Path path : allPaths) {
-            try {
-                String[] contents = path.toFile().list();
-                if (contents == null || contents.length == 0) {
-                    // Ignore empty directories
-                    continue;
-                }
-
-                String signature58 = path.getFileName().toString();
-                byte[] signature = Base58.decode(signature58);
-                TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-                if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) {
-                    continue;
-                }
-                arbitraryTransactionDataList.add((ArbitraryTransactionData) transactionData);
-
-            } catch (DataException e) {
-                continue;
-            }
-        }
-
-        // Sort by newest first
-        arbitraryTransactionDataList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed());
-
-        // Update cache
-        this.hostedTransactions = arbitraryTransactionDataList;
-
-        return ArbitraryTransactionUtils.limitOffsetTransactions(arbitraryTransactionDataList, limit, offset);
-    }
-
-    /**
-     * Warning: this method will walk through the entire data directory
-     * Do not call it too frequently as it could create high disk load
-     * in environments with a large amount of hosted data.
-     * @return a list of paths that are being hosted
-     */
-    public List<Path> findAllHostedPaths() {
-        Path dataPath = Paths.get(Settings.getInstance().getDataPath());
-        Path tempPath = Paths.get(Settings.getInstance().getTempDataPath());
-
-        // Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length
-        // Also exclude the _temp and _misc paths if present
-        List<Path> allPaths = new ArrayList<>();
-        try {
-            allPaths = Files.walk(dataPath, 3)
-                    .filter(Files::isDirectory)
-                    .filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString())
-                            && !path.toString().contains("_misc")
-                            && path.getFileName().toString().length() > 32)
-                    .collect(Collectors.toList());
-        }
-        catch (IOException e) {
-            LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
-        }
-
-        return allPaths;
-    }
-
-    public void invalidateHostedTransactionsCache() {
-        this.hostedTransactions = null;
-    }
-
-
-    // Size limits
-
-    /**
-     * Rate limit to reduce IO load
-     */
-    public boolean shouldCalculateDirectorySize(Long now) {
-        if (now == null) {
-            return false;
-        }
-        // If storage capacity is null, we need to calculate it
-        if (this.storageCapacity == null) {
-            return true;
-        }
-        // If we haven't checked for a while, we need to check it now
-        if (now - lastDirectorySizeCheck > DIRECTORY_SIZE_CHECK_INTERVAL) {
-            return true;
-        }
-
-        // We shouldn't check this time, as we want to reduce IO load on the SSD/HDD
-        return false;
-    }
-
-    public void calculateDirectorySize(Long now) {
-        if (now == null) {
-            return;
-        }
-
-        long totalSize = 0;
-        long remainingCapacity = 0;
-
-        // Calculate remaining capacity
-        try {
-            remainingCapacity = this.getRemainingUsableStorageCapacity();
-        } catch (IOException e) {
-            LOGGER.info("Unable to calculate remaining storage capacity: {}", e.getMessage());
-            return;
-        }
-
-        // Calculate total size of data directory
-        LOGGER.trace("Calculating data directory size...");
-        Path dataDirectoryPath = Paths.get(Settings.getInstance().getDataPath());
-        if (dataDirectoryPath.toFile().exists()) {
-            totalSize += FileUtils.sizeOfDirectory(dataDirectoryPath.toFile());
-        }
-
-        // Add total size of temp directory, if it's not already inside the data directory
-        Path tempDirectoryPath = Paths.get(Settings.getInstance().getTempDataPath());
-        if (tempDirectoryPath.toFile().exists()) {
-            if (!FilesystemUtils.isChild(tempDirectoryPath, dataDirectoryPath)) {
-                LOGGER.trace("Calculating temp directory size...");
-                totalSize += FileUtils.sizeOfDirectory(dataDirectoryPath.toFile());
-            }
-        }
-
-        this.totalDirectorySize = totalSize;
-        this.lastDirectorySizeCheck = now;
-
-        // It's essential that used space (this.totalDirectorySize) is included in the storage capacity
-        LOGGER.trace("Calculating total storage capacity...");
-        long storageCapacity = remainingCapacity + this.totalDirectorySize;
-
-        // Make sure to limit the storage capacity if the user is overriding it in the settings
-        if (Settings.getInstance().getMaxStorageCapacity() != null) {
-            storageCapacity = Math.min(storageCapacity, Settings.getInstance().getMaxStorageCapacity());
-        }
-        this.storageCapacity = storageCapacity;
-
-        LOGGER.info("Total used: {} bytes, Total capacity: {} bytes", this.totalDirectorySize, this.storageCapacity);
-    }
-
-    private long getRemainingUsableStorageCapacity() throws IOException {
-        // Create data directory if it doesn't exist so that we can perform calculations on it
-        Path dataDirectoryPath = Paths.get(Settings.getInstance().getDataPath());
-        if (!dataDirectoryPath.toFile().exists()) {
-            Files.createDirectories(dataDirectoryPath);
-        }
-
-        return dataDirectoryPath.toFile().getUsableSpace();
-    }
-
-    public long getTotalDirectorySize() {
-        return this.totalDirectorySize;
-    }
-
-    public boolean isStorageSpaceAvailable(double threshold) {
-        if (!this.isStorageCapacityCalculated()) {
-            return false;
-        }
-
-        long maxStorageCapacity = (long)((double)this.storageCapacity * threshold);
-        if (this.totalDirectorySize >= maxStorageCapacity) {
-            return false;
-        }
-        return true;
-    }
-
-    public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) {
-        if (!this.isStorageSpaceAvailable(threshold)) {
-            // No storage space available at all, so no need to check this name
-            return false;
-        }
-
-        if (name == null) {
-            // This transaction doesn't have a name, so fall back to total space limitations
-            return true;
-        }
-
-        int followedNamesCount = this.followedNamesCount();
-        if (followedNamesCount == 0) {
-            // Not following any names, so we have space
-            return true;
-        }
-
-        long totalSizeForName = 0;
-        long maxStoragePerName = this.storageCapacityPerName(threshold);
-
-        // Fetch all hosted transactions
-        List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository, null, null);
-        for (ArbitraryTransactionData transactionData : hostedTransactions) {
-            String transactionName = transactionData.getName();
-            if (!Objects.equals(name, transactionName)) {
-                // Transaction relates to a different name
-                continue;
-            }
-
-            totalSizeForName += transactionData.getSize();
-        }
-
-        // Have we reached the limit for this name?
-        if (totalSizeForName > maxStoragePerName) {
-            return false;
-        }
-
-        return true;
-    }
-
-    public long storageCapacityPerName(double threshold) {
-        int followedNamesCount = this.followedNamesCount();
-        if (followedNamesCount == 0) {
-            // Not following any names, so we have the total space available
-            return this.getStorageCapacityIncludingThreshold(threshold);
-        }
-
-        double maxStorageCapacity = (double)this.storageCapacity * threshold;
-        long maxStoragePerName = (long)(maxStorageCapacity / (double)followedNamesCount);
-
-        return maxStoragePerName;
-    }
-
-    public boolean isStorageCapacityCalculated() {
-        return (this.storageCapacity != null);
-    }
-
-    public Long getStorageCapacity() {
-        return this.storageCapacity;
-    }
-
-    public Long getStorageCapacityIncludingThreshold(double threshold) {
-        if (this.storageCapacity == null) {
-            return null;
-        }
-        return (long)(this.storageCapacity * threshold);
-    }
-}

src/main/java/org/qortal/controller/repository/AtStatesPruner.java

@@ -1,110 +0,0 @@
-package org.qortal.controller.repository;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.controller.Controller;
-import org.qortal.controller.Synchronizer;
-import org.qortal.data.block.BlockData;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.settings.Settings;
-import org.qortal.utils.NTP;
-
-public class AtStatesPruner implements Runnable {
-
-	private static final Logger LOGGER = LogManager.getLogger(AtStatesPruner.class);
-
-	@Override
-	public void run() {
-		Thread.currentThread().setName("AT States pruner");
-
-		boolean archiveMode = false;
-		if (!Settings.getInstance().isTopOnly()) {
-			// Top-only mode isn't enabled, but we might want to prune for the purposes of archiving
-			if (!Settings.getInstance().isArchiveEnabled()) {
-				// No pruning or archiving, so we must not prune anything
-				return;
-			}
-			else {
-				// We're allowed to prune blocks that have already been archived
-				archiveMode = true;
-			}
-		}
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			int pruneStartHeight = repository.getATRepository().getAtPruneHeight();
-
-			repository.discardChanges();
-			repository.getATRepository().rebuildLatestAtStates();
-
-			while (!Controller.isStopping()) {
-				repository.discardChanges();
-
-				Thread.sleep(Settings.getInstance().getAtStatesPruneInterval());
-
-				BlockData chainTip = Controller.getInstance().getChainTip();
-				if (chainTip == null || NTP.getTime() == null)
-					continue;
-
-				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Synchronizer.getInstance().isSynchronizing())
-					continue;
-
-				// Prune AT states for all blocks up until our latest minus pruneBlockLimit
-				final int ourLatestHeight = chainTip.getHeight();
-				int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit();
-
-				// In archive mode we are only allowed to trim blocks that have already been archived
-				if (archiveMode) {
-					upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1;
-
-					// TODO: validate that the actual archived data exists before pruning it?
-				}
-
-				int upperBatchHeight = pruneStartHeight + Settings.getInstance().getAtStatesPruneBatchSize();
-				int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight);
-
-				if (pruneStartHeight >= upperPruneHeight)
-					continue;
-
-				LOGGER.debug(String.format("Pruning AT states between blocks %d and %d...", pruneStartHeight, upperPruneHeight));
-
-				int numAtStatesPruned = repository.getATRepository().pruneAtStates(pruneStartHeight, upperPruneHeight);
-				repository.saveChanges();
-				int numAtStateDataRowsTrimmed = repository.getATRepository().trimAtStates(
-						pruneStartHeight, upperPruneHeight, Settings.getInstance().getAtStatesTrimLimit());
-				repository.saveChanges();
-
-				if (numAtStatesPruned > 0 || numAtStateDataRowsTrimmed > 0) {
-					final int finalPruneStartHeight = pruneStartHeight;
-					LOGGER.debug(() -> String.format("Pruned %d AT state%s between blocks %d and %d",
-							numAtStatesPruned, (numAtStatesPruned != 1 ? "s" : ""),
-							finalPruneStartHeight, upperPruneHeight));
-				} else {
-					// Can we move onto next batch?
-					if (upperPrunableHeight > upperBatchHeight) {
-						pruneStartHeight = upperBatchHeight;
-						repository.getATRepository().setAtPruneHeight(pruneStartHeight);
-						repository.getATRepository().rebuildLatestAtStates();
-						repository.saveChanges();
-
-						final int finalPruneStartHeight = pruneStartHeight;
-						LOGGER.debug(() -> String.format("Bumping AT state base prune height to %d", finalPruneStartHeight));
-					}
-					else {
-						// We've pruned up to the upper prunable height
-						// Back off for a while to save CPU for syncing
-						repository.discardChanges();
-						Thread.sleep(5*60*1000L);
-					}
-				}
-			}
-		} catch (DataException e) {
-			LOGGER.warn(String.format("Repository issue trying to prune AT states: %s", e.getMessage()));
-		} catch (InterruptedException e) {
-			// Time to exit
-		}
-	}
-
-}

src/main/java/org/qortal/controller/repository/BlockArchiver.java

@@ -1,114 +0,0 @@
-package org.qortal.controller.repository;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.controller.Controller;
-import org.qortal.controller.Synchronizer;
-import org.qortal.data.block.BlockData;
-import org.qortal.repository.*;
-import org.qortal.settings.Settings;
-import org.qortal.transform.TransformationException;
-import org.qortal.utils.NTP;
-
-import java.io.IOException;
-
-public class BlockArchiver implements Runnable {
-
-	private static final Logger LOGGER = LogManager.getLogger(BlockArchiver.class);
-
-	private static final long INITIAL_SLEEP_PERIOD = 0L; // TODO: 5 * 60 * 1000L + 1234L; // ms
-
-	public void run() {
-		Thread.currentThread().setName("Block archiver");
-
-		if (!Settings.getInstance().isArchiveEnabled()) {
-			return;
-		}
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			// Don't even start building until initial rush has ended
-			Thread.sleep(INITIAL_SLEEP_PERIOD);
-
-			int startHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight();
-
-			// Don't attempt to archive if we have no ATStatesHeightIndex, as it will be too slow
-			boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex();
-			if (!hasAtStatesHeightIndex) {
-				LOGGER.info("Unable to start block archiver due to missing ATStatesHeightIndex. Bootstrapping is recommended.");
-				repository.discardChanges();
-				return;
-			}
-
-			LOGGER.info("Starting block archiver from height {}...", startHeight);
-
-			while (!Controller.isStopping()) {
-				repository.discardChanges();
-
-				Thread.sleep(Settings.getInstance().getArchiveInterval());
-
-				BlockData chainTip = Controller.getInstance().getChainTip();
-				if (chainTip == null || NTP.getTime() == null) {
-					continue;
-				}
-
-				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Synchronizer.getInstance().isSynchronizing()) {
-					continue;
-				}
-
-				// Don't attempt to archive if we're not synced yet
-				final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
-				if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) {
-					continue;
-				}
-
-
-				// Build cache of blocks
-				try {
-					final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
-					BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository);
-					BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
-					switch (result) {
-						case OK:
-							// Increment block archive height
-							startHeight += writer.getWrittenCount();
-							repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight);
-							repository.saveChanges();
-							break;
-
-						case STOPPING:
-							return;
-
-							// We've reached the limit of the blocks we can archive
-							// Sleep for a while to allow more to become available
-						case NOT_ENOUGH_BLOCKS:
-							// We didn't reach our file size target, so that must mean that we don't have enough blocks
-							// yet or something went wrong. Sleep for a while and then try again.
-							repository.discardChanges();
-							Thread.sleep(60 * 60 * 1000L); // 1 hour
-							break;
-
-						case BLOCK_NOT_FOUND:
-							// We tried to archive a block that didn't exist. This is a major failure and likely means
-							// that a bootstrap or re-sync is needed. Try again every minute until then.
-							LOGGER.info("Error: block not found when building archive. If this error persists, " +
-									"a bootstrap or re-sync may be needed.");
-							repository.discardChanges();
-							Thread.sleep( 60 * 1000L); // 1 minute
-							break;
-					}
-
-				} catch (IOException | TransformationException e) {
-					LOGGER.info("Caught exception when creating block cache", e);
-				}
-
-			}
-		} catch (DataException e) {
-			LOGGER.info("Caught exception when creating block cache", e);
-		} catch (InterruptedException e) {
-			// Do nothing
-		}
-
-	}
-
-}

src/main/java/org/qortal/controller/repository/BlockPruner.java

@@ -1,115 +0,0 @@
-package org.qortal.controller.repository;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.controller.Controller;
-import org.qortal.controller.Synchronizer;
-import org.qortal.data.block.BlockData;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.settings.Settings;
-import org.qortal.utils.NTP;
-
-public class BlockPruner implements Runnable {
-
-	private static final Logger LOGGER = LogManager.getLogger(BlockPruner.class);
-
-	@Override
-	public void run() {
-		Thread.currentThread().setName("Block pruner");
-
-		boolean archiveMode = false;
-		if (!Settings.getInstance().isTopOnly()) {
-			// Top-only mode isn't enabled, but we might want to prune for the purposes of archiving
-			if (!Settings.getInstance().isArchiveEnabled()) {
-				// No pruning or archiving, so we must not prune anything
-				return;
-			}
-			else {
-				// We're allowed to prune blocks that have already been archived
-				archiveMode = true;
-			}
-		}
-
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			int pruneStartHeight = repository.getBlockRepository().getBlockPruneHeight();
-
-			// Don't attempt to prune if we have no ATStatesHeightIndex, as it will be too slow
-			boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex();
-			if (!hasAtStatesHeightIndex) {
-				LOGGER.info("Unable to start block pruner due to missing ATStatesHeightIndex. Bootstrapping is recommended.");
-				return;
-			}
-
-			while (!Controller.isStopping()) {
-				repository.discardChanges();
-
-				Thread.sleep(Settings.getInstance().getBlockPruneInterval());
-
-				BlockData chainTip = Controller.getInstance().getChainTip();
-				if (chainTip == null || NTP.getTime() == null)
-					continue;
-
-				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Synchronizer.getInstance().isSynchronizing()) {
-					continue;
-				}
-
-				// Don't attempt to prune if we're not synced yet
-				final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
-				if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) {
-					continue;
-				}
-
-				// Prune all blocks up until our latest minus pruneBlockLimit
-				final int ourLatestHeight = chainTip.getHeight();
-				int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit();
-
-				// In archive mode we are only allowed to trim blocks that have already been archived
-				if (archiveMode) {
-					upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1;
-				}
-
-				int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize();
-				int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight);
-
-				if (pruneStartHeight >= upperPruneHeight) {
-					continue;
-				}
-
-				LOGGER.debug(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight));
-
-				int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight);
-				repository.saveChanges();
-
-				if (numBlocksPruned > 0) {
-					LOGGER.debug(String.format("Pruned %d block%s between %d and %d",
-							numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""),
-							pruneStartHeight, upperPruneHeight));
-				} else {
-					final int nextPruneHeight = upperPruneHeight + 1;
-					repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight);
-					repository.saveChanges();
-					LOGGER.debug(String.format("Bumping block base prune height to %d", pruneStartHeight));
-
-					// Can we move onto next batch?
-					if (upperPrunableHeight > nextPruneHeight) {
-						pruneStartHeight = nextPruneHeight;
-					}
-					else {
-						// We've pruned up to the upper prunable height
-						// Back off for a while to save CPU for syncing
-						repository.discardChanges();
-						Thread.sleep(10*60*1000L);
-					}
-				}
-			}
-		} catch (DataException e) {
-			LOGGER.warn(String.format("Repository issue trying to prune blocks: %s", e.getMessage()));
-		} catch (InterruptedException e) {
-			// Time to exit
-		}
-	}
-
-}

src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java

@@ -1,372 +0,0 @@
-package org.qortal.controller.repository;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.account.PublicKeyAccount;
-import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
-import org.qortal.data.naming.NameData;
-import org.qortal.data.transaction.*;
-import org.qortal.naming.Name;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.transaction.Transaction.TransactionType;
-import org.qortal.utils.Unicode;
-
-import java.util.*;
-
-public class NamesDatabaseIntegrityCheck {
-
-    private static final Logger LOGGER = LogManager.getLogger(NamesDatabaseIntegrityCheck.class);
-
-    private static final List<TransactionType> ALL_NAME_TX_TYPE = Arrays.asList(
-            TransactionType.REGISTER_NAME,
-            TransactionType.UPDATE_NAME,
-            TransactionType.BUY_NAME,
-            TransactionType.SELL_NAME
-    );
-
-    private List<TransactionData> nameTransactions = new ArrayList<>();
-
-    public int rebuildName(String name, Repository repository) {
-        int modificationCount = 0;
-        try {
-            List<TransactionData> transactions = this.fetchAllTransactionsInvolvingName(name, repository);
-            if (transactions.isEmpty()) {
-                // This name was never registered, so there's nothing to do
-                return modificationCount;
-            }
-
-            // Loop through each past transaction and re-apply it to the Names table
-            for (TransactionData currentTransaction : transactions) {
-
-                // Process REGISTER_NAME transactions
-                if (currentTransaction.getType() == TransactionType.REGISTER_NAME) {
-                    RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) currentTransaction;
-                    Name nameObj = new Name(repository, registerNameTransactionData);
-                    nameObj.register();
-                    modificationCount++;
-                    LOGGER.trace("Processed REGISTER_NAME transaction for name {}", name);
-                }
-
-                // Process UPDATE_NAME transactions
-                if (currentTransaction.getType() == TransactionType.UPDATE_NAME) {
-                    UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) currentTransaction;
-
-                    if (Objects.equals(updateNameTransactionData.getNewName(), name) &&
-                            !Objects.equals(updateNameTransactionData.getName(), updateNameTransactionData.getNewName())) {
-                        // This renames an existing name, so we need to process that instead
-                        this.rebuildName(updateNameTransactionData.getName(), repository);
-                    }
-                    else {
-                        Name nameObj = new Name(repository, name);
-                        if (nameObj != null && nameObj.getNameData() != null) {
-                            nameObj.update(updateNameTransactionData);
-                            modificationCount++;
-                            LOGGER.trace("Processed UPDATE_NAME transaction for name {}", name);
-                        } else {
-                            // Something went wrong
-                            throw new DataException(String.format("Name data not found for name %s", updateNameTransactionData.getName()));
-                        }
-                    }
-                }
-
-                // Process SELL_NAME transactions
-                if (currentTransaction.getType() == TransactionType.SELL_NAME) {
-                    SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) currentTransaction;
-                    Name nameObj = new Name(repository, sellNameTransactionData.getName());
-                    if (nameObj != null && nameObj.getNameData() != null) {
-                        nameObj.sell(sellNameTransactionData);
-                        modificationCount++;
-                        LOGGER.trace("Processed SELL_NAME transaction for name {}", name);
-                    }
-                    else {
-                        // Something went wrong
-                        throw new DataException(String.format("Name data not found for name %s", sellNameTransactionData.getName()));
-                    }
-                }
-
-                // Process BUY_NAME transactions
-                if (currentTransaction.getType() == TransactionType.BUY_NAME) {
-                    BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) currentTransaction;
-                    Name nameObj = new Name(repository, buyNameTransactionData.getName());
-                    if (nameObj != null && nameObj.getNameData() != null) {
-                        nameObj.buy(buyNameTransactionData);
-                        modificationCount++;
-                        LOGGER.trace("Processed BUY_NAME transaction for name {}", name);
-                    }
-                    else {
-                        // Something went wrong
-                        throw new DataException(String.format("Name data not found for name %s", buyNameTransactionData.getName()));
-                    }
-                }
-            }
-
-        } catch (DataException e) {
-            LOGGER.info("Unable to run integrity check for name {}: {}", name, e.getMessage());
-        }
-
-        return modificationCount;
-    }
-
-    public int rebuildAllNames() {
-        int modificationCount = 0;
-        try (final Repository repository = RepositoryManager.getRepository()) {
-            List<String> names = this.fetchAllNames(repository);
-            for (String name : names) {
-                modificationCount += this.rebuildName(name, repository);
-            }
-            repository.saveChanges();
-        }
-        catch (DataException e) {
-            LOGGER.info("Error when running integrity check for all names: {}", e.getMessage());
-        }
-
-        //LOGGER.info("modificationCount: {}", modificationCount);
-        return modificationCount;
-    }
-
-    public void runIntegrityCheck() {
-        boolean integrityCheckFailed = false;
-        try (final Repository repository = RepositoryManager.getRepository()) {
-
-            // Fetch all the (confirmed) REGISTER_NAME transactions
-            List<RegisterNameTransactionData> registerNameTransactions = this.fetchRegisterNameTransactions();
-
-            // Loop through each REGISTER_NAME txn signature and request the full transaction data
-            for (RegisterNameTransactionData registerNameTransactionData : registerNameTransactions) {
-                String registeredName = registerNameTransactionData.getName();
-                NameData nameData = repository.getNameRepository().fromName(registeredName);
-
-                // Check to see if this name has been updated or bought at any point
-                TransactionData latestUpdate = this.fetchLatestModificationTransactionInvolvingName(registeredName, repository);
-                if (latestUpdate == null) {
-                    // Name was never updated once registered
-                    // We expect this name to still be registered to this transaction's creator
-
-                    if (nameData == null) {
-                        LOGGER.info("Error: registered name {} doesn't exist in Names table. Adding...", registeredName);
-                        integrityCheckFailed = true;
-                    }
-                    else {
-                        LOGGER.trace("Registered name {} is correctly registered", registeredName);
-                    }
-
-                    // Check the owner is correct
-                    PublicKeyAccount creator = new PublicKeyAccount(repository, registerNameTransactionData.getCreatorPublicKey());
-                    if (!Objects.equals(creator.getAddress(), nameData.getOwner())) {
-                        LOGGER.info("Error: registered name {} is owned by {}, but it should be {}",
-                                registeredName, nameData.getOwner(), creator.getAddress());
-                        integrityCheckFailed = true;
-                    }
-                    else {
-                        LOGGER.trace("Registered name {} has the correct owner", registeredName);
-                    }
-                }
-                else {
-                    // Check if owner is correct after update
-
-                    // Check for name updates
-                    if (latestUpdate.getType() == TransactionType.UPDATE_NAME) {
-                        UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) latestUpdate;
-                        PublicKeyAccount creator = new PublicKeyAccount(repository, updateNameTransactionData.getCreatorPublicKey());
-
-                        // When this name is the "new name", we expect the current owner to match the txn creator
-                        if (Objects.equals(updateNameTransactionData.getNewName(), registeredName)) {
-                            if (!Objects.equals(creator.getAddress(), nameData.getOwner())) {
-                                LOGGER.info("Error: registered name {} is owned by {}, but it should be {}",
-                                        registeredName, nameData.getOwner(), creator.getAddress());
-                                integrityCheckFailed = true;
-                            }
-                            else {
-                                LOGGER.trace("Registered name {} has the correct owner after being updated", registeredName);
-                            }
-                        }
-
-                        // When this name is the old name, we expect the "new name"'s owner to match the txn creator
-                        // The old name will then be unregistered, or re-registered.
-                        // FUTURE: check database integrity for names that have been updated and then the original name re-registered
-                        else if (Objects.equals(updateNameTransactionData.getName(), registeredName)) {
-                            String newName = updateNameTransactionData.getNewName();
-                            if (newName == null || newName.length() == 0) {
-                                // If new name is blank (or maybe null, just to be safe), it means that it stayed the same
-                                newName = registeredName;
-                            }
-                            NameData newNameData = repository.getNameRepository().fromName(newName);
-                            if (!Objects.equals(creator.getAddress(), newNameData.getOwner())) {
-                                LOGGER.info("Error: registered name {} is owned by {}, but it should be {}",
-                                        updateNameTransactionData.getNewName(), newNameData.getOwner(), creator.getAddress());
-                                integrityCheckFailed = true;
-                            }
-                            else {
-                                LOGGER.trace("Registered name {} has the correct owner after being updated", updateNameTransactionData.getNewName());
-                            }
-                        }
-
-                        else {
-                            LOGGER.info("Unhandled update case for name {}", registeredName);
-                        }
-                    }
-
-                    // Check for name buys
-                    else if (latestUpdate.getType() == TransactionType.BUY_NAME) {
-                        BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) latestUpdate;
-                        PublicKeyAccount creator = new PublicKeyAccount(repository, buyNameTransactionData.getCreatorPublicKey());
-                        if (!Objects.equals(creator.getAddress(), nameData.getOwner())) {
-                            LOGGER.info("Error: registered name {} is owned by {}, but it should be {}",
-                                    registeredName, nameData.getOwner(), creator.getAddress());
-                            integrityCheckFailed = true;
-                        }
-                        else {
-                            LOGGER.trace("Registered name {} has the correct owner after being bought", registeredName);
-                        }
-                    }
-
-                    // Check for name sells
-                    else if (latestUpdate.getType() == TransactionType.SELL_NAME) {
-                        SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) latestUpdate;
-                        PublicKeyAccount creator = new PublicKeyAccount(repository, sellNameTransactionData.getCreatorPublicKey());
-                        if (!Objects.equals(creator.getAddress(), nameData.getOwner())) {
-                            LOGGER.info("Error: registered name {} is owned by {}, but it should be {}",
-                                    registeredName, nameData.getOwner(), creator.getAddress());
-                            integrityCheckFailed = true;
-                        }
-                        else {
-                            LOGGER.trace("Registered name {} has the correct owner after being listed for sale", registeredName);
-                        }
-                    }
-
-                    else {
-                        LOGGER.info("Unhandled case for name {}", registeredName);
-                    }
-
-                }
-
-            }
-
-        } catch (DataException e) {
-            LOGGER.warn(String.format("Repository issue trying to trim online accounts signatures: %s", e.getMessage()));
-            integrityCheckFailed = true;
-        }
-
-        if (integrityCheckFailed) {
-            LOGGER.info("Registered names database integrity check failed. Bootstrapping is recommended.");
-        } else {
-            LOGGER.info("Registered names database integrity check passed.");
-        }
-    }
-
-    private List<RegisterNameTransactionData> fetchRegisterNameTransactions() {
-        List<RegisterNameTransactionData> registerNameTransactions = new ArrayList<>();
-
-        for (TransactionData transactionData : this.nameTransactions) {
-            if (transactionData.getType() == TransactionType.REGISTER_NAME) {
-                RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData;
-                registerNameTransactions.add(registerNameTransactionData);
-            }
-        }
-        return registerNameTransactions;
-    }
-
-    private void fetchAllNameTransactions(Repository repository) throws DataException {
-        List<TransactionData> nameTransactions = new ArrayList<>();
-
-        // Fetch all the confirmed REGISTER_NAME transaction signatures
-        List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(
-                null, null, null, ALL_NAME_TX_TYPE, null, null,
-                null, ConfirmationStatus.CONFIRMED, null, null, false);
-
-        for (byte[] signature : signatures) {
-            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-            nameTransactions.add(transactionData);
-        }
-        this.nameTransactions = nameTransactions;
-    }
-
-    public List<TransactionData> fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException {
-        List<byte[]> signatures = new ArrayList<>();
-        String reducedName = Unicode.sanitize(name);
-
-        List<byte[]> registerNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
-                TransactionType.REGISTER_NAME, Arrays.asList("(name = ? OR reduced_name = ?)"), Arrays.asList(name, reducedName));
-        signatures.addAll(registerNameTransactions);
-
-        List<byte[]> updateNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
-                TransactionType.UPDATE_NAME,
-                Arrays.asList("(name = ? OR new_name = ? OR (reduced_new_name != '' AND reduced_new_name = ?))"),
-                Arrays.asList(name, name, reducedName));
-        signatures.addAll(updateNameTransactions);
-
-        List<byte[]> sellNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
-                TransactionType.SELL_NAME, Arrays.asList("name = ?"), Arrays.asList(name));
-        signatures.addAll(sellNameTransactions);
-
-        List<byte[]> buyNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
-                TransactionType.BUY_NAME, Arrays.asList("name = ?"), Arrays.asList(name));
-        signatures.addAll(buyNameTransactions);
-
-        List<TransactionData> transactions = new ArrayList<>();
-        for (byte[] signature : signatures) {
-            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-            // Filter out any unconfirmed transactions
-            if (transactionData.getBlockHeight() != null && transactionData.getBlockHeight() > 0) {
-                transactions.add(transactionData);
-            }
-        }
-        return transactions;
-    }
-
-    private TransactionData fetchLatestModificationTransactionInvolvingName(String registeredName, Repository repository) throws DataException {
-        List<TransactionData> transactionsInvolvingName = this.fetchAllTransactionsInvolvingName(registeredName, repository);
-
-        // Get the latest update for this name (excluding REGISTER_NAME transactions)
-        TransactionData latestUpdateToName = transactionsInvolvingName.stream()
-                .filter(txn -> txn.getType() != TransactionType.REGISTER_NAME)
-                .max(Comparator.comparing(TransactionData::getTimestamp))
-                .orElse(null);
-
-        return latestUpdateToName;
-    }
-
-    private List<String> fetchAllNames(Repository repository) throws DataException {
-        List<String> names = new ArrayList<>();
-
-        // Fetch all the confirmed name transactions
-        if (this.nameTransactions.isEmpty()) {
-            this.fetchAllNameTransactions(repository);
-        }
-
-        for (TransactionData transactionData : this.nameTransactions) {
-
-            if ((transactionData instanceof RegisterNameTransactionData)) {
-                RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData;
-                if (!names.contains(registerNameTransactionData.getName())) {
-                    names.add(registerNameTransactionData.getName());
-                }
-            }
-            if ((transactionData instanceof UpdateNameTransactionData)) {
-                UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData;
-                if (!names.contains(updateNameTransactionData.getName())) {
-                    names.add(updateNameTransactionData.getName());
-                }
-                if (!names.contains(updateNameTransactionData.getNewName())) {
-                    names.add(updateNameTransactionData.getNewName());
-                }
-            }
-            if ((transactionData instanceof BuyNameTransactionData)) {
-                BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData;
-                if (!names.contains(buyNameTransactionData.getName())) {
-                    names.add(buyNameTransactionData.getName());
-                }
-            }
-            if ((transactionData instanceof SellNameTransactionData)) {
-                SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData;
-                if (!names.contains(sellNameTransactionData.getName())) {
-                    names.add(sellNameTransactionData.getName());
-                }
-            }
-        }
-        return names;
-    }
-
-}

src/main/java/org/qortal/controller/repository/PruneManager.java

@@ -1,160 +0,0 @@
-package org.qortal.controller.repository;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.qortal.controller.Controller;
-
-import org.qortal.data.block.BlockData;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.settings.Settings;
-import org.qortal.utils.DaemonThreadFactory;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-public class PruneManager {
-
-    private static final Logger LOGGER = LogManager.getLogger(PruneManager.class);
-
-    private static PruneManager instance;
-
-    private boolean isTopOnly = Settings.getInstance().isTopOnly();
-    private int pruneBlockLimit = Settings.getInstance().getPruneBlockLimit();
-
-    private ExecutorService executorService;
-
-    private PruneManager() {
-
-    }
-
-    public static synchronized PruneManager getInstance() {
-        if (instance == null)
-            instance = new PruneManager();
-
-        return instance;
-    }
-
-    public void start() {
-        this.executorService = Executors.newCachedThreadPool(new DaemonThreadFactory());
-
-        if (Settings.getInstance().isTopOnly()) {
-            // Top-only-sync
-            this.startTopOnlySyncMode();
-        }
-        else if (Settings.getInstance().isArchiveEnabled()) {
-            // Full node with block archive
-            this.startFullNodeWithBlockArchive();
-        }
-        else {
-            // Full node with full SQL support
-            this.startFullSQLNode();
-        }
-    }
-
-    /**
-     * Top-only-sync
-     * In this mode, we delete (prune) all blocks except
-     * a small number of recent ones. There is no need for
-     * trimming or archiving, because all relevant blocks
-     * are deleted.
-     */
-    private void startTopOnlySyncMode() {
-        this.startPruning();
-
-        // We don't need the block archive in top-only mode
-        this.deleteArchive();
-    }
-
-    /**
-     * Full node with block archive
-     * In this mode we archive trimmed blocks, and then
-     * prune archived blocks to keep the database small
-     */
-    private void startFullNodeWithBlockArchive() {
-        this.startTrimming();
-        this.startArchiving();
-        this.startPruning();
-    }
-
-    /**
-     * Full node with full SQL support
-     * In this mode we trim the database but don't prune
-     * or archive any data, because we want to maintain
-     * full SQL support of old blocks. This mode will not
-     * be actively maintained but can be used by those who
-     * need to perform SQL analysis on older blocks.
-     */
-    private void startFullSQLNode() {
-        this.startTrimming();
-    }
-
-
-    private void startPruning() {
-        this.executorService.execute(new AtStatesPruner());
-        this.executorService.execute(new BlockPruner());
-    }
-
-    private void startTrimming() {
-        this.executorService.execute(new AtStatesTrimmer());
-        this.executorService.execute(new OnlineAccountsSignaturesTrimmer());
-    }
-
-    private void startArchiving() {
-        this.executorService.execute(new BlockArchiver());
-    }
-
-    private void deleteArchive() {
-        if (!Settings.getInstance().isTopOnly()) {
-            LOGGER.error("Refusing to delete archive when not in top-only mode");
-        }
-
-        try {
-            Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive");
-            if (archivePath.toFile().exists()) {
-                LOGGER.info("Deleting block archive because we are in top-only mode...");
-                FileUtils.deleteDirectory(archivePath.toFile());
-            }
-
-        } catch (IOException e) {
-            LOGGER.info("Couldn't delete archive: {}", e.getMessage());
-        }
-    }
-
-    public void stop() {
-        this.executorService.shutdownNow();
-
-        try {
-            this.executorService.awaitTermination(2L, TimeUnit.SECONDS);
-        } catch (InterruptedException e) {
-            // We tried...
-        }
-    }
-
-    public boolean isBlockPruned(int height) throws DataException {
-        if (!this.isTopOnly) {
-            return false;
-        }
-
-        BlockData chainTip = Controller.getInstance().getChainTip();
-        if (chainTip == null) {
-            throw new DataException("Unable to determine chain tip when checking if a block is pruned");
-        }
-
-        if (height == 1) {
-            // We don't prune the genesis block
-            return false;
-        }
-
-        final int ourLatestHeight = chainTip.getHeight();
-        final int latestUnprunedHeight = ourLatestHeight - this.pruneBlockLimit;
-
-        return (height < latestUnprunedHeight);
-    }
-
-}

src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java

@@ -360,7 +360,6 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot {
 			case BOB_DONE:
 			case ALICE_REFUNDED:
 			case BOB_REFUNDED:
-			case ALICE_REFUNDING_A:
 				return true;
 
 			default:
@@ -1034,7 +1033,7 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot {
 			return;
 		}
 
-		byte[] secretA = BitcoinACCTv1.getInstance().findSecretA(repository, crossChainTradeData);
+		byte[] secretA = BitcoinACCTv1.findSecretA(repository, crossChainTradeData);
 		if (secretA == null) {
 			LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress()));
 			return;

src/main/java/org/qortal/controller/tradebot/DogecoinACCTv1TradeBot.java

@@ -1,885 +0,0 @@
-package org.qortal.controller.tradebot;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.bitcoinj.core.*;
-import org.bitcoinj.script.Script.ScriptType;
-import org.qortal.account.PrivateKeyAccount;
-import org.qortal.account.PublicKeyAccount;
-import org.qortal.api.model.crosschain.TradeBotCreateRequest;
-import org.qortal.asset.Asset;
-import org.qortal.crosschain.*;
-import org.qortal.crypto.Crypto;
-import org.qortal.data.at.ATData;
-import org.qortal.data.crosschain.CrossChainTradeData;
-import org.qortal.data.crosschain.TradeBotData;
-import org.qortal.data.transaction.BaseTransactionData;
-import org.qortal.data.transaction.DeployAtTransactionData;
-import org.qortal.data.transaction.MessageTransactionData;
-import org.qortal.group.Group;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.transaction.DeployAtTransaction;
-import org.qortal.transaction.MessageTransaction;
-import org.qortal.transaction.Transaction.ValidationResult;
-import org.qortal.transform.TransformationException;
-import org.qortal.transform.transaction.DeployAtTransactionTransformer;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static java.util.Arrays.stream;
-import static java.util.stream.Collectors.toMap;
-
-/**
- * Performing cross-chain trading steps on behalf of user.
- * <p>
- * We deal with three different independent state-spaces here:
- * <ul>
- * 	<li>Qortal blockchain</li>
- * 	<li>Foreign blockchain</li>
- * 	<li>Trade-bot entries</li>
- * </ul>
- */
-public class DogecoinACCTv1TradeBot implements AcctTradeBot {
-
-	private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv1TradeBot.class);
-
-	public enum State implements TradeBot.StateNameAndValueSupplier {
-		BOB_WAITING_FOR_AT_CONFIRM(10, false, false),
-		BOB_WAITING_FOR_MESSAGE(15, true, true),
-		BOB_WAITING_FOR_AT_REDEEM(25, true, true),
-		BOB_DONE(30, false, false),
-		BOB_REFUNDED(35, false, false),
-
-		ALICE_WAITING_FOR_AT_LOCK(85, true, true),
-		ALICE_DONE(95, false, false),
-		ALICE_REFUNDING_A(105, true, true),
-		ALICE_REFUNDED(110, false, false);
-
-		private static final Map<Integer, State> map = stream(State.values()).collect(toMap(state -> state.value, state -> state));
-
-		public final int value;
-		public final boolean requiresAtData;
-		public final boolean requiresTradeData;
-
-		State(int value, boolean requiresAtData, boolean requiresTradeData) {
-			this.value = value;
-			this.requiresAtData = requiresAtData;
-			this.requiresTradeData = requiresTradeData;
-		}
-
-		public static State valueOf(int value) {
-			return map.get(value);
-		}
-
-		@Override
-		public String getState() {
-			return this.name();
-		}
-
-		@Override
-		public int getStateValue() {
-			return this.value;
-		}
-	}
-
-	/** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */
-	private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms
-
-	private static DogecoinACCTv1TradeBot instance;
-
-	private final List<String> endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream()
-			.map(State::name)
-			.collect(Collectors.toUnmodifiableList());
-
-	private DogecoinACCTv1TradeBot() {
-	}
-
-	public static synchronized DogecoinACCTv1TradeBot getInstance() {
-		if (instance == null)
-			instance = new DogecoinACCTv1TradeBot();
-
-		return instance;
-	}
-
-	@Override
-	public List<String> getEndStates() {
-		return this.endStates;
-	}
-
-	/**
-	 * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for DOGE.
-	 * <p>
-	 * Generates:
-	 * <ul>
-	 * 	<li>new 'trade' private key</li>
-	 * </ul>
-	 * Derives:
-	 * <ul>
-	 * 	<li>'native' (as in Qortal) public key, public key hash, address (starting with Q)</li>
-	 * 	<li>'foreign' (as in Dogecoin) public key, public key hash</li>
-	 * </ul>
-	 * A Qortal AT is then constructed including the following as constants in the 'data segment':
-	 * <ul>
-	 * 	<li>'native'/Qortal 'trade' address - used as a MESSAGE contact</li>
-	 * 	<li>'foreign'/Dogecoin public key hash - used by Alice's P2SH scripts to allow redeem</li>
-	 * 	<li>QORT amount on offer by Bob</li>
-	 * 	<li>DOGE amount expected in return by Bob (from Alice)</li>
-	 * 	<li>trading timeout, in case things go wrong and everyone needs to refund</li>
-	 * </ul>
-	 * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network.
-	 * <p>
-	 * Trade-bot will wait for Bob's AT to be deployed before taking next step.
-	 * <p>
-	 * @param repository
-	 * @param tradeBotCreateRequest
-	 * @return raw, unsigned DEPLOY_AT transaction
-	 * @throws DataException
-	 */
-	public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException {
-		byte[] tradePrivateKey = TradeBot.generateTradePrivateKey();
-
-		byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey);
-		byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey);
-		String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey);
-
-		byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey);
-		byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey);
-
-		// Convert Dogecoin receiving address into public key hash (we only support P2PKH at this time)
-		Address dogecoinReceivingAddress;
-		try {
-			dogecoinReceivingAddress = Address.fromString(Dogecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress);
-		} catch (AddressFormatException e) {
-			throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress);
-		}
-		if (dogecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH)
-			throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress);
-
-		byte[] dogecoinReceivingAccountInfo = dogecoinReceivingAddress.getHash();
-
-		PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey);
-
-		// Deploy AT
-		long timestamp = NTP.getTime();
-		byte[] reference = creator.getLastReference();
-		long fee = 0L;
-		byte[] signature = null;
-		BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature);
-
-		String name = "QORT/DOGE ACCT";
-		String description = "QORT/DOGE cross-chain trade";
-		String aTType = "ACCT";
-		String tags = "ACCT QORT DOGE";
-		byte[] creationBytes = DogecoinACCTv1.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount,
-				tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout);
-		long amount = tradeBotCreateRequest.fundingQortAmount;
-
-		DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT);
-
-		DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData);
-		fee = deployAtTransaction.calcRecommendedFee();
-		deployAtTransactionData.setFee(fee);
-
-		DeployAtTransaction.ensureATAddress(deployAtTransactionData);
-		String atAddress = deployAtTransactionData.getAtAddress();
-
-		TradeBotData tradeBotData =  new TradeBotData(tradePrivateKey, DogecoinACCTv1.NAME,
-				State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value,
-				creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount,
-				tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress,
-				null, null,
-				SupportedBlockchain.DOGECOIN.name(),
-				tradeForeignPublicKey, tradeForeignPublicKeyHash,
-				tradeBotCreateRequest.foreignAmount, null, null, null, dogecoinReceivingAccountInfo);
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress));
-
-		// Attempt to backup the trade bot data
-		TradeBot.backupTradeBotData(repository, null);
-
-		// Return to user for signing and broadcast as we don't have their Qortal private key
-		try {
-			return DeployAtTransactionTransformer.toBytes(deployAtTransactionData);
-		} catch (TransformationException e) {
-			throw new DataException("Failed to transform DEPLOY_AT transaction?", e);
-		}
-	}
-
-	/**
-	 * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching DOGE to an existing offer.
-	 * <p>
-	 * Requires a chosen trade offer from Bob, passed by <tt>crossChainTradeData</tt>
-	 * and access to a Dogecoin wallet via <tt>xprv58</tt>.
-	 * <p>
-	 * The <tt>crossChainTradeData</tt> contains the current trade offer state
-	 * as extracted from the AT's data segment.
-	 * <p>
-	 * Access to a funded wallet is via a Dogecoin BIP32 hierarchical deterministic key,
-	 * passed via <tt>xprv58</tt>.
-	 * <b>This key will be stored in your node's database</b>
-	 * to allow trade-bot to create/fund the necessary P2SH transactions!
-	 * However, due to the nature of BIP32 keys, it is possible to give the trade-bot
-	 * only a subset of wallet access (see BIP32 for more details).
-	 * <p>
-	 * As an example, the xprv58 can be extract from a <i>legacy, password-less</i>
-	 * Electrum wallet by going to the console tab and entering:<br>
-	 * <tt>wallet.keystore.xprv</tt><br>
-	 * which should result in a base58 string starting with either 'xprv' (for Dogecoin main-net)
-	 * or 'tprv' for (Dogecoin test-net).
-	 * <p>
-	 * It is envisaged that the value in <tt>xprv58</tt> will actually come from a Qortal-UI-managed wallet.
-	 * <p>
-	 * If sufficient funds are available, <b>this method will actually fund the P2SH-A</b>
-	 * with the Dogecoin amount expected by 'Bob'.
-	 * <p>
-	 * If the Dogecoin transaction is successfully broadcast to the network then
-	 * we also send a MESSAGE to Bob's trade-bot to let them know.
-	 * <p>
-	 * The trade-bot entry is saved to the repository and the cross-chain trading process commences.
-	 * <p>
-	 * @param repository
-	 * @param crossChainTradeData chosen trade OFFER that Alice wants to match
-	 * @param xprv58 funded wallet xprv in base58
-	 * @return true if P2SH-A funding transaction successfully broadcast to Dogecoin network, false otherwise
-	 * @throws DataException
-	 */
-	public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException {
-		byte[] tradePrivateKey = TradeBot.generateTradePrivateKey();
-		byte[] secretA = TradeBot.generateSecret();
-		byte[] hashOfSecretA = Crypto.hash160(secretA);
-
-		byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey);
-		byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey);
-		String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey);
-
-		byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey);
-		byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey);
-		byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH
-
-		// We need to generate lockTime-A: add tradeTimeout to now
-		long now = NTP.getTime();
-		int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L);
-
-		TradeBotData tradeBotData =  new TradeBotData(tradePrivateKey, DogecoinACCTv1.NAME,
-				State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value,
-				receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount,
-				tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress,
-				secretA, hashOfSecretA,
-				SupportedBlockchain.DOGECOIN.name(),
-				tradeForeignPublicKey, tradeForeignPublicKeyHash,
-				crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash);
-
-		// Attempt to backup the trade bot data
-		// Include tradeBotData as an additional parameter, since it's not in the repository yet
-		TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData));
-
-		// Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount
-		long p2shFee;
-		try {
-			p2shFee = Dogecoin.getInstance().getP2shFee(now);
-		} catch (ForeignBlockchainException e) {
-			LOGGER.debug("Couldn't estimate Dogecoin fees?");
-			return ResponseResult.NETWORK_ISSUE;
-		}
-
-		// Fee for redeem/refund is subtracted from P2SH-A balance.
-		// Do not include fee for funding transaction as this is covered by buildSpend()
-		long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/;
-
-		// P2SH-A to be funded
-		byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA);
-		String p2shAddress = Dogecoin.getInstance().deriveP2shAddress(redeemScriptBytes);
-
-		// Build transaction for funding P2SH-A
-		Transaction p2shFundingTransaction = Dogecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA);
-		if (p2shFundingTransaction == null) {
-			LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?");
-			return ResponseResult.BALANCE_ISSUE;
-		}
-
-		try {
-			Dogecoin.getInstance().broadcastTransaction(p2shFundingTransaction);
-		} catch (ForeignBlockchainException e) {
-			// We couldn't fund P2SH-A at this time
-			LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?");
-			return ResponseResult.NETWORK_ISSUE;
-		}
-
-		// Attempt to send MESSAGE to Bob's Qortal trade address
-		byte[] messageData = DogecoinACCTv1.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA());
-		String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress;
-
-		boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData);
-		if (!isMessageAlreadySent) {
-			PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
-			MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false);
-
-			messageTransaction.computeNonce();
-			messageTransaction.sign(sender);
-
-			// reset repository state to prevent deadlock
-			repository.discardChanges();
-			ValidationResult result = messageTransaction.importAsUnconfirmed();
-
-			if (result != ValidationResult.OK) {
-				LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name()));
-				return ResponseResult.NETWORK_ISSUE;
-			}
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress));
-
-		return ResponseResult.OK;
-	}
-
-	@Override
-	public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException {
-		State tradeBotState = State.valueOf(tradeBotData.getStateValue());
-		if (tradeBotState == null)
-			return true;
-
-		// If the AT doesn't exist then we might as well let the user tidy up
-		if (!repository.getATRepository().exists(tradeBotData.getAtAddress()))
-			return true;
-
-		switch (tradeBotState) {
-			case BOB_WAITING_FOR_AT_CONFIRM:
-			case ALICE_DONE:
-			case BOB_DONE:
-			case ALICE_REFUNDED:
-			case BOB_REFUNDED:
-			case ALICE_REFUNDING_A:
-				return true;
-
-			default:
-				return false;
-		}
-	}
-
-	@Override
-	public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException {
-		State tradeBotState = State.valueOf(tradeBotData.getStateValue());
-		if (tradeBotState == null) {
-			LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress()));
-			return;
-		}
-
-		ATData atData = null;
-		CrossChainTradeData tradeData = null;
-
-		if (tradeBotState.requiresAtData) {
-			// Attempt to fetch AT data
-			atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress());
-			if (atData == null) {
-				LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress()));
-				return;
-			}
-
-			if (tradeBotState.requiresTradeData) {
-				tradeData = DogecoinACCTv1.getInstance().populateTradeData(repository, atData);
-				if (tradeData == null) {
-					LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress()));
-					return;
-				}
-			}
-		}
-
-		switch (tradeBotState) {
-			case BOB_WAITING_FOR_AT_CONFIRM:
-				handleBobWaitingForAtConfirm(repository, tradeBotData);
-				break;
-
-			case BOB_WAITING_FOR_MESSAGE:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case ALICE_WAITING_FOR_AT_LOCK:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case BOB_WAITING_FOR_AT_REDEEM:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case ALICE_DONE:
-			case BOB_DONE:
-				break;
-
-			case ALICE_REFUNDING_A:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case ALICE_REFUNDED:
-			case BOB_REFUNDED:
-				break;
-		}
-	}
-
-	/**
-	 * Trade-bot is waiting for Bob's AT to deploy.
-	 * <p>
-	 * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice.
-	 */
-	private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException {
-		if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) {
-			if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD)
-				return;
-
-			// We've waited ages for AT to be confirmed into a block but something has gone awry.
-			// After this long we assume transaction loss so give up with trade-bot entry too.
-			tradeBotData.setState(State.BOB_REFUNDED.name());
-			tradeBotData.setStateValue(State.BOB_REFUNDED.value);
-			tradeBotData.setTimestamp(NTP.getTime());
-			// We delete trade-bot entry here instead of saving, hence not using updateTradeBotState()
-			repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
-			repository.saveChanges();
-
-			LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress()));
-			TradeBot.notifyStateChange(tradeBotData);
-			return;
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE,
-				() -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress()));
-	}
-
-	/**
-	 * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info.
-	 * <p>
-	 * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund,
-	 * in which case trade-bot is done with this specific trade and finalizes on refunded state.
-	 * <p>
-	 * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot.
-	 * <p>
-	 * Details from Alice are used to derive P2SH-A address and this is checked for funding balance.
-	 * <p>
-	 * Assuming P2SH-A has at least expected Dogecoin balance,
-	 * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details.
-	 * <p>
-	 * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice.
-	 * <p>
-	 * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to
-	 * extract secret-A needed to redeem Alice's P2SH.
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		// If AT has finished then Bob likely cancelled his trade offer
-		if (atData.getIsFinished()) {
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED,
-					() -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress()));
-			return;
-		}
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		String address = tradeBotData.getTradeNativeAddress();
-		List<MessageTransactionData> messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null);
-
-		for (MessageTransactionData messageTransactionData : messageTransactionsData) {
-			if (messageTransactionData.isText())
-				continue;
-
-			// We're expecting: HASH160(secret-A), Alice's Dogecoin pubkeyhash and lockTime-A
-			byte[] messageData = messageTransactionData.getData();
-			DogecoinACCTv1.OfferMessageData offerMessageData = DogecoinACCTv1.extractOfferMessageData(messageData);
-			if (offerMessageData == null)
-				continue;
-
-			byte[] aliceForeignPublicKeyHash = offerMessageData.partnerDogecoinPKH;
-			byte[] hashOfSecretA = offerMessageData.hashOfSecretA;
-			int lockTimeA = (int) offerMessageData.lockTimeA;
-			long messageTimestamp = messageTransactionData.getTimestamp();
-			int refundTimeout = DogecoinACCTv1.calcRefundTimeout(messageTimestamp, lockTimeA);
-
-			// Determine P2SH-A address and confirm funded
-			byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA);
-			String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-			long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-			long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-			final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee;
-
-			BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-			switch (htlcStatusA) {
-				case UNFUNDED:
-				case FUNDING_IN_PROGRESS:
-					// There might be another MESSAGE from someone else with an actually funded P2SH-A...
-					continue;
-
-				case REDEEM_IN_PROGRESS:
-				case REDEEMED:
-					// We've already redeemed this?
-					TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE,
-							() -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA));
-					return;
-
-				case REFUND_IN_PROGRESS:
-				case REFUNDED:
-					// This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A...
-					continue;
-
-				case FUNDED:
-					// Fall-through out of switch...
-					break;
-			}
-
-			// Good to go - send MESSAGE to AT
-
-			String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey());
-
-			// Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume
-			byte[] outgoingMessageData = DogecoinACCTv1.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout);
-			String messageRecipient = tradeBotData.getAtAddress();
-
-			boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData);
-			if (!isMessageAlreadySent) {
-				PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
-				MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false);
-
-				outgoingMessageTransaction.computeNonce();
-				outgoingMessageTransaction.sign(sender);
-
-				// reset repository state to prevent deadlock
-				repository.discardChanges();
-				ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed();
-
-				if (result != ValidationResult.OK) {
-					LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name()));
-					return;
-				}
-			}
-
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM,
-					() -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress));
-
-			return;
-		}
-	}
-
-	/**
-	 * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only.
-	 * <p>
-	 * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow
-	 * this process has taken so long that we've reached P2SH-A's locktime, or that someone else
-	 * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process.
-	 * <p>
-	 * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct.
-	 * <p>
-	 * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice.
-	 * <p>
-	 * In revealing a valid secret-A, Bob can then redeem the DOGE funds from P2SH-A.
-	 * <p>
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData))
-			return;
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-		int lockTimeA = tradeBotData.getLockTimeA();
-
-		// Refund P2SH-A if we've passed lockTime-A
-		if (NTP.getTime() >= lockTimeA * 1000L) {
-			byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret());
-			String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-			long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-			long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-			long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-
-			BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-			switch (htlcStatusA) {
-				case UNFUNDED:
-				case FUNDING_IN_PROGRESS:
-				case FUNDED:
-					break;
-
-				case REDEEM_IN_PROGRESS:
-				case REDEEMED:
-					// Already redeemed?
-					TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-							() -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA));
-					return;
-
-				case REFUND_IN_PROGRESS:
-				case REFUNDED:
-					TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED,
-							() -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA));
-					return;
-
-			}
-
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A,
-					() -> atData.getIsFinished()
-					? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA)
-					: String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA));
-
-			return;
-		}
-
-		// We're waiting for AT to be in TRADE mode
-		if (crossChainTradeData.mode != AcctMode.TRADING)
-			return;
-
-		// AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above
-
-		// Find our MESSAGE to AT from previous state
-		List<MessageTransactionData> messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(),
-				crossChainTradeData.qortalCreatorTradeAddress, null, null, null);
-		if (messageTransactionsData == null || messageTransactionsData.isEmpty()) {
-			LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress));
-			return;
-		}
-
-		long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp();
-		int refundTimeout = DogecoinACCTv1.calcRefundTimeout(recipientMessageTimestamp, lockTimeA);
-
-		// Our calculated refundTimeout should match AT's refundTimeout
-		if (refundTimeout != crossChainTradeData.refundTimeout) {
-			LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout));
-			// We'll eventually refund
-			return;
-		}
-
-		// We're good to redeem AT
-
-		// Send 'redeem' MESSAGE to AT using both secret
-		byte[] secretA = tradeBotData.getSecret();
-		String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH
-		byte[] messageData = DogecoinACCTv1.buildRedeemMessage(secretA, qortalReceivingAddress);
-		String messageRecipient = tradeBotData.getAtAddress();
-
-		boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData);
-		if (!isMessageAlreadySent) {
-			PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
-			MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false);
-
-			messageTransaction.computeNonce();
-			messageTransaction.sign(sender);
-
-			// Reset repository state to prevent deadlock
-			repository.discardChanges();
-			ValidationResult result = messageTransaction.importAsUnconfirmed();
-
-			if (result != ValidationResult.OK) {
-				LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name()));
-				return;
-			}
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-				() -> String.format("Redeeming AT %s. Funds should arrive at %s",
-						tradeBotData.getAtAddress(), qortalReceivingAddress));
-	}
-
-	/**
-	 * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the DOGE funds from P2SH-A.
-	 * <p>
-	 * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case,
-	 * trade-bot is done with this specific trade and finalizes in refunded state.
-	 * <p>
-	 * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the DOGE funds from P2SH-A
-	 * to Bob's 'foreign'/Dogecoin trade legacy-format address, as derived from trade private key.
-	 * <p>
-	 * (This could potentially be 'improved' to send DOGE to any address of Bob's choosing by changing the transaction output).
-	 * <p>
-	 * If trade-bot successfully broadcasts the transaction, then this specific trade is done.
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		// AT should be 'finished' once Alice has redeemed QORT funds
-		if (!atData.getIsFinished())
-			// Not finished yet
-			return;
-
-		// If AT is REFUNDED or CANCELLED then something has gone wrong
-		if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) {
-			// Alice hasn't redeemed the QORT, so there is no point in trying to redeem the DOGE
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED,
-					() -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress()));
-
-			return;
-		}
-
-		byte[] secretA = DogecoinACCTv1.getInstance().findSecretA(repository, crossChainTradeData);
-		if (secretA == null) {
-			LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress()));
-			return;
-		}
-
-		// Use secret-A to redeem P2SH-A
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo();
-		int lockTimeA = crossChainTradeData.lockTimeA;
-		byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA);
-		String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-		// Fee for redeem/refund is subtracted from P2SH-A balance.
-		long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-		long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-		long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-		BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-		switch (htlcStatusA) {
-			case UNFUNDED:
-			case FUNDING_IN_PROGRESS:
-				// P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund
-				return;
-
-			case REDEEM_IN_PROGRESS:
-			case REDEEMED:
-				// Double-check that we have redeemed P2SH-A...
-				break;
-
-			case REFUND_IN_PROGRESS:
-			case REFUNDED:
-				// Wait for AT to auto-refund
-				return;
-
-			case FUNDED: {
-				Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount);
-				ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey());
-				List<TransactionOutput> fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA);
-
-				Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(dogecoin.getNetworkParameters(), redeemAmount, redeemKey,
-						fundingOutputs, redeemScriptA, secretA, receivingAccountInfo);
-
-				dogecoin.broadcastTransaction(p2shRedeemTransaction);
-				break;
-			}
-		}
-
-		String receivingAddress = dogecoin.pkhToAddress(receivingAccountInfo);
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE,
-				() -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress));
-	}
-
-	/**
-	 * Trade-bot is attempting to refund P2SH-A.
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		int lockTimeA = tradeBotData.getLockTimeA();
-
-		// We can't refund P2SH-A until lockTime-A has passed
-		if (NTP.getTime() <= lockTimeA * 1000L)
-			return;
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		// We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113)
-		int medianBlockTime = dogecoin.getMedianBlockTime();
-		if (medianBlockTime <= lockTimeA)
-			return;
-
-		byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret());
-		String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-		// Fee for redeem/refund is subtracted from P2SH-A balance.
-		long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-		long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-		long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-		BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-		switch (htlcStatusA) {
-			case UNFUNDED:
-			case FUNDING_IN_PROGRESS:
-				// Still waiting for P2SH-A to be funded...
-				return;
-
-			case REDEEM_IN_PROGRESS:
-			case REDEEMED:
-				// Too late!
-				TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-						() -> String.format("P2SH-A %s already spent!", p2shAddressA));
-				return;
-
-			case REFUND_IN_PROGRESS:
-			case REFUNDED:
-				break;
-
-			case FUNDED:{
-				Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount);
-				ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey());
-				List<TransactionOutput> fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA);
-
-				// Determine receive address for refund
-				String receiveAddress = dogecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey());
-				Address receiving = Address.fromString(dogecoin.getNetworkParameters(), receiveAddress);
-
-				Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(dogecoin.getNetworkParameters(), refundAmount, refundKey,
-						fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash());
-
-				dogecoin.broadcastTransaction(p2shRefundTransaction);
-				break;
-			}
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED,
-				() -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA));
-	}
-
-	/**
-	 * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else.
-	 * <p>
-	 * Will automatically update trade-bot state to <tt>ALICE_REFUNDING_A</tt> or <tt>ALICE_DONE</tt> as necessary.
-	 * 
-	 * @throws DataException
-	 * @throws ForeignBlockchainException
-	 */
-	private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		// This is OK
-		if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING)
-			return false;
-
-		boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress);
-
-		if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING)
-			if (isAtLockedToUs) {
-				// AT is trading with us - OK
-				return false;
-			} else {
-				TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A,
-						() -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress));
-
-				return true;
-			}
-
-		if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) {
-			// We've redeemed already?
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-					() -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress()));
-		} else {
-			// Any other state is not good, so start defensive refund
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A,
-					() -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress()));
-		}
-
-		return true;
-	}
-
-	private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) {
-		return (lockTimeA - tradeTimeout * 60) * 1000L;
-	}
-
-}

src/main/java/org/qortal/controller/tradebot/DogecoinACCTv2TradeBot.java

@@ -1,884 +0,0 @@
-package org.qortal.controller.tradebot;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.bitcoinj.core.*;
-import org.bitcoinj.script.Script.ScriptType;
-import org.qortal.account.PrivateKeyAccount;
-import org.qortal.account.PublicKeyAccount;
-import org.qortal.api.model.crosschain.TradeBotCreateRequest;
-import org.qortal.asset.Asset;
-import org.qortal.crosschain.*;
-import org.qortal.crypto.Crypto;
-import org.qortal.data.at.ATData;
-import org.qortal.data.crosschain.CrossChainTradeData;
-import org.qortal.data.crosschain.TradeBotData;
-import org.qortal.data.transaction.BaseTransactionData;
-import org.qortal.data.transaction.DeployAtTransactionData;
-import org.qortal.data.transaction.MessageTransactionData;
-import org.qortal.group.Group;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.transaction.DeployAtTransaction;
-import org.qortal.transaction.MessageTransaction;
-import org.qortal.transaction.Transaction.ValidationResult;
-import org.qortal.transform.TransformationException;
-import org.qortal.transform.transaction.DeployAtTransactionTransformer;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static java.util.Arrays.stream;
-import static java.util.stream.Collectors.toMap;
-
-/**
- * Performing cross-chain trading steps on behalf of user.
- * <p>
- * We deal with three different independent state-spaces here:
- * <ul>
- * 	<li>Qortal blockchain</li>
- * 	<li>Foreign blockchain</li>
- * 	<li>Trade-bot entries</li>
- * </ul>
- */
-public class DogecoinACCTv2TradeBot implements AcctTradeBot {
-
-	private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv2TradeBot.class);
-
-	public enum State implements TradeBot.StateNameAndValueSupplier {
-		BOB_WAITING_FOR_AT_CONFIRM(10, false, false),
-		BOB_WAITING_FOR_MESSAGE(15, true, true),
-		BOB_WAITING_FOR_AT_REDEEM(25, true, true),
-		BOB_DONE(30, false, false),
-		BOB_REFUNDED(35, false, false),
-
-		ALICE_WAITING_FOR_AT_LOCK(85, true, true),
-		ALICE_DONE(95, false, false),
-		ALICE_REFUNDING_A(105, true, true),
-		ALICE_REFUNDED(110, false, false);
-
-		private static final Map<Integer, State> map = stream(State.values()).collect(toMap(state -> state.value, state -> state));
-
-		public final int value;
-		public final boolean requiresAtData;
-		public final boolean requiresTradeData;
-
-		State(int value, boolean requiresAtData, boolean requiresTradeData) {
-			this.value = value;
-			this.requiresAtData = requiresAtData;
-			this.requiresTradeData = requiresTradeData;
-		}
-
-		public static State valueOf(int value) {
-			return map.get(value);
-		}
-
-		@Override
-		public String getState() {
-			return this.name();
-		}
-
-		@Override
-		public int getStateValue() {
-			return this.value;
-		}
-	}
-
-	/** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */
-	private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms
-
-	private static DogecoinACCTv2TradeBot instance;
-
-	private final List<String> endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream()
-			.map(State::name)
-			.collect(Collectors.toUnmodifiableList());
-
-	private DogecoinACCTv2TradeBot() {
-	}
-
-	public static synchronized DogecoinACCTv2TradeBot getInstance() {
-		if (instance == null)
-			instance = new DogecoinACCTv2TradeBot();
-
-		return instance;
-	}
-
-	@Override
-	public List<String> getEndStates() {
-		return this.endStates;
-	}
-
-	/**
-	 * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for DOGE.
-	 * <p>
-	 * Generates:
-	 * <ul>
-	 * 	<li>new 'trade' private key</li>
-	 * </ul>
-	 * Derives:
-	 * <ul>
-	 * 	<li>'native' (as in Qortal) public key, public key hash, address (starting with Q)</li>
-	 * 	<li>'foreign' (as in Dogecoin) public key, public key hash</li>
-	 * </ul>
-	 * A Qortal AT is then constructed including the following as constants in the 'data segment':
-	 * <ul>
-	 * 	<li>'native'/Qortal 'trade' address - used as a MESSAGE contact</li>
-	 * 	<li>'foreign'/Dogecoin public key hash - used by Alice's P2SH scripts to allow redeem</li>
-	 * 	<li>QORT amount on offer by Bob</li>
-	 * 	<li>DOGE amount expected in return by Bob (from Alice)</li>
-	 * 	<li>trading timeout, in case things go wrong and everyone needs to refund</li>
-	 * </ul>
-	 * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network.
-	 * <p>
-	 * Trade-bot will wait for Bob's AT to be deployed before taking next step.
-	 * <p>
-	 * @param repository
-	 * @param tradeBotCreateRequest
-	 * @return raw, unsigned DEPLOY_AT transaction
-	 * @throws DataException
-	 */
-	public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException {
-		byte[] tradePrivateKey = TradeBot.generateTradePrivateKey();
-
-		byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey);
-		byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey);
-		String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey);
-
-		byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey);
-		byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey);
-
-		// Convert Dogecoin receiving address into public key hash (we only support P2PKH at this time)
-		Address dogecoinReceivingAddress;
-		try {
-			dogecoinReceivingAddress = Address.fromString(Dogecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress);
-		} catch (AddressFormatException e) {
-			throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress);
-		}
-		if (dogecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH)
-			throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress);
-
-		byte[] dogecoinReceivingAccountInfo = dogecoinReceivingAddress.getHash();
-
-		PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey);
-
-		// Deploy AT
-		long timestamp = NTP.getTime();
-		byte[] reference = creator.getLastReference();
-		long fee = 0L;
-		byte[] signature = null;
-		BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature);
-
-		String name = "QORT/DOGE ACCT";
-		String description = "QORT/DOGE cross-chain trade";
-		String aTType = "ACCT";
-		String tags = "ACCT QORT DOGE";
-		byte[] creationBytes = DogecoinACCTv2.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount,
-				tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout);
-		long amount = tradeBotCreateRequest.fundingQortAmount;
-
-		DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT);
-
-		DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData);
-		fee = deployAtTransaction.calcRecommendedFee();
-		deployAtTransactionData.setFee(fee);
-
-		DeployAtTransaction.ensureATAddress(deployAtTransactionData);
-		String atAddress = deployAtTransactionData.getAtAddress();
-
-		TradeBotData tradeBotData =  new TradeBotData(tradePrivateKey, DogecoinACCTv2.NAME,
-				State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value,
-				creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount,
-				tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress,
-				null, null,
-				SupportedBlockchain.DOGECOIN.name(),
-				tradeForeignPublicKey, tradeForeignPublicKeyHash,
-				tradeBotCreateRequest.foreignAmount, null, null, null, dogecoinReceivingAccountInfo);
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress));
-
-		// Attempt to backup the trade bot data
-		TradeBot.backupTradeBotData(repository, null);
-
-		// Return to user for signing and broadcast as we don't have their Qortal private key
-		try {
-			return DeployAtTransactionTransformer.toBytes(deployAtTransactionData);
-		} catch (TransformationException e) {
-			throw new DataException("Failed to transform DEPLOY_AT transaction?", e);
-		}
-	}
-
-	/**
-	 * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching DOGE to an existing offer.
-	 * <p>
-	 * Requires a chosen trade offer from Bob, passed by <tt>crossChainTradeData</tt>
-	 * and access to a Dogecoin wallet via <tt>xprv58</tt>.
-	 * <p>
-	 * The <tt>crossChainTradeData</tt> contains the current trade offer state
-	 * as extracted from the AT's data segment.
-	 * <p>
-	 * Access to a funded wallet is via a Dogecoin BIP32 hierarchical deterministic key,
-	 * passed via <tt>xprv58</tt>.
-	 * <b>This key will be stored in your node's database</b>
-	 * to allow trade-bot to create/fund the necessary P2SH transactions!
-	 * However, due to the nature of BIP32 keys, it is possible to give the trade-bot
-	 * only a subset of wallet access (see BIP32 for more details).
-	 * <p>
-	 * As an example, the xprv58 can be extract from a <i>legacy, password-less</i>
-	 * Electrum wallet by going to the console tab and entering:<br>
-	 * <tt>wallet.keystore.xprv</tt><br>
-	 * which should result in a base58 string starting with either 'xprv' (for Dogecoin main-net)
-	 * or 'tprv' for (Dogecoin test-net).
-	 * <p>
-	 * It is envisaged that the value in <tt>xprv58</tt> will actually come from a Qortal-UI-managed wallet.
-	 * <p>
-	 * If sufficient funds are available, <b>this method will actually fund the P2SH-A</b>
-	 * with the Dogecoin amount expected by 'Bob'.
-	 * <p>
-	 * If the Dogecoin transaction is successfully broadcast to the network then
-	 * we also send a MESSAGE to Bob's trade-bot to let them know.
-	 * <p>
-	 * The trade-bot entry is saved to the repository and the cross-chain trading process commences.
-	 * <p>
-	 * @param repository
-	 * @param crossChainTradeData chosen trade OFFER that Alice wants to match
-	 * @param xprv58 funded wallet xprv in base58
-	 * @return true if P2SH-A funding transaction successfully broadcast to Dogecoin network, false otherwise
-	 * @throws DataException
-	 */
-	public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException {
-		byte[] tradePrivateKey = TradeBot.generateTradePrivateKey();
-		byte[] secretA = TradeBot.generateSecret();
-		byte[] hashOfSecretA = Crypto.hash160(secretA);
-
-		byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey);
-		byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey);
-		String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey);
-
-		byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey);
-		byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey);
-		byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH
-
-		// We need to generate lockTime-A: add tradeTimeout to now
-		long now = NTP.getTime();
-		int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L);
-
-		TradeBotData tradeBotData =  new TradeBotData(tradePrivateKey, DogecoinACCTv2.NAME,
-				State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value,
-				receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount,
-				tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress,
-				secretA, hashOfSecretA,
-				SupportedBlockchain.DOGECOIN.name(),
-				tradeForeignPublicKey, tradeForeignPublicKeyHash,
-				crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash);
-
-		// Attempt to backup the trade bot data
-		// Include tradeBotData as an additional parameter, since it's not in the repository yet
-		TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData));
-
-		// Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount
-		long p2shFee;
-		try {
-			p2shFee = Dogecoin.getInstance().getP2shFee(now);
-		} catch (ForeignBlockchainException e) {
-			LOGGER.debug("Couldn't estimate Dogecoin fees?");
-			return ResponseResult.NETWORK_ISSUE;
-		}
-
-		// Fee for redeem/refund is subtracted from P2SH-A balance.
-		// Do not include fee for funding transaction as this is covered by buildSpend()
-		long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/;
-
-		// P2SH-A to be funded
-		byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA);
-		String p2shAddress = Dogecoin.getInstance().deriveP2shAddress(redeemScriptBytes);
-
-		// Build transaction for funding P2SH-A
-		Transaction p2shFundingTransaction = Dogecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA);
-		if (p2shFundingTransaction == null) {
-			LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?");
-			return ResponseResult.BALANCE_ISSUE;
-		}
-
-		try {
-			Dogecoin.getInstance().broadcastTransaction(p2shFundingTransaction);
-		} catch (ForeignBlockchainException e) {
-			// We couldn't fund P2SH-A at this time
-			LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?");
-			return ResponseResult.NETWORK_ISSUE;
-		}
-
-		// Attempt to send MESSAGE to Bob's Qortal trade address
-		byte[] messageData = DogecoinACCTv2.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA());
-		String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress;
-
-		boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData);
-		if (!isMessageAlreadySent) {
-			PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
-			MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false);
-
-			messageTransaction.computeNonce();
-			messageTransaction.sign(sender);
-
-			// reset repository state to prevent deadlock
-			repository.discardChanges();
-			ValidationResult result = messageTransaction.importAsUnconfirmed();
-
-			if (result != ValidationResult.OK) {
-				LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name()));
-				return ResponseResult.NETWORK_ISSUE;
-			}
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress));
-
-		return ResponseResult.OK;
-	}
-
-	@Override
-	public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException {
-		State tradeBotState = State.valueOf(tradeBotData.getStateValue());
-		if (tradeBotState == null)
-			return true;
-
-		// If the AT doesn't exist then we might as well let the user tidy up
-		if (!repository.getATRepository().exists(tradeBotData.getAtAddress()))
-			return true;
-
-		switch (tradeBotState) {
-			case BOB_WAITING_FOR_AT_CONFIRM:
-			case ALICE_DONE:
-			case BOB_DONE:
-			case ALICE_REFUNDED:
-			case BOB_REFUNDED:
-				return true;
-
-			default:
-				return false;
-		}
-	}
-
-	@Override
-	public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException {
-		State tradeBotState = State.valueOf(tradeBotData.getStateValue());
-		if (tradeBotState == null) {
-			LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress()));
-			return;
-		}
-
-		ATData atData = null;
-		CrossChainTradeData tradeData = null;
-
-		if (tradeBotState.requiresAtData) {
-			// Attempt to fetch AT data
-			atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress());
-			if (atData == null) {
-				LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress()));
-				return;
-			}
-
-			if (tradeBotState.requiresTradeData) {
-				tradeData = DogecoinACCTv2.getInstance().populateTradeData(repository, atData);
-				if (tradeData == null) {
-					LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress()));
-					return;
-				}
-			}
-		}
-
-		switch (tradeBotState) {
-			case BOB_WAITING_FOR_AT_CONFIRM:
-				handleBobWaitingForAtConfirm(repository, tradeBotData);
-				break;
-
-			case BOB_WAITING_FOR_MESSAGE:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case ALICE_WAITING_FOR_AT_LOCK:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case BOB_WAITING_FOR_AT_REDEEM:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case ALICE_DONE:
-			case BOB_DONE:
-				break;
-
-			case ALICE_REFUNDING_A:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case ALICE_REFUNDED:
-			case BOB_REFUNDED:
-				break;
-		}
-	}
-
-	/**
-	 * Trade-bot is waiting for Bob's AT to deploy.
-	 * <p>
-	 * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice.
-	 */
-	private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException {
-		if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) {
-			if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD)
-				return;
-
-			// We've waited ages for AT to be confirmed into a block but something has gone awry.
-			// After this long we assume transaction loss so give up with trade-bot entry too.
-			tradeBotData.setState(State.BOB_REFUNDED.name());
-			tradeBotData.setStateValue(State.BOB_REFUNDED.value);
-			tradeBotData.setTimestamp(NTP.getTime());
-			// We delete trade-bot entry here instead of saving, hence not using updateTradeBotState()
-			repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
-			repository.saveChanges();
-
-			LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress()));
-			TradeBot.notifyStateChange(tradeBotData);
-			return;
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE,
-				() -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress()));
-	}
-
-	/**
-	 * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info.
-	 * <p>
-	 * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund,
-	 * in which case trade-bot is done with this specific trade and finalizes on refunded state.
-	 * <p>
-	 * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot.
-	 * <p>
-	 * Details from Alice are used to derive P2SH-A address and this is checked for funding balance.
-	 * <p>
-	 * Assuming P2SH-A has at least expected Dogecoin balance,
-	 * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details.
-	 * <p>
-	 * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice.
-	 * <p>
-	 * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to
-	 * extract secret-A needed to redeem Alice's P2SH.
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		// If AT has finished then Bob likely cancelled his trade offer
-		if (atData.getIsFinished()) {
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED,
-					() -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress()));
-			return;
-		}
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		String address = tradeBotData.getTradeNativeAddress();
-		List<MessageTransactionData> messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null);
-
-		for (MessageTransactionData messageTransactionData : messageTransactionsData) {
-			if (messageTransactionData.isText())
-				continue;
-
-			// We're expecting: HASH160(secret-A), Alice's Dogecoin pubkeyhash and lockTime-A
-			byte[] messageData = messageTransactionData.getData();
-			DogecoinACCTv2.OfferMessageData offerMessageData = DogecoinACCTv2.extractOfferMessageData(messageData);
-			if (offerMessageData == null)
-				continue;
-
-			byte[] aliceForeignPublicKeyHash = offerMessageData.partnerDogecoinPKH;
-			byte[] hashOfSecretA = offerMessageData.hashOfSecretA;
-			int lockTimeA = (int) offerMessageData.lockTimeA;
-			long messageTimestamp = messageTransactionData.getTimestamp();
-			int refundTimeout = DogecoinACCTv2.calcRefundTimeout(messageTimestamp, lockTimeA);
-
-			// Determine P2SH-A address and confirm funded
-			byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA);
-			String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-			long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-			long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-			final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee;
-
-			BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-			switch (htlcStatusA) {
-				case UNFUNDED:
-				case FUNDING_IN_PROGRESS:
-					// There might be another MESSAGE from someone else with an actually funded P2SH-A...
-					continue;
-
-				case REDEEM_IN_PROGRESS:
-				case REDEEMED:
-					// We've already redeemed this?
-					TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE,
-							() -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA));
-					return;
-
-				case REFUND_IN_PROGRESS:
-				case REFUNDED:
-					// This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A...
-					continue;
-
-				case FUNDED:
-					// Fall-through out of switch...
-					break;
-			}
-
-			// Good to go - send MESSAGE to AT
-
-			String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey());
-
-			// Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume
-			byte[] outgoingMessageData = DogecoinACCTv2.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout);
-			String messageRecipient = tradeBotData.getAtAddress();
-
-			boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData);
-			if (!isMessageAlreadySent) {
-				PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
-				MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false);
-
-				outgoingMessageTransaction.computeNonce();
-				outgoingMessageTransaction.sign(sender);
-
-				// reset repository state to prevent deadlock
-				repository.discardChanges();
-				ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed();
-
-				if (result != ValidationResult.OK) {
-					LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name()));
-					return;
-				}
-			}
-
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM,
-					() -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress));
-
-			return;
-		}
-	}
-
-	/**
-	 * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only.
-	 * <p>
-	 * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow
-	 * this process has taken so long that we've reached P2SH-A's locktime, or that someone else
-	 * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process.
-	 * <p>
-	 * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct.
-	 * <p>
-	 * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice.
-	 * <p>
-	 * In revealing a valid secret-A, Bob can then redeem the DOGE funds from P2SH-A.
-	 * <p>
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData))
-			return;
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-		int lockTimeA = tradeBotData.getLockTimeA();
-
-		// Refund P2SH-A if we've passed lockTime-A
-		if (NTP.getTime() >= lockTimeA * 1000L) {
-			byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret());
-			String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-			long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-			long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-			long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-
-			BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-			switch (htlcStatusA) {
-				case UNFUNDED:
-				case FUNDING_IN_PROGRESS:
-				case FUNDED:
-					break;
-
-				case REDEEM_IN_PROGRESS:
-				case REDEEMED:
-					// Already redeemed?
-					TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-							() -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA));
-					return;
-
-				case REFUND_IN_PROGRESS:
-				case REFUNDED:
-					TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED,
-							() -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA));
-					return;
-
-			}
-
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A,
-					() -> atData.getIsFinished()
-					? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA)
-					: String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA));
-
-			return;
-		}
-
-		// We're waiting for AT to be in TRADE mode
-		if (crossChainTradeData.mode != AcctMode.TRADING)
-			return;
-
-		// AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above
-
-		// Find our MESSAGE to AT from previous state
-		List<MessageTransactionData> messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(),
-				crossChainTradeData.qortalCreatorTradeAddress, null, null, null);
-		if (messageTransactionsData == null || messageTransactionsData.isEmpty()) {
-			LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress));
-			return;
-		}
-
-		long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp();
-		int refundTimeout = DogecoinACCTv2.calcRefundTimeout(recipientMessageTimestamp, lockTimeA);
-
-		// Our calculated refundTimeout should match AT's refundTimeout
-		if (refundTimeout != crossChainTradeData.refundTimeout) {
-			LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout));
-			// We'll eventually refund
-			return;
-		}
-
-		// We're good to redeem AT
-
-		// Send 'redeem' MESSAGE to AT using both secret
-		byte[] secretA = tradeBotData.getSecret();
-		String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH
-		byte[] messageData = DogecoinACCTv2.buildRedeemMessage(secretA, qortalReceivingAddress);
-		String messageRecipient = tradeBotData.getAtAddress();
-
-		boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData);
-		if (!isMessageAlreadySent) {
-			PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
-			MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false);
-
-			messageTransaction.computeNonce();
-			messageTransaction.sign(sender);
-
-			// Reset repository state to prevent deadlock
-			repository.discardChanges();
-			ValidationResult result = messageTransaction.importAsUnconfirmed();
-
-			if (result != ValidationResult.OK) {
-				LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name()));
-				return;
-			}
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-				() -> String.format("Redeeming AT %s. Funds should arrive at %s",
-						tradeBotData.getAtAddress(), qortalReceivingAddress));
-	}
-
-	/**
-	 * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the DOGE funds from P2SH-A.
-	 * <p>
-	 * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case,
-	 * trade-bot is done with this specific trade and finalizes in refunded state.
-	 * <p>
-	 * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the DOGE funds from P2SH-A
-	 * to Bob's 'foreign'/Dogecoin trade legacy-format address, as derived from trade private key.
-	 * <p>
-	 * (This could potentially be 'improved' to send DOGE to any address of Bob's choosing by changing the transaction output).
-	 * <p>
-	 * If trade-bot successfully broadcasts the transaction, then this specific trade is done.
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		// AT should be 'finished' once Alice has redeemed QORT funds
-		if (!atData.getIsFinished())
-			// Not finished yet
-			return;
-
-		// If AT is REFUNDED or CANCELLED then something has gone wrong
-		if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) {
-			// Alice hasn't redeemed the QORT, so there is no point in trying to redeem the DOGE
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED,
-					() -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress()));
-
-			return;
-		}
-
-		byte[] secretA = DogecoinACCTv2.getInstance().findSecretA(repository, crossChainTradeData);
-		if (secretA == null) {
-			LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress()));
-			return;
-		}
-
-		// Use secret-A to redeem P2SH-A
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo();
-		int lockTimeA = crossChainTradeData.lockTimeA;
-		byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA);
-		String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-		// Fee for redeem/refund is subtracted from P2SH-A balance.
-		long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-		long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-		long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-		BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-		switch (htlcStatusA) {
-			case UNFUNDED:
-			case FUNDING_IN_PROGRESS:
-				// P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund
-				return;
-
-			case REDEEM_IN_PROGRESS:
-			case REDEEMED:
-				// Double-check that we have redeemed P2SH-A...
-				break;
-
-			case REFUND_IN_PROGRESS:
-			case REFUNDED:
-				// Wait for AT to auto-refund
-				return;
-
-			case FUNDED: {
-				Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount);
-				ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey());
-				List<TransactionOutput> fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA);
-
-				Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(dogecoin.getNetworkParameters(), redeemAmount, redeemKey,
-						fundingOutputs, redeemScriptA, secretA, receivingAccountInfo);
-
-				dogecoin.broadcastTransaction(p2shRedeemTransaction);
-				break;
-			}
-		}
-
-		String receivingAddress = dogecoin.pkhToAddress(receivingAccountInfo);
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE,
-				() -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress));
-	}
-
-	/**
-	 * Trade-bot is attempting to refund P2SH-A.
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		int lockTimeA = tradeBotData.getLockTimeA();
-
-		// We can't refund P2SH-A until lockTime-A has passed
-		if (NTP.getTime() <= lockTimeA * 1000L)
-			return;
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		// We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113)
-		int medianBlockTime = dogecoin.getMedianBlockTime();
-		if (medianBlockTime <= lockTimeA)
-			return;
-
-		byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret());
-		String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-		// Fee for redeem/refund is subtracted from P2SH-A balance.
-		long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-		long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-		long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-		BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-		switch (htlcStatusA) {
-			case UNFUNDED:
-			case FUNDING_IN_PROGRESS:
-				// Still waiting for P2SH-A to be funded...
-				return;
-
-			case REDEEM_IN_PROGRESS:
-			case REDEEMED:
-				// Too late!
-				TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-						() -> String.format("P2SH-A %s already spent!", p2shAddressA));
-				return;
-
-			case REFUND_IN_PROGRESS:
-			case REFUNDED:
-				break;
-
-			case FUNDED:{
-				Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount);
-				ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey());
-				List<TransactionOutput> fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA);
-
-				// Determine receive address for refund
-				String receiveAddress = dogecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey());
-				Address receiving = Address.fromString(dogecoin.getNetworkParameters(), receiveAddress);
-
-				Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(dogecoin.getNetworkParameters(), refundAmount, refundKey,
-						fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash());
-
-				dogecoin.broadcastTransaction(p2shRefundTransaction);
-				break;
-			}
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED,
-				() -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA));
-	}
-
-	/**
-	 * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else.
-	 * <p>
-	 * Will automatically update trade-bot state to <tt>ALICE_REFUNDING_A</tt> or <tt>ALICE_DONE</tt> as necessary.
-	 * 
-	 * @throws DataException
-	 * @throws ForeignBlockchainException
-	 */
-	private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		// This is OK
-		if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING)
-			return false;
-
-		boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress);
-
-		if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING)
-			if (isAtLockedToUs) {
-				// AT is trading with us - OK
-				return false;
-			} else {
-				TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A,
-						() -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress));
-
-				return true;
-			}
-
-		if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) {
-			// We've redeemed already?
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-					() -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress()));
-		} else {
-			// Any other state is not good, so start defensive refund
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A,
-					() -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress()));
-		}
-
-		return true;
-	}
-
-	private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) {
-		return (lockTimeA - tradeTimeout * 60) * 1000L;
-	}
-
-}

src/main/java/org/qortal/controller/tradebot/DogecoinACCTv3TradeBot.java

@@ -1,885 +0,0 @@
-package org.qortal.controller.tradebot;
-
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.bitcoinj.core.*;
-import org.bitcoinj.script.Script.ScriptType;
-import org.qortal.account.PrivateKeyAccount;
-import org.qortal.account.PublicKeyAccount;
-import org.qortal.api.model.crosschain.TradeBotCreateRequest;
-import org.qortal.asset.Asset;
-import org.qortal.crosschain.*;
-import org.qortal.crypto.Crypto;
-import org.qortal.data.at.ATData;
-import org.qortal.data.crosschain.CrossChainTradeData;
-import org.qortal.data.crosschain.TradeBotData;
-import org.qortal.data.transaction.BaseTransactionData;
-import org.qortal.data.transaction.DeployAtTransactionData;
-import org.qortal.data.transaction.MessageTransactionData;
-import org.qortal.group.Group;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.transaction.DeployAtTransaction;
-import org.qortal.transaction.MessageTransaction;
-import org.qortal.transaction.Transaction.ValidationResult;
-import org.qortal.transform.TransformationException;
-import org.qortal.transform.transaction.DeployAtTransactionTransformer;
-import org.qortal.utils.Base58;
-import org.qortal.utils.NTP;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import static java.util.Arrays.stream;
-import static java.util.stream.Collectors.toMap;
-
-/**
- * Performing cross-chain trading steps on behalf of user.
- * <p>
- * We deal with three different independent state-spaces here:
- * <ul>
- * 	<li>Qortal blockchain</li>
- * 	<li>Foreign blockchain</li>
- * 	<li>Trade-bot entries</li>
- * </ul>
- */
-public class DogecoinACCTv3TradeBot implements AcctTradeBot {
-
-	private static final Logger LOGGER = LogManager.getLogger(DogecoinACCTv3TradeBot.class);
-
-	public enum State implements TradeBot.StateNameAndValueSupplier {
-		BOB_WAITING_FOR_AT_CONFIRM(10, false, false),
-		BOB_WAITING_FOR_MESSAGE(15, true, true),
-		BOB_WAITING_FOR_AT_REDEEM(25, true, true),
-		BOB_DONE(30, false, false),
-		BOB_REFUNDED(35, false, false),
-
-		ALICE_WAITING_FOR_AT_LOCK(85, true, true),
-		ALICE_DONE(95, false, false),
-		ALICE_REFUNDING_A(105, true, true),
-		ALICE_REFUNDED(110, false, false);
-
-		private static final Map<Integer, State> map = stream(State.values()).collect(toMap(state -> state.value, state -> state));
-
-		public final int value;
-		public final boolean requiresAtData;
-		public final boolean requiresTradeData;
-
-		State(int value, boolean requiresAtData, boolean requiresTradeData) {
-			this.value = value;
-			this.requiresAtData = requiresAtData;
-			this.requiresTradeData = requiresTradeData;
-		}
-
-		public static State valueOf(int value) {
-			return map.get(value);
-		}
-
-		@Override
-		public String getState() {
-			return this.name();
-		}
-
-		@Override
-		public int getStateValue() {
-			return this.value;
-		}
-	}
-
-	/** Maximum time Bob waits for his AT creation transaction to be confirmed into a block. (milliseconds) */
-	private static final long MAX_AT_CONFIRMATION_PERIOD = 24 * 60 * 60 * 1000L; // ms
-
-	private static DogecoinACCTv3TradeBot instance;
-
-	private final List<String> endStates = Arrays.asList(State.BOB_DONE, State.BOB_REFUNDED, State.ALICE_DONE, State.ALICE_REFUNDING_A, State.ALICE_REFUNDED).stream()
-			.map(State::name)
-			.collect(Collectors.toUnmodifiableList());
-
-	private DogecoinACCTv3TradeBot() {
-	}
-
-	public static synchronized DogecoinACCTv3TradeBot getInstance() {
-		if (instance == null)
-			instance = new DogecoinACCTv3TradeBot();
-
-		return instance;
-	}
-
-	@Override
-	public List<String> getEndStates() {
-		return this.endStates;
-	}
-
-	/**
-	 * Creates a new trade-bot entry from the "Bob" viewpoint, i.e. OFFERing QORT in exchange for DOGE.
-	 * <p>
-	 * Generates:
-	 * <ul>
-	 * 	<li>new 'trade' private key</li>
-	 * </ul>
-	 * Derives:
-	 * <ul>
-	 * 	<li>'native' (as in Qortal) public key, public key hash, address (starting with Q)</li>
-	 * 	<li>'foreign' (as in Dogecoin) public key, public key hash</li>
-	 * </ul>
-	 * A Qortal AT is then constructed including the following as constants in the 'data segment':
-	 * <ul>
-	 * 	<li>'native'/Qortal 'trade' address - used as a MESSAGE contact</li>
-	 * 	<li>'foreign'/Dogecoin public key hash - used by Alice's P2SH scripts to allow redeem</li>
-	 * 	<li>QORT amount on offer by Bob</li>
-	 * 	<li>DOGE amount expected in return by Bob (from Alice)</li>
-	 * 	<li>trading timeout, in case things go wrong and everyone needs to refund</li>
-	 * </ul>
-	 * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network.
-	 * <p>
-	 * Trade-bot will wait for Bob's AT to be deployed before taking next step.
-	 * <p>
-	 * @param repository
-	 * @param tradeBotCreateRequest
-	 * @return raw, unsigned DEPLOY_AT transaction
-	 * @throws DataException
-	 */
-	public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException {
-		byte[] tradePrivateKey = TradeBot.generateTradePrivateKey();
-
-		byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey);
-		byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey);
-		String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey);
-
-		byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey);
-		byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey);
-
-		// Convert Dogecoin receiving address into public key hash (we only support P2PKH at this time)
-		Address dogecoinReceivingAddress;
-		try {
-			dogecoinReceivingAddress = Address.fromString(Dogecoin.getInstance().getNetworkParameters(), tradeBotCreateRequest.receivingAddress);
-		} catch (AddressFormatException e) {
-			throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress);
-		}
-		if (dogecoinReceivingAddress.getOutputScriptType() != ScriptType.P2PKH)
-			throw new DataException("Unsupported Dogecoin receiving address: " + tradeBotCreateRequest.receivingAddress);
-
-		byte[] dogecoinReceivingAccountInfo = dogecoinReceivingAddress.getHash();
-
-		PublicKeyAccount creator = new PublicKeyAccount(repository, tradeBotCreateRequest.creatorPublicKey);
-
-		// Deploy AT
-		long timestamp = NTP.getTime();
-		byte[] reference = creator.getLastReference();
-		long fee = 0L;
-		byte[] signature = null;
-		BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, Group.NO_GROUP, reference, creator.getPublicKey(), fee, signature);
-
-		String name = "QORT/DOGE ACCT";
-		String description = "QORT/DOGE cross-chain trade";
-		String aTType = "ACCT";
-		String tags = "ACCT QORT DOGE";
-		byte[] creationBytes = DogecoinACCTv3.buildQortalAT(tradeNativeAddress, tradeForeignPublicKeyHash, tradeBotCreateRequest.qortAmount,
-				tradeBotCreateRequest.foreignAmount, tradeBotCreateRequest.tradeTimeout);
-		long amount = tradeBotCreateRequest.fundingQortAmount;
-
-		DeployAtTransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, aTType, tags, creationBytes, amount, Asset.QORT);
-
-		DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData);
-		fee = deployAtTransaction.calcRecommendedFee();
-		deployAtTransactionData.setFee(fee);
-
-		DeployAtTransaction.ensureATAddress(deployAtTransactionData);
-		String atAddress = deployAtTransactionData.getAtAddress();
-
-		TradeBotData tradeBotData =  new TradeBotData(tradePrivateKey, DogecoinACCTv3.NAME,
-				State.BOB_WAITING_FOR_AT_CONFIRM.name(), State.BOB_WAITING_FOR_AT_CONFIRM.value,
-				creator.getAddress(), atAddress, timestamp, tradeBotCreateRequest.qortAmount,
-				tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress,
-				null, null,
-				SupportedBlockchain.DOGECOIN.name(),
-				tradeForeignPublicKey, tradeForeignPublicKeyHash,
-				tradeBotCreateRequest.foreignAmount, null, null, null, dogecoinReceivingAccountInfo);
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress));
-
-		// Attempt to backup the trade bot data
-		TradeBot.backupTradeBotData(repository, null);
-
-		// Return to user for signing and broadcast as we don't have their Qortal private key
-		try {
-			return DeployAtTransactionTransformer.toBytes(deployAtTransactionData);
-		} catch (TransformationException e) {
-			throw new DataException("Failed to transform DEPLOY_AT transaction?", e);
-		}
-	}
-
-	/**
-	 * Creates a trade-bot entry from the 'Alice' viewpoint, i.e. matching DOGE to an existing offer.
-	 * <p>
-	 * Requires a chosen trade offer from Bob, passed by <tt>crossChainTradeData</tt>
-	 * and access to a Dogecoin wallet via <tt>xprv58</tt>.
-	 * <p>
-	 * The <tt>crossChainTradeData</tt> contains the current trade offer state
-	 * as extracted from the AT's data segment.
-	 * <p>
-	 * Access to a funded wallet is via a Dogecoin BIP32 hierarchical deterministic key,
-	 * passed via <tt>xprv58</tt>.
-	 * <b>This key will be stored in your node's database</b>
-	 * to allow trade-bot to create/fund the necessary P2SH transactions!
-	 * However, due to the nature of BIP32 keys, it is possible to give the trade-bot
-	 * only a subset of wallet access (see BIP32 for more details).
-	 * <p>
-	 * As an example, the xprv58 can be extract from a <i>legacy, password-less</i>
-	 * Electrum wallet by going to the console tab and entering:<br>
-	 * <tt>wallet.keystore.xprv</tt><br>
-	 * which should result in a base58 string starting with either 'xprv' (for Dogecoin main-net)
-	 * or 'tprv' for (Dogecoin test-net).
-	 * <p>
-	 * It is envisaged that the value in <tt>xprv58</tt> will actually come from a Qortal-UI-managed wallet.
-	 * <p>
-	 * If sufficient funds are available, <b>this method will actually fund the P2SH-A</b>
-	 * with the Dogecoin amount expected by 'Bob'.
-	 * <p>
-	 * If the Dogecoin transaction is successfully broadcast to the network then
-	 * we also send a MESSAGE to Bob's trade-bot to let them know.
-	 * <p>
-	 * The trade-bot entry is saved to the repository and the cross-chain trading process commences.
-	 * <p>
-	 * @param repository
-	 * @param crossChainTradeData chosen trade OFFER that Alice wants to match
-	 * @param xprv58 funded wallet xprv in base58
-	 * @return true if P2SH-A funding transaction successfully broadcast to Dogecoin network, false otherwise
-	 * @throws DataException
-	 */
-	public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct, CrossChainTradeData crossChainTradeData, String xprv58, String receivingAddress) throws DataException {
-		byte[] tradePrivateKey = TradeBot.generateTradePrivateKey();
-		byte[] secretA = TradeBot.generateSecret();
-		byte[] hashOfSecretA = Crypto.hash160(secretA);
-
-		byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey);
-		byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey);
-		String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey);
-
-		byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey);
-		byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey);
-		byte[] receivingPublicKeyHash = Base58.decode(receivingAddress); // Actually the whole address, not just PKH
-
-		// We need to generate lockTime-A: add tradeTimeout to now
-		long now = NTP.getTime();
-		int lockTimeA = crossChainTradeData.tradeTimeout * 60 + (int) (now / 1000L);
-
-		TradeBotData tradeBotData =  new TradeBotData(tradePrivateKey, DogecoinACCTv3.NAME,
-				State.ALICE_WAITING_FOR_AT_LOCK.name(), State.ALICE_WAITING_FOR_AT_LOCK.value,
-				receivingAddress, crossChainTradeData.qortalAtAddress, now, crossChainTradeData.qortAmount,
-				tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress,
-				secretA, hashOfSecretA,
-				SupportedBlockchain.DOGECOIN.name(),
-				tradeForeignPublicKey, tradeForeignPublicKeyHash,
-				crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash);
-
-		// Attempt to backup the trade bot data
-		// Include tradeBotData as an additional parameter, since it's not in the repository yet
-		TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData));
-
-		// Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount
-		long p2shFee;
-		try {
-			p2shFee = Dogecoin.getInstance().getP2shFee(now);
-		} catch (ForeignBlockchainException e) {
-			LOGGER.debug("Couldn't estimate Dogecoin fees?");
-			return ResponseResult.NETWORK_ISSUE;
-		}
-
-		// Fee for redeem/refund is subtracted from P2SH-A balance.
-		// Do not include fee for funding transaction as this is covered by buildSpend()
-		long amountA = crossChainTradeData.expectedForeignAmount + p2shFee /*redeeming/refunding P2SH-A*/;
-
-		// P2SH-A to be funded
-		byte[] redeemScriptBytes = BitcoinyHTLC.buildScript(tradeForeignPublicKeyHash, lockTimeA, crossChainTradeData.creatorForeignPKH, hashOfSecretA);
-		String p2shAddress = Dogecoin.getInstance().deriveP2shAddress(redeemScriptBytes);
-
-		// Build transaction for funding P2SH-A
-		Transaction p2shFundingTransaction = Dogecoin.getInstance().buildSpend(tradeBotData.getForeignKey(), p2shAddress, amountA);
-		if (p2shFundingTransaction == null) {
-			LOGGER.debug("Unable to build P2SH-A funding transaction - lack of funds?");
-			return ResponseResult.BALANCE_ISSUE;
-		}
-
-		try {
-			Dogecoin.getInstance().broadcastTransaction(p2shFundingTransaction);
-		} catch (ForeignBlockchainException e) {
-			// We couldn't fund P2SH-A at this time
-			LOGGER.debug("Couldn't broadcast P2SH-A funding transaction?");
-			return ResponseResult.NETWORK_ISSUE;
-		}
-
-		// Attempt to send MESSAGE to Bob's Qortal trade address
-		byte[] messageData = DogecoinACCTv3.buildOfferMessage(tradeBotData.getTradeForeignPublicKeyHash(), tradeBotData.getHashOfSecret(), tradeBotData.getLockTimeA());
-		String messageRecipient = crossChainTradeData.qortalCreatorTradeAddress;
-
-		boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData);
-		if (!isMessageAlreadySent) {
-			PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
-			MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false);
-
-			messageTransaction.computeNonce();
-			messageTransaction.sign(sender);
-
-			// reset repository state to prevent deadlock
-			repository.discardChanges();
-			ValidationResult result = messageTransaction.importAsUnconfirmed();
-
-			if (result != ValidationResult.OK) {
-				LOGGER.warn(() -> String.format("Unable to send MESSAGE to Bob's trade-bot %s: %s", messageRecipient, result.name()));
-				return ResponseResult.NETWORK_ISSUE;
-			}
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Funding P2SH-A %s. Messaged Bob. Waiting for AT-lock", p2shAddress));
-
-		return ResponseResult.OK;
-	}
-
-	@Override
-	public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException {
-		State tradeBotState = State.valueOf(tradeBotData.getStateValue());
-		if (tradeBotState == null)
-			return true;
-
-		// If the AT doesn't exist then we might as well let the user tidy up
-		if (!repository.getATRepository().exists(tradeBotData.getAtAddress()))
-			return true;
-
-		switch (tradeBotState) {
-			case BOB_WAITING_FOR_AT_CONFIRM:
-			case ALICE_DONE:
-			case BOB_DONE:
-			case ALICE_REFUNDED:
-			case BOB_REFUNDED:
-			case ALICE_REFUNDING_A:
-				return true;
-
-			default:
-				return false;
-		}
-	}
-
-	@Override
-	public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException {
-		State tradeBotState = State.valueOf(tradeBotData.getStateValue());
-		if (tradeBotState == null) {
-			LOGGER.info(() -> String.format("Trade-bot entry for AT %s has invalid state?", tradeBotData.getAtAddress()));
-			return;
-		}
-
-		ATData atData = null;
-		CrossChainTradeData tradeData = null;
-
-		if (tradeBotState.requiresAtData) {
-			// Attempt to fetch AT data
-			atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress());
-			if (atData == null) {
-				LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress()));
-				return;
-			}
-
-			if (tradeBotState.requiresTradeData) {
-				tradeData = DogecoinACCTv3.getInstance().populateTradeData(repository, atData);
-				if (tradeData == null) {
-					LOGGER.warn(() -> String.format("Unable to fetch ACCT trade data for AT %s from repository", tradeBotData.getAtAddress()));
-					return;
-				}
-			}
-		}
-
-		switch (tradeBotState) {
-			case BOB_WAITING_FOR_AT_CONFIRM:
-				handleBobWaitingForAtConfirm(repository, tradeBotData);
-				break;
-
-			case BOB_WAITING_FOR_MESSAGE:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleBobWaitingForMessage(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case ALICE_WAITING_FOR_AT_LOCK:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleAliceWaitingForAtLock(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case BOB_WAITING_FOR_AT_REDEEM:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleBobWaitingForAtRedeem(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case ALICE_DONE:
-			case BOB_DONE:
-				break;
-
-			case ALICE_REFUNDING_A:
-				TradeBot.getInstance().updatePresence(repository, tradeBotData, tradeData);
-				handleAliceRefundingP2shA(repository, tradeBotData, atData, tradeData);
-				break;
-
-			case ALICE_REFUNDED:
-			case BOB_REFUNDED:
-				break;
-		}
-	}
-
-	/**
-	 * Trade-bot is waiting for Bob's AT to deploy.
-	 * <p>
-	 * If AT is deployed, then trade-bot's next step is to wait for MESSAGE from Alice.
-	 */
-	private void handleBobWaitingForAtConfirm(Repository repository, TradeBotData tradeBotData) throws DataException {
-		if (!repository.getATRepository().exists(tradeBotData.getAtAddress())) {
-			if (NTP.getTime() - tradeBotData.getTimestamp() <= MAX_AT_CONFIRMATION_PERIOD)
-				return;
-
-			// We've waited ages for AT to be confirmed into a block but something has gone awry.
-			// After this long we assume transaction loss so give up with trade-bot entry too.
-			tradeBotData.setState(State.BOB_REFUNDED.name());
-			tradeBotData.setStateValue(State.BOB_REFUNDED.value);
-			tradeBotData.setTimestamp(NTP.getTime());
-			// We delete trade-bot entry here instead of saving, hence not using updateTradeBotState()
-			repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
-			repository.saveChanges();
-
-			LOGGER.info(() -> String.format("AT %s never confirmed. Giving up on trade", tradeBotData.getAtAddress()));
-			TradeBot.notifyStateChange(tradeBotData);
-			return;
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_MESSAGE,
-				() -> String.format("AT %s confirmed ready. Waiting for trade message", tradeBotData.getAtAddress()));
-	}
-
-	/**
-	 * Trade-bot is waiting for MESSAGE from Alice's trade-bot, containing Alice's trade info.
-	 * <p>
-	 * It's possible Bob has cancelling his trade offer, receiving an automatic QORT refund,
-	 * in which case trade-bot is done with this specific trade and finalizes on refunded state.
-	 * <p>
-	 * Assuming trade is still on offer, trade-bot checks the contents of MESSAGE from Alice's trade-bot.
-	 * <p>
-	 * Details from Alice are used to derive P2SH-A address and this is checked for funding balance.
-	 * <p>
-	 * Assuming P2SH-A has at least expected Dogecoin balance,
-	 * Bob's trade-bot constructs a zero-fee, PoW MESSAGE to send to Bob's AT with more trade details.
-	 * <p>
-	 * On processing this MESSAGE, Bob's AT should switch into 'TRADE' mode and only trade with Alice.
-	 * <p>
-	 * Trade-bot's next step is to wait for Alice to redeem the AT, which will allow Bob to
-	 * extract secret-A needed to redeem Alice's P2SH.
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleBobWaitingForMessage(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		// If AT has finished then Bob likely cancelled his trade offer
-		if (atData.getIsFinished()) {
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED,
-					() -> String.format("AT %s cancelled - trading aborted", tradeBotData.getAtAddress()));
-			return;
-		}
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		String address = tradeBotData.getTradeNativeAddress();
-		List<MessageTransactionData> messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(null, address, null, null, null);
-
-		for (MessageTransactionData messageTransactionData : messageTransactionsData) {
-			if (messageTransactionData.isText())
-				continue;
-
-			// We're expecting: HASH160(secret-A), Alice's Dogecoin pubkeyhash and lockTime-A
-			byte[] messageData = messageTransactionData.getData();
-			DogecoinACCTv3.OfferMessageData offerMessageData = DogecoinACCTv3.extractOfferMessageData(messageData);
-			if (offerMessageData == null)
-				continue;
-
-			byte[] aliceForeignPublicKeyHash = offerMessageData.partnerDogecoinPKH;
-			byte[] hashOfSecretA = offerMessageData.hashOfSecretA;
-			int lockTimeA = (int) offerMessageData.lockTimeA;
-			long messageTimestamp = messageTransactionData.getTimestamp();
-			int refundTimeout = DogecoinACCTv3.calcRefundTimeout(messageTimestamp, lockTimeA);
-
-			// Determine P2SH-A address and confirm funded
-			byte[] redeemScriptA = BitcoinyHTLC.buildScript(aliceForeignPublicKeyHash, lockTimeA, tradeBotData.getTradeForeignPublicKeyHash(), hashOfSecretA);
-			String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-			long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-			long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-			final long minimumAmountA = tradeBotData.getForeignAmount() + p2shFee;
-
-			BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-			switch (htlcStatusA) {
-				case UNFUNDED:
-				case FUNDING_IN_PROGRESS:
-					// There might be another MESSAGE from someone else with an actually funded P2SH-A...
-					continue;
-
-				case REDEEM_IN_PROGRESS:
-				case REDEEMED:
-					// We've already redeemed this?
-					TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE,
-							() -> String.format("P2SH-A %s already spent? Assuming trade complete", p2shAddressA));
-					return;
-
-				case REFUND_IN_PROGRESS:
-				case REFUNDED:
-					// This P2SH-A is burnt, but there might be another MESSAGE from someone else with an actually funded P2SH-A...
-					continue;
-
-				case FUNDED:
-					// Fall-through out of switch...
-					break;
-			}
-
-			// Good to go - send MESSAGE to AT
-
-			String aliceNativeAddress = Crypto.toAddress(messageTransactionData.getCreatorPublicKey());
-
-			// Build outgoing message, padding each part to 32 bytes to make it easier for AT to consume
-			byte[] outgoingMessageData = DogecoinACCTv3.buildTradeMessage(aliceNativeAddress, aliceForeignPublicKeyHash, hashOfSecretA, lockTimeA, refundTimeout);
-			String messageRecipient = tradeBotData.getAtAddress();
-
-			boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, outgoingMessageData);
-			if (!isMessageAlreadySent) {
-				PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
-				MessageTransaction outgoingMessageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, outgoingMessageData, false, false);
-
-				outgoingMessageTransaction.computeNonce();
-				outgoingMessageTransaction.sign(sender);
-
-				// reset repository state to prevent deadlock
-				repository.discardChanges();
-				ValidationResult result = outgoingMessageTransaction.importAsUnconfirmed();
-
-				if (result != ValidationResult.OK) {
-					LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name()));
-					return;
-				}
-			}
-
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_WAITING_FOR_AT_REDEEM,
-					() -> String.format("Locked AT %s to %s. Waiting for AT redeem", tradeBotData.getAtAddress(), aliceNativeAddress));
-
-			return;
-		}
-	}
-
-	/**
-	 * Trade-bot is waiting for Bob's AT to switch to TRADE mode and lock trade to Alice only.
-	 * <p>
-	 * It's possible that Bob has cancelled his trade offer in the mean time, or that somehow
-	 * this process has taken so long that we've reached P2SH-A's locktime, or that someone else
-	 * has managed to trade with Bob. In any of these cases, trade-bot switches to begin the refunding process.
-	 * <p>
-	 * Assuming Bob's AT is locked to Alice, trade-bot checks AT's state data to make sure it is correct.
-	 * <p>
-	 * If all is well, trade-bot then redeems AT using Alice's secret-A, releasing Bob's QORT to Alice.
-	 * <p>
-	 * In revealing a valid secret-A, Bob can then redeem the DOGE funds from P2SH-A.
-	 * <p>
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleAliceWaitingForAtLock(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		if (aliceUnexpectedState(repository, tradeBotData, atData, crossChainTradeData))
-			return;
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-		int lockTimeA = tradeBotData.getLockTimeA();
-
-		// Refund P2SH-A if we've passed lockTime-A
-		if (NTP.getTime() >= lockTimeA * 1000L) {
-			byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret());
-			String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-			long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-			long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-			long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-
-			BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-			switch (htlcStatusA) {
-				case UNFUNDED:
-				case FUNDING_IN_PROGRESS:
-				case FUNDED:
-					break;
-
-				case REDEEM_IN_PROGRESS:
-				case REDEEMED:
-					// Already redeemed?
-					TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-							() -> String.format("P2SH-A %s already spent? Assuming trade completed", p2shAddressA));
-					return;
-
-				case REFUND_IN_PROGRESS:
-				case REFUNDED:
-					TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED,
-							() -> String.format("P2SH-A %s already refunded. Trade aborted", p2shAddressA));
-					return;
-
-			}
-
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A,
-					() -> atData.getIsFinished()
-					? String.format("AT %s cancelled. Refunding P2SH-A %s - aborting trade", tradeBotData.getAtAddress(), p2shAddressA)
-					: String.format("LockTime-A reached, refunding P2SH-A %s - aborting trade", p2shAddressA));
-
-			return;
-		}
-
-		// We're waiting for AT to be in TRADE mode
-		if (crossChainTradeData.mode != AcctMode.TRADING)
-			return;
-
-		// AT is in TRADE mode and locked to us as checked by aliceUnexpectedState() above
-
-		// Find our MESSAGE to AT from previous state
-		List<MessageTransactionData> messageTransactionsData = repository.getMessageRepository().getMessagesByParticipants(tradeBotData.getTradeNativePublicKey(),
-				crossChainTradeData.qortalCreatorTradeAddress, null, null, null);
-		if (messageTransactionsData == null || messageTransactionsData.isEmpty()) {
-			LOGGER.warn(() -> String.format("Unable to find our message to trade creator %s?", crossChainTradeData.qortalCreatorTradeAddress));
-			return;
-		}
-
-		long recipientMessageTimestamp = messageTransactionsData.get(0).getTimestamp();
-		int refundTimeout = DogecoinACCTv3.calcRefundTimeout(recipientMessageTimestamp, lockTimeA);
-
-		// Our calculated refundTimeout should match AT's refundTimeout
-		if (refundTimeout != crossChainTradeData.refundTimeout) {
-			LOGGER.debug(() -> String.format("Trade AT refundTimeout '%d' doesn't match our refundTimeout '%d'", crossChainTradeData.refundTimeout, refundTimeout));
-			// We'll eventually refund
-			return;
-		}
-
-		// We're good to redeem AT
-
-		// Send 'redeem' MESSAGE to AT using both secret
-		byte[] secretA = tradeBotData.getSecret();
-		String qortalReceivingAddress = Base58.encode(tradeBotData.getReceivingAccountInfo()); // Actually contains whole address, not just PKH
-		byte[] messageData = DogecoinACCTv3.buildRedeemMessage(secretA, qortalReceivingAddress);
-		String messageRecipient = tradeBotData.getAtAddress();
-
-		boolean isMessageAlreadySent = repository.getMessageRepository().exists(tradeBotData.getTradeNativePublicKey(), messageRecipient, messageData);
-		if (!isMessageAlreadySent) {
-			PrivateKeyAccount sender = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
-			MessageTransaction messageTransaction = MessageTransaction.build(repository, sender, Group.NO_GROUP, messageRecipient, messageData, false, false);
-
-			messageTransaction.computeNonce();
-			messageTransaction.sign(sender);
-
-			// Reset repository state to prevent deadlock
-			repository.discardChanges();
-			ValidationResult result = messageTransaction.importAsUnconfirmed();
-
-			if (result != ValidationResult.OK) {
-				LOGGER.warn(() -> String.format("Unable to send MESSAGE to AT %s: %s", messageRecipient, result.name()));
-				return;
-			}
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-				() -> String.format("Redeeming AT %s. Funds should arrive at %s",
-						tradeBotData.getAtAddress(), qortalReceivingAddress));
-	}
-
-	/**
-	 * Trade-bot is waiting for Alice to redeem Bob's AT, thus revealing secret-A which is required to spend the DOGE funds from P2SH-A.
-	 * <p>
-	 * It's possible that Bob's AT has reached its trading timeout and automatically refunded QORT back to Bob. In which case,
-	 * trade-bot is done with this specific trade and finalizes in refunded state.
-	 * <p>
-	 * Assuming trade-bot can extract a valid secret-A from Alice's MESSAGE then trade-bot uses that to redeem the DOGE funds from P2SH-A
-	 * to Bob's 'foreign'/Dogecoin trade legacy-format address, as derived from trade private key.
-	 * <p>
-	 * (This could potentially be 'improved' to send DOGE to any address of Bob's choosing by changing the transaction output).
-	 * <p>
-	 * If trade-bot successfully broadcasts the transaction, then this specific trade is done.
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleBobWaitingForAtRedeem(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		// AT should be 'finished' once Alice has redeemed QORT funds
-		if (!atData.getIsFinished())
-			// Not finished yet
-			return;
-
-		// If AT is REFUNDED or CANCELLED then something has gone wrong
-		if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) {
-			// Alice hasn't redeemed the QORT, so there is no point in trying to redeem the DOGE
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED,
-					() -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress()));
-
-			return;
-		}
-
-		byte[] secretA = DogecoinACCTv3.getInstance().findSecretA(repository, crossChainTradeData);
-		if (secretA == null) {
-			LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress()));
-			return;
-		}
-
-		// Use secret-A to redeem P2SH-A
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		byte[] receivingAccountInfo = tradeBotData.getReceivingAccountInfo();
-		int lockTimeA = crossChainTradeData.lockTimeA;
-		byte[] redeemScriptA = BitcoinyHTLC.buildScript(crossChainTradeData.partnerForeignPKH, lockTimeA, crossChainTradeData.creatorForeignPKH, crossChainTradeData.hashOfSecretA);
-		String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-		// Fee for redeem/refund is subtracted from P2SH-A balance.
-		long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-		long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-		long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-		BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-		switch (htlcStatusA) {
-			case UNFUNDED:
-			case FUNDING_IN_PROGRESS:
-				// P2SH-A suddenly not funded? Our best bet at this point is to hope for AT auto-refund
-				return;
-
-			case REDEEM_IN_PROGRESS:
-			case REDEEMED:
-				// Double-check that we have redeemed P2SH-A...
-				break;
-
-			case REFUND_IN_PROGRESS:
-			case REFUNDED:
-				// Wait for AT to auto-refund
-				return;
-
-			case FUNDED: {
-				Coin redeemAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount);
-				ECKey redeemKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey());
-				List<TransactionOutput> fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA);
-
-				Transaction p2shRedeemTransaction = BitcoinyHTLC.buildRedeemTransaction(dogecoin.getNetworkParameters(), redeemAmount, redeemKey,
-						fundingOutputs, redeemScriptA, secretA, receivingAccountInfo);
-
-				dogecoin.broadcastTransaction(p2shRedeemTransaction);
-				break;
-			}
-		}
-
-		String receivingAddress = dogecoin.pkhToAddress(receivingAccountInfo);
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_DONE,
-				() -> String.format("P2SH-A %s redeemed. Funds should arrive at %s", tradeBotData.getAtAddress(), receivingAddress));
-	}
-
-	/**
-	 * Trade-bot is attempting to refund P2SH-A.
-	 * @throws ForeignBlockchainException
-	 */
-	private void handleAliceRefundingP2shA(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		int lockTimeA = tradeBotData.getLockTimeA();
-
-		// We can't refund P2SH-A until lockTime-A has passed
-		if (NTP.getTime() <= lockTimeA * 1000L)
-			return;
-
-		Dogecoin dogecoin = Dogecoin.getInstance();
-
-		// We can't refund P2SH-A until median block time has passed lockTime-A (see BIP113)
-		int medianBlockTime = dogecoin.getMedianBlockTime();
-		if (medianBlockTime <= lockTimeA)
-			return;
-
-		byte[] redeemScriptA = BitcoinyHTLC.buildScript(tradeBotData.getTradeForeignPublicKeyHash(), lockTimeA, crossChainTradeData.creatorForeignPKH, tradeBotData.getHashOfSecret());
-		String p2shAddressA = dogecoin.deriveP2shAddress(redeemScriptA);
-
-		// Fee for redeem/refund is subtracted from P2SH-A balance.
-		long feeTimestamp = calcFeeTimestamp(lockTimeA, crossChainTradeData.tradeTimeout);
-		long p2shFee = Dogecoin.getInstance().getP2shFee(feeTimestamp);
-		long minimumAmountA = crossChainTradeData.expectedForeignAmount + p2shFee;
-		BitcoinyHTLC.Status htlcStatusA = BitcoinyHTLC.determineHtlcStatus(dogecoin.getBlockchainProvider(), p2shAddressA, minimumAmountA);
-
-		switch (htlcStatusA) {
-			case UNFUNDED:
-			case FUNDING_IN_PROGRESS:
-				// Still waiting for P2SH-A to be funded...
-				return;
-
-			case REDEEM_IN_PROGRESS:
-			case REDEEMED:
-				// Too late!
-				TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-						() -> String.format("P2SH-A %s already spent!", p2shAddressA));
-				return;
-
-			case REFUND_IN_PROGRESS:
-			case REFUNDED:
-				break;
-
-			case FUNDED:{
-				Coin refundAmount = Coin.valueOf(crossChainTradeData.expectedForeignAmount);
-				ECKey refundKey = ECKey.fromPrivate(tradeBotData.getTradePrivateKey());
-				List<TransactionOutput> fundingOutputs = dogecoin.getUnspentOutputs(p2shAddressA);
-
-				// Determine receive address for refund
-				String receiveAddress = dogecoin.getUnusedReceiveAddress(tradeBotData.getForeignKey());
-				Address receiving = Address.fromString(dogecoin.getNetworkParameters(), receiveAddress);
-
-				Transaction p2shRefundTransaction = BitcoinyHTLC.buildRefundTransaction(dogecoin.getNetworkParameters(), refundAmount, refundKey,
-						fundingOutputs, redeemScriptA, lockTimeA, receiving.getHash());
-
-				dogecoin.broadcastTransaction(p2shRefundTransaction);
-				break;
-			}
-		}
-
-		TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDED,
-				() -> String.format("LockTime-A reached. Refunded P2SH-A %s. Trade aborted", p2shAddressA));
-	}
-
-	/**
-	 * Returns true if Alice finds AT unexpectedly cancelled, refunded, redeemed or locked to someone else.
-	 * <p>
-	 * Will automatically update trade-bot state to <tt>ALICE_REFUNDING_A</tt> or <tt>ALICE_DONE</tt> as necessary.
-	 * 
-	 * @throws DataException
-	 * @throws ForeignBlockchainException
-	 */
-	private boolean aliceUnexpectedState(Repository repository, TradeBotData tradeBotData,
-			ATData atData, CrossChainTradeData crossChainTradeData) throws DataException, ForeignBlockchainException {
-		// This is OK
-		if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.OFFERING)
-			return false;
-
-		boolean isAtLockedToUs = tradeBotData.getTradeNativeAddress().equals(crossChainTradeData.qortalPartnerAddress);
-
-		if (!atData.getIsFinished() && crossChainTradeData.mode == AcctMode.TRADING)
-			if (isAtLockedToUs) {
-				// AT is trading with us - OK
-				return false;
-			} else {
-				TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A,
-						() -> String.format("AT %s trading with someone else: %s. Refunding & aborting trade", tradeBotData.getAtAddress(), crossChainTradeData.qortalPartnerAddress));
-
-				return true;
-			}
-
-		if (atData.getIsFinished() && crossChainTradeData.mode == AcctMode.REDEEMED && isAtLockedToUs) {
-			// We've redeemed already?
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_DONE,
-					() -> String.format("AT %s already redeemed by us. Trade completed", tradeBotData.getAtAddress()));
-		} else {
-			// Any other state is not good, so start defensive refund
-			TradeBot.updateTradeBotState(repository, tradeBotData, State.ALICE_REFUNDING_A,
-					() -> String.format("AT %s cancelled/refunded/redeemed by someone else/invalid state. Refunding & aborting trade", tradeBotData.getAtAddress()));
-		}
-
-		return true;
-	}
-
-	private long calcFeeTimestamp(int lockTimeA, int tradeTimeout) {
-		return (lockTimeA - tradeTimeout * 60) * 1000L;
-	}
-
-}

src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java

@@ -212,7 +212,7 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
 		TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress));
 
 		// Attempt to backup the trade bot data
-		TradeBot.backupTradeBotData(repository, null);
+		TradeBot.backupTradeBotData(repository);
 
 		// Return to user for signing and broadcast as we don't have their Qortal private key
 		try {
@@ -287,8 +287,7 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
 				crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash);
 
 		// Attempt to backup the trade bot data
-		// Include tradeBotData as an additional parameter, since it's not in the repository yet
-		TradeBot.backupTradeBotData(repository, Arrays.asList(tradeBotData));
+		TradeBot.backupTradeBotData(repository);
 
 		// Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount
 		long p2shFee;
@@ -365,7 +364,6 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
 			case BOB_DONE:
 			case ALICE_REFUNDED:
 			case BOB_REFUNDED:
-			case ALICE_REFUNDING_A:
 				return true;
 
 			default:
@@ -727,7 +725,7 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
 			return;
 		}
 
-		byte[] secretA = LitecoinACCTv1.getInstance().findSecretA(repository, crossChainTradeData);
+		byte[] secretA = LitecoinACCTv1.findSecretA(repository, crossChainTradeData);
 		if (secretA == null) {
 			LOGGER.debug(() -> String.format("Unable to find secret-A from redeem message to AT %s?", tradeBotData.getAtAddress()));
 			return;