Merge pull request #254 from jschulthess/reticulum

Reticulum branch updates

.vscode/settings.json

@@ -1,3 +0,0 @@
-{
-    "java.compile.nullAnalysis.mode": "automatic"
-}

WindowsInstaller/Install Files/AppData/settings.json

@@ -1,3 +1,4 @@
 {
-  "apiDocumentationEnabled": true
+  "apiDocumentationEnabled": true,
+  "apiWhitelistEnabled": false
 }

WindowsInstaller/README.md

@@ -2,7 +2,9 @@
 
 ## Prerequisites
 
-* AdvancedInstaller v19.4 or better, and enterprise licence if translations are required
+* AdvancedInstaller v19.4 or better, and enterprise licence. 
+* Qortal has an open source license, however it currently (as of December 2024) only supports up to version 19. (We may need to reach out to Advanced Installer again to obtain a new license at some point, if needed. 
+* Reach out to @crowetic for links to the installer install files, and license. 
 * Installed AdoptOpenJDK v17 64bit, full JDK *not* JRE
 
 ## General build instructions
@@ -10,6 +12,12 @@
 If this is your first time opening the `qortal.aip` file then you might need to adjust
 configured paths, or create a dummy `D:` drive with the expected layout.
 
+Opening the aip file from within a clone of the qortal repo also works, if you have a separate windows machine setup to do the build. 
+
+You May need to change the location of the 'jre64' files inside Advanced Installer, if it is set to a path that your build machine doesn't have. 
+
+The Java Memory Arguments can be set manually, but as of December 2024 they have been reset back to system defaults. This should include G1GC Garbage Collector.
+
 Typical build procedure:
 
 * Place the `qortal.jar` file in `Install-Files\`

log4j2.properties

@@ -6,6 +6,34 @@ rootLogger.level = info
 rootLogger.appenderRef.console.ref = stdout
 rootLogger.appenderRef.rolling.ref = FILE
 
+### additional debug options (in case of problems eg. 202411
+
+#to see more QDN details - add the stuff below
+#logger.arbitrary.name = org.qortal.arbitrary
+#logger.arbitrary.level = trace
+
+#to see more QDN networking details - add stuff below
+#logger.arbitrarycontroller.name = org.qortal.controller.arbitrary
+#logger.arbitrarycontroller.level = debug
+
+# Support optional, Network Task debugging
+#logger.networkTask.name = org.qortal.network.task
+#logger.networkTask.level = debug
+
+# Support optional, Network Task tracing
+#logger.networkTask.name = org.qortal.network.task
+#logger.networkTask.level = trace
+
+# Support optional, Block debugging
+#logger.block.name = org.qortal.block
+#logger.block.level = debug
+
+# Support optional, Block tracing
+#logger.block.name = org.qortal.block
+#logger.block.level = trace
+
+### end additional debug options
+
 # Suppress extraneous bitcoinj library output
 logger.bitcoinj.name = org.bitcoinj
 logger.bitcoinj.level = error
@@ -18,6 +46,10 @@ logger.hsqldb.level = warn
 logger.hsqldbRepository.name = org.qortal.repository.hsqldb
 logger.hsqldbRepository.level = debug
 
+## Support optional, controller repository debugging
+#logger.controllerRepository.name = org.qortal.controller.repository
+#logger.controllerRepository.level = debug
+
 # Suppress extraneous Jersey warning
 logger.jerseyInject.name = org.glassfish.jersey.internal.inject.Providers
 logger.jerseyInject.level = off

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>4.6.1</version>
+	<version>4.7.1</version>
 	<packaging>jar</packaging>
 	<properties>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -16,7 +16,7 @@
 		<ciyam-at.version>1.4.2</ciyam-at.version>
 		<commons-net.version>3.8.0</commons-net.version>
 		<commons-text.version>1.12.0</commons-text.version>
-		<commons-io.version>2.17.0</commons-io.version>
+		<commons-io.version>2.18.0</commons-io.version>
 		<commons-compress.version>1.27.1</commons-compress.version>
 		<commons-lang3.version>3.17.0</commons-lang3.version>
 		<dagger.version>1.2.2</dagger.version>
@@ -26,9 +26,9 @@
 		<guava.version>33.3.1-jre</guava.version>
 		<hamcrest-library.version>2.2</hamcrest-library.version>
 		<homoglyph.version>1.2.1</homoglyph.version>
-		<hsqldb.version>2.5.1</hsqldb.version>
+		<hsqldb.version>2.7.4</hsqldb.version>
 		<icu4j.version>76.1</icu4j.version>
-		<java-diff-utils.version>4.12</java-diff-utils.version>
+		<java-diff-utils.version>4.15</java-diff-utils.version>
 		<javax.servlet-api.version>4.0.1</javax.servlet-api.version>
 		<jaxb-runtime.version>2.3.9</jaxb-runtime.version>
 		<jersey.version>2.42</jersey.version>
@@ -45,19 +45,24 @@
 		<maven-dependency-plugin.version>3.6.1</maven-dependency-plugin.version>
 		<maven-jar-plugin.version>3.4.2</maven-jar-plugin.version>
 		<maven-package-info-plugin.version>1.1.0</maven-package-info-plugin.version>
-		<maven-plugin.version>2.17.1</maven-plugin.version>
+		<maven-plugin.version>2.18.0</maven-plugin.version>
 		<maven-reproducible-build-plugin.version>0.17</maven-reproducible-build-plugin.version>
 		<maven-resources-plugin.version>3.3.1</maven-resources-plugin.version>
 		<maven-shade-plugin.version>3.6.0</maven-shade-plugin.version>
 		<maven-surefire-plugin.version>3.5.2</maven-surefire-plugin.version>
 		<protobuf.version>3.25.3</protobuf.version>
 		<replacer.version>1.5.3</replacer.version>
+		<reticulum.version>8623ce3</reticulum.version>
 		<simplemagic.version>1.17</simplemagic.version>
-		<slf4j.version>1.7.36</slf4j.version>
 		<swagger-api.version>2.0.10</swagger-api.version>
-		<swagger-ui.version>5.17.14</swagger-ui.version>
+		<swagger-ui.version>5.18.2</swagger-ui.version>
 		<upnp.version>1.2</upnp.version>
 		<xz.version>1.10</xz.version>
+   	<lombok.version>1.18.30</lombok.version>
+   	<jackson.version>2.16.1</jackson.version>
+		<slf4j.version>2.0.12</slf4j.version>
+    <nitrite.version>4.3.0</nitrite.version>
+    <junit.version>5.9.2</junit.version>
 	</properties>
 	<build>
 		<sourceDirectory>src/main/java</sourceDirectory>
@@ -426,13 +431,41 @@
 			<id>project.local</id>
 			<name>project</name>
 			<url>file:${project.basedir}/lib</url>
+      <snapshots>
+        <enabled>true</enabled>
+        <updatePolicy>always</updatePolicy>
+      </snapshots>
 		</repository>
 		<!-- jitpack for build-on-demand of altcoinj -->
 		<repository>
 			<id>jitpack.io</id>
 			<url>https://jitpack.io</url>
+      <snapshots>
+        <enabled>true</enabled>
+        <updatePolicy>always</updatePolicy>
+      </snapshots>
 		</repository>
 	</repositories>
+	<!--
+    <dependencyManagement>
+      <dependencies>
+        <dependency>
+          <groupId>org.dizitart</groupId>
+          <artifactId>nitrite-bom</artifactId>
+          <version>${nitrite.version}</version>
+          <scope>import</scope>
+          <type>pom</type>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-bom</artifactId>
+          <version>${log4j.version}</version>
+          <scope>import</scope>
+          <type>pom</type>
+        </dependency>
+      </dependencies>
+  	</dependencyManagement>
+    -->
 	<dependencies>
 		<!-- https://mvnrepository.com/artifact/org.codehaus.mojo/build-helper-maven-plugin -->
 		<dependency>
@@ -480,6 +513,13 @@
 			<artifactId>altcoinj</artifactId>
 			<version>${altcoinj.version}</version>
 		</dependency>
+		<!-- Build Reticulum from Source -->
+		<dependency>
+			<!--<groupId>com.github.sergst83</groupId>-->
+			<groupId>com.github.jschulthess</groupId>
+			<artifactId>reticulum-network-stack</artifactId>
+			<version>${reticulum.version}</version>
+		</dependency>
 		<!-- Utilities -->
 		<dependency>
 			<groupId>com.googlecode.json-simple</groupId>
@@ -558,35 +598,33 @@
 			<artifactId>guava</artifactId>
 			<version>${guava.version}</version>
 		</dependency>
-		<!-- Logging: log4j2 -->
-		<dependency>
-			<groupId>org.apache.logging.log4j</groupId>
-			<artifactId>log4j-core</artifactId>
-			<version>${log4j.version}</version>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.logging.log4j</groupId>
-			<artifactId>log4j-api</artifactId>
-			<version>${log4j.version}</version>
-		</dependency>
-		<!-- redirect slf4j to log4j2 -->
-		<dependency>
-			<groupId>org.apache.logging.log4j</groupId>
-			<artifactId>log4j-slf4j-impl</artifactId>
-			<version>${log4j.version}</version>
-		</dependency>
-		<!-- redirect java.utils.logging to log4j2 -->
-		<dependency>
-			<groupId>org.apache.logging.log4j</groupId>
-			<artifactId>log4j-jul</artifactId>
-			<version>${log4j.version}</version>
-		</dependency>
-		<!-- Logging: slf4j used by Jetty/Jersey -->
+    <!-- logging: slf4j -->
 		<dependency>
 			<groupId>org.slf4j</groupId>
 			<artifactId>slf4j-api</artifactId>
 			<version>${slf4j.version}</version>
 		</dependency>
+    <dependency>
+       <groupId>org.apache.logging.log4j</groupId>
+       <artifactId>log4j-slf4j2-impl</artifactId>
+       <version>${log4j.version}</version>
+    </dependency>
+		<!-- Logging: log4j2 -->
+    <dependency>
+       <groupId>org.apache.logging.log4j</groupId>
+       <artifactId>log4j-core</artifactId>
+       <version>${log4j.version}</version>
+    </dependency>
+    <dependency>
+    <groupId>org.apache.logging.log4j</groupId>
+       <artifactId>log4j-api</artifactId>
+       <version>${log4j.version}</version>
+    </dependency>
+    <dependency>
+       <groupId>org.apache.logging.log4j</groupId>
+       <artifactId>log4j-jul</artifactId>
+       <version>${log4j.version}</version>
+     </dependency> 
 		<!-- Servlet related -->
 		<dependency>
 			<groupId>javax.servlet</groupId>
@@ -728,6 +766,11 @@
 			<groupId>org.bouncycastle</groupId>
 			<artifactId>bctls-jdk15on</artifactId>
 			<version>${bouncycastle.version}</version>
+		</dependency><!-- https://mvnrepository.com/artifact/org.bouncycastle/bcprov-jdk15to18 -->
+		<dependency>
+		    <groupId>org.bouncycastle</groupId>
+		    <artifactId>bcprov-jdk15to18</artifactId>
+		    <version>${bouncycastle.version}</version>
 		</dependency>
 		<dependency>
 			<groupId>org.jsoup</groupId>
@@ -770,5 +813,123 @@
 			<artifactId>jaxb-runtime</artifactId>
 			<version>${jaxb-runtime.version}</version>
 		</dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
+    <dependency>
+      <groupId>commons-codec</groupId>
+      <artifactId>commons-codec</artifactId>
+      <version>1.16.1</version>
+    </dependency>
+    <!-- already declared earlier
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.14.0</version>
+    </dependency>
+    -->
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-collections4</artifactId>
+      <version>4.4</version>
+    </dependency>
+    <!-- already declared earlier
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-compress</artifactId>
+      <version>1.26.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+    </dependency>
+    -->
+    <!-- note: covered ? -->
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-yaml</artifactId>
+      <version>${jackson.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.yaml</groupId>
+      <artifactId>snakeyaml</artifactId>
+      <version>2.2</version>
+    </dependency>
+    <dependency>
+      <groupId>org.msgpack</groupId>
+      <artifactId>jackson-dataformat-msgpack</artifactId>
+      <version>0.9.8</version>
+    </dependency>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <version>${lombok.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.bouncycastle</groupId>
+      <artifactId>bcpkix-jdk15on</artifactId>
+      <version>1.70</version>
+    </dependency>
+    <dependency>
+      <groupId>com.macasaet.fernet</groupId>
+      <artifactId>fernet-java8</artifactId>
+      <version>1.5.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.igormaznitsa</groupId>
+      <artifactId>jbbp</artifactId>
+      <version>2.0.6</version>
+    </dependency>
+    <dependency>
+      <groupId>io.netty</groupId>
+      <artifactId>netty-all</artifactId>
+      <!--<version>4.1.106.Final</version>-->
+      <version>5.0.0.Alpha2</version>
+    </dependency>
+    <dependency>
+      <groupId>com.github.seancfoley</groupId>
+      <artifactId>ipaddress</artifactId>
+      <version>5.4.2</version>
+    </dependency>
+    <!-- Nitrite Modules -->
+    <dependency>
+        <groupId>org.dizitart</groupId>
+        <artifactId>nitrite</artifactId>
+        <version>${nitrite.version}</version>
+    </dependency>
+    <dependency>
+        <groupId>org.dizitart</groupId>
+        <artifactId>nitrite-mvstore-adapter</artifactId>
+        <version>${nitrite.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>com.h2database</groupId>
+      <artifactId>h2</artifactId>
+      <version>2.3.230</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.h2database</groupId>
+      <artifactId>h2-mvstore</artifactId>
+      <version>2.3.230</version>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>${junit.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-junit-jupiter</artifactId>
+      <version>5.10.0</version>
+      <scope>test</scope>
+    </dependency>
+
 	</dependencies>
 </project>

src/main/java/org/hsqldb/jdbc/HSQLDBPoolMonitored.java

@@ -0,0 +1,173 @@
+package org.hsqldb.jdbc;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.hsqldb.jdbc.pool.JDBCPooledConnection;
+import org.qortal.data.system.DbConnectionInfo;
+import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
+
+import javax.sql.ConnectionEvent;
+import javax.sql.PooledConnection;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+/**
+ * Class HSQLDBPoolMonitored
+ *
+ * This class uses the same logic as HSQLDBPool. The only difference is it monitors the state of every connection
+ * to the database. This is used for debugging purposes only.
+ */
+public class HSQLDBPoolMonitored extends HSQLDBPool {
+
+	private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepositoryFactory.class);
+
+	private static final String EMPTY = "Empty";
+	private static final String AVAILABLE = "Available";
+	private static final String ALLOCATED = "Allocated";
+
+	private ConcurrentHashMap<Integer, DbConnectionInfo> infoByIndex;
+
+	public HSQLDBPoolMonitored(int poolSize) {
+		super(poolSize);
+
+		this.infoByIndex = new ConcurrentHashMap<>(poolSize);
+	}
+
+	/**
+	 * Tries to retrieve a new connection using the properties that have already been
+	 * set.
+	 *
+	 * @return  a connection to the data source, or null if no spare connections in pool
+	 * @exception SQLException if a database access error occurs
+	 */
+	public Connection tryConnection() throws SQLException {
+		for (int i = 0; i < states.length(); i++) {
+			if (states.compareAndSet(i, RefState.available, RefState.allocated)) {
+				JDBCPooledConnection pooledConnection = connections[i];
+
+				if (pooledConnection == null)
+					// Probably shutdown situation
+					return null;
+
+				infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED));
+
+				return pooledConnection.getConnection();
+			}
+
+			if (states.compareAndSet(i, RefState.empty, RefState.allocated)) {
+				try {
+					JDBCPooledConnection pooledConnection = (JDBCPooledConnection) source.getPooledConnection();
+
+					if (pooledConnection == null)
+						// Probably shutdown situation
+						return null;
+
+					pooledConnection.addConnectionEventListener(this);
+					pooledConnection.addStatementEventListener(this);
+					connections[i] = pooledConnection;
+
+					infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED));
+
+					return pooledConnection.getConnection();
+				} catch (SQLException e) {
+					states.set(i, RefState.empty);
+					infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), EMPTY));
+				}
+			}
+		}
+
+		return null;
+	}
+
+	public Connection getConnection() throws SQLException {
+		int var1 = 300;
+		if (this.source.loginTimeout != 0) {
+			var1 = this.source.loginTimeout * 10;
+		}
+
+		if (this.closed) {
+			throw new SQLException("connection pool is closed");
+		} else {
+			for(int var2 = 0; var2 < var1; ++var2) {
+				for(int var3 = 0; var3 < this.states.length(); ++var3) {
+					if (this.states.compareAndSet(var3, 1, 2)) {
+						infoByIndex.put(var3, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED));
+						return this.connections[var3].getConnection();
+					}
+
+					if (this.states.compareAndSet(var3, 0, 2)) {
+						try {
+							JDBCPooledConnection var4 = (JDBCPooledConnection)this.source.getPooledConnection();
+							var4.addConnectionEventListener(this);
+							var4.addStatementEventListener(this);
+							this.connections[var3] = var4;
+
+							infoByIndex.put(var3, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), ALLOCATED));
+
+							return this.connections[var3].getConnection();
+						} catch (SQLException var6) {
+							this.states.set(var3, 0);
+							infoByIndex.put(var3, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), EMPTY));
+						}
+					}
+				}
+
+				try {
+					Thread.sleep(100L);
+				} catch (InterruptedException var5) {
+				}
+			}
+
+			throw JDBCUtil.invalidArgument();
+		}
+	}
+
+	public void connectionClosed(ConnectionEvent event) {
+		PooledConnection connection = (PooledConnection) event.getSource();
+
+		for (int i = 0; i < connections.length; i++) {
+			if (connections[i] == connection) {
+				states.set(i, RefState.available);
+				infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), AVAILABLE));
+				break;
+			}
+		}
+	}
+
+	public void connectionErrorOccurred(ConnectionEvent event) {
+		PooledConnection connection = (PooledConnection) event.getSource();
+
+		for (int i = 0; i < connections.length; i++) {
+			if (connections[i] == connection) {
+				states.set(i, RefState.allocated);
+				connections[i] = null;
+				states.set(i, RefState.empty);
+				infoByIndex.put(i, new DbConnectionInfo(System.currentTimeMillis(), Thread.currentThread().getName(), EMPTY));
+				break;
+			}
+		}
+	}
+
+	public List<DbConnectionInfo> getDbConnectionsStates() {
+
+		return infoByIndex.values().stream()
+			.sorted(Comparator.comparingLong(DbConnectionInfo::getUpdated))
+			.collect(Collectors.toList());
+	}
+
+	private int findConnectionIndex(ConnectionEvent connectionEvent) {
+		PooledConnection pooledConnection = (PooledConnection) connectionEvent.getSource();
+
+		for(int i = 0; i < this.connections.length; ++i) {
+			if (this.connections[i] == pooledConnection) {
+				return i;
+			}
+		}
+
+		return -1;
+	}
+}

src/main/java/org/qortal/account/Account.java

@@ -14,6 +14,7 @@ import org.qortal.repository.NameRepository;
 import org.qortal.repository.Repository;
 import org.qortal.settings.Settings;
 import org.qortal.utils.Base58;
+import org.qortal.utils.Groups;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -198,66 +199,85 @@ public class Account {
 
 	/** Returns whether account can be considered a "minting account".
 	 * <p>
-	 * To be considered a "minting account", the account needs to pass all of these tests:<br>
+	 * To be considered a "minting account", the account needs to pass some of these tests:<br>
 	 * <ul>
 	 * <li>account's level is at least <tt>minAccountLevelToMint</tt> from blockchain config</li>
-	 * <li>account's address have registered a name</li>
-	 * <li>account's address is member of minter group</li>
+	 * <li>account's address has registered a name</li>
+	 * <li>account's address is a member of the minter group</li>
 	 * </ul>
 	 *
+	 * @param isGroupValidated true if this account has already been validated for MINTER Group membership
 	 * @return true if account can be considered "minting account"
 	 * @throws DataException
 	 */
-	public boolean canMint() throws DataException {
+	public boolean canMint(boolean isGroupValidated) throws DataException {
 		AccountData accountData = this.repository.getAccountRepository().getAccount(this.address);
 		NameRepository nameRepository = this.repository.getNameRepository();
 		GroupRepository groupRepository = this.repository.getGroupRepository();
+		String myAddress = accountData.getAddress();
 
 		int blockchainHeight = this.repository.getBlockRepository().getBlockchainHeight();
-		int nameCheckHeight = BlockChain.getInstance().getOnlyMintWithNameHeight();
-		int levelToMint = BlockChain.getInstance().getMinAccountLevelToMint();
+
+		int levelToMint;
+
+		if( blockchainHeight >= BlockChain.getInstance().getIgnoreLevelForRewardShareHeight() ) {
+			levelToMint = 0;
+		}
+		else {
+			levelToMint = BlockChain.getInstance().getMinAccountLevelToMint();
+		}
+
 		int level = accountData.getLevel();
-		int groupIdToMint = BlockChain.getInstance().getMintingGroupId();
+		List<Integer> groupIdsToMint = Groups.getGroupIdsToMint( BlockChain.getInstance(), blockchainHeight );
+		int nameCheckHeight = BlockChain.getInstance().getOnlyMintWithNameHeight();
 		int groupCheckHeight = BlockChain.getInstance().getGroupMemberCheckHeight();
+		int removeNameCheckHeight = BlockChain.getInstance().getRemoveOnlyMintWithNameHeight();
 
-		String myAddress = accountData.getAddress();
-		List<NameData> myName = nameRepository.getNamesByOwner(myAddress);
-		boolean isMember = groupRepository.memberExists(groupIdToMint, myAddress);
+		// Can only mint if:
+		// Account's level is at least minAccountLevelToMint from blockchain config
+		if (blockchainHeight < nameCheckHeight) {
+			if (Account.isFounder(accountData.getFlags())) {
+				return accountData.getBlocksMintedPenalty() == 0;
+			} else {
+				return level >= levelToMint;
+			}
+		}
 
-		if (accountData == null)
-			return false;
+		// Can only mint on onlyMintWithNameHeight from blockchain config if:
+		// Account's level is at least minAccountLevelToMint from blockchain config
+		// Account's address has registered a name
+		if (blockchainHeight >= nameCheckHeight && blockchainHeight < groupCheckHeight) {
+			List<NameData> myName = nameRepository.getNamesByOwner(myAddress);
+			if (Account.isFounder(accountData.getFlags())) {
+				return accountData.getBlocksMintedPenalty() == 0 && !myName.isEmpty();
+			} else {
+				return level >= levelToMint && !myName.isEmpty();
+			}
+		}
 
-		// Can only mint if level is at least minAccountLevelToMint< from blockchain config
-		if (blockchainHeight < nameCheckHeight && level >= levelToMint)
-			return true;
+		// Can only mint on groupMemberCheckHeight from blockchain config if:
+		// Account's level is at least minAccountLevelToMint from blockchain config
+		// Account's address has registered a name
+		// Account's address is a member of the minter group
+		if (blockchainHeight >= groupCheckHeight && blockchainHeight < removeNameCheckHeight) {
+			List<NameData> myName = nameRepository.getNamesByOwner(myAddress);
+			if (Account.isFounder(accountData.getFlags())) {
+				return accountData.getBlocksMintedPenalty() == 0 && !myName.isEmpty() && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
+			} else {
+				return level >= levelToMint && !myName.isEmpty() && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
+			}
+		}
 
-		// Can only mint if have registered a name
-		if (blockchainHeight >= nameCheckHeight && blockchainHeight < groupCheckHeight && level >= levelToMint && !myName.isEmpty())
-			return true;
-
-		// Can only mint if have registered a name and is member of minter group id
-		if (blockchainHeight >= groupCheckHeight && level >= levelToMint && !myName.isEmpty() && isMember)
-			return true;
-
-		// Founders needs to pass same tests like minters
-		if (blockchainHeight < nameCheckHeight &&
-				Account.isFounder(accountData.getFlags()) &&
-				accountData.getBlocksMintedPenalty() == 0)
-			return true;
-
-		if (blockchainHeight >= nameCheckHeight &&
-				blockchainHeight < groupCheckHeight &&
-				Account.isFounder(accountData.getFlags()) &&
-				accountData.getBlocksMintedPenalty() == 0 &&
-				!myName.isEmpty())
-			return true;
-
-		if (blockchainHeight >= groupCheckHeight &&
-				Account.isFounder(accountData.getFlags()) &&
-				accountData.getBlocksMintedPenalty() == 0 &&
-				!myName.isEmpty() &&
-				isMember)
-			return true;
+		// Can only mint on removeOnlyMintWithNameHeight from blockchain config if:
+		// Account's level is at least minAccountLevelToMint from blockchain config
+		// Account's address is a member of the minter group
+		if (blockchainHeight >= removeNameCheckHeight) {
+			if (Account.isFounder(accountData.getFlags())) {
+				return accountData.getBlocksMintedPenalty() == 0 && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
+			} else {
+				return level >= levelToMint && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
+			}
+		}
 
 		return false;
 	}
@@ -272,7 +292,6 @@ public class Account {
 		return this.repository.getAccountRepository().getBlocksMintedPenaltyCount(this.address);
 	}
 
-
 	/** Returns whether account can build reward-shares.
 	 * <p>
 	 * To be able to create reward-shares, the account needs to pass at least one of these tests:<br>
@@ -286,6 +305,7 @@ public class Account {
 	 */
 	public boolean canRewardShare() throws DataException {
 		AccountData accountData = this.repository.getAccountRepository().getAccount(this.address);
+
 		if (accountData == null)
 			return false;
 
@@ -296,6 +316,9 @@ public class Account {
 		if (Account.isFounder(accountData.getFlags()) && accountData.getBlocksMintedPenalty() == 0)
 			return true;
 
+		if( this.repository.getBlockRepository().getBlockchainHeight() >= BlockChain.getInstance().getIgnoreLevelForRewardShareHeight() )
+			return true;
+
 		return false;
 	}
 
@@ -339,10 +362,28 @@ public class Account {
 	}
 
 	/**
-	 * Returns 'effective' minting level, or zero if reward-share does not exist.
+	 * Returns reward-share minting address, or unknown if reward-share does not exist.
 	 * 
 	 * @param repository
 	 * @param rewardSharePublicKey
+	 * @return address or unknown
+	 * @throws DataException
+	 */
+	public static String getRewardShareMintingAddress(Repository repository, byte[] rewardSharePublicKey) throws DataException {
+		// Find actual minter address
+		RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey);
+
+		if (rewardShareData == null)
+			return "Unknown";
+
+		return rewardShareData.getMinter();
+	}
+
+	/**
+	 * Returns 'effective' minting level, or zero if reward-share does not exist.
+	 *
+	 * @param repository
+	 * @param rewardSharePublicKey
 	 * @return 0+
 	 * @throws DataException
 	 */
@@ -355,6 +396,7 @@ public class Account {
 		Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
 		return rewardShareMinter.getEffectiveMintingLevel();
 	}
+
 	/**
 	 * Returns 'effective' minting level, with a fix for the zero level.
 	 * <p>

src/main/java/org/qortal/api/ApiService.java

@@ -194,6 +194,7 @@ public class ApiService {
 
 			context.addServlet(AdminStatusWebSocket.class, "/websockets/admin/status");
 			context.addServlet(BlocksWebSocket.class, "/websockets/blocks");
+			context.addServlet(DataMonitorSocket.class, "/websockets/datamonitor");
 			context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*");
 			context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages");
 			context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers");

src/main/java/org/qortal/api/model/ApiOnlineAccount.java

@@ -1,7 +1,13 @@
 package org.qortal.api.model;
 
+import org.qortal.account.Account;
+import org.qortal.repository.DataException;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.repository.Repository;
+
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
 
 // All properties to be converted to JSON via JAXB
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -47,4 +53,31 @@ public class ApiOnlineAccount {
 		return this.recipientAddress;
 	}
 
+	public int getMinterLevelFromPublicKey() {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			return Account.getRewardShareEffectiveMintingLevel(repository, this.rewardSharePublicKey);
+		} catch (DataException e) {
+			return 0;
+		}
+	}
+
+	public boolean getIsMember() {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			return repository.getGroupRepository().memberExists(694, getMinterAddress());
+		} catch (DataException e) {
+			return false;
+		}
+	}
+
+	// JAXB special
+	
+	@XmlElement(name = "minterLevel")
+	protected int getMinterLevel() {
+		return getMinterLevelFromPublicKey();
+	}
+
+	@XmlElement(name = "isMinterMember")
+	protected boolean getMinterMember() {
+		return getIsMember();
+	}
 }

src/main/java/org/qortal/api/model/BlockMintingInfo.java

@@ -9,6 +9,7 @@ import java.math.BigInteger;
 public class BlockMintingInfo {
 
 	public byte[] minterPublicKey;
+	public String minterAddress;
 	public int minterLevel;
 	public int onlineAccountsCount;
 	public BigDecimal maxDistance;
@@ -19,5 +20,4 @@ public class BlockMintingInfo {
 
 	public BlockMintingInfo() {
 	}
-
 }

src/main/java/org/qortal/api/model/CrossChainTradeLedgerEntry.java

@@ -0,0 +1,72 @@
+package org.qortal.api.model;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import org.qortal.data.crosschain.CrossChainTradeData;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class CrossChainTradeLedgerEntry {
+
+	private String market;
+
+	private String currency;
+
+	@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
+	private long quantity;
+
+	@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
+	private long feeAmount;
+
+	private String feeCurrency;
+
+	@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
+	private long totalPrice;
+
+	private long tradeTimestamp;
+
+	protected CrossChainTradeLedgerEntry() {
+		/* For JAXB */
+	}
+
+	public CrossChainTradeLedgerEntry(String market, String currency, long quantity, long feeAmount, String feeCurrency, long totalPrice, long tradeTimestamp) {
+		this.market = market;
+		this.currency = currency;
+		this.quantity = quantity;
+		this.feeAmount = feeAmount;
+		this.feeCurrency = feeCurrency;
+		this.totalPrice = totalPrice;
+		this.tradeTimestamp = tradeTimestamp;
+	}
+
+	public String getMarket() {
+		return market;
+	}
+
+	public String getCurrency() {
+		return currency;
+	}
+
+	public long getQuantity() {
+		return quantity;
+	}
+
+	public long getFeeAmount() {
+		return feeAmount;
+	}
+
+	public String getFeeCurrency() {
+		return feeCurrency;
+	}
+
+	public long getTotalPrice() {
+		return totalPrice;
+	}
+
+	public long getTradeTimestamp() {
+		return tradeTimestamp;
+	}
+}

src/main/java/org/qortal/api/model/DatasetStatus.java

@@ -0,0 +1,50 @@
+package org.qortal.api.model;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.Objects;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class DatasetStatus {
+
+    private String name;
+
+    private long count;
+
+    public DatasetStatus() {}
+
+    public DatasetStatus(String name, long count) {
+        this.name = name;
+        this.count = count;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public long getCount() {
+        return count;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DatasetStatus that = (DatasetStatus) o;
+        return count == that.count && Objects.equals(name, that.name);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, count);
+    }
+
+    @Override
+    public String toString() {
+        return "DatasetStatus{" +
+                "name='" + name + '\'' +
+                ", count=" + count +
+                '}';
+    }
+}

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -33,9 +33,13 @@ import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
 import org.qortal.controller.arbitrary.ArbitraryMetadataManager;
 import org.qortal.data.account.AccountData;
 import org.qortal.data.arbitrary.ArbitraryCategoryInfo;
+import org.qortal.data.arbitrary.ArbitraryDataIndexDetail;
+import org.qortal.data.arbitrary.ArbitraryDataIndexScoreKey;
+import org.qortal.data.arbitrary.ArbitraryDataIndexScorecard;
 import org.qortal.data.arbitrary.ArbitraryResourceData;
 import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
 import org.qortal.data.arbitrary.ArbitraryResourceStatus;
+import org.qortal.data.arbitrary.IndexCache;
 import org.qortal.data.naming.NameData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -69,8 +73,11 @@ import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.stream.Collectors;
 
 @Path("/arbitrary")
 @Tag(name = "Arbitrary")
@@ -172,6 +179,7 @@ public class ArbitraryResource {
 			@Parameter(description = "Name (searches name field only)") @QueryParam("name") List<String> names,
 			@Parameter(description = "Title (searches title metadata field only)") @QueryParam("title") String title,
 			@Parameter(description = "Description (searches description metadata field only)") @QueryParam("description") String description,
+			@Parameter(description = "Keyword (searches description metadata field by keywords)") @QueryParam("keywords") List<String> keywords,
 			@Parameter(description = "Prefix only (if true, only the beginning of fields are matched)") @QueryParam("prefix") Boolean prefixOnly,
 			@Parameter(description = "Exact match names only (if true, partial name matches are excluded)") @QueryParam("exactmatchnames") Boolean exactMatchNamesOnly,
 			@Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource,
@@ -212,7 +220,7 @@ public class ArbitraryResource {
 			}
 
 			List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
-					.searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly,
+					.searchArbitraryResources(service, query, identifier, names, title, description, keywords, usePrefixOnly,
 							exactMatchNames, defaultRes, mode, minLevel, followedOnly, excludeBlocked, includeMetadata, includeStatus,
 							before, after, limit, offset, reverse);
 
@@ -1185,6 +1193,90 @@ public class ArbitraryResource {
 		}
 	}
 
+	@GET
+	@Path("/indices")
+	@Operation(
+			summary = "Find matching arbitrary resource indices",
+			description = "",
+			responses = {
+					@ApiResponse(
+							description = "indices",
+							content = @Content(
+									array = @ArraySchema(
+											schema = @Schema(
+													implementation = ArbitraryDataIndexScorecard.class
+											)
+									)
+							)
+					)
+			}
+	)
+	public List<ArbitraryDataIndexScorecard> searchIndices(@QueryParam("terms") String[] terms) {
+
+		List<ArbitraryDataIndexDetail> indices = new ArrayList<>();
+
+		// get index details for each term
+		for( String term : terms ) {
+			List<ArbitraryDataIndexDetail> details = IndexCache.getInstance().getIndicesByTerm().get(term);
+
+			if( details != null ) {
+				indices.addAll(details);
+			}
+		}
+
+		// sum up the scores for each index with identical attributes
+		Map<ArbitraryDataIndexScoreKey, Double> scoreForKey
+			= indices.stream()
+				.collect(
+					Collectors.groupingBy(
+						index -> new ArbitraryDataIndexScoreKey(index.name, index.category, index.link),
+						Collectors.summingDouble(detail -> 1.0 / detail.rank)
+					)
+				);
+
+		// create scorecards for each index group and put them in descending order by score
+		List<ArbitraryDataIndexScorecard> scorecards
+			= scoreForKey.entrySet().stream().map(
+				entry
+				->
+				new ArbitraryDataIndexScorecard(
+					entry.getValue(),
+					entry.getKey().name,
+					entry.getKey().category,
+					entry.getKey().link)
+				)
+				.sorted(Comparator.comparingDouble(ArbitraryDataIndexScorecard::getScore).reversed())
+				.collect(Collectors.toList());
+
+		return scorecards;
+	}
+
+	@GET
+	@Path("/indices/{name}/{idPrefix}")
+	@Operation(
+			summary = "Find matching arbitrary resource indices for a registered name and identifier prefix",
+			description = "",
+			responses = {
+					@ApiResponse(
+							description = "indices",
+							content = @Content(
+									array = @ArraySchema(
+											schema = @Schema(
+													implementation = ArbitraryDataIndexDetail.class
+											)
+									)
+							)
+					)
+			}
+	)
+	public List<ArbitraryDataIndexDetail> searchIndicesByName(@PathParam("name") String name, @PathParam("idPrefix") String idPrefix) {
+
+		return
+			IndexCache.getInstance().getIndicesByIssuer()
+				.getOrDefault(name, new ArrayList<>(0)).stream()
+					.filter( indexDetail -> indexDetail.indexIdentifer.startsWith(idPrefix))
+					.collect(Collectors.toList());
+	}
 
 	// Shared methods
 

src/main/java/org/qortal/api/resource/AssetsResource.java

@@ -16,9 +16,13 @@ import org.qortal.api.model.AggregatedOrder;
 import org.qortal.api.model.TradeWithOrderInfo;
 import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
 import org.qortal.asset.Asset;
+import org.qortal.controller.hsqldb.HSQLDBBalanceRecorder;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.account.AccountBalanceData;
 import org.qortal.data.account.AccountData;
+import org.qortal.data.account.AddressAmountData;
+import org.qortal.data.account.BlockHeightRange;
+import org.qortal.data.account.BlockHeightRangeAddressAmounts;
 import org.qortal.data.asset.AssetData;
 import org.qortal.data.asset.OrderData;
 import org.qortal.data.asset.RecentTradeData;
@@ -33,6 +37,7 @@ import org.qortal.transaction.Transaction;
 import org.qortal.transaction.Transaction.ValidationResult;
 import org.qortal.transform.TransformationException;
 import org.qortal.transform.transaction.*;
+import org.qortal.utils.BalanceRecorderUtils;
 import org.qortal.utils.Base58;
 
 import javax.servlet.http.HttpServletRequest;
@@ -42,6 +47,7 @@ import javax.ws.rs.core.MediaType;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.stream.Collectors;
 
 @Path("/assets")
@@ -179,6 +185,122 @@ public class AssetsResource {
 		}
 	}
 
+	@GET
+	@Path("/balancedynamicranges")
+	@Operation(
+			summary = "Get balance dynamic ranges listed.",
+			description = ".",
+			responses = {
+					@ApiResponse(
+							content = @Content(
+									array = @ArraySchema(
+											schema = @Schema(
+													implementation = BlockHeightRange.class
+											)
+									)
+							)
+					)
+			}
+	)
+	public List<BlockHeightRange> getBalanceDynamicRanges(
+			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+			@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
+			@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) {
+
+		Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();
+
+		if( recorder.isPresent()) {
+			return recorder.get().getRanges(offset, limit, reverse);
+		}
+		else {
+			return new ArrayList<>(0);
+		}
+	}
+
+	@GET
+	@Path("/balancedynamicrange/{height}")
+	@Operation(
+			summary = "Get balance dynamic range for a given height.",
+			description = ".",
+			responses = {
+					@ApiResponse(
+							content = @Content(
+										schema = @Schema(
+												implementation = BlockHeightRange.class
+										)
+							)
+					)
+			}
+	)
+	@ApiErrors({
+			ApiError.INVALID_CRITERIA, ApiError.INVALID_DATA
+	})
+	public BlockHeightRange getBalanceDynamicRange(@PathParam("height") int height) {
+
+		Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();
+
+		if( recorder.isPresent()) {
+			Optional<BlockHeightRange> range = recorder.get().getRange(height);
+
+			if( range.isPresent() ) {
+				return range.get();
+			}
+			else {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+			}
+		}
+		else {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+		}
+	}
+
+	@GET
+	@Path("/balancedynamicamounts/{begin}/{end}")
+	@Operation(
+			summary = "Get balance dynamic ranges address amounts listed.",
+			description = ".",
+			responses = {
+					@ApiResponse(
+							content = @Content(
+									array = @ArraySchema(
+											schema = @Schema(
+													implementation = AddressAmountData.class
+											)
+									)
+							)
+					)
+			}
+	)
+	@ApiErrors({
+			ApiError.INVALID_CRITERIA, ApiError.INVALID_DATA
+	})
+	public List<AddressAmountData> getBalanceDynamicAddressAmounts(
+			@PathParam("begin") int begin,
+			@PathParam("end") int end,
+			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+			@Parameter(ref = "limit") @QueryParam("limit") Integer limit) {
+
+		Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();
+
+		if( recorder.isPresent()) {
+			Optional<BlockHeightRangeAddressAmounts> addressAmounts = recorder.get().getAddressAmounts(new BlockHeightRange(begin, end, false));
+
+			if( addressAmounts.isPresent() ) {
+				return addressAmounts.get().getAmounts().stream()
+					.sorted(BalanceRecorderUtils.ADDRESS_AMOUNT_DATA_COMPARATOR.reversed())
+					.skip(offset)
+					.limit(limit)
+					.collect(Collectors.toList());
+			}
+			else {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+			}
+		}
+		else {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+		}
+	}
+
 	@GET
 	@Path("/openorders/{assetid}/{otherassetid}")
 	@Operation(

src/main/java/org/qortal/api/resource/BlocksResource.java

@@ -19,6 +19,8 @@ import org.qortal.crypto.Crypto;
 import org.qortal.data.account.AccountData;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.block.BlockSummaryData;
+import org.qortal.data.block.DecodedOnlineAccountData;
+import org.qortal.data.network.OnlineAccountData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.repository.BlockArchiveReader;
 import org.qortal.repository.DataException;
@@ -27,6 +29,7 @@ import org.qortal.repository.RepositoryManager;
 import org.qortal.transform.TransformationException;
 import org.qortal.transform.block.BlockTransformer;
 import org.qortal.utils.Base58;
+import org.qortal.utils.Blocks;
 import org.qortal.utils.Triple;
 
 import javax.servlet.http.HttpServletRequest;
@@ -45,6 +48,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Set;
 
 @Path("/blocks")
 @Tag(name = "Blocks")
@@ -542,6 +546,7 @@ public class BlocksResource {
 				}
 			}
 
+			String minterAddress = Account.getRewardShareMintingAddress(repository, blockData.getMinterPublicKey());
 			int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
 			if (minterLevel == 0)
 				// This may be unavailable when requesting a trimmed block
@@ -554,6 +559,7 @@ public class BlocksResource {
 
 			BlockMintingInfo blockMintingInfo = new BlockMintingInfo();
 			blockMintingInfo.minterPublicKey = blockData.getMinterPublicKey();
+			blockMintingInfo.minterAddress = minterAddress;
 			blockMintingInfo.minterLevel = minterLevel;
 			blockMintingInfo.onlineAccountsCount = blockData.getOnlineAccountsCount();
 			blockMintingInfo.maxDistance = new BigDecimal(block.MAX_DISTANCE);
@@ -888,4 +894,49 @@ public class BlocksResource {
 		}
 	}
 
-}
+	@GET
+	@Path("/onlineaccounts/{height}")
+	@Operation(
+			summary = "Get online accounts for block",
+			description = "Returns the online accounts who submitted signatures for this block",
+			responses = {
+					@ApiResponse(
+							description = "online accounts",
+							content = @Content(
+									array = @ArraySchema(
+											schema = @Schema(
+													implementation = DecodedOnlineAccountData.class
+											)
+									)
+							)
+					)
+			}
+	)
+	@ApiErrors({
+			ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE
+	})
+	public Set<DecodedOnlineAccountData> getOnlineAccounts(@PathParam("height") int height) {
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+
+			// get block from database
+			BlockData blockData = repository.getBlockRepository().fromHeight(height);
+
+			// if block data is not in the database, then try the archive
+			if (blockData == null) {
+				blockData = repository.getBlockArchiveRepository().fromHeight(height);
+
+				// if the block is not in the database or the archive, then the block is unknown
+				if( blockData == null ) {
+					throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+				}
+			}
+
+			Set<DecodedOnlineAccountData> onlineAccounts = Blocks.getDecodedOnlineAccountsForBlock(repository, blockData);
+
+			return onlineAccounts;
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
+		}
+	}
+}

src/main/java/org/qortal/api/resource/ChatResource.java

@@ -234,17 +234,21 @@ public class ChatResource {
 		}
 	)
 	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE})
-	public ActiveChats getActiveChats(@PathParam("address") String address, @QueryParam("encoding") Encoding encoding) {
+	public ActiveChats getActiveChats(
+		@PathParam("address") String address,
+		@QueryParam("encoding") Encoding encoding,
+		@QueryParam("haschatreference") Boolean hasChatReference
+	) {
 		if (address == null || !Crypto.isValidAddress(address))
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
-
+	
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			return repository.getChatRepository().getActiveChats(address, encoding);
+			return repository.getChatRepository().getActiveChats(address, encoding, hasChatReference);
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
 	}
-
+	
 	@POST
 	@Operation(
 		summary = "Build raw, unsigned, CHAT transaction",

src/main/java/org/qortal/api/resource/CrossChainResource.java

@@ -10,11 +10,13 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
+import org.glassfish.jersey.media.multipart.ContentDisposition;
 import org.qortal.api.ApiError;
 import org.qortal.api.ApiErrors;
 import org.qortal.api.ApiExceptionFactory;
 import org.qortal.api.Security;
 import org.qortal.api.model.CrossChainCancelRequest;
+import org.qortal.api.model.CrossChainTradeLedgerEntry;
 import org.qortal.api.model.CrossChainTradeSummary;
 import org.qortal.controller.tradebot.TradeBot;
 import org.qortal.crosschain.ACCT;
@@ -44,14 +46,20 @@ import org.qortal.utils.Base58;
 import org.qortal.utils.ByteArray;
 import org.qortal.utils.NTP;
 
+import javax.servlet.ServletContext;
 import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
+import javax.ws.rs.core.HttpHeaders;
 import javax.ws.rs.core.MediaType;
+import java.io.IOException;
 import java.util.*;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
+
+
 @Path("/crosschain")
 @Tag(name = "Cross-Chain")
 public class CrossChainResource {
@@ -59,6 +67,13 @@ public class CrossChainResource {
 	@Context
 	HttpServletRequest request;
 
+	@Context
+	HttpServletResponse response;
+
+	@Context
+	ServletContext context;
+
+
 	@GET
 	@Path("/tradeoffers")
 	@Operation(
@@ -255,6 +270,12 @@ public class CrossChainResource {
 				description = "Only return trades that completed on/after this timestamp (milliseconds since epoch)",
 				example = "1597310000000"
 			) @QueryParam("minimumTimestamp") Long minimumTimestamp,
+			@Parameter(
+				description = "Optionally filter by buyer Qortal public key"
+			) @QueryParam("buyerPublicKey") String buyerPublicKey58,
+			@Parameter(
+				description = "Optionally filter by seller Qortal public key"
+			) @QueryParam("sellerPublicKey") String sellerPublicKey58,
 			@Parameter( ref = "limit") @QueryParam("limit") Integer limit,
 			@Parameter( ref = "offset" ) @QueryParam("offset") Integer offset,
 			@Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) {
@@ -266,6 +287,10 @@ public class CrossChainResource {
 		if (minimumTimestamp != null && minimumTimestamp <= 0)
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
+		// Decode public keys
+		byte[] buyerPublicKey = decodePublicKey(buyerPublicKey58);
+		byte[] sellerPublicKey = decodePublicKey(sellerPublicKey58);
+
 		final Boolean isFinished = Boolean.TRUE;
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -296,7 +321,7 @@ public class CrossChainResource {
 				byte[] codeHash = acctInfo.getKey().value;
 				ACCT acct = acctInfo.getValue().get();
 
-				List<ATStateData> atStates = repository.getATRepository().getMatchingFinalATStates(codeHash,
+				List<ATStateData> atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, buyerPublicKey, sellerPublicKey,
 						isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumFinalHeight,
 						limit, offset, reverse);
 
@@ -335,6 +360,120 @@ public class CrossChainResource {
 		}
 	}
 
+	/**
+	 * Decode Public Key
+	 *
+	 * @param publicKey58 the public key in a string
+	 *
+	 * @return the public key in bytes
+	 */
+	private byte[] decodePublicKey(String publicKey58) {
+
+		if( publicKey58 == null ) return null;
+		if( publicKey58.isEmpty() ) return new byte[0];
+
+		byte[] publicKey;
+		try {
+			publicKey = Base58.decode(publicKey58);
+		} catch (NumberFormatException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY, e);
+		}
+
+		// Correct size for public key?
+		if (publicKey.length != Transformer.PUBLIC_KEY_LENGTH)
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY);
+
+		return publicKey;
+	}
+
+	@GET
+	@Path("/ledger/{publicKey}")
+	@Operation(
+			summary = "Accounting entries for all trades.",
+			description = "Returns accounting entries for all completed cross-chain trades",
+			responses = {
+					@ApiResponse(
+							content = @Content(
+									schema = @Schema(
+											type = "string",
+											format = "byte"
+									)
+							)
+					)
+			}
+	)
+	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
+	public HttpServletResponse getLedgerEntries(
+			@PathParam("publicKey") String publicKey58,
+			@Parameter(
+					description = "Only return trades that completed on/after this timestamp (milliseconds since epoch)",
+					example = "1597310000000"
+			) @QueryParam("minimumTimestamp") Long minimumTimestamp) {
+
+		byte[] publicKey = decodePublicKey(publicKey58);
+
+		// minimumTimestamp (if given) needs to be positive
+		if (minimumTimestamp != null && minimumTimestamp <= 0)
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			Integer minimumFinalHeight = null;
+
+			if (minimumTimestamp != null) {
+				minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(minimumTimestamp);
+				// If not found in the block repository it will return either 0 or 1
+				if (minimumFinalHeight == 0 || minimumFinalHeight == 1) {
+					// Try the archive
+					minimumFinalHeight = repository.getBlockArchiveRepository().getHeightFromTimestamp(minimumTimestamp);
+				}
+
+				if (minimumFinalHeight == 0)
+					// We don't have any blocks since minimumTimestamp, let alone trades, so nothing to return
+					return response;
+
+				// height returned from repository is for block BEFORE timestamp
+				// but we want trades AFTER timestamp so bump height accordingly
+				minimumFinalHeight++;
+			}
+
+			List<CrossChainTradeLedgerEntry> crossChainTradeLedgerEntries = new ArrayList<>();
+
+			Map<ByteArray, Supplier<ACCT>> acctsByCodeHash = SupportedBlockchain.getAcctMap();
+
+			// collect ledger entries for each ACCT
+			for (Map.Entry<ByteArray, Supplier<ACCT>> acctInfo : acctsByCodeHash.entrySet()) {
+				byte[] codeHash = acctInfo.getKey().value;
+				ACCT acct = acctInfo.getValue().get();
+
+				// collect buys and sells
+				CrossChainUtils.collectLedgerEntries(publicKey, repository, minimumFinalHeight, crossChainTradeLedgerEntries, codeHash, acct, true);
+				CrossChainUtils.collectLedgerEntries(publicKey, repository, minimumFinalHeight, crossChainTradeLedgerEntries, codeHash, acct, false);
+			}
+
+			crossChainTradeLedgerEntries.sort((a, b) -> Longs.compare(a.getTradeTimestamp(), b.getTradeTimestamp()));
+
+			response.setStatus(HttpServletResponse.SC_OK);
+			response.setContentType("text/csv");
+			response.setHeader(
+				HttpHeaders.CONTENT_DISPOSITION,
+				ContentDisposition
+					.type("attachment")
+					.fileName(CrossChainUtils.createLedgerFileName(Crypto.toAddress(publicKey)))
+					.build()
+					.toString()
+			);
+
+			CrossChainUtils.writeToLedger( response.getWriter(), crossChainTradeLedgerEntries);
+
+			return response;
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		} catch (IOException e) {
+			response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+			return response;
+		}
+	}
+
 	@GET
 	@Path("/price/{blockchain}")
 	@Operation(

src/main/java/org/qortal/api/resource/CrossChainUtils.java

@@ -10,21 +10,36 @@ import org.bitcoinj.script.ScriptBuilder;
 
 import org.bouncycastle.util.Strings;
 import org.json.simple.JSONObject;
+import org.qortal.api.model.CrossChainTradeLedgerEntry;
 import org.qortal.api.model.crosschain.BitcoinyTBDRequest;
 import org.qortal.crosschain.*;
 import org.qortal.data.at.ATData;
+import org.qortal.data.at.ATStateData;
 import org.qortal.data.crosschain.*;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
+import org.qortal.utils.Amounts;
 import org.qortal.utils.BitTwiddling;
 
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.Writer;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
 import java.util.*;
 import java.util.stream.Collectors;
 
 
 public class CrossChainUtils {
+    public static final String QORT_CURRENCY_CODE = "QORT";
     private static final Logger LOGGER = LogManager.getLogger(CrossChainUtils.class);
     public static final String CORE_API_CALL = "Core API Call";
+    public static final String QORTAL_EXCHANGE_LABEL = "Qortal";
 
     public static ServerConfigurationInfo buildServerConfigurationInfo(Bitcoiny blockchain) {
 
@@ -632,4 +647,128 @@ public class CrossChainUtils {
         byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA);
         return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes);
     }
+
+    /**
+     * Write To Ledger
+     *
+     * @param writer the writer to the ledger
+     * @param entries the entries to write to the ledger
+     *
+     * @throws IOException
+     */
+    public static void writeToLedger(Writer writer, List<CrossChainTradeLedgerEntry> entries) throws IOException {
+
+        BufferedWriter bufferedWriter = new BufferedWriter(writer);
+
+        StringJoiner header = new StringJoiner(",");
+        header.add("Market");
+        header.add("Currency");
+        header.add("Quantity");
+        header.add("Commission Paid");
+        header.add("Commission Currency");
+        header.add("Total Price");
+        header.add("Date Time");
+        header.add("Exchange");
+
+        bufferedWriter.append(header.toString());
+
+        DateFormat dateFormatter = new SimpleDateFormat("yyyyMMdd HH:mm");
+        dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+        for( CrossChainTradeLedgerEntry entry : entries ) {
+            StringJoiner joiner = new StringJoiner(",");
+
+            joiner.add(entry.getMarket());
+            joiner.add(entry.getCurrency());
+            joiner.add(String.valueOf(Amounts.prettyAmount(entry.getQuantity())));
+            joiner.add(String.valueOf(Amounts.prettyAmount(entry.getFeeAmount())));
+            joiner.add(entry.getFeeCurrency());
+            joiner.add(String.valueOf(Amounts.prettyAmount(entry.getTotalPrice())));
+            joiner.add(dateFormatter.format(new Date(entry.getTradeTimestamp())));
+            joiner.add(QORTAL_EXCHANGE_LABEL);
+
+            bufferedWriter.newLine();
+            bufferedWriter.append(joiner.toString());
+        }
+
+        bufferedWriter.newLine();
+        bufferedWriter.flush();
+    }
+
+    /**
+     * Create Ledger File Name
+     *
+     * Create a file name the includes timestamp and address.
+     *
+     * @param address the address
+     *
+     * @return the file name created
+     */
+    public static String createLedgerFileName(String address) {
+        DateFormat dateFormatter = new SimpleDateFormat("yyyyMMddHHmmss");
+        String fileName = "ledger-" + address + "-" + dateFormatter.format(new Date());
+        return fileName;
+    }
+
+    /**
+     * Collect Ledger Entries
+     *
+     * @param publicKey the public key for the ledger entries, buy and sell
+     * @param repository the data repository
+     * @param minimumFinalHeight the minimum block height for entries to be collected
+     * @param entries the ledger entries to add to
+     * @param codeHash code hash for the entry blockchain
+     * @param acct the ACCT for the entry blockchain
+     * @param isBuy true collecting entries for a buy, otherwise false
+     *
+     * @throws DataException
+     */
+    public static void collectLedgerEntries(
+            byte[] publicKey,
+            Repository repository,
+            Integer minimumFinalHeight,
+            List<CrossChainTradeLedgerEntry> entries,
+            byte[] codeHash,
+            ACCT acct,
+            boolean isBuy) throws DataException {
+
+        // get all the final AT states for the code hash (foreign coin)
+        List<ATStateData> atStates
+            = repository.getATRepository().getMatchingFinalATStates(
+                codeHash,
+                isBuy ? publicKey : null,
+                !isBuy ? publicKey : null,
+                Boolean.TRUE, acct.getModeByteOffset(),
+                (long) AcctMode.REDEEMED.value,
+                minimumFinalHeight,
+                null, null, false
+        );
+
+        String foreignBlockchainCurrencyCode = acct.getBlockchain().getCurrencyCode();
+
+        // for each trade, build ledger entry, collect ledger entry
+        for (ATStateData atState : atStates) {
+            CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState);
+
+            // We also need block timestamp for use as trade timestamp
+            long localTimestamp = repository.getBlockRepository().getTimestampFromHeight(atState.getHeight());
+
+            if (localTimestamp == 0) {
+                // Try the archive
+                localTimestamp = repository.getBlockArchiveRepository().getTimestampFromHeight(atState.getHeight());
+            }
+
+            CrossChainTradeLedgerEntry ledgerEntry
+                = new CrossChainTradeLedgerEntry(
+                    isBuy ? QORT_CURRENCY_CODE : foreignBlockchainCurrencyCode,
+                    isBuy ? foreignBlockchainCurrencyCode : QORT_CURRENCY_CODE,
+                    isBuy ? crossChainTradeData.qortAmount : crossChainTradeData.expectedForeignAmount,
+                    0,
+                    foreignBlockchainCurrencyCode,
+                    isBuy ? crossChainTradeData.expectedForeignAmount : crossChainTradeData.qortAmount,
+                    localTimestamp);
+
+            entries.add(ledgerEntry);
+        }
+    }
 }

src/main/java/org/qortal/api/restricted/resource/AdminResource.java

@@ -32,6 +32,7 @@ import org.qortal.controller.Synchronizer.SynchronizationResult;
 import org.qortal.controller.repository.BlockArchiveRebuilder;
 import org.qortal.data.account.MintingAccountData;
 import org.qortal.data.account.RewardShareData;
+import org.qortal.data.system.DbConnectionInfo;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
 import org.qortal.network.PeerAddress;
@@ -40,6 +41,7 @@ import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
+import org.qortal.data.system.SystemInfo;
 import org.qortal.utils.Base58;
 import org.qortal.utils.NTP;
 
@@ -52,6 +54,7 @@ import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
@@ -459,7 +462,7 @@ public class AdminResource {
 
 			// Qortal: check reward-share's minting account is still allowed to mint
 			Account rewardShareMintingAccount = new Account(repository, rewardShareData.getMinter());
-			if (!rewardShareMintingAccount.canMint())
+			if (!rewardShareMintingAccount.canMint(false))
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.CANNOT_MINT);
 
 			MintingAccountData mintingAccountData = new MintingAccountData(mintingAccount.getPrivateKey(), mintingAccount.getPublicKey());
@@ -1064,4 +1067,50 @@ public class AdminResource {
 		return "true";
 	}
 
-}
+	@GET
+	@Path("/systeminfo")
+	@Operation(
+			summary = "System Information",
+			description = "System memory usage and available processors.",
+			responses = {
+					@ApiResponse(
+							description = "memory usage and available processors",
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = SystemInfo.class))
+					)
+			}
+	)
+	@ApiErrors({ApiError.REPOSITORY_ISSUE})
+	public SystemInfo getSystemInformation() {
+
+		SystemInfo info
+			= new SystemInfo(
+				Runtime.getRuntime().freeMemory(),
+				Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory(),
+				Runtime.getRuntime().totalMemory(),
+				Runtime.getRuntime().maxMemory(),
+				Runtime.getRuntime().availableProcessors());
+
+		return info;
+	}
+
+	@GET
+	@Path("/dbstates")
+	@Operation(
+			summary = "Get DB States",
+			description = "Get DB States",
+			responses = {
+					@ApiResponse(
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = DbConnectionInfo.class)))
+					)
+			}
+	)
+	public List<DbConnectionInfo> getDbConnectionsStates() {
+
+		try {
+			return Controller.REPOSITORY_FACTORY.getDbConnectionsStates();
+		} catch (Exception e) {
+			LOGGER.error(e.getMessage(), e);
+			return new ArrayList<>(0);
+		}
+	}
+}

src/main/java/org/qortal/api/websocket/ActiveChatsWebSocket.java

@@ -77,7 +77,9 @@ public class ActiveChatsWebSocket extends ApiWebSocket {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			ActiveChats activeChats = repository.getChatRepository().getActiveChats(ourAddress, getTargetEncoding(session));
+			Boolean hasChatReference = getHasChatReference(session);
+
+			ActiveChats activeChats = repository.getChatRepository().getActiveChats(ourAddress, getTargetEncoding(session), hasChatReference);
 
 			StringWriter stringWriter = new StringWriter();
 
@@ -103,4 +105,20 @@ public class ActiveChatsWebSocket extends ApiWebSocket {
 		return Encoding.valueOf(encoding);
 	}
 
+	private Boolean getHasChatReference(Session session) {
+		Map<String, List<String>> queryParams = session.getUpgradeRequest().getParameterMap();
+		List<String> hasChatReferenceList = queryParams.get("haschatreference");
+	
+		// Return null if not specified
+		if (hasChatReferenceList != null && hasChatReferenceList.size() == 1) {
+			String value = hasChatReferenceList.get(0).toLowerCase();
+			if (value.equals("true")) {
+				return true;
+			} else if (value.equals("false")) {
+				return false;
+			}
+		}
+		return null; // Ignored if not present
+	}
+
 }

src/main/java/org/qortal/api/websocket/DataMonitorSocket.java

@@ -0,0 +1,102 @@
+package org.qortal.api.websocket;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.eclipse.jetty.websocket.api.Session;
+import org.eclipse.jetty.websocket.api.WebSocketException;
+import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose;
+import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect;
+import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError;
+import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
+import org.eclipse.jetty.websocket.api.annotations.WebSocket;
+import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
+import org.qortal.api.ApiError;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.DataMonitorInfo;
+import org.qortal.event.DataMonitorEvent;
+import org.qortal.event.Event;
+import org.qortal.event.EventBus;
+import org.qortal.event.Listener;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.utils.Base58;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.List;
+
+@WebSocket
+@SuppressWarnings("serial")
+public class DataMonitorSocket extends ApiWebSocket implements Listener {
+
+	private static final Logger LOGGER = LogManager.getLogger(DataMonitorSocket.class);
+
+	@Override
+	public void configure(WebSocketServletFactory factory) {
+		LOGGER.info("configure");
+
+		factory.register(DataMonitorSocket.class);
+
+		EventBus.INSTANCE.addListener(this);
+	}
+
+	@Override
+	public void listen(Event event) {
+		if (!(event instanceof DataMonitorEvent))
+			return;
+
+		DataMonitorEvent dataMonitorEvent = (DataMonitorEvent) event;
+
+		for (Session session : getSessions())
+			sendDataEventSummary(session, buildInfo(dataMonitorEvent));
+	}
+
+	private DataMonitorInfo buildInfo(DataMonitorEvent dataMonitorEvent) {
+
+		return new DataMonitorInfo(
+			dataMonitorEvent.getTimestamp(),
+			dataMonitorEvent.getIdentifier(),
+			dataMonitorEvent.getName(),
+			dataMonitorEvent.getService(),
+			dataMonitorEvent.getDescription(),
+			dataMonitorEvent.getTransactionTimestamp(),
+			dataMonitorEvent.getLatestPutTimestamp()
+		);
+	}
+
+	@OnWebSocketConnect
+	@Override
+	public void onWebSocketConnect(Session session) {
+		super.onWebSocketConnect(session);
+	}
+
+	@OnWebSocketClose
+	@Override
+	public void onWebSocketClose(Session session, int statusCode, String reason) {
+		super.onWebSocketClose(session, statusCode, reason);
+	}
+
+	@OnWebSocketError
+	public void onWebSocketError(Session session, Throwable throwable) {
+		/* We ignore errors for now, but method here to silence log spam */
+	}
+
+	@OnWebSocketMessage
+	public void onWebSocketMessage(Session session, String message) {
+		LOGGER.info("onWebSocketMessage: message = " + message);
+	}
+
+	private void sendDataEventSummary(Session session, DataMonitorInfo dataMonitorInfo) {
+		StringWriter stringWriter = new StringWriter();
+
+		try {
+			marshall(stringWriter, dataMonitorInfo);
+
+			session.getRemote().sendStringByFuture(stringWriter.toString());
+		} catch (IOException | WebSocketException e) {
+			// No output this time
+		}
+	}
+
+}

src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java

@@ -98,7 +98,7 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener {
 						byte[] codeHash = acctInfo.getKey().value;
 						ACCT acct = acctInfo.getValue().get();
 
-						List<ATStateData> atStates = repository.getATRepository().getMatchingFinalATStates(codeHash,
+						List<ATStateData> atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, null, null,
 								isFinished, dataByteOffset, expectedValue, minimumFinalHeight,
 								null, null, null);
 
@@ -259,7 +259,7 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener {
 				ACCT acct = acctInfo.getValue().get();
 
 				Integer dataByteOffset = acct.getModeByteOffset();
-				List<ATStateData> initialAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash,
+				List<ATStateData> initialAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, null, null,
 					isFinished, dataByteOffset, expectedValue, minimumFinalHeight,
 					null, null, null);
 
@@ -298,7 +298,7 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener {
 				byte[] codeHash = acctInfo.getKey().value;
 				ACCT acct = acctInfo.getValue().get();
 
-				List<ATStateData> historicAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash,
+				List<ATStateData> historicAtStates = repository.getATRepository().getMatchingFinalATStates(codeHash, null, null,
 					isFinished, dataByteOffset, expectedValue, minimumFinalHeight,
 					null, null, null);
 

src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java

@@ -439,7 +439,15 @@ public class ArbitraryDataReader {
             // Ensure the complete hash matches the joined chunks
             if (!Arrays.equals(arbitraryDataFile.digest(), transactionData.getData())) {
                 // Delete the invalid file
-                arbitraryDataFile.delete();
+                LOGGER.info("Deleting invalid file: path = " + arbitraryDataFile.getFilePath());
+
+                if( arbitraryDataFile.delete() ) {
+                    LOGGER.info("Deleted invalid file successfully: path = " + arbitraryDataFile.getFilePath());
+                }
+                else {
+                    LOGGER.warn("Could not delete invalid file: path = " + arbitraryDataFile.getFilePath());
+                }
+
                 throw new DataException("Unable to validate complete file hash");
             }
         }

src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java

@@ -168,7 +168,7 @@ public class ArbitraryDataRenderer {
                 byte[] data = Files.readAllBytes(filePath); // TODO: limit file size that can be read into memory
                 HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, includeResourceIdInPrefix, data, qdnContext, service, identifier, theme, usingCustomRouting);
                 htmlParser.addAdditionalHeaderTags();
-                response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:;");
+                response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; font-src 'self' data:; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' wss:;");
                 response.setContentType(context.getMimeType(filename));
                 response.setContentLength(htmlParser.getData().length);
                 response.getOutputStream().write(htmlParser.getData());

src/main/java/org/qortal/block/Block.java

@@ -23,12 +23,11 @@ import org.qortal.data.at.ATStateData;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.block.BlockSummaryData;
 import org.qortal.data.block.BlockTransactionData;
+import org.qortal.data.group.GroupAdminData;
 import org.qortal.data.network.OnlineAccountData;
 import org.qortal.data.transaction.TransactionData;
-import org.qortal.repository.ATRepository;
-import org.qortal.repository.DataException;
-import org.qortal.repository.Repository;
-import org.qortal.repository.TransactionRepository;
+import org.qortal.group.Group;
+import org.qortal.repository.*;
 import org.qortal.settings.Settings;
 import org.qortal.transaction.AtTransaction;
 import org.qortal.transaction.Transaction;
@@ -40,6 +39,7 @@ import org.qortal.transform.block.BlockTransformer;
 import org.qortal.transform.transaction.TransactionTransformer;
 import org.qortal.utils.Amounts;
 import org.qortal.utils.Base58;
+import org.qortal.utils.Groups;
 import org.qortal.utils.NTP;
 
 import java.io.ByteArrayOutputStream;
@@ -144,11 +144,14 @@ public class Block {
 		private final Account mintingAccount;
 		private final AccountData mintingAccountData;
 		private final boolean isMinterFounder;
+		private final boolean isMinterMember;
 
 		private final Account recipientAccount;
 		private final AccountData recipientAccountData;
 
-		ExpandedAccount(Repository repository, RewardShareData rewardShareData) throws DataException {
+		final BlockChain blockChain = BlockChain.getInstance();
+
+		ExpandedAccount(Repository repository, RewardShareData rewardShareData, int blockHeight) throws DataException {
 			this.rewardShareData = rewardShareData;
 			this.sharePercent = this.rewardShareData.getSharePercent();
 
@@ -157,6 +160,12 @@ public class Block {
 			this.isMinterFounder = Account.isFounder(mintingAccountData.getFlags());
 
 			this.isRecipientAlsoMinter = this.rewardShareData.getRecipient().equals(this.mintingAccount.getAddress());
+			this.isMinterMember
+				= Groups.memberExistsInAnyGroup(
+					repository.getGroupRepository(),
+					Groups.getGroupIdsToMint(BlockChain.getInstance(), blockHeight),
+					this.mintingAccount.getAddress()
+			);
 
 			if (this.isRecipientAlsoMinter) {
 				// Self-share: minter is also recipient
@@ -169,6 +178,19 @@ public class Block {
 			}
 		}
 
+		/**
+		 * Get Effective Minting Level
+		 *
+		 * @return the effective minting level, if a data exception is thrown, it catches the exception and returns a zero
+		 */
+		public int getEffectiveMintingLevel() {
+			try {
+				return this.mintingAccount.getEffectiveMintingLevel();
+			} catch (DataException e) {
+				return 0;
+			}
+		}
+
 		public Account getMintingAccount() {
 			return this.mintingAccount;
 		}
@@ -181,19 +203,23 @@ public class Block {
 		 * <p>
 		 * This is a method, not a final variable, because account's level can change between construction and call,
 		 * e.g. during Block.process() where account levels are bumped right before Block.distributeBlockReward().
-		 * 
+		 *
 		 *  @return account-level share "bin" from blockchain config, or null if founder / none found
 		 */
 		public AccountLevelShareBin getShareBin(int blockHeight) {
-			if (this.isMinterFounder)
+			if (this.isMinterFounder && blockHeight < BlockChain.getInstance().getAdminsReplaceFoundersHeight())
 				return null;
 
 			final int accountLevel = this.mintingAccountData.getLevel();
 			if (accountLevel <= 0)
 				return null; // level 0 isn't included in any share bins
 
+			if (blockHeight >= blockChain.getFixBatchRewardHeight()) {
+				if (!this.isMinterMember)
+					return null; // not member of minter group isn't included in any share bins
+			}
+
 			// Select the correct set of share bins based on block height
-			final BlockChain blockChain = BlockChain.getInstance();
 			final AccountLevelShareBin[] shareBinsByLevel = (blockHeight >= blockChain.getSharesByLevelV2Height()) ?
 					blockChain.getShareBinsByAccountLevelV2() : blockChain.getShareBinsByAccountLevelV1();
 
@@ -262,7 +288,7 @@ public class Block {
 	 * Constructs new Block without loading transactions and AT states.
 	 * <p>
 	 * Transactions and AT states are loaded on first call to getTransactions() or getATStates() respectively.
-	 * 
+	 *
 	 * @param repository
 	 * @param blockData
 	 */
@@ -333,7 +359,7 @@ public class Block {
 
 	/**
 	 * Constructs new Block with empty transaction list, using passed minter account.
-	 * 
+	 *
 	 * @param repository
 	 * @param blockData
 	 * @param minter
@@ -351,7 +377,7 @@ public class Block {
 	 * This constructor typically used when minting a new block.
 	 * <p>
 	 * Note that CIYAM ATs will be executed and AT-Transactions prepended to this block, along with AT state data and fees.
-	 * 
+	 *
 	 * @param repository
 	 * @param parentBlockData
 	 * @param minter
@@ -377,7 +403,7 @@ public class Block {
 		byte[] encodedOnlineAccounts = new byte[0];
 		int onlineAccountsCount = 0;
 		byte[] onlineAccountsSignatures = null;
-		
+
 		if (isBatchRewardDistributionBlock(height)) {
 			// Batch reward distribution block - copy online accounts from recent block with highest online accounts count
 
@@ -398,7 +424,9 @@ public class Block {
 			onlineAccounts.removeIf(a -> a.getNonce() == null || a.getNonce() < 0);
 
 			// After feature trigger, remove any online accounts that are level 0
-			if (height >= BlockChain.getInstance().getOnlineAccountMinterLevelValidationHeight()) {
+			// but only if they are before the ignore level feature trigger
+			if (height < BlockChain.getInstance().getIgnoreLevelForRewardShareHeight() &&
+					height >= BlockChain.getInstance().getOnlineAccountMinterLevelValidationHeight()) {
 				onlineAccounts.removeIf(a -> {
 					try {
 						return Account.getRewardShareEffectiveMintingLevel(repository, a.getPublicKey()) == 0;
@@ -409,6 +437,21 @@ public class Block {
 				});
 			}
 
+			// After feature trigger, remove any online accounts that are not minter group member
+			if (height >= BlockChain.getInstance().getGroupMemberCheckHeight()) {
+				onlineAccounts.removeIf(a -> {
+					try {
+						List<Integer> groupIdsToMint = Groups.getGroupIdsToMint(BlockChain.getInstance(), height);
+						String address = Account.getRewardShareMintingAddress(repository, a.getPublicKey());
+						boolean isMinterGroupMember = Groups.memberExistsInAnyGroup(repository.getGroupRepository(), groupIdsToMint, address);
+						return !isMinterGroupMember;
+					} catch (DataException e) {
+						// Something went wrong, so remove the account
+						return true;
+					}
+				});
+			}
+
 			if (onlineAccounts.isEmpty()) {
 				LOGGER.debug("No online accounts - not even our own?");
 				return null;
@@ -512,7 +555,7 @@ public class Block {
 	 * Mints new block using this block as template, but with different minting account.
 	 * <p>
 	 * NOTE: uses the same transactions list, AT states, etc.
-	 * 
+	 *
 	 * @param minter
 	 * @return
 	 * @throws DataException
@@ -598,7 +641,7 @@ public class Block {
 
 	/**
 	 * Return composite block signature (minterSignature + transactionsSignature).
-	 * 
+	 *
 	 * @return byte[], or null if either component signature is null.
 	 */
 	public byte[] getSignature() {
@@ -613,7 +656,7 @@ public class Block {
 	 * <p>
 	 * We're starting with version 4 as a nod to being newer than successor Qora,
 	 * whose latest block version was 3.
-	 * 
+	 *
 	 * @return 1, 2, 3 or 4
 	 */
 	public int getNextBlockVersion() {
@@ -627,7 +670,7 @@ public class Block {
 	 * Return block's transactions.
 	 * <p>
 	 * If the block was loaded from repository then it's possible this method will call the repository to fetch the transactions if not done already.
-	 * 
+	 *
 	 * @return
 	 * @throws DataException
 	 */
@@ -661,7 +704,7 @@ public class Block {
 	 * If the block was loaded from repository then it's possible this method will call the repository to fetch the AT states if not done already.
 	 * <p>
 	 * <b>Note:</b> AT states fetched from repository only contain summary info, not actual data like serialized state data or AT creation timestamps!
-	 * 
+	 *
 	 * @return
 	 * @throws DataException
 	 */
@@ -697,7 +740,7 @@ public class Block {
 	 * <p>
 	 * Typically called as part of Block.process() or Block.orphan()
 	 * so ideally after any calls to Block.isValid().
-	 * 
+	 *
 	 * @throws DataException
 	 */
 	public List<ExpandedAccount> getExpandedAccounts() throws DataException {
@@ -715,10 +758,12 @@ public class Block {
 
 		List<ExpandedAccount> expandedAccounts = new ArrayList<>();
 
-		for (RewardShareData rewardShare : this.cachedOnlineRewardShares)
-			expandedAccounts.add(new ExpandedAccount(repository, rewardShare));
+		for (RewardShareData rewardShare : this.cachedOnlineRewardShares) {
+			expandedAccounts.add(new ExpandedAccount(repository, rewardShare, this.blockData.getHeight()));
+		}
 
 		this.cachedExpandedAccounts = expandedAccounts;
+		LOGGER.trace(() -> String.format("Online reward-shares after expanded accounts %s", this.cachedOnlineRewardShares));
 
 		return this.cachedExpandedAccounts;
 	}
@@ -727,7 +772,7 @@ public class Block {
 
 	/**
 	 * Load parent block's data from repository via this block's reference.
-	 * 
+	 *
 	 * @return parent's BlockData, or null if no parent found
 	 * @throws DataException
 	 */
@@ -741,7 +786,7 @@ public class Block {
 
 	/**
 	 * Load child block's data from repository via this block's signature.
-	 * 
+	 *
 	 * @return child's BlockData, or null if no parent found
 	 * @throws DataException
 	 */
@@ -761,7 +806,7 @@ public class Block {
 	 * Used when constructing a new block during minting.
 	 * <p>
 	 * Requires block's {@code minter} being a {@code PrivateKeyAccount} so block's transactions signature can be recalculated.
-	 * 
+	 *
 	 * @param transactionData
 	 * @return true if transaction successfully added to block, false otherwise
 	 * @throws IllegalStateException
@@ -814,7 +859,7 @@ public class Block {
 	 * Used when constructing a new block during minting.
 	 * <p>
 	 * Requires block's {@code minter} being a {@code PrivateKeyAccount} so block's transactions signature can be recalculated.
-	 * 
+	 *
 	 * @param transactionData
 	 * @throws IllegalStateException
 	 *             if block's {@code minter} is not a {@code PrivateKeyAccount}.
@@ -859,7 +904,7 @@ public class Block {
 	 * previous block's minter signature + minter's public key + (encoded) online-accounts data
 	 * <p>
 	 * (Previous block's minter signature is extracted from this block's reference).
-	 * 
+	 *
 	 * @throws IllegalStateException
 	 *             if block's {@code minter} is not a {@code PrivateKeyAccount}.
 	 * @throws RuntimeException
@@ -876,7 +921,7 @@ public class Block {
 	 * Recalculate block's transactions signature.
 	 * <p>
 	 * Requires block's {@code minter} being a {@code PrivateKeyAccount}.
-	 * 
+	 *
 	 * @throws IllegalStateException
 	 *             if block's {@code minter} is not a {@code PrivateKeyAccount}.
 	 * @throws RuntimeException
@@ -998,7 +1043,7 @@ public class Block {
 	 * Recalculate block's minter and transactions signatures, thus giving block full signature.
 	 * <p>
 	 * Note: Block instance must have been constructed with a <tt>PrivateKeyAccount</tt> minter or this call will throw an <tt>IllegalStateException</tt>.
-	 * 
+	 *
 	 * @throws IllegalStateException
 	 *             if block's {@code minter} is not a {@code PrivateKeyAccount}.
 	 */
@@ -1011,7 +1056,7 @@ public class Block {
 
 	/**
 	 * Returns whether this block's signatures are valid.
-	 * 
+	 *
 	 * @return true if both minter and transaction signatures are valid, false otherwise
 	 */
 	public boolean isSignatureValid() {
@@ -1035,7 +1080,7 @@ public class Block {
 	 * <p>
 	 * Used by BlockMinter to check whether it's time to mint a new block,
 	 * and also used by Block.isValid for checks (if not a testchain).
-	 * 
+	 *
 	 * @return ValidationResult.OK if timestamp valid, or some other ValidationResult otherwise.
 	 * @throws DataException
 	 */
@@ -1124,14 +1169,32 @@ public class Block {
 		if (onlineRewardShares == null)
 			return ValidationResult.ONLINE_ACCOUNT_UNKNOWN;
 
-		// After feature trigger, require all online account minters to be greater than level 0
-		if (this.getBlockData().getHeight() >= BlockChain.getInstance().getOnlineAccountMinterLevelValidationHeight()) {
-			List<ExpandedAccount> expandedAccounts = this.getExpandedAccounts();
+		// After feature trigger, require all online account minters to be greater than level 0,
+		// but only if it is before the feature trigger where we ignore level again
+		if (this.blockData.getHeight() < BlockChain.getInstance().getIgnoreLevelForRewardShareHeight() &&
+			this.getBlockData().getHeight() >= BlockChain.getInstance().getOnlineAccountMinterLevelValidationHeight()) {
+			List<ExpandedAccount> expandedAccounts
+					= this.getExpandedAccounts().stream()
+					.filter(expandedAccount -> expandedAccount.isMinterMember)
+					.collect(Collectors.toList());
+
 			for (ExpandedAccount account : expandedAccounts) {
 				if (account.getMintingAccount().getEffectiveMintingLevel() == 0)
 					return ValidationResult.ONLINE_ACCOUNTS_INVALID;
+
+				if (this.getBlockData().getHeight() >= BlockChain.getInstance().getFixBatchRewardHeight()) {
+					if (!account.isMinterMember)
+						return ValidationResult.ONLINE_ACCOUNTS_INVALID;
+				}
 			}
 		}
+		else if (this.blockData.getHeight() >= BlockChain.getInstance().getIgnoreLevelForRewardShareHeight()){
+			Optional<ExpandedAccount> anyInvalidAccount
+					= this.getExpandedAccounts().stream()
+					.filter(account -> !account.isMinterMember)
+					.findAny();
+			if( anyInvalidAccount.isPresent() ) return ValidationResult.ONLINE_ACCOUNTS_INVALID;
+		}
 
 		// If block is past a certain age then we simply assume the signatures were correct
 		long signatureRequirementThreshold = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMinLifetime();
@@ -1215,7 +1278,7 @@ public class Block {
 	 * <p>
 	 * Checks block's transactions by testing their validity then processing them.<br>
 	 * Hence uses a repository savepoint during execution.
-	 * 
+	 *
 	 * @return ValidationResult.OK if block is valid, or some other ValidationResult otherwise.
 	 * @throws DataException
 	 */
@@ -1258,6 +1321,7 @@ public class Block {
 
 		// Online Accounts
 		ValidationResult onlineAccountsResult = this.areOnlineAccountsValid();
+		LOGGER.trace("Accounts valid = {}", onlineAccountsResult);
 		if (onlineAccountsResult != ValidationResult.OK)
 			return onlineAccountsResult;
 
@@ -1346,7 +1410,7 @@ public class Block {
 				// Check transaction can even be processed
 				validationResult = transaction.isProcessable();
 				if (validationResult != Transaction.ValidationResult.OK) {
-					LOGGER.info(String.format("Error during transaction validation, tx %s: %s", Base58.encode(transactionData.getSignature()), validationResult.name()));
+					LOGGER.debug(String.format("Error during transaction validation, tx %s: %s", Base58.encode(transactionData.getSignature()), validationResult.name()));
 					return ValidationResult.TRANSACTION_INVALID;
 				}
 
@@ -1386,7 +1450,7 @@ public class Block {
 	 * <p>
 	 * NOTE: will execute ATs locally if not already done.<br>
 	 * This is so we have locally-generated AT states for comparison.
-	 * 
+	 *
 	 * @return OK, or some AT-related validation result
 	 * @throws DataException
 	 */
@@ -1462,11 +1526,11 @@ public class Block {
 	 * Note: this method does not store new AT state data into repository - that is handled by <tt>process()</tt>.
 	 * <p>
 	 * This method is not needed if fetching an existing block from the repository as AT state data will be loaded from repository as well.
-	 * 
+	 *
 	 * @see #isValid()
-	 * 
+	 *
 	 * @throws DataException
-	 * 
+	 *
 	 */
 	private void executeATs() throws DataException {
 		// We're expecting a lack of AT state data at this point.
@@ -1518,7 +1582,7 @@ public class Block {
 			return false;
 
 		Account mintingAccount = new PublicKeyAccount(this.repository, rewardShareData.getMinterPublicKey());
-		return mintingAccount.canMint();
+		return mintingAccount.canMint(false);
 	}
 
 	/**
@@ -1538,7 +1602,7 @@ public class Block {
 
 	/**
 	 * Process block, and its transactions, adding them to the blockchain.
-	 * 
+	 *
 	 * @throws DataException
 	 */
 	public void process() throws DataException {
@@ -1547,6 +1611,7 @@ public class Block {
 		this.blockData.setHeight(blockchainHeight + 1);
 
 		LOGGER.trace(() -> String.format("Processing block %d", this.blockData.getHeight()));
+		LOGGER.trace(() -> String.format("Online Reward Shares in process %s", this.cachedOnlineRewardShares));
 
 		if (this.blockData.getHeight() > 1) {
 
@@ -1618,7 +1683,17 @@ public class Block {
 		final List<Integer> cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel();
 		final int maximumLevel = cumulativeBlocksByLevel.size() - 1;
 
-		final List<ExpandedAccount> expandedAccounts = this.getExpandedAccounts();
+		final List<ExpandedAccount> expandedAccounts;
+
+		if (this.getBlockData().getHeight() < BlockChain.getInstance().getFixBatchRewardHeight()) {
+			expandedAccounts = this.getExpandedAccounts().stream().collect(Collectors.toList());
+		}
+		else {
+			expandedAccounts
+				= this.getExpandedAccounts().stream()
+					.filter(expandedAccount -> expandedAccount.isMinterMember)
+					.collect(Collectors.toList());
+		}
 
 		Set<AccountData> allUniqueExpandedAccounts = new HashSet<>();
 		for (ExpandedAccount expandedAccount : expandedAccounts) {
@@ -1839,7 +1914,7 @@ public class Block {
 
 	/**
 	 * Removes block from blockchain undoing transactions and adding them to unconfirmed pile.
-	 * 
+	 *
 	 * @throws DataException
 	 */
 	public void orphan() throws DataException {
@@ -1879,7 +1954,7 @@ public class Block {
 					SelfSponsorshipAlgoV3Block.orphanAccountPenalties(this);
 				}
 			}
-			
+
 			// Account levels and block rewards are only processed/orphaned on block reward distribution blocks
 			if (this.isRewardDistributionBlock()) {
 				// Block rewards, including transaction fees, removed after transactions undone
@@ -2018,7 +2093,17 @@ public class Block {
 		final List<Integer> cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel();
 		final int maximumLevel = cumulativeBlocksByLevel.size() - 1;
 
-		final List<ExpandedAccount> expandedAccounts = this.getExpandedAccounts();
+		final List<ExpandedAccount> expandedAccounts;
+
+		if (this.getBlockData().getHeight() < BlockChain.getInstance().getFixBatchRewardHeight()) {
+			expandedAccounts = this.getExpandedAccounts().stream().collect(Collectors.toList());
+		}
+		else {
+			expandedAccounts
+				= this.getExpandedAccounts().stream()
+					.filter(expandedAccount -> expandedAccount.isMinterMember)
+					.collect(Collectors.toList());
+		}
 
 		Set<AccountData> allUniqueExpandedAccounts = new HashSet<>();
 		for (ExpandedAccount expandedAccount : expandedAccounts) {
@@ -2213,6 +2298,7 @@ public class Block {
 		List<AccountBalanceData> accountBalanceDeltas = balanceChanges.entrySet().stream()
 				.map(entry -> new AccountBalanceData(entry.getKey(), Asset.QORT, entry.getValue()))
 				.collect(Collectors.toList());
+		LOGGER.trace("Account Balance Deltas: {}", accountBalanceDeltas);
 		this.repository.getAccountRepository().modifyAssetBalances(accountBalanceDeltas);
 	}
 
@@ -2221,34 +2307,44 @@ public class Block {
 		List<BlockRewardCandidate> rewardCandidates = new ArrayList<>();
 
 		// All online accounts
-		final List<ExpandedAccount> expandedAccounts = this.getExpandedAccounts();
+		final List<ExpandedAccount> expandedAccounts;
+
+		if (this.getBlockData().getHeight() < BlockChain.getInstance().getFixBatchRewardHeight()) {
+			expandedAccounts = this.getExpandedAccounts().stream().collect(Collectors.toList());
+		}
+		else {
+			expandedAccounts
+				= this.getExpandedAccounts().stream()
+					.filter(expandedAccount -> expandedAccount.isMinterMember)
+					.collect(Collectors.toList());
+		}
 
 		/*
 		 * Distribution rules:
-		 * 
+		 *
 		 * Distribution is based on the minting account of 'online' reward-shares.
-		 * 
+		 *
 		 * If ANY founders are online, then they receive the leftover non-distributed reward.
 		 * If NO founders are online, then account-level-based rewards are scaled up so 100% of reward is allocated.
-		 * 
+		 *
 		 * If ANY non-maxxed legacy QORA holders exist then they are always allocated their fixed share (e.g. 20%).
-		 * 
+		 *
 		 * There has to be either at least one 'online' account for blocks to be minted
 		 * so there is always either one account-level-based or founder reward candidate.
-		 * 
+		 *
 		 * Examples:
-		 * 
+		 *
 		 * With at least one founder online:
 		 * Level 1/2 accounts: 5%
 		 * Legacy QORA holders: 20%
 		 * Founders: ~75%
-		 * 
+		 *
 		 * No online founders:
 		 * Level 1/2 accounts: 5%
 		 * Level 5/6 accounts: 15%
 		 * Legacy QORA holders: 20%
 		 * Total: 40%
-		 * 
+		 *
 		 * After scaling account-level-based shares to fill 100%:
 		 * Level 1/2 accounts: 20%
 		 * Level 5/6 accounts: 60%
@@ -2264,7 +2360,6 @@ public class Block {
 		// Select the correct set of share bins based on block height
 		List<AccountLevelShareBin> accountLevelShareBinsForBlock = (this.blockData.getHeight() >= BlockChain.getInstance().getSharesByLevelV2Height()) ?
 				BlockChain.getInstance().getAccountLevelShareBinsV2() : BlockChain.getInstance().getAccountLevelShareBinsV1();
-
 		// Determine reward candidates based on account level
 		// This needs a deep copy, so the shares can be modified when tiers aren't activated yet
 		List<AccountLevelShareBin> accountLevelShareBins = new ArrayList<>();
@@ -2347,7 +2442,7 @@ public class Block {
 		final long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShareAtHeight(this.blockData.getHeight());
 
 		// Perform account-level-based reward scaling if appropriate
-		if (!haveFounders) {
+		if (!haveFounders && this.blockData.getHeight() < BlockChain.getInstance().getAdminsReplaceFoundersHeight() ) {
 			// Recalculate distribution ratios based on candidates
 
 			// Nothing shared? This shouldn't happen
@@ -2383,18 +2478,103 @@ public class Block {
 		}
 
 		// Add founders as reward candidate if appropriate
-		if (haveFounders) {
+		if (haveFounders && this.blockData.getHeight() < BlockChain.getInstance().getAdminsReplaceFoundersHeight()) {
 			// Yes: add to reward candidates list
 			BlockRewardDistributor founderDistributor = (distributionAmount, balanceChanges) -> distributeBlockRewardShare(distributionAmount, onlineFounderAccounts, balanceChanges);
 
 			final long foundersShare = 1_00000000 - totalShares;
 			BlockRewardCandidate rewardCandidate = new BlockRewardCandidate("Founders", foundersShare, founderDistributor);
 			rewardCandidates.add(rewardCandidate);
+			LOGGER.info("logging foundersShare prior to reward modifications {}",foundersShare);
+		}
+		else if (this.blockData.getHeight() >= BlockChain.getInstance().getAdminsReplaceFoundersHeight()) {
+			try (final Repository repository = RepositoryManager.getRepository()) {
+				GroupRepository groupRepository = repository.getGroupRepository();
+
+				List<Integer> mintingGroupIds = Groups.getGroupIdsToMint(BlockChain.getInstance(), this.blockData.getHeight());
+
+				// all minter admins
+				List<String> minterAdmins = Groups.getAllAdmins(groupRepository, mintingGroupIds);
+
+				// all minter admins that are online
+				List<ExpandedAccount> onlineMinterAdminAccounts
+					= expandedAccounts.stream()
+						.filter(expandedAccount ->  minterAdmins.contains(expandedAccount.getMintingAccount().getAddress()))
+						.collect(Collectors.toList());
+
+				long minterAdminShare;
+
+				if( onlineMinterAdminAccounts.isEmpty() ) {
+					minterAdminShare = 0;
+				}
+				else {
+					BlockRewardDistributor minterAdminDistributor
+							= (distributionAmount, balanceChanges)
+							->
+							distributeBlockRewardShare(distributionAmount, onlineMinterAdminAccounts, balanceChanges);
+
+					long adminShare = 1_00000000 - totalShares;
+					LOGGER.info("initial total Shares: {}", totalShares);
+					LOGGER.info("logging adminShare after hardfork, this is the primary reward that will be split {}", adminShare);
+
+					minterAdminShare = adminShare / 2;
+					BlockRewardCandidate minterAdminRewardCandidate
+							= new BlockRewardCandidate("Minter Admins", minterAdminShare, minterAdminDistributor);
+					rewardCandidates.add(minterAdminRewardCandidate);
+
+					totalShares += minterAdminShare;
+				}
+
+				LOGGER.info("MINTER ADMIN SHARE: {}",minterAdminShare);
+
+				// all dev admins
+				List<String> devAdminAddresses
+						= groupRepository.getGroupAdmins(1).stream()
+						.map(GroupAdminData::getAdmin)
+						.collect(Collectors.toList());
+
+				LOGGER.info("Removing NULL Account Address, Dev Admin Count = {}", devAdminAddresses.size());
+				devAdminAddresses.removeIf( address -> Group.NULL_OWNER_ADDRESS.equals(address) );
+				LOGGER.info("Removed NULL Account Address, Dev Admin Count = {}", devAdminAddresses.size());
+
+				BlockRewardDistributor devAdminDistributor
+					= (distributionAmount, balanceChanges) -> distributeToAccounts(distributionAmount, devAdminAddresses, balanceChanges);
+
+				long devAdminShare = 1_00000000 - totalShares;
+				LOGGER.info("DEV ADMIN SHARE: {}",devAdminShare);
+				BlockRewardCandidate devAdminRewardCandidate
+					= new BlockRewardCandidate("Dev Admins", devAdminShare,devAdminDistributor);
+				rewardCandidates.add(devAdminRewardCandidate);
+			}
 		}
 
 		return rewardCandidates;
 	}
 
+	/**
+	 * Distribute To Accounts
+	 *
+	 * Merges distribute shares to a map of distribution shares.
+	 *
+	 * @param distributionAmount the amount to distribute
+	 * @param accountAddressess the addresses to distribute to
+	 * @param balanceChanges the map of distribution shares, this gets appended to
+	 *
+	 * @return the total amount mapped to addresses for distribution
+	 */
+	public static long distributeToAccounts(long distributionAmount, List<String> accountAddressess, Map<String, Long> balanceChanges) {
+
+		if( accountAddressess.isEmpty() ) return 0;
+
+		long distibutionShare = distributionAmount / accountAddressess.size();
+
+		for(String accountAddress : accountAddressess ) {
+			balanceChanges.merge(accountAddress, distibutionShare, Long::sum);
+		}
+
+		return distibutionShare * accountAddressess.size();
+	}
+
 	private static long distributeBlockRewardShare(long distributionAmount, List<ExpandedAccount> accounts, Map<String, Long> balanceChanges) {
 		// Collate all expanded accounts by minting account
 		Map<String, List<ExpandedAccount>> accountsByMinter = new HashMap<>();
@@ -2554,9 +2734,11 @@ public class Block {
 				return;
 
 			int minterLevel = Account.getRewardShareEffectiveMintingLevel(this.repository, this.getMinter().getPublicKey());
+			String minterAddress = Account.getRewardShareMintingAddress(this.repository, this.getMinter().getPublicKey());
 
 			LOGGER.debug(String.format("======= BLOCK %d (%.8s) =======", this.getBlockData().getHeight(), Base58.encode(this.getSignature())));
 			LOGGER.debug(String.format("Timestamp: %d", this.getBlockData().getTimestamp()));
+			LOGGER.debug(String.format("Minter address: %s", minterAddress));
 			LOGGER.debug(String.format("Minter level: %d", minterLevel));
 			LOGGER.debug(String.format("Online accounts: %d", this.getBlockData().getOnlineAccountsCount()));
 			LOGGER.debug(String.format("AT count: %d", this.getBlockData().getATCount()));

src/main/java/org/qortal/block/BlockChain.java

@@ -71,6 +71,7 @@ public class BlockChain {
 		transactionV6Timestamp,
 		disableReferenceTimestamp,
 		increaseOnlineAccountsDifficultyTimestamp,
+		decreaseOnlineAccountsDifficultyTimestamp,
 		onlineAccountMinterLevelValidationHeight,
 		selfSponsorshipAlgoV1Height,
 		selfSponsorshipAlgoV2Height,
@@ -85,7 +86,13 @@ public class BlockChain {
 		disableRewardshareHeight,
 		enableRewardshareHeight,
 		onlyMintWithNameHeight,
-		groupMemberCheckHeight
+		removeOnlyMintWithNameHeight,
+		groupMemberCheckHeight,
+		fixBatchRewardHeight,
+		adminsReplaceFoundersHeight,
+		nullGroupMembershipHeight,
+		ignoreLevelForRewardShareHeight,
+		adminQueryFixHeight
 	}
 
 	// Custom transaction fees
@@ -205,7 +212,13 @@ public class BlockChain {
 	private int minAccountLevelToRewardShare;
 	private int maxRewardSharesPerFounderMintingAccount;
 	private int founderEffectiveMintingLevel;
-	private int mintingGroupId;
+
+	public static class IdsForHeight {
+		public int height;
+		public List<Integer> ids;
+	}
+
+	private List<IdsForHeight> mintingGroupIds;
 
 	/** Minimum time to retain online account signatures (ms) for block validity checks. */
 	private long onlineAccountSignaturesMinLifetime;
@@ -217,6 +230,10 @@ public class BlockChain {
 	 * featureTriggers because unit tests need to set this value via Reflection. */
 	private long onlineAccountsModulusV2Timestamp;
 
+	/** Feature trigger timestamp for ONLINE_ACCOUNTS_MODULUS time interval decrease. Can't use
+	 * featureTriggers because unit tests need to set this value via Reflection. */
+	private long onlineAccountsModulusV3Timestamp;
+
 	/** Snapshot timestamp for self sponsorship algo V1 */
 	private long selfSponsorshipAlgoV1SnapshotTimestamp;
 
@@ -403,6 +420,10 @@ public class BlockChain {
 		return this.onlineAccountsModulusV2Timestamp;
 	}
 
+	public long getOnlineAccountsModulusV3Timestamp() {
+		return this.onlineAccountsModulusV3Timestamp;
+	}
+
 	/* Block reward batching */
 	public long getBlockRewardBatchStartHeight() {
 		return this.blockRewardBatchStartHeight;
@@ -529,8 +550,8 @@ public class BlockChain {
 		return this.onlineAccountSignaturesMaxLifetime;
 	}
 
-	public int getMintingGroupId() {
-		return this.mintingGroupId;
+	public List<IdsForHeight> getMintingGroupIds() {
+		return mintingGroupIds;
 	}
 
 	public CiyamAtSettings getCiyamAtSettings() {
@@ -579,6 +600,10 @@ public class BlockChain {
 		return this.featureTriggers.get(FeatureTrigger.increaseOnlineAccountsDifficultyTimestamp.name()).longValue();
 	}
 
+	public long getDecreaseOnlineAccountsDifficultyTimestamp() {
+		return this.featureTriggers.get(FeatureTrigger.decreaseOnlineAccountsDifficultyTimestamp.name()).longValue();
+	}
+
 	public int getSelfSponsorshipAlgoV1Height() {
 		return this.featureTriggers.get(FeatureTrigger.selfSponsorshipAlgoV1Height.name()).intValue();
 	}
@@ -635,10 +660,34 @@ public class BlockChain {
 		return this.featureTriggers.get(FeatureTrigger.onlyMintWithNameHeight.name()).intValue();
 	}
 
+	public int getRemoveOnlyMintWithNameHeight() {
+		return this.featureTriggers.get(FeatureTrigger.removeOnlyMintWithNameHeight.name()).intValue();
+	}
+
 	public int getGroupMemberCheckHeight() {
 		return this.featureTriggers.get(FeatureTrigger.groupMemberCheckHeight.name()).intValue();
 	}
 
+	public int getFixBatchRewardHeight() {
+		return this.featureTriggers.get(FeatureTrigger.fixBatchRewardHeight.name()).intValue();
+	}
+
+	public int getAdminsReplaceFoundersHeight() {
+		return this.featureTriggers.get(FeatureTrigger.adminsReplaceFoundersHeight.name()).intValue();
+	}
+
+	public int getNullGroupMembershipHeight() {
+		return this.featureTriggers.get(FeatureTrigger.nullGroupMembershipHeight.name()).intValue();
+	}
+
+	public int getIgnoreLevelForRewardShareHeight() {
+		return this.featureTriggers.get(FeatureTrigger.ignoreLevelForRewardShareHeight.name()).intValue();
+	}
+
+	public int getAdminQueryFixHeight() {
+		return this.featureTriggers.get(FeatureTrigger.adminQueryFixHeight.name()).intValue();
+	}
+
 	// More complex getters for aspects that change by height or timestamp
 
 	public long getRewardAtHeight(int ourHeight) {

src/main/java/org/qortal/controller/BlockMinter.java

@@ -97,364 +97,375 @@ public class BlockMinter extends Thread {
 
 		final boolean isSingleNodeTestnet = Settings.getInstance().isSingleNodeTestnet();
 
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			// Going to need this a lot...
-			BlockRepository blockRepository = repository.getBlockRepository();
-
-			// Flags for tracking change in whether minting is possible,
-			// so we can notify Controller, and further update SysTray, etc.
-			boolean isMintingPossible = false;
-			boolean wasMintingPossible = isMintingPossible;
+		// Flags for tracking change in whether minting is possible,
+		// so we can notify Controller, and further update SysTray, etc.
+		boolean isMintingPossible = false;
+		boolean wasMintingPossible = isMintingPossible;
+		try {
 			while (running) {
-				if (isMintingPossible != wasMintingPossible)
-					Controller.getInstance().onMintingPossibleChange(isMintingPossible);
+				// recreate repository for new loop iteration
+				try (final Repository repository = RepositoryManager.getRepository()) {
 
-				wasMintingPossible = isMintingPossible;
+					// Going to need this a lot...
+					BlockRepository blockRepository = repository.getBlockRepository();
 
-				try {
-					// Free up any repository locks
-					repository.discardChanges();
+					if (isMintingPossible != wasMintingPossible)
+						Controller.getInstance().onMintingPossibleChange(isMintingPossible);
 
-					// Sleep for a while.
-					// It's faster on single node testnets, to allow lots of blocks to be minted quickly.
-					Thread.sleep(isSingleNodeTestnet ? 50 : 1000);
-
-					isMintingPossible = false;
-
-					final Long now = NTP.getTime();
-					if (now == null)
-						continue;
-
-					final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
-					if (minLatestBlockTimestamp == null)
-						continue;
-
-					List<MintingAccountData> mintingAccountsData = repository.getAccountRepository().getMintingAccounts();
-					// No minting accounts?
-					if (mintingAccountsData.isEmpty())
-						continue;
-
-					// Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level
-					// Note that minting accounts are actually reward-shares in Qortal
-					Iterator<MintingAccountData> madi = mintingAccountsData.iterator();
-					while (madi.hasNext()) {
-						MintingAccountData mintingAccountData = madi.next();
-
-						RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
-						if (rewardShareData == null) {
-							// Reward-share doesn't exist - probably cancelled but not yet removed from node's list of minting accounts
-							madi.remove();
-							continue;
-						}
-
-						Account mintingAccount = new Account(repository, rewardShareData.getMinter());
-						if (!mintingAccount.canMint()) {
-							// Minting-account component of reward-share can no longer mint - disregard
-							madi.remove();
-							continue;
-						}
-
-						// Optional (non-validated) prevention of block submissions below a defined level.
-						// This is an unvalidated version of Blockchain.minAccountLevelToMint
-						// and exists only to reduce block candidates by default.
-						int level = mintingAccount.getEffectiveMintingLevel();
-						if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
-							madi.remove();
-                        }
-					}
-
-					// Needs a mutable copy of the unmodifiableList
-					List<Peer> peers = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers());
-					BlockData lastBlockData = blockRepository.getLastBlock();
-
-					// Disregard peers that have "misbehaved" recently
-					peers.removeIf(Controller.hasMisbehaved);
-
-					// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
-					// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
-					if (!Synchronizer.getInstance().getRecoveryMode())
-						peers.removeIf(Controller.hasNoRecentBlock);
-
-					// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
-					if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
-						continue;
-
-					// If we are stuck on an invalid block, we should allow an alternative to be minted
-					boolean recoverInvalidBlock = false;
-					if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) {
-						// We've had at least one invalid block
-						long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived;
-						long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived;
-						if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) {
-							if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) {
-								// Last valid block was more than 10 mins ago, but we've had an invalid block since then
-								// Assume that the chain has stalled because there is no alternative valid candidate
-								// Enter recovery mode to allow alternative, valid candidates to be minted
-								recoverInvalidBlock = true;
-							}
-						}
-					}
-
-					// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
-					if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
-						if (!Synchronizer.getInstance().getRecoveryMode() && !recoverInvalidBlock)
-							continue;
-
-					// There are enough peers with a recent block and our latest block is recent
-					// so go ahead and mint a block if possible.
-					isMintingPossible = true;
-
-					// Check blockchain hasn't changed
-					if (previousBlockData == null || !Arrays.equals(previousBlockData.getSignature(), lastBlockData.getSignature())) {
-						previousBlockData = lastBlockData;
-						newBlocks.clear();
-
-						// Reduce log timeout
-						logTimeout = 10 * 1000L;
-
-						// Last low weight block is no longer valid
-						parentSignatureForLastLowWeightBlock = null;
-					}
-
-					// Discard accounts we have already built blocks with
-					mintingAccountsData.removeIf(mintingAccountData -> newBlocks.stream().anyMatch(newBlock -> Arrays.equals(newBlock.getBlockData().getMinterPublicKey(), mintingAccountData.getPublicKey())));
-
-					// Do we need to build any potential new blocks?
-					List<PrivateKeyAccount> newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
-
-					// We might need to sit the next block out, if one of our minting accounts signed the previous one
-					// Skip this check for single node testnets, since they definitely need to mint every block
-					byte[] previousBlockMinter = previousBlockData.getMinterPublicKey();
-					boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter));
-					if (mintedLastBlock && !isSingleNodeTestnet) {
-						LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one"));
-						continue;
-					}
-
-					if (parentSignatureForLastLowWeightBlock != null) {
-						// The last iteration found a higher weight block in the network, so sleep for a while
-						// to allow is to sync the higher weight chain. We are sleeping here rather than when
-						// detected as we don't want to hold the blockchain lock open.
-						LOGGER.info("Sleeping for 10 seconds...");
-						Thread.sleep(10 * 1000L);
-					}
-
-					for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
-						// First block does the AT heavy-lifting
-						if (newBlocks.isEmpty()) {
-							Block newBlock = Block.mint(repository, previousBlockData, mintingAccount);
-							if (newBlock == null) {
-								// For some reason we can't mint right now
-								moderatedLog(() -> LOGGER.info("Couldn't build a to-be-minted block"));
-								continue;
-							}
-
-							newBlocks.add(newBlock);
-						} else {
-							// The blocks for other minters require less effort...
-							Block newBlock = newBlocks.get(0).remint(mintingAccount);
-							if (newBlock == null) {
-								// For some reason we can't mint right now
-								moderatedLog(() -> LOGGER.error("Couldn't rebuild a to-be-minted block"));
-								continue;
-							}
-
-							newBlocks.add(newBlock);
-						}
-					}
-
-					// No potential block candidates?
-					if (newBlocks.isEmpty())
-						continue;
-
-					// Make sure we're the only thread modifying the blockchain
-					ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
-					if (!blockchainLock.tryLock(30, TimeUnit.SECONDS)) {
-						LOGGER.debug("Couldn't acquire blockchain lock even after waiting 30 seconds");
-						continue;
-					}
-
-					boolean newBlockMinted = false;
-					Block newBlock = null;
+					wasMintingPossible = isMintingPossible;
 
 					try {
-						// Clear repository session state so we have latest view of data
+						// reset the repository, to the repository recreated for this loop iteration
+						for( Block newBlock : newBlocks ) newBlock.setRepository(repository);
+
+						// Free up any repository locks
 						repository.discardChanges();
 
-						// Now that we have blockchain lock, do final check that chain hasn't changed
-						BlockData latestBlockData = blockRepository.getLastBlock();
-						if (!Arrays.equals(lastBlockData.getSignature(), latestBlockData.getSignature()))
+						// Sleep for a while.
+						// It's faster on single node testnets, to allow lots of blocks to be minted quickly.
+						Thread.sleep(isSingleNodeTestnet ? 50 : 1000);
+
+						isMintingPossible = false;
+
+						final Long now = NTP.getTime();
+						if (now == null)
 							continue;
 
-						List<Block> goodBlocks = new ArrayList<>();
-						boolean wasInvalidBlockDiscarded = false;
-						Iterator<Block> newBlocksIterator = newBlocks.iterator();
+						final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
+						if (minLatestBlockTimestamp == null)
+							continue;
 
-						while (newBlocksIterator.hasNext()) {
-							Block testBlock = newBlocksIterator.next();
+						List<MintingAccountData> mintingAccountsData = repository.getAccountRepository().getMintingAccounts();
+						// No minting accounts?
+						if (mintingAccountsData.isEmpty())
+							continue;
 
-							// Is new block's timestamp valid yet?
-							// We do a separate check as some timestamp checks are skipped for testchains
-							if (testBlock.isTimestampValid() != ValidationResult.OK)
+						// Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level
+						// Note that minting accounts are actually reward-shares in Qortal
+						Iterator<MintingAccountData> madi = mintingAccountsData.iterator();
+						while (madi.hasNext()) {
+							MintingAccountData mintingAccountData = madi.next();
+
+							RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
+							if (rewardShareData == null) {
+								// Reward-share doesn't exist - probably cancelled but not yet removed from node's list of minting accounts
+								madi.remove();
+								continue;
+							}
+
+							Account mintingAccount = new Account(repository, rewardShareData.getMinter());
+							if (!mintingAccount.canMint(true)) {
+								// Minting-account component of reward-share can no longer mint - disregard
+								madi.remove();
+								continue;
+							}
+
+							// Optional (non-validated) prevention of block submissions below a defined level.
+							// This is an unvalidated version of Blockchain.minAccountLevelToMint
+							// and exists only to reduce block candidates by default.
+							int level = mintingAccount.getEffectiveMintingLevel();
+							if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
+								madi.remove();
+							}
+						}
+
+						// Needs a mutable copy of the unmodifiableList
+						List<Peer> peers = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers());
+						BlockData lastBlockData = blockRepository.getLastBlock();
+
+						// Disregard peers that have "misbehaved" recently
+						peers.removeIf(Controller.hasMisbehaved);
+
+						// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
+						// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
+						if (!Synchronizer.getInstance().getRecoveryMode())
+							peers.removeIf(Controller.hasNoRecentBlock);
+
+						// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
+						if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
+							continue;
+
+						// If we are stuck on an invalid block, we should allow an alternative to be minted
+						boolean recoverInvalidBlock = false;
+						if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) {
+							// We've had at least one invalid block
+							long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived;
+							long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived;
+							if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) {
+								if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) {
+									// Last valid block was more than 10 mins ago, but we've had an invalid block since then
+									// Assume that the chain has stalled because there is no alternative valid candidate
+									// Enter recovery mode to allow alternative, valid candidates to be minted
+									recoverInvalidBlock = true;
+								}
+							}
+						}
+
+						// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
+						if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
+							if (!Synchronizer.getInstance().getRecoveryMode() && !recoverInvalidBlock)
 								continue;
 
-							testBlock.preProcess();
+						// There are enough peers with a recent block and our latest block is recent
+						// so go ahead and mint a block if possible.
+						isMintingPossible = true;
 
-							// Is new block valid yet? (Before adding unconfirmed transactions)
-							ValidationResult result = testBlock.isValid();
-							if (result != ValidationResult.OK) {
-								moderatedLog(() -> LOGGER.error(String.format("To-be-minted block invalid '%s' before adding transactions?", result.name())));
+						// Check blockchain hasn't changed
+						if (previousBlockData == null || !Arrays.equals(previousBlockData.getSignature(), lastBlockData.getSignature())) {
+							previousBlockData = lastBlockData;
+							newBlocks.clear();
 
-								newBlocksIterator.remove();
-								wasInvalidBlockDiscarded = true;
-								/*
-								 * Bail out fast so that we loop around from the top again.
-								 * This gives BlockMinter the possibility to remint this candidate block using another block from newBlocks,
-								 * via the Blocks.remint() method, which avoids having to re-process Block ATs all over again.
-								 * Particularly useful if some aspect of Blocks changes due a timestamp-based feature-trigger (see BlockChain class).
-								 */
-								break;
-							}
+							// Reduce log timeout
+							logTimeout = 10 * 1000L;
 
-							goodBlocks.add(testBlock);
+							// Last low weight block is no longer valid
+							parentSignatureForLastLowWeightBlock = null;
 						}
 
-						if (wasInvalidBlockDiscarded || goodBlocks.isEmpty())
+						// Discard accounts we have already built blocks with
+						mintingAccountsData.removeIf(mintingAccountData -> newBlocks.stream().anyMatch(newBlock -> Arrays.equals(newBlock.getBlockData().getMinterPublicKey(), mintingAccountData.getPublicKey())));
+
+						// Do we need to build any potential new blocks?
+						List<PrivateKeyAccount> newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
+
+						// We might need to sit the next block out, if one of our minting accounts signed the previous one
+						// Skip this check for single node testnets, since they definitely need to mint every block
+						byte[] previousBlockMinter = previousBlockData.getMinterPublicKey();
+						boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter));
+						if (mintedLastBlock && !isSingleNodeTestnet) {
+							LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one"));
 							continue;
-
-						// Pick best block
-						final int parentHeight = previousBlockData.getHeight();
-						final byte[] parentBlockSignature = previousBlockData.getSignature();
-
-						BigInteger bestWeight = null;
-
-						for (int bi = 0; bi < goodBlocks.size(); ++bi) {
-							BlockData blockData = goodBlocks.get(bi).getBlockData();
-
-							BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
-							int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
-							blockSummaryData.setMinterLevel(minterLevel);
-
-							BigInteger blockWeight = Block.calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData);
-
-							if (bestWeight == null || blockWeight.compareTo(bestWeight) < 0) {
-								newBlock = goodBlocks.get(bi);
-								bestWeight = blockWeight;
-							}
 						}
 
-						try {
-							if (this.higherWeightChainExists(repository, bestWeight)) {
+						if (parentSignatureForLastLowWeightBlock != null) {
+							// The last iteration found a higher weight block in the network, so sleep for a while
+							// to allow is to sync the higher weight chain. We are sleeping here rather than when
+							// detected as we don't want to hold the blockchain lock open.
+							LOGGER.info("Sleeping for 10 seconds...");
+							Thread.sleep(10 * 1000L);
+						}
 
-								// Check if the base block has updated since the last time we were here
-								if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null ||
-										!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) {
-									// We've switched to a different chain, so reset the timer
-									timeOfLastLowWeightBlock = NTP.getTime();
-								}
-								parentSignatureForLastLowWeightBlock = previousBlockData.getSignature();
-
-								// If less than 30 seconds has passed since first detection the higher weight chain,
-								// we should skip our block submission to give us the opportunity to sync to the better chain
-								if (NTP.getTime() - timeOfLastLowWeightBlock < 30 * 1000L) {
-									LOGGER.info("Higher weight chain found in peers, so not signing a block this round");
-									LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock);
+						for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
+							// First block does the AT heavy-lifting
+							if (newBlocks.isEmpty()) {
+								Block newBlock = Block.mint(repository, previousBlockData, mintingAccount);
+								if (newBlock == null) {
+									// For some reason we can't mint right now
+									moderatedLog(() -> LOGGER.info("Couldn't build a to-be-minted block"));
 									continue;
-								} else {
-									// More than 30 seconds have passed, so we should submit our block candidate anyway.
-									LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate...");
 								}
+
+								newBlocks.add(newBlock);
 							} else {
-								LOGGER.debug("No higher weight chain found in peers");
+								// The blocks for other minters require less effort...
+								Block newBlock = newBlocks.get(0).remint(mintingAccount);
+								if (newBlock == null) {
+									// For some reason we can't mint right now
+									moderatedLog(() -> LOGGER.error("Couldn't rebuild a to-be-minted block"));
+									continue;
+								}
+
+								newBlocks.add(newBlock);
 							}
-						} catch (DataException e) {
-							LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
 						}
 
-						// Discard any uncommitted changes as a result of the higher weight chain detection
-						repository.discardChanges();
+						// No potential block candidates?
+						if (newBlocks.isEmpty())
+							continue;
 
-						// Clear variables that track low weight blocks
-						parentSignatureForLastLowWeightBlock = null;
-						timeOfLastLowWeightBlock = null;
-
-						Long unconfirmedStartTime = NTP.getTime();
-
-						// Add unconfirmed transactions
-						addUnconfirmedTransactions(repository, newBlock);
-
-						LOGGER.info(String.format("Adding %d unconfirmed transactions took %d ms", newBlock.getTransactions().size(), (NTP.getTime()-unconfirmedStartTime)));
-
-						// Sign to create block's signature
-						newBlock.sign();
-
-						// Is newBlock still valid?
-						ValidationResult validationResult = newBlock.isValid();
-						if (validationResult != ValidationResult.OK) {
-							// No longer valid? Report and discard
-							LOGGER.error(String.format("To-be-minted block now invalid '%s' after adding unconfirmed transactions?", validationResult.name()));
-
-							// Rebuild block candidates, just to be sure
-							newBlocks.clear();
+						// Make sure we're the only thread modifying the blockchain
+						ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+						if (!blockchainLock.tryLock(30, TimeUnit.SECONDS)) {
+							LOGGER.debug("Couldn't acquire blockchain lock even after waiting 30 seconds");
 							continue;
 						}
 
-						// Add to blockchain - something else will notice and broadcast new block to network
+						boolean newBlockMinted = false;
+						Block newBlock = null;
+
 						try {
-							newBlock.process();
+							// Clear repository session state so we have latest view of data
+							repository.discardChanges();
 
-							repository.saveChanges();
+							// Now that we have blockchain lock, do final check that chain hasn't changed
+							BlockData latestBlockData = blockRepository.getLastBlock();
+							if (!Arrays.equals(lastBlockData.getSignature(), latestBlockData.getSignature()))
+								continue;
 
-							LOGGER.info(String.format("Minted new block: %d", newBlock.getBlockData().getHeight()));
+							List<Block> goodBlocks = new ArrayList<>();
+							boolean wasInvalidBlockDiscarded = false;
+							Iterator<Block> newBlocksIterator = newBlocks.iterator();
 
-							RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey());
+							while (newBlocksIterator.hasNext()) {
+								Block testBlock = newBlocksIterator.next();
 
-							if (rewardShareData != null) {
-								LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s",
-										newBlock.getBlockData().getHeight(),
-										Base58.encode(newBlock.getBlockData().getSignature()),
-										Base58.encode(newBlock.getParent().getSignature()),
-										rewardShareData.getMinter(),
-										rewardShareData.getRecipient()));
-							} else {
-								LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s",
-										newBlock.getBlockData().getHeight(),
-										Base58.encode(newBlock.getBlockData().getSignature()),
-										Base58.encode(newBlock.getParent().getSignature()),
-										newBlock.getMinter().getAddress()));
+								// Is new block's timestamp valid yet?
+								// We do a separate check as some timestamp checks are skipped for testchains
+								if (testBlock.isTimestampValid() != ValidationResult.OK)
+									continue;
+
+								testBlock.preProcess();
+
+								// Is new block valid yet? (Before adding unconfirmed transactions)
+								ValidationResult result = testBlock.isValid();
+								if (result != ValidationResult.OK) {
+									moderatedLog(() -> LOGGER.error(String.format("To-be-minted block invalid '%s' before adding transactions?", result.name())));
+
+									newBlocksIterator.remove();
+									wasInvalidBlockDiscarded = true;
+									/*
+									 * Bail out fast so that we loop around from the top again.
+									 * This gives BlockMinter the possibility to remint this candidate block using another block from newBlocks,
+									 * via the Blocks.remint() method, which avoids having to re-process Block ATs all over again.
+									 * Particularly useful if some aspect of Blocks changes due a timestamp-based feature-trigger (see BlockChain class).
+									 */
+									break;
+								}
+
+								goodBlocks.add(testBlock);
 							}
 
-							// Notify network after we're released blockchain lock
-							newBlockMinted = true;
+							if (wasInvalidBlockDiscarded || goodBlocks.isEmpty())
+								continue;
 
-							// Notify Controller
-							repository.discardChanges(); // clear transaction status to prevent deadlocks
-							Controller.getInstance().onNewBlock(newBlock.getBlockData());
-						} catch (DataException e) {
-							// Unable to process block - report and discard
-							LOGGER.error("Unable to process newly minted block?", e);
-							newBlocks.clear();
-						} catch (ArithmeticException e) {
-							// Unable to process block - report and discard
-							LOGGER.error("Unable to process newly minted block?", e);
-							newBlocks.clear();
+							// Pick best block
+							final int parentHeight = previousBlockData.getHeight();
+							final byte[] parentBlockSignature = previousBlockData.getSignature();
+
+							BigInteger bestWeight = null;
+
+							for (int bi = 0; bi < goodBlocks.size(); ++bi) {
+								BlockData blockData = goodBlocks.get(bi).getBlockData();
+
+								BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
+								int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
+								blockSummaryData.setMinterLevel(minterLevel);
+
+								BigInteger blockWeight = Block.calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData);
+
+								if (bestWeight == null || blockWeight.compareTo(bestWeight) < 0) {
+									newBlock = goodBlocks.get(bi);
+									bestWeight = blockWeight;
+								}
+							}
+
+							try {
+								if (this.higherWeightChainExists(repository, bestWeight)) {
+
+									// Check if the base block has updated since the last time we were here
+									if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null ||
+											!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) {
+										// We've switched to a different chain, so reset the timer
+										timeOfLastLowWeightBlock = NTP.getTime();
+									}
+									parentSignatureForLastLowWeightBlock = previousBlockData.getSignature();
+
+									// If less than 30 seconds has passed since first detection the higher weight chain,
+									// we should skip our block submission to give us the opportunity to sync to the better chain
+									if (NTP.getTime() - timeOfLastLowWeightBlock < 30 * 1000L) {
+										LOGGER.info("Higher weight chain found in peers, so not signing a block this round");
+										LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock);
+										continue;
+									} else {
+										// More than 30 seconds have passed, so we should submit our block candidate anyway.
+										LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate...");
+									}
+								} else {
+									LOGGER.debug("No higher weight chain found in peers");
+								}
+							} catch (DataException e) {
+								LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
+							}
+
+							// Discard any uncommitted changes as a result of the higher weight chain detection
+							repository.discardChanges();
+
+							// Clear variables that track low weight blocks
+							parentSignatureForLastLowWeightBlock = null;
+							timeOfLastLowWeightBlock = null;
+
+							Long unconfirmedStartTime = NTP.getTime();
+
+							// Add unconfirmed transactions
+							addUnconfirmedTransactions(repository, newBlock);
+
+							LOGGER.info(String.format("Adding %d unconfirmed transactions took %d ms", newBlock.getTransactions().size(), (NTP.getTime() - unconfirmedStartTime)));
+
+							// Sign to create block's signature
+							newBlock.sign();
+
+							// Is newBlock still valid?
+							ValidationResult validationResult = newBlock.isValid();
+							if (validationResult != ValidationResult.OK) {
+								// No longer valid? Report and discard
+								LOGGER.error(String.format("To-be-minted block now invalid '%s' after adding unconfirmed transactions?", validationResult.name()));
+
+								// Rebuild block candidates, just to be sure
+								newBlocks.clear();
+								continue;
+							}
+
+							// Add to blockchain - something else will notice and broadcast new block to network
+							try {
+								newBlock.process();
+
+								repository.saveChanges();
+
+								LOGGER.info(String.format("Minted new block: %d", newBlock.getBlockData().getHeight()));
+
+								RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey());
+
+								if (rewardShareData != null) {
+									LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s",
+											newBlock.getBlockData().getHeight(),
+											Base58.encode(newBlock.getBlockData().getSignature()),
+											Base58.encode(newBlock.getParent().getSignature()),
+											rewardShareData.getMinter(),
+											rewardShareData.getRecipient()));
+								} else {
+									LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s",
+											newBlock.getBlockData().getHeight(),
+											Base58.encode(newBlock.getBlockData().getSignature()),
+											Base58.encode(newBlock.getParent().getSignature()),
+											newBlock.getMinter().getAddress()));
+								}
+
+								// Notify network after we're released blockchain lock
+								newBlockMinted = true;
+
+								// Notify Controller
+								repository.discardChanges(); // clear transaction status to prevent deadlocks
+								Controller.getInstance().onNewBlock(newBlock.getBlockData());
+							} catch (DataException e) {
+								// Unable to process block - report and discard
+								LOGGER.error("Unable to process newly minted block?", e);
+								newBlocks.clear();
+							} catch (ArithmeticException e) {
+								// Unable to process block - report and discard
+								LOGGER.error("Unable to process newly minted block?", e);
+								newBlocks.clear();
+							}
+						} finally {
+							blockchainLock.unlock();
 						}
-					} finally {
-						blockchainLock.unlock();
-					}
 
-					if (newBlockMinted) {
-						// Broadcast our new chain to network
-						Network.getInstance().broadcastOurChain();
-					}
+						if (newBlockMinted) {
+							// Broadcast our new chain to network
+							Network.getInstance().broadcastOurChain();
+						}
 
-				} catch (InterruptedException e) {
-					// We've been interrupted - time to exit
-					return;
+					} catch (InterruptedException e) {
+						// We've been interrupted - time to exit
+						return;
+					}
+				} catch (DataException e) {
+					LOGGER.warn("Repository issue while running block minter - NO LONGER MINTING", e);
+				} catch (Exception e) {
+					LOGGER.error(e.getMessage(), e);
 				}
 			}
-		} catch (DataException e) {
-			LOGGER.warn("Repository issue while running block minter - NO LONGER MINTING", e);
+		} catch (Exception e) {
+			LOGGER.error(e.getMessage(), e);
 		}
 	}
 

src/main/java/org/qortal/controller/OnlineAccountsManager.java

@@ -13,6 +13,7 @@ import org.qortal.crypto.MemoryPoW;
 import org.qortal.crypto.Qortal25519Extras;
 import org.qortal.data.account.MintingAccountData;
 import org.qortal.data.account.RewardShareData;
+import org.qortal.data.group.GroupMemberData;
 import org.qortal.data.network.OnlineAccountData;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
@@ -24,6 +25,7 @@ import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
 import org.qortal.utils.Base58;
+import org.qortal.utils.Groups;
 import org.qortal.utils.NTP;
 import org.qortal.utils.NamedThreadFactory;
 
@@ -44,6 +46,7 @@ public class OnlineAccountsManager {
      */
     private static final long ONLINE_TIMESTAMP_MODULUS_V1 = 5 * 60 * 1000L;
     private static final long ONLINE_TIMESTAMP_MODULUS_V2 = 30 * 60 * 1000L;
+    private static final long ONLINE_TIMESTAMP_MODULUS_V3 = 10 * 60 * 1000L;
 
     /**
      * How many 'current' timestamp-sets of online accounts we cache.
@@ -67,12 +70,13 @@ public class OnlineAccountsManager {
     private static final long ONLINE_ACCOUNTS_COMPUTE_INITIAL_SLEEP_INTERVAL = 30 * 1000L; // ms
 
     // MemoryPoW - mainnet
-    public static final int POW_BUFFER_SIZE = 1 * 1024 * 1024; // bytes
+    public static final int POW_BUFFER_SIZE = 1024 * 1024; // bytes
     public static final int POW_DIFFICULTY_V1 = 18; // leading zero bits
     public static final int POW_DIFFICULTY_V2 = 19; // leading zero bits
+    public static final int POW_DIFFICULTY_V3 = 6; // leading zero bits
 
     // MemoryPoW - testnet
-    public static final int POW_BUFFER_SIZE_TESTNET = 1 * 1024 * 1024; // bytes
+    public static final int POW_BUFFER_SIZE_TESTNET = 1024 * 1024; // bytes
     public static final int POW_DIFFICULTY_TESTNET = 5; // leading zero bits
 
     // IMPORTANT: if we ever need to dynamically modify the buffer size using a feature trigger, the
@@ -106,11 +110,15 @@ public class OnlineAccountsManager {
 
     public static long getOnlineTimestampModulus() {
         Long now = NTP.getTime();
-        if (now != null && now >= BlockChain.getInstance().getOnlineAccountsModulusV2Timestamp()) {
+        if (now != null && now >= BlockChain.getInstance().getOnlineAccountsModulusV2Timestamp() && now < BlockChain.getInstance().getOnlineAccountsModulusV3Timestamp()) {
             return ONLINE_TIMESTAMP_MODULUS_V2;
         }
+        if (now != null && now >= BlockChain.getInstance().getOnlineAccountsModulusV3Timestamp()) {
+            return ONLINE_TIMESTAMP_MODULUS_V3;
+        }
         return ONLINE_TIMESTAMP_MODULUS_V1;
     }
+
     public static Long getCurrentOnlineAccountTimestamp() {
         Long now = NTP.getTime();
         if (now == null)
@@ -135,9 +143,12 @@ public class OnlineAccountsManager {
         if (Settings.getInstance().isTestNet())
             return POW_DIFFICULTY_TESTNET;
 
-        if (timestamp >= BlockChain.getInstance().getIncreaseOnlineAccountsDifficultyTimestamp())
+        if (timestamp >= BlockChain.getInstance().getIncreaseOnlineAccountsDifficultyTimestamp() && timestamp < BlockChain.getInstance().getDecreaseOnlineAccountsDifficultyTimestamp())
             return POW_DIFFICULTY_V2;
 
+        if (timestamp >= BlockChain.getInstance().getDecreaseOnlineAccountsDifficultyTimestamp())
+            return POW_DIFFICULTY_V3;
+
         return POW_DIFFICULTY_V1;
     }
 
@@ -215,6 +226,15 @@ public class OnlineAccountsManager {
         Set<OnlineAccountData> onlineAccountsToAdd = new HashSet<>();
         Set<OnlineAccountData> onlineAccountsToRemove = new HashSet<>();
         try (final Repository repository = RepositoryManager.getRepository()) {
+
+            int blockHeight = repository.getBlockRepository().getBlockchainHeight();
+
+            List<String> mintingGroupMemberAddresses
+                = Groups.getAllMembers(
+                    repository.getGroupRepository(),
+                    Groups.getGroupIdsToMint(BlockChain.getInstance(), blockHeight)
+            );
+
             for (OnlineAccountData onlineAccountData : this.onlineAccountsImportQueue) {
                 if (isStopping)
                     return;
@@ -227,7 +247,7 @@ public class OnlineAccountsManager {
                     continue;
                 }
 
-                boolean isValid = this.isValidCurrentAccount(repository, onlineAccountData);
+                boolean isValid = this.isValidCurrentAccount(repository, mintingGroupMemberAddresses, onlineAccountData);
                 if (isValid)
                     onlineAccountsToAdd.add(onlineAccountData);
 
@@ -306,7 +326,7 @@ public class OnlineAccountsManager {
         return inplaceArray;
     }
 
-    private static boolean isValidCurrentAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException {
+    private static boolean isValidCurrentAccount(Repository repository, List<String> mintingGroupMemberAddresses, OnlineAccountData onlineAccountData) throws DataException {
         final Long now = NTP.getTime();
         if (now == null)
             return false;
@@ -341,9 +361,14 @@ public class OnlineAccountsManager {
             LOGGER.trace(() -> String.format("Rejecting unknown online reward-share public key %s", Base58.encode(rewardSharePublicKey)));
             return false;
         }
+        // reject account address that are not in the MINTER Group
+        else if( !mintingGroupMemberAddresses.contains(rewardShareData.getMinter())) {
+            LOGGER.trace(() -> String.format("Rejecting online reward-share that is not in MINTER Group, account %s", rewardShareData.getMinter()));
+            return false;
+        }
 
         Account mintingAccount = new Account(repository, rewardShareData.getMinter());
-        if (!mintingAccount.canMint()) {
+        if (!mintingAccount.canMint(true)) {  // group validation is a few lines above
             // Minting-account component of reward-share can no longer mint - disregard
             LOGGER.trace(() -> String.format("Rejecting online reward-share with non-minting account %s", mintingAccount.getAddress()));
             return false;
@@ -530,7 +555,7 @@ public class OnlineAccountsManager {
                     }
 
                     Account mintingAccount = new Account(repository, rewardShareData.getMinter());
-                    if (!mintingAccount.canMint()) {
+                    if (!mintingAccount.canMint(true)) {
                         // Minting-account component of reward-share can no longer mint - disregard
                         iterator.remove();
                         continue;

src/main/java/org/qortal/controller/RNSTransactionImporter.java

@@ -0,0 +1,460 @@
+package org.qortal.controller;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.data.block.BlockData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.GetTransactionMessage;
+import org.qortal.network.message.Message;
+import org.qortal.network.message.TransactionMessage;
+import org.qortal.network.message.TransactionSignaturesMessage;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.transaction.Transaction;
+import org.qortal.transform.TransformationException;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+
+import java.util.*;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+
+public class RNSTransactionImporter extends Thread {
+
+    private static final Logger LOGGER = LogManager.getLogger(RNSTransactionImporter.class);
+
+    private static RNSTransactionImporter instance;
+    private volatile boolean isStopping = false;
+
+    private static final int MAX_INCOMING_TRANSACTIONS = 5000;
+
+    /** Minimum time before considering an invalid unconfirmed transaction as "stale" */
+    public static final long INVALID_TRANSACTION_STALE_TIMEOUT = 30 * 60 * 1000L; // ms
+    /** Minimum frequency to re-request stale unconfirmed transactions from peers, to recheck validity */
+    public static final long INVALID_TRANSACTION_RECHECK_INTERVAL = 60 * 60 * 1000L; // ms\
+    /** Minimum frequency to re-request expired unconfirmed transactions from peers, to recheck validity
+     * This mainly exists to stop expired transactions from bloating the list */
+    public static final long EXPIRED_TRANSACTION_RECHECK_INTERVAL = 10 * 60 * 1000L; // ms
+
+
+    /** Map of incoming transaction that are in the import queue. Key is transaction data, value is whether signature has been validated. */
+    private final Map<TransactionData, Boolean> incomingTransactions = Collections.synchronizedMap(new HashMap<>());
+
+    /** Map of recent invalid unconfirmed transactions. Key is base58 transaction signature, value is do-not-request expiry timestamp. */
+    private final Map<String, Long> invalidUnconfirmedTransactions = Collections.synchronizedMap(new HashMap<>());
+
+    /** Cached list of unconfirmed transactions, used when counting per creator. This is replaced regularly */
+    public static List<TransactionData> unconfirmedTransactionsCache = null;
+
+
+    public static synchronized RNSTransactionImporter getInstance() {
+        if (instance == null) {
+            instance = new RNSTransactionImporter();
+        }
+
+        return instance;
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Transaction Importer");
+
+        try {
+            while (!Controller.isStopping()) {
+                Thread.sleep(500L);
+
+                // Process incoming transactions queue
+                validateTransactionsInQueue();
+                importTransactionsInQueue();
+
+                // Clean up invalid incoming transactions list
+                cleanupInvalidTransactionsList(NTP.getTime());
+            }
+        } catch (InterruptedException e) {
+            // Fall through to exit thread
+        }
+    }
+
+    public void shutdown() {
+        isStopping = true;
+        this.interrupt();
+    }
+
+
+    // Incoming transactions queue
+
+    private boolean incomingTransactionQueueContains(byte[] signature) {
+        synchronized (incomingTransactions) {
+            return incomingTransactions.keySet().stream().anyMatch(t -> Arrays.equals(t.getSignature(), signature));
+        }
+    }
+
+    private void removeIncomingTransaction(byte[] signature) {
+        incomingTransactions.keySet().removeIf(t -> Arrays.equals(t.getSignature(), signature));
+    }
+
+    /**
+     * Retrieve all pending unconfirmed transactions that have had their signatures validated.
+     * @return a list of TransactionData objects, with valid signatures.
+     */
+    private List<TransactionData> getCachedSigValidTransactions() {
+        synchronized (this.incomingTransactions) {
+            return this.incomingTransactions.entrySet().stream()
+                    .filter(t -> Boolean.TRUE.equals(t.getValue()))
+                    .map(Map.Entry::getKey)
+                    .collect(Collectors.toList());
+        }
+    }
+
+    /**
+     * Validate the signatures of any transactions pending import, then update their
+     * entries in the queue to mark them as valid/invalid.
+     *
+     * No database lock is required.
+     */
+    private void validateTransactionsInQueue() {
+        if (this.incomingTransactions.isEmpty()) {
+            // Nothing to do?
+            return;
+        }
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            // Take a snapshot of incomingTransactions, so we don't need to lock it while processing
+            Map<TransactionData, Boolean> incomingTransactionsCopy = Map.copyOf(this.incomingTransactions);
+
+            int unvalidatedCount = Collections.frequency(incomingTransactionsCopy.values(), Boolean.FALSE);
+            int validatedCount = 0;
+
+            if (unvalidatedCount > 0) {
+                LOGGER.debug("Validating signatures in incoming transactions queue (size {})...", unvalidatedCount);
+            }
+
+            // A list of all currently pending transactions that have valid signatures
+            List<Transaction> sigValidTransactions = new ArrayList<>();
+
+            // A list of signatures that became valid in this round
+            List<byte[]> newlyValidSignatures = new ArrayList<>();
+
+            boolean isLiteNode = Settings.getInstance().isLite();
+
+            // We need the latest block in order to check for expired transactions
+            BlockData latestBlock = Controller.getInstance().getChainTip();
+
+            // Signature validation round - does not require blockchain lock
+            for (Map.Entry<TransactionData, Boolean> transactionEntry : incomingTransactionsCopy.entrySet()) {
+                // Quick exit?
+                if (isStopping) {
+                    return;
+                }
+
+                TransactionData transactionData = transactionEntry.getKey();
+                Transaction transaction = Transaction.fromData(repository, transactionData);
+                String signature58 = Base58.encode(transactionData.getSignature());
+
+                Long now = NTP.getTime();
+                if (now == null) {
+                    return;
+                }
+
+                // Drop expired transactions before they are considered "sig valid"
+                if (latestBlock != null && transaction.getDeadline() <= latestBlock.getTimestamp()) {
+                    LOGGER.debug("Removing expired {} transaction {} from import queue", transactionData.getType().name(), signature58);
+                    removeIncomingTransaction(transactionData.getSignature());
+                    invalidUnconfirmedTransactions.put(signature58, (now + EXPIRED_TRANSACTION_RECHECK_INTERVAL));
+                    continue;
+                }
+
+                // Only validate signature if we haven't already done so
+                Boolean isSigValid = transactionEntry.getValue();
+                if (!Boolean.TRUE.equals(isSigValid)) {
+                    if (isLiteNode) {
+                        // Lite nodes can't easily validate transactions, so for now we will have to assume that everything is valid
+                        sigValidTransactions.add(transaction);
+                        newlyValidSignatures.add(transactionData.getSignature());
+                        // Add mark signature as valid if transaction still exists in import queue
+                        incomingTransactions.computeIfPresent(transactionData, (k, v) -> Boolean.TRUE);
+                        continue;
+                    }
+
+                    if (!transaction.isSignatureValid()) {
+                        LOGGER.debug("Ignoring {} transaction {} with invalid signature", transactionData.getType().name(), signature58);
+                        removeIncomingTransaction(transactionData.getSignature());
+
+                        // Also add to invalidIncomingTransactions map
+                        now = NTP.getTime();
+                        if (now != null) {
+                            Long expiry = now + INVALID_TRANSACTION_RECHECK_INTERVAL;
+                            LOGGER.trace("Adding invalid transaction {} to invalidUnconfirmedTransactions...", signature58);
+                            // Add to invalidUnconfirmedTransactions so that we don't keep requesting it
+                            invalidUnconfirmedTransactions.put(signature58, expiry);
+                        }
+
+                        // We're done with this transaction
+                        continue;
+                    }
+
+                    // Count the number that were validated in this round, for logging purposes
+                    validatedCount++;
+
+                    // Add mark signature as valid if transaction still exists in import queue
+                    incomingTransactions.computeIfPresent(transactionData, (k, v) -> Boolean.TRUE);
+
+                    // Signature validated in this round
+                    newlyValidSignatures.add(transactionData.getSignature());
+
+                } else {
+                    LOGGER.trace(() -> String.format("Transaction %s known to have valid signature", Base58.encode(transactionData.getSignature())));
+                }
+
+                // Signature valid - add to shortlist
+                sigValidTransactions.add(transaction);
+            }
+
+            if (unvalidatedCount > 0) {
+                LOGGER.debug("Finished validating signatures in incoming transactions queue (valid this round: {}, total pending import: {})...", validatedCount, sigValidTransactions.size());
+            }
+
+        } catch (DataException e) {
+            LOGGER.error("Repository issue while processing incoming transactions", e);
+        }
+    }
+
+    /**
+     * Import any transactions in the queue that have valid signatures.
+     *
+     * A database lock is required.
+     */
+    private void importTransactionsInQueue() {
+        List<TransactionData> sigValidTransactions = this.getCachedSigValidTransactions();
+        if (sigValidTransactions.isEmpty()) {
+            // Don't bother locking if there are no new transactions to process
+            return;
+        }
+
+        if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) {
+            // Prioritize syncing, and don't attempt to lock
+            return;
+        }
+
+        ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+        if (!blockchainLock.tryLock()) {
+            LOGGER.debug("Too busy to import incoming transactions queue");
+            return;
+        }
+
+        LOGGER.debug("Importing incoming transactions queue (size {})...", sigValidTransactions.size());
+
+        int processedCount = 0;
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Use a single copy of the unconfirmed transactions list for each cycle, to speed up constant lookups
+            // when counting unconfirmed transactions by creator.
+            List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions();
+            unconfirmedTransactions.removeIf(t -> t.getType() == Transaction.TransactionType.CHAT);
+            unconfirmedTransactionsCache = unconfirmedTransactions;
+
+            // A list of signatures were imported in this round
+            List<byte[]> newlyImportedSignatures = new ArrayList<>();
+
+            // Import transactions with valid signatures
+            try {
+                for (int i = 0; i < sigValidTransactions.size(); ++i) {
+                    if (isStopping) {
+                        return;
+                    }
+
+                    if (Synchronizer.getInstance().isSyncRequestPending()) {
+                        LOGGER.debug("Breaking out of transaction importing with {} remaining, because a sync request is pending", sigValidTransactions.size() - i);
+                        return;
+                    }
+
+                    TransactionData transactionData = sigValidTransactions.get(i);
+                    Transaction transaction = Transaction.fromData(repository, transactionData);
+
+                    Transaction.ValidationResult validationResult = transaction.importAsUnconfirmed();
+                    processedCount++;
+
+                    switch (validationResult) {
+                        case TRANSACTION_ALREADY_EXISTS: {
+                            LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature())));
+                            break;
+                        }
+
+                        case NO_BLOCKCHAIN_LOCK: {
+                            // Is this even possible considering we acquired blockchain lock above?
+                            LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction %s", Base58.encode(transactionData.getSignature())));
+                            break;
+                        }
+
+                        case OK: {
+                            LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature())));
+
+                            // Add to the unconfirmed transactions cache
+                            if (transactionData.getType() != Transaction.TransactionType.CHAT && unconfirmedTransactionsCache != null) {
+                                unconfirmedTransactionsCache.add(transactionData);
+                            }
+
+                            // Signature imported in this round
+                            newlyImportedSignatures.add(transactionData.getSignature());
+
+                            break;
+                        }
+
+                        // All other invalid cases:
+                        default: {
+                            final String signature58 = Base58.encode(transactionData.getSignature());
+                            LOGGER.debug(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58));
+
+                            Long now = NTP.getTime();
+                            if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) {
+                                Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL;
+
+                                if (validationResult == Transaction.ValidationResult.TIMESTAMP_TOO_OLD) {
+                                    // Use shorter recheck interval for expired transactions
+                                    expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL;
+                                }
+
+                                Long expiry = now + expiryLength;
+                                LOGGER.trace("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58);
+                                // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it
+                                invalidUnconfirmedTransactions.put(signature58, expiry);
+                            }
+                        }
+                    }
+
+                    // Transaction has been processed, even if only to reject it
+                    removeIncomingTransaction(transactionData.getSignature());
+                }
+
+                if (!newlyImportedSignatures.isEmpty()) {
+                    LOGGER.debug("Broadcasting {} newly imported signatures", newlyImportedSignatures.size());
+                    Message newTransactionSignatureMessage = new TransactionSignaturesMessage(newlyImportedSignatures);
+                    RNSNetwork.getInstance().broadcast(broadcastPeer -> newTransactionSignatureMessage);
+                }
+            } finally {
+                LOGGER.debug("Finished importing {} incoming transaction{}", processedCount, (processedCount == 1 ? "" : "s"));
+                blockchainLock.unlock();
+
+                // Clear the unconfirmed transaction cache so new data can be populated in the next cycle
+                unconfirmedTransactionsCache = null;
+            }
+        } catch (DataException e) {
+            LOGGER.error("Repository issue while importing incoming transactions", e);
+        }
+    }
+
+    private void cleanupInvalidTransactionsList(Long now) {
+        if (now == null) {
+            return;
+        }
+        // Periodically remove invalid unconfirmed transactions from the list, so that they can be fetched again
+        invalidUnconfirmedTransactions.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < now);
+    }
+
+
+    // Network handlers
+
+    public void onNetworkTransactionMessage(RNSPeer peer, Message message) {
+        TransactionMessage transactionMessage = (TransactionMessage) message;
+        TransactionData transactionData = transactionMessage.getTransactionData();
+
+        if (this.incomingTransactions.size() < MAX_INCOMING_TRANSACTIONS) {
+            synchronized (this.incomingTransactions) {
+                if (!incomingTransactionQueueContains(transactionData.getSignature())) {
+                    this.incomingTransactions.put(transactionData, Boolean.FALSE);
+                }
+            }
+        }
+    }
+
+    public void onNetworkGetTransactionMessage(RNSPeer peer, Message message) {
+        GetTransactionMessage getTransactionMessage = (GetTransactionMessage) message;
+        byte[] signature = getTransactionMessage.getSignature();
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            // Firstly check the sig-valid transactions that are currently queued for import
+            TransactionData transactionData = this.getCachedSigValidTransactions().stream()
+                    .filter(t -> Arrays.equals(signature, t.getSignature()))
+                    .findFirst().orElse(null);
+
+            if (transactionData == null) {
+                // Not found in import queue, so try the database
+                transactionData = repository.getTransactionRepository().fromSignature(signature);
+            }
+
+            if (transactionData == null) {
+                // Still not found - so we don't have this transaction
+                LOGGER.debug(() -> String.format("Ignoring GET_TRANSACTION request from peer %s for unknown transaction %s", peer, Base58.encode(signature)));
+                // Send no response at all???
+                return;
+            }
+
+            Message transactionMessage = new TransactionMessage(transactionData);
+            transactionMessage.setId(message.getId());
+            peer.sendMessage(transactionMessage);
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e);
+        } catch (TransformationException e) {
+            LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e);
+        }
+    }
+
+    public void onNetworkGetUnconfirmedTransactionsMessage(RNSPeer peer, Message message) {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            List<byte[]> signatures = Collections.emptyList();
+
+            // If we're NOT up-to-date then don't send out unconfirmed transactions
+            // as it's possible they are already included in a later block that we don't have.
+            if (Controller.getInstance().isUpToDate())
+                signatures = repository.getTransactionRepository().getUnconfirmedTransactionSignatures();
+
+            Message transactionSignaturesMessage = new TransactionSignaturesMessage(signatures);
+            peer.sendMessage(transactionSignaturesMessage);
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while sending unconfirmed transaction signatures to peer %s", peer), e);
+        }
+    }
+
+    public void onNetworkTransactionSignaturesMessage(RNSPeer peer, Message message) {
+        TransactionSignaturesMessage transactionSignaturesMessage = (TransactionSignaturesMessage) message;
+        List<byte[]> signatures = transactionSignaturesMessage.getSignatures();
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            for (byte[] signature : signatures) {
+                String signature58 = Base58.encode(signature);
+                if (invalidUnconfirmedTransactions.containsKey(signature58)) {
+                    // Previously invalid transaction - don't keep requesting it
+                    // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks
+                    continue;
+                }
+
+                // Ignore if this transaction is in the queue
+                if (incomingTransactionQueueContains(signature)) {
+                    LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peer));
+                    continue;
+                }
+
+                // Do we have it already? (Before requesting transaction data itself)
+                if (repository.getTransactionRepository().exists(signature)) {
+                    LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer));
+                    continue;
+                }
+
+                // Check isInterrupted() here and exit fast
+                if (Thread.currentThread().isInterrupted())
+                    return;
+
+                // Fetch actual transaction data from peer
+                Message getTransactionMessage = new GetTransactionMessage(signature);
+                peer.sendMessage(getTransactionMessage);
+            }
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while processing unconfirmed transactions from peer %s", peer), e);
+        }
+    }
+
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java

@@ -2,22 +2,30 @@ package org.qortal.controller.arbitrary;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.qortal.api.resource.TransactionsResource;
 import org.qortal.controller.Controller;
 import org.qortal.data.arbitrary.ArbitraryResourceData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.event.DataMonitorEvent;
+import org.qortal.event.EventBus;
 import org.qortal.gui.SplashFrame;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
 import org.qortal.transaction.ArbitraryTransaction;
-import org.qortal.transaction.Transaction;
 import org.qortal.utils.Base58;
 
+import java.text.NumberFormat;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 public class ArbitraryDataCacheManager extends Thread {
 
@@ -29,6 +37,11 @@ public class ArbitraryDataCacheManager extends Thread {
     /** Queue of arbitrary transactions that require cache updates */
     private final List<ArbitraryTransactionData> updateQueue = Collections.synchronizedList(new ArrayList<>());
 
+    private static final NumberFormat FORMATTER = NumberFormat.getNumberInstance();
+
+    static {
+        FORMATTER.setGroupingUsed(true);
+    }
 
     public static synchronized ArbitraryDataCacheManager getInstance() {
         if (instance == null) {
@@ -45,17 +58,22 @@ public class ArbitraryDataCacheManager extends Thread {
 
         try {
             while (!Controller.isStopping()) {
-                Thread.sleep(500L);
+                try {
+                    Thread.sleep(500L);
 
-                // Process queue
-                processResourceQueue();
+                    // Process queue
+                    processResourceQueue();
+                } catch (Exception e) {
+                    LOGGER.error(e.getMessage(), e);
+                    Thread.sleep(600_000L); // wait 10 minutes to continue
+                }
             }
-        } catch (InterruptedException e) {
-            // Fall through to exit thread
-        }
 
-        // Clear queue before terminating thread
-        processResourceQueue();
+            // Clear queue before terminating thread
+            processResourceQueue();
+        } catch (Exception e) {
+            LOGGER.error(e.getMessage(), e);
+        }
     }
 
     public void shutdown() {
@@ -85,14 +103,25 @@ public class ArbitraryDataCacheManager extends Thread {
                 // Update arbitrary resource caches
                 try {
                     ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
-                    arbitraryTransaction.updateArbitraryResourceCache(repository);
-                    arbitraryTransaction.updateArbitraryMetadataCache(repository);
+                    arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0));
                     repository.saveChanges();
 
                     // Update status as separate commit, as this is more prone to failure
                     arbitraryTransaction.updateArbitraryResourceStatus(repository);
                     repository.saveChanges();
 
+                    EventBus.INSTANCE.notify(
+                        new DataMonitorEvent(
+                            System.currentTimeMillis(),
+                            transactionData.getIdentifier(),
+                            transactionData.getName(),
+                            transactionData.getService().name(),
+                            "updated resource cache and status, queue",
+                            transactionData.getTimestamp(),
+                            transactionData.getTimestamp()
+                        )
+                    );
+
                     LOGGER.debug(() -> String.format("Finished processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature())));
 
                 } catch (DataException e) {
@@ -103,6 +132,9 @@ public class ArbitraryDataCacheManager extends Thread {
         } catch (DataException e) {
             LOGGER.error("Repository issue while processing arbitrary resource cache updates", e);
         }
+        catch (Exception e) {
+            LOGGER.error(e.getMessage(), e);
+        }
     }
 
     public void addToUpdateQueue(ArbitraryTransactionData transactionData) {
@@ -148,34 +180,66 @@ public class ArbitraryDataCacheManager extends Thread {
             LOGGER.info("Building arbitrary resources cache...");
             SplashFrame.getInstance().updateStatus("Building QDN cache - please wait...");
 
-            final int batchSize = 100;
+            final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
             int offset = 0;
 
+            List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder
+                    = repository.getArbitraryRepository().getLatestArbitraryTransactions();
+
+            LOGGER.info("arbitrary transactions: count = " + allArbitraryTransactionsInDescendingOrder.size());
+
+            List<ArbitraryResourceData> resources = repository.getArbitraryRepository().getArbitraryResources(null, null, true);
+
+            Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper = new HashMap<>(resources.size());
+            for( ArbitraryResourceData resource : resources ) {
+                resourceByWrapper.put(
+                    new ArbitraryTransactionDataHashWrapper(resource.service.value, resource.name, resource.identifier),
+                    resource
+                );
+            }
+
+            LOGGER.info("arbitrary resources: count = " + resourceByWrapper.size());
+
+            Set<ArbitraryTransactionDataHashWrapper> latestTransactionsWrapped = new HashSet<>(allArbitraryTransactionsInDescendingOrder.size());
+
             // Loop through all ARBITRARY transactions, and determine latest state
             while (!Controller.isStopping()) {
-                LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
+                LOGGER.info(
+                    "Fetching arbitrary transactions {} - {} / {} Total",
+                    FORMATTER.format(offset),
+                    FORMATTER.format(offset+batchSize-1),
+                    FORMATTER.format(allArbitraryTransactionsInDescendingOrder.size())
+                );
 
-                List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false);
-                if (signatures.isEmpty()) {
+                List<ArbitraryTransactionData> transactionsToProcess
+                    = allArbitraryTransactionsInDescendingOrder.stream()
+                        .skip(offset)
+                        .limit(batchSize)
+                        .collect(Collectors.toList());
+
+                if (transactionsToProcess.isEmpty()) {
                     // Complete
                     break;
                 }
 
-                // Expand signatures to transactions
-                for (byte[] signature : signatures) {
-                    ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository
-                            .getTransactionRepository().fromSignature(signature);
+                try {
+                    for( ArbitraryTransactionData transactionData : transactionsToProcess) {
+                        if (transactionData.getService() == null) {
+                            // Unsupported service - ignore this resource
+                            continue;
+                        }
 
-                    if (transactionData.getService() == null) {
-                        // Unsupported service - ignore this resource
-                        continue;
+                        latestTransactionsWrapped.add(new ArbitraryTransactionDataHashWrapper(transactionData));
+
+                        // Update arbitrary resource caches
+                        ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
+                        arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, latestTransactionsWrapped, resourceByWrapper);
                     }
-
-                    // Update arbitrary resource caches
-                    ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
-                    arbitraryTransaction.updateArbitraryResourceCache(repository);
-                    arbitraryTransaction.updateArbitraryMetadataCache(repository);
                     repository.saveChanges();
+                } catch (DataException e) {
+                    repository.discardChanges();
+
+                    LOGGER.error(e.getMessage(), e);
                 }
                 offset += batchSize;
             }
@@ -193,6 +257,11 @@ public class ArbitraryDataCacheManager extends Thread {
             repository.discardChanges();
             throw new DataException("Build of arbitrary resources cache failed.");
         }
+        catch (Exception e) {
+            LOGGER.error(e.getMessage(), e);
+
+            return false;
+        }
     }
 
     private boolean refreshArbitraryStatuses(Repository repository) throws DataException {
@@ -200,27 +269,48 @@ public class ArbitraryDataCacheManager extends Thread {
             LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions...");
             SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait...");
 
-            final int batchSize = 100;
+            final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
             int offset = 0;
 
+            List<ArbitraryTransactionData> allHostedTransactions
+                = ArbitraryDataStorageManager.getInstance()
+                    .listAllHostedTransactions(repository, null, null);
+
             // Loop through all ARBITRARY transactions, and determine latest state
             while (!Controller.isStopping()) {
-                LOGGER.info("Fetching hosted transactions {} - {}", offset, offset+batchSize-1);
+                LOGGER.info(
+                    "Fetching hosted transactions {} - {} / {} Total",
+                    FORMATTER.format(offset),
+                    FORMATTER.format(offset+batchSize-1),
+                    FORMATTER.format(allHostedTransactions.size())
+                );
+
+                List<ArbitraryTransactionData> hostedTransactions
+                    = allHostedTransactions.stream()
+                        .skip(offset)
+                        .limit(batchSize)
+                        .collect(Collectors.toList());
 
-                List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, batchSize, offset);
                 if (hostedTransactions.isEmpty()) {
                     // Complete
                     break;
                 }
 
-                // Loop through hosted transactions
-                for (ArbitraryTransactionData transactionData : hostedTransactions) {
+                try {
+                    // Loop through hosted transactions
+                    for (ArbitraryTransactionData transactionData : hostedTransactions) {
 
-                    // Determine status and update cache
-                    ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
-                    arbitraryTransaction.updateArbitraryResourceStatus(repository);
+                        // Determine status and update cache
+                        ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
+                        arbitraryTransaction.updateArbitraryResourceStatus(repository);
+                    }
                     repository.saveChanges();
+                } catch (DataException e) {
+                    repository.discardChanges();
+
+                    LOGGER.error(e.getMessage(), e);
                 }
+
                 offset += batchSize;
             }
 
@@ -234,6 +324,11 @@ public class ArbitraryDataCacheManager extends Thread {
             repository.discardChanges();
             throw new DataException("Refresh of arbitrary resource statuses failed.");
         }
+        catch (Exception e) {
+            LOGGER.error(e.getMessage(), e);
+
+            return false;
+        }
     }
 
 }

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java

@@ -2,9 +2,10 @@ package org.qortal.controller.arbitrary;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
+import org.qortal.event.DataMonitorEvent;
+import org.qortal.event.EventBus;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
@@ -21,8 +22,12 @@ import java.nio.file.Paths;
 import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
 
 import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD;
 
@@ -77,6 +82,19 @@ public class ArbitraryDataCleanupManager extends Thread {
 		final int limit = 100;
 		int offset = 0;
 
+		List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			allArbitraryTransactionsInDescendingOrder
+					= repository.getArbitraryRepository()
+					.getLatestArbitraryTransactions();
+		} catch( Exception e) {
+			LOGGER.error(e.getMessage(), e);
+			allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
+		}
+
+		Set<ArbitraryTransactionData> processedTransactions = new HashSet<>();
+
 		try {
 			while (!isStopping) {
 				Thread.sleep(30000);
@@ -107,27 +125,31 @@ public class ArbitraryDataCleanupManager extends Thread {
 
 				// Any arbitrary transactions we want to fetch data for?
 				try (final Repository repository = RepositoryManager.getRepository()) {
-					List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
-					// LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
+					List<ArbitraryTransactionData> transactions = allArbitraryTransactionsInDescendingOrder.stream().skip(offset).limit(limit).collect(Collectors.toList());
 					if (isStopping) {
 						return;
 					}
 
-					if (signatures == null || signatures.isEmpty()) {
+					if (transactions == null || transactions.isEmpty()) {
 						offset = 0;
-						continue;
+						allArbitraryTransactionsInDescendingOrder
+								= repository.getArbitraryRepository()
+								.getLatestArbitraryTransactions();
+						transactions = allArbitraryTransactionsInDescendingOrder.stream().limit(limit).collect(Collectors.toList());
+						processedTransactions.clear();
 					}
+
 					offset += limit;
 					now = NTP.getTime();
 
 					// Loop through the signatures in this batch
-					for (int i=0; i<signatures.size(); i++) {
+					for (int i=0; i<transactions.size(); i++) {
 						if (isStopping) {
 							return;
 						}
 
-						byte[] signature = signatures.get(i);
-						if (signature == null) {
+						ArbitraryTransactionData arbitraryTransactionData = transactions.get(i);
+						if (arbitraryTransactionData == null) {
 							continue;
 						}
 
@@ -136,9 +158,7 @@ public class ArbitraryDataCleanupManager extends Thread {
 							Thread.sleep(5000);
 						}
 
-						// Fetch the transaction data
-						ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
-						if (arbitraryTransactionData == null || arbitraryTransactionData.getService() == null) {
+						if (arbitraryTransactionData.getService() == null) {
 							continue;
 						}
 
@@ -147,6 +167,8 @@ public class ArbitraryDataCleanupManager extends Thread {
 							continue;
 						}
 
+						boolean mostRecentTransaction = processedTransactions.add(arbitraryTransactionData);
+
 						// Check if we have the complete file
 						boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData);
 
@@ -167,20 +189,54 @@ public class ArbitraryDataCleanupManager extends Thread {
 							LOGGER.info("Deleting transaction {} because we can't host its data",
 									Base58.encode(arbitraryTransactionData.getSignature()));
 							ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
+
+							EventBus.INSTANCE.notify(
+								new DataMonitorEvent(
+									System.currentTimeMillis(),
+									arbitraryTransactionData.getIdentifier(),
+									arbitraryTransactionData.getName(),
+									arbitraryTransactionData.getService().name(),
+									"can't store data, deleting",
+									arbitraryTransactionData.getTimestamp(),
+									arbitraryTransactionData.getTimestamp()
+								)
+							);
 							continue;
 						}
 
 						// Check to see if we have had a more recent PUT
-						boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
-						if (hasMoreRecentPutTransaction) {
+						if (!mostRecentTransaction) {
 							// There is a more recent PUT transaction than the one we are currently processing.
 							// When a PUT is issued, it replaces any layers that would have been there before.
 							// Therefore any data relating to this older transaction is no longer needed.
 							LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " +
 											"Deleting all files associated with the earlier transaction.", arbitraryTransactionData.getService(),
-									arbitraryTransactionData.getName(), Base58.encode(signature)));
+									arbitraryTransactionData.getName(), Base58.encode(arbitraryTransactionData.getSignature())));
 
 							ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
+
+							Optional<ArbitraryTransactionData> moreRecentPutTransaction
+								= processedTransactions.stream()
+									.filter(data -> data.equals(arbitraryTransactionData))
+									.findAny();
+
+							if( moreRecentPutTransaction.isPresent() ) {
+								EventBus.INSTANCE.notify(
+									new DataMonitorEvent(
+										System.currentTimeMillis(),
+										arbitraryTransactionData.getIdentifier(),
+										arbitraryTransactionData.getName(),
+										arbitraryTransactionData.getService().name(),
+										"deleting data due to replacement",
+										arbitraryTransactionData.getTimestamp(),
+										moreRecentPutTransaction.get().getTimestamp()
+									)
+								);
+							}
+							else {
+								LOGGER.warn("Something went wrong with the most recent put transaction determination!");
+							}
+
 							continue;
 						}
 
@@ -199,7 +255,21 @@ public class ArbitraryDataCleanupManager extends Thread {
 							LOGGER.debug(String.format("Transaction %s has complete file and all chunks",
 									Base58.encode(arbitraryTransactionData.getSignature())));
 
-							ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
+							boolean wasDeleted = ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
+
+							if( wasDeleted ) {
+								EventBus.INSTANCE.notify(
+									new DataMonitorEvent(
+										System.currentTimeMillis(),
+										arbitraryTransactionData.getIdentifier(),
+										arbitraryTransactionData.getName(),
+										arbitraryTransactionData.getService().name(),
+										"deleting file, retaining chunks",
+										arbitraryTransactionData.getTimestamp(),
+										arbitraryTransactionData.getTimestamp()
+									)
+								);
+							}
 							continue;
 						}
 
@@ -237,17 +307,6 @@ public class ArbitraryDataCleanupManager extends Thread {
 						this.storageLimitReached(repository);
 					}
 
-					// Delete random data associated with name if we're over our storage limit for this name
-					// Use the DELETION_THRESHOLD, for the same reasons as above
-					for (String followedName : ListUtils.followedNames()) {
-						if (isStopping) {
-							return;
-						}
-						if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) {
-							this.storageLimitReachedForName(repository, followedName);
-						}
-					}
-
 				} catch (DataException e) {
 					LOGGER.error("Repository issue when cleaning up arbitrary transaction data", e);
 				}
@@ -326,25 +385,6 @@ public class ArbitraryDataCleanupManager extends Thread {
 		// FUTURE: consider reducing the expiry time of the reader cache
 	}
 
-	public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException {
-		// We think that the storage limit has been reached for supplied name - but we should double check
-		if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) {
-			// We have space available for this name, so don't delete anything
-			return;
-		}
-
-		// Delete a batch of random chunks associated with this name
-		// This reduces the chance of too many nodes deleting the same chunk
-		// when they reach their storage limit
-		Path dataPath = Paths.get(Settings.getInstance().getDataPath());
-		for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
-			if (isStopping) {
-				return;
-			}
-			this.deleteRandomFile(repository, dataPath.toFile(), name);
-		}
-	}
-
 	/**
 	 * Iteratively walk through given directory and delete a single random file
 	 *
@@ -423,6 +463,7 @@ public class ArbitraryDataCleanupManager extends Thread {
 				}
 
 				LOGGER.info("Deleting random file {} because we have reached max storage capacity...", randomItem.toString());
+				fireRandomItemDeletionNotification(randomItem, repository, "Deleting random file, because we have reached max storage capacity");
 				boolean success = randomItem.delete();
 				if (success) {
 					try {
@@ -437,6 +478,35 @@ public class ArbitraryDataCleanupManager extends Thread {
 		return false;
 	}
 
+	private void fireRandomItemDeletionNotification(File randomItem, Repository repository, String reason) {
+		try {
+			Path parentFileNamePath = randomItem.toPath().toAbsolutePath().getParent().getFileName();
+			if (parentFileNamePath != null) {
+				String signature58 = parentFileNamePath.toString();
+				byte[] signature = Base58.decode(signature58);
+				TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+				if (transactionData != null && transactionData.getType() == Transaction.TransactionType.ARBITRARY) {
+					ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+
+					EventBus.INSTANCE.notify(
+						new DataMonitorEvent(
+							System.currentTimeMillis(),
+							arbitraryTransactionData.getIdentifier(),
+							arbitraryTransactionData.getName(),
+							arbitraryTransactionData.getService().name(),
+							reason,
+							arbitraryTransactionData.getTimestamp(),
+							arbitraryTransactionData.getTimestamp()
+						)
+					);
+				}
+			}
+
+		} catch (Exception e) {
+			LOGGER.error(e.getMessage(), e);
+		}
+	}
+
 	private void cleanupTempDirectory(String folder, long now, long minAge) {
 		String baseDir = Settings.getInstance().getTempDataPath();
 		Path tempDir = Paths.get(baseDir, folder);

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataExamination.java

@@ -0,0 +1,21 @@
+package org.qortal.controller.arbitrary;
+
+public class ArbitraryDataExamination {
+
+    private boolean pass;
+
+    private String notes;
+
+    public ArbitraryDataExamination(boolean pass, String notes) {
+        this.pass = pass;
+        this.notes = notes;
+    }
+
+    public boolean isPass() {
+        return pass;
+    }
+
+    public String getNotes() {
+        return notes;
+    }
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java

@@ -5,6 +5,8 @@ import org.apache.logging.log4j.Logger;
 import org.qortal.controller.Controller;
 import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo;
 import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.event.DataMonitorEvent;
+import org.qortal.event.EventBus;
 import org.qortal.network.Peer;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -10,6 +10,8 @@ import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.Controller;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
+import org.qortal.event.DataMonitorEvent;
+import org.qortal.event.EventBus;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
 import org.qortal.repository.DataException;
@@ -28,6 +30,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
+import java.util.stream.Collectors;
 
 public class ArbitraryDataManager extends Thread {
 
@@ -195,13 +198,35 @@ public class ArbitraryDataManager extends Thread {
 		final int limit = 100;
 		int offset = 0;
 
+		List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+
+			if( name == null ) {
+				allArbitraryTransactionsInDescendingOrder
+						= repository.getArbitraryRepository()
+						.getLatestArbitraryTransactions();
+			}
+			else {
+				allArbitraryTransactionsInDescendingOrder
+						= repository.getArbitraryRepository()
+						.getLatestArbitraryTransactionsByName(name);
+			}
+		} catch( Exception e) {
+			LOGGER.error(e.getMessage(), e);
+			allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
+		}
+
+		// collect processed transactions in a set to ensure outdated data transactions do not get fetched
+		Set<ArbitraryTransactionDataHashWrapper> processedTransactions = new HashSet<>();
+
 		while (!isStopping) {
 			Thread.sleep(1000L);
 
 			// Any arbitrary transactions we want to fetch data for?
 			try (final Repository repository = RepositoryManager.getRepository()) {
-				List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true);
-				// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
+				List<byte[]> signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions);
+
 				if (signatures == null || signatures.isEmpty()) {
 					offset = 0;
 					break;
@@ -223,14 +248,38 @@ public class ArbitraryDataManager extends Thread {
 					ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
 
 					// Skip transactions that we don't need to proactively store data for
-					if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) {
+					ArbitraryDataExamination arbitraryDataExamination = storageManager.shouldPreFetchData(repository, arbitraryTransactionData);
+					if (!arbitraryDataExamination.isPass()) {
 						iterator.remove();
+
+						EventBus.INSTANCE.notify(
+							new DataMonitorEvent(
+								System.currentTimeMillis(),
+								arbitraryTransactionData.getIdentifier(),
+								arbitraryTransactionData.getName(),
+								arbitraryTransactionData.getService().name(),
+								arbitraryDataExamination.getNotes(),
+								arbitraryTransactionData.getTimestamp(),
+								arbitraryTransactionData.getTimestamp()
+							)
+						);
 						continue;
 					}
 
 					// Remove transactions that we already have local data for
 					if (hasLocalData(arbitraryTransaction)) {
 						iterator.remove();
+						EventBus.INSTANCE.notify(
+							new DataMonitorEvent(
+								System.currentTimeMillis(),
+								arbitraryTransactionData.getIdentifier(),
+								arbitraryTransactionData.getName(),
+								arbitraryTransactionData.getService().name(),
+								"already have local data, skipping",
+								arbitraryTransactionData.getTimestamp(),
+								arbitraryTransactionData.getTimestamp()
+							)
+						);
                     }
 				}
 
@@ -248,8 +297,21 @@ public class ArbitraryDataManager extends Thread {
 
 				// Check to see if we have had a more recent PUT
 				ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
-				boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
-				if (hasMoreRecentPutTransaction) {
+
+				Optional<ArbitraryTransactionData> moreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
+
+				if (moreRecentPutTransaction.isPresent()) {
+					EventBus.INSTANCE.notify(
+						new DataMonitorEvent(
+							System.currentTimeMillis(),
+							arbitraryTransactionData.getIdentifier(),
+							arbitraryTransactionData.getName(),
+							arbitraryTransactionData.getService().name(),
+							"not fetching old data",
+							arbitraryTransactionData.getTimestamp(),
+							moreRecentPutTransaction.get().getTimestamp()
+						)
+					);
 					// There is a more recent PUT transaction than the one we are currently processing.
 					// When a PUT is issued, it replaces any layers that would have been there before.
 					// Therefore any data relating to this older transaction is no longer needed and we
@@ -257,10 +319,34 @@ public class ArbitraryDataManager extends Thread {
 					continue;
 				}
 
+				EventBus.INSTANCE.notify(
+					new DataMonitorEvent(
+						System.currentTimeMillis(),
+						arbitraryTransactionData.getIdentifier(),
+						arbitraryTransactionData.getName(),
+						arbitraryTransactionData.getService().name(),
+						"fetching data",
+						arbitraryTransactionData.getTimestamp(),
+						arbitraryTransactionData.getTimestamp()
+					)
+				);
+
 				// Ask our connected peers if they have files for this signature
 				// This process automatically then fetches the files themselves if a peer is found
 				fetchData(arbitraryTransactionData);
 
+				EventBus.INSTANCE.notify(
+					new DataMonitorEvent(
+						System.currentTimeMillis(),
+						arbitraryTransactionData.getIdentifier(),
+						arbitraryTransactionData.getName(),
+						arbitraryTransactionData.getService().name(),
+						"fetched data",
+						arbitraryTransactionData.getTimestamp(),
+						arbitraryTransactionData.getTimestamp()
+					)
+				);
+
 			} catch (DataException e) {
 				LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
 			}
@@ -274,6 +360,20 @@ public class ArbitraryDataManager extends Thread {
 		final int limit = 100;
 		int offset = 0;
 
+		List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			allArbitraryTransactionsInDescendingOrder
+					= repository.getArbitraryRepository()
+						.getLatestArbitraryTransactions();
+		} catch( Exception e) {
+			LOGGER.error(e.getMessage(), e);
+			allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
+		}
+
+		// collect processed transactions in a set to ensure outdated data transactions do not get fetched
+		Set<ArbitraryTransactionDataHashWrapper> processedTransactions = new HashSet<>();
+
 		while (!isStopping) {
 			final int minSeconds = 3;
 			final int maxSeconds = 10;
@@ -282,8 +382,8 @@ public class ArbitraryDataManager extends Thread {
 
 			// Any arbitrary transactions we want to fetch data for?
 			try (final Repository repository = RepositoryManager.getRepository()) {
-				List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
-				// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
+				List<byte[]> signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions);
+
 				if (signatures == null || signatures.isEmpty()) {
 					offset = 0;
 					break;
@@ -328,26 +428,74 @@ public class ArbitraryDataManager extends Thread {
 					continue;
 				}
 
-				// Check to see if we have had a more recent PUT
+				// No longer need to see if we have had a more recent PUT since we compared the transactions to process
+				// to the transactions previously processed, so we can fetch the transactiondata, notify the event bus,
+				// fetch the metadata and notify the event bus again
 				ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
-				boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
-				if (hasMoreRecentPutTransaction) {
-					// There is a more recent PUT transaction than the one we are currently processing.
-					// When a PUT is issued, it replaces any layers that would have been there before.
-					// Therefore any data relating to this older transaction is no longer needed and we
-					// shouldn't fetch it from the network.
-					continue;
-				}
 
 				// Ask our connected peers if they have metadata for this signature
 				fetchMetadata(arbitraryTransactionData);
 
+				EventBus.INSTANCE.notify(
+					new DataMonitorEvent(
+						System.currentTimeMillis(),
+						arbitraryTransactionData.getIdentifier(),
+						arbitraryTransactionData.getName(),
+						arbitraryTransactionData.getService().name(),
+						"fetched metadata",
+						arbitraryTransactionData.getTimestamp(),
+						arbitraryTransactionData.getTimestamp()
+					)
+				);
 			} catch (DataException e) {
 				LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
+			} catch (Exception e) {
+				LOGGER.error(e.getMessage(), e);
 			}
 		}
 	}
 
+	private static List<byte[]> processTransactionsForSignatures(
+			int limit,
+			int offset,
+			List<ArbitraryTransactionData> transactionsInDescendingOrder,
+			Set<ArbitraryTransactionDataHashWrapper> processedTransactions) {
+		// these transactions are in descending order, latest transactions come first
+		List<ArbitraryTransactionData> transactions
+				= transactionsInDescendingOrder.stream()
+					.skip(offset)
+					.limit(limit)
+					.collect(Collectors.toList());
+
+		// wrap the transactions, so they can be used for hashing and comparing
+		// Class ArbitraryTransactionDataHashWrapper supports hashCode() and equals(...) for this purpose
+		List<ArbitraryTransactionDataHashWrapper> wrappedTransactions
+				= transactions.stream()
+					.map(transaction -> new ArbitraryTransactionDataHashWrapper(transaction))
+					.collect(Collectors.toList());
+
+		// create a set of wrappers and populate it first to last, so that all outdated transactions get rejected
+		Set<ArbitraryTransactionDataHashWrapper> transactionsToProcess = new HashSet<>(wrappedTransactions.size());
+		for(ArbitraryTransactionDataHashWrapper wrappedTransaction : wrappedTransactions) {
+			transactionsToProcess.add(wrappedTransaction);
+		}
+
+		// remove the matches for previously processed transactions,
+		// because these transactions have had updates that have already been processed
+		transactionsToProcess.removeAll(processedTransactions);
+
+		// add to processed transactions to compare and remove matches from future processing iterations
+		processedTransactions.addAll(transactionsToProcess);
+
+		List<byte[]> signatures
+				= transactionsToProcess.stream()
+					.map(transactionToProcess -> transactionToProcess.getData()
+					.getSignature())
+					.collect(Collectors.toList());
+
+		return signatures;
+	}
+
 	private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) {
 		try {
 			TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java

@@ -155,31 +155,24 @@ public class ArbitraryDataStorageManager extends Thread {
      * @param arbitraryTransactionData - the transaction
      * @return boolean - whether to prefetch or not
      */
-    public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
+    public ArbitraryDataExamination shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
         String name = arbitraryTransactionData.getName();
 
         // Only fetch data associated with hashes, as we already have RAW_DATA
         if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) {
-            return false;
+            return new ArbitraryDataExamination(false, "Only fetch data associated with hashes");
         }
 
         // Don't fetch anything more if we're (nearly) out of space
         // Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
         // avoid a fetch/delete loop
         if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) {
-            return false;
-        }
-
-        // Don't fetch anything if we're (nearly) out of space for this name
-        // Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
-        // avoid a fetch/delete loop
-        if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
-            return false;
+            return new ArbitraryDataExamination(false,"Don't fetch anything more if we're (nearly) out of space");
         }
 
         // Don't store data unless it's an allowed type (public/private)
         if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
-            return false;
+            return new ArbitraryDataExamination(false, "Don't store data unless it's an allowed type (public/private)");
         }
 
         // Handle transactions without names differently
@@ -189,21 +182,21 @@ public class ArbitraryDataStorageManager extends Thread {
 
         // Never fetch data from blocked names, even if they are followed
         if (ListUtils.isNameBlocked(name)) {
-            return false;
+            return new ArbitraryDataExamination(false, "blocked name");
         }
 
         switch (Settings.getInstance().getStoragePolicy()) {
             case FOLLOWED:
             case FOLLOWED_OR_VIEWED:
-                return ListUtils.isFollowingName(name);
+                return new ArbitraryDataExamination(ListUtils.isFollowingName(name), Settings.getInstance().getStoragePolicy().name());
                 
             case ALL:
-                return true;
+                return new ArbitraryDataExamination(true, Settings.getInstance().getStoragePolicy().name());
 
             case NONE:
             case VIEWED:
             default:
-                return false;
+                return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
         }
     }
 
@@ -214,17 +207,17 @@ public class ArbitraryDataStorageManager extends Thread {
      *
      * @return boolean - whether the storage policy allows for unnamed data
      */
-    private boolean shouldPreFetchDataWithoutName() {
+    private ArbitraryDataExamination shouldPreFetchDataWithoutName() {
         switch (Settings.getInstance().getStoragePolicy()) {
             case ALL:
-                return true;
+                return new ArbitraryDataExamination(true, "Fetching all data");
 
             case NONE:
             case VIEWED:
             case FOLLOWED:
             case FOLLOWED_OR_VIEWED:
             default:
-                return false;
+                return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
         }
     }
 
@@ -484,51 +477,6 @@ public class ArbitraryDataStorageManager extends Thread {
         return true;
     }
 
-    public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) {
-        if (!this.isStorageSpaceAvailable(threshold)) {
-            // No storage space available at all, so no need to check this name
-            return false;
-        }
-
-        if (Settings.getInstance().getStoragePolicy() == StoragePolicy.ALL) {
-            // Using storage policy ALL, so don't limit anything per name
-            return true;
-        }
-
-        if (name == null) {
-            // This transaction doesn't have a name, so fall back to total space limitations
-            return true;
-        }
-
-        int followedNamesCount = ListUtils.followedNamesCount();
-        if (followedNamesCount == 0) {
-            // Not following any names, so we have space
-            return true;
-        }
-
-        long totalSizeForName = 0;
-        long maxStoragePerName = this.storageCapacityPerName(threshold);
-
-        // Fetch all hosted transactions
-        List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository, null, null);
-        for (ArbitraryTransactionData transactionData : hostedTransactions) {
-            String transactionName = transactionData.getName();
-            if (!Objects.equals(name, transactionName)) {
-                // Transaction relates to a different name
-                continue;
-            }
-
-            totalSizeForName += transactionData.getSize();
-        }
-
-        // Have we reached the limit for this name?
-        if (totalSizeForName > maxStoragePerName) {
-            return false;
-        }
-
-        return true;
-    }
-
     public long storageCapacityPerName(double threshold) {
         int followedNamesCount = ListUtils.followedNamesCount();
         if (followedNamesCount == 0) {

src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java

@@ -0,0 +1,48 @@
+package org.qortal.controller.arbitrary;
+
+import org.qortal.arbitrary.misc.Service;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+
+import java.util.Objects;
+
+public class ArbitraryTransactionDataHashWrapper {
+
+    private ArbitraryTransactionData data;
+
+    private int service;
+
+    private String name;
+
+    private String identifier;
+
+    public ArbitraryTransactionDataHashWrapper(ArbitraryTransactionData data) {
+        this.data = data;
+
+        this.service = data.getService().value;
+        this.name = data.getName();
+        this.identifier = data.getIdentifier();
+    }
+
+    public ArbitraryTransactionDataHashWrapper(int service, String name, String identifier) {
+        this.service = service;
+        this.name = name;
+        this.identifier = identifier;
+    }
+
+    public ArbitraryTransactionData getData() {
+        return data;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ArbitraryTransactionDataHashWrapper that = (ArbitraryTransactionDataHashWrapper) o;
+        return service == that.service && name.equals(that.name) && Objects.equals(identifier, that.identifier);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(service, name, identifier);
+    }
+}

src/main/java/org/qortal/controller/arbitrary/RNSArbitraryDataFileListManager.java

@@ -0,0 +1,731 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.arbitrary.ArbitraryDataFileChunk;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.RNSArbitraryDirectConnectionInfo;
+import org.qortal.data.arbitrary.RNSArbitraryFileListResponseInfo;
+import org.qortal.data.arbitrary.RNSArbitraryRelayInfo;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.ArbitraryDataFileListMessage;
+import org.qortal.network.message.GetArbitraryDataFileListMessage;
+import org.qortal.network.message.Message;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+import org.qortal.utils.ListUtils;
+import org.qortal.utils.NTP;
+import org.qortal.utils.Triple;
+
+import java.util.*;
+
+import static org.qortal.controller.arbitrary.RNSArbitraryDataFileManager.MAX_FILE_HASH_RESPONSES;
+
+public class RNSArbitraryDataFileListManager {
+
+    private static final Logger LOGGER = LogManager.getLogger(RNSArbitraryDataFileListManager.class);
+
+    private static RNSArbitraryDataFileListManager instance;
+
+    private static String MIN_PEER_VERSION_FOR_FILE_LIST_STATS = "3.2.0";
+
+    /**
+     * Map of recent incoming requests for ARBITRARY transaction data file lists.
+     * <p>
+     * Key is original request's message ID<br>
+     * Value is Triple&lt;transaction signature in base58, first requesting peer, first request's timestamp&gt;
+     * <p>
+     * If peer is null then either:<br>
+     * <ul>
+     * <li>we are the original requesting peer</li>
+     * <li>we have already sent data payload to original requesting peer.</li>
+     * </ul>
+     * If signature is null then we have already received the file list and either:<br>
+     * <ul>
+     * <li>we are the original requesting peer and have processed it</li>
+     * <li>we have forwarded the file list</li>
+     * </ul>
+     */
+    public Map<Integer, Triple<String, RNSPeer, Long>> arbitraryDataFileListRequests = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of in progress arbitrary data signature requests
+     * Key: string - the signature encoded in base58
+     * Value: Triple<networkBroadcastCount, directPeerRequestCount, lastAttemptTimestamp>
+     */
+    private Map<String, Triple<Integer, Integer, Long>> arbitraryDataSignatureRequests = Collections.synchronizedMap(new HashMap<>());
+
+
+    /** Maximum number of seconds that a file list relay request is able to exist on the network */
+    public static long RELAY_REQUEST_MAX_DURATION = 5000L;
+    /** Maximum number of hops that a file list relay request is allowed to make */
+    public static int RELAY_REQUEST_MAX_HOPS = 4;
+
+    /** Minimum peer version to use relay */
+    public static String RELAY_MIN_PEER_VERSION = "3.4.0";
+
+
+    private RNSArbitraryDataFileListManager() {
+    }
+
+    public static RNSArbitraryDataFileListManager getInstance() {
+        if (instance == null)
+            instance = new RNSArbitraryDataFileListManager();
+
+        return instance;
+    }
+
+
+    public void cleanupRequestCache(Long now) {
+        if (now == null) {
+            return;
+        }
+        final long requestMinimumTimestamp = now - ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT;
+        arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < requestMinimumTimestamp);
+    }
+
+
+    // Track file list lookups by signature
+
+    private boolean shouldMakeFileListRequestForSignature(String signature58) {
+        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        // Extract the components
+        Integer networkBroadcastCount = request.getA();
+        // Integer directPeerRequestCount = request.getB();
+        Long lastAttemptTimestamp = request.getC();
+
+        if (lastAttemptTimestamp == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
+
+        // Allow a second attempt after 15 seconds, and another after 30 seconds
+        if (timeSinceLastAttempt > 15 * 1000L) {
+            // We haven't tried for at least 15 seconds
+
+            if (networkBroadcastCount < 3) {
+                // We've made less than 3 total attempts
+                return true;
+            }
+        }
+
+        // Then allow another 5 attempts, each 1 minute apart
+        if (timeSinceLastAttempt > 60 * 1000L) {
+            // We haven't tried for at least 1 minute
+
+            if (networkBroadcastCount < 8) {
+                // We've made less than 8 total attempts
+                return true;
+            }
+        }
+
+        // Then allow another 8 attempts, each 15 minutes apart
+        if (timeSinceLastAttempt > 15 * 60 * 1000L) {
+            // We haven't tried for at least 15 minutes
+
+            if (networkBroadcastCount < 16) {
+                // We've made less than 16 total attempts
+                return true;
+            }
+        }
+
+        // From then on, only try once every 6 hours, to reduce network spam
+        if (timeSinceLastAttempt > 6 * 60 * 60 * 1000L) {
+            // We haven't tried for at least 6 hours
+            return true;
+        }
+
+        return false;
+    }
+
+    private boolean shouldMakeDirectFileRequestsForSignature(String signature58) {
+        if (!Settings.getInstance().isDirectDataRetrievalEnabled()) {
+            // Direct connections are disabled in the settings
+            return false;
+        }
+
+        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        // Extract the components
+        //Integer networkBroadcastCount = request.getA();
+        Integer directPeerRequestCount = request.getB();
+        Long lastAttemptTimestamp = request.getC();
+
+        if (lastAttemptTimestamp == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        if (directPeerRequestCount == 0) {
+            // We haven't tried asking peers directly yet, so we should
+            return true;
+        }
+
+        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
+        if (timeSinceLastAttempt > 10 * 1000L) {
+            // We haven't tried for at least 10 seconds
+            if (directPeerRequestCount < 5) {
+                // We've made less than 5 total attempts
+                return true;
+            }
+        }
+
+        if (timeSinceLastAttempt > 5 * 60 * 1000L) {
+            // We haven't tried for at least 5 minutes
+            if (directPeerRequestCount < 10) {
+                // We've made less than 10 total attempts
+                return true;
+            }
+        }
+
+        if (timeSinceLastAttempt > 60 * 60 * 1000L) {
+            // We haven't tried for at least 1 hour
+            return true;
+        }
+
+        return false;
+    }
+
+    public boolean isSignatureRateLimited(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        return !this.shouldMakeFileListRequestForSignature(signature58)
+                && !this.shouldMakeDirectFileRequestsForSignature(signature58);
+    }
+
+    public long lastRequestForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        Triple<Integer, Integer, Long> request = arbitraryDataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return 0;
+        }
+
+        // Extract the components
+        Long lastAttemptTimestamp = request.getC();
+        if (lastAttemptTimestamp != null) {
+            return  lastAttemptTimestamp;
+        }
+        return 0;
+    }
+
+    public void addToSignatureRequests(String signature58, boolean incrementNetworkRequests, boolean incrementPeerRequests) {
+        Triple<Integer, Integer, Long> request  = arbitraryDataSignatureRequests.get(signature58);
+        Long now = NTP.getTime();
+
+        if (request == null) {
+            // No entry yet
+            Triple<Integer, Integer, Long> newRequest = new Triple<>(0, 0, now);
+            arbitraryDataSignatureRequests.put(signature58, newRequest);
+        }
+        else {
+            // There is an existing entry
+            if (incrementNetworkRequests) {
+                request.setA(request.getA() + 1);
+            }
+            if (incrementPeerRequests) {
+                request.setB(request.getB() + 1);
+            }
+            request.setC(now);
+            arbitraryDataSignatureRequests.put(signature58, request);
+        }
+    }
+
+    public void removeFromSignatureRequests(String signature58) {
+        arbitraryDataSignatureRequests.remove(signature58);
+    }
+
+
+    // Lookup file lists by signature (and optionally hashes)
+
+    public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) {
+        byte[] signature = arbitraryTransactionData.getSignature();
+        String signature58 = Base58.encode(signature);
+
+        // Require an NTP sync
+        Long now = NTP.getTime();
+        if (now == null) {
+            return false;
+        }
+
+        // If we've already tried too many times in a short space of time, make sure to give up
+        if (!this.shouldMakeFileListRequestForSignature(signature58)) {
+            // Check if we should make direct connections to peers
+            if (this.shouldMakeDirectFileRequestsForSignature(signature58)) {
+                return RNSArbitraryDataFileManager.getInstance().fetchDataFilesFromPeersForSignature(signature);
+            }
+
+            LOGGER.trace("Skipping file list request for signature {} due to rate limit", signature58);
+            return false;
+        }
+        this.addToSignatureRequests(signature58, true, false);
+
+        //List<Peer> handshakedPeers = Network.getInstance().getImmutableHandshakedPeers();
+        List<RNSPeer> handshakedPeers = RNSNetwork.getInstance().getLinkedPeers();
+        List<byte[]> missingHashes = null;
+
+        // Find hashes that we are missing
+        try {
+            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
+            missingHashes = arbitraryDataFile.missingHashes();
+        } catch (DataException e) {
+            // Leave missingHashes as null, so that all hashes are requested
+        }
+        int hashCount = missingHashes != null ? missingHashes.size() : 0;
+
+        LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size()));
+
+        //// Send our address as requestingPeer, to allow for potential direct connections with seeds/peers
+        //String requestingPeer = Network.getInstance().getOurExternalIpAddressAndPort();
+        String requestingPeer = null;
+
+        // Build request
+        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, missingHashes, now, 0, requestingPeer);
+
+        // Save our request into requests map
+        Triple<String, RNSPeer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
+
+        // Assign random ID to this message
+        int id;
+        do {
+            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
+
+            // Put queue into map (keyed by message ID) so we can poll for a response
+            // If putIfAbsent() doesn't return null, then this ID is already taken
+        } while (arbitraryDataFileListRequests.put(id, requestEntry) != null);
+        getArbitraryDataFileListMessage.setId(id);
+
+        // Broadcast request
+        RNSNetwork.getInstance().broadcast(peer -> getArbitraryDataFileListMessage);
+
+        // Poll to see if data has arrived
+        final long singleWait = 100;
+        long totalWait = 0;
+        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
+            try {
+                Thread.sleep(singleWait);
+            } catch (InterruptedException e) {
+                break;
+            }
+
+            requestEntry = arbitraryDataFileListRequests.get(id);
+            if (requestEntry == null)
+                return false;
+
+            if (requestEntry.getA() == null)
+                break;
+
+            totalWait += singleWait;
+        }
+        return true;
+    }
+
+    public boolean fetchArbitraryDataFileList(RNSPeer peer, byte[] signature) {
+        String signature58 = Base58.encode(signature);
+
+        // Require an NTP sync
+        Long now = NTP.getTime();
+        if (now == null) {
+            return false;
+        }
+
+        int hashCount = 0;
+        LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to peer %s...", signature58, hashCount, peer));
+
+        // Build request
+        // Use a time in the past, so that the recipient peer doesn't try and relay it
+        // Also, set hashes to null since it's easier to request all hashes than it is to determine which ones we need
+        // This could be optimized in the future
+        long timestamp = now - 60000L;
+        List<byte[]> hashes = null;
+        Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, timestamp, 0, null);
+
+        // Save our request into requests map
+        Triple<String, RNSPeer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
+
+        // Assign random ID to this message
+        int id;
+        do {
+            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
+
+            // Put queue into map (keyed by message ID) so we can poll for a response
+            // If putIfAbsent() doesn't return null, then this ID is already taken
+        } while (arbitraryDataFileListRequests.put(id, requestEntry) != null);
+        getArbitraryDataFileListMessage.setId(id);
+
+        // Send the request
+        peer.sendMessage(getArbitraryDataFileListMessage);
+
+        // Poll to see if data has arrived
+        final long singleWait = 100;
+        long totalWait = 0;
+        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
+            try {
+                Thread.sleep(singleWait);
+            } catch (InterruptedException e) {
+                break;
+            }
+
+            requestEntry = arbitraryDataFileListRequests.get(id);
+            if (requestEntry == null)
+                return false;
+
+            if (requestEntry.getA() == null)
+                break;
+
+            totalWait += singleWait;
+        }
+        return true;
+    }
+
+    public void deleteFileListRequestsForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        for (Iterator<Map.Entry<Integer, Triple<String, RNSPeer, Long>>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) {
+            Map.Entry<Integer, Triple<String, RNSPeer, Long>> entry = it.next();
+            if (entry == null || entry.getKey() == null || entry.getValue() != null) {
+                continue;
+            }
+            if (Objects.equals(entry.getValue().getA(), signature58)) {
+                // Update requests map to reflect that we've received all chunks
+                Triple<String, RNSPeer, Long> newEntry = new Triple<>(null, null, entry.getValue().getC());
+                arbitraryDataFileListRequests.put(entry.getKey(), newEntry);
+            }
+        }
+    }
+
+    // Network handlers
+
+    public void onNetworkArbitraryDataFileListMessage(RNSPeer peer, Message message) {
+        // Don't process if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
+        LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
+
+        if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) {
+            long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime();
+            LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}",
+                    totalRequestTime, arbitraryDataFileListMessage.getRequestHops(),
+                    arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
+        }
+
+        // Do we have a pending request for this data?
+        Triple<String, RNSPeer, Long> request = arbitraryDataFileListRequests.get(message.getId());
+        if (request == null || request.getA() == null) {
+            return;
+        }
+        boolean isRelayRequest = (request.getB() != null);
+
+        // Does this message's signature match what we're expecting?
+        byte[] signature = arbitraryDataFileListMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        if (!request.getA().equals(signature58)) {
+            return;
+        }
+
+        List<byte[]> hashes = arbitraryDataFileListMessage.getHashes();
+        if (hashes == null || hashes.isEmpty()) {
+            return;
+        }
+
+        ArbitraryTransactionData arbitraryTransactionData = null;
+
+        // Check transaction exists and hashes are correct
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+            if (!(transactionData instanceof ArbitraryTransactionData))
+                return;
+
+            arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+
+//          // Load data file(s)
+//          ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
+//
+//			// Check all hashes exist
+//			for (byte[] hash : hashes) {
+//				//LOGGER.debug("Received hash {}", Base58.encode(hash));
+//				if (!arbitraryDataFile.containsChunk(hash)) {
+//					// Check the hash against the complete file
+//					if (!Arrays.equals(arbitraryDataFile.getHash(), hash)) {
+//						LOGGER.info("Received non-matching chunk hash {} for signature {}. This could happen if we haven't obtained the metadata file yet.", Base58.encode(hash), signature58);
+//						return;
+//					}
+//				}
+//			}
+
+            if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) {
+                Long now = NTP.getTime();
+
+                if (RNSArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.size() < MAX_FILE_HASH_RESPONSES) {
+                    // Keep track of the hashes this peer reports to have access to
+                    for (byte[] hash : hashes) {
+                        String hash58 = Base58.encode(hash);
+
+                        // Treat null request hops as 100, so that they are able to be sorted (and put to the end of the list)
+                        int requestHops = arbitraryDataFileListMessage.getRequestHops() != null ? arbitraryDataFileListMessage.getRequestHops() : 100;
+
+                        RNSArbitraryFileListResponseInfo responseInfo = new RNSArbitraryFileListResponseInfo(hash58, signature58,
+                                peer, now, arbitraryDataFileListMessage.getRequestTime(), requestHops);
+
+                        RNSArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.add(responseInfo);
+                    }
+                }
+
+                // Keep track of the source peer, for direct connections
+                if (arbitraryDataFileListMessage.getPeerAddress() != null) {
+                    RNSArbitraryDataFileManager.getInstance().addDirectConnectionInfoIfUnique(
+                            new RNSArbitraryDirectConnectionInfo(signature, arbitraryDataFileListMessage.getPeerAddress(), hashes, now));
+                }
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e);
+        }
+
+        // Forwarding
+        if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
+            boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
+            if (!isBlocked) {
+                RNSPeer requestingPeer = request.getB();
+                if (requestingPeer != null) {
+                    Long requestTime = arbitraryDataFileListMessage.getRequestTime();
+                    Integer requestHops = arbitraryDataFileListMessage.getRequestHops();
+
+                    // Add each hash to our local mapping so we know who to ask later
+                    Long now = NTP.getTime();
+                    for (byte[] hash : hashes) {
+                        String hash58 = Base58.encode(hash);
+                        RNSArbitraryRelayInfo relayInfo = new RNSArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops);
+                        RNSArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo);
+                    }
+
+                    // Bump requestHops if it exists
+                    if (requestHops != null) {
+                        requestHops++;
+                    }
+
+                    ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage;
+
+                    //// TODO - rework for Reticulum
+                    //// Remove optional parameters if the requesting peer doesn't support it yet
+                    //// A message with less statistical data is better than no message at all
+                    //if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
+                    //    forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+                    //} else {
+                    //    forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops,
+                    //            arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
+                    //}
+                    forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops,
+                            arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
+                    forwardArbitraryDataFileListMessage.setId(message.getId());
+
+                    // Forward to requesting peer
+                    LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer);
+                    //if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) {
+                    //    requestingPeer.disconnect("failed to forward arbitrary data file list");
+                    //}
+                    requestingPeer.sendMessage(forwardArbitraryDataFileListMessage);
+                }
+            }
+        }
+    }
+
+    public void onNetworkGetArbitraryDataFileListMessage(RNSPeer peer, Message message) {
+        // Don't respond if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        Controller.getInstance().stats.getArbitraryDataFileListMessageStats.requests.incrementAndGet();
+
+        GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message;
+        byte[] signature = getArbitraryDataFileListMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        Long now = NTP.getTime();
+        Triple<String, RNSPeer, Long> newEntry = new Triple<>(signature58, peer, now);
+
+        // If we've seen this request recently, then ignore
+        if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) {
+            LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58);
+            return;
+        }
+
+        List<byte[]> requestedHashes = getArbitraryDataFileListMessage.getHashes();
+        int hashCount = requestedHashes != null ? requestedHashes.size() : 0;
+        String requestingPeer = getArbitraryDataFileListMessage.getRequestingPeer();
+
+        if (requestingPeer != null) {
+            LOGGER.debug("Received hash list request with {} hashes from peer {} (requesting peer {}) for signature {}", hashCount, peer, requestingPeer, signature58);
+        }
+        else {
+            LOGGER.debug("Received hash list request with {} hashes from peer {} for signature {}", hashCount, peer, signature58);
+        }
+
+        List<byte[]> hashes = new ArrayList<>();
+        ArbitraryTransactionData transactionData = null;
+        boolean allChunksExist = false;
+        boolean hasMetadata = false;
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Firstly we need to lookup this file on chain to get a list of its hashes
+            transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature);
+            if (transactionData instanceof ArbitraryTransactionData) {
+
+                // Check if we're even allowed to serve data for this transaction
+                if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
+
+                    // Load file(s) and add any that exist to the list of hashes
+                    ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(transactionData);
+
+                    // If the peer didn't supply a hash list, we need to return all hashes for this transaction
+                    if (requestedHashes == null || requestedHashes.isEmpty()) {
+                        requestedHashes = new ArrayList<>();
+
+                        // Add the metadata file
+                        if (arbitraryDataFile.getMetadataHash() != null) {
+                            requestedHashes.add(arbitraryDataFile.getMetadataHash());
+                            hasMetadata = true;
+                        }
+
+                        // Add the chunk hashes
+                        if (!arbitraryDataFile.getChunkHashes().isEmpty()) {
+                            requestedHashes.addAll(arbitraryDataFile.getChunkHashes());
+                        }
+                        // Add complete file if there are no hashes
+                        else {
+                            requestedHashes.add(arbitraryDataFile.getHash());
+                        }
+                    }
+
+                    // Assume all chunks exists, unless one can't be found below
+                    allChunksExist = true;
+
+                    for (byte[] requestedHash : requestedHashes) {
+                        ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature);
+                        if (chunk.exists()) {
+                            hashes.add(chunk.getHash());
+                            //LOGGER.trace("Added hash {}", chunk.getHash58());
+                        } else {
+                            LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58());
+                            allChunksExist = false;
+                        }
+                    }
+                }
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e);
+        }
+
+        // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that,
+        // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam.
+        if (hasMetadata && hashes.size() == 1) {
+            hashes.clear();
+        }
+
+        // We should only respond if we have at least one hash
+        if (!hashes.isEmpty()) {
+
+            // Firstly we should keep track of the requesting peer, to allow for potential direct connections later
+            RNSArbitraryDataFileManager.getInstance().addRecentDataRequest(requestingPeer);
+
+            // We have all the chunks, so update requests map to reflect that we've sent it
+            // There is no need to keep track of the request, as we can serve all the chunks
+            if (allChunksExist) {
+                newEntry = new Triple<>(null, null, now);
+                arbitraryDataFileListRequests.put(message.getId(), newEntry);
+            }
+
+            //String ourAddress = RNSNetwork.getInstance().getOurExternalIpAddressAndPort();
+            String ourAddress = RNSNetwork.getInstance().getBaseDestination().getHexHash();
+            ArbitraryDataFileListMessage arbitraryDataFileListMessage;
+
+            // TODO: rework for Reticulum
+            // Remove optional parameters if the requesting peer doesn't support it yet
+            // A message with less statistical data is better than no message at all
+            //if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
+            //    arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+            //} else {
+            //    arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature,
+            //            hashes, NTP.getTime(), 0, ourAddress, true);
+            //}
+            arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+
+            arbitraryDataFileListMessage.setId(message.getId());
+
+            //if (!peer.sendMessage(arbitraryDataFileListMessage)) {
+            //    LOGGER.debug("Couldn't send list of hashes");
+            //    peer.disconnect("failed to send list of hashes");
+            //    return;
+            //}
+            peer.sendMessage(arbitraryDataFileListMessage);
+            LOGGER.debug("Sent list of hashes (count: {})", hashes.size());
+
+            if (allChunksExist) {
+                // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
+                LOGGER.debug("No need for any forwarding because file list request is fully served");
+                return;
+            }
+
+        }
+
+        // We may need to forward this request on
+        boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName()));
+        if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
+            // In relay mode - so ask our other peers if they have it
+
+            long requestTime = getArbitraryDataFileListMessage.getRequestTime();
+            int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1;
+            long totalRequestTime = now - requestTime;
+
+            if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
+                // Relay request hasn't timed out yet, so can potentially be rebroadcast
+                if (requestHops < RELAY_REQUEST_MAX_HOPS) {
+                    // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
+
+                    Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer);
+                    relayGetArbitraryDataFileListMessage.setId(message.getId());
+
+                    LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
+                    //Network.getInstance().broadcast(
+                    //        broadcastPeer ->
+                    //                !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null :
+                    //                broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryDataFileListMessage
+                    //);
+                    RNSNetwork.getInstance().broadcast(broadcastPeer -> relayGetArbitraryDataFileListMessage);
+
+                }
+                else {
+                    // This relay request has reached the maximum number of allowed hops
+                }
+            }
+            else {
+                // This relay request has timed out
+            }
+        }
+    }
+
+}

src/main/java/org/qortal/controller/arbitrary/RNSArbitraryDataFileManager.java

@@ -0,0 +1,639 @@
+package org.qortal.controller.arbitrary;
+
+import com.google.common.net.InetAddresses;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.RNSArbitraryDirectConnectionInfo;
+import org.qortal.data.arbitrary.RNSArbitraryFileListResponseInfo;
+import org.qortal.data.arbitrary.RNSArbitraryRelayInfo;
+import org.qortal.data.network.PeerData;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.*;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+
+import java.security.SecureRandom;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
+
+public class RNSArbitraryDataFileManager extends Thread {
+
+    private static final Logger LOGGER = LogManager.getLogger(RNSArbitraryDataFileManager.class);
+
+    private static RNSArbitraryDataFileManager instance;
+    private volatile boolean isStopping = false;
+
+
+    /**
+     * Map to keep track of our in progress (outgoing) arbitrary data file requests
+     */
+    public Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of hashes that we might need to relay
+     */
+    public final List<RNSArbitraryRelayInfo> arbitraryRelayMap = Collections.synchronizedList(new ArrayList<>());
+
+    /**
+     * List to keep track of any arbitrary data file hash responses
+     */
+    public final List<RNSArbitraryFileListResponseInfo> arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>());
+
+    /**
+     * List to keep track of peers potentially available for direct connections, based on recent requests
+     */
+    private final List<RNSArbitraryDirectConnectionInfo> directConnectionInfo = Collections.synchronizedList(new ArrayList<>());
+
+    /**
+     * Map to keep track of peers requesting QDN data that we hold.
+     * Key = peer address string, value = time of last request.
+     * This allows for additional "burst" connections beyond existing limits.
+     */
+    private Map<String, Long> recentDataRequests = Collections.synchronizedMap(new HashMap<>());
+
+
+    public static int MAX_FILE_HASH_RESPONSES = 1000;
+
+
+    private RNSArbitraryDataFileManager() {
+    }
+
+    public static RNSArbitraryDataFileManager getInstance() {
+        if (instance == null)
+            instance = new RNSArbitraryDataFileManager();
+
+        return instance;
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data File Manager");
+
+        try {
+            // Use a fixed thread pool to execute the arbitrary data file requests
+            int threadCount = 5;
+            ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount);
+            for (int i = 0; i < threadCount; i++) {
+                arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread());
+            }
+
+            while (!isStopping) {
+                // Nothing to do yet
+                Thread.sleep(1000);
+            }
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
+        }
+    }
+
+    public void shutdown() {
+        isStopping = true;
+        this.interrupt();
+    }
+
+
+    public void cleanupRequestCache(Long now) {
+        if (now == null) {
+            return;
+        }
+        final long requestMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_REQUEST_TIMEOUT;
+        arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < requestMinimumTimestamp);
+
+        final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT;
+        arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp);
+        arbitraryDataFileHashResponses.removeIf(entry -> entry.getTimestamp() < relayMinimumTimestamp);
+
+        final long directConnectionInfoMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_DIRECT_CONNECTION_INFO_TIMEOUT;
+        directConnectionInfo.removeIf(entry -> entry.getTimestamp() < directConnectionInfoMinimumTimestamp);
+
+        final long recentDataRequestMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RECENT_DATA_REQUESTS_TIMEOUT;
+        recentDataRequests.entrySet().removeIf(entry -> entry.getValue() < recentDataRequestMinimumTimestamp);
+    }
+
+
+
+    // Fetch data files by hash
+
+    public boolean fetchArbitraryDataFiles(Repository repository,
+                                           RNSPeer peer,
+                                           byte[] signature,
+                                           ArbitraryTransactionData arbitraryTransactionData,
+                                           List<byte[]> hashes) throws DataException {
+
+        // Load data file(s)
+        ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromTransactionData(arbitraryTransactionData);
+        boolean receivedAtLeastOneFile = false;
+
+        // Now fetch actual data from this peer
+        for (byte[] hash : hashes) {
+            if (isStopping) {
+                return false;
+            }
+            String hash58 = Base58.encode(hash);
+            if (!arbitraryDataFile.chunkExists(hash)) {
+                // Only request the file if we aren't already requesting it from someone else
+                if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
+                    LOGGER.debug("Requesting data file {} from peer {}", hash58, peer);
+                    Long startTime = NTP.getTime();
+                    ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, null, arbitraryTransactionData, signature, hash, null);
+                    Long endTime = NTP.getTime();
+                    if (receivedArbitraryDataFile != null) {
+                        LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFile.getHash58(), peer, (endTime-startTime));
+                        receivedAtLeastOneFile = true;
+
+                        // Remove this hash from arbitraryDataFileHashResponses now that we have received it
+                        arbitraryDataFileHashResponses.remove(hash58);
+                    }
+                    else {
+                        LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime));
+
+                        // Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it
+                        arbitraryDataFileHashResponses.remove(hash58);
+
+                        // Stop asking for files from this peer
+                        break;
+                    }
+                }
+                else {
+                    LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer);
+                }
+            }
+            else {
+                // Remove this hash from arbitraryDataFileHashResponses because we have a local copy
+                arbitraryDataFileHashResponses.remove(hash58);
+            }
+        }
+
+        if (receivedAtLeastOneFile) {
+            // Invalidate the hosted transactions cache as we are now hosting something new
+            ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache();
+
+            // Check if we have all the files we need for this transaction
+            if (arbitraryDataFile.allFilesExist()) {
+
+                // We have all the chunks for this transaction, so we should invalidate the transaction's name's
+                // data cache so that it is rebuilt the next time we serve it
+                ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData);
+            }
+        }
+
+        return receivedAtLeastOneFile;
+    }
+
+    private ArbitraryDataFile fetchArbitraryDataFile(RNSPeer peer, RNSPeer requestingPeer, ArbitraryTransactionData arbitraryTransactionData, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
+        ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature);
+        boolean fileAlreadyExists = existingFile.exists();
+        String hash58 = Base58.encode(hash);
+        ArbitraryDataFile arbitraryDataFile;
+
+        // Fetch the file if it doesn't exist locally
+        if (!fileAlreadyExists) {
+            LOGGER.debug(String.format("Fetching data file %.8s from peer %s", hash58, peer));
+            arbitraryDataFileRequests.put(hash58, NTP.getTime());
+            Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash);
+
+            Message response = null;
+            // TODO - revisit with RNS
+            try {
+                response = peer.getResponseWithTimeout(getArbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT);
+            } catch (InterruptedException e) {
+                // Will return below due to null response
+            }
+            arbitraryDataFileRequests.remove(hash58);
+            LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58));
+
+            // We may need to remove the file list request, if we have all the files for this transaction
+            this.handleFileListRequests(signature);
+
+            if (response == null) {
+                LOGGER.debug("Received null response from peer {}", peer);
+                return null;
+            }
+            if (response.getType() != MessageType.ARBITRARY_DATA_FILE) {
+                LOGGER.debug("Received response with invalid type: {} from peer {}", response.getType(), peer);
+                return null;
+            }
+
+            ArbitraryDataFileMessage peersArbitraryDataFileMessage = (ArbitraryDataFileMessage) response;
+            arbitraryDataFile = peersArbitraryDataFileMessage.getArbitraryDataFile();
+        } else {
+            LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58));
+            arbitraryDataFile = existingFile;
+        }
+
+        if (arbitraryDataFile == null) {
+            // We don't have a file, so give up here
+            return null;
+        }
+
+        // We might want to forward the request to the peer that originally requested it
+        this.handleArbitraryDataFileForwarding(requestingPeer, new ArbitraryDataFileMessage(signature, arbitraryDataFile), originalMessage);
+
+        boolean isRelayRequest = (requestingPeer != null);
+        if (isRelayRequest) {
+            if (!fileAlreadyExists) {
+                // File didn't exist locally before the request, and it's a forwarding request, so delete it if it exists.
+                // It shouldn't exist on the filesystem yet, but leaving this here just in case.
+                arbitraryDataFile.delete(10);
+            }
+        }
+        else {
+            arbitraryDataFile.save();
+        }
+
+        // If this is a metadata file then we need to update the cache
+        if (arbitraryTransactionData != null && arbitraryTransactionData.getMetadataHash() != null) {
+            if (Arrays.equals(arbitraryTransactionData.getMetadataHash(), hash)) {
+                ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
+            }
+        }
+
+        return arbitraryDataFile;
+    }
+
+    private void handleFileListRequests(byte[] signature) {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Fetch the transaction data
+            ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+            if (arbitraryTransactionData == null) {
+                return;
+            }
+
+            boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData);
+
+            if (allChunksExist) {
+                // Update requests map to reflect that we've received all chunks
+                RNSArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature);
+            }
+
+        } catch (DataException e) {
+            LOGGER.debug("Unable to handle file list requests: {}", e.getMessage());
+        }
+    }
+
+    public void handleArbitraryDataFileForwarding(RNSPeer requestingPeer, Message message, Message originalMessage) {
+        // Return if there is no originally requesting peer to forward to
+        if (requestingPeer == null) {
+            return;
+        }
+
+        // Return if we're not in relay mode or if this request doesn't need forwarding
+        if (!Settings.getInstance().isRelayModeEnabled()) {
+            return;
+        }
+
+        LOGGER.debug("Received arbitrary data file - forwarding is needed");
+
+        // The ID needs to match that of the original request
+        message.setId(originalMessage.getId());
+
+        //if (!requestingPeer.sendMessageWithTimeout(message, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT)) {
+        //    LOGGER.debug("Failed to forward arbitrary data file to peer {}", requestingPeer);
+        //    requestingPeer.disconnect("failed to forward arbitrary data file");
+        //}
+        //else {
+        //    LOGGER.debug("Forwarded arbitrary data file to peer {}", requestingPeer);
+        //}
+        requestingPeer.sendMessageWithTimeout(message, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT);
+    }
+
+
+    // Fetch data directly from peers
+
+    private List<RNSArbitraryDirectConnectionInfo> getDirectConnectionInfoForSignature(byte[] signature) {
+        synchronized (directConnectionInfo) {
+            return directConnectionInfo.stream().filter(i -> Arrays.equals(i.getSignature(), signature)).collect(Collectors.toList());
+        }
+    }
+
+    /**
+     * Add an ArbitraryDirectConnectionInfo item, but only if one with this peer-signature combination
+     * doesn't already exist.
+     * @param connectionInfo - the direct connection info to add
+     */
+    public void addDirectConnectionInfoIfUnique(RNSArbitraryDirectConnectionInfo connectionInfo) {
+        boolean peerAlreadyExists;
+        synchronized (directConnectionInfo) {
+            peerAlreadyExists = directConnectionInfo.stream()
+                    .anyMatch(i -> Arrays.equals(i.getSignature(), connectionInfo.getSignature())
+                            && Objects.equals(i.getPeerAddress(), connectionInfo.getPeerAddress()));
+        }
+        if (!peerAlreadyExists) {
+            directConnectionInfo.add(connectionInfo);
+        }
+    }
+
+    private void removeDirectConnectionInfo(RNSArbitraryDirectConnectionInfo connectionInfo) {
+        this.directConnectionInfo.remove(connectionInfo);
+    }
+
+    public boolean fetchDataFilesFromPeersForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+
+        boolean success = false;
+
+        try {
+            while (!success) {
+                if (isStopping) {
+                    return false;
+                }
+                Thread.sleep(500L);
+
+                // Firstly fetch peers that claim to be hosting files for this signature
+                List<RNSArbitraryDirectConnectionInfo> connectionInfoList = getDirectConnectionInfoForSignature(signature);
+                if (connectionInfoList == null || connectionInfoList.isEmpty()) {
+                    LOGGER.debug("No remaining direct connection peers found for signature {}", signature58);
+                    return false;
+                }
+
+                LOGGER.debug("Attempting a direct peer connection for signature {}...", signature58);
+
+                // Peers found, so pick one with the highest number of chunks
+                Comparator<RNSArbitraryDirectConnectionInfo> highestChunkCountFirstComparator =
+                        Comparator.comparingInt(RNSArbitraryDirectConnectionInfo::getHashCount).reversed();
+                RNSArbitraryDirectConnectionInfo directConnectionInfo = connectionInfoList.stream()
+                        .sorted(highestChunkCountFirstComparator).findFirst().orElse(null);
+
+                if (directConnectionInfo == null) {
+                    return false;
+                }
+
+                // Remove from the list so that a different peer is tried next time
+                removeDirectConnectionInfo(directConnectionInfo);
+
+                //// TODO - rework this section (RNS network address?)
+                //String peerAddressString = directConnectionInfo.getPeerAddress();
+                //
+                //// Parse the peer address to find the host and port
+                //String host = null;
+                //int port = -1;
+                //String[] parts = peerAddressString.split(":");
+                //if (parts.length > 1) {
+                //    host = parts[0];
+                //    port = Integer.parseInt(parts[1]);
+                //} else {
+                //    // Assume no port included
+                //    host = peerAddressString;
+                //    // Use default listen port
+                //    port = Settings.getInstance().getDefaultListenPort();
+                //}
+                //
+                //String peerAddressStringWithPort = String.format("%s:%d", host, port);
+                //success = Network.getInstance().requestDataFromPeer(peerAddressStringWithPort, signature);
+                //
+                //int defaultPort = Settings.getInstance().getDefaultListenPort();
+                //
+                //// If unsuccessful, and using a non-standard port, try a second connection with the default listen port,
+                //// since almost all nodes use that. This is a workaround to account for any ephemeral ports that may
+                //// have made it into the dataset.
+                //if (!success) {
+                //    if (host != null && port > 0) {
+                //        if (port != defaultPort) {
+                //            String newPeerAddressString = String.format("%s:%d", host, defaultPort);
+                //            success = Network.getInstance().requestDataFromPeer(newPeerAddressString, signature);
+                //        }
+                //    }
+                //}
+                //
+                //// If _still_ unsuccessful, try matching the peer's IP address with some known peers, and then connect
+                //// to each of those in turn until one succeeds.
+                //if (!success) {
+                //    if (host != null) {
+                //        final String finalHost = host;
+                //        List<PeerData> knownPeers = Network.getInstance().getAllKnownPeers().stream()
+                //                .filter(knownPeerData -> knownPeerData.getAddress().getHost().equals(finalHost))
+                //                .collect(Collectors.toList());
+                //        // Loop through each match and attempt a connection
+                //        for (PeerData matchingPeer : knownPeers) {
+                //            String matchingPeerAddress = matchingPeer.getAddress().toString();
+                //            int matchingPeerPort = matchingPeer.getAddress().getPort();
+                //            // Make sure that it's not a port we've already tried
+                //            if (matchingPeerPort != port && matchingPeerPort != defaultPort) {
+                //                success = Network.getInstance().requestDataFromPeer(matchingPeerAddress, signature);
+                //                if (success) {
+                //                    // Successfully connected, so stop making connections
+                //                    break;
+                //                }
+                //            }
+                //        }
+                //    }
+                //}
+
+                if (success) {
+                    // We were able to connect with a peer, so track the request
+                    RNSArbitraryDataFileListManager.getInstance().addToSignatureRequests(signature58, false, true);
+                }
+
+            }
+        } catch (InterruptedException e) {
+            // Do nothing
+        }
+
+        return success;
+    }
+
+
+    // Relays
+
+    private List<RNSArbitraryRelayInfo> getRelayInfoListForHash(String hash58) {
+        synchronized (arbitraryRelayMap) {
+            return arbitraryRelayMap.stream()
+                    .filter(relayInfo -> Objects.equals(relayInfo.getHash58(), hash58))
+                    .collect(Collectors.toList());
+        }
+    }
+
+    private RNSArbitraryRelayInfo getOptimalRelayInfoEntryForHash(String hash58) {
+        LOGGER.trace("Fetching relay info for hash: {}", hash58);
+        List<RNSArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
+        if (relayInfoList != null && !relayInfoList.isEmpty()) {
+
+            // Remove any with null requestHops
+            relayInfoList.removeIf(r -> r.getRequestHops() == null);
+
+            // If list is now empty, then just return one at random
+            if (relayInfoList.isEmpty()) {
+                return this.getRandomRelayInfoEntryForHash(hash58);
+            }
+
+            // Sort by number of hops (lowest first)
+            relayInfoList.sort(Comparator.comparingInt(RNSArbitraryRelayInfo::getRequestHops));
+
+            // FUTURE: secondary sort by requestTime?
+
+            RNSArbitraryRelayInfo relayInfo = relayInfoList.get(0);
+
+            LOGGER.trace("Returning optimal relay info for hash: {} (requestHops {})", hash58, relayInfo.getRequestHops());
+            return relayInfo;
+        }
+        LOGGER.trace("No relay info exists for hash: {}", hash58);
+        return null;
+    }
+
+    private RNSArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) {
+        LOGGER.trace("Fetching random relay info for hash: {}", hash58);
+        List<RNSArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
+        if (relayInfoList != null && !relayInfoList.isEmpty()) {
+
+            // Pick random item
+            int index = new SecureRandom().nextInt(relayInfoList.size());
+            LOGGER.trace("Returning random relay info for hash: {} (index {})", hash58, index);
+            return relayInfoList.get(index);
+        }
+        LOGGER.trace("No relay info exists for hash: {}", hash58);
+        return null;
+    }
+
+    public void addToRelayMap(RNSArbitraryRelayInfo newEntry) {
+        if (newEntry == null || !newEntry.isValid()) {
+            return;
+        }
+
+        // Remove existing entry for this peer if it exists, to renew the timestamp
+        this.removeFromRelayMap(newEntry);
+
+        // Re-add
+        arbitraryRelayMap.add(newEntry);
+        LOGGER.debug("Added entry to relay map: {}", newEntry);
+    }
+
+    private void removeFromRelayMap(RNSArbitraryRelayInfo entry) {
+        arbitraryRelayMap.removeIf(relayInfo -> relayInfo.equals(entry));
+    }
+
+
+    // Peers requesting QDN data from us
+
+    /**
+     * Add an address string of a peer that is trying to request data from us.
+     * @param peerAddress
+     */
+    public void addRecentDataRequest(String peerAddress) {
+        if (peerAddress == null) {
+            return;
+        }
+
+        Long now = NTP.getTime();
+        if (now == null) {
+            return;
+        }
+
+        // Make sure to remove the port, since it isn't guaranteed to match next time
+        String[] parts = peerAddress.split(":");
+        if (parts.length == 0) {
+            return;
+        }
+        String host = parts[0];
+        if (!InetAddresses.isInetAddress(host)) {
+            // Invalid host
+            return;
+        }
+
+        this.recentDataRequests.put(host, now);
+    }
+
+    public boolean isPeerRequestingData(String peerAddressWithoutPort) {
+        return this.recentDataRequests.containsKey(peerAddressWithoutPort);
+    }
+
+    public boolean hasPendingDataRequest() {
+        return !this.recentDataRequests.isEmpty();
+    }
+
+
+    // Network handlers
+
+    public void onNetworkGetArbitraryDataFileMessage(RNSPeer peer, Message message) {
+        // Don't respond if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        GetArbitraryDataFileMessage getArbitraryDataFileMessage = (GetArbitraryDataFileMessage) message;
+        byte[] hash = getArbitraryDataFileMessage.getHash();
+        String hash58 = Base58.encode(hash);
+        byte[] signature = getArbitraryDataFileMessage.getSignature();
+        Controller.getInstance().stats.getArbitraryDataFileMessageStats.requests.incrementAndGet();
+
+        LOGGER.debug("Received GetArbitraryDataFileMessage from peer {} for hash {}", peer, Base58.encode(hash));
+
+        try {
+            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
+            RNSArbitraryRelayInfo relayInfo = this.getOptimalRelayInfoEntryForHash(hash58);
+
+            if (arbitraryDataFile.exists()) {
+                LOGGER.trace("Hash {} exists", hash58);
+
+                // We can serve the file directly as we already have it
+                LOGGER.debug("Sending file {}...", arbitraryDataFile);
+                ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, arbitraryDataFile);
+                arbitraryDataFileMessage.setId(message.getId());
+                //if (!peer.sendMessageWithTimeout(arbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT)) {
+                //    LOGGER.debug("Couldn't send file {}", arbitraryDataFile);
+                //    peer.disconnect("failed to send file");
+                //}
+                //else {
+                //    LOGGER.debug("Sent file {}", arbitraryDataFile);
+                //}
+                peer.sendMessageWithTimeout(arbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT);
+            }
+            //// TODO: rework (doesn't work with Reticulum)
+            //else if (relayInfo != null) {
+            //    LOGGER.debug("We have relay info for hash {}", Base58.encode(hash));
+            //    // We need to ask this peer for the file
+            //    Peer peerToAsk = relayInfo.getPeer();
+            //    if (peerToAsk != null) {
+            //
+            //        // Forward the message to this peer
+            //        LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58);
+            //        // No need to pass arbitraryTransactionData below because this is only used for metadata caching,
+            //        // and metadata isn't retained when relaying.
+            //        this.fetchArbitraryDataFile(peerToAsk, peer, null, signature, hash, message);
+            //    }
+            //    else {
+            //        LOGGER.debug("Peer {} not found in relay info", peer);
+            //    }
+            //}
+            else {
+                LOGGER.debug("Hash {} doesn't exist and we don't have relay info", hash58);
+
+                // We don't have this file
+                Controller.getInstance().stats.getArbitraryDataFileMessageStats.unknownFiles.getAndIncrement();
+
+                // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout
+                LOGGER.debug(String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile));
+
+                //// Send generic 'unknown' message as it's very short
+                //Message fileUnknownMessage = peer.getPeersVersion() >= GenericUnknownMessage.MINIMUM_PEER_VERSION
+                //        ? new GenericUnknownMessage()
+                //        : new BlockSummariesMessage(Collections.emptyList());
+                //fileUnknownMessage.setId(message.getId());
+                //if (!peer.sendMessage(fileUnknownMessage)) {
+                //    LOGGER.debug("Couldn't sent file-unknown response");
+                //    peer.disconnect("failed to send file-unknown response");
+                //}
+                //else {
+                //    LOGGER.debug("Sent file-unknown response for file {}", arbitraryDataFile);
+                //}
+                Message fileUnknownMessage = new GenericUnknownMessage();
+                peer.sendMessage(fileUnknownMessage);
+            }
+        }
+        catch (DataException e) {
+            LOGGER.debug("Unable to handle request for arbitrary data file: {}", hash58);
+        }
+    }
+
+}

src/main/java/org/qortal/controller/arbitrary/RNSArbitraryDataFileRequestThread.java

@@ -0,0 +1,130 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.RNSArbitraryFileListResponseInfo;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.event.DataMonitorEvent;
+import org.qortal.event.EventBus;
+import org.qortal.network.RNSPeer;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.utils.ArbitraryTransactionUtils;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Iterator;
+
+import static java.lang.Thread.NORM_PRIORITY;
+
+public class RNSArbitraryDataFileRequestThread implements Runnable {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class);
+
+    public RNSArbitraryDataFileRequestThread() {
+
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data File Request Thread");
+        Thread.currentThread().setPriority(NORM_PRIORITY);
+
+        try {
+            while (!Controller.isStopping()) {
+                Long now = NTP.getTime();
+                this.processFileHashes(now);
+            }
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
+        }
+    }
+
+    private void processFileHashes(Long now) throws InterruptedException {
+		if (Controller.isStopping()) {
+            return;
+        }
+
+        RNSArbitraryDataFileManager arbitraryDataFileManager = RNSArbitraryDataFileManager.getInstance();
+        String signature58 = null;
+        String hash58 = null;
+        RNSPeer peer = null;
+        boolean shouldProcess = false;
+
+        synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) {
+            if (!arbitraryDataFileManager.arbitraryDataFileHashResponses.isEmpty()) {
+
+                // Sort by lowest number of node hops first
+                Comparator<RNSArbitraryFileListResponseInfo> lowestHopsFirstComparator =
+                        Comparator.comparingInt(RNSArbitraryFileListResponseInfo::getRequestHops);
+                arbitraryDataFileManager.arbitraryDataFileHashResponses.sort(lowestHopsFirstComparator);
+
+                Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.iterator();
+                while (iterator.hasNext()) {
+                    if (Controller.isStopping()) {
+                        return;
+                    }
+
+                    RNSArbitraryFileListResponseInfo responseInfo = (RNSArbitraryFileListResponseInfo) iterator.next();
+                    if (responseInfo == null) {
+                        iterator.remove();
+                        continue;
+                    }
+
+                    hash58 = responseInfo.getHash58();
+                    peer = responseInfo.getPeer();
+                    signature58 = responseInfo.getSignature58();
+                    Long timestamp = responseInfo.getTimestamp();
+
+                    if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) {
+                        // Ignore - to be deleted
+                        iterator.remove();
+                        continue;
+                    }
+
+                    // Skip if already requesting, but don't remove, as we might want to retry later
+                    if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) {
+                        // Already requesting - leave this attempt for later
+                        continue;
+                    }
+
+                    // We want to process this file
+                    shouldProcess = true;
+                    iterator.remove();
+                    break;
+                }
+            }
+        }
+
+        if (!shouldProcess) {
+            // Nothing to do
+            Thread.sleep(1000L);
+            return;
+        }
+
+        byte[] hash = Base58.decode(hash58);
+        byte[] signature = Base58.decode(signature58);
+
+        // Fetch the transaction data
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+            if (arbitraryTransactionData == null) {
+                return;
+            }
+
+            if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) {
+                return;
+            }
+
+            LOGGER.trace("Fetching file {} from peer {} via request thread...", hash58, peer);
+            arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash));
+
+        } catch (DataException e) {
+            LOGGER.debug("Unable to process file hashes: {}", e.getMessage());
+        }
+    }
+}

src/main/java/org/qortal/controller/arbitrary/RNSArbitraryMetadataManager.java

@@ -0,0 +1,481 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.arbitrary.ArbitraryDataResource;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.controller.Controller;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.ArbitraryMetadataMessage;
+import org.qortal.network.message.GetArbitraryMetadataMessage;
+import org.qortal.network.message.Message;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+import org.qortal.utils.ListUtils;
+import org.qortal.utils.NTP;
+import org.qortal.utils.Triple;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.qortal.controller.arbitrary.ArbitraryDataFileListManager.*;
+
+public class RNSArbitraryMetadataManager {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryMetadataManager.class);
+
+    private static RNSArbitraryMetadataManager instance;
+
+    /**
+     * Map of recent incoming requests for ARBITRARY transaction metadata.
+     * <p>
+     * Key is original request's message ID<br>
+     * Value is Triple&lt;transaction signature in base58, first requesting peer, first request's timestamp&gt;
+     * <p>
+     * If peer is null then either:<br>
+     * <ul>
+     * <li>we are the original requesting peer</li>
+     * <li>we have already sent data payload to original requesting peer.</li>
+     * </ul>
+     * If signature is null then we have already received the file list and either:<br>
+     * <ul>
+     * <li>we are the original requesting peer and have processed it</li>
+     * <li>we have forwarded the metadata</li>
+     * </ul>
+     */
+    public Map<Integer, Triple<String, RNSPeer, Long>> arbitraryMetadataRequests = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of in progress arbitrary metadata requests
+     * Key: string - the signature encoded in base58
+     * Value: Triple<networkBroadcastCount, directPeerRequestCount, lastAttemptTimestamp>
+     */
+    private Map<String, Triple<Integer, Integer, Long>> arbitraryMetadataSignatureRequests = Collections.synchronizedMap(new HashMap<>());
+
+
+    private RNSArbitraryMetadataManager() {
+    }
+
+    public static RNSArbitraryMetadataManager getInstance() {
+        if (instance == null)
+            instance = new RNSArbitraryMetadataManager();
+
+        return instance;
+    }
+
+    public void cleanupRequestCache(Long now) {
+        if (now == null) {
+            return;
+        }
+        final long requestMinimumTimestamp = now - ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT;
+        arbitraryMetadataRequests.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < requestMinimumTimestamp);
+    }
+
+
+    public ArbitraryDataTransactionMetadata fetchMetadata(ArbitraryDataResource arbitraryDataResource, boolean useRateLimiter) {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            // Find latest transaction
+            ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
+                    .getLatestTransaction(arbitraryDataResource.getResourceId(), arbitraryDataResource.getService(),
+                            null, arbitraryDataResource.getIdentifier());
+
+            if (latestTransaction != null) {
+                byte[] signature = latestTransaction.getSignature();
+                byte[] metadataHash = latestTransaction.getMetadataHash();
+                if (metadataHash == null) {
+                    // This resource doesn't have metadata
+                    throw new IllegalArgumentException("This resource doesn't have metadata");
+                }
+
+                ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+                if (!metadataFile.exists()) {
+                    // Request from network
+                    this.fetchArbitraryMetadata(latestTransaction, useRateLimiter);
+                }
+
+                // Now check again as it may have been downloaded above
+                if (metadataFile.exists()) {
+                    // Use local copy
+                    ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
+                    try {
+                        transactionMetadata.read();
+                    } catch (DataException e) {
+                        // Invalid file, so delete it
+                        LOGGER.info("Deleting invalid metadata file due to exception: {}", e.getMessage());
+                        transactionMetadata.delete();
+                        return null;
+                    }
+                    return transactionMetadata;
+                }
+            }
+
+        } catch (DataException | IOException e) {
+            LOGGER.error("Repository issue when fetching arbitrary transaction metadata", e);
+        }
+
+        return null;
+    }
+
+
+    // Request metadata from network
+
+    public byte[] fetchArbitraryMetadata(ArbitraryTransactionData arbitraryTransactionData, boolean useRateLimiter) {
+        byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
+        if (metadataHash == null) {
+            return null;
+        }
+
+        byte[] signature = arbitraryTransactionData.getSignature();
+        String signature58 = Base58.encode(signature);
+
+        // Require an NTP sync
+        Long now = NTP.getTime();
+        if (now == null) {
+            return null;
+        }
+
+        // If we've already tried too many times in a short space of time, make sure to give up
+        if (useRateLimiter && !this.shouldMakeMetadataRequestForSignature(signature58)) {
+            LOGGER.trace("Skipping metadata request for signature {} due to rate limit", signature58);
+            return null;
+        }
+        this.addToSignatureRequests(signature58, true, false);
+
+        //List<Peer> handshakedPeers = Network.getInstance().getImmutableHandshakedPeers();
+        List<RNSPeer> handshakedPeers = RNSNetwork.getInstance().getLinkedPeers();
+        LOGGER.debug(String.format("Sending metadata request for signature %s to %d peers...", signature58, handshakedPeers.size()));
+
+        // Build request
+        Message getArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, now, 0);
+
+        // Save our request into requests map
+        Triple<String, RNSPeer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
+
+        // Assign random ID to this message
+        int id;
+        do {
+            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
+
+            // Put queue into map (keyed by message ID) so we can poll for a response
+            // If putIfAbsent() doesn't return null, then this ID is already taken
+        } while (arbitraryMetadataRequests.put(id, requestEntry) != null);
+        getArbitraryMetadataMessage.setId(id);
+
+        // Broadcast request
+        RNSNetwork.getInstance().broadcast(peer -> getArbitraryMetadataMessage);
+
+        // Poll to see if data has arrived
+        final long singleWait = 100;
+        long totalWait = 0;
+        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
+            try {
+                Thread.sleep(singleWait);
+            } catch (InterruptedException e) {
+                break;
+            }
+
+            requestEntry = arbitraryMetadataRequests.get(id);
+            if (requestEntry == null)
+                return null;
+
+            if (requestEntry.getA() == null)
+                break;
+
+            totalWait += singleWait;
+        }
+
+        try {
+            ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+            if (metadataFile.exists()) {
+                return metadataFile.getBytes();
+            }
+        } catch (DataException e) {
+            // Do nothing
+        }
+
+        return null;
+    }
+
+
+    // Track metadata lookups by signature
+
+    private boolean shouldMakeMetadataRequestForSignature(String signature58) {
+        Triple<Integer, Integer, Long> request = arbitraryMetadataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        // Extract the components
+        Integer networkBroadcastCount = request.getA();
+        // Integer directPeerRequestCount = request.getB();
+        Long lastAttemptTimestamp = request.getC();
+
+        if (lastAttemptTimestamp == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
+
+        // Allow a second attempt after 60 seconds
+        if (timeSinceLastAttempt > 60 * 1000L) {
+            // We haven't tried for at least 60 seconds
+
+            if (networkBroadcastCount < 2) {
+                // We've made less than 2 total attempts
+                return true;
+            }
+        }
+
+        // Then allow another attempt after 60 minutes
+        if (timeSinceLastAttempt > 60 * 60 * 1000L) {
+            // We haven't tried for at least 60 minutes
+
+            if (networkBroadcastCount < 3) {
+                // We've made less than 3 total attempts
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    public boolean isSignatureRateLimited(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        return !this.shouldMakeMetadataRequestForSignature(signature58);
+    }
+
+    public long lastRequestForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        Triple<Integer, Integer, Long> request = arbitraryMetadataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return 0;
+        }
+
+        // Extract the components
+        Long lastAttemptTimestamp = request.getC();
+        if (lastAttemptTimestamp != null) {
+            return  lastAttemptTimestamp;
+        }
+        return 0;
+    }
+
+    public void addToSignatureRequests(String signature58, boolean incrementNetworkRequests, boolean incrementPeerRequests) {
+        Triple<Integer, Integer, Long> request  = arbitraryMetadataSignatureRequests.get(signature58);
+        Long now = NTP.getTime();
+
+        if (request == null) {
+            // No entry yet
+            Triple<Integer, Integer, Long> newRequest = new Triple<>(0, 0, now);
+            arbitraryMetadataSignatureRequests.put(signature58, newRequest);
+        }
+        else {
+            // There is an existing entry
+            if (incrementNetworkRequests) {
+                request.setA(request.getA() + 1);
+            }
+            if (incrementPeerRequests) {
+                request.setB(request.getB() + 1);
+            }
+            request.setC(now);
+            arbitraryMetadataSignatureRequests.put(signature58, request);
+        }
+    }
+
+    public void removeFromSignatureRequests(String signature58) {
+        arbitraryMetadataSignatureRequests.remove(signature58);
+    }
+
+
+    // Network handlers
+
+    public void onNetworkArbitraryMetadataMessage(RNSPeer peer, Message message) {
+        // Don't process if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        ArbitraryMetadataMessage arbitraryMetadataMessage = (ArbitraryMetadataMessage) message;
+        LOGGER.debug("Received metadata from peer {}", peer);
+
+        // Do we have a pending request for this data?
+        Triple<String, RNSPeer, Long> request = arbitraryMetadataRequests.get(message.getId());
+        if (request == null || request.getA() == null) {
+            return;
+        }
+        boolean isRelayRequest = (request.getB() != null);
+
+        // Does this message's signature match what we're expecting?
+        byte[] signature = arbitraryMetadataMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        if (!request.getA().equals(signature58)) {
+            return;
+        }
+
+        // Update requests map to reflect that we've received this metadata
+        Triple<String, RNSPeer, Long> newEntry = new Triple<>(null, null, request.getC());
+        arbitraryMetadataRequests.put(message.getId(), newEntry);
+
+        // Get transaction info
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+            if (!(transactionData instanceof ArbitraryTransactionData)) {
+                return;
+            }
+            ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+
+            // Check if the name is blocked
+            boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
+
+            // Save if not blocked
+            ArbitraryDataFile arbitraryMetadataFile = arbitraryMetadataMessage.getArbitraryMetadataFile();
+            if (!isBlocked && arbitraryMetadataFile != null) {
+                arbitraryMetadataFile.save();
+            }
+
+            // Forwarding
+            if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
+                if (!isBlocked) {
+                    RNSPeer requestingPeer = request.getB();
+                    if (requestingPeer != null) {
+
+                        ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile());
+                        forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId());
+
+                        // Forward to requesting peer
+                        LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
+                        //if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
+                        //    requestingPeer.disconnect("failed to forward arbitrary metadata");
+                        //}
+                        requestingPeer.sendMessage(forwardArbitraryMetadataMessage);
+                    }
+                }
+            }
+
+            // Add to resource queue to update arbitrary resource caches
+            if (arbitraryTransactionData != null) {
+                ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while saving arbitrary transaction metadata from peer %s", peer), e);
+        }
+    }
+
+    public void onNetworkGetArbitraryMetadataMessage(RNSPeer peer, Message message) {
+        // Don't respond if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet();
+
+        GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) message;
+        byte[] signature = getArbitraryMetadataMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        Long now = NTP.getTime();
+        Triple<String, RNSPeer, Long> newEntry = new Triple<>(signature58, peer, now);
+
+        // If we've seen this request recently, then ignore
+        if (arbitraryMetadataRequests.putIfAbsent(message.getId(), newEntry) != null) {
+            LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peer, signature58);
+            return;
+        }
+
+        LOGGER.debug("Received metadata request from peer {} for signature {}", peer, signature58);
+
+        ArbitraryTransactionData transactionData = null;
+        ArbitraryDataFile metadataFile = null;
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Firstly we need to lookup this file on chain to get its metadata hash
+            transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature);
+            if (transactionData instanceof ArbitraryTransactionData) {
+
+                // Check if we're even allowed to serve metadata for this transaction
+                if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
+
+                    byte[] metadataHash = transactionData.getMetadataHash();
+                    if (metadataHash != null) {
+
+                        // Load metadata file
+                        metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+                    }
+                }
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while fetching arbitrary metadata for peer %s", peer), e);
+        }
+
+        // We should only respond if we have the metadata file
+        if (metadataFile != null && metadataFile.exists()) {
+
+            // We have the metadata file, so update requests map to reflect that we've sent it
+            newEntry = new Triple<>(null, null, now);
+            arbitraryMetadataRequests.put(message.getId(), newEntry);
+
+            ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, metadataFile);
+            arbitraryMetadataMessage.setId(message.getId());
+            //if (!peer.sendMessage(arbitraryMetadataMessage)) {
+            //    LOGGER.debug("Couldn't send metadata");
+            //    peer.disconnect("failed to send metadata");
+            //    return;
+            //}
+            peer.sendMessage(arbitraryMetadataMessage);
+            LOGGER.debug("Sent metadata");
+
+            // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
+            LOGGER.debug("No need for any forwarding because metadata request is fully served");
+            return;
+
+        }
+
+        // We may need to forward this request on
+        boolean isBlocked = (transactionData == null || ListUtils.isNameBlocked(transactionData.getName()));
+        if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
+            // In relay mode - so ask our other peers if they have it
+
+            long requestTime = getArbitraryMetadataMessage.getRequestTime();
+            int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1;
+            long totalRequestTime = now - requestTime;
+
+            if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
+                // Relay request hasn't timed out yet, so can potentially be rebroadcast
+                if (requestHops < RELAY_REQUEST_MAX_HOPS) {
+                    // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
+
+                    Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops);
+                    relayGetArbitraryMetadataMessage.setId(message.getId());
+
+                    LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
+                    //Network.getInstance().broadcast(
+                    //        broadcastPeer ->
+                    //                !broadcastPeer.isAtLeastVersion(RELAY_MIN_PEER_VERSION) ? null :
+                    //                broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) ? null : relayGetArbitraryMetadataMessage);
+                    RNSNetwork.getInstance().broadcast(broadcastPeer -> relayGetArbitraryMetadataMessage);
+
+                }
+                else {
+                    // This relay request has reached the maximum number of allowed hops
+                }
+            }
+            else {
+                // This relay request has timed out
+            }
+        }
+    }
+
+}

src/main/java/org/qortal/controller/arbitrary/RebuildArbitraryResourceCacheTask.java

@@ -0,0 +1,33 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+
+import java.util.TimerTask;
+
+public class RebuildArbitraryResourceCacheTask extends TimerTask {
+
+    private static final Logger LOGGER = LogManager.getLogger(RebuildArbitraryResourceCacheTask.class);
+
+    public static final long MILLIS_IN_HOUR = 60 * 60 * 1000;
+
+    public static final long MILLIS_IN_MINUTE = 60 * 1000;
+
+    private static final String REBUILD_ARBITRARY_RESOURCE_CACHE_TASK = "Rebuild Arbitrary Resource Cache Task";
+
+    @Override
+    public void run() {
+
+        Thread.currentThread().setName(REBUILD_ARBITRARY_RESOURCE_CACHE_TASK);
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, true);
+        }
+        catch( DataException e ) {
+            LOGGER.error(e.getMessage(), e);
+        }
+    }
+}

src/main/java/org/qortal/controller/hsqldb/HSQLDBBalanceRecorder.java

@@ -0,0 +1,139 @@
+package org.qortal.controller.hsqldb;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.util.PropertySource;
+import org.qortal.data.account.AccountBalanceData;
+import org.qortal.data.account.BlockHeightRange;
+import org.qortal.data.account.BlockHeightRangeAddressAmounts;
+import org.qortal.repository.hsqldb.HSQLDBCacheUtils;
+import org.qortal.settings.Settings;
+import org.qortal.utils.BalanceRecorderUtils;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
+
+public class HSQLDBBalanceRecorder extends Thread{
+
+    private static final Logger LOGGER = LogManager.getLogger(HSQLDBBalanceRecorder.class);
+
+    private static HSQLDBBalanceRecorder SINGLETON = null;
+
+    private ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight = new ConcurrentHashMap<>();
+
+    private ConcurrentHashMap<String, List<AccountBalanceData>> balancesByAddress = new ConcurrentHashMap<>();
+
+    private CopyOnWriteArrayList<BlockHeightRangeAddressAmounts> balanceDynamics = new CopyOnWriteArrayList<>();
+
+    private int priorityRequested;
+    private int frequency;
+    private int capacity;
+
+    private HSQLDBBalanceRecorder( int priorityRequested, int frequency, int capacity) {
+
+        super("Balance Recorder");
+
+        this.priorityRequested = priorityRequested;
+        this.frequency = frequency;
+        this.capacity = capacity;
+    }
+
+    public static Optional<HSQLDBBalanceRecorder> getInstance() {
+
+        if( SINGLETON == null ) {
+
+            SINGLETON
+                = new HSQLDBBalanceRecorder(
+                    Settings.getInstance().getBalanceRecorderPriority(),
+                    Settings.getInstance().getBalanceRecorderFrequency(),
+                    Settings.getInstance().getBalanceRecorderCapacity()
+            );
+
+        }
+        else if( SINGLETON == null ) {
+
+            return Optional.empty();
+        }
+
+        return Optional.of(SINGLETON);
+    }
+
+    @Override
+    public void run() {
+
+        Thread.currentThread().setName("Balance Recorder");
+
+        HSQLDBCacheUtils.startRecordingBalances(this.balancesByHeight, this.balanceDynamics, this.priorityRequested, this.frequency, this.capacity);
+    }
+
+    public List<BlockHeightRangeAddressAmounts> getLatestDynamics(int limit, long offset) {
+
+        List<BlockHeightRangeAddressAmounts> latest = this.balanceDynamics.stream()
+                .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_ADDRESS_AMOUNTS_COMPARATOR.reversed())
+                .skip(offset)
+                .limit(limit)
+                .collect(Collectors.toList());
+
+        return latest;
+    }
+
+    public List<BlockHeightRange> getRanges(Integer offset, Integer limit, Boolean reverse) {
+
+        if( reverse ) {
+            return this.balanceDynamics.stream()
+                    .map(BlockHeightRangeAddressAmounts::getRange)
+                    .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_COMPARATOR.reversed())
+                    .skip(offset)
+                    .limit(limit)
+                    .collect(Collectors.toList());
+        }
+        else {
+            return this.balanceDynamics.stream()
+                    .map(BlockHeightRangeAddressAmounts::getRange)
+                    .sorted(BalanceRecorderUtils.BLOCK_HEIGHT_RANGE_COMPARATOR)
+                    .skip(offset)
+                    .limit(limit)
+                    .collect(Collectors.toList());
+        }
+    }
+
+    public Optional<BlockHeightRangeAddressAmounts> getAddressAmounts(BlockHeightRange range) {
+
+        return this.balanceDynamics.stream()
+            .filter( dynamic -> dynamic.getRange().equals(range))
+            .findAny();
+    }
+
+    public Optional<BlockHeightRange> getRange( int height ) {
+        return this.balanceDynamics.stream()
+            .map(BlockHeightRangeAddressAmounts::getRange)
+            .filter( range -> range.getBegin() < height && range.getEnd() >= height )
+            .findAny();
+    }
+
+    private Optional<Integer> getLastHeight() {
+        return this.balancesByHeight.keySet().stream().sorted(Comparator.reverseOrder()).findFirst();
+    }
+
+    public List<Integer> getBlocksRecorded() {
+
+        return this.balancesByHeight.keySet().stream().collect(Collectors.toList());
+    }
+
+    public List<AccountBalanceData> getAccountBalanceRecordings(String address) {
+        return this.balancesByAddress.get(address);
+    }
+
+    @Override
+    public String toString() {
+        return "HSQLDBBalanceRecorder{" +
+                "priorityRequested=" + priorityRequested +
+                ", frequency=" + frequency +
+                ", capacity=" + capacity +
+                '}';
+    }
+}

src/main/java/org/qortal/controller/hsqldb/HSQLDBDataCacheManager.java

@@ -8,11 +8,7 @@ import org.qortal.settings.Settings;
 
 public class HSQLDBDataCacheManager extends Thread{
 
-    private HSQLDBRepository respository;
-
-    public HSQLDBDataCacheManager(HSQLDBRepository respository) {
-        this.respository = respository;
-    }
+    public HSQLDBDataCacheManager() {}
 
     @Override
     public void run() {
@@ -20,8 +16,7 @@ public class HSQLDBDataCacheManager extends Thread{
 
         HSQLDBCacheUtils.startCaching(
             Settings.getInstance().getDbCacheThreadPriority(),
-            Settings.getInstance().getDbCacheFrequency(),
-            this.respository
+            Settings.getInstance().getDbCacheFrequency()
         );
     }
 }

src/main/java/org/qortal/controller/repository/AtStatesPruner.java

@@ -39,15 +39,24 @@ public class AtStatesPruner implements Runnable {
 			}
 		}
 
+		int pruneStartHeight;
+		int maxLatestAtStatesHeight;
+
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			int pruneStartHeight = repository.getATRepository().getAtPruneHeight();
-			int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
+			pruneStartHeight = repository.getATRepository().getAtPruneHeight();
+			maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
 
 			repository.discardChanges();
 			repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
 			repository.saveChanges();
+		} catch (Exception e) {
+			LOGGER.error("AT States Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
+			return;
+		}
+
+		while (!Controller.isStopping()) {
+			try (final Repository repository = RepositoryManager.getRepository()) {
 
-			while (!Controller.isStopping()) {
 				try {
 					repository.discardChanges();
 
@@ -102,28 +111,25 @@ public class AtStatesPruner implements Runnable {
 
 							final int finalPruneStartHeight = pruneStartHeight;
 							LOGGER.info(() -> String.format("Bumping AT state base prune height to %d", finalPruneStartHeight));
-						}
-						else {
+						} else {
 							// We've pruned up to the upper prunable height
 							// Back off for a while to save CPU for syncing
 							repository.discardChanges();
-							Thread.sleep(5*60*1000L);
+							Thread.sleep(5 * 60 * 1000L);
 						}
 					}
 				} catch (InterruptedException e) {
-					if(Controller.isStopping()) {
+					if (Controller.isStopping()) {
 						LOGGER.info("AT States Pruning Shutting Down");
-					}
-					else {
+					} else {
 						LOGGER.warn("AT States Pruning interrupted. Trying again. Report this error immediately to the developers.", e);
 					}
 				} catch (Exception e) {
 					LOGGER.warn("AT States Pruning stopped working. Trying again. Report this error immediately to the developers.", e);
 				}
+			} catch(Exception e){
+				LOGGER.error("AT States Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 			}
-		} catch (Exception e) {
-			LOGGER.error("AT States Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 		}
 	}
-
 }

src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java

@@ -26,15 +26,23 @@ public class AtStatesTrimmer implements Runnable {
 			return;
 		}
 
+		int trimStartHeight;
+		int maxLatestAtStatesHeight;
+
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			int trimStartHeight = repository.getATRepository().getAtTrimHeight();
-			int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
+			trimStartHeight = repository.getATRepository().getAtTrimHeight();
+			maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
 
 			repository.discardChanges();
 			repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
 			repository.saveChanges();
+		} catch (Exception e) {
+			LOGGER.error("AT States Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
+			return;
+		}
 
-			while (!Controller.isStopping()) {
+		while (!Controller.isStopping()) {
+			try (final Repository repository = RepositoryManager.getRepository()) {
 				try {
 					repository.discardChanges();
 
@@ -92,9 +100,9 @@ public class AtStatesTrimmer implements Runnable {
 				} catch (Exception e) {
 					LOGGER.warn("AT States Trimming stopped working. Trying again. Report this error immediately to the developers.", e);
 				}
+			} catch (Exception e) {
+				LOGGER.error("AT States Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 			}
-		} catch (Exception e) {
-			LOGGER.error("AT States Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 		}
 	}
 

src/main/java/org/qortal/controller/repository/BlockArchiver.java

@@ -30,11 +30,13 @@ public class BlockArchiver implements Runnable {
 			return;
 		}
 
+		int startHeight;
+
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			// Don't even start building until initial rush has ended
 			Thread.sleep(INITIAL_SLEEP_PERIOD);
 
-			int startHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight();
+			startHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight();
 
 			// Don't attempt to archive if we have no ATStatesHeightIndex, as it will be too slow
 			boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex();
@@ -43,10 +45,16 @@ public class BlockArchiver implements Runnable {
 				repository.discardChanges();
 				return;
 			}
+		} catch (Exception e) {
+			LOGGER.error("Block Archiving is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
+			return;
+		}
 
-			LOGGER.info("Starting block archiver from height {}...", startHeight);
+		LOGGER.info("Starting block archiver from height {}...", startHeight);
+
+		while (!Controller.isStopping()) {
+			try (final Repository repository = RepositoryManager.getRepository()) {
 
-			while (!Controller.isStopping()) {
 				try {
 					repository.discardChanges();
 
@@ -107,20 +115,17 @@ public class BlockArchiver implements Runnable {
 						LOGGER.info("Caught exception when creating block cache", e);
 					}
 				} catch (InterruptedException e) {
-					if(Controller.isStopping()) {
+					if (Controller.isStopping()) {
 						LOGGER.info("Block Archiving Shutting Down");
-					}
-					else {
+					} else {
 						LOGGER.warn("Block Archiving interrupted. Trying again. Report this error immediately to the developers.", e);
 					}
 				} catch (Exception e) {
 					LOGGER.warn("Block Archiving stopped working. Trying again. Report this error immediately to the developers.", e);
 				}
+			} catch(Exception e){
+				LOGGER.error("Block Archiving is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 			}
-		} catch (Exception e) {
-			LOGGER.error("Block Archiving is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 		}
-
 	}
-
 }

src/main/java/org/qortal/controller/repository/BlockPruner.java

@@ -39,8 +39,10 @@ public class BlockPruner implements Runnable {
 			}
 		}
 
+		int pruneStartHeight;
+
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			int pruneStartHeight = repository.getBlockRepository().getBlockPruneHeight();
+			pruneStartHeight = repository.getBlockRepository().getBlockPruneHeight();
 
 			// Don't attempt to prune if we have no ATStatesHeightIndex, as it will be too slow
 			boolean hasAtStatesHeightIndex = repository.getATRepository().hasAtStatesHeightIndex();
@@ -48,8 +50,15 @@ public class BlockPruner implements Runnable {
 				LOGGER.info("Unable to start block pruner due to missing ATStatesHeightIndex. Bootstrapping is recommended.");
 				return;
 			}
+		} catch (Exception e) {
+			LOGGER.error("Block Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
+			return;
+		}
+
+		while (!Controller.isStopping()) {
+
+			try (final Repository repository = RepositoryManager.getRepository()) {
 
-			while (!Controller.isStopping()) {
 				try {
 					repository.discardChanges();
 
@@ -122,10 +131,9 @@ public class BlockPruner implements Runnable {
 				} catch (Exception e) {
 					LOGGER.warn("Block Pruning stopped working. Trying again. Report this error immediately to the developers.", e);
 				}
+			} catch(Exception e){
+				LOGGER.error("Block Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 			}
-		} catch (Exception e) {
-			LOGGER.error("Block Pruning is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 		}
 	}
-
 }

src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java

@@ -28,13 +28,21 @@ public class OnlineAccountsSignaturesTrimmer implements Runnable {
 			return;
 		}
 
+		int trimStartHeight;
+
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			// Don't even start trimming until initial rush has ended
 			Thread.sleep(INITIAL_SLEEP_PERIOD);
 
-			int trimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight();
+			trimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight();
+		} catch (Exception e) {
+			LOGGER.error("Online Accounts Signatures Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
+			return;
+		}
+
+		while (!Controller.isStopping()) {
+			try (final Repository repository = RepositoryManager.getRepository()) {
 
-			while (!Controller.isStopping()) {
 				try {
 					repository.discardChanges();
 
@@ -88,10 +96,9 @@ public class OnlineAccountsSignaturesTrimmer implements Runnable {
 				} catch (Exception e) {
 					LOGGER.warn("Online Accounts Signatures Trimming stopped working. Trying again. Report this error immediately to the developers.", e);
 				}
+			} catch (Exception e) {
+				LOGGER.error("Online Accounts Signatures Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 			}
-		} catch (Exception e) {
-			LOGGER.error("Online Accounts Signatures Trimming is not working! Not trying again. Restart ASAP. Report this error immediately to the developers.", e);
 		}
 	}
-
 }

src/main/java/org/qortal/controller/tradebot/RNSTradeBot.java

@@ -0,0 +1,778 @@
+package org.qortal.controller.tradebot;
+
+import com.google.common.primitives.Longs;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.bitcoinj.core.ECKey;
+import org.qortal.account.PrivateKeyAccount;
+import org.qortal.api.model.crosschain.TradeBotCreateRequest;
+import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
+import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult;
+import org.qortal.crosschain.*;
+import org.qortal.crypto.Crypto;
+import org.qortal.data.at.ATData;
+import org.qortal.data.crosschain.CrossChainTradeData;
+import org.qortal.data.crosschain.TradeBotData;
+import org.qortal.data.network.TradePresenceData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.event.Event;
+import org.qortal.event.EventBus;
+import org.qortal.event.Listener;
+import org.qortal.gui.SysTray;
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.GetTradePresencesMessage;
+import org.qortal.network.message.Message;
+import org.qortal.network.message.TradePresencesMessage;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.repository.hsqldb.HSQLDBImportExport;
+import org.qortal.settings.Settings;
+import org.qortal.transaction.Transaction;
+import org.qortal.utils.ByteArray;
+import org.qortal.utils.NTP;
+
+import java.awt.TrayIcon.MessageType;
+import java.security.SecureRandom;
+import java.util.*;
+import java.util.function.Supplier;
+
+/**
+ * Performing cross-chain trading steps on behalf of user.
+ * <p>
+ * We deal with three different independent state-spaces here:
+ * <ul>
+ * 	<li>Qortal blockchain</li>
+ * 	<li>Foreign blockchain</li>
+ * 	<li>Trade-bot entries</li>
+ * </ul>
+ */
+public class RNSTradeBot implements Listener {
+
+	private static final Logger LOGGER = LogManager.getLogger(TradeBot.class);
+	private static final Random RANDOM = new SecureRandom();
+
+	/** Maximum lifetime of trade presence timestamp. 30 mins in ms. */
+	private static final long PRESENCE_LIFETIME = 30 * 60 * 1000L;
+	/** How soon before expiry of our own trade presence timestamp that we want to trigger renewal. 5 mins in ms. */
+	private static final long EARLY_RENEWAL_PERIOD = 5 * 60 * 1000L;
+	/** Trade presence timestamps are rounded up to this nearest interval. Bigger values improve grouping of entries in [GET_]TRADE_PRESENCES network messages. 15 mins in ms. */
+	private static final long EXPIRY_ROUNDING = 15 * 60 * 1000L;
+	/** How often we want to broadcast our list of all known trade presences to peers. 5 mins in ms. */
+	private static final long PRESENCE_BROADCAST_INTERVAL = 5 * 60 * 1000L;
+
+	public interface StateNameAndValueSupplier {
+		public String getState();
+		public int getStateValue();
+	}
+
+	public static class StateChangeEvent implements Event {
+		private final TradeBotData tradeBotData;
+
+		public StateChangeEvent(TradeBotData tradeBotData) {
+			this.tradeBotData = tradeBotData;
+		}
+
+		public TradeBotData getTradeBotData() {
+			return this.tradeBotData;
+		}
+	}
+
+	public static class TradePresenceEvent implements Event {
+		private final TradePresenceData tradePresenceData;
+
+		public TradePresenceEvent(TradePresenceData tradePresenceData) {
+			this.tradePresenceData = tradePresenceData;
+		}
+
+		public TradePresenceData getTradePresenceData() {
+			return this.tradePresenceData;
+		}
+	}
+
+	private static final Map<Class<? extends ACCT>, Supplier<AcctTradeBot>> acctTradeBotSuppliers = new HashMap<>();
+	static {
+		acctTradeBotSuppliers.put(BitcoinACCTv1.class, BitcoinACCTv1TradeBot::getInstance);
+		acctTradeBotSuppliers.put(BitcoinACCTv3.class, BitcoinACCTv3TradeBot::getInstance);
+		acctTradeBotSuppliers.put(LitecoinACCTv1.class, LitecoinACCTv1TradeBot::getInstance);
+		acctTradeBotSuppliers.put(LitecoinACCTv3.class, LitecoinACCTv3TradeBot::getInstance);
+		acctTradeBotSuppliers.put(DogecoinACCTv1.class, DogecoinACCTv1TradeBot::getInstance);
+		acctTradeBotSuppliers.put(DogecoinACCTv3.class, DogecoinACCTv3TradeBot::getInstance);
+		acctTradeBotSuppliers.put(DigibyteACCTv3.class, DigibyteACCTv3TradeBot::getInstance);
+		acctTradeBotSuppliers.put(RavencoinACCTv3.class, RavencoinACCTv3TradeBot::getInstance);
+		acctTradeBotSuppliers.put(PirateChainACCTv3.class, PirateChainACCTv3TradeBot::getInstance);
+	}
+
+	private static RNSTradeBot instance;
+
+	private final Map<ByteArray, Long> ourTradePresenceTimestampsByPubkey = Collections.synchronizedMap(new HashMap<>());
+	private final List<TradePresenceData> pendingTradePresences = Collections.synchronizedList(new ArrayList<>());
+
+	private final Map<ByteArray, TradePresenceData> allTradePresencesByPubkey = Collections.synchronizedMap(new HashMap<>());
+	private Map<ByteArray, TradePresenceData> safeAllTradePresencesByPubkey = Collections.emptyMap();
+	private long nextTradePresenceBroadcastTimestamp = 0L;
+
+	private Map<String, Long> failedTrades = new HashMap<>();
+	private Map<String, Long> validTrades = new HashMap<>();
+
+	private RNSTradeBot() {
+		EventBus.INSTANCE.addListener(event -> RNSTradeBot.getInstance().listen(event));
+	}
+
+	public static synchronized RNSTradeBot getInstance() {
+		if (instance == null)
+			instance = new RNSTradeBot();
+
+		return instance;
+	}
+
+	public ACCT getAcctUsingAtData(ATData atData) {
+		byte[] codeHash = atData.getCodeHash();
+		if (codeHash == null)
+			return null;
+
+		return SupportedBlockchain.getAcctByCodeHash(codeHash);
+	}
+
+	public CrossChainTradeData populateTradeData(Repository repository, ATData atData) throws DataException {
+		ACCT acct = this.getAcctUsingAtData(atData);
+		if (acct == null)
+			return null;
+
+		return acct.populateTradeData(repository, atData);
+	}
+
+	/**
+	 * Creates a new trade-bot entry from the "Bob" viewpoint,
+	 * i.e. OFFERing QORT in exchange for foreign blockchain currency.
+	 * <p>
+	 * Generates:
+	 * <ul>
+	 * 	<li>new 'trade' private key</li>
+	 * 	<li>secret(s)</li>
+	 * </ul>
+	 * Derives:
+	 * <ul>
+	 * 	<li>'native' (as in Qortal) public key, public key hash, address (starting with Q)</li>
+	 * 	<li>'foreign' public key, public key hash</li>
+	 *	<li>hash(es) of secret(s)</li>
+	 * </ul>
+	 * A Qortal AT is then constructed including the following as constants in the 'data segment':
+	 * <ul>
+	 * 	<li>'native' (Qortal) 'trade' address - used to MESSAGE AT</li>
+	 * 	<li>'foreign' public key hash - used by Alice's to allow redeem of currency on foreign blockchain</li>
+	 * 	<li>hash(es) of secret(s) - used by AT (optional) and foreign blockchain as needed</li>
+	 * 	<li>QORT amount on offer by Bob</li>
+	 * 	<li>foreign currency amount expected in return by Bob (from Alice)</li>
+	 * 	<li>trading timeout, in case things go wrong and everyone needs to refund</li>
+	 * </ul>
+	 * Returns a DEPLOY_AT transaction that needs to be signed and broadcast to the Qortal network.
+	 * <p>
+	 * Trade-bot will wait for Bob's AT to be deployed before taking next step.
+	 * <p>
+	 * @param repository
+	 * @param tradeBotCreateRequest
+	 * @return raw, unsigned DEPLOY_AT transaction
+	 * @throws DataException
+	 */
+	public byte[] createTrade(Repository repository, TradeBotCreateRequest tradeBotCreateRequest) throws DataException {
+		// Fetch latest ACCT version for requested foreign blockchain
+		ACCT acct = tradeBotCreateRequest.foreignBlockchain.getLatestAcct();
+
+		AcctTradeBot acctTradeBot = findTradeBotForAcct(acct);
+		if (acctTradeBot == null)
+			return null;
+
+		return acctTradeBot.createTrade(repository, tradeBotCreateRequest);
+	}
+
+	/**
+	 * Creates a trade-bot entry from the 'Alice' viewpoint,
+	 * i.e. matching foreign blockchain currency to an existing QORT offer.
+	 * <p>
+	 * Requires a chosen trade offer from Bob, passed by <tt>crossChainTradeData</tt>
+	 * and access to a foreign blockchain wallet via <tt>foreignKey</tt>.
+	 * <p>
+	 * @param repository
+	 * @param crossChainTradeData chosen trade OFFER that Alice wants to match
+	 * @param foreignKey foreign blockchain wallet key
+	 * @throws DataException
+	 */
+	public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct,
+			CrossChainTradeData crossChainTradeData, String foreignKey, String receivingAddress) throws DataException {
+		AcctTradeBot acctTradeBot = findTradeBotForAcct(acct);
+		if (acctTradeBot == null) {
+			LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot for AT %s", atData.getATAddress()));
+			return ResponseResult.NETWORK_ISSUE;
+		}
+
+		// Check Alice doesn't already have an existing, on-going trade-bot entry for this AT.
+		if (repository.getCrossChainRepository().existsTradeWithAtExcludingStates(atData.getATAddress(), acctTradeBot.getEndStates()))
+			return ResponseResult.TRADE_ALREADY_EXISTS;
+
+		return acctTradeBot.startResponse(repository, atData, acct, crossChainTradeData, foreignKey, receivingAddress);
+	}
+
+	/**
+	 * Creates a trade-bot entries from the 'Alice' viewpoint,
+	 * i.e. matching foreign blockchain currency to existing QORT offers.
+	 * <p>
+	 * Requires chosen trade offers from Bob, passed by <tt>crossChainTradeData</tt>
+	 * and access to a foreign blockchain wallet via <tt>foreignKey</tt>.
+	 * <p>
+	 * @param repository
+	 * @param crossChainTradeDataList chosen trade OFFERs that Alice wants to match
+	 * @param receiveAddress Alice's Qortal address to receive her QORT
+	 * @param foreignKey foreign blockchain wallet key
+	 * @param bitcoiny
+	 * @throws DataException
+	 */
+	public ResponseResult startResponseMultiple(
+			Repository repository,
+			ACCT acct,
+			List<CrossChainTradeData> crossChainTradeDataList,
+			String receiveAddress,
+			String foreignKey,
+			Bitcoiny bitcoiny) throws DataException {
+		AcctTradeBot acctTradeBot = findTradeBotForAcct(acct);
+		if (acctTradeBot == null) {
+			LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot for %s", acct.getBlockchain()));
+			return ResponseResult.NETWORK_ISSUE;
+		}
+
+		for( CrossChainTradeData tradeData : crossChainTradeDataList) {
+			// Check Alice doesn't already have an existing, on-going trade-bot entry for this AT.
+			if (repository.getCrossChainRepository().existsTradeWithAtExcludingStates(tradeData.qortalAtAddress, acctTradeBot.getEndStates()))
+				return ResponseResult.TRADE_ALREADY_EXISTS;
+		}
+		return TradeBotUtils.startResponseMultiple(repository, acct, crossChainTradeDataList, receiveAddress, foreignKey, bitcoiny);
+	}
+
+	public boolean deleteEntry(Repository repository, byte[] tradePrivateKey) throws DataException {
+		TradeBotData tradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
+		if (tradeBotData == null)
+			// Can't delete what we don't have!
+			return false;
+
+		boolean canDelete = false;
+
+		ACCT acct = SupportedBlockchain.getAcctByName(tradeBotData.getAcctName());
+		if (acct == null)
+			// We can't/no longer support this ACCT
+			canDelete = true;
+		else {
+			AcctTradeBot acctTradeBot = findTradeBotForAcct(acct);
+			canDelete = acctTradeBot == null || acctTradeBot.canDelete(repository, tradeBotData);
+		}
+
+		if (canDelete) {
+			repository.getCrossChainRepository().delete(tradePrivateKey);
+			repository.saveChanges();
+		}
+
+		return canDelete;
+	}
+
+	@Override
+	public void listen(Event event) {
+		if (!(event instanceof Synchronizer.NewChainTipEvent))
+			return;
+
+		// Don't process trade bots or broadcast presence timestamps if our chain is more than 60 minutes old
+		final Long minLatestBlockTimestamp = NTP.getTime() - (60 * 60 * 1000L);
+		if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp))
+			return;
+
+		synchronized (this) {
+			expireOldPresenceTimestamps();
+
+			List<TradeBotData> allTradeBotData;
+
+			try (final Repository repository = RepositoryManager.getRepository()) {
+				allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
+			} catch (DataException e) {
+				LOGGER.error("Couldn't run trade bot due to repository issue", e);
+				return;
+			}
+
+			for (TradeBotData tradeBotData : allTradeBotData)
+				try (final Repository repository = RepositoryManager.getRepository()) {
+					// Find ACCT-specific trade-bot for this entry
+					ACCT acct = SupportedBlockchain.getAcctByName(tradeBotData.getAcctName());
+					if (acct == null) {
+						LOGGER.debug(() -> String.format("Couldn't find ACCT matching name %s", tradeBotData.getAcctName()));
+						continue;
+					}
+
+					AcctTradeBot acctTradeBot = findTradeBotForAcct(acct);
+					if (acctTradeBot == null) {
+						LOGGER.debug(() -> String.format("Couldn't find ACCT trade-bot matching name %s", tradeBotData.getAcctName()));
+						continue;
+					}
+
+					acctTradeBot.progress(repository, tradeBotData);
+				} catch (DataException e) {
+					LOGGER.error("Couldn't run trade bot due to repository issue", e);
+				} catch (ForeignBlockchainException e) {
+					LOGGER.warn(() -> String.format("Foreign blockchain issue processing trade-bot entry for AT %s: %s", tradeBotData.getAtAddress(), e.getMessage()));
+				}
+
+			broadcastPresenceTimestamps();
+		}
+	}
+
+	public static byte[] generateTradePrivateKey() {
+		// The private key is used for both Curve25519 and secp256k1 so needs to be valid for both.
+		// Curve25519 accepts any seed, so generate a valid secp256k1 key and use that.
+		return new ECKey().getPrivKeyBytes();
+	}
+
+	public static byte[] deriveTradeNativePublicKey(byte[] privateKey) {
+		return Crypto.toPublicKey(privateKey);
+	}
+
+	public static byte[] deriveTradeForeignPublicKey(byte[] privateKey) {
+		return ECKey.fromPrivate(privateKey).getPubKey();
+	}
+
+	/*package*/ public static byte[] generateSecret() {
+		byte[] secret = new byte[32];
+		RANDOM.nextBytes(secret);
+		return secret;
+	}
+
+	/*package*/ static void backupTradeBotData(Repository repository, List<TradeBotData> additional) {
+		// Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure
+		try {
+			LOGGER.info("About to backup trade bot data...");
+			HSQLDBImportExport.backupTradeBotStates(repository, additional);
+		} catch (DataException e) {
+			LOGGER.info(String.format("Repository issue when exporting trade bot data: %s", e.getMessage()));
+		}
+	}
+
+	/** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */
+	/*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData,
+			String newState, int newStateValue, Supplier<String> logMessageSupplier) throws DataException {
+		tradeBotData.setState(newState);
+		tradeBotData.setStateValue(newStateValue);
+		tradeBotData.setTimestamp(NTP.getTime());
+		repository.getCrossChainRepository().save(tradeBotData);
+		repository.saveChanges();
+
+		if (Settings.getInstance().isTradebotSystrayEnabled())
+			SysTray.getInstance().showMessage("Trade-Bot", String.format("%s: %s", tradeBotData.getAtAddress(), newState), MessageType.INFO);
+
+		if (logMessageSupplier != null)
+			LOGGER.info(logMessageSupplier.get());
+
+		LOGGER.debug(() -> String.format("new state for trade-bot entry based on AT %s: %s", tradeBotData.getAtAddress(), newState));
+
+		notifyStateChange(tradeBotData);
+	}
+
+	/** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */
+	/*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, StateNameAndValueSupplier newStateSupplier, Supplier<String> logMessageSupplier) throws DataException {
+		updateTradeBotState(repository, tradeBotData, newStateSupplier.getState(), newStateSupplier.getStateValue(), logMessageSupplier);
+	}
+
+	/** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */
+	/*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, Supplier<String> logMessageSupplier) throws DataException {
+		updateTradeBotState(repository, tradeBotData, tradeBotData.getState(), tradeBotData.getStateValue(), logMessageSupplier);
+	}
+
+	/*package*/ static void notifyStateChange(TradeBotData tradeBotData) {
+		StateChangeEvent stateChangeEvent = new StateChangeEvent(tradeBotData);
+		EventBus.INSTANCE.notify(stateChangeEvent);
+	}
+
+	/*package*/ static AcctTradeBot findTradeBotForAcct(ACCT acct) {
+		Supplier<AcctTradeBot> acctTradeBotSupplier = acctTradeBotSuppliers.get(acct.getClass());
+		if (acctTradeBotSupplier == null)
+			return null;
+
+		return acctTradeBotSupplier.get();
+	}
+
+	// PRESENCE-related
+
+	public Collection<TradePresenceData> getAllTradePresences() {
+		return this.safeAllTradePresencesByPubkey.values();
+	}
+
+	/** Trade presence timestamps expire in the 'future' so any that reach 'now' have expired and are removed. */
+	private void expireOldPresenceTimestamps() {
+		long now = NTP.getTime();
+
+		int allRemovedCount = 0;
+		synchronized (this.allTradePresencesByPubkey) {
+			int preRemoveCount = this.allTradePresencesByPubkey.size();
+			this.allTradePresencesByPubkey.values().removeIf(tradePresenceData -> tradePresenceData.getTimestamp() <= now);
+			allRemovedCount = this.allTradePresencesByPubkey.size() - preRemoveCount;
+		}
+
+		int ourRemovedCount = 0;
+		synchronized (this.ourTradePresenceTimestampsByPubkey) {
+			int preRemoveCount = this.ourTradePresenceTimestampsByPubkey.size();
+			this.ourTradePresenceTimestampsByPubkey.values().removeIf(timestamp -> timestamp < now);
+			ourRemovedCount = this.ourTradePresenceTimestampsByPubkey.size() - preRemoveCount;
+		}
+
+		if (allRemovedCount > 0)
+			LOGGER.debug("Removed {} expired trade presences, of which {} ours", allRemovedCount, ourRemovedCount);
+	}
+
+	/*package*/ void updatePresence(Repository repository, TradeBotData tradeBotData, CrossChainTradeData tradeData)
+			throws DataException {
+		String atAddress = tradeBotData.getAtAddress();
+
+		PrivateKeyAccount tradeNativeAccount = new PrivateKeyAccount(repository, tradeBotData.getTradePrivateKey());
+		String signerAddress = tradeNativeAccount.getAddress();
+
+		/*
+		* There's no point in Alice trying to broadcast presence for an AT that isn't locked to her,
+		* as other peers won't be able to verify as signing public key isn't yet in the AT's data segment.
+		*/
+		if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) {
+			// Signer is neither Bob, nor trade locked to Alice
+			LOGGER.trace("Can't provide trade presence for our AT {} as it's not yet locked to Alice", atAddress);
+			return;
+		}
+
+		long now = NTP.getTime();
+		long newExpiry = generateExpiry(now);
+		ByteArray pubkeyByteArray = ByteArray.wrap(tradeNativeAccount.getPublicKey());
+
+		// If map entry's timestamp is missing, or within early renewal period, use the new expiry - otherwise use existing timestamp.
+		synchronized (this.ourTradePresenceTimestampsByPubkey) {
+			Long currentTimestamp = this.ourTradePresenceTimestampsByPubkey.get(pubkeyByteArray);
+
+			if (currentTimestamp != null && currentTimestamp - now > EARLY_RENEWAL_PERIOD) {
+				// timestamp still good
+				LOGGER.trace("Current trade presence timestamp {} still good for our trade {}", currentTimestamp, atAddress);
+				return;
+			}
+
+			this.ourTradePresenceTimestampsByPubkey.put(pubkeyByteArray, newExpiry);
+		}
+
+		// Create signature
+		byte[] signature = tradeNativeAccount.sign(Longs.toByteArray(newExpiry));
+
+		// Add new trade presence to queue to be broadcast around network
+		TradePresenceData tradePresenceData = new TradePresenceData(newExpiry, tradeNativeAccount.getPublicKey(), signature, atAddress);
+		this.pendingTradePresences.add(tradePresenceData);
+
+		this.allTradePresencesByPubkey.put(pubkeyByteArray, tradePresenceData);
+		rebuildSafeAllTradePresences();
+
+		LOGGER.trace("New trade presence timestamp {} for our trade {}", newExpiry, atAddress);
+
+		EventBus.INSTANCE.notify(new TradePresenceEvent(tradePresenceData));
+	}
+
+	private void rebuildSafeAllTradePresences() {
+		synchronized (this.allTradePresencesByPubkey) {
+			// Collect into a *new* unmodifiable map.
+			this.safeAllTradePresencesByPubkey = Map.copyOf(this.allTradePresencesByPubkey);
+		}
+	}
+
+	private void broadcastPresenceTimestamps() {
+		// If we have new trade presences that are pending broadcast, send those as a priority
+		if (!this.pendingTradePresences.isEmpty()) {
+			// Create a copy for Network to safely use in another thread
+			List<TradePresenceData> safeTradePresences;
+			synchronized (this.pendingTradePresences) {
+				safeTradePresences = List.copyOf(this.pendingTradePresences);
+				this.pendingTradePresences.clear();
+			}
+
+			LOGGER.debug("Broadcasting {} new trade presences", safeTradePresences.size());
+
+			TradePresencesMessage tradePresencesMessage = new TradePresencesMessage(safeTradePresences);
+			RNSNetwork.getInstance().broadcast(peer -> tradePresencesMessage);
+
+			return;
+		}
+
+		// As we have no new trade presences, check whether it's time to do a general broadcast
+		Long now = NTP.getTime();
+		if (now == null || now < nextTradePresenceBroadcastTimestamp)
+			return;
+
+		nextTradePresenceBroadcastTimestamp = now + PRESENCE_BROADCAST_INTERVAL;
+
+		List<TradePresenceData> safeTradePresences = List.copyOf(this.safeAllTradePresencesByPubkey.values());
+
+		LOGGER.debug("Broadcasting all {} known trade presences. Next broadcast timestamp: {}",
+				safeTradePresences.size(), nextTradePresenceBroadcastTimestamp
+		);
+
+		GetTradePresencesMessage getTradePresencesMessage = new GetTradePresencesMessage(safeTradePresences);
+		RNSNetwork.getInstance().broadcast(peer -> getTradePresencesMessage);
+	}
+
+	// Network message processing
+
+	public void onGetTradePresencesMessage(RNSPeer peer, Message message) {
+		GetTradePresencesMessage getTradePresencesMessage = (GetTradePresencesMessage) message;
+
+		List<TradePresenceData> peersTradePresences = getTradePresencesMessage.getTradePresences();
+
+		// Create mutable copy from safe snapshot
+		Map<ByteArray, TradePresenceData> entriesUnknownToPeer = new HashMap<>(this.safeAllTradePresencesByPubkey);
+		int knownCount = entriesUnknownToPeer.size();
+
+		for (TradePresenceData peersTradePresence : peersTradePresences) {
+			ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey());
+
+			TradePresenceData ourEntry = entriesUnknownToPeer.get(pubkeyByteArray);
+
+			if (ourEntry != null && ourEntry.getTimestamp() == peersTradePresence.getTimestamp())
+				entriesUnknownToPeer.remove(pubkeyByteArray);
+		}
+
+		if (entriesUnknownToPeer.isEmpty())
+			return;
+
+		LOGGER.debug("Sending {} trade presences to peer {} after excluding their {} from known {}",
+				entriesUnknownToPeer.size(), peer, peersTradePresences.size(), knownCount
+		);
+
+		// Send complement to peer
+		List<TradePresenceData> safeTradePresences = List.copyOf(entriesUnknownToPeer.values());
+		Message responseMessage = new TradePresencesMessage(safeTradePresences);
+		//if (!peer.sendMessage(responseMessage)) {
+		//	peer.disconnect("failed to send TRADE_PRESENCES response");
+		//	return;
+		//}
+		peer.sendMessage(responseMessage);
+	}
+
+	public void onTradePresencesMessage(RNSPeer peer, Message message) {
+		TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) message;
+
+		List<TradePresenceData> peersTradePresences = tradePresencesMessage.getTradePresences();
+
+		long now = NTP.getTime();
+		// Timestamps before this are too far into the past
+		long pastThreshold = now;
+		// Timestamps after this are too far into the future
+		long futureThreshold = now + PRESENCE_LIFETIME;
+
+		Map<ByteArray, Supplier<ACCT>> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap();
+
+		int newCount = 0;
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			for (TradePresenceData peersTradePresence : peersTradePresences) {
+				long timestamp = peersTradePresence.getTimestamp();
+
+				// Ignore if timestamp is out of bounds
+				if (timestamp < pastThreshold || timestamp > futureThreshold) {
+					if (timestamp < pastThreshold)
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold
+								);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold
+						);
+
+					continue;
+				}
+
+				ByteArray pubkeyByteArray = ByteArray.wrap(peersTradePresence.getPublicKey());
+
+				// Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older
+				TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray);
+				if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) {
+					if (timestamp == existingTradeData.getTimestamp())
+						LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before",
+								peersTradePresence.getAtAddress(), peer, timestamp
+						);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp()
+						);
+
+					continue;
+				}
+
+				// Check timestamp signature
+				byte[] timestampSignature = peersTradePresence.getSignature();
+				byte[] timestampBytes = Longs.toByteArray(timestamp);
+				byte[] publicKey = peersTradePresence.getPublicKey();
+				if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				ATData atData = repository.getATRepository().fromATAddress(peersTradePresence.getAtAddress());
+				if (atData == null || atData.getIsFrozen() || atData.getIsFinished()) {
+					if (atData == null)
+						LOGGER.trace("Ignoring trade presence {} from peer {} as AT doesn't exist",
+								peersTradePresence.getAtAddress(), peer
+						);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as AT is frozen or finished",
+								peersTradePresence.getAtAddress(), peer
+						);
+
+					continue;
+				}
+
+				ByteArray atCodeHash = ByteArray.wrap(atData.getCodeHash());
+				Supplier<ACCT> acctSupplier = acctSuppliersByCodeHash.get(atCodeHash);
+				if (acctSupplier == null) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as AT isn't a known ACCT?",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				CrossChainTradeData tradeData = acctSupplier.get().populateTradeData(repository, atData);
+				if (tradeData == null) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as trade data not found?",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				// Convert signer's public key to address form
+				String signerAddress = peersTradePresence.getTradeAddress();
+
+				// Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form)
+				if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				// This is new to us
+				this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence);
+				++newCount;
+
+				LOGGER.trace("Added trade presence {} from peer {} with timestamp {}",
+						peersTradePresence.getAtAddress(), peer, timestamp
+				);
+
+				EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence));
+			}
+		} catch (DataException e) {
+			LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e);
+		}
+
+		if (newCount > 0) {
+			LOGGER.debug("New trade presences: {}, all trade presences: {}", newCount, allTradePresencesByPubkey.size());
+			rebuildSafeAllTradePresences();
+		}
+	}
+
+	public void bridgePresence(long timestamp, byte[] publicKey, byte[] signature, String atAddress) {
+		long expiry = generateExpiry(timestamp);
+		ByteArray pubkeyByteArray = ByteArray.wrap(publicKey);
+
+		TradePresenceData fakeTradePresenceData = new TradePresenceData(expiry, publicKey, signature, atAddress);
+
+		// Only bridge if trade presence expiry timestamp is newer
+		TradePresenceData computedTradePresenceData = this.allTradePresencesByPubkey.compute(pubkeyByteArray, (k, v) ->
+				v == null || v.getTimestamp() < expiry ? fakeTradePresenceData : v
+		);
+
+		if (computedTradePresenceData == fakeTradePresenceData) {
+			LOGGER.trace("Bridged PRESENCE transaction for trade {} with timestamp {}", atAddress, expiry);
+			rebuildSafeAllTradePresences();
+
+			EventBus.INSTANCE.notify(new TradePresenceEvent(fakeTradePresenceData));
+		}
+	}
+
+	/** Decorates a CrossChainTradeData object with Alice / Bob trade-bot presence timestamp, if available. */
+	public void decorateTradeDataWithPresence(CrossChainTradeData crossChainTradeData) {
+		// Match by AT address, then check for Bob vs Alice
+		this.safeAllTradePresencesByPubkey.values().stream()
+				.filter(tradePresenceData -> tradePresenceData.getAtAddress().equals(crossChainTradeData.qortalAtAddress))
+				.forEach(tradePresenceData -> {
+					String signerAddress = tradePresenceData.getTradeAddress();
+
+					// Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form)
+					if (signerAddress.equals(crossChainTradeData.qortalCreatorTradeAddress))
+						crossChainTradeData.creatorPresenceExpiry = tradePresenceData.getTimestamp();
+					else if (signerAddress.equals(crossChainTradeData.qortalPartnerAddress))
+						crossChainTradeData.partnerPresenceExpiry = tradePresenceData.getTimestamp();
+				});
+	}
+
+	/** Removes any trades that have had multiple failures */
+	public List<CrossChainTradeData> removeFailedTrades(Repository repository, List<CrossChainTradeData> crossChainTrades) {
+		Long now = NTP.getTime();
+		if (now == null) {
+			return crossChainTrades;
+		}
+
+		List<CrossChainTradeData> updatedCrossChainTrades = new ArrayList<>(crossChainTrades);
+		int getMaxTradeOfferAttempts = Settings.getInstance().getMaxTradeOfferAttempts();
+
+		for (CrossChainTradeData crossChainTradeData : crossChainTrades) {
+			// We only care about trades in the OFFERING state
+			if (crossChainTradeData.mode != AcctMode.OFFERING) {
+				failedTrades.remove(crossChainTradeData.qortalAtAddress);
+				validTrades.remove(crossChainTradeData.qortalAtAddress);
+				continue;
+			}
+
+			// Return recently cached values if they exist
+			Long failedTimestamp = failedTrades.get(crossChainTradeData.qortalAtAddress);
+			if (failedTimestamp != null && now - failedTimestamp < 60 * 60 * 1000L) {
+				updatedCrossChainTrades.remove(crossChainTradeData);
+				//LOGGER.info("Removing cached failed trade AT {}", crossChainTradeData.qortalAtAddress);
+				continue;
+			}
+			Long validTimestamp = validTrades.get(crossChainTradeData.qortalAtAddress);
+			if (validTimestamp != null && now - validTimestamp < 60 * 60 * 1000L) {
+				//LOGGER.info("NOT removing cached valid trade AT {}", crossChainTradeData.qortalAtAddress);
+				continue;
+			}
+
+			try {
+				List<TransactionData> transactions = repository.getTransactionRepository().getUnconfirmedTransactions(Arrays.asList(Transaction.TransactionType.MESSAGE), null, null, null, null);
+
+				for (TransactionData transactionData : transactions) {
+					// Treat as failed if buy attempt was more than 60 mins ago (as it's still in the OFFERING state)
+					if (transactionData.getRecipient().equals(crossChainTradeData.qortalCreatorTradeAddress) && now - transactionData.getTimestamp() > 60*60*1000L) {
+						failedTrades.put(crossChainTradeData.qortalAtAddress, now);
+						updatedCrossChainTrades.remove(crossChainTradeData);
+					} else {
+						validTrades.put(crossChainTradeData.qortalAtAddress, now);
+					}
+				}
+
+			} catch (DataException e) {
+				LOGGER.info("Unable to determine failed state of AT {}", crossChainTradeData.qortalAtAddress);
+            }
+		}
+
+		return updatedCrossChainTrades;
+	}
+
+	public boolean isFailedTrade(Repository repository, CrossChainTradeData crossChainTradeData) {
+		List<CrossChainTradeData> results = removeFailedTrades(repository, Arrays.asList(crossChainTradeData));
+		return results.isEmpty();
+	}
+
+	private long generateExpiry(long timestamp) {
+		return ((timestamp - 1) / EXPIRY_ROUNDING) * EXPIRY_ROUNDING + PRESENCE_LIFETIME;
+	}
+
+}

src/main/java/org/qortal/crosschain/Bitcoiny.java

@@ -83,6 +83,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 		return this.bitcoinjContext;
 	}
 
+	@Override
 	public String getCurrencyCode() {
 		return this.currencyCode;
 	}

src/main/java/org/qortal/crosschain/ForeignBlockchain.java

@@ -2,6 +2,8 @@ package org.qortal.crosschain;
 
 public interface ForeignBlockchain {
 
+	public String getCurrencyCode();
+
 	public boolean isValidAddress(String address);
 
 	public boolean isValidWalletKey(String walletKey);

src/main/java/org/qortal/data/account/AddressAmountData.java

@@ -0,0 +1,54 @@
+package org.qortal.data.account;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+import java.util.Objects;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class AddressAmountData {
+
+    private String address;
+
+    @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
+    private long amount;
+
+    public AddressAmountData() {
+    }
+
+    public AddressAmountData(String address, long amount) {
+
+        this.address = address;
+        this.amount = amount;
+    }
+
+    public String getAddress() {
+        return address;
+    }
+
+    public long getAmount() {
+        return amount;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AddressAmountData that = (AddressAmountData) o;
+        return amount == that.amount && Objects.equals(address, that.address);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(address, amount);
+    }
+
+    @Override
+    public String toString() {
+        return "AddressAmountData{" +
+                "address='" + address + '\'' +
+                ", amount=" + amount +
+                '}';
+    }
+}

src/main/java/org/qortal/data/account/BlockHeightRange.java

@@ -0,0 +1,59 @@
+package org.qortal.data.account;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.Objects;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class BlockHeightRange {
+
+    private int begin;
+
+    private int end;
+
+    private boolean isRewardDistribution;
+
+    public BlockHeightRange() {
+    }
+
+    public BlockHeightRange(int begin, int end, boolean isRewardDistribution) {
+        this.begin = begin;
+        this.end = end;
+        this.isRewardDistribution = isRewardDistribution;
+    }
+
+    public int getBegin() {
+        return begin;
+    }
+
+    public int getEnd() {
+        return end;
+    }
+
+    public boolean isRewardDistribution() {
+        return isRewardDistribution;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        BlockHeightRange that = (BlockHeightRange) o;
+        return begin == that.begin && end == that.end;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(begin, end);
+    }
+
+    @Override
+    public String toString() {
+        return "BlockHeightRange{" +
+                "begin=" + begin +
+                ", end=" + end +
+                ", isRewardDistribution=" + isRewardDistribution +
+                '}';
+    }
+}

src/main/java/org/qortal/data/account/BlockHeightRangeAddressAmounts.java

@@ -0,0 +1,52 @@
+package org.qortal.data.account;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.List;
+import java.util.Objects;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class BlockHeightRangeAddressAmounts {
+
+    private BlockHeightRange range;
+
+    private List<AddressAmountData> amounts;
+
+    public BlockHeightRangeAddressAmounts() {
+    }
+
+    public BlockHeightRangeAddressAmounts(BlockHeightRange range, List<AddressAmountData> amounts) {
+        this.range = range;
+        this.amounts = amounts;
+    }
+
+    public BlockHeightRange getRange() {
+        return range;
+    }
+
+    public List<AddressAmountData> getAmounts() {
+        return amounts;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        BlockHeightRangeAddressAmounts that = (BlockHeightRangeAddressAmounts) o;
+        return Objects.equals(range, that.range) && Objects.equals(amounts, that.amounts);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(range, amounts);
+    }
+
+    @Override
+    public String toString() {
+        return "BlockHeightRangeAddressAmounts{" +
+                "range=" + range +
+                ", amounts=" + amounts +
+                '}';
+    }
+}

src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java

@@ -0,0 +1,34 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.arbitrary.misc.Service;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ArbitraryDataIndex {
+
+    public String t;
+    public String n;
+    public int c;
+    public String l;
+
+    public ArbitraryDataIndex() {}
+
+    public ArbitraryDataIndex(String t, String n, int c, String l) {
+        this.t = t;
+        this.n = n;
+        this.c = c;
+        this.l = l;
+    }
+
+    @Override
+    public String toString() {
+        return "ArbitraryDataIndex{" +
+                "t='" + t + '\'' +
+                ", n='" + n + '\'' +
+                ", c=" + c +
+                ", l='" + l + '\'' +
+                '}';
+    }
+}

src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java

@@ -0,0 +1,41 @@
+package org.qortal.data.arbitrary;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ArbitraryDataIndexDetail {
+
+    public String issuer;
+    public int rank;
+    public String term;
+    public String name;
+    public int category;
+    public String link;
+    public String indexIdentifer;
+
+    public ArbitraryDataIndexDetail() {}
+
+    public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index, String indexIdentifer) {
+        this.issuer = issuer;
+        this.rank = rank;
+        this.term = index.t;
+        this.name = index.n;
+        this.category = index.c;
+        this.link = index.l;
+        this.indexIdentifer = indexIdentifer;
+    }
+
+    @Override
+    public String toString() {
+        return "ArbitraryDataIndexDetail{" +
+                "issuer='" + issuer + '\'' +
+                ", rank=" + rank +
+                ", term='" + term + '\'' +
+                ", name='" + name + '\'' +
+                ", category=" + category +
+                ", link='" + link + '\'' +
+                ", indexIdentifer='" + indexIdentifer + '\'' +
+                '}';
+    }
+}

src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java

@@ -0,0 +1,38 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.arbitrary.misc.Service;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.Objects;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ArbitraryDataIndexScoreKey {
+
+    public String name;
+    public int category;
+    public String link;
+
+    public ArbitraryDataIndexScoreKey() {}
+
+    public ArbitraryDataIndexScoreKey(String name, int category, String link) {
+        this.name = name;
+        this.category = category;
+        this.link = link;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ArbitraryDataIndexScoreKey that = (ArbitraryDataIndexScoreKey) o;
+        return category == that.category && Objects.equals(name, that.name) && Objects.equals(link, that.link);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, category, link);
+    }
+
+
+}

src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java

@@ -0,0 +1,38 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.arbitrary.misc.Service;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ArbitraryDataIndexScorecard {
+
+    public double score;
+    public String name;
+    public int category;
+    public String link;
+
+    public ArbitraryDataIndexScorecard() {}
+
+    public ArbitraryDataIndexScorecard(double score, String name, int category, String link) {
+        this.score = score;
+        this.name = name;
+        this.category = category;
+        this.link = link;
+    }
+
+    public double getScore() {
+        return score;
+    }
+
+    @Override
+    public String toString() {
+        return "ArbitraryDataIndexScorecard{" +
+                "score=" + score +
+                ", name='" + name + '\'' +
+                ", category=" + category +
+                ", link='" + link + '\'' +
+                '}';
+    }
+}

src/main/java/org/qortal/data/arbitrary/DataMonitorInfo.java

@@ -0,0 +1,57 @@
+package org.qortal.data.arbitrary;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class DataMonitorInfo {
+    private long timestamp;
+    private String identifier;
+    private String name;
+    private String service;
+    private String description;
+    private long transactionTimestamp;
+    private long latestPutTimestamp;
+
+    public DataMonitorInfo() {
+    }
+
+    public DataMonitorInfo(long timestamp, String identifier, String name, String service, String description, long transactionTimestamp, long latestPutTimestamp) {
+
+        this.timestamp = timestamp;
+        this.identifier = identifier;
+        this.name = name;
+        this.service = service;
+        this.description = description;
+        this.transactionTimestamp = transactionTimestamp;
+        this.latestPutTimestamp = latestPutTimestamp;
+    }
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public String getIdentifier() {
+        return identifier;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getService() {
+        return service;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public long getTransactionTimestamp() {
+        return transactionTimestamp;
+    }
+
+    public long getLatestPutTimestamp() {
+        return latestPutTimestamp;
+    }
+}

src/main/java/org/qortal/data/arbitrary/IndexCache.java

@@ -0,0 +1,23 @@
+package org.qortal.data.arbitrary;
+
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class IndexCache {
+
+    public static final IndexCache SINGLETON = new IndexCache();
+    private ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> indicesByTerm = new ConcurrentHashMap<>();
+    private ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> indicesByIssuer = new ConcurrentHashMap<>();
+
+    public static IndexCache getInstance() {
+        return SINGLETON;
+    }
+
+    public ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> getIndicesByTerm() {
+        return indicesByTerm;
+    }
+
+    public ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> getIndicesByIssuer() {
+        return indicesByIssuer;
+    }
+}

src/main/java/org/qortal/data/arbitrary/RNSArbitraryDirectConnectionInfo.java

@@ -0,0 +1,59 @@
+package org.qortal.data.arbitrary;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+
+public class RNSArbitraryDirectConnectionInfo {
+
+    private final byte[] signature;
+    private final String peerAddress;
+    private final List<byte[]> hashes;
+    private final long timestamp;
+
+    public RNSArbitraryDirectConnectionInfo(byte[] signature, String peerAddress, List<byte[]> hashes, long timestamp) {
+        this.signature = signature;
+        this.peerAddress = peerAddress;
+        this.hashes = hashes;
+        this.timestamp = timestamp;
+    }
+
+    public byte[] getSignature() {
+        return this.signature;
+    }
+
+    public String getPeerAddress() {
+        return this.peerAddress;
+    }
+
+    public List<byte[]> getHashes() {
+        return this.hashes;
+    }
+    
+    public long getTimestamp() {
+        return this.timestamp;
+    }
+
+    public int getHashCount() {
+        if (this.hashes == null) {
+            return 0;
+        }
+        return this.hashes.size();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this)
+            return true;
+
+        if (!(other instanceof ArbitraryDirectConnectionInfo))
+            return false;
+
+        ArbitraryDirectConnectionInfo otherDirectConnectionInfo = (ArbitraryDirectConnectionInfo) other;
+
+        return Arrays.equals(this.signature, otherDirectConnectionInfo.getSignature())
+                && Objects.equals(this.peerAddress, otherDirectConnectionInfo.getPeerAddress())
+                && Objects.equals(this.hashes, otherDirectConnectionInfo.getHashes())
+                && Objects.equals(this.timestamp, otherDirectConnectionInfo.getTimestamp());
+    }
+}

src/main/java/org/qortal/data/arbitrary/RNSArbitraryFileListResponseInfo.java

@@ -0,0 +1,11 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.network.RNSPeer;
+
+public class RNSArbitraryFileListResponseInfo extends  RNSArbitraryRelayInfo {
+
+    public RNSArbitraryFileListResponseInfo(String hash58, String signature58, RNSPeer peer, Long timestamp, Long requestTime, Integer requestHops) {
+        super(hash58, signature58, peer, timestamp, requestTime, requestHops);
+    }
+
+}

src/main/java/org/qortal/data/arbitrary/RNSArbitraryRelayInfo.java

@@ -0,0 +1,73 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.network.RNSPeer;
+
+import java.util.Objects;
+
+public class RNSArbitraryRelayInfo {
+
+    private final String hash58;
+    private final String signature58;
+    private final RNSPeer peer;
+    private final Long timestamp;
+    private final Long requestTime;
+    private final Integer requestHops;
+
+    public RNSArbitraryRelayInfo(String hash58, String signature58, RNSPeer peer, Long timestamp, Long requestTime, Integer requestHops) {
+        this.hash58 = hash58;
+        this.signature58 = signature58;
+        this.peer = peer;
+        this.timestamp = timestamp;
+        this.requestTime = requestTime;
+        this.requestHops = requestHops;
+    }
+
+    public boolean isValid() {
+        return this.getHash58() != null && this.getSignature58() != null
+                && this.getPeer() != null && this.getTimestamp() != null;
+    }
+
+    public String getHash58() {
+        return this.hash58;
+    }
+
+    public String getSignature58() {
+        return signature58;
+    }
+
+    public RNSPeer getPeer() {
+        return peer;
+    }
+
+    public Long getTimestamp() {
+        return timestamp;
+    }
+
+    public Long getRequestTime() {
+        return this.requestTime;
+    }
+
+    public Integer getRequestHops() {
+        return this.requestHops;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == this)
+            return true;
+
+        if (!(other instanceof RNSArbitraryRelayInfo))
+            return false;
+
+        RNSArbitraryRelayInfo otherRelayInfo = (RNSArbitraryRelayInfo) other;
+
+        return this.peer == otherRelayInfo.getPeer()
+                && Objects.equals(this.hash58, otherRelayInfo.getHash58())
+                && Objects.equals(this.signature58, otherRelayInfo.getSignature58());
+    }
+}

src/main/java/org/qortal/data/block/BlockData.java

@@ -1,8 +1,11 @@
 package org.qortal.data.block;
 
 import com.google.common.primitives.Bytes;
+import org.qortal.account.Account;
 import org.qortal.block.BlockChain;
-import org.qortal.crypto.Crypto;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
 import org.qortal.utils.NTP;
 
@@ -224,7 +227,7 @@ public class BlockData implements Serializable {
 		}
 		return 0;
 	}
-
+	
 	public boolean isTrimmed() {
 		long onlineAccountSignaturesTrimmedTimestamp = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime();
 		long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime();
@@ -232,11 +235,31 @@ public class BlockData implements Serializable {
 		return blockTimestamp < onlineAccountSignaturesTrimmedTimestamp && blockTimestamp < currentTrimmableTimestamp;
 	}
 
+	public String getMinterAddressFromPublicKey() {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			return Account.getRewardShareMintingAddress(repository, this.minterPublicKey);
+		} catch (DataException e) {
+			return "Unknown";
+		}
+	}
+
+	public int getMinterLevelFromPublicKey() {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			return Account.getRewardShareEffectiveMintingLevel(repository, this.minterPublicKey);
+		} catch (DataException e) {
+			return 0;
+		}
+	}
+	
 	// JAXB special
 
 	@XmlElement(name = "minterAddress")
 	protected String getMinterAddress() {
-		return Crypto.toAddress(this.minterPublicKey);
+		return getMinterAddressFromPublicKey();
 	}
 
+	@XmlElement(name = "minterLevel")
+	protected int getMinterLevel() {
+		return getMinterLevelFromPublicKey();
+	}
 }

src/main/java/org/qortal/data/block/DecodedOnlineAccountData.java

@@ -0,0 +1,85 @@
+package org.qortal.data.block;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.Objects;
+
+// All properties to be converted to JSON via JAX-RS
+@XmlAccessorType(XmlAccessType.FIELD)
+public class DecodedOnlineAccountData {
+
+    private long onlineTimestamp;
+    private String minter;
+    private String recipient;
+    private int sharePercent;
+    private boolean minterGroupMember;
+    private String name;
+    private int level;
+
+    public DecodedOnlineAccountData() {
+    }
+
+    public DecodedOnlineAccountData(long onlineTimestamp, String minter, String recipient, int sharePercent, boolean minterGroupMember, String name, int level) {
+        this.onlineTimestamp = onlineTimestamp;
+        this.minter = minter;
+        this.recipient = recipient;
+        this.sharePercent = sharePercent;
+        this.minterGroupMember = minterGroupMember;
+        this.name = name;
+        this.level = level;
+    }
+
+    public long getOnlineTimestamp() {
+        return onlineTimestamp;
+    }
+
+    public String getMinter() {
+        return minter;
+    }
+
+    public String getRecipient() {
+        return recipient;
+    }
+
+    public int getSharePercent() {
+        return sharePercent;
+    }
+
+    public boolean isMinterGroupMember() {
+        return minterGroupMember;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public int getLevel() {
+        return level;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        DecodedOnlineAccountData that = (DecodedOnlineAccountData) o;
+        return onlineTimestamp == that.onlineTimestamp && sharePercent == that.sharePercent && minterGroupMember == that.minterGroupMember && level == that.level && Objects.equals(minter, that.minter) && Objects.equals(recipient, that.recipient) && Objects.equals(name, that.name);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(onlineTimestamp, minter, recipient, sharePercent, minterGroupMember, name, level);
+    }
+
+    @Override
+    public String toString() {
+        return "DecodedOnlineAccountData{" +
+                "onlineTimestamp=" + onlineTimestamp +
+                ", minter='" + minter + '\'' +
+                ", recipient='" + recipient + '\'' +
+                ", sharePercent=" + sharePercent +
+                ", minterGroupMember=" + minterGroupMember +
+                ", name='" + name + '\'' +
+                ", level=" + level +
+                '}';
+    }
+}

src/main/java/org/qortal/data/network/RNSPeerData.java

@@ -0,0 +1,117 @@
+package org.qortal.data.network;
+
+import io.swagger.v3.oas.annotations.media.Schema;
+import org.qortal.network.PeerAddress;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlTransient;
+import static org.apache.commons.codec.binary.Hex.encodeHexString;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class RNSPeerData {
+
+	//public static final int MAX_PEER_ADDRESS_SIZE = 255;
+
+	// Properties
+
+	//// Don't expose this via JAXB - use pretty getter instead
+	//@XmlTransient
+	//@Schema(hidden = true)
+	//private PeerAddress peerAddress;
+	private byte[] peerAddress;
+
+	private Long lastAttempted;
+	private Long lastConnected;
+	private Long lastMisbehaved;
+	private Long addedWhen;
+	private String addedBy;
+
+	/** The number of consecutive times we failed to sync with this peer */
+	private int failedSyncCount = 0;
+
+	// Constructors
+
+	// necessary for JAXB serialization
+	protected RNSPeerData() {
+	}
+
+	public RNSPeerData(byte[] peerAddress, Long lastAttempted, Long lastConnected, Long lastMisbehaved, Long addedWhen, String addedBy) {
+		this.peerAddress = peerAddress;
+		this.lastAttempted = lastAttempted;
+		this.lastConnected = lastConnected;
+		this.lastMisbehaved = lastMisbehaved;
+		this.addedWhen = addedWhen;
+		this.addedBy = addedBy;
+	}
+
+	public RNSPeerData(byte[] peerAddress, Long addedWhen, String addedBy) {
+		this(peerAddress, null, null, null, addedWhen, addedBy);
+	}
+
+	public RNSPeerData(byte[] peerAddress) {
+		this(peerAddress, null, null, null, null, null);
+	}
+
+	// Getters / setters
+
+	// Don't let JAXB use this getter
+	@XmlTransient
+	@Schema(hidden = true)
+	public byte[] getAddress() {
+		return this.peerAddress;
+	}
+
+	public Long getLastAttempted() {
+		return this.lastAttempted;
+	}
+
+	public void setLastAttempted(Long lastAttempted) {
+		this.lastAttempted = lastAttempted;
+	}
+
+	public Long getLastConnected() {
+		return this.lastConnected;
+	}
+
+	public void setLastConnected(Long lastConnected) {
+		this.lastConnected = lastConnected;
+	}
+
+	public Long getLastMisbehaved() {
+		return this.lastMisbehaved;
+	}
+
+	public void setLastMisbehaved(Long lastMisbehaved) {
+		this.lastMisbehaved = lastMisbehaved;
+	}
+
+	public Long getAddedWhen() {
+		return this.addedWhen;
+	}
+
+	public String getAddedBy() {
+		return this.addedBy;
+	}
+
+	public int getFailedSyncCount() {
+		return this.failedSyncCount;
+	}
+
+	public void setFailedSyncCount(int failedSyncCount) {
+		this.failedSyncCount = failedSyncCount;
+	}
+
+	public void incrementFailedSyncCount() {
+		this.failedSyncCount++;
+	}
+
+	// Pretty peerAddress getter for JAXB
+	@XmlElement(name = "address")
+	protected String getPrettyAddress() {
+		return encodeHexString(this.peerAddress);
+	}
+
+}

src/main/java/org/qortal/data/system/DbConnectionInfo.java

@@ -0,0 +1,35 @@
+package org.qortal.data.system;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class DbConnectionInfo {
+
+    private long updated;
+
+    private String owner;
+
+    private String state;
+
+    public DbConnectionInfo() {
+    }
+
+    public DbConnectionInfo(long timeOpened, String owner, String state) {
+        this.updated = timeOpened;
+        this.owner = owner;
+        this.state = state;
+    }
+
+    public long getUpdated() {
+        return updated;
+    }
+
+    public String getOwner() {
+        return owner;
+    }
+
+    public String getState() {
+        return state;
+    }
+}

src/main/java/org/qortal/data/system/SystemInfo.java

@@ -0,0 +1,49 @@
+package org.qortal.data.system;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SystemInfo {
+
+    private long freeMemory;
+
+    private long memoryInUse;
+
+    private long totalMemory;
+
+    private long maxMemory;
+
+    private int availableProcessors;
+
+    public SystemInfo() {
+    }
+
+    public SystemInfo(long freeMemory, long memoryInUse, long totalMemory, long maxMemory, int availableProcessors) {
+        this.freeMemory = freeMemory;
+        this.memoryInUse = memoryInUse;
+        this.totalMemory = totalMemory;
+        this.maxMemory = maxMemory;
+        this.availableProcessors = availableProcessors;
+    }
+
+    public long getFreeMemory() {
+        return freeMemory;
+    }
+
+    public long getMemoryInUse() {
+        return memoryInUse;
+    }
+
+    public long getTotalMemory() {
+        return totalMemory;
+    }
+
+    public long getMaxMemory() {
+        return maxMemory;
+    }
+
+    public int getAvailableProcessors() {
+        return availableProcessors;
+    }
+}

src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java

@@ -200,4 +200,26 @@ public class ArbitraryTransactionData extends TransactionData {
 		return this.payments;
 	}
 
+	@Override
+	public String toString() {
+		return "ArbitraryTransactionData{" +
+				"version=" + version +
+				", service=" + service +
+				", nonce=" + nonce +
+				", size=" + size +
+				", name='" + name + '\'' +
+				", identifier='" + identifier + '\'' +
+				", method=" + method +
+				", compression=" + compression +
+				", dataType=" + dataType +
+				", type=" + type +
+				", timestamp=" + timestamp +
+				", fee=" + fee +
+				", txGroupId=" + txGroupId +
+				", blockHeight=" + blockHeight +
+				", blockSequence=" + blockSequence +
+				", approvalStatus=" + approvalStatus +
+				", approvalHeight=" + approvalHeight +
+				'}';
+	}
 }

src/main/java/org/qortal/event/DataMonitorEvent.java

@@ -0,0 +1,57 @@
+package org.qortal.event;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class DataMonitorEvent implements Event{
+    private long timestamp;
+    private String identifier;
+    private String name;
+    private String service;
+    private String description;
+    private long transactionTimestamp;
+    private long latestPutTimestamp;
+
+    public DataMonitorEvent() {
+    }
+
+    public DataMonitorEvent(long timestamp, String identifier, String name, String service, String description, long transactionTimestamp, long latestPutTimestamp) {
+
+        this.timestamp = timestamp;
+        this.identifier = identifier;
+        this.name = name;
+        this.service = service;
+        this.description = description;
+        this.transactionTimestamp = transactionTimestamp;
+        this.latestPutTimestamp = latestPutTimestamp;
+    }
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public String getIdentifier() {
+        return identifier;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getService() {
+        return service;
+    }
+
+    public String getDescription() {
+        return description;
+    }
+
+    public long getTransactionTimestamp() {
+        return transactionTimestamp;
+    }
+
+    public long getLatestPutTimestamp() {
+        return latestPutTimestamp;
+    }
+}

src/main/java/org/qortal/group/Group.java

@@ -2,6 +2,7 @@ package org.qortal.group;
 
 import org.qortal.account.Account;
 import org.qortal.account.PublicKeyAccount;
+import org.qortal.block.BlockChain;
 import org.qortal.controller.Controller;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.group.*;
@@ -150,7 +151,12 @@ public class Group {
 	// Adminship
 
 	private GroupAdminData getAdmin(String admin) throws DataException {
-		return groupRepository.getAdmin(this.groupData.getGroupId(), admin);
+		if( repository.getBlockRepository().getBlockchainHeight() < BlockChain.getInstance().getAdminQueryFixHeight()) {
+			return groupRepository.getAdminFaulty(this.groupData.getGroupId(), admin);
+		}
+		else {
+			return groupRepository.getAdmin(this.groupData.getGroupId(), admin);
+		}
 	}
 
 	private boolean adminExists(String admin) throws DataException {
@@ -668,8 +674,8 @@ public class Group {
 	public void uninvite(GroupInviteTransactionData groupInviteTransactionData) throws DataException {
 		String invitee = groupInviteTransactionData.getInvitee();
 
-		// If member exists then they were added when invite matched join request
-		if (this.memberExists(invitee)) {
+		// If member exists and the join request is present then they were added when invite matched join request
+		if (this.memberExists(invitee) && groupInviteTransactionData.getJoinReference() != null) {
 			// Rebuild join request using cached reference to transaction that created join request.
 			this.rebuildJoinRequest(invitee, groupInviteTransactionData.getJoinReference());
 

src/main/java/org/qortal/network/RNSCommon.java

@@ -0,0 +1,31 @@
+package org.qortal.network;
+
+public class RNSCommon {
+
+    /**
+     * Destination application name
+     */
+    public static String MAINNET_APP_NAME = "qortal";      // production
+    public static String TESTNET_APP_NAME = "qortaltest";  // test net
+
+    /**
+     * Configuration path relative to the Qortal launch directory
+     */
+    public static String defaultRNSConfigPath = ".reticulum";
+    public static String defaultRNSConfigPathTestnet = ".reticulum_test";
+
+    /**
+     * Default config
+     */
+    public static String defaultRNSConfig = "reticulum_default_config.yml";
+    public static String defaultRNSConfigTestnet = "reticulum_default_testnet_config.yml";
+
+    ///**
+    // * Qortal RNS Destinations
+    // */
+    //public enum RNSDestinations {
+    //    BASE,
+    //    QDN;
+    //}
+    
+}

src/main/java/org/qortal/network/RNSNetwork.java

@@ -0,0 +1,845 @@
+package org.qortal.network;
+
+import io.reticulum.Reticulum;
+import io.reticulum.Transport;
+import io.reticulum.interfaces.ConnectionInterface;
+import io.reticulum.destination.Destination;
+import io.reticulum.destination.DestinationType;
+import io.reticulum.destination.Direction;
+import io.reticulum.destination.ProofStrategy;
+import io.reticulum.identity.Identity;
+import io.reticulum.link.Link;
+import io.reticulum.link.LinkStatus;
+//import io.reticulum.constant.LinkConstant;
+//import static io.reticulum.constant.ReticulumConstant.MTU;
+import io.reticulum.buffer.Buffer;
+import io.reticulum.buffer.BufferedRWPair;
+import io.reticulum.packet.Packet;
+import io.reticulum.packet.PacketReceipt;
+import io.reticulum.packet.PacketReceiptStatus;
+import io.reticulum.transport.AnnounceHandler;
+//import static io.reticulum.link.TeardownSession.DESTINATION_CLOSED;
+//import static io.reticulum.link.TeardownSession.INITIATOR_CLOSED;
+import static io.reticulum.link.TeardownSession.TIMEOUT;
+import static io.reticulum.link.LinkStatus.ACTIVE;
+import static io.reticulum.link.LinkStatus.STALE;
+import static io.reticulum.link.LinkStatus.CLOSED;
+import static io.reticulum.link.LinkStatus.PENDING;
+import static io.reticulum.link.LinkStatus.HANDSHAKE;
+//import static io.reticulum.packet.PacketContextType.LINKCLOSE;
+//import static io.reticulum.identity.IdentityKnownDestination.recall;
+import static io.reticulum.utils.IdentityUtils.concatArrays;
+//import static io.reticulum.constant.ReticulumConstant.TRUNCATED_HASHLENGTH;
+import static io.reticulum.constant.ReticulumConstant.CONFIG_FILE_NAME;
+import lombok.Data;
+//import lombok.Setter;
+//import lombok.Getter;
+import lombok.Synchronized;
+
+import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.StandardCopyOption;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.WRITE;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.channels.SelectionKey;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+//import static java.util.Objects.isNull;
+//import static java.util.Objects.isNull;
+import static java.util.Objects.nonNull;
+//import static org.apache.commons.lang3.BooleanUtils.isTrue;
+//import static org.apache.commons.lang3.BooleanUtils.isFalse;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Iterator;
+//import java.util.Random;
+//import java.util.Scanner;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicLong;
+//import java.util.concurrent.locks.Lock;
+//import java.util.concurrent.locks.ReentrantLock;
+import java.util.Objects;
+import java.util.function.Function;
+import java.time.Instant;
+
+import static org.apache.commons.codec.binary.Hex.encodeHexString;
+import org.qortal.utils.ExecuteProduceConsume;
+import org.qortal.utils.ExecuteProduceConsume.StatsSnapshot;
+import org.qortal.utils.NTP;
+import org.qortal.utils.NamedThreadFactory;
+import org.qortal.network.message.Message;
+import org.qortal.network.message.BlockSummariesV2Message;
+import org.qortal.network.message.TransactionSignaturesMessage;
+import org.qortal.network.message.GetUnconfirmedTransactionsMessage;
+import org.qortal.network.task.RNSBroadcastTask;
+import org.qortal.network.task.RNSPrunePeersTask;
+import org.qortal.data.network.RNSPeerData;
+import org.qortal.controller.Controller;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.data.block.BlockData;
+import org.qortal.data.block.BlockSummaryData;
+import org.qortal.data.transaction.TransactionData;
+
+// logging
+import lombok.extern.slf4j.Slf4j;
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
+
+@Data
+@Slf4j
+public class RNSNetwork {
+
+    Reticulum reticulum;
+    //private static final String APP_NAME = "qortal";
+    static final String APP_NAME = Settings.getInstance().isTestNet() ? RNSCommon.TESTNET_APP_NAME: RNSCommon.MAINNET_APP_NAME;
+    //static final String defaultConfigPath = ".reticulum"; // if empty will look in Reticulums default paths
+    static final String defaultConfigPath = Settings.getInstance().isTestNet() ? RNSCommon.defaultRNSConfigPathTestnet: RNSCommon.defaultRNSConfigPath;
+    private final int MAX_PEERS = Settings.getInstance().getReticulumMaxPeers();
+    private final int MIN_DESIRED_PEERS = Settings.getInstance().getReticulumMinDesiredPeers();
+    // How long [ms] between pruning of peers
+	private long PRUNE_INTERVAL = 1 * 64 * 1000L; // ms;
+    
+    Identity serverIdentity;
+    public Destination baseDestination;
+    private volatile boolean isShuttingDown = false;
+
+    /**
+     * Maintain two lists for each subset of peers
+     *  => a synchronizedList, modified when peers are added/removed
+     *  => an immutable List, automatically rebuild to mirror synchronizedList, served to consumers
+     *  linkedPeers are "initiators" (containing initiator reticulum Link), actively doing work.
+     *  incomimgPeers are "non-initiators", the passive end of bidirectional Reticulum Buffers.
+     */
+    private final List<RNSPeer> linkedPeers = Collections.synchronizedList(new ArrayList<>());
+    private List<RNSPeer> immutableLinkedPeers = Collections.emptyList();
+    private final List<RNSPeer> incomingPeers = Collections.synchronizedList(new ArrayList<>());
+    private List<RNSPeer> immutableIncomingPeers = Collections.emptyList();
+
+    private final ExecuteProduceConsume rnsNetworkEPC;
+    private static final long NETWORK_EPC_KEEPALIVE = 1000L; // 1 second
+    private int totalThreadCount = 0;
+    private final int reticulumMaxNetworkThreadPoolSize = Settings.getInstance().getReticulumMaxNetworkThreadPoolSize();
+
+    // replicating a feature from Network.class needed in for base Message.java,
+    // just in case the classic TCP/IP Networking is turned off.
+    private static final byte[] MAINNET_MESSAGE_MAGIC = new byte[]{0x51, 0x4f, 0x52, 0x54}; // QORT
+    private static final byte[] TESTNET_MESSAGE_MAGIC = new byte[]{0x71, 0x6f, 0x72, 0x54}; // qorT
+    private static final int BROADCAST_CHAIN_TIP_DEPTH = 7; // (~1440 bytes)
+    /**
+     * How long between informational broadcasts to all ACTIVE peers, in milliseconds.
+     */
+    private static final long BROADCAST_INTERVAL = 30 * 1000L; // ms
+    /**
+     * Link low-level ping interval and timeout
+     */
+    private static final long LINK_PING_INTERVAL = 55 * 1000L; // ms
+    private static final long LINK_UNREACHABLE_TIMEOUT = 3 * LINK_PING_INTERVAL;
+
+    //private static final Logger logger = LoggerFactory.getLogger(RNSNetwork.class);
+    
+    // Constructor
+    private RNSNetwork () {
+        log.info("RNSNetwork constructor");
+        try {
+            //String configPath = new java.io.File(defaultConfigPath).getCanonicalPath();
+            log.info("creating config from {}", defaultConfigPath);
+            initConfig(defaultConfigPath);
+            //reticulum = new Reticulum(configPath);
+            reticulum = new Reticulum(defaultConfigPath);
+            var identitiesPath = reticulum.getStoragePath().resolve("identities");
+            if (Files.notExists(identitiesPath)) {
+                Files.createDirectories(identitiesPath);
+            }
+        } catch (IOException e) {
+            log.error("unable to create Reticulum network", e);
+        }
+        log.info("reticulum instance created");
+        log.info("reticulum instance created: {}", reticulum);
+
+        //        Settings.getInstance().getMaxRNSNetworkThreadPoolSize(),   // statically set to 5 below
+        ExecutorService RNSNetworkExecutor = new ThreadPoolExecutor(1,
+                reticulumMaxNetworkThreadPoolSize,
+                NETWORK_EPC_KEEPALIVE, TimeUnit.SECONDS,
+                new SynchronousQueue<Runnable>(),
+                new NamedThreadFactory("RNSNetwork-EPC", Settings.getInstance().getNetworkThreadPriority()));
+        rnsNetworkEPC = new RNSNetworkProcessor(RNSNetworkExecutor);
+    }
+
+    // Note: potentially create persistent serverIdentity (utility rnid) and load it from file
+    public void start() throws IOException, DataException {
+
+        // create identity either from file or new (creating new keys)
+        var serverIdentityPath = reticulum.getStoragePath().resolve("identities/"+APP_NAME);
+        if (Files.isReadable(serverIdentityPath)) {
+            serverIdentity = Identity.fromFile(serverIdentityPath);
+            log.info("server identity loaded from file {}", serverIdentityPath);
+        } else {
+            serverIdentity = new Identity();
+            log.info("APP_NAME: {}, storage path: {}", APP_NAME, serverIdentityPath);
+            log.info("new server identity created dynamically.");
+            // save it back to file by default for next start (possibly add setting to override)
+            try {
+                Files.write(serverIdentityPath, serverIdentity.getPrivateKey(), CREATE, WRITE);
+                log.info("serverIdentity written back to file");
+            } catch (IOException e) {
+                log.error("Error while saving serverIdentity to {}", serverIdentityPath, e);
+            }
+        }
+        log.debug("Server Identity: {}", serverIdentity.toString());
+
+        // show the ifac_size of the configured interfaces (debug code)
+        for (ConnectionInterface i: Transport.getInstance().getInterfaces() ) {
+            log.info("interface {}, length: {}", i.getInterfaceName(), i.getIfacSize());
+        }
+
+        baseDestination = new Destination(
+            serverIdentity,
+            Direction.IN,
+            DestinationType.SINGLE,
+            APP_NAME,
+            "core"
+        );
+        //// idea for other entry point (needs AnnounceHandler with appropriate aspect)
+        //dataDestination = new Destination(
+        //    serverIdentity,
+        //    Direction.IN,
+        //    DestinationType.SINGLE,
+        //    APP_NAME,
+        //    "qdn"
+        //);
+        log.info("Destination {} {} running", encodeHexString(baseDestination.getHash()), baseDestination.getName());
+   
+        baseDestination.setProofStrategy(ProofStrategy.PROVE_ALL);
+        baseDestination.setAcceptLinkRequests(true);
+        
+        baseDestination.setLinkEstablishedCallback(this::clientConnected);
+        Transport.getInstance().registerAnnounceHandler(new QAnnounceHandler());
+        log.debug("announceHandlers: {}", Transport.getInstance().getAnnounceHandlers());
+        // do a first announce
+        baseDestination.announce();
+        log.debug("Sent initial announce from {} ({})", encodeHexString(baseDestination.getHash()), baseDestination.getName());
+
+        // Start up first networking thread (the "server loop", the "Tasks engine")
+        rnsNetworkEPC.start();
+    }
+
+    private void initConfig(String configDir) throws IOException {
+        File configDir1 = new File(configDir);
+        if (!configDir1.exists()) {
+            configDir1.mkdir();
+        }
+        var configPath = Path.of(configDir1.getAbsolutePath());
+        Path configFile = configPath.resolve(CONFIG_FILE_NAME);
+
+        if (Files.notExists(configFile)) {
+            var defaultConfig = this.getClass().getClassLoader().getResourceAsStream(RNSCommon.defaultRNSConfig);
+            if (Settings.getInstance().isTestNet()) {
+                defaultConfig = this.getClass().getClassLoader().getResourceAsStream(RNSCommon.defaultRNSConfigTestnet);
+            }
+            Files.copy(defaultConfig, configFile, StandardCopyOption.REPLACE_EXISTING);
+        }
+    }
+
+    public void broadcast(Function<RNSPeer, Message> peerMessageBuilder) {
+        for (RNSPeer peer : getActiveImmutableLinkedPeers()) {
+            if (this.isShuttingDown) {
+                return;
+            }
+    
+            Message message = peerMessageBuilder.apply(peer);
+    
+            if (message == null) {
+                continue;
+            }
+    
+            var pl = peer.getPeerLink();
+            if (nonNull(pl) && (pl.getStatus() == ACTIVE)) {
+                peer.sendMessage(message);
+            }
+        }
+    }
+
+    public void broadcastOurChain() {
+        BlockData latestBlockData = Controller.getInstance().getChainTip();
+        int latestHeight = latestBlockData.getHeight();
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            List<BlockSummaryData> latestBlockSummaries = repository.getBlockRepository().getBlockSummaries(latestHeight - BROADCAST_CHAIN_TIP_DEPTH, latestHeight);
+            Message latestBlockSummariesMessage = new BlockSummariesV2Message(latestBlockSummaries);
+
+            broadcast(broadcastPeer -> latestBlockSummariesMessage);
+        } catch (DataException e) {
+            log.warn("Couldn't broadcast our chain tip info", e);
+        }
+    }
+
+    public Message buildNewTransactionMessage(RNSPeer peer, TransactionData transactionData) {
+        // In V2 we send out transaction signature only and peers can decide whether to request the full transaction
+        return new TransactionSignaturesMessage(Collections.singletonList(transactionData.getSignature()));
+    }
+
+    public Message buildGetUnconfirmedTransactionsMessage(RNSPeer peer) {
+        return new GetUnconfirmedTransactionsMessage();
+    }
+
+    public void shutdown() {
+        this.isShuttingDown = true;
+        log.info("shutting down Reticulum");
+        
+        // gracefully close links of peers that point to us
+        for (RNSPeer p: incomingPeers) {
+            var pl = p.getPeerLink();
+            if (nonNull(pl) & (pl.getStatus() == ACTIVE)) {
+                p.sendCloseToRemote(pl);
+            }
+        }
+        // Disconnect peers gracefully and terminate Reticulum
+        for (RNSPeer p: linkedPeers) {
+            log.info("shutting down peer: {}", encodeHexString(p.getDestinationHash()));
+            //log.debug("peer: {}", p);
+            p.shutdown();
+            try {
+                TimeUnit.SECONDS.sleep(1); // allow for peers to disconnect gracefully
+            } catch (InterruptedException e) {
+                log.error("exception: ", e);
+            }
+            //var pl = p.getPeerLink();
+            //if (nonNull(pl) & (pl.getStatus() == ACTIVE)) {
+            //    pl.teardown();
+            //}
+        }
+        // Stop processing threads (the "server loop")
+        try {
+            if (!this.rnsNetworkEPC.shutdown(5000)) {
+                log.warn("RNSNetwork threads failed to terminate");
+            }
+        } catch (InterruptedException e) {
+            log.warn("Interrupted while waiting for RNS networking threads to terminate");
+        }
+        // Note: we still need to get the packet timeout callback to work...
+        reticulum.exitHandler();
+    }
+
+    public void sendCloseToRemote(Link link) {
+        if (nonNull(link)) {
+            var data = concatArrays("close::".getBytes(UTF_8),link.getDestination().getHash());
+            Packet closePacket = new Packet(link, data);
+            var packetReceipt = closePacket.send();
+            packetReceipt.setDeliveryCallback(this::closePacketDelivered);
+            packetReceipt.setTimeoutCallback(this::packetTimedOut);
+        } else {
+            log.debug("can't send to null link");
+        }
+    }
+
+    public void closePacketDelivered(PacketReceipt receipt) {
+        var rttString = "";
+        if (receipt.getStatus() == PacketReceiptStatus.DELIVERED) {
+            var rtt = receipt.getRtt();    // rtt (Java) is in miliseconds
+            //log.info("qqp - packetDelivered - rtt: {}", rtt);
+            if (rtt >= 1000) {
+                rtt = Math.round((float) rtt / 1000);
+                rttString = String.format("%d seconds", rtt);
+            } else {
+                rttString = String.format("%d miliseconds", rtt);
+            }
+            log.info("Shutdown packet confirmation received from {}, round-trip time is {}",
+                    encodeHexString(receipt.getDestination().getHash()), rttString);
+        }
+    }
+
+    public void packetTimedOut(PacketReceipt receipt) {
+        log.info("packet timed out, receipt status: {}", receipt.getStatus());
+    }
+
+    public void clientConnected(Link link) {
+        //link.setLinkClosedCallback(this::clientDisconnected);
+        //link.setPacketCallback(this::serverPacketReceived);
+        log.info("clientConnected - link hash: {}, {}", link.getHash(), encodeHexString(link.getHash()));
+        RNSPeer newPeer = new RNSPeer(link);
+        newPeer.setPeerLinkHash(link.getHash());
+        newPeer.setMessageMagic(getMessageMagic());
+        // make sure the peer has a channel and buffer
+        newPeer.getOrInitPeerBuffer();
+        addIncomingPeer(newPeer);
+        log.info("***> Client connected, link: {}", encodeHexString(link.getLinkId()));
+    }
+
+    public void clientDisconnected(Link link) {
+        log.info("***> Client disconnected");
+    }
+
+    public void serverPacketReceived(byte[] message, Packet packet) {
+        var msgText = new String(message, StandardCharsets.UTF_8);
+        log.info("Received data on link - message: {}, destinationHash: {}", msgText, encodeHexString(packet.getDestinationHash()));
+    }
+
+    //public void announceBaseDestination () {
+    //    getBaseDestination().announce();
+    //}
+
+    private class QAnnounceHandler implements AnnounceHandler {
+        @Override
+        public String getAspectFilter() {
+            return "qortal.core";
+        }
+
+        @Override
+        @Synchronized
+        public void receivedAnnounce(byte[] destinationHash, Identity announcedIdentity, byte[] appData) {
+            var peerExists = false;
+            var activePeerCount = 0; 
+
+            log.info("Received an announce from {}", encodeHexString(destinationHash));
+
+            if (nonNull(appData)) {
+                log.debug("The announce contained the following app data: {}", new String(appData, UTF_8));
+            }
+
+            // add to peer list if we can use more peers
+            //synchronized (this) {
+            var lps =  RNSNetwork.getInstance().getImmutableLinkedPeers();
+            for (RNSPeer p: lps) {
+                var pl = p.getPeerLink();
+                if ((nonNull(pl) && (pl.getStatus() == ACTIVE))) {
+                    activePeerCount = activePeerCount + 1;
+                }
+            }
+            if (activePeerCount < MAX_PEERS) {
+                for (RNSPeer p: lps) {
+                    if (Arrays.equals(p.getDestinationHash(), destinationHash)) {
+                        log.info("QAnnounceHandler - peer exists - found peer matching destinationHash");
+                        if (nonNull(p.getPeerLink())) {
+                            log.info("peer link: {}, status: {}",
+                                    encodeHexString(p.getPeerLink().getLinkId()), p.getPeerLink().getStatus());
+                        }
+                        peerExists = true;
+                        if (p.getPeerLink().getStatus() != ACTIVE) {
+                            p.getOrInitPeerLink();
+                        }
+                        break;
+                    } else {
+                        if (nonNull(p.getPeerLink())) {
+                            log.info("QAnnounceHandler - other peer - link: {}, status: {}",
+                                    encodeHexString(p.getPeerLink().getLinkId()), p.getPeerLink().getStatus());
+                            if (p.getPeerLink().getStatus() == CLOSED) {
+                                // mark peer for deletion on nexe pruning
+                                p.setDeleteMe(true);
+                            }
+                        } else {
+                            log.info("QAnnounceHandler - peer link is null");
+                        }
+                    }
+                }
+                if (!peerExists) {
+                    RNSPeer newPeer = new RNSPeer(destinationHash);
+                    newPeer.setServerIdentity(announcedIdentity);
+                    newPeer.setIsInitiator(true);
+                    newPeer.setMessageMagic(getMessageMagic());
+                    addLinkedPeer(newPeer);
+                    log.info("added new RNSPeer, destinationHash: {}", encodeHexString(destinationHash));
+                }
+            }
+            // Chance to announce instead of waiting for next pruning.
+            // Note: good in theory but leads to ping-pong of announces => not a good idea!
+            //maybeAnnounce(getBaseDestination());
+        }
+    }
+
+    // Main thread
+    class RNSNetworkProcessor extends ExecuteProduceConsume {
+
+        //private final Logger logger = LoggerFactory.getLogger(RNSNetworkProcessor.class);
+
+        private final AtomicLong nextConnectTaskTimestamp = new AtomicLong(0L); // ms - try first connect once NTP syncs
+        private final AtomicLong nextBroadcastTimestamp = new AtomicLong(0L); // ms - try first broadcast once NTP syncs
+        private final AtomicLong nextPingTimestamp = new AtomicLong(0L); // ms - try first low-level Ping
+        private final AtomicLong nextPruneTimestamp = new AtomicLong(0L); // ms - try first low-level Ping
+
+        private Iterator<SelectionKey> channelIterator = null;
+
+        RNSNetworkProcessor(ExecutorService executor) {
+            super(executor);
+            final Long now = NTP.getTime();
+            nextPruneTimestamp.set(now + PRUNE_INTERVAL/2);
+        }
+
+        @Override
+        protected void onSpawnFailure() {
+            // For debugging:
+            // ExecutorDumper.dump(this.executor, 3, ExecuteProduceConsume.class);
+        }
+
+        @Override
+        protected Task produceTask(boolean canBlock) throws InterruptedException {
+            Task task;
+
+            //// TODO: Needed? Figure out how to add pending messages in RNSPeer
+            ////        (RNSPeer: pendingMessages.offer(message))
+            //task = maybeProducePeerMessageTask();
+            //if (task != null) {
+            //    return task;
+            //}
+            
+            final Long now = NTP.getTime();
+            
+            // ping task (Link+Channel+Buffer)
+            task = maybeProducePeerPingTask(now);
+            if (task != null) {
+                return task;
+            }
+            
+            task = maybeProduceBroadcastTask(now);
+            if (task != null) {
+                return task;
+            }
+
+            //// Prune stuck/slow/old peers (moved from Controller)
+            //task = maybeProduceRNSPrunePeersTask(now);
+            //if (task != null) {
+            //    return task;
+            //}
+
+            return null;
+        }
+
+        ////private Task maybeProducePeerMessageTask() {
+        ////    return getImmutableConnectedPeers().stream()
+        ////            .map(Peer::getMessageTask)
+        ////            .filter(Objects::nonNull)
+        ////            .findFirst()
+        ////            .orElse(null);
+        ////}
+        ////private Task maybeProducePeerMessageTask() {
+        ////    return getImmutableIncomingPeers().stream()
+        ////            .map(RNSPeer::getMessageTask)
+        ////            .filter(RNSPeer::isAvailable)
+        ////            .findFirst()
+        ////            .orElse(null);
+        ////}
+        //// Note: we might not need this. All messages handled asynchronously in Reticulum
+        ////       (RNSPeer peerBufferReady callback)
+        //private Task maybeProducePeerMessageTask() {
+        //    return getActiveImmutableLinkedPeers().stream()
+        //            .map(RNSPeer::getMessageTask)
+        //            .filter(Objects::nonNull)
+        //            .findFirst()
+        //            .orElse(null);
+        //}
+
+        //private Task maybeProducePeerPingTask(Long now) {
+        //    return getImmutableHandshakedPeers().stream()
+        //            .map(peer -> peer.getPingTask(now))
+        //            .filter(Objects::nonNull)
+        //            .findFirst()
+        //            .orElse(null);
+        //}
+        private Task maybeProducePeerPingTask(Long now) {
+            //var ilp = getImmutableLinkedPeers().stream()
+            //        .map(peer -> peer.getPingTask(now))
+            //        .filter(Objects::nonNull)
+            //        .findFirst()
+            //        .orElse(null);
+            //if (nonNull(ilp)) {
+            //    log.info("ilp - {}", ilp);
+            //}
+            //return ilp;
+            return getActiveImmutableLinkedPeers().stream()
+                    .map(peer -> peer.getPingTask(now))
+                    .filter(Objects::nonNull)
+                    .findFirst()
+                    .orElse(null);
+        }
+        
+        private Task maybeProduceBroadcastTask(Long now) {
+            if (now == null || now < nextBroadcastTimestamp.get()) {
+                return null;
+            }
+            
+            nextBroadcastTimestamp.set(now + BROADCAST_INTERVAL);
+            return new RNSBroadcastTask();
+        }
+
+        private Task maybeProduceRNSPrunePeersTask(Long now) {
+            if (now == null || now < nextPruneTimestamp.get()) {
+                return null;
+            }
+            
+            nextPruneTimestamp.set(now + PRUNE_INTERVAL);
+            return new RNSPrunePeersTask();
+        }
+    }
+
+    private static class SingletonContainer {
+        private static final RNSNetwork INSTANCE = new RNSNetwork();
+    }
+
+    public static RNSNetwork getInstance() {
+        return SingletonContainer.INSTANCE;
+    }
+
+    public List<RNSPeer> getActiveImmutableLinkedPeers() {
+        List<RNSPeer> activePeers = Collections.synchronizedList(new ArrayList<>());
+        for (RNSPeer p: this.immutableLinkedPeers) {
+            if (nonNull(p.getPeerLink()) && (p.getPeerLink().getStatus() == ACTIVE)) {
+                activePeers.add(p);
+            }
+        }
+        return activePeers;
+    }
+
+    // note: we already have a lobok getter for this
+    //public List<RNSPeer> getImmutableLinkedPeers() {
+    //    return this.immutableLinkedPeers;
+    //}
+
+    public void addLinkedPeer(RNSPeer peer) {
+        this.linkedPeers.add(peer);
+        this.immutableLinkedPeers = List.copyOf(this.linkedPeers); // thread safe
+    }
+
+    public void removeLinkedPeer(RNSPeer peer) {
+        //if (nonNull(peer.getPeerBuffer())) {
+        //    peer.getPeerBuffer().close();
+        //}
+        if (nonNull(peer.getPeerLink())) {
+            peer.getPeerLink().teardown();
+        }
+        var p = this.linkedPeers.remove(this.linkedPeers.indexOf(peer)); // thread safe
+        this.immutableLinkedPeers = List.copyOf(this.linkedPeers);
+    }
+
+    // note: we already have a lobok getter for this
+    //public List<RNSPeer> getLinkedPeers() {
+    //    //synchronized(this.linkedPeers) {
+    //        //return new ArrayList<>(this.linkedPeers);
+    //        return this.linkedPeers;
+    //    //}
+    //}
+
+    public void addIncomingPeer(RNSPeer peer) {
+        this.incomingPeers.add(peer);
+        this.immutableIncomingPeers = List.copyOf(this.incomingPeers);
+    }
+
+    public void removeIncomingPeer(RNSPeer peer) {
+        if (nonNull(peer.getPeerLink())) {
+            peer.getPeerLink().teardown();
+        }
+        var p = this.incomingPeers.remove(this.incomingPeers.indexOf(peer));
+        this.immutableIncomingPeers = List.copyOf(this.incomingPeers);
+    }
+
+    // note: we already have a lobok getter for this
+    //public List<RNSPeer> getIncomingPeers() {
+    //    return this.incomingPeers;
+    //}
+    //public List<RNSPeer> getImmutableIncomingPeers() {
+    //    return this.immutableIncomingPeers;
+    //}
+
+    // TODO, methods for: getAvailablePeer
+
+    private Boolean isUnreachable(RNSPeer peer) {
+        var result = peer.getDeleteMe();
+        var now = Instant.now();
+        var peerLastAccessTimestamp = peer.getLastAccessTimestamp();
+        if (peerLastAccessTimestamp.isBefore(now.minusMillis(LINK_UNREACHABLE_TIMEOUT))) {
+            result = true;
+        }
+        return result;
+    }
+
+    public void peerMisbehaved(RNSPeer peer) {
+        RNSPeerData peerData = peer.getPeerData();
+        peerData.setLastMisbehaved(NTP.getTime());
+
+        //// Only update repository if outbound/initiator peer
+        //if (peer.getIsInitiator()) {
+        //    try (Repository repository = RepositoryManager.getRepository()) {
+        //        synchronized (this.allKnownPeers) {
+        //            repository.getNetworkRepository().save(peerData);
+        //            repository.saveChanges();
+        //        }
+        //    } catch (DataException e) {
+        //        log.warn("Repository issue while updating peer synchronization info", e);
+        //    }
+        //}
+    }
+
+    public List<RNSPeer> getNonActiveIncomingPeers() {
+        var ips = getIncomingPeers();
+        List<RNSPeer> result = Collections.synchronizedList(new ArrayList<>());
+        Link pl;
+        for (RNSPeer p: ips) {
+            pl = p.getPeerLink();
+            if (nonNull(pl)) {
+                if (pl.getStatus() != ACTIVE) {
+                    result.add(p);
+                }
+            } else {
+                result.add(p);
+            }
+        }
+        return result;
+    }
+
+    //@Synchronized
+    public void prunePeers() throws DataException {
+        // prune initiator peers
+        //var peerList = getImmutableLinkedPeers();
+        var initiatorPeerList = getImmutableLinkedPeers();
+        var initiatorActivePeerList = getActiveImmutableLinkedPeers();
+        var incomingPeerList = getImmutableIncomingPeers();
+        var numActiveIncomingPeers = incomingPeerList.size() - getNonActiveIncomingPeers().size();
+        log.info("number of links (linkedPeers (active) / incomingPeers (active) before prunig: {} ({}), {} ({})",
+                initiatorPeerList.size(), getActiveImmutableLinkedPeers().size(),
+                incomingPeerList.size(), numActiveIncomingPeers);
+        for (RNSPeer p: initiatorActivePeerList) {
+            var pLink = p.getOrInitPeerLink();
+            p.pingRemote();
+        }
+        for (RNSPeer p : initiatorPeerList) {
+            var pLink = p.getPeerLink();
+            if (nonNull(pLink)) {
+                if (p.getPeerTimedOut()) {
+                    // options: keep in case peer reconnects or remove => we'll remove it
+                    removeLinkedPeer(p);
+                    continue;
+                }
+                if (pLink.getStatus() == ACTIVE) {
+                    continue;
+                }
+                if ((pLink.getStatus() == CLOSED) || (p.getDeleteMe()))  {
+                    removeLinkedPeer(p);
+                    continue;
+                }
+                if (pLink.getStatus() == PENDING) {
+                    pLink.teardown();
+                    removeLinkedPeer(p);
+                    continue;
+                }
+            }
+        }
+        // prune non-initiator peers
+        List<RNSPeer> inaps = getNonActiveIncomingPeers();
+        incomingPeerList = this.incomingPeers;
+        for (RNSPeer p: incomingPeerList) {
+            var pLink = p.getOrInitPeerLink();
+            if (nonNull(pLink) && (pLink.getStatus() == ACTIVE)) {
+                // make false active links to timeout (and teardown in timeout callback)
+                // note: actual removal of peer happens on the following pruning run.
+                p.pingRemote();
+            }
+        }
+        for (RNSPeer p: inaps) {
+            var pLink = p.getPeerLink();
+            if (nonNull(pLink)) {
+                // could be eg. PENDING
+                pLink.teardown();
+            }
+            removeIncomingPeer(p);
+        }
+        initiatorPeerList = getImmutableLinkedPeers();
+        initiatorActivePeerList = getActiveImmutableLinkedPeers();
+        incomingPeerList = getImmutableIncomingPeers();
+        numActiveIncomingPeers = incomingPeerList.size() - getNonActiveIncomingPeers().size();
+        log.info("number of links (linkedPeers (active) / incomingPeers (active) after prunig: {} ({}), {} ({})",
+                initiatorPeerList.size(), getActiveImmutableLinkedPeers().size(),
+                incomingPeerList.size(), numActiveIncomingPeers);
+        maybeAnnounce(getBaseDestination());
+    }
+
+    public void maybeAnnounce(Destination d) {
+        var activePeers = getActiveImmutableLinkedPeers().size();
+        if (activePeers <= MIN_DESIRED_PEERS) {
+            log.info("Active peers ({}) <= desired peers ({}). Announcing", activePeers, MIN_DESIRED_PEERS);
+            d.announce();
+        }
+    }
+
+    /**
+     * Helper methods
+     */
+
+    public RNSPeer findPeerByLink(Link link) {
+        //List<RNSPeer> lps =  RNSNetwork.getInstance().getLinkedPeers();
+        List<RNSPeer> lps =  RNSNetwork.getInstance().getImmutableLinkedPeers();
+        RNSPeer peer = null;
+        for (RNSPeer p : lps) {
+            var pLink = p.getPeerLink();
+            if (nonNull(pLink)) {
+                if (Arrays.equals(pLink.getDestination().getHash(),link.getDestination().getHash())) {
+                    log.info("found peer matching destinationHash: {}", encodeHexString(link.getDestination().getHash()));
+                    peer = p;
+                    break;
+                }
+            }
+        }
+        return peer;
+    }
+
+    public RNSPeer findPeerByDestinationHash(byte[] dhash) {
+        //List<RNSPeer> lps =  RNSNetwork.getInstance().getLinkedPeers();
+        List<RNSPeer> lps =  RNSNetwork.getInstance().getImmutableLinkedPeers();
+        RNSPeer peer = null;
+        for (RNSPeer p : lps) {
+            if (Arrays.equals(p.getDestinationHash(), dhash)) {
+                log.info("found peer matching destinationHash: {}", encodeHexString(dhash));
+                peer = p;
+                break;
+            }
+        }
+        return peer;
+    }
+
+    //public void removePeer(RNSPeer peer) {
+    //    List<RNSPeer> peerList = this.linkedPeers;
+    //    if (nonNull(peer)) {
+    //        peerList.remove(peer);
+    //    }
+    //}
+
+    public byte[] getMessageMagic() {
+        return Settings.getInstance().isTestNet() ? TESTNET_MESSAGE_MAGIC : MAINNET_MESSAGE_MAGIC;
+    }
+
+    public String getOurNodeId() {
+        return this.serverIdentity.toString();
+    }
+
+    protected byte[] getOurPublicKey() {
+        return this.serverIdentity.getPublicKey();
+    }
+
+    // Network methods Reticulum implementation
+
+    /** Builds either (legacy) HeightV2Message or (newer) BlockSummariesV2Message, depending on peer version.
+     *
+     *  @return Message, or null if DataException was thrown.
+     */
+    public Message buildHeightOrChainTipInfo(RNSPeer peer) {
+        // peer only used for version check
+        int latestHeight = Controller.getInstance().getChainHeight();
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            List<BlockSummaryData> latestBlockSummaries = repository.getBlockRepository().getBlockSummaries(latestHeight - BROADCAST_CHAIN_TIP_DEPTH, latestHeight);
+            return new BlockSummariesV2Message(latestBlockSummaries);
+        } catch (DataException e) {
+            return null;
+        }
+    }
+
+}
+

src/main/java/org/qortal/network/RNSPeer.java

@@ -0,0 +1,894 @@
+package org.qortal.network;
+
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
+
+import static java.util.Objects.isNull;
+import static java.util.Objects.nonNull;
+//import java.io.IOException;
+import java.time.Instant;
+import java.util.*;
+
+//import io.reticulum.Reticulum;
+//import org.qortal.network.RNSNetwork;
+import io.reticulum.link.Link;
+import io.reticulum.link.RequestReceipt;
+import io.reticulum.packet.PacketReceiptStatus;
+import io.reticulum.packet.Packet;
+import io.reticulum.packet.PacketReceipt;
+import io.reticulum.identity.Identity;
+import io.reticulum.channel.Channel;
+import io.reticulum.destination.Destination;
+import io.reticulum.destination.DestinationType;
+import io.reticulum.destination.Direction;
+import io.reticulum.destination.ProofStrategy;
+import io.reticulum.resource.Resource;
+import static io.reticulum.link.TeardownSession.INITIATOR_CLOSED;
+import static io.reticulum.link.TeardownSession.DESTINATION_CLOSED;
+import static io.reticulum.link.TeardownSession.TIMEOUT;
+import static io.reticulum.link.LinkStatus.ACTIVE;
+//import static io.reticulum.link.LinkStatus.CLOSED;
+import static io.reticulum.identity.IdentityKnownDestination.recall;
+//import static io.reticulum.identity.IdentityKnownDestination.recallAppData;
+import io.reticulum.buffer.Buffer;
+import io.reticulum.buffer.BufferedRWPair;
+import static io.reticulum.utils.IdentityUtils.concatArrays;
+
+import org.qortal.controller.Controller;
+import org.qortal.data.block.BlockSummaryData;
+import org.qortal.data.block.CommonBlockData;
+import org.qortal.data.network.RNSPeerData;
+import org.qortal.network.message.Message;
+import org.qortal.network.message.MessageType;
+import org.qortal.network.message.PingMessage;
+import org.qortal.network.message.*;
+import org.qortal.network.message.MessageException;
+import org.qortal.network.task.RNSMessageTask;
+import org.qortal.network.task.RNSPingTask;
+import org.qortal.settings.Settings;
+import org.qortal.utils.ExecuteProduceConsume.Task;
+import org.qortal.utils.NTP;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import java.util.concurrent.*;
+import java.util.Arrays;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.commons.codec.binary.Hex.encodeHexString;
+import static org.apache.commons.lang3.ArrayUtils.subarray;
+import static org.apache.commons.lang3.BooleanUtils.isFalse;
+import static org.apache.commons.lang3.BooleanUtils.isTrue;
+
+import lombok.extern.slf4j.Slf4j;
+import lombok.Setter;
+import lombok.Data;
+import lombok.AccessLevel;
+//import lombok.Synchronized;
+//
+//import org.qortal.network.message.Message;
+//import org.qortal.network.message.MessageException;
+
+import java.util.concurrent.atomic.LongAdder;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import java.lang.IllegalStateException;
+
+@Data
+@Slf4j
+public class RNSPeer {
+
+    static final String APP_NAME = Settings.getInstance().isTestNet() ? RNSCommon.TESTNET_APP_NAME: RNSCommon.MAINNET_APP_NAME;
+    //static final String defaultConfigPath = new String(".reticulum");
+    //static final String defaultConfigPath = RNSCommon.defaultRNSConfigPath;
+
+    private byte[] destinationHash;   // remote destination hash
+    Destination peerDestination;      // OUT destination created for this
+    private Identity serverIdentity;
+    @Setter(AccessLevel.PACKAGE) private Instant creationTimestamp;
+    @Setter(AccessLevel.PACKAGE) private Instant lastAccessTimestamp;
+    @Setter(AccessLevel.PACKAGE) private Instant lastLinkProbeTimestamp;
+    Link peerLink;
+    byte[] peerLinkHash;
+    BufferedRWPair peerBuffer;
+    int receiveStreamId = 0;
+    int sendStreamId = 0;
+    private Boolean isInitiator;
+    private Boolean deleteMe = false;
+    //private Boolean isVacant = true;
+    private Long lastPacketRtt = null;
+    //private byte[] emptyBuffer = {0,0,0,0,0};
+
+    private Double requestResponseProgress;
+    @Setter(AccessLevel.PACKAGE) private Boolean peerTimedOut = false;
+
+    // for qortal networking
+    private static final int RESPONSE_TIMEOUT = 3000; // [ms]
+    private static final int PING_INTERVAL = 55_000; // [ms]
+    private static final long LINK_PING_INTERVAL = 55 * 1000L; // ms
+    private byte[] messageMagic;  // set in message creating classes
+    private Long lastPing = null;      // last (packet) ping roundtrip time [ms]
+    private Long lastPingSent = null;  // time last (packet) ping was sent, or null if not started.
+    @Setter(AccessLevel.PACKAGE) private Instant lastPingResponseReceived = null; // time last (packet) ping succeeded
+    private Map<Integer, BlockingQueue<Message>> replyQueues;
+    private LinkedBlockingQueue<Message> pendingMessages;
+    private boolean syncInProgress = false;
+    private RNSPeerData peerData = null;
+    private long linkEstablishedTime = -1L; // equivalent of (tcpip) Peer 'handshakeComplete'
+    // Versioning
+    public static final Pattern VERSION_PATTERN = Pattern.compile(Controller.VERSION_PREFIX
+            + "(\\d{1,3})\\.(\\d{1,5})\\.(\\d{1,5})");
+    /* Pending signature requests */
+    private List<byte[]> pendingSignatureRequests = Collections.synchronizedList(new ArrayList<>());
+    /**
+     * Latest block info as reported by peer.
+     */
+    private List<BlockSummaryData> peersChainTipData = Collections.emptyList();
+    /**
+     * Our common block with this peer
+     */
+    private CommonBlockData commonBlockData;
+    /**
+     * Last time we detected this peer as TOO_DIVERGENT
+     */
+    private Long lastTooDivergentTime;
+    ///**
+    // * Known starting sequences for data received over buffer
+    // */
+    //private byte[] SEQ_REQUEST_CONFIRM_ID = new byte[]{0x53, 0x52, 0x65, 0x71, 0x43, 0x49, 0x44}; // SReqCID
+    //private byte[] SEQ_RESPONSE_CONFIRM_ID = new byte[]{0x53, 0x52, 0x65, 0x73, 0x70, 0x43, 0x49, 0x44}; // SRespCID
+
+    // Message stats
+    private static class MessageStats {
+        public final LongAdder count = new LongAdder();
+        public final LongAdder totalBytes = new LongAdder();
+    }
+
+    private final Map<MessageType, RNSPeer.MessageStats> receivedMessageStats = new ConcurrentHashMap<>();
+    private final Map<MessageType, RNSPeer.MessageStats> sentMessageStats = new ConcurrentHashMap<>();
+
+    /**
+     * Constructor for initiator peers
+     */
+    public RNSPeer(byte[] dhash) {
+        this.destinationHash = dhash;
+        this.serverIdentity = recall(dhash);
+        initPeerLink();
+        //setCreationTimestamp(System.currentTimeMillis());
+        this.creationTimestamp = Instant.now();
+        //this.isVacant = true;
+        this.replyQueues = new ConcurrentHashMap<>();
+        this.pendingMessages = new LinkedBlockingQueue<>();
+        this.peerData = new RNSPeerData(dhash);
+    }
+
+    /**
+     * Constructor for non-initiator peers
+     */
+    public RNSPeer(Link link) {
+        this.peerLink = link;
+        //this.peerLinkId = link.getLinkId();
+        this.peerDestination = link.getDestination();
+        this.destinationHash = link.getDestination().getHash();
+        this.serverIdentity = link.getRemoteIdentity();
+
+        this.creationTimestamp = Instant.now();
+        this.lastAccessTimestamp = Instant.now();
+        this.lastLinkProbeTimestamp = null;
+        this.isInitiator = false;
+        //this.isVacant = false;
+
+        //this.peerLink.setLinkEstablishedCallback(this::linkEstablished);
+        //this.peerLink.setLinkClosedCallback(this::linkClosed);
+        //this.peerLink.setPacketCallback(this::linkPacketReceived);
+        this.peerData = new RNSPeerData(this.destinationHash);
+    }
+    public void initPeerLink() {
+        peerDestination = new Destination(
+            this.serverIdentity,
+            Direction.OUT, 
+            DestinationType.SINGLE,
+            APP_NAME,
+            "core"
+        );
+        peerDestination.setProofStrategy(ProofStrategy.PROVE_ALL);
+
+        this.creationTimestamp = Instant.now();
+        this.lastAccessTimestamp = Instant.now();
+        this.lastLinkProbeTimestamp = null;
+        this.isInitiator = true;
+
+        this.peerLink = new Link(peerDestination);
+
+        this.peerLink.setLinkEstablishedCallback(this::linkEstablished);
+        this.peerLink.setLinkClosedCallback(this::linkClosed);
+        this.peerLink.setPacketCallback(this::linkPacketReceived);
+    }
+
+    @Override
+    public String toString() {
+        // for messages we want an address-like string representation
+        if (nonNull(this.peerLink)) {
+            return this.getPeerLink().toString();
+        } else {
+            return encodeHexString(this.getDestinationHash());
+        }
+    }
+
+    public BufferedRWPair getOrInitPeerBuffer() {
+        var channel = this.peerLink.getChannel();
+        if (nonNull(this.peerBuffer)) {
+            //log.info("peerBuffer exists: {}, link status: {}", this.peerBuffer, this.peerLink.getStatus());
+            try {
+                log.trace("peerBuffer exists: {}, link status: {}", this.peerBuffer, this.peerLink.getStatus());
+            } catch (IllegalStateException e) {
+                // Exception thrown by Reticulum if the buffer is unusable (Channel, Link, etc)
+                // This is a chance to correct links status when doing a RNSPingTask
+                log.warn("can't establish Channel/Buffer (remote peer down?), closing link: {}");
+                this.peerBuffer.close();
+                this.peerLink.teardown();
+                this.peerLink = null;
+                //log.error("(handled) IllegalStateException - can't establish Channel/Buffer: {}", e);
+            }
+        }
+        else {
+            log.info("creating buffer - peerLink status: {}, channel: {}", this.peerLink.getStatus(), channel);
+            this.peerBuffer = Buffer.createBidirectionalBuffer(receiveStreamId, sendStreamId, channel, this::peerBufferReady);
+        }
+        return getPeerBuffer();
+    }
+
+    public Link getOrInitPeerLink() {
+        if (this.peerLink.getStatus() == ACTIVE) {
+            lastAccessTimestamp = Instant.now();
+            //return this.peerLink;
+        } else {
+            initPeerLink();
+        }
+        return this.peerLink;
+    }
+
+    public void shutdown() {
+        if (nonNull(this.peerLink)) {
+            log.info("shutdown - peerLink: {}, status: {}", peerLink, peerLink.getStatus());
+            if (peerLink.getStatus() == ACTIVE) {
+                if (nonNull(this.peerBuffer)) {
+                    this.peerBuffer.close();
+                    this.peerBuffer = null;
+                }
+                this.peerLink.teardown();
+            } else {
+                log.info("shutdown - status (non-ACTIVE): {}", peerLink.getStatus());
+            }
+            this.peerLink = null;
+        }
+        this.deleteMe = true;
+    }
+
+    public Channel getChannel() {
+        if (isNull(getPeerLink())) {
+            log.warn("link is null.");
+            return null;
+        }
+        setLastAccessTimestamp(Instant.now());
+        return getPeerLink().getChannel();
+    }
+
+    public Boolean getIsInitiator() {
+        return this.isInitiator;
+    }
+
+    /** Link callbacks */
+    public void linkEstablished(Link link) {
+        this.linkEstablishedTime = System.currentTimeMillis();
+        link.setLinkClosedCallback(this::linkClosed);
+        log.info("peerLink {} established (link: {}) with peer: hash - {}, link destination hash: {}", 
+            encodeHexString(peerLink.getLinkId()), encodeHexString(link.getLinkId()), encodeHexString(destinationHash),
+            encodeHexString(link.getDestination().getHash()));
+        if (isInitiator) {
+            startPings();
+        }
+    }
+    
+    public void linkClosed(Link link) {
+        if (link.getTeardownReason() == TIMEOUT) {
+            log.info("The link timed out");
+            this.peerTimedOut = true;
+            this.peerBuffer = null;
+        } else if (link.getTeardownReason() == INITIATOR_CLOSED) {
+            log.info("Link closed callback: The initiator closed the link");
+            log.info("peerLink {} closed (link: {}), link destination hash: {}",
+                encodeHexString(peerLink.getLinkId()), encodeHexString(link.getLinkId()), encodeHexString(link.getDestination().getHash()));
+            this.peerBuffer = null;
+        } else if (link.getTeardownReason() == DESTINATION_CLOSED) {
+            log.info("Link closed callback: The link was closed by the peer, removing peer");
+            log.info("peerLink {} closed (link: {}), link destination hash: {}",
+                encodeHexString(peerLink.getLinkId()), encodeHexString(link.getLinkId()), encodeHexString(link.getDestination().getHash()));
+            this.peerBuffer = null;
+        } else {
+            log.info("Link closed callback");
+        }
+    }
+    
+    public void linkPacketReceived(byte[] message, Packet packet) {
+        var msgText = new String(message, StandardCharsets.UTF_8);
+        if (msgText.equals("ping")) {
+            log.info("received ping on link");
+            this.lastLinkProbeTimestamp = Instant.now();
+        } else if (msgText.startsWith("close::")) {
+            var targetPeerHash = subarray(message, 7, message.length);
+            log.info("peer dest hash: {}, target hash: {}",
+                encodeHexString(destinationHash),
+                encodeHexString(targetPeerHash));
+            if (Arrays.equals(destinationHash, targetPeerHash)) {
+                log.info("closing link: {}", peerLink.getDestination().getHexHash());
+                if (nonNull(this.peerBuffer)) {
+                    this.peerBuffer.close();
+                    this.peerBuffer = null;
+                }
+                this.peerLink.teardown();
+            }
+        } else if (msgText.startsWith("open::")) {
+            var targetPeerHash = subarray(message, 7, message.length);
+            log.info("peer dest hash: {}, target hash: {}",
+                encodeHexString(destinationHash),
+                encodeHexString(targetPeerHash));
+            if (Arrays.equals(destinationHash, targetPeerHash)) {
+                log.info("closing link: {}", peerLink.getDestination().getHexHash());
+                getOrInitPeerLink();
+            }
+        }
+    }
+
+    /*
+     * Callback from buffer when buffer has data available
+     *
+     * :param readyBytes: The number of bytes ready to read
+     */
+    public void peerBufferReady(Integer readyBytes) {
+        // get the message data
+        byte[] data = this.peerBuffer.read(readyBytes);
+        ByteBuffer bb = ByteBuffer.wrap(data);
+        //log.info("data length: {}, MAGIC: {}, data: {}, ByteBuffer: {}", data.length, this.messageMagic, data, bb);
+        //log.info("data length: {}, MAGIC: {}, ByteBuffer: {}", data.length, this.messageMagic, bb);
+        //log.trace("peerBufferReady - data bytes: {}", data.length);
+        this.lastAccessTimestamp = Instant.now();
+
+        //if (ByteBuffer.wrap(data, 0, emptyBuffer.length).equals(ByteBuffer.wrap(emptyBuffer, 0, emptyBuffer.length))) {
+        //    log.info("peerBufferReady - empty buffer detected (length: {})", data.length);
+        //}
+        //else {
+        //if (Arrays.equals(SEQ_REQUEST_CONFIRM_ID, Arrays.copyOfRange(data, 0, SEQ_REQUEST_CONFIRM_ID.length))) {
+        //    // a non-initiator peer requested to confirm sending of a packet
+        //    var messageId = subarray(data, SEQ_REQUEST_CONFIRM_ID.length + 1, data.length);
+        //    log.info("received request to confirm message id, id: {}", messageId);
+        //    var confirmData = concatArrays(SEQ_RESPONSE_CONFIRM_ID, "::",data.getBytes(UTF_8), messageId.getBytes(UTF_8));
+        //    this.peerBuffer.write(confirmData);
+        //    this.peerBuffer.flush();
+        //} else if (Arrays.equals(SEQ_RESPONSE_CONFIRM_ID, Arrays.copyOfRange(data, 0, SEQ_RESPONSE_CONFIRM_ID.lenth))) {
+        //    // an initiator peer receiving the confirmation
+        //    var messageId = subarray(data, SEQ_RESPONSE_CONFIRM_ID.length + 1, data.length);
+        //    this.replyQueues.remove(messageId);
+        //} else {
+            try {
+                //log.info("***> creating message from {} bytes", data.length);
+                Message message = Message.fromByteBuffer(bb);
+                //log.info("*=> type {} message received ({} bytes): {}", message.getType(), data.length, message);
+                log.info("*=> type {} message received ({} bytes, id: {})", message.getType(), data.length, message.getId());
+
+                // Handle message based on type
+                switch (message.getType()) {
+                    // Do we need this ? (seems like a TCP scenario only thing)
+                    // Does any RNSPeer ever require an other RNSPeer's peer list?
+                    //case GET_PEERS:
+                    //    //onGetPeersMessage(peer, message);
+                    //    onGetRNSPeersMessage(peer, message);
+                    //    break;
+
+                    case PING:
+                        this.lastPingResponseReceived = Instant.now();
+                        if (isFalse(this.isInitiator)) {
+                            onPingMessage(this, message);
+                        }
+                        break;
+
+                    case PONG:
+                        log.trace("PONG received");
+                        addToQueue(message);  // as response in blocking queue for ping getResponse
+                        break;
+
+                    // Do we need this ? (no need to relay peer list...)
+                    //case PEERS_V2:
+                    //    onPeersV2Message(peer, message);
+                    //    break;
+
+                    case BLOCK_SUMMARIES:
+                        // from Synchronizer
+                        addToQueue(message);
+
+                    case BLOCK_SUMMARIES_V2:
+                        // from Synchronizer
+                        addToQueue(message);
+
+                    case SIGNATURES:
+                        // from Synchronizer
+                        addToQueue(message);
+
+                    case BLOCK:
+                        // from Synchronizer
+                        addToQueue(message);
+
+                    case BLOCK_V2:
+                        // from Synchronizer
+                        addToQueue(message);
+
+                    default:
+                        log.info("default - type {} message received ({} bytes)", message.getType(), data.length);
+                        // Bump up to controller for possible action
+                        addToQueue(message);
+                        Controller.getInstance().onRNSNetworkMessage(this, message);
+                        break;
+                }
+            } catch (MessageException e) {
+                //log.error("{} from peer {}", e.getMessage(), this);
+                log.error("{} from peer {}, closing link", e, this);
+                //log.info("{} from peer {}", e, this);
+                // don't take any chances:
+                // can happen if link is closed by peer in which case we close this side of the link
+                this.peerData.setLastMisbehaved(NTP.getTime());
+                shutdown();
+            }
+        //}
+    }
+
+    /**
+     * we need to queue all incoming messages that follow request/response
+     * with explicit handling of the response message.
+     */
+    public void addToQueue(Message message) {
+        if (message.getType() == MessageType.UNSUPPORTED) {
+            log.trace("discarding/skipping UNSUPPORTED message");
+            return;
+        }
+        BlockingQueue<Message> queue = this.replyQueues.get(message.getId());
+        if (queue != null) {
+            // Adding message to queue will unblock thread waiting for response
+            this.replyQueues.get(message.getId()).add(message);
+            // Consumed elsewhere (getResponseWithTimeout)
+            log.info("addToQueue - queue size: {}, message type: {} (id: {})", queue.size(), message.getType(), message.getId());
+        }
+        else if (!this.pendingMessages.offer(message)) {
+            log.info("[{}] Busy, no room to queue message from peer {} - discarding",
+                    this.peerLink, this);
+        }
+    }
+
+    /**
+     * Set a packet to remote with the message format "close::<our_destination_hash>"
+     * This method is only useful for non-initiator links to close the remote initiator.
+     *
+     * @param link
+     */
+    public void sendCloseToRemote(Link link) {
+        var baseDestination = RNSNetwork.getInstance().getBaseDestination();
+        if (nonNull(link) & (isFalse(link.isInitiator()))) {
+            // Note: if part of link we need to get the baseDesitination hash
+            //var data = concatArrays("close::".getBytes(UTF_8),link.getDestination().getHash());
+            var data = concatArrays("close::".getBytes(UTF_8), baseDestination.getHash());
+            Packet closePacket = new Packet(link, data);
+            var packetReceipt = closePacket.send();
+            packetReceipt.setDeliveryCallback(this::closePacketDelivered);
+            packetReceipt.setTimeout(1000L);
+            packetReceipt.setTimeoutCallback(this::packetTimedOut);
+        } else {
+            log.debug("can't send to null link");
+        }
+    }
+
+    /** PacketReceipt callbacks */
+    public void closePacketDelivered(PacketReceipt receipt) {
+        var rttString = new String("");
+        if (receipt.getStatus() == PacketReceiptStatus.DELIVERED) {
+            var rtt = receipt.getRtt();    // rtt (Java) is in milliseconds
+            this.lastPacketRtt = rtt;
+            if (rtt >= 1000) {
+                rtt = Math.round(rtt / 1000);
+                rttString = String.format("%d seconds", rtt);
+            } else {
+                rttString = String.format("%d miliseconds", rtt);
+            }
+            log.info("Shutdown packet confirmation received from {}, round-trip time is {}",
+                    encodeHexString(receipt.getDestination().getHash()), rttString);
+        }
+    }
+
+    public void packetDelivered(PacketReceipt receipt) {
+        var rttString = "";
+        //log.info("packet delivered callback, receipt: {}", receipt);
+        if (receipt.getStatus() == PacketReceiptStatus.DELIVERED) {
+            var rtt = receipt.getRtt();    // rtt (Java) is in milliseconds
+            this.lastPacketRtt = rtt;
+            //log.info("qqp - packetDelivered - rtt: {}", rtt);
+            if (rtt >= 1000) {
+                rtt = Math.round((float) rtt / 1000);
+                rttString = String.format("%d seconds", rtt);
+            } else {
+                rttString = String.format("%d milliseconds", rtt);
+            }
+            if (getIsInitiator()) {
+                // reporting round trip time in one direction is enough
+                log.info("Valid reply received from {}, round-trip time is {}",
+                        encodeHexString(receipt.getDestination().getHash()), rttString);
+            }
+            this.lastAccessTimestamp = Instant.now();
+        }
+    }
+
+    public void packetTimedOut(PacketReceipt receipt) {
+        //log.info("packet timed out, receipt status: {}", receipt.getStatus());
+        if (receipt.getStatus() == PacketReceiptStatus.FAILED) {
+            log.info("packet timed out, receipt status: {}", PacketReceiptStatus.FAILED);
+            this.peerTimedOut = true;
+            this.peerLink.teardown();
+        }
+        //this.peerTimedOut = true;
+        //this.peerLink.teardown();
+    }
+
+    /** Link Request callbacks */ 
+    public void linkRequestResponseReceived(RequestReceipt rr) {
+        log.info("Response received");
+    }
+
+    public void linkRequestResponseProgress(RequestReceipt rr) {
+        this.requestResponseProgress = rr.getProgress();
+        log.debug("Response progress set");
+    }
+
+    public void linkRequestFailed(RequestReceipt rr) {
+        log.error("Request failed");
+    }
+
+    /** Link Resource callbacks */
+    // Resource: allow arbitrary amounts of data to be passed over a link with
+    // sequencing, compression, coordination and checksumming handled automatically
+    //public Boolean linkResourceAdvertised(Resource resource) {
+    //    log.debug("Resource advertised");
+    //}
+    public void linkResourceTransferStarted(Resource resource) {
+        log.debug("Resource transfer started");
+    }
+    public void linkResourceTransferConcluded(Resource resource) {
+        log.debug("Resource transfer complete");
+    }
+
+    /** Utility methods */
+    public void pingRemote() {
+        var link = this.peerLink;
+        if (nonNull(link)) {
+            if (peerLink.getStatus() == ACTIVE) {
+                log.info("pinging remote (direct, 1 packet): {}", encodeHexString(link.getLinkId()));
+                var data = "ping".getBytes(UTF_8);
+                link.setPacketCallback(this::linkPacketReceived);
+                Packet pingPacket = new Packet(link, data);
+                PacketReceipt packetReceipt = pingPacket.send();
+                packetReceipt.setDeliveryCallback(this::packetDelivered);
+                // Note: don't setTimeout, we want it to timeout with FAIL if not deliverable
+                //packetReceipt.setTimeout(5000L);
+                packetReceipt.setTimeoutCallback(this::packetTimedOut);
+            } else {
+                log.info("can't send ping to a peer {} with (link) status: {}",
+                    encodeHexString(peerLink.getDestination().getHash()), peerLink.getStatus());
+            }
+        }
+    }
+
+    //public void shutdownLink(Link link) {
+    //    var data = "shutdown".getBytes(UTF_8);
+    //    Packet shutdownPacket = new Packet(link, data);
+    //    PacketReceipt packetReceipt = shutdownPacket.send();
+    //    packetReceipt.setTimeout(2000L);
+    //    packetReceipt.setTimeoutCallback(this::packetTimedOut);
+    //    packetReceipt.setDeliveryCallback(this::shutdownPacketDelivered);
+    //}
+
+    /** qortal networking specific (Tasks) */
+
+    private void onPingMessage(RNSPeer peer, Message message) {
+        PingMessage pingMessage = (PingMessage) message;
+    
+        try {
+            PongMessage pongMessage = new PongMessage();
+            pongMessage.setId(message.getId());  // use the ping message id (for ping getResponse)
+            this.peerBuffer.write(pongMessage.toBytes());
+            this.peerBuffer.flush();
+            this.lastAccessTimestamp = Instant.now();
+        } catch (MessageException e) {
+            //log.error("{} from peer {}", e.getMessage(), this);
+            log.error("{} from peer {}", e, this);
+        }
+    }
+
+    /**
+     * Send message to peer and await response, using default RESPONSE_TIMEOUT.
+     * <p>
+     * Message is assigned a random ID and sent.
+     * Responses are handled by registered callbacks.
+     * <p>
+     * Note: The method is called "get..." to match the original method name
+     *
+     * @param message message to send
+     * @return <code>Message</code> if valid response received; <code>null</code> if not or error/exception occurs
+     * @throws InterruptedException if interrupted while waiting
+     */
+    public Message getResponse(Message message) throws InterruptedException {
+        //log.info("RNSPingTask action - pinging peer {}", encodeHexString(getDestinationHash()));
+        return getResponseWithTimeout(message, RESPONSE_TIMEOUT);
+    }
+
+    /**
+     * Send message to peer and await response.
+     * <p>
+     * Message is assigned a random ID and sent.
+     * If a response with matching ID is received then it is returned to caller.
+     * <p>
+     * If no response with matching ID within timeout, or some other error/exception occurs,
+     * then return <code>null</code>.<br>
+     * (Assume peer will be rapidly disconnected after this).
+     *
+     * @param message message to send
+     * @return <code>Message</code> if valid response received; <code>null</code> if not or error/exception occurs
+     * @throws InterruptedException if interrupted while waiting
+     */
+    public Message getResponseWithTimeout(Message message, int timeout) throws InterruptedException {
+        BlockingQueue<Message> blockingQueue = new ArrayBlockingQueue<>(1);
+        // Assign random ID to this message
+        Random random = new Random();
+        int id;
+        do {
+            id = random.nextInt(Integer.MAX_VALUE - 1) + 1;
+
+            // Put queue into map (keyed by message ID) so we can poll for a response
+            // If putIfAbsent() doesn't return null, then this ID is already taken
+        } while (this.replyQueues.putIfAbsent(id, blockingQueue) != null);
+        message.setId(id);
+        //log.info("getResponse - before send {} message, random id is {}", message.getType(), id);
+
+        // Try to send message
+        if (!this.sendMessageWithTimeout(message, timeout)) {
+            this.replyQueues.remove(id);
+            return null;
+        }
+        //log.info("getResponse - after send");
+
+        try {
+            return blockingQueue.poll(timeout, TimeUnit.MILLISECONDS);
+        } finally {
+            this.replyQueues.remove(id);
+            //log.info("getResponse - regular - id removed from replyQueues");
+        }
+    }
+
+    /**
+     * Attempt to send Message to peer using the buffer and a custom timeout.
+     *
+     * @param message message to be sent
+     * @return <code>true</code> if message successfully sent; <code>false</code> otherwise
+     */
+    public boolean sendMessageWithTimeout(Message message, int timeout) {
+        try {
+            // send the message
+            log.trace("Sending {} message with ID {} to peer {}", message.getType().name(), message.getId(), this);
+            var peerBuffer = getOrInitPeerBuffer();
+            this.peerBuffer.write(message.toBytes());
+            this.peerBuffer.flush();
+            //// send a message to confirm receipt over the buffer
+            //var messageId = message.getId();
+            //var confirmData = concatArrays(SEQ_REQUEST_CONFIRM_ID,"::".getBytes(UTF_8), messageId.getBytes(UTF_8));
+            //this.peerBuffer.write(confirmData);
+            //this.peerBuffer.flush();
+            return true;
+        //} catch (InterruptedException e) {
+        //    // Send failure
+        //    return false;
+        } catch (IllegalStateException e) {
+            //log.warn("Can't write to buffer (remote buffer down?)");
+            this.peerLink.teardown();
+            this.peerBuffer = null;
+            log.error("IllegalStateException - can't write to buffer: {}", e);
+            return false;
+        } catch (MessageException e) {
+            log.error(e.getMessage(), e);
+            return false;
+        }
+    }
+
+    protected Task getMessageTask() {
+        /*
+         * If our peerLink is not in ACTIVE node and there is a message yet to be
+         * processed then don't produce another message task.
+         * This allows us to process remaining messages sequentially.
+         */
+        if (this.peerLink.getStatus() != ACTIVE) {
+            return null;
+        }
+
+        final Message nextMessage = this.pendingMessages.poll();
+
+        if (nextMessage == null) {
+            return null;
+        }
+
+        // Return a task to process message in queue
+        return new RNSMessageTask(this, nextMessage);
+    }
+
+    /**
+     * Send a Qortal message using a Reticulum Buffer
+     * 
+     * @param message message to be sent
+     * @return <code>true</code> if message successfully sent; <code>false</code> otherwise
+     */
+    //@Synchronized
+    public boolean sendMessage(Message message) {
+        try {
+            log.trace("Sending {} message with ID {} to peer {}", message.getType().name(), message.getId(), this.toString());
+            //log.info("Sending {} message with ID {} to peer {}", message.getType().name(), message.getId(), this.toString());
+            var peerBuffer = getOrInitPeerBuffer();
+            peerBuffer.write(message.toBytes());
+            peerBuffer.flush();
+            return true;
+        } catch (IllegalStateException e) {
+            this.peerLink.teardown();
+            this.peerBuffer = null;
+            log.error("IllegalStateException - can't write to buffer: {}", e);
+            return false;
+        } catch (MessageException e) {
+            log.error(e.getMessage(), e);
+            return false;
+        }
+    }
+
+    protected void startPings() {
+        log.trace("[{}] Enabling pings for peer {}",
+                peerLink.getDestination().getHexHash(), this.toString());
+        this.lastPingSent = NTP.getTime();
+    }
+
+    protected Task getPingTask(Long now) {
+        // Pings not enabled yet?
+        if (now == null || this.lastPingSent == null) {
+            return null;
+        }
+
+        // ping only possible over ACTIVE Link
+        if (nonNull(this.peerLink)) {
+            if (this.peerLink.getStatus() != ACTIVE) {
+                return null;
+            }
+        } else {
+            return null;
+        }
+
+        // Time to send another ping?
+        if (now < this.lastPingSent + PING_INTERVAL) {
+            return null; // Not yet
+        }
+
+        // Not strictly true, but prevents this peer from being immediately chosen again
+        this.lastPingSent = now;
+
+        return new RNSPingTask(this, now);
+    }
+
+    // low-level Link (packet) ping
+    protected Link getPingLinks(Long now) {
+        if (now == null || this.lastPingSent == null) {
+            return null;
+        }
+
+        // ping only possible over ACTIVE link
+        if (nonNull(this.peerLink)) {
+            if (this.peerLink.getStatus() != ACTIVE) {
+                return null;
+            }
+        } else {
+            return null;
+        }
+
+        if (now < this.lastPingSent + LINK_PING_INTERVAL) {
+            return null;
+        }
+
+        this.lastPingSent = now;
+
+        return this.peerLink;
+
+    }
+
+    // Peer methods reticulum implementations
+    public BlockSummaryData getChainTipData() {
+        List<BlockSummaryData> chainTipSummaries = this.peersChainTipData;
+
+        if (chainTipSummaries.isEmpty())
+            return null;
+
+        // Return last entry, which should have greatest height
+        return chainTipSummaries.get(chainTipSummaries.size() - 1);
+    }
+
+    public void setChainTipData(BlockSummaryData chainTipData) {
+        this.peersChainTipData = Collections.singletonList(chainTipData);
+    }
+
+    public List<BlockSummaryData> getChainTipSummaries() {
+        return this.peersChainTipData;
+    }
+
+    public void setChainTipSummaries(List<BlockSummaryData> chainTipSummaries) {
+        this.peersChainTipData = List.copyOf(chainTipSummaries);
+    }
+
+    public CommonBlockData getCommonBlockData() {
+        return this.commonBlockData;
+    }
+
+    public void setCommonBlockData(CommonBlockData commonBlockData) {
+        this.commonBlockData = commonBlockData;
+    }
+
+    // Common block data
+    public boolean canUseCachedCommonBlockData() {
+        BlockSummaryData peerChainTipData = this.getChainTipData();
+        if (peerChainTipData == null || peerChainTipData.getSignature() == null)
+            return false;
+         CommonBlockData commonBlockData = this.getCommonBlockData();
+        if (commonBlockData == null)
+            return false;
+         BlockSummaryData commonBlockChainTipData = commonBlockData.getChainTipData();
+        if (commonBlockChainTipData == null || commonBlockChainTipData.getSignature() == null)
+            return false;
+         if (!Arrays.equals(peerChainTipData.getSignature(), commonBlockChainTipData.getSignature()))
+            return false;
+         return true;
+    }
+
+    // Pending signature requests
+    public void addPendingSignatureRequest(byte[] signature) {
+        // Check if we already have this signature in the list
+        for (byte[] existingSignature : this.pendingSignatureRequests) {
+            if (Arrays.equals(existingSignature, signature )) {
+                return;
+            }
+        }
+        this.pendingSignatureRequests.add(signature);
+    }
+
+    public void removePendingSignatureRequest(byte[] signature) {
+        Iterator iterator = this.pendingSignatureRequests.iterator();
+        while (iterator.hasNext()) {
+            byte[] existingSignature = (byte[]) iterator.next();
+            if (Arrays.equals(existingSignature, signature)) {
+                iterator.remove();
+            }
+        }
+    }
+
+    public List<byte[]> getPendingSignatureRequests() {
+        return this.pendingSignatureRequests;
+    }
+
+    // Details used by API
+    public long getConnectionEstablishedTime() {
+        return linkEstablishedTime;
+    }
+
+    public long getConnectionAge() {
+        if (linkEstablishedTime > 0L) {
+            return System.currentTimeMillis() - linkEstablishedTime;
+        }
+        return linkEstablishedTime;
+    }
+}

src/main/java/org/qortal/network/RNSPrunePeersTask.java

@@ -0,0 +1,27 @@
+package org.qortal.network.task;
+
+import org.qortal.controller.Controller;
+//import org.qortal.network.RNSNetwork;
+//import org.qortal.repository.DataException;
+import org.qortal.utils.ExecuteProduceConsume.Task;
+
+public class RNSPrunePeersTask implements Task {
+    public RNSPrunePeersTask() {
+    }
+
+    @Override
+    public String getName() {
+        return "PrunePeersTask";
+    }
+
+    @Override
+    public void perform() throws InterruptedException {
+        Controller.getInstance().doRNSPrunePeers();
+        //try {
+        //    log.debug("Pruning peers...");
+        //    RNSNetwork.getInstance().prunePeers();
+        //} catch (DataException e) {
+        //    log.warn(String.format("Repository issue when trying to prune peers: %s", e.getMessage()));
+        //}
+    }
+}

src/main/java/org/qortal/network/message/Message.java

@@ -9,6 +9,8 @@ import java.io.IOException;
 import java.nio.BufferUnderflowException;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 /**
  * Network message for sending over network, or unpacked data received from network.
@@ -33,6 +35,7 @@ import java.util.Arrays;
  * </p>
  */
 public abstract class Message {
+    private static final Logger LOGGER = LogManager.getLogger(Message.class);
 
 	// MAGIC(4) + TYPE(4) + HAS-ID(1) + ID?(4) + DATA-SIZE(4) + CHECKSUM?(4) + DATA?(*)
 	private static final int MAGIC_LENGTH = 4;
@@ -95,9 +98,11 @@ public abstract class Message {
 			byte[] messageMagic = new byte[MAGIC_LENGTH];
 			readOnlyBuffer.get(messageMagic);
 
-			if (!Arrays.equals(messageMagic, Network.getInstance().getMessageMagic()))
+			if (!Arrays.equals(messageMagic, Network.getInstance().getMessageMagic())) {
+				LOGGER.info("xyz - mM: {}, Network getMessageMagic: {}", messageMagic, Network.getInstance().getMessageMagic());
 				// Didn't receive correct Message "magic"
 				throw new MessageException("Received incorrect message 'magic'");
+			}
 
 			// Find supporting object
 			int typeValue = readOnlyBuffer.getInt();

src/main/java/org/qortal/network/task/PeerConnectTask.java

@@ -4,10 +4,15 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
+import org.qortal.utils.DaemonThreadFactory;
 import org.qortal.utils.ExecuteProduceConsume.Task;
 
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
 public class PeerConnectTask implements Task {
     private static final Logger LOGGER = LogManager.getLogger(PeerConnectTask.class);
+    private static final ExecutorService connectionExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory(8));
 
     private final Peer peer;
     private final String name;
@@ -24,6 +29,24 @@ public class PeerConnectTask implements Task {
 
     @Override
     public void perform() throws InterruptedException {
-        Network.getInstance().connectPeer(peer);
+        // Submit connection task to a dedicated thread pool for non-blocking I/O
+        connectionExecutor.submit(() -> {
+            try {
+                connectPeerAsync(peer);
+            } catch (InterruptedException e) {
+                LOGGER.error("Connection attempt interrupted for peer {}", peer, e);
+                Thread.currentThread().interrupt();  // Reset interrupt flag
+            }
+        });
+    }
+
+    private void connectPeerAsync(Peer peer) throws InterruptedException {
+        // Perform peer connection in a separate thread to avoid blocking main task execution
+        try {
+            Network.getInstance().connectPeer(peer);
+            LOGGER.trace("Successfully connected to peer {}", peer);
+        } catch (Exception e) {
+            LOGGER.error("Error connecting to peer {}", peer, e);
+        }
     }
 }

src/main/java/org/qortal/network/task/RNSBroadcastTask.java

@@ -0,0 +1,19 @@
+package org.qortal.network.task;
+
+import org.qortal.controller.Controller;
+import org.qortal.utils.ExecuteProduceConsume.Task;
+
+public class RNSBroadcastTask implements Task {
+    public RNSBroadcastTask() {
+    }
+
+    @Override
+    public String getName() {
+        return "BroadcastTask";
+    }
+
+    @Override
+    public void perform() throws InterruptedException {
+        Controller.getInstance().doRNSNetworkBroadcast();
+    }
+}

src/main/java/org/qortal/network/task/RNSMessageTask.java

@@ -0,0 +1,30 @@
+package org.qortal.network.task;
+
+import org.qortal.network.RNSNetwork;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.Message;
+import org.qortal.utils.ExecuteProduceConsume.Task;
+
+public class RNSMessageTask implements Task {
+    private final RNSPeer peer;
+    private final Message nextMessage;
+    private final String name;
+
+    public RNSMessageTask(RNSPeer peer, Message nextMessage) {
+        this.peer = peer;
+        this.nextMessage = nextMessage;
+        this.name = "MessageTask::" + peer + "::" + nextMessage.getType();
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public void perform() throws InterruptedException {
+        //RNSNetwork.getInstance().onMessage(peer, nextMessage);
+        // TODO: what do we do in the Reticulum case?
+        // Note: this is automatically handled (asynchronously) by the RNSPeer peerBufferReady callback
+    }
+}

src/main/java/org/qortal/network/task/RNSPingTask.java

@@ -0,0 +1,44 @@
+package org.qortal.network.task;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.network.RNSPeer;
+import org.qortal.network.message.Message;
+import org.qortal.network.message.MessageType;
+import org.qortal.network.message.PingMessage;
+import org.qortal.network.message.MessageException;
+import org.qortal.utils.ExecuteProduceConsume.Task;
+import org.qortal.utils.NTP;
+
+public class RNSPingTask implements Task {
+    private static final Logger LOGGER = LogManager.getLogger(PingTask.class);
+
+    private final RNSPeer peer;
+    private final Long now;
+    private final String name;
+
+    public RNSPingTask(RNSPeer peer, Long now) {
+        this.peer = peer;
+        this.now = now;
+        this.name = "PingTask::" + peer;
+    }
+
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    public void perform() throws InterruptedException {
+        PingMessage pingMessage = new PingMessage();
+
+        // Note: Even though getResponse would work, we can use
+        //       peer.sendMessage(pingMessage) using Reticulum buffer instead.
+        //       More efficient and saves room for other request/response tasks.
+        peer.getResponse(pingMessage);
+        //peer.sendMessage(pingMessage);
+
+        //// task is not over here (Reticulum is asynchronous)
+        //peer.setLastPing(NTP.getTime() - now);
+    }
+}

src/main/java/org/qortal/repository/ATRepository.java

@@ -76,9 +76,9 @@ public interface ATRepository {
 	 * Although <tt>expectedValue</tt>, if provided, is natively an unsigned long,
 	 * the data segment comparison is done via unsigned hex string.
 	 */
-	public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, Boolean isFinished,
-			Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
-			Integer limit, Integer offset, Boolean reverse) throws DataException;
+	public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished,
+													  Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
+													  Integer limit, Integer offset, Boolean reverse) throws DataException;
 
 	/**
 	 * Returns final ATStateData for ATs matching codeHash (required)

src/main/java/org/qortal/repository/ArbitraryRepository.java

@@ -27,6 +27,10 @@ public interface ArbitraryRepository {
 
 	public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
 
+	List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException;
+
+	List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName(String name) throws DataException;
+
 	public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException;
 
 	public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
@@ -42,7 +46,7 @@ public interface ArbitraryRepository {
 
 	public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException;
 
-	public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException;
+	public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, List<String> keywords, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException;
 
 	List<ArbitraryResourceData> searchArbitraryResourcesSimple(
 			Service service,

src/main/java/org/qortal/repository/ChatRepository.java

@@ -22,6 +22,6 @@ public interface ChatRepository {
 
 	public ChatMessage toChatMessage(ChatTransactionData chatTransactionData, Encoding encoding) throws DataException;
 
-	public ActiveChats getActiveChats(String address, Encoding encoding) throws DataException;
+	public ActiveChats getActiveChats(String address, Encoding encoding, Boolean hasChatReference) throws DataException;
 
 }

src/main/java/org/qortal/repository/GroupRepository.java

@@ -48,6 +48,8 @@ public interface GroupRepository {
 
 	// Group Admins
 
+	public GroupAdminData getAdminFaulty(int groupId, String address) throws DataException;
+
 	public GroupAdminData getAdmin(int groupId, String address) throws DataException;
 
 	public boolean adminExists(int groupId, String address) throws DataException;

src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java

@@ -1,9 +1,11 @@
 package org.qortal.repository.hsqldb;
 
 import com.google.common.primitives.Longs;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.controller.Controller;
+import org.qortal.crypto.Crypto;
 import org.qortal.data.at.ATData;
 import org.qortal.data.at.ATStateData;
 import org.qortal.repository.ATRepository;
@@ -16,6 +18,8 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
 
+import org.qortal.data.account.AccountData;
+
 public class HSQLDBATRepository implements ATRepository {
 
 	private static final Logger LOGGER = LogManager.getLogger(HSQLDBATRepository.class);
@@ -400,9 +404,9 @@ public class HSQLDBATRepository implements ATRepository {
 	}
 
 	@Override
-	public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, Boolean isFinished,
-			Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
-			Integer limit, Integer offset, Boolean reverse) throws DataException {
+	public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished,
+			  Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
+			  Integer limit, Integer offset, Boolean reverse) throws DataException {
 		StringBuilder sql = new StringBuilder(1024);
 		List<Object> bindParams = new ArrayList<>();
 
@@ -421,10 +425,14 @@ public class HSQLDBATRepository implements ATRepository {
 
 		// Order by AT_address and height to use compound primary key as index
 		// Both must be the same direction (DESC) also
-		sql.append("ORDER BY ATStates.AT_address DESC, ATStates.height DESC "
-					+ "LIMIT 1 "
-				+ ") AS FinalATStates "
-				+ "WHERE code_hash = ? ");
+		sql.append("ORDER BY ATStates.height DESC LIMIT 1) AS FinalATStates ");
+
+		// Optional JOIN with ATTRANSACTIONS for buyerAddress
+		if (buyerPublicKey != null && buyerPublicKey.length > 0) {
+			sql.append("JOIN ATTRANSACTIONS tx ON tx.at_address = ATs.AT_address ");
+		}
+	
+		sql.append("WHERE ATs.code_hash = ? ");
 		bindParams.add(codeHash);
 
 		if (isFinished != null) {
@@ -443,6 +451,20 @@ public class HSQLDBATRepository implements ATRepository {
 			bindParams.add(rawExpectedValue);
 		}
 
+		if (buyerPublicKey != null && buyerPublicKey.length > 0 ) {
+			// the buyer must be the recipient of the transaction and not the creator of the AT
+			sql.append("AND tx.recipient = ? AND ATs.creator != ? ");
+
+			bindParams.add(Crypto.toAddress(buyerPublicKey));
+			bindParams.add(buyerPublicKey);
+		}
+
+
+		if (sellerPublicKey != null && sellerPublicKey.length > 0) {
+			sql.append("AND ATs.creator = ? ");
+			bindParams.add(sellerPublicKey);
+		}
+
 		sql.append(" ORDER BY FinalATStates.height ");
 		if (reverse != null && reverse)
 			sql.append("DESC");
@@ -483,7 +505,7 @@ public class HSQLDBATRepository implements ATRepository {
 			Integer dataByteOffset, Long expectedValue,
 			int minimumCount, int maximumCount, long minimumPeriod) throws DataException {
 		// We need most recent entry first so we can use its timestamp to slice further results
-		List<ATStateData> mostRecentStates = this.getMatchingFinalATStates(codeHash, isFinished,
+		List<ATStateData> mostRecentStates = this.getMatchingFinalATStates(codeHash, null, null, isFinished,
 				dataByteOffset, expectedValue, null,
 				1, 0, true);
 

src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java

@@ -7,7 +7,6 @@ import org.qortal.arbitrary.ArbitraryDataFile;
 import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
 import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
-import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.data.arbitrary.ArbitraryResourceCache;
 import org.qortal.data.arbitrary.ArbitraryResourceData;
 import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
@@ -29,6 +28,7 @@ import org.qortal.utils.ListUtils;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -227,6 +227,144 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 		}
 	}
 
+	@Override
+	public List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException {
+		String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
+				"tx_group_id, block_height, approval_status, approval_height, " +
+				"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
+				"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
+				"JOIN Transactions USING (signature) " +
+				"WHERE name IS NOT NULL " +
+				"ORDER BY created_when DESC";
+		List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
+			if (resultSet == null)
+				return new ArrayList<>(0);
+
+			do {
+				byte[] reference = resultSet.getBytes(2);
+				byte[] signature = resultSet.getBytes(3);
+				byte[] creatorPublicKey = resultSet.getBytes(4);
+				long timestamp = resultSet.getLong(5);
+
+				Long fee = resultSet.getLong(6);
+				if (fee == 0 && resultSet.wasNull())
+					fee = null;
+
+				int txGroupId = resultSet.getInt(7);
+
+				Integer blockHeight = resultSet.getInt(8);
+				if (blockHeight == 0 && resultSet.wasNull())
+					blockHeight = null;
+
+				ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
+				Integer approvalHeight = resultSet.getInt(10);
+				if (approvalHeight == 0 && resultSet.wasNull())
+					approvalHeight = null;
+
+				BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
+
+				int version = resultSet.getInt(11);
+				int nonce = resultSet.getInt(12);
+				int serviceInt = resultSet.getInt(13);
+				int size = resultSet.getInt(14);
+				boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
+				DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
+				byte[] data = resultSet.getBytes(16);
+				byte[] metadataHash = resultSet.getBytes(17);
+				String nameResult = resultSet.getString(18);
+				String identifierResult = resultSet.getString(19);
+				Method method = Method.valueOf(resultSet.getInt(20));
+				byte[] secret = resultSet.getBytes(21);
+				Compression compression = Compression.valueOf(resultSet.getInt(22));
+				// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
+
+				ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
+						version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
+						compression, data, dataType, metadataHash, null);
+
+				arbitraryTransactionData.add(transactionData);
+			} while (resultSet.next());
+
+			return arbitraryTransactionData;
+		} catch (SQLException e) {
+			throw new DataException("Unable to fetch arbitrary transactions from repository", e);
+		} catch (Exception e) {
+			LOGGER.error(e.getMessage(), e);
+			return new ArrayList<>(0);
+		}
+	}
+
+	@Override
+	public List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName( String name ) throws DataException {
+		String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
+				"tx_group_id, block_height, approval_status, approval_height, " +
+				"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
+				"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
+				"JOIN Transactions USING (signature) " +
+				"WHERE name = ? " +
+				"ORDER BY created_when DESC";
+		List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql, name)) {
+			if (resultSet == null)
+				return new ArrayList<>(0);
+
+			do {
+				byte[] reference = resultSet.getBytes(2);
+				byte[] signature = resultSet.getBytes(3);
+				byte[] creatorPublicKey = resultSet.getBytes(4);
+				long timestamp = resultSet.getLong(5);
+
+				Long fee = resultSet.getLong(6);
+				if (fee == 0 && resultSet.wasNull())
+					fee = null;
+
+				int txGroupId = resultSet.getInt(7);
+
+				Integer blockHeight = resultSet.getInt(8);
+				if (blockHeight == 0 && resultSet.wasNull())
+					blockHeight = null;
+
+				ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
+				Integer approvalHeight = resultSet.getInt(10);
+				if (approvalHeight == 0 && resultSet.wasNull())
+					approvalHeight = null;
+
+				BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
+
+				int version = resultSet.getInt(11);
+				int nonce = resultSet.getInt(12);
+				int serviceInt = resultSet.getInt(13);
+				int size = resultSet.getInt(14);
+				boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
+				DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
+				byte[] data = resultSet.getBytes(16);
+				byte[] metadataHash = resultSet.getBytes(17);
+				String nameResult = resultSet.getString(18);
+				String identifierResult = resultSet.getString(19);
+				Method method = Method.valueOf(resultSet.getInt(20));
+				byte[] secret = resultSet.getBytes(21);
+				Compression compression = Compression.valueOf(resultSet.getInt(22));
+				// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
+
+				ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
+						version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
+						compression, data, dataType, metadataHash, null);
+
+				arbitraryTransactionData.add(transactionData);
+			} while (resultSet.next());
+
+			return arbitraryTransactionData;
+		} catch (SQLException e) {
+			throw new DataException("Unable to fetch arbitrary transactions from repository", e);
+		} catch (Exception e) {
+			LOGGER.error(e.getMessage(), e);
+			return new ArrayList<>(0);
+		}
+	}
+
 	private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException {
 		if (name == null || service == null) {
 			// Required fields
@@ -724,12 +862,11 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 	}
 
 	@Override
-	public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, boolean prefixOnly,
+	public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, List<String> keywords, boolean prefixOnly,
 																List<String> exactMatchNames, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked,
 																Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException {
 
 		if(Settings.getInstance().isDbCacheEnabled()) {
-
 			List<ArbitraryResourceData> list
 				= HSQLDBCacheUtils.callCache(
 					ArbitraryResourceCache.getInstance(),
@@ -751,6 +888,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 						Optional.ofNullable(description),
 						prefixOnly,
 						Optional.ofNullable(exactMatchNames),
+						Optional.ofNullable(keywords),
 						defaultResource,
 						Optional.ofNullable(minLevel),
 						Optional.ofNullable(() -> ListUtils.followedNames()),
@@ -771,6 +909,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			}
 		}
 
+
 		StringBuilder sql = new StringBuilder(512);
 		List<Object> bindParams = new ArrayList<>();
 
@@ -857,6 +996,26 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			bindParams.add(queryWildcard);
 		}
 
+		if (keywords != null && !keywords.isEmpty()) {
+			List<String> searchKeywords = new ArrayList<>(keywords); 
+		
+			List<String> conditions = new ArrayList<>();
+			List<String> bindValues = new ArrayList<>();
+		
+			for (int i = 0; i < searchKeywords.size(); i++) {
+				conditions.add("LOWER(description) LIKE ?"); 
+				bindValues.add("%" + searchKeywords.get(i).trim().toLowerCase() + "%"); 
+			}
+		
+			String finalCondition = String.join(" OR ", conditions);
+			sql.append(" AND (").append(finalCondition).append(")");
+		
+			bindParams.addAll(bindValues); 
+		}
+		
+		
+		
+
 		// Handle name searches
 		if (names != null && !names.isEmpty()) {
 			sql.append(" AND (");

src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java

@@ -3,12 +3,24 @@ package org.qortal.repository.hsqldb;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.api.SearchMode;
+import org.qortal.api.resource.TransactionsResource;
 import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
+import org.qortal.controller.Controller;
+import org.qortal.data.account.AccountBalanceData;
+import org.qortal.data.account.AddressAmountData;
+import org.qortal.data.account.BlockHeightRange;
+import org.qortal.data.account.BlockHeightRangeAddressAmounts;
 import org.qortal.data.arbitrary.ArbitraryResourceCache;
 import org.qortal.data.arbitrary.ArbitraryResourceData;
 import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
 import org.qortal.data.arbitrary.ArbitraryResourceStatus;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.BalanceRecorderUtils;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -25,6 +37,7 @@ import java.util.Optional;
 import java.util.Timer;
 import java.util.TimerTask;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
@@ -48,6 +61,11 @@ public class HSQLDBCacheUtils {
         }
     };
     private static final String DEFAULT_IDENTIFIER = "default";
+    private static final int ZERO = 0;
+    public static final String DB_CACHE_TIMER = "DB Cache Timer";
+    public static final String DB_CACHE_TIMER_TASK = "DB Cache Timer Task";
+    public static final String BALANCE_RECORDER_TIMER = "Balance Recorder Timer";
+    public static final String BALANCE_RECORDER_TIMER_TASK = "Balance Recorder Timer Task";
 
     /**
      *
@@ -149,6 +167,7 @@ public class HSQLDBCacheUtils {
             Optional<String> description,
             boolean prefixOnly,
             Optional<List<String>> exactMatchNames,
+            Optional<List<String>> keywords,
             boolean defaultResource,
             Optional<Integer> minLevel,
             Optional<Supplier<List<String>>> includeOnly,
@@ -162,7 +181,18 @@ public class HSQLDBCacheUtils {
             Optional<Boolean> reverse) {
 
         // retain only candidates with names
-        Stream<ArbitraryResourceData> stream = candidates.stream().filter(candidate -> candidate.name != null);
+        Stream<ArbitraryResourceData> stream = candidates.stream().filter(candidate -> candidate.name != null );
+
+        if(after.isPresent()) {
+            stream = stream.filter( candidate -> candidate.created > after.get().longValue() );
+        }
+
+        if(before.isPresent()) {
+            stream = stream.filter( candidate -> candidate.created < before.get().longValue() );
+        }
+
+        if(exclude.isPresent())
+            stream = stream.filter( candidate -> !exclude.get().get().contains( candidate.name ));
 
         // filter by service
         if( service.isPresent() )
@@ -186,6 +216,36 @@ public class HSQLDBCacheUtils {
         stream = filterTerm(title, data -> data.metadata != null ? data.metadata.getTitle() : null, prefixOnly, stream);
         stream = filterTerm(description, data -> data.metadata != null ? data.metadata.getDescription() : null, prefixOnly, stream);
 
+        // New: Filter by keywords if provided
+        if (keywords.isPresent() && !keywords.get().isEmpty()) {
+            List<String> searchKeywords = keywords.get().stream()
+                .map(String::toLowerCase)
+                .collect(Collectors.toList());
+
+            stream = stream.filter(candidate -> {
+                
+                if (candidate.metadata != null && candidate.metadata.getDescription() != null) {
+                    String descriptionLower = candidate.metadata.getDescription().toLowerCase();
+                    return searchKeywords.stream().anyMatch(descriptionLower::contains);
+                }
+                return false;
+            });
+        }
+
+        if (keywords.isPresent() && !keywords.get().isEmpty()) {
+            List<String> searchKeywords = keywords.get().stream()
+                .map(String::toLowerCase)
+                .collect(Collectors.toList());
+
+            stream = stream.filter(candidate -> {
+                if (candidate.metadata != null && candidate.metadata.getDescription() != null) {
+            String descriptionLower = candidate.metadata.getDescription().toLowerCase();
+            return searchKeywords.stream().anyMatch(descriptionLower::contains);
+                }
+                return false;
+            });
+        }
+
         // if exact names is set, retain resources with exact names
         if( exactMatchNames.isPresent() && !exactMatchNames.get().isEmpty()) {
 
@@ -241,15 +301,58 @@ public class HSQLDBCacheUtils {
         // truncate to limit
         if( limit.isPresent() && limit.get() > 0 ) stream = stream.limit(limit.get());
 
-        // include metadata
-        if( includeMetadata.isEmpty() || !includeMetadata.get() )
-            stream = stream.peek( candidate -> candidate.metadata = null );
+        List<ArbitraryResourceData> listCopy1 = stream.collect(Collectors.toList());
 
-        // include status
-        if( includeStatus.isEmpty() || !includeStatus.get() )
-            stream = stream.peek( candidate -> candidate.status = null);
+        List<ArbitraryResourceData> listCopy2 = new ArrayList<>(listCopy1.size());
 
-        return stream.collect(Collectors.toList());
+        // remove metadata from the first copy
+        if( includeMetadata.isEmpty() || !includeMetadata.get() ) {
+            for( ArbitraryResourceData data : listCopy1 ) {
+                ArbitraryResourceData copy = new ArbitraryResourceData();
+                copy.name = data.name;
+                copy.service = data.service;
+                copy.identifier = data.identifier;
+                copy.status = data.status;
+                copy.metadata = null;
+
+                copy.size = data.size;
+                copy.created = data.created;
+                copy.updated = data.updated;
+
+                listCopy2.add(copy);
+            }
+        }
+        // put the list copy 1 into the second copy
+        else {
+            listCopy2.addAll(listCopy1);
+        }
+
+        // remove status from final copy
+        if( includeStatus.isEmpty() || !includeStatus.get() ) {
+
+            List<ArbitraryResourceData> finalCopy = new ArrayList<>(listCopy2.size());
+
+            for( ArbitraryResourceData data : listCopy2 ) {
+                ArbitraryResourceData copy = new ArbitraryResourceData();
+                copy.name = data.name;
+                copy.service = data.service;
+                copy.identifier = data.identifier;
+                copy.status = null;
+                copy.metadata = data.metadata;
+
+                copy.size = data.size;
+                copy.created = data.created;
+                copy.updated = data.updated;
+
+                finalCopy.add(copy);
+            }
+
+            return finalCopy;
+        }
+        // keep status included by returning the second copy
+        else {
+            return listCopy2;
+        }
     }
 
     /**
@@ -351,13 +454,200 @@ public class HSQLDBCacheUtils {
      * Start Caching
      *
      * @param priorityRequested the thread priority to fill cache in
-     * @param frequency the frequency to fill the cache (in seconds)
-     * @param respository the data source
+     * @param frequency         the frequency to fill the cache (in seconds)
      *
      * @return the data cache
      */
-    public static void startCaching(int priorityRequested, int frequency, HSQLDBRepository respository) {
+    public static void startCaching(int priorityRequested, int frequency) {
 
+        Timer timer = buildTimer(DB_CACHE_TIMER, priorityRequested);
+
+        TimerTask task = new TimerTask() {
+            @Override
+            public void run() {
+
+                Thread.currentThread().setName(DB_CACHE_TIMER_TASK);
+
+                try (final HSQLDBRepository respository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) {
+                    fillCache(ArbitraryResourceCache.getInstance(), respository);
+                }
+                catch( DataException e ) {
+                    LOGGER.error(e.getMessage(), e);
+                }
+            }
+        };
+
+        // delay 1 second
+        timer.scheduleAtFixedRate(task, 1000, frequency * 1000);
+    }
+
+    /**
+     * Start Recording Balances
+     *
+     * @param balancesByHeight height -> account balances
+     * @param balanceDynamics every balance dynamic
+     * @param priorityRequested the requested thread priority
+     * @param frequency the recording frequencies, in minutes
+     * @param capacity the maximum size of balanceDynamics
+     */
+    public static void startRecordingBalances(
+            final ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight,
+            CopyOnWriteArrayList<BlockHeightRangeAddressAmounts> balanceDynamics,
+            int priorityRequested,
+            int frequency,
+            int capacity) {
+
+        Timer timer = buildTimer(BALANCE_RECORDER_TIMER, priorityRequested);
+
+        TimerTask task = new TimerTask() {
+            @Override
+            public void run() {
+
+                Thread.currentThread().setName(BALANCE_RECORDER_TIMER_TASK);
+
+                int currentHeight = recordCurrentBalances(balancesByHeight);
+
+                LOGGER.debug("recorded balances: height = " + currentHeight);
+
+                // remove invalidated recordings, recording after current height
+                BalanceRecorderUtils.removeRecordingsAboveHeight(currentHeight, balancesByHeight);
+
+                // remove invalidated dynamics, on or after current height
+                BalanceRecorderUtils.removeDynamicsOnOrAboveHeight(currentHeight, balanceDynamics);
+
+                // if there are 2 or more recordings, then produce balance dynamics for the first 2 recordings
+                if( balancesByHeight.size() > 1 ) {
+
+                    Optional<Integer> priorHeight = BalanceRecorderUtils.getPriorHeight(currentHeight, balancesByHeight);
+
+                    // if there is a prior height
+                    if(priorHeight.isPresent()) {
+
+                        boolean isRewardDistribution = BalanceRecorderUtils.isRewardDistributionRange(priorHeight.get(), currentHeight);
+
+                        // if this range has a reward recording block or if other blocks are enabled for recording
+                        if( isRewardDistribution || !Settings.getInstance().isRewardRecordingOnly() ) {
+                            produceBalanceDynamics(currentHeight, priorHeight, isRewardDistribution, balancesByHeight, balanceDynamics, capacity);
+                        }
+                    }
+                    else {
+                        LOGGER.warn("Expecting prior height and nothing was discovered, current height = " + currentHeight);
+                    }
+                }
+                // else this should be the first recording
+                else {
+                    LOGGER.info("first balance recording completed");
+                }
+            }
+        };
+
+        // wait 5 minutes
+        timer.scheduleAtFixedRate(task, 300_000, frequency * 60_000);
+    }
+
+    private static void produceBalanceDynamics(int currentHeight, Optional<Integer> priorHeight, boolean isRewardDistribution, ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight, CopyOnWriteArrayList<BlockHeightRangeAddressAmounts> balanceDynamics, int capacity) {
+        BlockHeightRange blockHeightRange = new BlockHeightRange(priorHeight.get(), currentHeight, isRewardDistribution);
+
+        LOGGER.debug("building dynamics for block heights: range = " + blockHeightRange);
+
+        List<AccountBalanceData> currentBalances = balancesByHeight.get(currentHeight);
+
+        ArrayList<TransactionData> transactions = getTransactionDataForBlocks(blockHeightRange);
+
+        LOGGER.info("transactions counted for balance adjustments: count = " + transactions.size());
+        List<AddressAmountData> currentDynamics
+            = BalanceRecorderUtils.buildBalanceDynamics(
+                currentBalances,
+                balancesByHeight.get(priorHeight.get()),
+                Settings.getInstance().getMinimumBalanceRecording(),
+                transactions);
+
+        LOGGER.debug("dynamics built: count = " + currentDynamics.size());
+
+        if(LOGGER.isDebugEnabled())
+            currentDynamics.stream()
+                .sorted(Comparator.comparingLong(AddressAmountData::getAmount).reversed())
+                .limit(Settings.getInstance().getTopBalanceLoggingLimit())
+                .forEach(top5Dynamic -> LOGGER.debug("Top Dynamics = " + top5Dynamic));
+
+        BlockHeightRangeAddressAmounts amounts
+            = new BlockHeightRangeAddressAmounts( blockHeightRange, currentDynamics );
+
+        balanceDynamics.add(amounts);
+
+        BalanceRecorderUtils.removeRecordingsBelowHeight(currentHeight - Settings.getInstance().getBalanceRecorderRollbackAllowance(), balancesByHeight);
+
+        while(balanceDynamics.size() > capacity) {
+            BlockHeightRangeAddressAmounts oldestDynamics = BalanceRecorderUtils.removeOldestDynamics(balanceDynamics);
+
+            LOGGER.debug("removing oldest dynamics: range " + oldestDynamics.getRange());
+        }
+    }
+
+    private static ArrayList<TransactionData> getTransactionDataForBlocks(BlockHeightRange blockHeightRange) {
+        ArrayList<TransactionData> transactions;
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            List<byte[]> signatures
+                = repository.getTransactionRepository().getSignaturesMatchingCriteria(
+                    blockHeightRange.getBegin() + 1, blockHeightRange.getEnd() - blockHeightRange.getBegin(),
+                    null, null,null, null, null,
+                    TransactionsResource.ConfirmationStatus.CONFIRMED,
+                    null, null, null);
+
+            transactions = new ArrayList<>(signatures.size());
+            for (byte[] signature : signatures) {
+                transactions.add(repository.getTransactionRepository().fromSignature(signature));
+            }
+
+            LOGGER.debug(String.format("Found %s transactions for " + blockHeightRange, transactions.size()));
+        } catch (Exception e) {
+            transactions = new ArrayList<>(0);
+            LOGGER.warn("Problems getting transactions for balance recording: " + e.getMessage());
+        }
+        return transactions;
+    }
+
+    private static int recordCurrentBalances(ConcurrentHashMap<Integer, List<AccountBalanceData>> balancesByHeight) {
+        int currentHeight;
+
+        try (final HSQLDBRepository repository = (HSQLDBRepository) Controller.REPOSITORY_FACTORY.getRepository()) {
+
+            // get current balances
+            List<AccountBalanceData> accountBalances = getAccountBalances(repository);
+
+            // get anyone of the balances
+            Optional<AccountBalanceData> data = accountBalances.stream().findAny();
+
+            // if there are any balances, then record them
+            if (data.isPresent()) {
+                // map all new balances to the current height
+                balancesByHeight.put(data.get().getHeight(), accountBalances);
+
+                currentHeight =  data.get().getHeight();
+            }
+            else {
+                currentHeight = Integer.MAX_VALUE;
+            }
+        } catch (DataException e) {
+            LOGGER.error(e.getMessage(), e);
+            currentHeight = Integer.MAX_VALUE;
+        }
+
+        return currentHeight;
+    }
+
+    /**
+     * Build Timer
+     *
+     * Build a timer for scheduling a timer task.
+     *
+     * @param name the name for the thread running the timer task
+     * @param priorityRequested the priority for the thread running the timer task
+     *
+     * @return a timer for scheduling a timer task
+     */
+    private static Timer buildTimer( final String name, int priorityRequested) {
         // ensure priority is in between 1-10
         final int priority = Math.max(0, Math.min(10, priorityRequested));
 
@@ -365,7 +655,7 @@ public class HSQLDBCacheUtils {
         Timer timer = new Timer(true) { // 'true' to make the Timer daemon
             @Override
             public void schedule(TimerTask task, long delay) {
-                Thread thread = new Thread(task) {
+                Thread thread = new Thread(task, name) {
                     @Override
                     public void run() {
                         this.setPriority(priority);
@@ -376,17 +666,7 @@ public class HSQLDBCacheUtils {
                 thread.start();
             }
         };
-
-        TimerTask task = new TimerTask() {
-            @Override
-            public void run() {
-
-                fillCache(ArbitraryResourceCache.getInstance(), respository);
-            }
-        };
-
-        // delay 1 second
-        timer.scheduleAtFixedRate(task, 1000, frequency * 1000);
+        return timer;
     }
 
     /**
@@ -541,4 +821,43 @@ public class HSQLDBCacheUtils {
 
         return resources;
     }
+
+    public static List<AccountBalanceData> getAccountBalances(HSQLDBRepository repository) {
+
+        StringBuilder sql = new StringBuilder();
+
+        sql.append("SELECT account, balance, height ");
+        sql.append("FROM ACCOUNTBALANCES as balances ");
+        sql.append("JOIN (SELECT height FROM BLOCKS ORDER BY height DESC LIMIT 1) AS max_height ON true ");
+        sql.append("WHERE asset_id=0");
+
+        List<AccountBalanceData> data = new ArrayList<>();
+
+        LOGGER.info( "Getting account balances ...");
+
+        try {
+            Statement statement = repository.connection.createStatement();
+
+            ResultSet resultSet = statement.executeQuery(sql.toString());
+
+            if (resultSet == null || !resultSet.next())
+                return new ArrayList<>(0);
+
+            do {
+                String account = resultSet.getString(1);
+                long balance = resultSet.getLong(2);
+                int height = resultSet.getInt(3);
+
+                data.add(new AccountBalanceData(account, ZERO, balance, height));
+            } while (resultSet.next());
+        } catch (SQLException e) {
+            LOGGER.warn(e.getMessage());
+        } catch (Exception e) {
+            LOGGER.error(e.getMessage(), e);
+        }
+
+        LOGGER.info("Retrieved account balances: count = " + data.size());
+
+        return data;
+    }
 }

src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java

@@ -23,7 +23,7 @@ public class HSQLDBChatRepository implements ChatRepository {
 	public HSQLDBChatRepository(HSQLDBRepository repository) {
 		this.repository = repository;
 	}
-
+	
 	@Override
 	public List<ChatMessage> getMessagesMatchingCriteria(Long before, Long after, Integer txGroupId, byte[] referenceBytes,
 														 byte[] chatReferenceBytes, Boolean hasChatReference, List<String> involving, String senderAddress,
@@ -176,14 +176,14 @@ public class HSQLDBChatRepository implements ChatRepository {
 	}
 
 	@Override
-	public ActiveChats getActiveChats(String address, Encoding encoding) throws DataException {
-		List<GroupChat> groupChats = getActiveGroupChats(address, encoding);
-		List<DirectChat> directChats = getActiveDirectChats(address);
+	public ActiveChats getActiveChats(String address, Encoding encoding, Boolean hasChatReference) throws DataException {
+		List<GroupChat> groupChats = getActiveGroupChats(address, encoding, hasChatReference);
+		List<DirectChat> directChats = getActiveDirectChats(address, hasChatReference);
 
 		return new ActiveChats(groupChats, directChats);
 	}
-
-	private List<GroupChat> getActiveGroupChats(String address, Encoding encoding) throws DataException {
+	
+	private List<GroupChat> getActiveGroupChats(String address, Encoding encoding, Boolean hasChatReference) throws DataException {
 		// Find groups where address is a member and potential latest message details
 		String groupsSql = "SELECT group_id, group_name, latest_timestamp, sender, sender_name, signature, data "
 				+ "FROM GroupMembers "
@@ -194,11 +194,19 @@ public class HSQLDBChatRepository implements ChatRepository {
 					+ "JOIN Transactions USING (signature) "
 					+ "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender "
 					// NOTE: We need to qualify "Groups.group_id" here to avoid "General error" bug in HSQLDB v2.5.0
-					+ "WHERE tx_group_id = Groups.group_id AND type = " + TransactionType.CHAT.value + " "
-					+ "ORDER BY created_when DESC "
+					+ "WHERE tx_group_id = Groups.group_id AND type = " + TransactionType.CHAT.value + " ";
+
+					if (hasChatReference != null) {
+						if (hasChatReference) {
+							groupsSql += "AND chat_reference IS NOT NULL ";
+						} else {
+							groupsSql += "AND chat_reference IS NULL ";
+						}
+					}
+					groupsSql += "ORDER BY created_when DESC "
 					+ "LIMIT 1"
-				+ ") AS LatestMessages ON TRUE "
-				+ "WHERE address = ?";
+					+ ") AS LatestMessages ON TRUE "
+					+ "WHERE address = ?";
 
 		List<GroupChat> groupChats = new ArrayList<>();
 		try (ResultSet resultSet = this.repository.checkedExecute(groupsSql, address)) {
@@ -230,8 +238,16 @@ public class HSQLDBChatRepository implements ChatRepository {
 				+ "JOIN Transactions USING (signature) "
 				+ "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender "
 				+ "WHERE tx_group_id = 0 "
-				+ "AND recipient IS NULL "
-				+ "ORDER BY created_when DESC "
+				+ "AND recipient IS NULL ";
+
+				if (hasChatReference != null) {
+					if (hasChatReference) {
+						grouplessSql += "AND chat_reference IS NOT NULL ";
+					} else {
+						grouplessSql += "AND chat_reference IS NULL ";
+					}
+				}
+				grouplessSql += "ORDER BY created_when DESC "
 				+ "LIMIT 1";
 
 		try (ResultSet resultSet = this.repository.checkedExecute(grouplessSql)) {
@@ -259,7 +275,7 @@ public class HSQLDBChatRepository implements ChatRepository {
 		return groupChats;
 	}
 
-	private List<DirectChat> getActiveDirectChats(String address) throws DataException {
+	private List<DirectChat> getActiveDirectChats(String address, Boolean hasChatReference) throws DataException {
 		// Find chat messages involving address
 		String directSql = "SELECT other_address, name, latest_timestamp, sender, sender_name "
 				+ "FROM ("
@@ -275,11 +291,21 @@ public class HSQLDBChatRepository implements ChatRepository {
 					+ "NATURAL JOIN Transactions "
 					+ "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender "
 					+ "WHERE (sender = other_address AND recipient = ?) "
-					+ "OR (sender = ? AND recipient = other_address) "
-					+ "ORDER BY created_when DESC "
-					+ "LIMIT 1"
-				+ ") AS LatestMessages "
-				+ "LEFT OUTER JOIN Names ON owner = other_address";
+					+ "OR (sender = ? AND recipient = other_address) ";
+
+			    // Apply hasChatReference filter
+				if (hasChatReference != null) {
+					if (hasChatReference) {
+						directSql += "AND chat_reference IS NOT NULL ";
+					} else {
+						directSql += "AND chat_reference IS NULL ";
+					}
+				}
+			
+				directSql += "ORDER BY created_when DESC "
+						+ "LIMIT 1"
+						+ ") AS LatestMessages "
+						+ "LEFT OUTER JOIN Names ON owner = other_address";
 
 		Object[] bindParams = new Object[] { address, address, address, address };
 

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -454,40 +454,41 @@ public class HSQLDBDatabaseUpdates {
 
 				case 12:
 					// Groups
-					stmt.execute("CREATE TABLE Groups (group_id GroupID, owner QortalAddress NOT NULL, group_name GroupName NOT NULL, "
+					// NOTE: We need to set Groups to `GROUPS` here to avoid SQL Standard Keywords in HSQLDB v2.7.4
+					stmt.execute("CREATE TABLE `GROUPS` (group_id GroupID, owner QortalAddress NOT NULL, group_name GroupName NOT NULL, "
 							+ "created_when EpochMillis NOT NULL, updated_when EpochMillis, is_open BOOLEAN NOT NULL, "
 							+ "approval_threshold TINYINT NOT NULL, min_block_delay INTEGER NOT NULL, max_block_delay INTEGER NOT NULL, "
 							+ "reference Signature, creation_group_id GroupID, reduced_group_name GroupName NOT NULL, "
 							+ "description GenericDescription NOT NULL, PRIMARY KEY (group_id))");
 					// For finding groups by name
-					stmt.execute("CREATE INDEX GroupNameIndex on Groups (group_name)");
+					stmt.execute("CREATE INDEX GroupNameIndex on `GROUPS` (group_name)");
 					// For finding groups by reduced name
-					stmt.execute("CREATE INDEX GroupReducedNameIndex on Groups (reduced_group_name)");
+					stmt.execute("CREATE INDEX GroupReducedNameIndex on `GROUPS` (reduced_group_name)");
 					// For finding groups by owner
-					stmt.execute("CREATE INDEX GroupOwnerIndex ON Groups (owner)");
+					stmt.execute("CREATE INDEX GroupOwnerIndex ON `GROUPS` (owner)");
 
 					// We need a corresponding trigger to make sure new group_id values are assigned sequentially starting from 1
-					stmt.execute("CREATE TRIGGER Group_ID_Trigger BEFORE INSERT ON Groups "
+					stmt.execute("CREATE TRIGGER Group_ID_Trigger BEFORE INSERT ON `GROUPS` "
 							+ "REFERENCING NEW ROW AS new_row FOR EACH ROW WHEN (new_row.group_id IS NULL) "
-							+ "SET new_row.group_id = (SELECT IFNULL(MAX(group_id) + 1, 1) FROM Groups)");
+							+ "SET new_row.group_id = (SELECT IFNULL(MAX(group_id) + 1, 1) FROM `GROUPS`)");
 
 					// Admins
 					stmt.execute("CREATE TABLE GroupAdmins (group_id GroupID, admin QortalAddress, reference Signature NOT NULL, "
-							+ "PRIMARY KEY (group_id, admin), FOREIGN KEY (group_id) REFERENCES Groups (group_id) ON DELETE CASCADE)");
+							+ "PRIMARY KEY (group_id, admin), FOREIGN KEY (group_id) REFERENCES `GROUPS` (group_id) ON DELETE CASCADE)");
 					// For finding groups by admin address
 					stmt.execute("CREATE INDEX GroupAdminIndex ON GroupAdmins (admin)");
 
 					// Members
 					stmt.execute("CREATE TABLE GroupMembers (group_id GroupID, address QortalAddress, "
 							+ "joined_when EpochMillis NOT NULL, reference Signature NOT NULL, "
-							+ "PRIMARY KEY (group_id, address), FOREIGN KEY (group_id) REFERENCES Groups (group_id) ON DELETE CASCADE)");
+							+ "PRIMARY KEY (group_id, address), FOREIGN KEY (group_id) REFERENCES `GROUPS` (group_id) ON DELETE CASCADE)");
 					// For finding groups by member address
 					stmt.execute("CREATE INDEX GroupMemberIndex ON GroupMembers (address)");
 
 					// Invites
 					stmt.execute("CREATE TABLE GroupInvites (group_id GroupID, inviter QortalAddress, invitee QortalAddress, "
 							+ "expires_when EpochMillis, reference Signature, "
-							+ "PRIMARY KEY (group_id, invitee), FOREIGN KEY (group_id) REFERENCES Groups (group_id) ON DELETE CASCADE)");
+							+ "PRIMARY KEY (group_id, invitee), FOREIGN KEY (group_id) REFERENCES `GROUPS` (group_id) ON DELETE CASCADE)");
 					// For finding invites sent by inviter
 					stmt.execute("CREATE INDEX GroupInviteInviterIndex ON GroupInvites (inviter)");
 					// For finding invites by group
@@ -503,7 +504,7 @@ public class HSQLDBDatabaseUpdates {
 					// NULL expires_when means does not expire!
 					stmt.execute("CREATE TABLE GroupBans (group_id GroupID, offender QortalAddress, admin QortalAddress NOT NULL, "
 							+ "banned_when EpochMillis NOT NULL, reason GenericDescription NOT NULL, expires_when EpochMillis, reference Signature NOT NULL, "
-							+ "PRIMARY KEY (group_id, offender), FOREIGN KEY (group_id) REFERENCES Groups (group_id) ON DELETE CASCADE)");
+							+ "PRIMARY KEY (group_id, offender), FOREIGN KEY (group_id) REFERENCES `GROUPS` (group_id) ON DELETE CASCADE)");
 					// For expiry maintenance
 					stmt.execute("CREATE INDEX GroupBanExpiryIndex ON GroupBans (expires_when)");
 					break;

src/main/java/org/qortal/repository/hsqldb/HSQLDBGroupRepository.java

@@ -351,7 +351,7 @@ public class HSQLDBGroupRepository implements GroupRepository {
 	// Group Admins
 
 	@Override
-	public GroupAdminData getAdmin(int groupId, String address) throws DataException {
+	public GroupAdminData getAdminFaulty(int groupId, String address) throws DataException {
 		try (ResultSet resultSet = this.repository.checkedExecute("SELECT admin, reference FROM GroupAdmins WHERE group_id = ?", groupId)) {
 			if (resultSet == null)
 				return null;
@@ -365,6 +365,21 @@ public class HSQLDBGroupRepository implements GroupRepository {
 		}
 	}
 
+	@Override
+	public GroupAdminData getAdmin(int groupId, String address) throws DataException {
+		try (ResultSet resultSet = this.repository.checkedExecute("SELECT admin, reference FROM GroupAdmins WHERE group_id = ? AND admin = ?", groupId, address)) {
+			if (resultSet == null)
+				return null;
+
+			String admin = resultSet.getString(1);
+			byte[] reference = resultSet.getBytes(2);
+
+			return new GroupAdminData(groupId, admin, reference);
+		} catch (SQLException e) {
+			throw new DataException("Unable to fetch group admin from repository", e);
+		}
+	}
+
 	@Override
 	public boolean adminExists(int groupId, String address) throws DataException {
 		try {

src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java

@@ -5,6 +5,8 @@ import org.apache.logging.log4j.Logger;
 import org.hsqldb.HsqlException;
 import org.hsqldb.error.ErrorCode;
 import org.hsqldb.jdbc.HSQLDBPool;
+import org.hsqldb.jdbc.HSQLDBPoolMonitored;
+import org.qortal.data.system.DbConnectionInfo;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryFactory;
@@ -14,6 +16,8 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Properties;
 
 public class HSQLDBRepositoryFactory implements RepositoryFactory {
@@ -57,7 +61,13 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory {
 			HSQLDBRepository.attemptRecovery(connectionUrl, "backup");
 		}
 
-		this.connectionPool = new HSQLDBPool(Settings.getInstance().getRepositoryConnectionPoolSize());
+		if(Settings.getInstance().isConnectionPoolMonitorEnabled()) {
+			this.connectionPool = new HSQLDBPoolMonitored(Settings.getInstance().getRepositoryConnectionPoolSize());
+		}
+		else {
+			this.connectionPool = new HSQLDBPool(Settings.getInstance().getRepositoryConnectionPoolSize());
+		}
+
 		this.connectionPool.setUrl(this.connectionUrl);
 
 		Properties properties = new Properties();
@@ -153,4 +163,19 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory {
 		return HSQLDBRepository.isDeadlockException(e);
 	}
 
+	/**
+	 * Get Connection States
+	 *
+	 * Get the database connection states, if database connection pool monitoring is enabled.
+	 *
+	 * @return the connection states if enabled, otherwise an empty list
+	 */
+	public List<DbConnectionInfo> getDbConnectionsStates() {
+		if( Settings.getInstance().isConnectionPoolMonitorEnabled() ) {
+			return ((HSQLDBPoolMonitored) this.connectionPool).getDbConnectionsStates();
+		}
+		else {
+			return new ArrayList<>(0);
+		}
+	}
 }

src/main/java/org/qortal/settings/Settings.java

@@ -213,7 +213,7 @@ public class Settings {
 	public long recoveryModeTimeout = 9999999999999L;
 
 	/** Minimum peer version number required in order to sync with them */
-	private String minPeerVersion = "4.6.0";
+	private String minPeerVersion = "4.6.5";
 	/** Whether to allow connections with peers below minPeerVersion
 	 * If true, we won't sync with them but they can still sync with us, and will show in the peers list
 	 * If false, sync will be blocked both ways, and they will not appear in the peers list */
@@ -222,7 +222,7 @@ public class Settings {
 	/** Minimum time (in seconds) that we should attempt to remain connected to a peer for */
 	private int minPeerConnectionTime = 2 * 60 * 60; // seconds
 	/** Maximum time (in seconds) that we should attempt to remain connected to a peer for */
-	private int maxPeerConnectionTime = 4 * 60 * 60; // seconds
+	private int maxPeerConnectionTime = 6 * 60 * 60; // seconds
 	/** Maximum time (in seconds) that a peer should remain connected when requesting QDN data */
 	private int maxDataPeerConnectionTime = 30 * 60; // seconds
 
@@ -281,7 +281,10 @@ public class Settings {
 	// Auto-update sources
 	private String[] autoUpdateRepos = new String[] {
 		"https://github.com/Qortal/qortal/raw/%s/qortal.update",
-		"https://raw.githubusercontent.com@151.101.16.133/Qortal/qortal/%s/qortal.update"
+		"https://raw.githubusercontent.com@151.101.16.133/Qortal/qortal/%s/qortal.update",
+	    "https://qortal.link/Auto-Update/%s/qortal.update",
+	    "https://qortal.name/Auto-Update/%s/qortal.update",
+	    "https://update.qortal.org/Auto-Update/%s/qortal.update"
 	};
 
 	// Lists
@@ -383,7 +386,7 @@ public class Settings {
 	/**
 	 * DB Cache Enabled?
 	 */
-	private boolean dbCacheEnabled = false;
+	private boolean dbCacheEnabled = true;
 
 	/**
 	 * DB Cache Thread Priority
@@ -441,6 +444,107 @@ public class Settings {
 	 */
 	private long archivingPause = 3000;
 
+	/**
+	 * Enable Balance Recorder?
+	 *
+	 * True for balance recording, otherwise false.
+	 */
+	private boolean balanceRecorderEnabled = false;
+
+	/**
+	 * Balance Recorder Priority
+	 *
+	 * The thread priority (1 is lowest, 10 is highest) of the balance recorder thread, if enabled.
+	 */
+	private int balanceRecorderPriority = 1;
+
+	/**
+	 * Balance Recorder Frequency
+	 *
+	 * How often the balances will be recorded, if enabled, measured in minutes.
+	 */
+	private int balanceRecorderFrequency = 20;
+
+	/**
+	 * Balance Recorder Capacity
+	 *
+	 * The number of balance recorder ranges will be held in memory.
+	 */
+	private int balanceRecorderCapacity = 1000;
+
+	/**
+	 * Minimum Balance Recording
+	 *
+	 * The minimum recored balance change in Qortoshis (1/100000000 QORT)
+	 */
+    private long minimumBalanceRecording = 100000000;
+
+	/**
+	 * Top Balance Logging Limit
+	 *
+	 * When logging the number limit of top balance changes to show in the logs for any given block range.
+	 */
+	private long topBalanceLoggingLimit = 100;
+
+	/**
+	 * Balance Recorder Rollback Allowance
+	 *
+	 * If the balance recorder is enabled, it must protect its prior balances by this number of blocks in case of
+	 * a blockchain rollback and reorganization.
+	 */
+	private int balanceRecorderRollbackAllowance = 100;
+
+	/**
+	 * Is Reward Recording Only
+	 *
+	 * Set true to only retain the recordings that cover reward distributions, otherwise set false.
+	 */
+    private boolean rewardRecordingOnly = true;
+
+	/**
+	 * Is The Connection Monitored?
+	 *
+	 * Is the database connection pooled monitored?
+	 */
+    private boolean connectionPoolMonitorEnabled = false;
+
+	/**
+	 * Buiild Arbitrary Resources Batch Size
+	 *
+	 * The number resources to batch per iteration when rebuilding.
+	 */
+	private int buildArbitraryResourcesBatchSize = 200;
+
+	/**
+	 * Arbitrary Indexing Priority
+	 *
+	 * The thread priority when indexing arbirary resources.
+	 */
+    private int arbitraryIndexingPriority = 5;
+
+	/**
+	 * Arbitrary Indexing Frequency (In Minutes)
+	 *
+	 * The frequency at which the arbitrary indices are cached.
+	 */
+	private int arbitraryIndexingFrequency = 10;
+
+    private boolean rebuildArbitraryResourceCacheTaskEnabled = false;
+
+	/**
+	 * Rebuild Arbitrary Resource Cache Task Delay (In Minutes)
+	 *
+	 * Waiting period before the first rebuild task is started.
+	 */
+	private int rebuildArbitraryResourceCacheTaskDelay = 300;
+
+	/**
+	 * Rebuild Arbitrary Resource Cache Task Period (In Hours)
+	 *
+	 * The frequency the arbitrary resource cache is rebuilt.
+	 */
+	private int rebuildArbitraryResourceCacheTaskPeriod = 24;
+
 	// Domain mapping
 	public static class ThreadLimit {
 		private String messageType;
@@ -510,6 +614,17 @@ public class Settings {
 		}
 	}
 
+	// Related to mesh networking
+
+	/** Preferred network: "tcpip" or "reticulum" */
+	private String preferredNetwork = "reticulum";
+	/** Maximum number of Reticulum peers allowed. */
+	private int reticulumMaxPeers = 55;
+	/** Minimum number of Reticulum peers desired. */
+	private int reticulumMinDesiredPeers = 8;
+	/** Maximum number of task executor network threads */
+	private int reticulumMaxNetworkThreadPoolSize = 89;
+
 	// Constructors
 
 	private Settings() {
@@ -1230,4 +1345,80 @@ public class Settings {
 	public long getArchivingPause() {
 		return archivingPause;
 	}
+
+	public int getBalanceRecorderPriority() {
+		return balanceRecorderPriority;
+	}
+
+	public int getBalanceRecorderFrequency() {
+		return balanceRecorderFrequency;
+	}
+
+	public int getBalanceRecorderCapacity() {
+		return balanceRecorderCapacity;
+	}
+
+	public boolean isBalanceRecorderEnabled() {
+		return balanceRecorderEnabled;
+	}
+
+	public long getMinimumBalanceRecording() {
+		return minimumBalanceRecording;
+	}
+
+	public long getTopBalanceLoggingLimit() {
+		return topBalanceLoggingLimit;
+	}
+
+	public int getBalanceRecorderRollbackAllowance() {
+		return balanceRecorderRollbackAllowance;
+	}
+
+	public boolean isRewardRecordingOnly() {
+		return rewardRecordingOnly;
+	}
+
+	public boolean isConnectionPoolMonitorEnabled() {
+		return connectionPoolMonitorEnabled;
+	}
+
+	public String getPreferredNetwork () {
+		return this.preferredNetwork.toLowerCase(Locale.getDefault());
+	}
+
+	public int getReticulumMaxPeers() {
+		return this.reticulumMaxPeers;
+	}
+
+	public int getReticulumMinDesiredPeers() {
+		return this.reticulumMinDesiredPeers;
+	}
+
+	public int getReticulumMaxNetworkThreadPoolSize() {
+		return this.reticulumMaxNetworkThreadPoolSize;
+  	}
+
+	public int getBuildArbitraryResourcesBatchSize() {
+		return buildArbitraryResourcesBatchSize;
+	}
+
+	public int getArbitraryIndexingPriority() {
+		return arbitraryIndexingPriority;
+	}
+
+	public int getArbitraryIndexingFrequency() {
+		return arbitraryIndexingFrequency;
+	}
+
+	public boolean isRebuildArbitraryResourceCacheTaskEnabled() {
+		return rebuildArbitraryResourceCacheTaskEnabled;
+	}
+
+	public int getRebuildArbitraryResourceCacheTaskDelay() {
+		return rebuildArbitraryResourceCacheTaskDelay;
+	}
+
+	public int getRebuildArbitraryResourceCacheTaskPeriod() {
+		return rebuildArbitraryResourceCacheTaskPeriod;
+	}
 }

src/main/java/org/qortal/transaction/ArbitraryTransaction.java

@@ -9,6 +9,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.block.BlockChain;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
+import org.qortal.controller.arbitrary.ArbitraryTransactionDataHashWrapper;
 import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
 import org.qortal.crypto.Crypto;
 import org.qortal.crypto.MemoryPoW;
@@ -31,8 +32,12 @@ import org.qortal.utils.ArbitraryTransactionUtils;
 import org.qortal.utils.NTP;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
+import java.util.Set;
 import java.util.stream.Collectors;
 
 public class ArbitraryTransaction extends Transaction {
@@ -303,8 +308,13 @@ public class ArbitraryTransaction extends Transaction {
 			// Add/update arbitrary resource caches, but don't update the status as this involves time-consuming
 			// disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when
 			// accessing the resource.
-			this.updateArbitraryResourceCache(repository);
-			this.updateArbitraryMetadataCache(repository);
+			// Also, must add this transaction as a latest transaction, since the it has not been saved to the
+			// repository yet.
+			this.updateArbitraryResourceCacheIncludingMetadata(
+				repository,
+				Set.of(new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData)),
+				new HashMap<>(0)
+			);
 
 			repository.saveChanges();
 
@@ -360,7 +370,10 @@ public class ArbitraryTransaction extends Transaction {
 	 *
 	 * @throws DataException
 	 */
-	public void updateArbitraryResourceCache(Repository repository) throws DataException {
+	public void updateArbitraryResourceCacheIncludingMetadata(
+			Repository repository,
+			Set<ArbitraryTransactionDataHashWrapper> latestTransactionWrappers,
+			Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper) throws DataException {
 		// Don't cache resources without a name (such as auto updates)
 		if (arbitraryTransactionData.getName() == null) {
 			return;
@@ -385,17 +398,33 @@ public class ArbitraryTransaction extends Transaction {
 		arbitraryResourceData.name = name;
 		arbitraryResourceData.identifier = identifier;
 
-		// Get the latest transaction
-		ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
-		if (latestTransactionData == null) {
-			// We don't have a latest transaction, so delete from cache
-			repository.getArbitraryRepository().delete(arbitraryResourceData);
-			return;
-		}
+		final ArbitraryTransactionDataHashWrapper wrapper = new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData);
 
-		// Get existing cached entry if it exists
-		ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository()
-				.getArbitraryResource(service, name, identifier);
+		ArbitraryTransactionData latestTransactionData;
+		if( latestTransactionWrappers.contains(wrapper)) {
+			latestTransactionData
+				= latestTransactionWrappers.stream()
+					.filter( latestWrapper -> latestWrapper.equals(wrapper))
+					.findAny().get()
+					.getData();
+		}
+		else {
+			// Get the latest transaction
+			latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
+			if (latestTransactionData == null) {
+				LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData);
+				// We don't have a latest transaction, so delete from cache
+				repository.getArbitraryRepository().delete(arbitraryResourceData);
+				return;
+			}
+		}
+		ArbitraryResourceData existingArbitraryResourceData = resourceByWrapper.get(wrapper);
+
+		if( existingArbitraryResourceData == null ) {
+			// Get existing cached entry if it exists
+			existingArbitraryResourceData = repository.getArbitraryRepository()
+					.getArbitraryResource(service, name, identifier);
+		}
 
 		// Check for existing cached data
 		if (existingArbitraryResourceData == null) {
@@ -404,6 +433,7 @@ public class ArbitraryTransaction extends Transaction {
 			arbitraryResourceData.updated = null;
 		}
 		else {
+			resourceByWrapper.put(wrapper, existingArbitraryResourceData);
 			// An entry already exists - update created time from current transaction if this is older
 			arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp());
 
@@ -421,6 +451,34 @@ public class ArbitraryTransaction extends Transaction {
 
 		// Save
 		repository.getArbitraryRepository().save(arbitraryResourceData);
+
+		// Update metadata for latest transaction if it is local
+		if (latestTransactionData.getMetadataHash() != null) {
+			ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
+			if (metadataFile.exists()) {
+				ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
+				try {
+					transactionMetadata.read();
+
+					ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+					metadata.setArbitraryResourceData(arbitraryResourceData);
+					metadata.setTitle(transactionMetadata.getTitle());
+					metadata.setDescription(transactionMetadata.getDescription());
+					metadata.setCategory(transactionMetadata.getCategory());
+					metadata.setTags(transactionMetadata.getTags());
+					repository.getArbitraryRepository().save(metadata);
+
+				} catch (IOException e) {
+					// Ignore, as we can add it again later
+				}
+			} else {
+				// We don't have a local copy of this metadata file, so delete it from the cache
+				// It will be re-added if the file later arrives via the network
+				ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+				metadata.setArbitraryResourceData(arbitraryResourceData);
+				repository.getArbitraryRepository().delete(metadata);
+			}
+		}
 	}
 
 	public void updateArbitraryResourceStatus(Repository repository) throws DataException {
@@ -455,60 +513,4 @@ public class ArbitraryTransaction extends Transaction {
 		repository.getArbitraryRepository().setStatus(arbitraryResourceData, status);
 	}
 
-	public void updateArbitraryMetadataCache(Repository repository) throws DataException {
-		// Get the latest transaction
-		ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
-		if (latestTransactionData == null) {
-			// We don't have a latest transaction, so give up
-			return;
-		}
-
-		Service service = latestTransactionData.getService();
-		String name = latestTransactionData.getName();
-		String identifier = latestTransactionData.getIdentifier();
-
-		if (service == null) {
-			// Unsupported service - ignore this resource
-			return;
-		}
-
-		// In the cache we store null identifiers as "default", as it is part of the primary key
-		if (identifier == null) {
-			identifier = "default";
-		}
-
-		ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
-		arbitraryResourceData.service = service;
-		arbitraryResourceData.name = name;
-		arbitraryResourceData.identifier = identifier;
-
-		// Update metadata for latest transaction if it is local
-		if (latestTransactionData.getMetadataHash() != null) {
-			ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
-			if (metadataFile.exists()) {
-				ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
-				try {
-					transactionMetadata.read();
-
-					ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
-					metadata.setArbitraryResourceData(arbitraryResourceData);
-					metadata.setTitle(transactionMetadata.getTitle());
-					metadata.setDescription(transactionMetadata.getDescription());
-					metadata.setCategory(transactionMetadata.getCategory());
-					metadata.setTags(transactionMetadata.getTags());
-					repository.getArbitraryRepository().save(metadata);
-
-				} catch (IOException e) {
-					// Ignore, as we can add it again later
-				}
-			} else {
-				// We don't have a local copy of this metadata file, so delete it from the cache
-				// It will be re-added if the file later arrives via the network
-				ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
-				metadata.setArbitraryResourceData(arbitraryResourceData);
-				repository.getArbitraryRepository().delete(metadata);
-			}
-		}
-	}
-
 }

src/main/java/org/qortal/transaction/CancelGroupBanTransaction.java

@@ -2,6 +2,7 @@ package org.qortal.transaction;
 
 import org.qortal.account.Account;
 import org.qortal.asset.Asset;
+import org.qortal.block.BlockChain;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.group.GroupData;
 import org.qortal.data.transaction.CancelGroupBanTransactionData;
@@ -12,6 +13,7 @@ import org.qortal.repository.Repository;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 public class CancelGroupBanTransaction extends Transaction {
 
@@ -70,9 +72,26 @@ public class CancelGroupBanTransaction extends Transaction {
 		if (!this.repository.getGroupRepository().adminExists(groupId, admin.getAddress()))
 			return ValidationResult.NOT_GROUP_ADMIN;
 
-		// Can't unban if not group's current owner
-		if (!admin.getAddress().equals(groupData.getOwner()))
-			return ValidationResult.INVALID_GROUP_OWNER;
+		if( this.repository.getBlockRepository().getBlockchainHeight() < BlockChain.getInstance().getNullGroupMembershipHeight() ) {
+			// Can't cancel ban if not group's current owner
+			if (!admin.getAddress().equals(groupData.getOwner()))
+				return ValidationResult.INVALID_GROUP_OWNER;
+		}
+		// if( this.repository.getBlockRepository().getBlockchainHeight() >= BlockChain.getInstance().getNullGroupMembershipHeight() )
+		else {
+			String groupOwner = this.repository.getGroupRepository().getOwner(groupId);
+			boolean groupOwnedByNullAccount = Objects.equals(groupOwner, Group.NULL_OWNER_ADDRESS);
+
+			// if null ownership group, then check for admin approval
+			if(groupOwnedByNullAccount ) {
+				// Require approval if transaction relates to a group owned by the null account
+				if (!this.needsGroupApproval())
+					return ValidationResult.GROUP_APPROVAL_REQUIRED;
+			}
+			// Can't cancel ban if not group's current owner
+			else if (!admin.getAddress().equals(groupData.getOwner()))
+				return ValidationResult.INVALID_GROUP_OWNER;
+		}
 
 		Account member = getMember();
 

src/main/java/org/qortal/transaction/CancelGroupInviteTransaction.java

@@ -2,6 +2,7 @@ package org.qortal.transaction;
 
 import org.qortal.account.Account;
 import org.qortal.asset.Asset;
+import org.qortal.block.BlockChain;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.group.GroupData;
 import org.qortal.data.transaction.CancelGroupInviteTransactionData;
@@ -12,6 +13,7 @@ import org.qortal.repository.Repository;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 public class CancelGroupInviteTransaction extends Transaction {
 
@@ -80,6 +82,16 @@ public class CancelGroupInviteTransaction extends Transaction {
 		if (admin.getConfirmedBalance(Asset.QORT) < this.cancelGroupInviteTransactionData.getFee())
 			return ValidationResult.NO_BALANCE;
 
+		// if null ownership group, then check for admin approval
+		if( this.repository.getBlockRepository().getBlockchainHeight() >= BlockChain.getInstance().getNullGroupMembershipHeight() ) {
+			String groupOwner = this.repository.getGroupRepository().getOwner(groupId);
+			boolean groupOwnedByNullAccount = Objects.equals(groupOwner, Group.NULL_OWNER_ADDRESS);
+
+			// Require approval if transaction relates to a group owned by the null account
+			if (groupOwnedByNullAccount && !this.needsGroupApproval())
+				return ValidationResult.GROUP_APPROVAL_REQUIRED;
+		}
+
 		return ValidationResult.OK;
 	}