Rework of arbitraryDataFileHashResponses to use a list rather than a map (limited to 1000) items. Sort the list by routes with the lowest number of peer hops first, to try and prioritize those which are easiest and quickest to reach.

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java

@@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.ArbitraryDataFile;
 import org.qortal.arbitrary.ArbitraryDataFileChunk;
 import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo;
 import org.qortal.data.arbitrary.ArbitraryRelayInfo;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -23,6 +24,8 @@ import org.qortal.utils.Triple;
 
 import java.util.*;
 
+import static org.qortal.controller.arbitrary.ArbitraryDataFileManager.MAX_FILE_HASH_RESPONSES;
+
 public class ArbitraryDataFileListManager {
 
     private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileListManager.class);
@@ -458,12 +461,20 @@ public class ArbitraryDataFileListManager {
 //			}
 
             if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) {
-                // Keep track of the hashes this peer reports to have access to
+                if (ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.size() < MAX_FILE_HASH_RESPONSES) {
-                Long now = NTP.getTime();
+                    // Keep track of the hashes this peer reports to have access to
-                for (byte[] hash : hashes) {
+                    Long now = NTP.getTime();
-                    String hash58 = Base58.encode(hash);
+                    for (byte[] hash : hashes) {
-                    String sig58 = Base58.encode(signature);
+                        String hash58 = Base58.encode(hash);
-                    ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.put(hash58, new Triple<>(peer, sig58, now));
+
+                        // Treat null request hops as 100, so that they are able to be sorted (and put to the end of the list)
+                        int requestHops = arbitraryDataFileListMessage.getRequestHops() != null ? arbitraryDataFileListMessage.getRequestHops() : 100;
+
+                        ArbitraryFileListResponseInfo responseInfo = new ArbitraryFileListResponseInfo(hash58, signature58,
+                                peer, now, arbitraryDataFileListMessage.getRequestTime(), requestHops);
+
+                        ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.add(responseInfo);
+                    }
                 }
 
                 // Go and fetch the actual data, since this isn't a relay request

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java

@@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.ArbitraryDataFile;
 import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo;
 import org.qortal.data.arbitrary.ArbitraryRelayInfo;
 import org.qortal.data.network.ArbitraryPeerData;
 import org.qortal.data.network.PeerData;
@@ -18,7 +19,6 @@ import org.qortal.settings.Settings;
 import org.qortal.utils.ArbitraryTransactionUtils;
 import org.qortal.utils.Base58;
 import org.qortal.utils.NTP;
-import org.qortal.utils.Triple;
 
 import java.security.SecureRandom;
 import java.util.*;
@@ -45,11 +45,11 @@ public class ArbitraryDataFileManager extends Thread {
     public List<ArbitraryRelayInfo> arbitraryRelayMap = Collections.synchronizedList(new ArrayList<>());
 
     /**
-     * Map to keep track of any arbitrary data file hash responses
+     * List to keep track of any arbitrary data file hash responses
-     * Key: string - the hash encoded in base58
-     * Value: Triple<respondingPeer, signature58, timeResponded>
      */
-    public Map<String, Triple<Peer, String, Long>> arbitraryDataFileHashResponses = Collections.synchronizedMap(new HashMap<>());
+    public List<ArbitraryFileListResponseInfo> arbitraryDataFileHashResponses = Collections.synchronizedList(new ArrayList<>());
+
+    public static int MAX_FILE_HASH_RESPONSES = 1000;
 
 
     private ArbitraryDataFileManager() {
@@ -98,7 +98,7 @@ public class ArbitraryDataFileManager extends Thread {
 
         final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT;
         arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp);
-        arbitraryDataFileHashResponses.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp);
+        arbitraryDataFileHashResponses.removeIf(entry -> entry.getTimestamp() < relayMinimumTimestamp);
     }
 
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java

@@ -3,6 +3,7 @@ package org.qortal.controller.arbitrary;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.network.Peer;
 import org.qortal.repository.DataException;
@@ -11,11 +12,9 @@ import org.qortal.repository.RepositoryManager;
 import org.qortal.utils.ArbitraryTransactionUtils;
 import org.qortal.utils.Base58;
 import org.qortal.utils.NTP;
-import org.qortal.utils.Triple;
 
-import java.util.Arrays;
+import java.util.*;
-import java.util.Iterator;
+import java.util.stream.Collectors;
-import java.util.Map;
 
 public class ArbitraryDataFileRequestThread implements Runnable {
 
@@ -51,45 +50,49 @@ public class ArbitraryDataFileRequestThread implements Runnable {
         boolean shouldProcess = false;
 
         synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) {
-            Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator();
+            if (!arbitraryDataFileManager.arbitraryDataFileHashResponses.isEmpty()) {
-            while (iterator.hasNext()) {
-                if (Controller.isStopping()) {
-                    return;
-                }
 
-                Map.Entry entry = (Map.Entry) iterator.next();
+                // Sort by lowest number of node hops first
-                if (entry == null || entry.getKey() == null || entry.getValue() == null) {
+                Comparator<ArbitraryFileListResponseInfo> lowestHopsFirstComparator =
+                        Comparator.comparingInt(ArbitraryFileListResponseInfo::getRequestHops);
+                arbitraryDataFileManager.arbitraryDataFileHashResponses = arbitraryDataFileManager.arbitraryDataFileHashResponses
+                        .stream().sorted(lowestHopsFirstComparator)
+                        .collect(Collectors.toCollection(() -> Collections.synchronizedList(new ArrayList<>())));
+
+                Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.iterator();
+                while (iterator.hasNext()) {
+                    if (Controller.isStopping()) {
+                        return;
+                    }
+
+                    ArbitraryFileListResponseInfo responseInfo = (ArbitraryFileListResponseInfo) iterator.next();
+                    if (responseInfo == null) {
+                        iterator.remove();
+                        continue;
+                    }
+
+                    hash58 = responseInfo.getHash58();
+                    peer = responseInfo.getPeer();
+                    signature58 = responseInfo.getSignature58();
+                    Long timestamp = responseInfo.getTimestamp();
+
+                    if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) {
+                        // Ignore - to be deleted
+                        iterator.remove();
+                        continue;
+                    }
+
+                    // Skip if already requesting, but don't remove, as we might want to retry later
+                    if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) {
+                        // Already requesting - leave this attempt for later
+                        continue;
+                    }
+
+                    // We want to process this file
+                    shouldProcess = true;
                     iterator.remove();
-                    continue;
+                    break;
                 }
-
-                hash58 = (String) entry.getKey();
-                Triple<Peer, String, Long> value = (Triple<Peer, String, Long>) entry.getValue();
-                if (value == null) {
-                    iterator.remove();
-                    continue;
-                }
-
-                peer = value.getA();
-                signature58 = value.getB();
-                Long timestamp = value.getC();
-
-                if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) {
-                    // Ignore - to be deleted
-                    iterator.remove();
-                    continue;
-                }
-
-                // Skip if already requesting, but don't remove, as we might want to retry later
-                if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) {
-                    // Already requesting - leave this attempt for later
-                    continue;
-                }
-
-                // We want to process this file
-                shouldProcess = true;
-                iterator.remove();
-                break;
             }
         }
 

src/main/java/org/qortal/data/arbitrary/ArbitraryFileListResponseInfo.java

@@ -0,0 +1,11 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.network.Peer;
+
+public class ArbitraryFileListResponseInfo extends  ArbitraryRelayInfo {
+
+    public ArbitraryFileListResponseInfo(String hash58, String signature58, Peer peer, Long timestamp, Long requestTime, Integer requestHops) {
+        super(hash58, signature58, peer, timestamp, requestTime, requestHops);
+    }
+
+}