Refactored to move all build-related code to a new ArbitraryDataBuildManager class

This is will be used to coordinate all build processes and threads. This way it keeps it separate from the ArbitraryDataManager class, which was getting a bit cluttered.

src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java

@@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.crypto.AES;
 import org.qortal.data.transaction.ArbitraryTransactionData;
@@ -70,7 +71,7 @@ public class ArbitraryDataReader {
         // cached data, as it may not be fully built
         ArbitraryDataBuildQueueItem queueItem =
                 new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service);
-        if (ArbitraryDataManager.getInstance().isInBuildQueue(queueItem)) {
+        if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
             return false;
         }
 
@@ -97,7 +98,7 @@ public class ArbitraryDataReader {
     public boolean loadAsynchronously() {
         ArbitraryDataBuildQueueItem queueItem =
                 new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service);
-        return ArbitraryDataManager.getInstance().addToBuildQueue(queueItem);
+        return ArbitraryDataBuildManager.getInstance().addToBuildQueue(queueItem);
     }
 
     /**
@@ -134,13 +135,13 @@ public class ArbitraryDataReader {
     }
 
     private void preExecute() {
-        ArbitraryDataManager.getInstance().setBuildInProgress(true);
+        ArbitraryDataBuildManager.getInstance().setBuildInProgress(true);
         this.createWorkingDirectory();
         this.createUncompressedDirectory();
     }
 
     private void postExecute() {
-        ArbitraryDataManager.getInstance().setBuildInProgress(false);
+        ArbitraryDataBuildManager.getInstance().setBuildInProgress(false);
     }
 
     private void createWorkingDirectory() {

src/main/java/org/qortal/controller/Controller.java

@@ -17,7 +17,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Deque;
-import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedList;
@@ -50,6 +49,7 @@ import org.qortal.api.DomainMapService;
 import org.qortal.block.Block;
 import org.qortal.block.BlockChain;
 import org.qortal.block.BlockChain.BlockTimingByHeight;
+import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
 import org.qortal.controller.arbitrary.ArbitraryDataCleanupManager;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.controller.Synchronizer.SynchronizationResult;
@@ -81,27 +81,6 @@ import org.qortal.transaction.Transaction.TransactionType;
 import org.qortal.transaction.Transaction.ValidationResult;
 import org.qortal.utils.*;
 
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import java.awt.TrayIcon.MessageType;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.SecureRandom;
-import java.security.Security;
-import java.time.LocalDateTime;
-import java.time.ZoneOffset;
-import java.time.format.DateTimeFormatter;
-import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.Function;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
 import static org.qortal.network.Peer.FETCH_BLOCKS_TIMEOUT;
 
 public class Controller extends Thread {
@@ -508,6 +487,7 @@ public class Controller extends Thread {
 		// Arbitrary data controllers
 		LOGGER.info("Starting arbitrary-transaction controllers");
 		ArbitraryDataManager.getInstance().start();
+		ArbitraryDataBuildManager.getInstance().start();
 		ArbitraryDataCleanupManager.getInstance().start();
 
 		// Auto-update service?
@@ -602,7 +582,7 @@ public class Controller extends Thread {
 				// Clean up arbitrary data request cache
 				ArbitraryDataManager.getInstance().cleanupRequestCache(now);
 				// Clean up arbitrary data queues and lists
-				ArbitraryDataManager.getInstance().cleanupQueues(now);
+				ArbitraryDataBuildManager.getInstance().cleanupQueues(now);
 
 				// Time to 'checkpoint' uncommitted repository writes?
 				if (now >= repositoryCheckpointTimestamp + repositoryCheckpointInterval) {
@@ -1084,6 +1064,7 @@ public class Controller extends Thread {
 				// Arbitrary data controllers
 				LOGGER.info("Shutting down arbitrary-transaction controllers");
 				ArbitraryDataManager.getInstance().shutdown();
+				ArbitraryDataBuildManager.getInstance().shutdown();
 				ArbitraryDataCleanupManager.getInstance().shutdown();
 
 				if (blockMinter != null) {

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java

@@ -0,0 +1,187 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
+import org.qortal.utils.NTP;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+public class ArbitraryDataBuildManager extends Thread {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class);
+
+    private static ArbitraryDataBuildManager instance;
+
+    private volatile boolean isStopping = false;
+    private boolean buildInProgress = false;
+
+    /**
+     * Map to keep track of arbitrary transaction resources currently being built (or queued).
+     */
+    public Map<String, ArbitraryDataBuildQueueItem> arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of failed arbitrary transaction builds.
+     */
+    public Map<String, ArbitraryDataBuildQueueItem> arbitraryDataFailedBuilds = Collections.synchronizedMap(new HashMap<>());
+
+
+    public ArbitraryDataBuildManager() {
+
+    }
+
+    @Override
+    public void run() {
+        try {
+            // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
+            // This can be expanded to have multiple threads processing the build queue when needed
+            ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
+            arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
+
+            while (!isStopping) {
+                // Nothing to do yet
+                Thread.sleep(5000);
+            }
+
+        } catch (InterruptedException e) {
+            // Fall-through to exit thread...
+        }
+    }
+
+    public static ArbitraryDataBuildManager getInstance() {
+        if (instance == null)
+            instance = new ArbitraryDataBuildManager();
+
+        return instance;
+    }
+
+    public void shutdown() {
+        isStopping = true;
+        this.interrupt();
+    }
+
+
+    public void cleanupQueues(Long now) {
+        if (now == null) {
+            return;
+        }
+        arbitraryDataBuildQueue.entrySet().removeIf(entry -> entry.getValue().hasReachedBuildTimeout(now));
+        arbitraryDataFailedBuilds.entrySet().removeIf(entry -> entry.getValue().hasReachedFailureTimeout(now));
+    }
+
+    // Build queue
+
+    public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
+        String resourceId = queueItem.getResourceId();
+        if (resourceId == null) {
+            return false;
+        }
+        resourceId = resourceId.toLowerCase();
+
+        if (this.arbitraryDataBuildQueue == null) {
+            return false;
+        }
+
+        if (NTP.getTime() == null) {
+            // Can't use queues until we have synced the time
+            return false;
+        }
+
+        // Don't add builds that have failed recently
+        if (this.isInFailedBuildsList(queueItem)) {
+            return false;
+        }
+
+        if (this.arbitraryDataBuildQueue.put(resourceId, queueItem) != null) {
+            // Already in queue
+            return true;
+        }
+
+        LOGGER.info("Added {} to build queue", resourceId);
+
+        // Added to queue
+        return true;
+    }
+
+    public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
+        String resourceId = queueItem.getResourceId();
+        if (resourceId == null) {
+            return false;
+        }
+
+        if (this.arbitraryDataBuildQueue == null) {
+            return false;
+        }
+
+        if (this.arbitraryDataBuildQueue.containsKey(resourceId)) {
+            // Already in queue
+            return true;
+        }
+
+        // Not in queue
+        return false;
+    }
+
+
+    // Failed builds
+
+    public boolean addToFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) {
+        String resourceId = queueItem.getResourceId();
+        if (resourceId == null) {
+            return false;
+        }
+
+        if (this.arbitraryDataFailedBuilds == null) {
+            return false;
+        }
+
+        if (NTP.getTime() == null) {
+            // Can't use queues until we have synced the time
+            return false;
+        }
+
+        if (this.arbitraryDataFailedBuilds.put(resourceId, queueItem) != null) {
+            // Already in list
+            return true;
+        }
+
+        LOGGER.info("Added {} to failed builds list", resourceId);
+
+        // Added to queue
+        return true;
+    }
+
+    public boolean isInFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) {
+        String resourceId = queueItem.getResourceId();
+        if (resourceId == null) {
+            return false;
+        }
+        resourceId = resourceId.toLowerCase();
+
+        if (this.arbitraryDataFailedBuilds == null) {
+            return false;
+        }
+
+        if (this.arbitraryDataFailedBuilds.containsKey(resourceId)) {
+            // Already in list
+            return true;
+        }
+
+        // Not in list
+        return false;
+    }
+
+
+    public void setBuildInProgress(boolean buildInProgress) {
+        this.buildInProgress = buildInProgress;
+    }
+
+    public boolean getBuildInProgress() {
+        return this.buildInProgress;
+    }
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

@@ -21,21 +21,21 @@ public class ArbitraryDataBuilderThread implements Runnable {
 
     public void run() {
         Thread.currentThread().setName("Arbitrary Data Build Manager");
-        ArbitraryDataManager arbitraryDataManager = ArbitraryDataManager.getInstance();
+        ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance();
 
         while (!Controller.isStopping()) {
             try {
                 Thread.sleep(1000);
 
-                if (arbitraryDataManager.arbitraryDataBuildQueue == null) {
+                if (buildManager.arbitraryDataBuildQueue == null) {
                     continue;
                 }
-                if (arbitraryDataManager.arbitraryDataBuildQueue.isEmpty()) {
+                if (buildManager.arbitraryDataBuildQueue.isEmpty()) {
                     continue;
                 }
 
                 // Find resources that are queued for building
-                Map.Entry<String, ArbitraryDataBuildQueueItem> next = arbitraryDataManager.arbitraryDataBuildQueue
+                Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
                         .entrySet().stream()
                         .filter(e -> e.getValue().isQueued())
                         .findFirst().get();
@@ -57,7 +57,7 @@ public class ArbitraryDataBuilderThread implements Runnable {
                 }
 
                 // Ignore builds that have failed recently
-                if (ArbitraryDataManager.getInstance().isInFailedBuildsList(queueItem)) {
+                if (buildManager.isInFailedBuildsList(queueItem)) {
                     continue;
                 }
 
@@ -73,7 +73,7 @@ public class ArbitraryDataBuilderThread implements Runnable {
                     LOGGER.info("Error building {}: {}", queueItem, e.getMessage());
                     // Something went wrong - so remove it from the queue, and add to failed builds list
                     queueItem.setFailed(true);
-                    ArbitraryDataManager.getInstance().addToFailedBuildsList(queueItem);
+                    buildManager.addToFailedBuildsList(queueItem);
                     this.removeFromQueue(resourceId);
                 }
 
@@ -87,6 +87,6 @@ public class ArbitraryDataBuilderThread implements Runnable {
         if (resourceId == null) {
             return;
         }
-        ArbitraryDataManager.getInstance().arbitraryDataBuildQueue.remove(resourceId.toLowerCase());
+        ArbitraryDataBuildManager.getInstance().arbitraryDataBuildQueue.remove(resourceId.toLowerCase());
     }
 }

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java

@@ -110,7 +110,7 @@ public class ArbitraryDataCleanupManager extends Thread {
 						}
 
 						// Don't interfere with the filesystem whilst a build is in progress
-						if (ArbitraryDataManager.getInstance().getBuildInProgress()) {
+						if (ArbitraryDataBuildManager.getInstance().getBuildInProgress()) {
 							Thread.sleep(5000);
 						}
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -1,13 +1,10 @@
 package org.qortal.controller.arbitrary;
 
 import java.util.*;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
-import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
 import org.qortal.controller.Controller;
 import org.qortal.data.network.ArbitraryPeerData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
@@ -38,8 +35,6 @@ public class ArbitraryDataManager extends Thread {
 	private static ArbitraryDataManager instance;
 	private final Object peerDataLock = new Object();
 
-	private boolean buildInProgress = false;
-
 	private volatile boolean isStopping = false;
 
 	/**
@@ -78,15 +73,6 @@ public class ArbitraryDataManager extends Thread {
 	 */
 	private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes
 
-	/**
-	 * Map to keep track of arbitrary transaction resources currently being built (or queued).
-	 */
-	public Map<String, ArbitraryDataBuildQueueItem> arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>());
-
-	/**
-	 * Map to keep track of failed arbitrary transaction builds.
-	 */
-	public Map<String, ArbitraryDataBuildQueueItem> arbitraryDataFailedBuilds = Collections.synchronizedMap(new HashMap<>());
 
 
 	private ArbitraryDataManager() {
@@ -103,11 +89,6 @@ public class ArbitraryDataManager extends Thread {
 	public void run() {
 		Thread.currentThread().setName("Arbitrary Data Manager");
 
-		// Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
-		// This can be expanded to have multiple threads processing the build queue when needed
-		ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
-		arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
-
 		// Keep a reference to the storage manager as we will need this a lot
 		ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
 
@@ -313,14 +294,6 @@ public class ArbitraryDataManager extends Thread {
 		arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() < requestMinimumTimestamp);
 	}
 
-	public void cleanupQueues(Long now) {
-		if (now == null) {
-			return;
-		}
-		arbitraryDataBuildQueue.entrySet().removeIf(entry -> entry.getValue().hasReachedBuildTimeout(now));
-		arbitraryDataFailedBuilds.entrySet().removeIf(entry -> entry.getValue().hasReachedFailureTimeout(now));
-	}
-
 
 	// Arbitrary data resource cache
 	public boolean isResourceCached(String resourceId) {
@@ -373,117 +346,6 @@ public class ArbitraryDataManager extends Thread {
 		this.arbitraryDataCachedResources.put(resourceId, timestamp);
 	}
 
-	// Build queue
-
-	public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
-		String resourceId = queueItem.getResourceId();
-		if (resourceId == null) {
-			return false;
-		}
-		resourceId = resourceId.toLowerCase();
-
-		if (this.arbitraryDataBuildQueue == null) {
-			return false;
-		}
-
-		if (NTP.getTime() == null) {
-			// Can't use queues until we have synced the time
-			return false;
-		}
-
-		// Don't add builds that have failed recently
-		if (this.isInFailedBuildsList(queueItem)) {
-			return false;
-		}
-
-		if (this.arbitraryDataBuildQueue.put(resourceId, queueItem) != null) {
-			// Already in queue
-			return true;
-		}
-
-		LOGGER.info("Added {} to build queue", resourceId);
-
-		// Added to queue
-		return true;
-	}
-
-	public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
-		String resourceId = queueItem.getResourceId();
-		if (resourceId == null) {
-			return false;
-		}
-
-		if (this.arbitraryDataBuildQueue == null) {
-			return false;
-		}
-
-		if (this.arbitraryDataBuildQueue.containsKey(resourceId)) {
-			// Already in queue
-			return true;
-		}
-
-		// Not in queue
-		return false;
-	}
-
-
-	// Failed builds
-
-	public boolean addToFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) {
-		String resourceId = queueItem.getResourceId();
-		if (resourceId == null) {
-			return false;
-		}
-
-		if (this.arbitraryDataFailedBuilds == null) {
-			return false;
-		}
-
-		if (NTP.getTime() == null) {
-			// Can't use queues until we have synced the time
-			return false;
-		}
-
-		if (this.arbitraryDataFailedBuilds.put(resourceId, queueItem) != null) {
-			// Already in list
-			return true;
-		}
-
-		LOGGER.info("Added {} to failed builds list", resourceId);
-
-		// Added to queue
-		return true;
-	}
-
-	public boolean isInFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) {
-		String resourceId = queueItem.getResourceId();
-		if (resourceId == null) {
-			return false;
-		}
-		resourceId = resourceId.toLowerCase();
-
-		if (this.arbitraryDataFailedBuilds == null) {
-			return false;
-		}
-
-		if (this.arbitraryDataFailedBuilds.containsKey(resourceId)) {
-			// Already in list
-			return true;
-		}
-
-		// Not in list
-		return false;
-	}
-
-
-	public void setBuildInProgress(boolean buildInProgress) {
-		this.buildInProgress = buildInProgress;
-	}
-
-	public boolean getBuildInProgress() {
-		return this.buildInProgress;
-	}
-
 
 	// Network handlers
 
@@ -611,8 +473,9 @@ public class ArbitraryDataManager extends Thread {
 					}
 
 					// Also remove from the failed builds queue in case it previously failed due to missing chunks
-					if (this.arbitraryDataFailedBuilds.containsKey(resourceId)) {
+					ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance();
-						this.arbitraryDataFailedBuilds.remove(resourceId);
+					if (buildManager.arbitraryDataFailedBuilds.containsKey(resourceId)) {
+						buildManager.arbitraryDataFailedBuilds.remove(resourceId);
 					}
 				}