From abfeafc823e502591ca0c65cf5be115e34302bc2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 3 Nov 2021 19:56:52 +0000 Subject: [PATCH] Refactored to move all build-related code to a new ArbitraryDataBuildManager class This is will be used to coordinate all build processes and threads. This way it keeps it separate from the ArbitraryDataManager class, which was getting a bit cluttered. --- .../qortal/arbitrary/ArbitraryDataReader.java | 9 +- .../org/qortal/controller/Controller.java | 27 +-- .../arbitrary/ArbitraryDataBuildManager.java | 187 ++++++++++++++++++ .../arbitrary/ArbitraryDataBuilderThread.java | 14 +- .../ArbitraryDataCleanupManager.java | 2 +- .../arbitrary/ArbitraryDataManager.java | 143 +------------- 6 files changed, 207 insertions(+), 175 deletions(-) create mode 100644 src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index a4820e37..7c9b3a9c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.crypto.AES; import org.qortal.data.transaction.ArbitraryTransactionData; @@ -70,7 +71,7 @@ public class ArbitraryDataReader { // cached data, as it may not be fully built ArbitraryDataBuildQueueItem queueItem = new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service); - if (ArbitraryDataManager.getInstance().isInBuildQueue(queueItem)) { + if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { return false; } @@ -97,7 +98,7 @@ public class ArbitraryDataReader { public boolean loadAsynchronously() { ArbitraryDataBuildQueueItem queueItem = new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service); - return ArbitraryDataManager.getInstance().addToBuildQueue(queueItem); + return ArbitraryDataBuildManager.getInstance().addToBuildQueue(queueItem); } /** @@ -134,13 +135,13 @@ public class ArbitraryDataReader { } private void preExecute() { - ArbitraryDataManager.getInstance().setBuildInProgress(true); + ArbitraryDataBuildManager.getInstance().setBuildInProgress(true); this.createWorkingDirectory(); this.createUncompressedDirectory(); } private void postExecute() { - ArbitraryDataManager.getInstance().setBuildInProgress(false); + ArbitraryDataBuildManager.getInstance().setBuildInProgress(false); } private void createWorkingDirectory() { diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 0805b4a3..7c89a405 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -17,7 +17,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Deque; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -50,6 +49,7 @@ import org.qortal.api.DomainMapService; import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; +import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; import org.qortal.controller.arbitrary.ArbitraryDataCleanupManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.controller.Synchronizer.SynchronizationResult; @@ -81,27 +81,6 @@ import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.ValidationResult; import org.qortal.utils.*; -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import java.awt.TrayIcon.MessageType; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.security.SecureRandom; -import java.security.Security; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Function; -import java.util.function.Predicate; -import java.util.stream.Collectors; - import static org.qortal.network.Peer.FETCH_BLOCKS_TIMEOUT; public class Controller extends Thread { @@ -508,6 +487,7 @@ public class Controller extends Thread { // Arbitrary data controllers LOGGER.info("Starting arbitrary-transaction controllers"); ArbitraryDataManager.getInstance().start(); + ArbitraryDataBuildManager.getInstance().start(); ArbitraryDataCleanupManager.getInstance().start(); // Auto-update service? @@ -602,7 +582,7 @@ public class Controller extends Thread { // Clean up arbitrary data request cache ArbitraryDataManager.getInstance().cleanupRequestCache(now); // Clean up arbitrary data queues and lists - ArbitraryDataManager.getInstance().cleanupQueues(now); + ArbitraryDataBuildManager.getInstance().cleanupQueues(now); // Time to 'checkpoint' uncommitted repository writes? if (now >= repositoryCheckpointTimestamp + repositoryCheckpointInterval) { @@ -1084,6 +1064,7 @@ public class Controller extends Thread { // Arbitrary data controllers LOGGER.info("Shutting down arbitrary-transaction controllers"); ArbitraryDataManager.getInstance().shutdown(); + ArbitraryDataBuildManager.getInstance().shutdown(); ArbitraryDataCleanupManager.getInstance().shutdown(); if (blockMinter != null) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java new file mode 100644 index 00000000..20606adc --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -0,0 +1,187 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; +import org.qortal.utils.NTP; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +public class ArbitraryDataBuildManager extends Thread { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class); + + private static ArbitraryDataBuildManager instance; + + private volatile boolean isStopping = false; + private boolean buildInProgress = false; + + /** + * Map to keep track of arbitrary transaction resources currently being built (or queued). + */ + public Map arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>()); + + /** + * Map to keep track of failed arbitrary transaction builds. + */ + public Map arbitraryDataFailedBuilds = Collections.synchronizedMap(new HashMap<>()); + + + public ArbitraryDataBuildManager() { + + } + + @Override + public void run() { + try { + // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) + // This can be expanded to have multiple threads processing the build queue when needed + ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); + arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread()); + + while (!isStopping) { + // Nothing to do yet + Thread.sleep(5000); + } + + } catch (InterruptedException e) { + // Fall-through to exit thread... + } + } + + public static ArbitraryDataBuildManager getInstance() { + if (instance == null) + instance = new ArbitraryDataBuildManager(); + + return instance; + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + } + + + public void cleanupQueues(Long now) { + if (now == null) { + return; + } + arbitraryDataBuildQueue.entrySet().removeIf(entry -> entry.getValue().hasReachedBuildTimeout(now)); + arbitraryDataFailedBuilds.entrySet().removeIf(entry -> entry.getValue().hasReachedFailureTimeout(now)); + } + + // Build queue + + public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + resourceId = resourceId.toLowerCase(); + + if (this.arbitraryDataBuildQueue == null) { + return false; + } + + if (NTP.getTime() == null) { + // Can't use queues until we have synced the time + return false; + } + + // Don't add builds that have failed recently + if (this.isInFailedBuildsList(queueItem)) { + return false; + } + + if (this.arbitraryDataBuildQueue.put(resourceId, queueItem) != null) { + // Already in queue + return true; + } + + LOGGER.info("Added {} to build queue", resourceId); + + // Added to queue + return true; + } + + public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + + if (this.arbitraryDataBuildQueue == null) { + return false; + } + + if (this.arbitraryDataBuildQueue.containsKey(resourceId)) { + // Already in queue + return true; + } + + // Not in queue + return false; + } + + + // Failed builds + + public boolean addToFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds == null) { + return false; + } + + if (NTP.getTime() == null) { + // Can't use queues until we have synced the time + return false; + } + + if (this.arbitraryDataFailedBuilds.put(resourceId, queueItem) != null) { + // Already in list + return true; + } + + LOGGER.info("Added {} to failed builds list", resourceId); + + // Added to queue + return true; + } + + public boolean isInFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + resourceId = resourceId.toLowerCase(); + + if (this.arbitraryDataFailedBuilds == null) { + return false; + } + + if (this.arbitraryDataFailedBuilds.containsKey(resourceId)) { + // Already in list + return true; + } + + // Not in list + return false; + } + + + public void setBuildInProgress(boolean buildInProgress) { + this.buildInProgress = buildInProgress; + } + + public boolean getBuildInProgress() { + return this.buildInProgress; + } +} diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index be4371c9..64866f7f 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -21,21 +21,21 @@ public class ArbitraryDataBuilderThread implements Runnable { public void run() { Thread.currentThread().setName("Arbitrary Data Build Manager"); - ArbitraryDataManager arbitraryDataManager = ArbitraryDataManager.getInstance(); + ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance(); while (!Controller.isStopping()) { try { Thread.sleep(1000); - if (arbitraryDataManager.arbitraryDataBuildQueue == null) { + if (buildManager.arbitraryDataBuildQueue == null) { continue; } - if (arbitraryDataManager.arbitraryDataBuildQueue.isEmpty()) { + if (buildManager.arbitraryDataBuildQueue.isEmpty()) { continue; } // Find resources that are queued for building - Map.Entry next = arbitraryDataManager.arbitraryDataBuildQueue + Map.Entry next = buildManager.arbitraryDataBuildQueue .entrySet().stream() .filter(e -> e.getValue().isQueued()) .findFirst().get(); @@ -57,7 +57,7 @@ public class ArbitraryDataBuilderThread implements Runnable { } // Ignore builds that have failed recently - if (ArbitraryDataManager.getInstance().isInFailedBuildsList(queueItem)) { + if (buildManager.isInFailedBuildsList(queueItem)) { continue; } @@ -73,7 +73,7 @@ public class ArbitraryDataBuilderThread implements Runnable { LOGGER.info("Error building {}: {}", queueItem, e.getMessage()); // Something went wrong - so remove it from the queue, and add to failed builds list queueItem.setFailed(true); - ArbitraryDataManager.getInstance().addToFailedBuildsList(queueItem); + buildManager.addToFailedBuildsList(queueItem); this.removeFromQueue(resourceId); } @@ -87,6 +87,6 @@ public class ArbitraryDataBuilderThread implements Runnable { if (resourceId == null) { return; } - ArbitraryDataManager.getInstance().arbitraryDataBuildQueue.remove(resourceId.toLowerCase()); + ArbitraryDataBuildManager.getInstance().arbitraryDataBuildQueue.remove(resourceId.toLowerCase()); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 8dfe27bb..2cef289d 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -110,7 +110,7 @@ public class ArbitraryDataCleanupManager extends Thread { } // Don't interfere with the filesystem whilst a build is in progress - if (ArbitraryDataManager.getInstance().getBuildInProgress()) { + if (ArbitraryDataBuildManager.getInstance().getBuildInProgress()) { Thread.sleep(5000); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 8c511104..bc729ad2 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -1,13 +1,10 @@ package org.qortal.controller.arbitrary; import java.util.*; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; -import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; import org.qortal.controller.Controller; import org.qortal.data.network.ArbitraryPeerData; import org.qortal.data.transaction.ArbitraryTransactionData; @@ -38,8 +35,6 @@ public class ArbitraryDataManager extends Thread { private static ArbitraryDataManager instance; private final Object peerDataLock = new Object(); - private boolean buildInProgress = false; - private volatile boolean isStopping = false; /** @@ -78,15 +73,6 @@ public class ArbitraryDataManager extends Thread { */ private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes - /** - * Map to keep track of arbitrary transaction resources currently being built (or queued). - */ - public Map arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>()); - - /** - * Map to keep track of failed arbitrary transaction builds. - */ - public Map arbitraryDataFailedBuilds = Collections.synchronizedMap(new HashMap<>()); private ArbitraryDataManager() { @@ -103,11 +89,6 @@ public class ArbitraryDataManager extends Thread { public void run() { Thread.currentThread().setName("Arbitrary Data Manager"); - // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) - // This can be expanded to have multiple threads processing the build queue when needed - ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); - arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread()); - // Keep a reference to the storage manager as we will need this a lot ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance(); @@ -313,14 +294,6 @@ public class ArbitraryDataManager extends Thread { arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() < requestMinimumTimestamp); } - public void cleanupQueues(Long now) { - if (now == null) { - return; - } - arbitraryDataBuildQueue.entrySet().removeIf(entry -> entry.getValue().hasReachedBuildTimeout(now)); - arbitraryDataFailedBuilds.entrySet().removeIf(entry -> entry.getValue().hasReachedFailureTimeout(now)); - } - // Arbitrary data resource cache public boolean isResourceCached(String resourceId) { @@ -373,117 +346,6 @@ public class ArbitraryDataManager extends Thread { this.arbitraryDataCachedResources.put(resourceId, timestamp); } - // Build queue - - public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) { - String resourceId = queueItem.getResourceId(); - if (resourceId == null) { - return false; - } - resourceId = resourceId.toLowerCase(); - - if (this.arbitraryDataBuildQueue == null) { - return false; - } - - if (NTP.getTime() == null) { - // Can't use queues until we have synced the time - return false; - } - - // Don't add builds that have failed recently - if (this.isInFailedBuildsList(queueItem)) { - return false; - } - - if (this.arbitraryDataBuildQueue.put(resourceId, queueItem) != null) { - // Already in queue - return true; - } - - LOGGER.info("Added {} to build queue", resourceId); - - // Added to queue - return true; - } - - public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) { - String resourceId = queueItem.getResourceId(); - if (resourceId == null) { - return false; - } - - if (this.arbitraryDataBuildQueue == null) { - return false; - } - - if (this.arbitraryDataBuildQueue.containsKey(resourceId)) { - // Already in queue - return true; - } - - // Not in queue - return false; - } - - - // Failed builds - - public boolean addToFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) { - String resourceId = queueItem.getResourceId(); - if (resourceId == null) { - return false; - } - - if (this.arbitraryDataFailedBuilds == null) { - return false; - } - - if (NTP.getTime() == null) { - // Can't use queues until we have synced the time - return false; - } - - if (this.arbitraryDataFailedBuilds.put(resourceId, queueItem) != null) { - // Already in list - return true; - } - - LOGGER.info("Added {} to failed builds list", resourceId); - - // Added to queue - return true; - } - - public boolean isInFailedBuildsList(ArbitraryDataBuildQueueItem queueItem) { - String resourceId = queueItem.getResourceId(); - if (resourceId == null) { - return false; - } - resourceId = resourceId.toLowerCase(); - - if (this.arbitraryDataFailedBuilds == null) { - return false; - } - - if (this.arbitraryDataFailedBuilds.containsKey(resourceId)) { - // Already in list - return true; - } - - // Not in list - return false; - } - - - public void setBuildInProgress(boolean buildInProgress) { - this.buildInProgress = buildInProgress; - } - - public boolean getBuildInProgress() { - return this.buildInProgress; - } - // Network handlers @@ -611,8 +473,9 @@ public class ArbitraryDataManager extends Thread { } // Also remove from the failed builds queue in case it previously failed due to missing chunks - if (this.arbitraryDataFailedBuilds.containsKey(resourceId)) { - this.arbitraryDataFailedBuilds.remove(resourceId); + ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance(); + if (buildManager.arbitraryDataFailedBuilds.containsKey(resourceId)) { + buildManager.arbitraryDataFailedBuilds.remove(resourceId); } }