crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Use 5 builder threads, so that one slow resource (e.g. a thumbnail) doesn't hold up the other queued build items.

Browse Source 
This can be replaced with a task-based approach longer term.
This commit is contained in:
CalDescent 2022-02-11 13:58:45 +00:00
parent c0c50f2e18
commit fc82f0b622
3 changed files with 40 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
View File

@ -27,13 +27,20 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
this.creationTimestamp = NTP.getTime();
}
public void prepareForBuild() {
this.buildStartTimestamp = NTP.getTime();
}
public void build() throws IOException, DataException, MissingDataException {
Long now = NTP.getTime();
if (now == null) {
this.buildStartTimestamp = null;
throw new DataException("NTP time hasn't synced yet");
}
this.buildStartTimestamp = now;
if (this.buildStartTimestamp == null) {
this.buildStartTimestamp = now;
}
ArbitraryDataReader arbitraryDataReader =
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);

7
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java
View File

@ -42,8 +42,11 @@ public class ArbitraryDataBuildManager extends Thread {
try {
// Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
// This can be expanded to have multiple threads processing the build queue when needed
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
int threadCount = 5;
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(threadCount);
for (int i = 0; i < threadCount; i++) {
arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
}
while (!isStopping) {
// Nothing to do yet

50
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java
View File

@ -36,37 +36,41 @@ public class ArbitraryDataBuilderThread implements Runnable {
continue;
}
Map.Entry<String, ArbitraryDataBuildQueueItem> next = null;
// Find resources that are queued for building
synchronized (buildManager.arbitraryDataBuildQueue) {
next = buildManager.arbitraryDataBuildQueue
.entrySet().stream()
.filter(e -> e.getValue().isQueued())
.findFirst().orElse(null);
}
if (next == null) {
continue;
}
Long now = NTP.getTime();
if (now == null) {
continue;
}
ArbitraryDataBuildQueueItem queueItem = next.getValue();
ArbitraryDataBuildQueueItem queueItem = null;
if (queueItem == null) {
this.removeFromQueue(queueItem);
// Find resources that are queued for building
synchronized (buildManager.arbitraryDataBuildQueue) {
Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
.entrySet().stream()
.filter(e -> e.getValue().isQueued())
.findFirst().orElse(null);
if (next == null) {
continue;
}
queueItem = next.getValue();
if (queueItem == null) {
this.removeFromQueue(queueItem);
continue;
}
// Ignore builds that have failed recently
if (buildManager.isInFailedBuildsList(queueItem)) {
this.removeFromQueue(queueItem);
continue;
}
// Set the start timestamp, to prevent other threads from building it at the same time
queueItem.prepareForBuild();
}
// Ignore builds that have failed recently
if (buildManager.isInFailedBuildsList(queueItem)) {
continue;
}
try {
// Perform the build
LOGGER.info("Building {}...", queueItem);