crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Added pagination to ArbitraryDataManager

Browse Source 
This improves scalability but isn't sufficient for a long term solution. TODO: It probably makes sense to add an additional query for recent transactions only, so that they are fetched quickly.
This commit is contained in:
CalDescent 2021-08-20 07:58:55 +01:00
parent d22a03f1a5
commit d01cdeded8
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -104,6 +104,10 @@ public class ArbitraryDataManager extends Thread {
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager()); arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager());
// Paginate queries when fetching arbitrary transactions
final int limit = 100;
int offset = 0;
try { try {
while (!isStopping) { while (!isStopping) {
Thread.sleep(2000); Thread.sleep(2000);
@ -120,10 +124,13 @@ public class ArbitraryDataManager extends Thread {
// Any arbitrary transactions we want to fetch data for? // Any arbitrary transactions we want to fetch data for?
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, null, null, true); List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, limit, offset, true);
// LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
if (signatures == null || signatures.isEmpty()) { if (signatures == null || signatures.isEmpty()) {
offset = 0;
continue; continue;
} }
offset += limit;
// Filter out those that already have local data // Filter out those that already have local data
signatures.removeIf(signature -> hasLocalData(repository, signature)); signatures.removeIf(signature -> hasLocalData(repository, signature));