From a3753c01bc4c3c73f02630496e5fe18dba1be156 Mon Sep 17 00:00:00 2001 From: proto <34919827+protoniuman@users.noreply.github.com> Date: Tue, 22 Feb 2022 15:50:46 +0100 Subject: [PATCH 1/5] Add search functionality to hosted resources --- .../api/resource/ArbitraryResource.java | 16 ++++- .../ArbitraryDataStorageManager.java | 58 +++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 8031bf83..f67423dd 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -432,14 +432,24 @@ public class ArbitraryResource { @HeaderParam(Security.API_KEY_HEADER) String apiKey, @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, - @Parameter(ref = "offset") @QueryParam("offset") Integer offset) { + @Parameter(ref = "offset") @QueryParam("offset") Integer offset, + @Parameter(ref = "query") @QueryParam("query") String query) { + Security.checkApiCallAllowed(request); List resources = new ArrayList<>(); try (final Repository repository = RepositoryManager.getRepository()) { + + List transactionDataList; + + if(query.equals("")){ + transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); + }else{ + transactionDataList = ArbitraryDataStorageManager.getInstance().searchHostedTransactions(repository,query, limit, offset); + } + - List transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); for (ArbitraryTransactionData transactionData : transactionDataList) { ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); arbitraryResourceInfo.name = transactionData.getName(); @@ -461,6 +471,8 @@ public class ArbitraryResource { } } + + @DELETE @Path("/resource/{service}/{name}/{identifier}") @Operation( diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index e2649cbe..75e37692 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -47,6 +47,9 @@ public class ArbitraryDataStorageManager extends Thread { private List hostedTransactions; + private String searchQuery; + private List searchResultsTransactions; + private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes /** Treat storage as full at 90% usage, to reduce risk of going over the limit. @@ -301,6 +304,61 @@ public class ArbitraryDataStorageManager extends Thread { return ArbitraryTransactionUtils.limitOffsetTransactions(arbitraryTransactionDataList, limit, offset); } + + /** + * searchHostedTransactions + * Allow to run a query against hosted data names and return matches if there are any + * @param repository + * @param query + * @param limit + * @param offset + * @return + */ + + public List searchHostedTransactions(Repository repository, String query, Integer limit, Integer offset) { + // Load from cache if we can (results that exists for the same query), to avoid disk reads + if (this.searchResultsTransactions != null && this.searchQuery.equals(query.toLowerCase())) { + return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset); + } + this.searchQuery=query.toLowerCase();//set the searchQuery so that it can be checked on the next call + + List searchResultsList = new ArrayList<>(); + + // Find all hosted paths + List allPaths = this.findAllHostedPaths(); + + // Loop through each path and attempt to match it to a signature, and check if it's a match with our search query + for (Path path : allPaths) { + try { + String[] contents = path.toFile().list(); + if (contents == null || contents.length == 0) { + // Ignore empty directories + continue; + } + + String signature58 = path.getFileName().toString(); + byte[] signature = Base58.decode(signature58); + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) { + continue; + } + ArbitraryTransactionData arbitraryTransactionData =(ArbitraryTransactionData) transactionData; + if(arbitraryTransactionData.getName().toLowerCase().contains(this.searchQuery) || arbitraryTransactionData.getIdentifier().toLowerCase().contains(this.searchQuery)) + searchResultsList.add(arbitraryTransactionData); + + } catch (DataException e) { + continue; + } + } + + // Sort by newest first + searchResultsList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed()); + + // Update cache + this.searchResultsTransactions = searchResultsList; + + return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset); + } /** * Warning: this method will walk through the entire data directory From 782904a9711268448533e5174051d441b34c07ed Mon Sep 17 00:00:00 2001 From: proto <34919827+protoniuman@users.noreply.github.com> Date: Tue, 22 Feb 2022 17:54:08 +0100 Subject: [PATCH 2/5] improvement to the search on hosted resources 1) use the cached version instead of rescanning all the files 2) separating the loading (which include files scanning) and listing logic --- .../api/resource/ArbitraryResource.java | 2 +- .../ArbitraryDataStorageManager.java | 62 +++++++++---------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index f67423dd..d6eb71ff 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -443,7 +443,7 @@ public class ArbitraryResource { List transactionDataList; - if(query.equals("")){ + if(query==null || query.equals("")){ transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); }else{ transactionDataList = ArbitraryDataStorageManager.getInstance().searchHostedTransactions(repository,query, limit, offset); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 75e37692..fc97fdf8 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -261,14 +261,8 @@ public class ArbitraryDataStorageManager extends Thread { } - // Hosted data - - public List listAllHostedTransactions(Repository repository, Integer limit, Integer offset) { - // Load from cache if we can, to avoid disk reads - if (this.hostedTransactions != null) { - return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); - } - + public List loadAllHostedTransactions(Repository repository){ + List arbitraryTransactionDataList = new ArrayList<>(); // Find all hosted paths @@ -299,10 +293,21 @@ public class ArbitraryDataStorageManager extends Thread { // Sort by newest first arbitraryTransactionDataList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed()); - // Update cache - this.hostedTransactions = arbitraryTransactionDataList; - return ArbitraryTransactionUtils.limitOffsetTransactions(arbitraryTransactionDataList, limit, offset); + return arbitraryTransactionDataList; + } + // Hosted data + + public List listAllHostedTransactions(Repository repository, Integer limit, Integer offset) { + // Load from cache if we can, to avoid disk reads + + if (this.hostedTransactions != null) { + return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); + } + + this.hostedTransactions = this.loadAllHostedTransactions(repository); + + return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset); } /** @@ -316,37 +321,28 @@ public class ArbitraryDataStorageManager extends Thread { */ public List searchHostedTransactions(Repository repository, String query, Integer limit, Integer offset) { - // Load from cache if we can (results that exists for the same query), to avoid disk reads + // Load from results cache if we can (results that exists for the same query), to avoid disk reads if (this.searchResultsTransactions != null && this.searchQuery.equals(query.toLowerCase())) { return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset); } + + // Using cache if we can, to avoid disk reads + if (this.hostedTransactions == null) { + this.hostedTransactions = this.loadAllHostedTransactions(repository); + } + this.searchQuery=query.toLowerCase();//set the searchQuery so that it can be checked on the next call List searchResultsList = new ArrayList<>(); - // Find all hosted paths - List allPaths = this.findAllHostedPaths(); - - // Loop through each path and attempt to match it to a signature, and check if it's a match with our search query - for (Path path : allPaths) { + // Loop through cached hostedTransactions + for (ArbitraryTransactionData atd : this.hostedTransactions) { try { - String[] contents = path.toFile().list(); - if (contents == null || contents.length == 0) { - // Ignore empty directories - continue; - } + + if(atd.getName().toLowerCase().contains(this.searchQuery) || atd.getIdentifier().toLowerCase().contains(this.searchQuery)) + searchResultsList.add(atd); - String signature58 = path.getFileName().toString(); - byte[] signature = Base58.decode(signature58); - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) { - continue; - } - ArbitraryTransactionData arbitraryTransactionData =(ArbitraryTransactionData) transactionData; - if(arbitraryTransactionData.getName().toLowerCase().contains(this.searchQuery) || arbitraryTransactionData.getIdentifier().toLowerCase().contains(this.searchQuery)) - searchResultsList.add(arbitraryTransactionData); - - } catch (DataException e) { + } catch (Exception e) { continue; } } From c65a63fc7ef940e0e5f5e9ca6933f6593d16d4c4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 26 Feb 2022 13:59:53 +0000 Subject: [PATCH 3/5] Fixed "query" parameter error in swagger documentation --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index d6eb71ff..5c07fdd1 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -433,7 +433,7 @@ public class ArbitraryResource { @Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus, @Parameter(ref = "limit") @QueryParam("limit") Integer limit, @Parameter(ref = "offset") @QueryParam("offset") Integer offset, - @Parameter(ref = "query") @QueryParam("query") String query) { + @QueryParam("query") String query) { Security.checkApiCallAllowed(request); From badb57699142c79f7812e70b53aedc993cfaec3a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 26 Feb 2022 14:04:35 +0000 Subject: [PATCH 4/5] Fixed exception when identifier is null. Also handling null names as this may be a future scenario. --- .../arbitrary/ArbitraryDataStorageManager.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index fc97fdf8..63b70691 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -338,9 +338,12 @@ public class ArbitraryDataStorageManager extends Thread { // Loop through cached hostedTransactions for (ArbitraryTransactionData atd : this.hostedTransactions) { try { - - if(atd.getName().toLowerCase().contains(this.searchQuery) || atd.getIdentifier().toLowerCase().contains(this.searchQuery)) - searchResultsList.add(atd); + if (atd.getName() != null && atd.getName().toLowerCase().contains(this.searchQuery)) { + searchResultsList.add(atd); + } + else if (atd.getIdentifier() != null && atd.getIdentifier().toLowerCase().contains(this.searchQuery)) { + searchResultsList.add(atd); + } } catch (Exception e) { continue; From 7740f3da7eade7903dd0179eb81ee2001ad90d21 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 26 Feb 2022 14:05:28 +0000 Subject: [PATCH 5/5] Small formatting tweaks, for consistency with existing code. --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 5 ++--- .../controller/arbitrary/ArbitraryDataStorageManager.java | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 5c07fdd1..ba39825c 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -443,13 +443,12 @@ public class ArbitraryResource { List transactionDataList; - if(query==null || query.equals("")){ + if (query == null || query.equals("")) { transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset); - }else{ + } else { transactionDataList = ArbitraryDataStorageManager.getInstance().searchHostedTransactions(repository,query, limit, offset); } - for (ArbitraryTransactionData transactionData : transactionDataList) { ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo(); arbitraryResourceInfo.name = transactionData.getName(); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 63b70691..4b975b40 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -293,7 +293,6 @@ public class ArbitraryDataStorageManager extends Thread { // Sort by newest first arbitraryTransactionDataList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed()); - return arbitraryTransactionDataList; } // Hosted data @@ -331,7 +330,7 @@ public class ArbitraryDataStorageManager extends Thread { this.hostedTransactions = this.loadAllHostedTransactions(repository); } - this.searchQuery=query.toLowerCase();//set the searchQuery so that it can be checked on the next call + this.searchQuery = query.toLowerCase(); //set the searchQuery so that it can be checked on the next call List searchResultsList = new ArrayList<>();