From 04642452185c7df23fc0d3258f64d81f555eb747 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 10 Dec 2021 11:33:26 +0000 Subject: [PATCH] Simplified lists feature to be referenced by a single name, instead of the confusing "category" and "resourceName" properties. This makes them extremely generic, improves filenames, and makes it easier to create custom lists. It doesn't have backwards support, but the lists feature isn't working properly in core 2.1+ anyway. --- .../qortal/api/resource/ListsResource.java | 80 +++++-------------- .../arbitrary/ArbitraryDataResource.java | 2 +- .../arbitrary/ArbitraryDataManager.java | 2 +- .../ArbitraryDataStorageManager.java | 8 +- .../java/org/qortal/list/ResourceList.java | 34 +++----- .../org/qortal/list/ResourceListManager.java | 41 +++++----- .../qortal/transaction/ChatTransaction.java | 4 +- .../ArbitraryDataStorageCapacityTests.java | 12 +-- .../ArbitraryDataStoragePolicyTests.java | 20 ++--- 9 files changed, 78 insertions(+), 125 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ListsResource.java b/src/main/java/org/qortal/api/resource/ListsResource.java index ad7bd54f..485dbb84 100644 --- a/src/main/java/org/qortal/api/resource/ListsResource.java +++ b/src/main/java/org/qortal/api/resource/ListsResource.java @@ -33,10 +33,9 @@ public class ListsResource { @POST - @Path("/{category}/{resourceName}") + @Path("/{listName}") @Operation( summary = "Add items to a new or existing list", - description = "Example categories are 'blacklist' or 'followed'. Example resource names are 'addresses' or 'names'", requestBody = @RequestBody( required = true, content = @Content( @@ -57,12 +56,11 @@ public class ListsResource { ) @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) @SecurityRequirement(name = "apiKey") - public String addItemstoList(@PathParam("category") String category, - @PathParam("resourceName") String resourceName, + public String addItemstoList(@PathParam("listName") String listName, ListRequest listRequest) { Security.checkApiCallAllowed(request); - if (category == null || resourceName == null) { + if (listName == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); } @@ -73,64 +71,33 @@ public class ListsResource { int successCount = 0; int errorCount = 0; - try (final Repository repository = RepositoryManager.getRepository()) { + for (String item : listRequest.items) { - for (String item : listRequest.items) { - - // Validate addresses - if (resourceName.equals("address") || resourceName.equals("addresses")) { - - if (!Crypto.isValidAddress(item)) { - errorCount++; - continue; - } - - AccountData accountData = repository.getAccountRepository().getAccount(item); - // Not found? - if (accountData == null) { - errorCount++; - continue; - } - } - - // Validate names - if (resourceName.equals("name") || resourceName.equals("names")) { - if (!repository.getNameRepository().nameExists(item)) { - errorCount++; - continue; - } - } - - // Valid address, so go ahead and blacklist it - boolean success = ResourceListManager.getInstance().addToList(category, resourceName, item, false); - if (success) { - successCount++; - } - else { - errorCount++; - } + boolean success = ResourceListManager.getInstance().addToList(listName, item, false); + if (success) { + successCount++; + } + else { + errorCount++; } - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } if (successCount > 0 && errorCount == 0) { - // All were successful, so save the blacklist - ResourceListManager.getInstance().saveList(category, resourceName); + // All were successful, so save the list + ResourceListManager.getInstance().saveList(listName); return "true"; } else { // Something went wrong, so revert - ResourceListManager.getInstance().revertList(category, resourceName); + ResourceListManager.getInstance().revertList(listName); return "false"; } } @DELETE - @Path("/{category}/{resourceName}") + @Path("/{listName}") @Operation( summary = "Remove one or more items from a list", - description = "Example categories are 'blacklist' or 'followed'. Example resource names are 'addresses' or 'names'", requestBody = @RequestBody( required = true, content = @Content( @@ -151,8 +118,7 @@ public class ListsResource { ) @ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE}) @SecurityRequirement(name = "apiKey") - public String removeItemsFromList(@PathParam("category") String category, - @PathParam("resourceName") String resourceName, + public String removeItemsFromList(@PathParam("listName") String listName, ListRequest listRequest) { Security.checkApiCallAllowed(request); @@ -167,7 +133,7 @@ public class ListsResource { // Attempt to remove the item // Don't save as we will do this at the end of the process - boolean success = ResourceListManager.getInstance().removeFromList(category, resourceName, address, false); + boolean success = ResourceListManager.getInstance().removeFromList(listName, address, false); if (success) { successCount++; } @@ -177,22 +143,21 @@ public class ListsResource { } if (successCount > 0 && errorCount == 0) { - // All were successful, so save the blacklist - ResourceListManager.getInstance().saveList(category, resourceName); + // All were successful, so save the list + ResourceListManager.getInstance().saveList(listName); return "true"; } else { // Something went wrong, so revert - ResourceListManager.getInstance().revertList(category, resourceName); + ResourceListManager.getInstance().revertList(listName); return "false"; } } @GET - @Path("/{category}/{resourceName}") + @Path("/{listName}") @Operation( summary = "Fetch all items in a list", - description = "Example categories are 'blacklist' or 'followed'. Example resource names are 'addresses' or 'names'", responses = { @ApiResponse( description = "A JSON array of items", @@ -201,10 +166,9 @@ public class ListsResource { } ) @SecurityRequirement(name = "apiKey") - public String getItemsInList(@PathParam("category") String category, - @PathParam("resourceName") String resourceName) { + public String getItemsInList(@PathParam("listName") String listName) { Security.checkApiCallAllowed(request); - return ResourceListManager.getInstance().getJSONStringForList(category, resourceName); + return ResourceListManager.getInstance().getJSONStringForList(listName); } } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 4df92f34..5922af56 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -43,7 +43,7 @@ public class ArbitraryDataResource { // Check if the name is blacklisted if (ResourceListManager.getInstance() - .listContains("blacklist", "names", this.resourceId, false)) { + .listContains("blacklistedNames", this.resourceId, false)) { return new ArbitraryResourceSummary(ArbitraryResourceStatus.BLACKLISTED); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index d3b60531..2902f90d 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -142,7 +142,7 @@ public class ArbitraryDataManager extends Thread { private void processNames() { // Fetch latest list of followed names - List followedNames = ResourceListManager.getInstance().getStringsInList("followed", "names"); + List followedNames = ResourceListManager.getInstance().getStringsInList("followedNames"); if (followedNames == null || followedNames.isEmpty()) { return; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index e5e7d6e6..9ec88045 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -223,19 +223,19 @@ public class ArbitraryDataStorageManager extends Thread { } public boolean isNameInBlacklist(String name) { - return ResourceListManager.getInstance().listContains("blacklist", "names", name, false); + return ResourceListManager.getInstance().listContains("blacklistedNames", name, false); } private boolean isFollowingName(String name) { - return ResourceListManager.getInstance().listContains("followed", "names", name, false); + return ResourceListManager.getInstance().listContains("followedNames", name, false); } public List followedNames() { - return ResourceListManager.getInstance().getStringsInList("followed", "names"); + return ResourceListManager.getInstance().getStringsInList("followedNames"); } private int followedNamesCount() { - return ResourceListManager.getInstance().getItemCountForList("followed", "names"); + return ResourceListManager.getInstance().getItemCountForList("followedNames"); } diff --git a/src/main/java/org/qortal/list/ResourceList.java b/src/main/java/org/qortal/list/ResourceList.java index 0550a2ff..eb50957f 100644 --- a/src/main/java/org/qortal/list/ResourceList.java +++ b/src/main/java/org/qortal/list/ResourceList.java @@ -19,8 +19,7 @@ public class ResourceList { private static final Logger LOGGER = LogManager.getLogger(ResourceList.class); - private String category; - private String resourceName; + private String name; private List list = new ArrayList<>(); /** @@ -29,13 +28,11 @@ public class ResourceList { * This can be used for local blocking, or even for curating and sharing content lists * Lists are backed off to JSON files (in the lists folder) to ease sharing between nodes and users * - * @param category - for instance "blacklist", "whitelist", or "userlist" - * @param resourceName - for instance "address", "poll", or "group" + * @param name - the name of the list, for instance "addressblacklist" * @throws IOException */ - public ResourceList(String category, String resourceName) throws IOException { - this.category = category; - this.resourceName = resourceName; + public ResourceList(String name) throws IOException { + this.name = name; this.load(); } @@ -43,17 +40,14 @@ public class ResourceList { /* Filesystem */ private Path getFilePath() { - String pathString = String.format("%s%s%s_%s.json", Settings.getInstance().getListsPath(), - File.separator, this.category, this.resourceName); + String pathString = String.format("%s%s%s.json", Settings.getInstance().getListsPath(), + File.separator, this.name); return Paths.get(pathString); } public void save() throws IOException { - if (this.resourceName == null) { - throw new IllegalStateException("Can't save list with missing resource name"); - } - if (this.category == null) { - throw new IllegalStateException("Can't save list with missing category"); + if (this.name == null) { + throw new IllegalStateException("Can't save list with missing name"); } String jsonString = ResourceList.listToJSONString(this.list); Path filePath = this.getFilePath(); @@ -91,7 +85,7 @@ public class ResourceList { try { return this.load(); } catch (IOException e) { - LOGGER.info("Unable to revert {} {}", this.resourceName, this.category); + LOGGER.info("Unable to revert list {}: {}", this.name, e.getMessage()); } return false; } @@ -159,12 +153,8 @@ public class ResourceList { return ResourceList.listToJSONString(this.list); } - public String getCategory() { - return this.category; - } - - public String getResourceName() { - return this.resourceName; + public String getName() { + return this.name; } public List getList() { @@ -172,7 +162,7 @@ public class ResourceList { } public String toString() { - return String.format("%s %s", this.category, this.resourceName); + return this.name; } } diff --git a/src/main/java/org/qortal/list/ResourceListManager.java b/src/main/java/org/qortal/list/ResourceListManager.java index c71dfdab..4d4d559d 100644 --- a/src/main/java/org/qortal/list/ResourceListManager.java +++ b/src/main/java/org/qortal/list/ResourceListManager.java @@ -26,10 +26,9 @@ public class ResourceListManager { return instance; } - private ResourceList getList(String category, String resourceName) { + private ResourceList getList(String listName) { for (ResourceList list : this.lists) { - if (Objects.equals(list.getCategory(), category) && - Objects.equals(list.getResourceName(), resourceName)) { + if (Objects.equals(list.getName(), listName)) { return list; } } @@ -37,19 +36,19 @@ public class ResourceListManager { // List doesn't exist in array yet, so create it // This will load any existing data from the filesystem try { - ResourceList list = new ResourceList(category, resourceName); + ResourceList list = new ResourceList(listName); this.lists.add(list); return list; } catch (IOException e) { - LOGGER.info("Unable to load or create list {} {}: {}", category, resourceName, e.getMessage()); + LOGGER.info("Unable to load or create list {}: {}", listName, e.getMessage()); return null; } } - public boolean addToList(String category, String resourceName, String item, boolean save) { - ResourceList list = this.getList(category, resourceName); + public boolean addToList(String listName, String item, boolean save) { + ResourceList list = this.getList(listName); if (list == null) { return false; } @@ -67,8 +66,8 @@ public class ResourceListManager { } } - public boolean removeFromList(String category, String resourceName, String item, boolean save) { - ResourceList list = this.getList(category, resourceName); + public boolean removeFromList(String listName, String item, boolean save) { + ResourceList list = this.getList(listName); if (list == null) { return false; } @@ -87,16 +86,16 @@ public class ResourceListManager { } } - public boolean listContains(String category, String resourceName, String item, boolean caseSensitive) { - ResourceList list = this.getList(category, resourceName); + public boolean listContains(String listName, String item, boolean caseSensitive) { + ResourceList list = this.getList(listName); if (list == null) { return false; } return list.contains(item, caseSensitive); } - public void saveList(String category, String resourceName) { - ResourceList list = this.getList(category, resourceName); + public void saveList(String listName) { + ResourceList list = this.getList(listName); if (list == null) { return; } @@ -109,32 +108,32 @@ public class ResourceListManager { } } - public void revertList(String category, String resourceName) { - ResourceList list = this.getList(category, resourceName); + public void revertList(String listName) { + ResourceList list = this.getList(listName); if (list == null) { return; } list.revert(); } - public String getJSONStringForList(String category, String resourceName) { - ResourceList list = this.getList(category, resourceName); + public String getJSONStringForList(String listName) { + ResourceList list = this.getList(listName); if (list == null) { return null; } return list.getJSONString(); } - public List getStringsInList(String category, String resourceName) { - ResourceList list = this.getList(category, resourceName); + public List getStringsInList(String listName) { + ResourceList list = this.getList(listName); if (list == null) { return null; } return list.getList(); } - public int getItemCountForList(String category, String resourceName) { - ResourceList list = this.getList(category, resourceName); + public int getItemCountForList(String listName) { + ResourceList list = this.getList(listName); if (list == null) { return 0; } diff --git a/src/main/java/org/qortal/transaction/ChatTransaction.java b/src/main/java/org/qortal/transaction/ChatTransaction.java index f96e3ca2..130e35f1 100644 --- a/src/main/java/org/qortal/transaction/ChatTransaction.java +++ b/src/main/java/org/qortal/transaction/ChatTransaction.java @@ -147,7 +147,7 @@ public class ChatTransaction extends Transaction { // Check for blacklisted author by address ResourceListManager listManager = ResourceListManager.getInstance(); - if (listManager.listContains("blacklist", "addresses", this.chatTransactionData.getSender(), true)) { + if (listManager.listContains("blacklistedAddresses", this.chatTransactionData.getSender(), true)) { return ValidationResult.ADDRESS_IN_BLACKLIST; } @@ -156,7 +156,7 @@ public class ChatTransaction extends Transaction { if (names != null && names.size() > 0) { for (NameData nameData : names) { if (nameData != null && nameData.getName() != null) { - if (listManager.listContains("blacklist", "names", nameData.getName(), false)) { + if (listManager.listContains("blacklistedNames", nameData.getName(), false)) { return ValidationResult.NAME_IN_BLACKLIST; } } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java index d5da47f9..e9691924 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java @@ -96,18 +96,18 @@ public class ArbitraryDataStorageCapacityTests extends Common { assertTrue(storageManager.isStorageCapacityCalculated()); // Storage capacity should initially equal the total - assertEquals(0, resourceListManager.getItemCountForList("followed", "names")); + assertEquals(0, resourceListManager.getItemCountForList("followedNames")); long totalStorageCapacity = storageManager.getStorageCapacityIncludingThreshold(storageFullThreshold); assertEquals(totalStorageCapacity, storageManager.storageCapacityPerName(storageFullThreshold)); // Follow some names - assertTrue(resourceListManager.addToList("followed", "names", "Test1", false)); - assertTrue(resourceListManager.addToList("followed", "names", "Test2", false)); - assertTrue(resourceListManager.addToList("followed", "names", "Test3", false)); - assertTrue(resourceListManager.addToList("followed", "names", "Test4", false)); + assertTrue(resourceListManager.addToList("followedNames", "Test1", false)); + assertTrue(resourceListManager.addToList("followedNames", "Test2", false)); + assertTrue(resourceListManager.addToList("followedNames", "Test3", false)); + assertTrue(resourceListManager.addToList("followedNames", "Test4", false)); // Ensure the followed name count is correct - assertEquals(4, resourceListManager.getItemCountForList("followed", "names")); + assertEquals(4, resourceListManager.getItemCountForList("followedNames")); // Storage space per name should be the total storage capacity divided by the number of names long expectedStorageCapacityPerName = (long)(totalStorageCapacity / 4.0f); diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java index a68c1773..5c88956e 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java @@ -65,7 +65,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); // Add name to followed list - assertTrue(ResourceListManager.getInstance().addToList("followed", "names", name, false)); + assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.FOLLOWED_AND_VIEWED, Settings.getInstance().getStoragePolicy()); @@ -73,7 +73,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); // Now unfollow the name - assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false)); + assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(transactionData)); @@ -98,7 +98,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); // Add name to followed list - assertTrue(ResourceListManager.getInstance().addToList("followed", "names", name, false)); + assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy()); @@ -106,7 +106,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); // Now unfollow the name - assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false)); + assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We shouldn't store or pre-fetch data for this transaction assertFalse(storageManager.canStoreData(transactionData)); @@ -131,7 +131,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); // Add name to followed list - assertTrue(ResourceListManager.getInstance().addToList("followed", "names", name, false)); + assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); // We should store but not pre-fetch data for this transaction assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy()); @@ -139,7 +139,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); // Now unfollow the name - assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false)); + assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store but not pre-fetch data for this transaction assertTrue(storageManager.canStoreData(transactionData)); @@ -164,7 +164,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); // Add name to followed list - assertTrue(ResourceListManager.getInstance().addToList("followed", "names", name, false)); + assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy()); @@ -172,7 +172,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); // Now unfollow the name - assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false)); + assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store and pre-fetch data for this transaction assertTrue(storageManager.canStoreData(transactionData)); @@ -197,7 +197,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); // Add name to followed list - assertTrue(ResourceListManager.getInstance().addToList("followed", "names", name, false)); + assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); // We shouldn't store or pre-fetch data for this transaction assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy()); @@ -205,7 +205,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); // Now unfollow the name - assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false)); + assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We shouldn't store or pre-fetch data for this transaction assertFalse(storageManager.canStoreData(transactionData));