From ea10eec92678c68ce5e6fde30cffb16e14645dd5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Jan 2022 12:38:19 +0000 Subject: [PATCH 01/12] Fixed bug in auto update process - use the API key when stopping the node. Luckily this code is included in the new JAR, not the old one, so we should be able to regain auto update ability by issuing a new update. --- src/main/java/org/qortal/ApplyUpdate.java | 61 +++++++++++++++++++++-- src/main/java/org/qortal/api/ApiKey.java | 9 ++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/ApplyUpdate.java b/src/main/java/org/qortal/ApplyUpdate.java index edd6d924..ece44ba4 100644 --- a/src/main/java/org/qortal/ApplyUpdate.java +++ b/src/main/java/org/qortal/ApplyUpdate.java @@ -7,15 +7,15 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.security.Security; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.api.ApiKey; import org.qortal.api.ApiRequest; +import org.qortal.api.ApiService; import org.qortal.controller.AutoUpdate; import org.qortal.settings.Settings; @@ -70,14 +70,43 @@ public class ApplyUpdate { String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/"; LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri)); + // The /admin/stop endpoint requires an API key, which may or may not be already generated + boolean apiKeyNewlyGenerated = false; + ApiKey apiKey = null; + try { + apiKey = ApiService.getInstance().getApiKey(); + if (apiKey == null) { + apiKey = new ApiKey(); + if (!apiKey.generated()) { + apiKey.generate(); + apiKeyNewlyGenerated = true; + LOGGER.info("Generated API key"); + } + } + } catch (IOException e) { + LOGGER.info("Error loading API key: {}", e.getMessage()); + } + + // Create GET params + Map params = new HashMap<>(); + if (apiKey != null) { + params.put("apiKey", apiKey.toString()); + } + + // Attempt to stop the node int attempt; for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) { final int attemptForLogging = attempt; LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS)); - String response = ApiRequest.perform(baseUri + "admin/stop", null); - if (response == null) + String response = ApiRequest.perform(baseUri + "admin/stop", params); + if (response == null) { // No response - consider node shut down + if (apiKeyNewlyGenerated) { + // API key was newly generated for this auto update, so we need to remove it + ApplyUpdate.removeGeneratedApiKey(); + } return true; + } LOGGER.info(() -> String.format("Response from API: %s", response)); @@ -89,6 +118,11 @@ public class ApplyUpdate { } } + if (apiKeyNewlyGenerated) { + // API key was newly generated for this auto update, so we need to remove it + ApplyUpdate.removeGeneratedApiKey(); + } + if (attempt == MAX_ATTEMPTS) { LOGGER.error("Failed to shutdown node - giving up"); return false; @@ -97,6 +131,23 @@ public class ApplyUpdate { return true; } + private static void removeGeneratedApiKey() { + try { + LOGGER.info("Removing newly generated API key..."); + + ApiKey apiKey = ApiService.getInstance().getApiKey(); + if (apiKey == null) { + apiKey = new ApiKey(); + } + + // Delete the API key since it was only generated for this auto update + apiKey.delete(); + + } catch (IOException e) { + LOGGER.info("Error loading or deleting API key: {}", e.getMessage()); + } + } + private static void replaceJar() { // Assuming current working directory contains the JAR files Path realJar = Paths.get(JAR_FILENAME); diff --git a/src/main/java/org/qortal/api/ApiKey.java b/src/main/java/org/qortal/api/ApiKey.java index 6a79dd20..3f7cfe35 100644 --- a/src/main/java/org/qortal/api/ApiKey.java +++ b/src/main/java/org/qortal/api/ApiKey.java @@ -81,6 +81,15 @@ public class ApiKey { writer.close(); } + public void delete() throws IOException { + this.apiKey = null; + + Path filePath = this.getFilePath(); + if (Files.exists(filePath)) { + Files.delete(filePath); + } + } + public boolean generated() { return (this.apiKey != null); From ff6ec83b1cc94e81e6212d2ec64bc262f2ad97fc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Jan 2022 12:48:37 +0000 Subject: [PATCH 02/12] Removed localAuthBypassEnabled override in unit tests. Hopefully this will allow us to proactively catch any missing API keys in the future. --- src/test/java/org/qortal/test/api/AdminApiTests.java | 7 ++++++- src/test/resources/test-settings-v2.json | 3 +-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/qortal/test/api/AdminApiTests.java b/src/test/java/org/qortal/test/api/AdminApiTests.java index 89b1464a..b3e6da03 100644 --- a/src/test/java/org/qortal/test/api/AdminApiTests.java +++ b/src/test/java/org/qortal/test/api/AdminApiTests.java @@ -2,10 +2,12 @@ package org.qortal.test.api; import static org.junit.Assert.*; +import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.Before; import org.junit.Test; import org.qortal.api.resource.AdminResource; import org.qortal.repository.DataException; +import org.qortal.settings.Settings; import org.qortal.test.common.ApiCommon; import org.qortal.test.common.Common; @@ -29,7 +31,10 @@ public class AdminApiTests extends ApiCommon { } @Test - public void testSummary() { + public void testSummary() throws IllegalAccessException { + // Set localAuthBypassEnabled to true, since we don't need to test authentication here + FieldUtils.writeField(Settings.getInstance(), "localAuthBypassEnabled", true, true); + assertNotNull(this.adminResource.summary("testApiKey")); } diff --git a/src/test/resources/test-settings-v2.json b/src/test/resources/test-settings-v2.json index 7802f598..b2ad3db8 100644 --- a/src/test/resources/test-settings-v2.json +++ b/src/test/resources/test-settings-v2.json @@ -15,6 +15,5 @@ "tempDataPath": "data-test/_temp", "listsPath": "lists-test", "storagePolicy": "FOLLOWED_OR_VIEWED", - "maxStorageCapacity": 104857600, - "localAuthBypassEnabled": true + "maxStorageCapacity": 104857600 } From 6f7c8d96b9c77ed911e3822c8b324fb59836ec8e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Jan 2022 12:57:28 +0000 Subject: [PATCH 03/12] Removed ApiService instance creation in ApplyUpdate as it wasn't really needed, and probably not sensible to instantiate it here. --- src/main/java/org/qortal/ApplyUpdate.java | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/qortal/ApplyUpdate.java b/src/main/java/org/qortal/ApplyUpdate.java index ece44ba4..90171191 100644 --- a/src/main/java/org/qortal/ApplyUpdate.java +++ b/src/main/java/org/qortal/ApplyUpdate.java @@ -15,7 +15,6 @@ import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; import org.qortal.api.ApiKey; import org.qortal.api.ApiRequest; -import org.qortal.api.ApiService; import org.qortal.controller.AutoUpdate; import org.qortal.settings.Settings; @@ -74,14 +73,11 @@ public class ApplyUpdate { boolean apiKeyNewlyGenerated = false; ApiKey apiKey = null; try { - apiKey = ApiService.getInstance().getApiKey(); - if (apiKey == null) { - apiKey = new ApiKey(); - if (!apiKey.generated()) { - apiKey.generate(); - apiKeyNewlyGenerated = true; - LOGGER.info("Generated API key"); - } + apiKey = new ApiKey(); + if (!apiKey.generated()) { + apiKey.generate(); + apiKeyNewlyGenerated = true; + LOGGER.info("Generated API key"); } } catch (IOException e) { LOGGER.info("Error loading API key: {}", e.getMessage()); @@ -135,12 +131,8 @@ public class ApplyUpdate { try { LOGGER.info("Removing newly generated API key..."); - ApiKey apiKey = ApiService.getInstance().getApiKey(); - if (apiKey == null) { - apiKey = new ApiKey(); - } - // Delete the API key since it was only generated for this auto update + ApiKey apiKey = new ApiKey(); apiKey.delete(); } catch (IOException e) { From 311fe98f44e3118635f9eb341450554b6ee8ec4b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Jan 2022 13:19:49 +0000 Subject: [PATCH 04/12] Bump version to 3.0.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a2a790fa..b192942b 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.0.2 + 3.0.3 jar true From af06774ba66921a2e4647b3744b22ae847a6e185 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Jan 2022 22:53:35 +0000 Subject: [PATCH 05/12] Clear the cache when deleting data, so that it disappears from the data management screen. --- src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 65c92cc6..0ece14a5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -6,6 +6,7 @@ import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.list.ResourceListManager; @@ -116,6 +117,9 @@ public class ArbitraryDataResource { // Also delete cached data for the entire resource this.deleteCache(); + // Invalidate the hosted transactions cache as we have removed an item + ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache(); + return true; } catch (DataException | IOException e) { From 9daf7a6668c2b8b612b5c2cbebfb7b952dfab88a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 26 Jan 2022 22:40:34 +0000 Subject: [PATCH 06/12] Synchronize lists, to prevent an occasional ConcurrentModificationException --- src/main/java/org/qortal/list/ResourceList.java | 3 ++- src/main/java/org/qortal/list/ResourceListManager.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/list/ResourceList.java b/src/main/java/org/qortal/list/ResourceList.java index fbdc8470..099aa168 100644 --- a/src/main/java/org/qortal/list/ResourceList.java +++ b/src/main/java/org/qortal/list/ResourceList.java @@ -13,6 +13,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class ResourceList { @@ -20,7 +21,7 @@ public class ResourceList { private static final Logger LOGGER = LogManager.getLogger(ResourceList.class); private String name; - private List list = new ArrayList<>(); + private List list = Collections.synchronizedList(new ArrayList<>()); /** * ResourceList diff --git a/src/main/java/org/qortal/list/ResourceListManager.java b/src/main/java/org/qortal/list/ResourceListManager.java index 4d4d559d..4182f87c 100644 --- a/src/main/java/org/qortal/list/ResourceListManager.java +++ b/src/main/java/org/qortal/list/ResourceListManager.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -13,7 +14,7 @@ public class ResourceListManager { private static final Logger LOGGER = LogManager.getLogger(ResourceListManager.class); private static ResourceListManager instance; - private List lists = new ArrayList<>(); + private List lists = Collections.synchronizedList(new ArrayList<>()); public ResourceListManager() { From 4e71ae0e598c75e3610aee41d2a7950909cf3a39 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 18:10:24 +0000 Subject: [PATCH 07/12] Allow QDN data to be served without authentication by setting "qdnAuthBypassEnabled":true This allows the GET /arbitrary/{service}/{name} and GET /{service}/{name}/{identifier} endpoints to operate without any authentication. Useful for those who are running public QDN nodes and need to serve data over http(s). --- .../org/qortal/api/resource/ArbitraryResource.java | 12 ++++++++++-- src/main/java/org/qortal/settings/Settings.java | 7 +++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 82618152..1cb9c1c3 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -575,7 +575,11 @@ public class ArbitraryResource { @PathParam("name") String name, @QueryParam("filepath") String filepath, @QueryParam("rebuild") boolean rebuild) { - Security.checkApiCallAllowed(request); + + // Authentication can be bypassed in the settings, for those running public QDN nodes + if (!Settings.getInstance().isQDNAuthBypassEnabled()) { + Security.checkApiCallAllowed(request); + } return this.download(service, name, null, filepath, rebuild); } @@ -604,7 +608,11 @@ public class ArbitraryResource { @PathParam("identifier") String identifier, @QueryParam("filepath") String filepath, @QueryParam("rebuild") boolean rebuild) { - Security.checkApiCallAllowed(request); + + // Authentication can be bypassed in the settings, for those running public QDN nodes + if (!Settings.getInstance().isQDNAuthBypassEnabled()) { + Security.checkApiCallAllowed(request); + } return this.download(service, name, identifier, filepath, rebuild); } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 3bd7cef5..41b69114 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -308,6 +308,9 @@ public class Settings { /** Maximum total size of hosted data, in bytes. Unlimited if null */ private Long maxStorageCapacity = null; + /** Whether to serve QDN data without authentication */ + private boolean qdnAuthBypassEnabled = false; + // Domain mapping public static class DomainMap { private String domain; @@ -884,4 +887,8 @@ public class Settings { public Long getMaxStorageCapacity() { return this.maxStorageCapacity; } + + public boolean isQDNAuthBypassEnabled() { + return this.qdnAuthBypassEnabled; + } } From 3303e41a399306dfb320d3d78685947ea102c471 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 18:12:21 +0000 Subject: [PATCH 08/12] Fixed unhandled case in GET /arbitrary/{service}/{name}* endpoints --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 1cb9c1c3..d542b89c 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1057,6 +1057,10 @@ public class ArbitraryResource { // This is a single file resource filepath = files[0]; } + else { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, + "filepath is required for resources containing more than one file"); + } } // TODO: limit file size that can be read into memory From 344704b6bf69861ff0b087327cc7d889e1efef88 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 18:15:00 +0000 Subject: [PATCH 09/12] MIN_LEVEL_FOR_BLOCK_SUBMISSION temporarily increased to 6. This is to hopefully improve network stability whilst a more advanced solution is being worked on. It also allows us to collect some data on how well the network behaves when there are less block candidates. It should have no effect on minting rewards (other than any side effects as a result of improved network stability). --- src/main/java/org/qortal/controller/BlockMinter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 4eea91a9..a20cf9ae 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -51,7 +51,7 @@ public class BlockMinter extends Thread { // Min account level to submit blocks // This is an unvalidated version of Blockchain.minAccountLevelToMint // and exists only to reduce block candidates by default. - private static int MIN_LEVEL_FOR_BLOCK_SUBMISSION = 3; + private static int MIN_LEVEL_FOR_BLOCK_SUBMISSION = 6; // Constructors From a0ba016171bab433f5ef641cd4560bd4deb3b0c6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 18:42:52 +0000 Subject: [PATCH 10/12] Fixed case sensitive ordering issue in websites list. --- .../org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 0087ce23..ccf4691b 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -330,7 +330,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { bindParams.add(name); } - sql.append(" GROUP BY name, service, identifier ORDER BY name"); + sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); if (reverse != null && reverse) { sql.append(" DESC"); From 7808a1553ef568a1d18c9173a433cd44b27391af Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 18:46:59 +0000 Subject: [PATCH 11/12] Fixed case sensitive ordering issues in other aspects of QDN. --- .../qortal/repository/hsqldb/HSQLDBArbitraryRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index ccf4691b..a7da66ae 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -401,7 +401,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { bindParams.add(queryWildcard); } - sql.append(" GROUP BY name, service, identifier ORDER BY name"); + sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); if (reverse != null && reverse) { sql.append(" DESC"); @@ -465,7 +465,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { sql.append(" AND (identifier = ? OR (? IS NULL))"); } - sql.append(" GROUP BY name ORDER BY name"); + sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); if (reverse != null && reverse) { sql.append(" DESC"); From e5c12b18afe1244111746690899190a5b06245b9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 19:58:39 +0000 Subject: [PATCH 12/12] Bump version to 3.0.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b192942b..798d68ea 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.0.3 + 3.0.4 jar true