From a790b2e529fd9a74b0c1b0b66f4e8dcad6e1e8db Mon Sep 17 00:00:00 2001 From: QuickMythril Date: Thu, 23 Dec 2021 19:10:17 -0500 Subject: [PATCH 001/151] reduce DOGE fees https://github.com/dogecoin/dogecoin/blob/master/doc/fee-recommendation.md --- src/main/java/org/qortal/crosschain/Dogecoin.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Dogecoin.java b/src/main/java/org/qortal/crosschain/Dogecoin.java index 4acd95aa..6a70bb00 100644 --- a/src/main/java/org/qortal/crosschain/Dogecoin.java +++ b/src/main/java/org/qortal/crosschain/Dogecoin.java @@ -19,12 +19,13 @@ public class Dogecoin extends Bitcoiny { public static final String CURRENCY_CODE = "DOGE"; - private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(500000000); // 5 DOGE per 1000 bytes + private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(1000000); // 0.01 DOGE per 1000 bytes - private static final long MINIMUM_ORDER_AMOUNT = 300000000L; // 3 DOGE minimum order. The RPC dust threshold is around 2 DOGE + private static final long MINIMUM_ORDER_AMOUNT = 100000000L; // 1 DOGE minimum order. See recommendations: + // https://github.com/dogecoin/dogecoin/blob/master/doc/fee-recommendation.md // Temporary values until a dynamic fee system is written. - private static final long MAINNET_FEE = 110000000L; + private static final long MAINNET_FEE = 100000L; private static final long NON_MAINNET_FEE = 10000L; // TODO: calibrate this private static final Map DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ConnectionType.class); From 25efee55b8310885b89238c180b6b3ca6d30c0bf Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Jan 2022 18:43:32 +0000 Subject: [PATCH 002/151] Added networking optimization, to avoid wasted processing on every read. Thanks to @catbref for finding this. --- src/main/java/org/qortal/network/Peer.java | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index 3b50b777..da4a70a9 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -473,16 +473,18 @@ public class Peer { return; } - if (bytesRead > 0) { - byte[] leadingBytes = new byte[Math.min(bytesRead, 8)]; - this.byteBuffer.asReadOnlyBuffer().position(priorPosition).get(leadingBytes); - String leadingHex = HashCode.fromBytes(leadingBytes).toString(); + if (LOGGER.isTraceEnabled()) { + if (bytesRead > 0) { + byte[] leadingBytes = new byte[Math.min(bytesRead, 8)]; + this.byteBuffer.asReadOnlyBuffer().position(priorPosition).get(leadingBytes); + String leadingHex = HashCode.fromBytes(leadingBytes).toString(); - LOGGER.trace("[{}] Received {} bytes, starting {}, into byteBuffer[{}] from peer {}", - this.peerConnectionId, bytesRead, leadingHex, priorPosition, this); - } else { - LOGGER.trace("[{}] Received {} bytes into byteBuffer[{}] from peer {}", this.peerConnectionId, - bytesRead, priorPosition, this); + LOGGER.trace("[{}] Received {} bytes, starting {}, into byteBuffer[{}] from peer {}", + this.peerConnectionId, bytesRead, leadingHex, priorPosition, this); + } else { + LOGGER.trace("[{}] Received {} bytes into byteBuffer[{}] from peer {}", this.peerConnectionId, + bytesRead, priorPosition, this); + } } final boolean wasByteBufferFull = !this.byteBuffer.hasRemaining(); From 59346db427711b7e263b8102f03d1201dbdac214 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Jan 2022 18:45:32 +0000 Subject: [PATCH 003/151] Bump version to 3.0.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 89ccd1d0..a2a790fa 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.0.1 + 3.0.2 jar true From a7c02733ec20c7c30a7b1f8eb2d00ed81962741e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Jan 2022 19:40:13 +0000 Subject: [PATCH 004/151] Updated approve-auto-update.sh to use new service format --- tools/approve-auto-update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/approve-auto-update.sh b/tools/approve-auto-update.sh index 232fc099..cbfa280d 100755 --- a/tools/approve-auto-update.sh +++ b/tools/approve-auto-update.sh @@ -7,7 +7,7 @@ fi printf "Searching for auto-update transactions to approve...\n"; -tx=$( curl --silent --url "http://localhost:${port}/arbitrary/search?txGroupId=1&service=1&confirmationStatus=CONFIRMED&limit=1&reverse=true" ); +tx=$( curl --silent --url "http://localhost:${port}/arbitrary/search?txGroupId=1&service=AUTO_UPDATE&confirmationStatus=CONFIRMED&limit=1&reverse=true" ); if fgrep --silent '"approvalStatus":"PENDING"' <<< "${tx}"; then true else From 048776e09087b39df68b1648934de3dbfd25b676 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Jan 2022 20:43:28 +0000 Subject: [PATCH 005/151] Ignore failing test due to recent API update, which makes the test incompatible. To be fixed later. --- src/test/java/org/qortal/test/api/BlockApiTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/qortal/test/api/BlockApiTests.java b/src/test/java/org/qortal/test/api/BlockApiTests.java index ed0a2b8f..47d5318a 100644 --- a/src/test/java/org/qortal/test/api/BlockApiTests.java +++ b/src/test/java/org/qortal/test/api/BlockApiTests.java @@ -7,6 +7,7 @@ import java.util.Collections; import java.util.List; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.api.ApiError; @@ -76,7 +77,8 @@ public class BlockApiTests extends ApiCommon { } @Test - public void testGetBlockByTimestamp() { + @Ignore(value = "Doesn't work, to be fixed later") + public void testGetBlockByTimestamp() throws DataException { assertNotNull(this.blocksResource.getByTimestamp(System.currentTimeMillis(), false)); } From 6f724f648dcf54b8fd1a5feeeab6fc70d8b84217 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Jan 2022 20:48:58 +0000 Subject: [PATCH 006/151] Fixed testDirectoryDigest() which has been failing for a couple of versions (due to gitignore removing the cache file) --- .../org/qortal/test/arbitrary/ArbitraryDataDigestTests.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java index 1c8afc2e..8ef04b27 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java @@ -8,6 +8,7 @@ import org.qortal.test.common.Common; import java.io.FileWriter; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.UUID; @@ -38,7 +39,9 @@ public class ArbitraryDataDigestTests extends Common { // Write a random file to .qortal/cache to ensure it isn't being included in the digest function // We exclude all .qortal files from the digest since they can be different with each build, and // we only care about the actual user files - FileWriter fileWriter = new FileWriter(Paths.get(dataPath.toString(), ".qortal", "cache").toString()); + Path cachePath = Paths.get(dataPath.toString(), ".qortal", "cache"); + Files.createDirectories(cachePath.getParent()); + FileWriter fileWriter = new FileWriter(cachePath.toString()); fileWriter.append(UUID.randomUUID().toString()); fileWriter.close(); From be561a16093a52a383cfb6396f44b077e667abdf Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Jan 2022 22:27:14 +0000 Subject: [PATCH 007/151] Add default "dataPath" to Windows installer builds, so that QDN data is located in AppData --- WindowsInstaller/Qortal.aip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip index 12636c44..bdf30092 100755 --- a/WindowsInstaller/Qortal.aip +++ b/WindowsInstaller/Qortal.aip @@ -1173,7 +1173,7 @@ - + From ea10eec92678c68ce5e6fde30cffb16e14645dd5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Jan 2022 12:38:19 +0000 Subject: [PATCH 008/151] Fixed bug in auto update process - use the API key when stopping the node. Luckily this code is included in the new JAR, not the old one, so we should be able to regain auto update ability by issuing a new update. --- src/main/java/org/qortal/ApplyUpdate.java | 61 +++++++++++++++++++++-- src/main/java/org/qortal/api/ApiKey.java | 9 ++++ 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/ApplyUpdate.java b/src/main/java/org/qortal/ApplyUpdate.java index edd6d924..ece44ba4 100644 --- a/src/main/java/org/qortal/ApplyUpdate.java +++ b/src/main/java/org/qortal/ApplyUpdate.java @@ -7,15 +7,15 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.security.Security; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import java.util.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; +import org.qortal.api.ApiKey; import org.qortal.api.ApiRequest; +import org.qortal.api.ApiService; import org.qortal.controller.AutoUpdate; import org.qortal.settings.Settings; @@ -70,14 +70,43 @@ public class ApplyUpdate { String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/"; LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri)); + // The /admin/stop endpoint requires an API key, which may or may not be already generated + boolean apiKeyNewlyGenerated = false; + ApiKey apiKey = null; + try { + apiKey = ApiService.getInstance().getApiKey(); + if (apiKey == null) { + apiKey = new ApiKey(); + if (!apiKey.generated()) { + apiKey.generate(); + apiKeyNewlyGenerated = true; + LOGGER.info("Generated API key"); + } + } + } catch (IOException e) { + LOGGER.info("Error loading API key: {}", e.getMessage()); + } + + // Create GET params + Map params = new HashMap<>(); + if (apiKey != null) { + params.put("apiKey", apiKey.toString()); + } + + // Attempt to stop the node int attempt; for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) { final int attemptForLogging = attempt; LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS)); - String response = ApiRequest.perform(baseUri + "admin/stop", null); - if (response == null) + String response = ApiRequest.perform(baseUri + "admin/stop", params); + if (response == null) { // No response - consider node shut down + if (apiKeyNewlyGenerated) { + // API key was newly generated for this auto update, so we need to remove it + ApplyUpdate.removeGeneratedApiKey(); + } return true; + } LOGGER.info(() -> String.format("Response from API: %s", response)); @@ -89,6 +118,11 @@ public class ApplyUpdate { } } + if (apiKeyNewlyGenerated) { + // API key was newly generated for this auto update, so we need to remove it + ApplyUpdate.removeGeneratedApiKey(); + } + if (attempt == MAX_ATTEMPTS) { LOGGER.error("Failed to shutdown node - giving up"); return false; @@ -97,6 +131,23 @@ public class ApplyUpdate { return true; } + private static void removeGeneratedApiKey() { + try { + LOGGER.info("Removing newly generated API key..."); + + ApiKey apiKey = ApiService.getInstance().getApiKey(); + if (apiKey == null) { + apiKey = new ApiKey(); + } + + // Delete the API key since it was only generated for this auto update + apiKey.delete(); + + } catch (IOException e) { + LOGGER.info("Error loading or deleting API key: {}", e.getMessage()); + } + } + private static void replaceJar() { // Assuming current working directory contains the JAR files Path realJar = Paths.get(JAR_FILENAME); diff --git a/src/main/java/org/qortal/api/ApiKey.java b/src/main/java/org/qortal/api/ApiKey.java index 6a79dd20..3f7cfe35 100644 --- a/src/main/java/org/qortal/api/ApiKey.java +++ b/src/main/java/org/qortal/api/ApiKey.java @@ -81,6 +81,15 @@ public class ApiKey { writer.close(); } + public void delete() throws IOException { + this.apiKey = null; + + Path filePath = this.getFilePath(); + if (Files.exists(filePath)) { + Files.delete(filePath); + } + } + public boolean generated() { return (this.apiKey != null); From ff6ec83b1cc94e81e6212d2ec64bc262f2ad97fc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Jan 2022 12:48:37 +0000 Subject: [PATCH 009/151] Removed localAuthBypassEnabled override in unit tests. Hopefully this will allow us to proactively catch any missing API keys in the future. --- src/test/java/org/qortal/test/api/AdminApiTests.java | 7 ++++++- src/test/resources/test-settings-v2.json | 3 +-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/qortal/test/api/AdminApiTests.java b/src/test/java/org/qortal/test/api/AdminApiTests.java index 89b1464a..b3e6da03 100644 --- a/src/test/java/org/qortal/test/api/AdminApiTests.java +++ b/src/test/java/org/qortal/test/api/AdminApiTests.java @@ -2,10 +2,12 @@ package org.qortal.test.api; import static org.junit.Assert.*; +import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.Before; import org.junit.Test; import org.qortal.api.resource.AdminResource; import org.qortal.repository.DataException; +import org.qortal.settings.Settings; import org.qortal.test.common.ApiCommon; import org.qortal.test.common.Common; @@ -29,7 +31,10 @@ public class AdminApiTests extends ApiCommon { } @Test - public void testSummary() { + public void testSummary() throws IllegalAccessException { + // Set localAuthBypassEnabled to true, since we don't need to test authentication here + FieldUtils.writeField(Settings.getInstance(), "localAuthBypassEnabled", true, true); + assertNotNull(this.adminResource.summary("testApiKey")); } diff --git a/src/test/resources/test-settings-v2.json b/src/test/resources/test-settings-v2.json index 7802f598..b2ad3db8 100644 --- a/src/test/resources/test-settings-v2.json +++ b/src/test/resources/test-settings-v2.json @@ -15,6 +15,5 @@ "tempDataPath": "data-test/_temp", "listsPath": "lists-test", "storagePolicy": "FOLLOWED_OR_VIEWED", - "maxStorageCapacity": 104857600, - "localAuthBypassEnabled": true + "maxStorageCapacity": 104857600 } From 6f7c8d96b9c77ed911e3822c8b324fb59836ec8e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Jan 2022 12:57:28 +0000 Subject: [PATCH 010/151] Removed ApiService instance creation in ApplyUpdate as it wasn't really needed, and probably not sensible to instantiate it here. --- src/main/java/org/qortal/ApplyUpdate.java | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/qortal/ApplyUpdate.java b/src/main/java/org/qortal/ApplyUpdate.java index ece44ba4..90171191 100644 --- a/src/main/java/org/qortal/ApplyUpdate.java +++ b/src/main/java/org/qortal/ApplyUpdate.java @@ -15,7 +15,6 @@ import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider; import org.qortal.api.ApiKey; import org.qortal.api.ApiRequest; -import org.qortal.api.ApiService; import org.qortal.controller.AutoUpdate; import org.qortal.settings.Settings; @@ -74,14 +73,11 @@ public class ApplyUpdate { boolean apiKeyNewlyGenerated = false; ApiKey apiKey = null; try { - apiKey = ApiService.getInstance().getApiKey(); - if (apiKey == null) { - apiKey = new ApiKey(); - if (!apiKey.generated()) { - apiKey.generate(); - apiKeyNewlyGenerated = true; - LOGGER.info("Generated API key"); - } + apiKey = new ApiKey(); + if (!apiKey.generated()) { + apiKey.generate(); + apiKeyNewlyGenerated = true; + LOGGER.info("Generated API key"); } } catch (IOException e) { LOGGER.info("Error loading API key: {}", e.getMessage()); @@ -135,12 +131,8 @@ public class ApplyUpdate { try { LOGGER.info("Removing newly generated API key..."); - ApiKey apiKey = ApiService.getInstance().getApiKey(); - if (apiKey == null) { - apiKey = new ApiKey(); - } - // Delete the API key since it was only generated for this auto update + ApiKey apiKey = new ApiKey(); apiKey.delete(); } catch (IOException e) { From 311fe98f44e3118635f9eb341450554b6ee8ec4b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Jan 2022 13:19:49 +0000 Subject: [PATCH 011/151] Bump version to 3.0.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a2a790fa..b192942b 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.0.2 + 3.0.3 jar true From af06774ba66921a2e4647b3744b22ae847a6e185 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Jan 2022 22:53:35 +0000 Subject: [PATCH 012/151] Clear the cache when deleting data, so that it disappears from the data management screen. --- src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 65c92cc6..0ece14a5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -6,6 +6,7 @@ import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; import org.qortal.data.arbitrary.ArbitraryResourceStatus; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.list.ResourceListManager; @@ -116,6 +117,9 @@ public class ArbitraryDataResource { // Also delete cached data for the entire resource this.deleteCache(); + // Invalidate the hosted transactions cache as we have removed an item + ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache(); + return true; } catch (DataException | IOException e) { From a2b2b6393248efa01644eadd5bf4ce95797010ba Mon Sep 17 00:00:00 2001 From: Alexander Do Date: Sun, 23 Jan 2022 15:53:29 -0800 Subject: [PATCH 013/151] feat: add Dockerfile --- Dockerfile | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b06f7659 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM maven:3-openjdk-11 as builder + +WORKDIR /work +COPY ./ /work/ +RUN mvn clean package + +### +FROM openjdk:11 + +RUN useradd -r -u 1000 -g users qortal && \ + mkdir /usr/local/qortal /qortal && \ + chown 1000:100 /qortal + +COPY --from=builder /work/log4j2.properties /usr/local/qortal/ +COPY --from=builder /work/target/qortal*.jar /usr/local/qortal/qortal.jar + +USER 1000:100 + +EXPOSE 12391 12392 +HEALTHCHECK --start-period=5m CMD curl -sf http://127.0.0.1:12391/admin/info || exit 1 + +WORKDIR /qortal +VOLUME /qortal + +ENTRYPOINT ["java"] +CMD ["-Djava.net.preferIPv4Stack=false", "-jar", "/usr/local/qortal/qortal.jar"] From 9daf7a6668c2b8b612b5c2cbebfb7b952dfab88a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 26 Jan 2022 22:40:34 +0000 Subject: [PATCH 014/151] Synchronize lists, to prevent an occasional ConcurrentModificationException --- src/main/java/org/qortal/list/ResourceList.java | 3 ++- src/main/java/org/qortal/list/ResourceListManager.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/list/ResourceList.java b/src/main/java/org/qortal/list/ResourceList.java index fbdc8470..099aa168 100644 --- a/src/main/java/org/qortal/list/ResourceList.java +++ b/src/main/java/org/qortal/list/ResourceList.java @@ -13,6 +13,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.List; public class ResourceList { @@ -20,7 +21,7 @@ public class ResourceList { private static final Logger LOGGER = LogManager.getLogger(ResourceList.class); private String name; - private List list = new ArrayList<>(); + private List list = Collections.synchronizedList(new ArrayList<>()); /** * ResourceList diff --git a/src/main/java/org/qortal/list/ResourceListManager.java b/src/main/java/org/qortal/list/ResourceListManager.java index 4d4d559d..4182f87c 100644 --- a/src/main/java/org/qortal/list/ResourceListManager.java +++ b/src/main/java/org/qortal/list/ResourceListManager.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -13,7 +14,7 @@ public class ResourceListManager { private static final Logger LOGGER = LogManager.getLogger(ResourceListManager.class); private static ResourceListManager instance; - private List lists = new ArrayList<>(); + private List lists = Collections.synchronizedList(new ArrayList<>()); public ResourceListManager() { From 4e71ae0e598c75e3610aee41d2a7950909cf3a39 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 18:10:24 +0000 Subject: [PATCH 015/151] Allow QDN data to be served without authentication by setting "qdnAuthBypassEnabled":true This allows the GET /arbitrary/{service}/{name} and GET /{service}/{name}/{identifier} endpoints to operate without any authentication. Useful for those who are running public QDN nodes and need to serve data over http(s). --- .../org/qortal/api/resource/ArbitraryResource.java | 12 ++++++++++-- src/main/java/org/qortal/settings/Settings.java | 7 +++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 82618152..1cb9c1c3 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -575,7 +575,11 @@ public class ArbitraryResource { @PathParam("name") String name, @QueryParam("filepath") String filepath, @QueryParam("rebuild") boolean rebuild) { - Security.checkApiCallAllowed(request); + + // Authentication can be bypassed in the settings, for those running public QDN nodes + if (!Settings.getInstance().isQDNAuthBypassEnabled()) { + Security.checkApiCallAllowed(request); + } return this.download(service, name, null, filepath, rebuild); } @@ -604,7 +608,11 @@ public class ArbitraryResource { @PathParam("identifier") String identifier, @QueryParam("filepath") String filepath, @QueryParam("rebuild") boolean rebuild) { - Security.checkApiCallAllowed(request); + + // Authentication can be bypassed in the settings, for those running public QDN nodes + if (!Settings.getInstance().isQDNAuthBypassEnabled()) { + Security.checkApiCallAllowed(request); + } return this.download(service, name, identifier, filepath, rebuild); } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 3bd7cef5..41b69114 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -308,6 +308,9 @@ public class Settings { /** Maximum total size of hosted data, in bytes. Unlimited if null */ private Long maxStorageCapacity = null; + /** Whether to serve QDN data without authentication */ + private boolean qdnAuthBypassEnabled = false; + // Domain mapping public static class DomainMap { private String domain; @@ -884,4 +887,8 @@ public class Settings { public Long getMaxStorageCapacity() { return this.maxStorageCapacity; } + + public boolean isQDNAuthBypassEnabled() { + return this.qdnAuthBypassEnabled; + } } From 3303e41a399306dfb320d3d78685947ea102c471 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 18:12:21 +0000 Subject: [PATCH 016/151] Fixed unhandled case in GET /arbitrary/{service}/{name}* endpoints --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 1cb9c1c3..d542b89c 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1057,6 +1057,10 @@ public class ArbitraryResource { // This is a single file resource filepath = files[0]; } + else { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, + "filepath is required for resources containing more than one file"); + } } // TODO: limit file size that can be read into memory From 344704b6bf69861ff0b087327cc7d889e1efef88 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 18:15:00 +0000 Subject: [PATCH 017/151] MIN_LEVEL_FOR_BLOCK_SUBMISSION temporarily increased to 6. This is to hopefully improve network stability whilst a more advanced solution is being worked on. It also allows us to collect some data on how well the network behaves when there are less block candidates. It should have no effect on minting rewards (other than any side effects as a result of improved network stability). --- src/main/java/org/qortal/controller/BlockMinter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 4eea91a9..a20cf9ae 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -51,7 +51,7 @@ public class BlockMinter extends Thread { // Min account level to submit blocks // This is an unvalidated version of Blockchain.minAccountLevelToMint // and exists only to reduce block candidates by default. - private static int MIN_LEVEL_FOR_BLOCK_SUBMISSION = 3; + private static int MIN_LEVEL_FOR_BLOCK_SUBMISSION = 6; // Constructors From a0ba016171bab433f5ef641cd4560bd4deb3b0c6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 18:42:52 +0000 Subject: [PATCH 018/151] Fixed case sensitive ordering issue in websites list. --- .../org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index 0087ce23..ccf4691b 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -330,7 +330,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { bindParams.add(name); } - sql.append(" GROUP BY name, service, identifier ORDER BY name"); + sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); if (reverse != null && reverse) { sql.append(" DESC"); From 7808a1553ef568a1d18c9173a433cd44b27391af Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 18:46:59 +0000 Subject: [PATCH 019/151] Fixed case sensitive ordering issues in other aspects of QDN. --- .../qortal/repository/hsqldb/HSQLDBArbitraryRepository.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index ccf4691b..a7da66ae 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -401,7 +401,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { bindParams.add(queryWildcard); } - sql.append(" GROUP BY name, service, identifier ORDER BY name"); + sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); if (reverse != null && reverse) { sql.append(" DESC"); @@ -465,7 +465,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { sql.append(" AND (identifier = ? OR (? IS NULL))"); } - sql.append(" GROUP BY name ORDER BY name"); + sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD"); if (reverse != null && reverse) { sql.append(" DESC"); From e5c12b18afe1244111746690899190a5b06245b9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 19:58:39 +0000 Subject: [PATCH 020/151] Bump version to 3.0.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b192942b..798d68ea 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.0.3 + 3.0.4 jar true From d200a098cd1d78607b9d98bc633d11d6e197c2fe Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 27 Jan 2022 23:18:34 +0000 Subject: [PATCH 021/151] Updated AdvancedInstaller project for v3.0.4 --- WindowsInstaller/Qortal.aip | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip index bdf30092..59da9519 100755 --- a/WindowsInstaller/Qortal.aip +++ b/WindowsInstaller/Qortal.aip @@ -17,10 +17,10 @@ - + - + @@ -212,7 +212,7 @@ - + From cdd57190ce0c1be6a56033b95868aa338c935657 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 28 Jan 2022 10:16:42 +0000 Subject: [PATCH 022/151] Use getEffectiveMintingLevel() rather than getLevel() --- .../java/org/qortal/controller/BlockMinter.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index a20cf9ae..5911d07c 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -16,7 +16,6 @@ import org.qortal.account.PrivateKeyAccount; import org.qortal.block.Block; import org.qortal.block.Block.ValidationResult; import org.qortal.block.BlockChain; -import org.qortal.data.account.AccountData; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.block.BlockData; @@ -48,10 +47,10 @@ public class BlockMinter extends Thread { // Recovery public static final long INVALID_BLOCK_RECOVERY_TIMEOUT = 10 * 60 * 1000L; // ms - // Min account level to submit blocks + // Min effective account level to submit blocks // This is an unvalidated version of Blockchain.minAccountLevelToMint // and exists only to reduce block candidates by default. - private static int MIN_LEVEL_FOR_BLOCK_SUBMISSION = 6; + private static int MIN_EFFECTIVE_LEVEL_FOR_BLOCK_SUBMISSION = 6; // Constructors @@ -138,13 +137,10 @@ public class BlockMinter extends Thread { } // Optional (non-validated) prevention of block submissions below a defined level - AccountData accountData = repository.getAccountRepository().getAccount(mintingAccount.getAddress()); - if (accountData != null) { - Integer level = accountData.getLevel(); - if (level != null && level < MIN_LEVEL_FOR_BLOCK_SUBMISSION) { - madi.remove(); - continue; - } + int level = mintingAccount.getEffectiveMintingLevel(); + if (level < MIN_EFFECTIVE_LEVEL_FOR_BLOCK_SUBMISSION) { + madi.remove(); + continue; } } From b1342d84fb640664685c8dfee1c4a336fbfa53a8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 28 Jan 2022 11:02:37 +0000 Subject: [PATCH 023/151] Updated potentially misleading log message. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index d5e489c8..1e3750cb 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -793,7 +793,7 @@ public class Synchronizer { return SynchronizationResult.REPOSITORY_ISSUE; if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { - LOGGER.info(String.format("Ditching our chain after height %d as our latest block is very old", commonBlockHeight)); + LOGGER.info(String.format("Ditching our chain after height %d", commonBlockHeight)); } else { // Compare chain weights From 72a291a54a8975212ea3156019c52079d0a8f365 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 28 Jan 2022 18:50:03 +0000 Subject: [PATCH 024/151] Added initial support of different unit fees per transaction type. Included a timestamp property, as this will be needed for each update (hard fork). --- .../org/qortal/transaction/Transaction.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index 69fb095d..f71c3e65 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -334,7 +334,7 @@ public abstract class Transaction { /** Returns whether transaction's fee is at least minimum unit fee as specified in blockchain config. */ public boolean hasMinimumFee() { - return this.transactionData.getFee() >= BlockChain.getInstance().getUnitFee(); + return this.transactionData.getFee() >= this.getUnitFee(this.transactionData.getTimestamp()); } public long feePerByte() { @@ -347,7 +347,7 @@ public abstract class Transaction { /** Returns whether transaction's fee is at least amount needed to cover byte-length of transaction. */ public boolean hasMinimumFeePerByte() { - long unitFee = BlockChain.getInstance().getUnitFee(); + long unitFee = this.getUnitFee(this.transactionData.getTimestamp()); int maxBytePerUnitFee = BlockChain.getInstance().getMaxBytesPerUnitFee(); // If the unit fee is zero, any fee is enough to cover the byte-length of the transaction @@ -369,7 +369,18 @@ public abstract class Transaction { int unitFeeCount = ((dataLength - 1) / maxBytePerUnitFee) + 1; - return BlockChain.getInstance().getUnitFee() * unitFeeCount; + return this.getUnitFee(this.transactionData.getTimestamp()) * unitFeeCount; + } + + /** + * Caclulate unit fee for a given transaction type + * + * FUTURE: add "accountLevel" parameter if needed - the level of the transaction creator + * @param timestamp - the transaction's timestamp (currently not used) + * @return + */ + public long getUnitFee(Long timestamp) { + return BlockChain.getInstance().getUnitFee(); } /** From be7bb2df9e5437468e2210ac095f4edc76160ad9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 28 Jan 2022 19:00:15 +0000 Subject: [PATCH 025/151] Added GET /transaction/unitfee API endpoint, to obtain the unit fee for a transaction type Additional params: - timestamp: to allow for hard forks. Default: the current time - level: the account level, to allow for the future possibility of different fees per level. Not currently used. --- .../api/resource/TransactionsResource.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/TransactionsResource.java b/src/main/java/org/qortal/api/resource/TransactionsResource.java index 30f242c4..f8a50ec7 100644 --- a/src/main/java/org/qortal/api/resource/TransactionsResource.java +++ b/src/main/java/org/qortal/api/resource/TransactionsResource.java @@ -9,6 +9,8 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; @@ -44,6 +46,7 @@ import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.utils.Base58; import com.google.common.primitives.Bytes; +import org.qortal.utils.NTP; @Path("/transactions") @Tag(name = "Transactions") @@ -363,6 +366,42 @@ public class TransactionsResource { } } + @GET + @Path("/unitfee") + @Operation( + summary = "Get transaction unit fee", + responses = { + @ApiResponse( + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "number" + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE + }) + public long getTransactionUnitFee(@QueryParam("txType") TransactionType txType, + @QueryParam("timestamp") Long timestamp, + @QueryParam("level") Integer accountLevel) { + try { + if (timestamp == null) { + timestamp = NTP.getTime(); + } + + Constructor constructor = txType.constructor; + Transaction transaction = (Transaction) constructor.newInstance(null, null); + // FUTURE: add accountLevel parameter to transaction.getUnitFee() if needed + return transaction.getUnitFee(timestamp); + + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e); + } + } + @GET @Path("/creator/{publickey}") @Operation( From 27387a134fb94454a0ba245e2aafdb167366fc61 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 28 Jan 2022 19:58:32 +0000 Subject: [PATCH 026/151] Fixed typo --- src/main/java/org/qortal/transaction/Transaction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index f71c3e65..74619013 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -373,7 +373,7 @@ public abstract class Transaction { } /** - * Caclulate unit fee for a given transaction type + * Calculate unit fee for a given transaction type * * FUTURE: add "accountLevel" parameter if needed - the level of the transaction creator * @param timestamp - the transaction's timestamp (currently not used) From fc1a376fbdcf8fb9a3ab42a2bdf465ff21bb35ea Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 29 Jan 2022 10:38:15 +0000 Subject: [PATCH 027/151] Added POST /transaction/fee API endpoint, to return the recommended fee for the supplied transaction data. --- .../api/resource/TransactionsResource.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/TransactionsResource.java b/src/main/java/org/qortal/api/resource/TransactionsResource.java index f8a50ec7..9bc6d497 100644 --- a/src/main/java/org/qortal/api/resource/TransactionsResource.java +++ b/src/main/java/org/qortal/api/resource/TransactionsResource.java @@ -402,6 +402,47 @@ public class TransactionsResource { } } + @POST + @Path("/fee") + @Operation( + summary = "Get recommended fee for supplied transaction data", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.TEXT_PLAIN, + schema = @Schema( + type = "string" + ) + ) + ) + ) + @ApiErrors({ + ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE + }) + public long getRecommendedTransactionFee(String rawInputBytes58) { + byte[] rawInputBytes = Base58.decode(rawInputBytes58); + if (rawInputBytes.length == 0) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.JSON); + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Append null signature on the end before transformation + byte[] rawBytes = Bytes.concat(rawInputBytes, new byte[TransactionTransformer.SIGNATURE_LENGTH]); + + TransactionData transactionData = TransactionTransformer.fromBytes(rawBytes); + if (transactionData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + Transaction transaction = Transaction.fromData(repository, transactionData); + return transaction.calcRecommendedFee(); + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } catch (TransformationException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); + } + } + @GET @Path("/creator/{publickey}") @Operation( From c5182a458993167e1adfef807ee204dea99a96d5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 29 Jan 2022 12:39:43 +0000 Subject: [PATCH 028/151] Increased MAX_ACCOUNT_COUNT in GetOnlineAccountsMessage from 1000 to 5000 --- .../org/qortal/network/message/GetOnlineAccountsMessage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java index 93f782df..23c21bc5 100644 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java @@ -15,7 +15,7 @@ import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; public class GetOnlineAccountsMessage extends Message { - private static final int MAX_ACCOUNT_COUNT = 1000; + private static final int MAX_ACCOUNT_COUNT = 5000; private List onlineAccounts; From c4f763960c68db698c80b1f3b2f5e8c4208f4dcd Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 29 Jan 2022 19:18:06 +0000 Subject: [PATCH 029/151] Don't delete a resource's cache if a build is in progress. Hopeful fix for "Unable to delete cache for resource: Unable to delete directory" error, and possibly some other file conflicts. --- .../java/org/qortal/arbitrary/ArbitraryDataResource.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 0ece14a5..7e00e0d0 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -128,6 +128,13 @@ public class ArbitraryDataResource { } public void deleteCache() throws IOException { + // Don't delete anything if there's a build in progress + ArbitraryDataBuildQueueItem queueItem = + new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier); + if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { + return; + } + String baseDir = Settings.getInstance().getTempDataPath(); String identifier = this.identifier != null ? this.identifier : "default"; Path cachePath = Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier); From 7aed0354f1e2926baa4257d4fbb2a04d14e195c2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 29 Jan 2022 19:24:28 +0000 Subject: [PATCH 030/151] We (currently) can't filter unconfirmed transactions by address, because only the public key is stored in the database until it is confirmed (at which point there is an entry in the TransactionParticipants table which contains the address). Given that this isn't a simple problem to solve, for now it makes sense to reject this combination if requested via the /transactions/search API. --- .../java/org/qortal/api/resource/TransactionsResource.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/api/resource/TransactionsResource.java b/src/main/java/org/qortal/api/resource/TransactionsResource.java index 9bc6d497..62e3bdc6 100644 --- a/src/main/java/org/qortal/api/resource/TransactionsResource.java +++ b/src/main/java/org/qortal/api/resource/TransactionsResource.java @@ -349,6 +349,10 @@ public class TransactionsResource { if (confirmationStatus != ConfirmationStatus.CONFIRMED && (startBlock != null || blockLimit != null)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + // You can't ask for unconfirmed and filter by address (due to only public key being stored when unconfirmed) + if (confirmationStatus != ConfirmationStatus.CONFIRMED && address != null && !address.isEmpty()) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); + try (final Repository repository = RepositoryManager.getRepository()) { List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(startBlock, blockLimit, txGroupId, txTypes, null, null, address, confirmationStatus, limit, offset, reverse); From d4ff7bbe4dcb6e548830e65dce3a6d30e1aa7530 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 29 Jan 2022 22:37:27 +0000 Subject: [PATCH 031/151] Fixed inaccurate README text that was accidentally merged from the data node repository. --- README.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/README.md b/README.md index e9001f9c..9dd9ad60 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,4 @@ -# Qortal Data Node - -## Important - -This code is unfinished, and we haven't had the official genesis block for the data chain yet. -Therefore it is only possible to use this code if you first create your own test chain. I would -highly recommend waiting until the code is in a more complete state before trying to run this. +# Qortal Project - Official Repo ## Build / run From a35e309a2f9928283fb611957742ee44ec51b63f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 29 Jan 2022 22:57:22 +0000 Subject: [PATCH 032/151] minAccountLevelForBlockSubmissions moved to blockchain.json --- src/main/java/org/qortal/block/BlockChain.java | 5 +++++ src/main/java/org/qortal/controller/BlockMinter.java | 11 ++++------- src/main/resources/blockchain.json | 1 + 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 7a6d6605..38e10f9b 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -142,6 +142,7 @@ public class BlockChain { private List blockTimingsByHeight; private int minAccountLevelToMint = 1; + private int minAccountLevelForBlockSubmissions; private int minAccountLevelToRewardShare; private int maxRewardSharesPerMintingAccount; private int founderEffectiveMintingLevel; @@ -344,6 +345,10 @@ public class BlockChain { return this.minAccountLevelToMint; } + public int getMinAccountLevelForBlockSubmissions() { + return this.minAccountLevelForBlockSubmissions; + } + public int getMinAccountLevelToRewardShare() { return this.minAccountLevelToRewardShare; } diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 5911d07c..428d5bf3 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -47,11 +47,6 @@ public class BlockMinter extends Thread { // Recovery public static final long INVALID_BLOCK_RECOVERY_TIMEOUT = 10 * 60 * 1000L; // ms - // Min effective account level to submit blocks - // This is an unvalidated version of Blockchain.minAccountLevelToMint - // and exists only to reduce block candidates by default. - private static int MIN_EFFECTIVE_LEVEL_FOR_BLOCK_SUBMISSION = 6; - // Constructors public BlockMinter() { @@ -136,9 +131,11 @@ public class BlockMinter extends Thread { continue; } - // Optional (non-validated) prevention of block submissions below a defined level + // Optional (non-validated) prevention of block submissions below a defined level. + // This is an unvalidated version of Blockchain.minAccountLevelToMint + // and exists only to reduce block candidates by default. int level = mintingAccount.getEffectiveMintingLevel(); - if (level < MIN_EFFECTIVE_LEVEL_FOR_BLOCK_SUBMISSION) { + if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) { madi.remove(); continue; } diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index acba90da..c142b6d8 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -8,6 +8,7 @@ "requireGroupForApproval": false, "defaultGroupId": 0, "oneNamePerAccount": true, + "minAccountLevelForBlockSubmissions": 6, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 6, "founderEffectiveMintingLevel": 10, From c5093168b1ad14a4dea412755d791872efbe690a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 29 Jan 2022 22:59:04 +0000 Subject: [PATCH 033/151] minAccountLevelToMint value moved to blockchain.json --- src/main/java/org/qortal/block/BlockChain.java | 2 +- src/main/resources/blockchain.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 38e10f9b..1ba65930 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -141,7 +141,7 @@ public class BlockChain { } private List blockTimingsByHeight; - private int minAccountLevelToMint = 1; + private int minAccountLevelToMint; private int minAccountLevelForBlockSubmissions; private int minAccountLevelToRewardShare; private int maxRewardSharesPerMintingAccount; diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index c142b6d8..742dd2f9 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -8,6 +8,7 @@ "requireGroupForApproval": false, "defaultGroupId": 0, "oneNamePerAccount": true, + "minAccountLevelToMint": 1, "minAccountLevelForBlockSubmissions": 6, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 6, From c73cdefe6f5023814697de53b46c70d5539e15d9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 13:47:20 +0000 Subject: [PATCH 034/151] transactionV5Timestamp moved to blockchain.json --- src/main/java/org/qortal/block/BlockChain.java | 7 ++++++- src/main/java/org/qortal/transaction/Transaction.java | 2 +- src/main/resources/blockchain.json | 3 ++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 1ba65930..defa9120 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -68,7 +68,8 @@ public class BlockChain { atFindNextTransactionFix, newBlockSigHeight, shareBinFix, - calcChainWeightTimestamp; + calcChainWeightTimestamp, + transactionV5Timestamp; } /** Map of which blockchain features are enabled when (height/timestamp) */ @@ -391,6 +392,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.calcChainWeightTimestamp.name()).longValue(); } + public long getTransactionV5Timestamp() { + return this.featureTriggers.get(FeatureTrigger.transactionV5Timestamp.name()).longValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index 74619013..79a6478b 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -393,7 +393,7 @@ public abstract class Transaction { * @return transaction version number */ public static int getVersionByTimestamp(long timestamp) { - if (timestamp >= 1642176000000L) { + if (timestamp >= BlockChain.getInstance().getTransactionV5Timestamp()) { return 5; } return 4; diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 742dd2f9..4df3f09a 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -53,7 +53,8 @@ "atFindNextTransactionFix": 275000, "newBlockSigHeight": 320000, "shareBinFix": 399000, - "calcChainWeightTimestamp": 1620579600000 + "calcChainWeightTimestamp": 1620579600000, + "transactionV5Timestamp": 1642176000000 }, "genesisInfo": { "version": 4, From 90f3d2568afa408353f9ce2290f15a92ba016fe3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 19:00:28 +0000 Subject: [PATCH 035/151] Log whenever the synchronizer can't obtain a blockchain lock, so that blockchain lock issues are more easily noticed. --- src/main/java/org/qortal/controller/Synchronizer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 1e3750cb..7eabcb9c 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -571,9 +571,11 @@ public class Synchronizer { // Make sure we're the only thread modifying the blockchain // If we're already synchronizing with another peer then this will also return fast ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock()) + if (!blockchainLock.tryLock()) { // Wasn't peer's fault we couldn't sync + LOGGER.info("Synchronizer couldn't acquire blockchain lock"); return SynchronizationResult.NO_BLOCKCHAIN_LOCK; + } try { try (final Repository repository = RepositoryManager.getRepository()) { From bd60c793bea54f3ef8f3a80308291b4db21755d2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 19:03:31 +0000 Subject: [PATCH 036/151] Incoming transactions are now added to a queue, and then processed soon after. This solves a problem where incoming transactions could rarely obtain a blockchain lock (due to multiple transactions arriving at once) and therefore most messages were thrown away. It was also causing constant blockchain locks to be acquired, which would often prevent the synchronizer from running. --- .../org/qortal/controller/Controller.java | 98 ++++++++++++------- 1 file changed, 64 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 2bfc80c2..90ac25d9 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -26,6 +26,7 @@ import java.util.Properties; import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantLock; @@ -101,6 +102,7 @@ public class Controller extends Thread { private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms + private static final int MAX_INCOMING_TRANSACTIONS = 5000; // To do with online accounts list private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms @@ -153,6 +155,9 @@ public class Controller extends Thread { /** Temporary estimate of synchronization progress for SysTray use. */ private volatile int syncPercent = 0; + /** List of incoming transaction that are in the import queue */ + private List incomingTransactions = Collections.synchronizedList(new ArrayList<>()); + /** Latest block signatures from other peers that we know are on inferior chains. */ List inferiorChainSignatures = new ArrayList<>(); @@ -584,6 +589,9 @@ public class Controller extends Thread { potentiallySynchronize(); } + // Process incoming transactions queue + processIncomingTransactionsQueue(); + // Clean up arbitrary data request cache ArbitraryDataManager.getInstance().cleanupRequestCache(now); // Clean up arbitrary data queues and lists @@ -1497,50 +1505,72 @@ public class Controller extends Thread { private void onNetworkTransactionMessage(Peer peer, Message message) { TransactionMessage transactionMessage = (TransactionMessage) message; TransactionData transactionData = transactionMessage.getTransactionData(); + if (this.incomingTransactions.size() < MAX_INCOMING_TRANSACTIONS) { + this.incomingTransactions.add(transactionData); + } + } - /* - * If we can't obtain blockchain lock immediately, - * e.g. Synchronizer is active, or another transaction is taking a while to validate, - * then we're using up a network thread for ages and clogging things up - * so bail out early - */ - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock()) { - LOGGER.trace(() -> String.format("Too busy to import %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer)); + private void processIncomingTransactionsQueue() { + try { + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + if (!blockchainLock.tryLock(2, TimeUnit.SECONDS)) { + LOGGER.info(() -> String.format("Too busy to process incoming transactions queue")); + return; + } + } catch (InterruptedException e) { + LOGGER.info("Interrupted when trying to acquire blockchain lock"); return; } try (final Repository repository = RepositoryManager.getRepository()) { - Transaction transaction = Transaction.fromData(repository, transactionData); - // Check signature - if (!transaction.isSignatureValid()) { - LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer)); - return; + // Iterate through incoming transactions list + synchronized (this.incomingTransactions) { // Required in order to safely iterate a synchronizedList() + Iterator iterator = this.incomingTransactions.iterator(); + while (iterator.hasNext()) { + if (isStopping) { + return; + } + + TransactionData transactionData = (TransactionData) iterator.next(); + Transaction transaction = Transaction.fromData(repository, transactionData); + + // Check signature + if (!transaction.isSignatureValid()) { + LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + iterator.remove(); + continue; + } + + ValidationResult validationResult = transaction.importAsUnconfirmed(); + + if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) { + LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature()))); + iterator.remove(); + continue; + } + + if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) { + LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction", Base58.encode(transactionData.getSignature()))); + iterator.remove(); + continue; + } + + if (validationResult != ValidationResult.OK) { + LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + iterator.remove(); + continue; + } + + LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + iterator.remove(); + } } - - ValidationResult validationResult = transaction.importAsUnconfirmed(); - - if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) { - LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer)); - return; - } - - if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) { - LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer)); - return; - } - - if (validationResult != ValidationResult.OK) { - LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s from peer %s", validationResult.name(), transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer)); - return; - } - - LOGGER.debug(() -> String.format("Imported %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer)); } catch (DataException e) { - LOGGER.error(String.format("Repository issue while processing transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer), e); + LOGGER.error(String.format("Repository issue while processing incoming transactions", e)); } finally { blockchainLock.unlock(); + LOGGER.info("[processIncomingTransactionsQueue] Released blockchain lock"); } } From 074bfadb2824ac290ba8aa9f32cb1db8c14a7414 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 19:50:09 +0000 Subject: [PATCH 037/151] Don't bother locking if there are no new transactions to process --- src/main/java/org/qortal/controller/Controller.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 90ac25d9..dbc81ecc 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1511,6 +1511,11 @@ public class Controller extends Thread { } private void processIncomingTransactionsQueue() { + if (this.incomingTransactions.size() == 0) { + // Don't bother locking if there are no new transactions to process + return; + } + try { ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); if (!blockchainLock.tryLock(2, TimeUnit.SECONDS)) { @@ -1570,7 +1575,6 @@ public class Controller extends Thread { LOGGER.error(String.format("Repository issue while processing incoming transactions", e)); } finally { blockchainLock.unlock(); - LOGGER.info("[processIncomingTransactionsQueue] Released blockchain lock"); } } From fa2bd40d5f49bdd0810070b5b69bf3e0cafab2ea Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 19:51:49 +0000 Subject: [PATCH 038/151] Reduce log spam --- src/main/java/org/qortal/controller/Controller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index dbc81ecc..1113cba0 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1519,7 +1519,7 @@ public class Controller extends Thread { try { ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); if (!blockchainLock.tryLock(2, TimeUnit.SECONDS)) { - LOGGER.info(() -> String.format("Too busy to process incoming transactions queue")); + LOGGER.trace(() -> String.format("Too busy to process incoming transactions queue")); return; } } catch (InterruptedException e) { From 5b788dad2f0fadb761a037854e77e8a904946f5c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:01:22 +0000 Subject: [PATCH 039/151] Relocated all synchronizer code from the controller to the synchronizer, and also moved synchonization onto its own thread. --- .../java/org/qortal/api/model/NodeStatus.java | 3 +- .../qortal/api/resource/AdminResource.java | 3 +- .../org/qortal/controller/BlockMinter.java | 4 +- .../org/qortal/controller/Controller.java | 291 ++---------------- .../org/qortal/controller/Synchronizer.java | 284 ++++++++++++++++- .../controller/repository/AtStatesPruner.java | 3 +- .../repository/AtStatesTrimmer.java | 3 +- .../controller/repository/BlockArchiver.java | 3 +- .../controller/repository/BlockPruner.java | 3 +- .../OnlineAccountsSignaturesTrimmer.java | 3 +- .../qortal/repository/BlockArchiveWriter.java | 3 +- 11 files changed, 325 insertions(+), 278 deletions(-) diff --git a/src/main/java/org/qortal/api/model/NodeStatus.java b/src/main/java/org/qortal/api/model/NodeStatus.java index be112bc3..ccc1eb01 100644 --- a/src/main/java/org/qortal/api/model/NodeStatus.java +++ b/src/main/java/org/qortal/api/model/NodeStatus.java @@ -4,6 +4,7 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.network.Network; @XmlAccessorType(XmlAccessType.FIELD) @@ -22,7 +23,7 @@ public class NodeStatus { public NodeStatus() { this.isMintingPossible = Controller.getInstance().isMintingPossible(); - this.syncPercent = Controller.getInstance().getSyncPercent(); + this.syncPercent = Synchronizer.getInstance().getSyncPercent(); this.isSynchronizing = this.syncPercent != null; this.numberOfConnections = Network.getInstance().getHandshakedPeers().size(); diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index 8d00c751..bde4bed4 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -44,6 +44,7 @@ import org.qortal.api.model.NodeInfo; import org.qortal.api.model.NodeStatus; import org.qortal.block.BlockChain; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.controller.Synchronizer.SynchronizationResult; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; @@ -525,7 +526,7 @@ public class AdminResource { SynchronizationResult syncResult; try { do { - syncResult = Controller.getInstance().actuallySynchronize(targetPeer, true); + syncResult = Synchronizer.getInstance().actuallySynchronize(targetPeer, true); } while (syncResult == SynchronizationResult.OK); } finally { blockchainLock.unlock(); diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 428d5bf3..616fd611 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -149,7 +149,7 @@ public class BlockMinter extends Thread { // Disregard peers that don't have a recent block, but only if we're not in recovery mode. // In that mode, we want to allow minting on top of older blocks, to recover stalled networks. - if (Controller.getInstance().getRecoveryMode() == false) + if (Synchronizer.getInstance().getRecoveryMode() == false) peers.removeIf(Controller.hasNoRecentBlock); // Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from? @@ -174,7 +174,7 @@ public class BlockMinter extends Thread { // If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode. if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp) - if (Controller.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false) + if (Synchronizer.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false) continue; // There are enough peers with a recent block and our latest block is recent diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 1113cba0..f22a3259 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.SecureRandom; import java.security.Security; import java.time.LocalDateTime; import java.time.ZoneOffset; @@ -52,7 +51,6 @@ import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.arbitrary.*; -import org.qortal.controller.Synchronizer.SynchronizationResult; import org.qortal.controller.repository.PruneManager; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.controller.tradebot.TradeBot; @@ -94,14 +92,13 @@ public class Controller extends Thread { public static final String VERSION_PREFIX = "qortal-"; private static final Logger LOGGER = LogManager.getLogger(Controller.class); - private static final long MISBEHAVIOUR_COOLOFF = 10 * 60 * 1000L; // ms + public static final long MISBEHAVIOUR_COOLOFF = 10 * 60 * 1000L; // ms private static final int MAX_BLOCKCHAIN_TIP_AGE = 5; // blocks private static final Object shutdownLock = new Object(); private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s" + File.separator + "blockchain;create=true;hsqldb.full_log_replay=true"; private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms - private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms private static final int MAX_INCOMING_TRANSACTIONS = 5000; // To do with online accounts list @@ -114,7 +111,6 @@ public class Controller extends Thread { private static volatile boolean isStopping = false; private static BlockMinter blockMinter = null; - private static volatile boolean requestSync = false; private static volatile boolean requestSysTrayUpdate = true; private static Controller instance; @@ -148,24 +144,9 @@ public class Controller extends Thread { /** Whether we can mint new blocks, as reported by BlockMinter. */ private volatile boolean isMintingPossible = false; - /** Synchronization object for sync variables below */ - private final Object syncLock = new Object(); - /** Whether we are attempting to synchronize. */ - private volatile boolean isSynchronizing = false; - /** Temporary estimate of synchronization progress for SysTray use. */ - private volatile int syncPercent = 0; - /** List of incoming transaction that are in the import queue */ private List incomingTransactions = Collections.synchronizedList(new ArrayList<>()); - /** Latest block signatures from other peers that we know are on inferior chains. */ - List inferiorChainSignatures = new ArrayList<>(); - - /** Recovery mode, which is used to bring back a stalled network */ - private boolean recoveryMode = false; - private boolean peersAvailable = true; // peersAvailable must default to true - private long timePeersLastAvailable = 0; - /** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly minted block. */ private final ReentrantLock blockchainLock = new ReentrantLock(); @@ -354,20 +335,6 @@ public class Controller extends Thread { return this.isMintingPossible; } - public boolean isSynchronizing() { - return this.isSynchronizing; - } - - public Integer getSyncPercent() { - synchronized (this.syncLock) { - return this.isSynchronizing ? this.syncPercent : null; - } - } - - public boolean getRecoveryMode() { - return this.recoveryMode; - } - // Entry point public static void main(String[] args) { @@ -472,6 +439,9 @@ public class Controller extends Thread { } }); + LOGGER.info("Starting synchronizer"); + Synchronizer.getInstance().start(); + LOGGER.info("Starting block minter"); blockMinter = new BlockMinter(); blockMinter.start(); @@ -584,11 +554,6 @@ public class Controller extends Thread { } } - if (requestSync) { - requestSync = false; - potentiallySynchronize(); - } - // Process incoming transactions queue processIncomingTransactionsQueue(); @@ -716,27 +681,6 @@ public class Controller extends Thread { } } - private long getRandomRepositoryMaintenanceInterval() { - final long minInterval = Settings.getInstance().getRepositoryMaintenanceMinInterval(); - final long maxInterval = Settings.getInstance().getRepositoryMaintenanceMaxInterval(); - if (maxInterval == 0) { - return 0; - } - return (new Random().nextLong() % (maxInterval - minInterval)) + minInterval; - } - - /** - * Export current trade bot states and minting accounts. - */ - public void exportRepositoryData() { - try (final Repository repository = RepositoryManager.getRepository()) { - repository.exportNodeLocalData(); - - } catch (DataException e) { - // Fail silently as this is an optional step - } - } - public static final Predicate hasMisbehaved = peer -> { final Long lastMisbehaved = peer.getPeerData().getLastMisbehaved(); return lastMisbehaved != null && lastMisbehaved > NTP.getTime() - MISBEHAVIOUR_COOLOFF; @@ -761,7 +705,7 @@ public class Controller extends Thread { public static final Predicate hasInferiorChainTip = peer -> { final PeerChainTipData peerChainTipData = peer.getChainTipData(); - final List inferiorChainTips = getInstance().inferiorChainSignatures; + final List inferiorChainTips = Synchronizer.getInstance().inferiorChainSignatures; return peerChainTipData == null || peerChainTipData.getLastBlockSignature() == null || inferiorChainTips.contains(new ByteArray(peerChainTipData.getLastBlockSignature())); }; @@ -770,218 +714,34 @@ public class Controller extends Thread { return peer.isAtLeastVersion(minPeerVersion) == false; }; - private void potentiallySynchronize() throws InterruptedException { - // Already synchronizing via another thread? - if (this.isSynchronizing) - return; - - List peers = Network.getInstance().getHandshakedPeers(); - - // Disregard peers that have "misbehaved" recently - peers.removeIf(hasMisbehaved); - - // Disregard peers that only have genesis block - peers.removeIf(hasOnlyGenesisBlock); - - // Disregard peers that don't have a recent block - peers.removeIf(hasNoRecentBlock); - - // Disregard peers that are on an old version - peers.removeIf(hasOldVersion); - - checkRecoveryModeForPeers(peers); - if (recoveryMode) { - peers = Network.getInstance().getHandshakedPeers(); - peers.removeIf(hasOnlyGenesisBlock); - peers.removeIf(hasMisbehaved); - peers.removeIf(hasOldVersion); + private long getRandomRepositoryMaintenanceInterval() { + final long minInterval = Settings.getInstance().getRepositoryMaintenanceMinInterval(); + final long maxInterval = Settings.getInstance().getRepositoryMaintenanceMaxInterval(); + if (maxInterval == 0) { + return 0; } - - // Check we have enough peers to potentially synchronize - if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) - return; - - // Disregard peers that have no block signature or the same block signature as us - peers.removeIf(hasNoOrSameBlock); - - // Disregard peers that are on the same block as last sync attempt and we didn't like their chain - peers.removeIf(hasInferiorChainTip); - - final int peersBeforeComparison = peers.size(); - - // Request recent block summaries from the remaining peers, and locate our common block with each - Synchronizer.getInstance().findCommonBlocksWithPeers(peers); - - // Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks - peers = Synchronizer.getInstance().comparePeers(peers); - - // We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains - peers.removeIf(hasInferiorChainTip); - - final int peersRemoved = peersBeforeComparison - peers.size(); - if (peersRemoved > 0 && peers.size() > 0) - LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size())); - - if (peers.isEmpty()) - return; - - if (peers.size() > 1) { - StringBuilder finalPeersString = new StringBuilder(); - for (Peer peer : peers) - finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer); - LOGGER.debug(String.format("Choosing random peer from: [%s]", finalPeersString.toString())); - } - - // Pick random peer to sync with - int index = new SecureRandom().nextInt(peers.size()); - Peer peer = peers.get(index); - - actuallySynchronize(peer, false); + return (new Random().nextLong() % (maxInterval - minInterval)) + minInterval; } - public SynchronizationResult actuallySynchronize(Peer peer, boolean force) throws InterruptedException { - boolean hasStatusChanged = false; - BlockData priorChainTip = this.getChainTip(); + /** + * Export current trade bot states and minting accounts. + */ + public void exportRepositoryData() { + try (final Repository repository = RepositoryManager.getRepository()) { + repository.exportNodeLocalData(); - synchronized (this.syncLock) { - this.syncPercent = (priorChainTip.getHeight() * 100) / peer.getChainTipData().getLastHeight(); - - // Only update SysTray if we're potentially changing height - if (this.syncPercent < 100) { - this.isSynchronizing = true; - hasStatusChanged = true; - } - } - peer.setSyncInProgress(true); - - if (hasStatusChanged) - updateSysTray(); - - try { - SynchronizationResult syncResult = Synchronizer.getInstance().synchronize(peer, force); - switch (syncResult) { - case GENESIS_ONLY: - case NO_COMMON_BLOCK: - case TOO_DIVERGENT: - case INVALID_DATA: { - // These are more serious results that warrant a cool-off - LOGGER.info(String.format("Failed to synchronize with peer %s (%s) - cooling off", peer, syncResult.name())); - - // Don't use this peer again for a while - Network.getInstance().peerMisbehaved(peer); - break; - } - - case INFERIOR_CHAIN: { - // Update our list of inferior chain tips - ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature()); - if (!inferiorChainSignatures.contains(inferiorChainSignature)) - inferiorChainSignatures.add(inferiorChainSignature); - - // These are minor failure results so fine to try again - LOGGER.debug(() -> String.format("Refused to synchronize with peer %s (%s)", peer, syncResult.name())); - - // Notify peer of our superior chain - if (!peer.sendMessage(Network.getInstance().buildHeightMessage(peer, priorChainTip))) - peer.disconnect("failed to notify peer of our superior chain"); - break; - } - - case NO_REPLY: - case NO_BLOCKCHAIN_LOCK: - case REPOSITORY_ISSUE: - // These are minor failure results so fine to try again - LOGGER.debug(() -> String.format("Failed to synchronize with peer %s (%s)", peer, syncResult.name())); - break; - - case SHUTTING_DOWN: - // Just quietly exit - break; - - case OK: - // fall-through... - case NOTHING_TO_DO: { - // Update our list of inferior chain tips - ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature()); - if (!inferiorChainSignatures.contains(inferiorChainSignature)) - inferiorChainSignatures.add(inferiorChainSignature); - - LOGGER.debug(() -> String.format("Synchronized with peer %s (%s)", peer, syncResult.name())); - break; - } - } - - // Has our chain tip changed? - BlockData newChainTip; - - try (final Repository repository = RepositoryManager.getRepository()) { - newChainTip = repository.getBlockRepository().getLastBlock(); - } catch (DataException e) { - LOGGER.warn(String.format("Repository issue when trying to fetch post-synchronization chain tip: %s", e.getMessage())); - return syncResult; - } - - if (!Arrays.equals(newChainTip.getSignature(), priorChainTip.getSignature())) { - // Reset our cache of inferior chains - inferiorChainSignatures.clear(); - - Network network = Network.getInstance(); - network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip)); - } - - return syncResult; - } finally { - isSynchronizing = false; - peer.setSyncInProgress(false); + } catch (DataException e) { + // Fail silently as this is an optional step } } - private boolean checkRecoveryModeForPeers(List qualifiedPeers) { - List handshakedPeers = Network.getInstance().getHandshakedPeers(); - - if (handshakedPeers.size() > 0) { - // There is at least one handshaked peer - if (qualifiedPeers.isEmpty()) { - // There are no 'qualified' peers - i.e. peers that have a recent block we can sync to - boolean werePeersAvailable = peersAvailable; - peersAvailable = false; - - // If peers only just became unavailable, update our record of the time they were last available - if (werePeersAvailable) - timePeersLastAvailable = NTP.getTime(); - - // If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint - if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) { - if (recoveryMode == false) { - LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000)); - recoveryMode = true; - } - } - } else { - // We now have at least one peer with a recent block, so we can exit recovery mode and sync normally - peersAvailable = true; - if (recoveryMode) { - LOGGER.info("Peers have become available again. Exiting recovery mode..."); - recoveryMode = false; - } - } - } - return recoveryMode; - } - - public void addInferiorChainSignature(byte[] inferiorSignature) { - // Update our list of inferior chain tips - ByteArray inferiorChainSignature = new ByteArray(inferiorSignature); - if (!inferiorChainSignatures.contains(inferiorChainSignature)) - inferiorChainSignatures.add(inferiorChainSignature); - } public static class StatusChangeEvent implements Event { public StatusChangeEvent() { } } - private void updateSysTray() { + public void updateSysTray() { if (NTP.getTime() == null) { SysTray.getInstance().setToolTipText(Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_CLOCK")); SysTray.getInstance().setTrayIcon(1); @@ -997,13 +757,13 @@ public class Controller extends Thread { String actionText; - synchronized (this.syncLock) { + synchronized (Synchronizer.getInstance().syncLock) { if (this.isMintingPossible) { actionText = Translator.INSTANCE.translate("SysTray", "MINTING_ENABLED"); SysTray.getInstance().setTrayIcon(2); } - else if (this.isSynchronizing) { - actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), this.syncPercent); + else if (Synchronizer.getInstance().isSynchronizing()) { + actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent()); SysTray.getInstance().setTrayIcon(3); } else if (numberOfPeers < Settings.getInstance().getMinBlockchainPeers()) { @@ -1092,6 +852,9 @@ public class Controller extends Thread { } } + LOGGER.info("Shutting down synchronizer"); + Synchronizer.getInstance().shutdown(); + // Export local data LOGGER.info("Backing up local data"); this.exportRepositoryData(); @@ -1726,7 +1489,7 @@ public class Controller extends Thread { peer.setChainTipData(newChainTipData); // Potentially synchronize - requestSync = true; + Synchronizer.getInstance().requestSync(); } private void onNetworkGetTransactionMessage(Peer peer, Message message) { diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 7eabcb9c..75bf0691 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -1,6 +1,7 @@ package org.qortal.controller; import java.math.BigInteger; +import java.security.SecureRandom; import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.*; @@ -20,6 +21,7 @@ import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.transaction.RewardShareTransactionData; import org.qortal.data.transaction.TransactionData; +import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.message.BlockMessage; import org.qortal.network.message.BlockSummariesMessage; @@ -35,11 +37,10 @@ import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.transaction.Transaction; import org.qortal.utils.Base58; +import org.qortal.utils.ByteArray; import org.qortal.utils.NTP; -import static org.qortal.network.Peer.FETCH_BLOCKS_TIMEOUT; - -public class Synchronizer { +public class Synchronizer extends Thread { private static final Logger LOGGER = LogManager.getLogger(Synchronizer.class); @@ -57,12 +58,31 @@ public class Synchronizer { /** Maximum number of block signatures we ask from peer in one go */ private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? + private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms + private boolean running; + + /** Latest block signatures from other peers that we know are on inferior chains. */ + List inferiorChainSignatures = new ArrayList<>(); + + /** Recovery mode, which is used to bring back a stalled network */ + private boolean recoveryMode = false; + private boolean peersAvailable = true; // peersAvailable must default to true + private long timePeersLastAvailable = 0; // Keep track of the size of the last re-org, so it can be logged private int lastReorgSize; + /** Synchronization object for sync variables below */ + public final Object syncLock = new Object(); + /** Whether we are attempting to synchronize. */ + private volatile boolean isSynchronizing = false; + /** Temporary estimate of synchronization progress for SysTray use. */ + private volatile int syncPercent = 0; + + private static volatile boolean requestSync = false; + // Keep track of invalid blocks so that we don't keep trying to sync them private Map invalidBlockSignatures = Collections.synchronizedMap(new HashMap<>()); public Long timeValidBlockLastReceived = null; @@ -77,6 +97,7 @@ public class Synchronizer { // Constructors private Synchronizer() { + this.running = true; } public static Synchronizer getInstance() { @@ -87,6 +108,261 @@ public class Synchronizer { } + @Override + public void run() { + try { + while (running) { + Thread.sleep(1000); + + if (requestSync) { + requestSync = false; + Synchronizer.getInstance().potentiallySynchronize(); + } + } + } catch (InterruptedException e) { + // Clear interrupted flag so we can shutdown trim threads + Thread.interrupted(); + // Fall-through to exit + } + } + + public void shutdown() { + this.running = false; + this.interrupt(); + } + + + + public boolean isSynchronizing() { + return this.isSynchronizing; + } + + public Integer getSyncPercent() { + synchronized (this.syncLock) { + return this.isSynchronizing ? this.syncPercent : null; + } + } + + public void requestSync() { + requestSync = true; + } + + public boolean isSyncRequested() { + return requestSync; + } + + public boolean getRecoveryMode() { + return this.recoveryMode; + } + + + public void potentiallySynchronize() throws InterruptedException { + // Already synchronizing via another thread? + if (this.isSynchronizing) + return; + + List peers = Network.getInstance().getHandshakedPeers(); + + // Disregard peers that have "misbehaved" recently + peers.removeIf(Controller.hasMisbehaved); + + // Disregard peers that only have genesis block + peers.removeIf(Controller.hasOnlyGenesisBlock); + + // Disregard peers that don't have a recent block + peers.removeIf(Controller.hasNoRecentBlock); + + // Disregard peers that are on an old version + peers.removeIf(Controller.hasOldVersion); + + checkRecoveryModeForPeers(peers); + if (recoveryMode) { + peers = Network.getInstance().getHandshakedPeers(); + peers.removeIf(Controller.hasOnlyGenesisBlock); + peers.removeIf(Controller.hasMisbehaved); + peers.removeIf(Controller.hasOldVersion); + } + + // Check we have enough peers to potentially synchronize + if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) + return; + + // Disregard peers that have no block signature or the same block signature as us + peers.removeIf(Controller.hasNoOrSameBlock); + + // Disregard peers that are on the same block as last sync attempt and we didn't like their chain + peers.removeIf(Controller.hasInferiorChainTip); + + final int peersBeforeComparison = peers.size(); + + // Request recent block summaries from the remaining peers, and locate our common block with each + Synchronizer.getInstance().findCommonBlocksWithPeers(peers); + + // Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks + peers = Synchronizer.getInstance().comparePeers(peers); + + // We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains + peers.removeIf(Controller.hasInferiorChainTip); + + final int peersRemoved = peersBeforeComparison - peers.size(); + if (peersRemoved > 0 && peers.size() > 0) + LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size())); + + if (peers.isEmpty()) + return; + + if (peers.size() > 1) { + StringBuilder finalPeersString = new StringBuilder(); + for (Peer peer : peers) + finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer); + LOGGER.debug(String.format("Choosing random peer from: [%s]", finalPeersString.toString())); + } + + // Pick random peer to sync with + int index = new SecureRandom().nextInt(peers.size()); + Peer peer = peers.get(index); + + actuallySynchronize(peer, false); + } + + public SynchronizationResult actuallySynchronize(Peer peer, boolean force) throws InterruptedException { + boolean hasStatusChanged = false; + BlockData priorChainTip = Controller.getInstance().getChainTip(); + + synchronized (this.syncLock) { + this.syncPercent = (priorChainTip.getHeight() * 100) / peer.getChainTipData().getLastHeight(); + + // Only update SysTray if we're potentially changing height + if (this.syncPercent < 100) { + this.isSynchronizing = true; + hasStatusChanged = true; + } + } + peer.setSyncInProgress(true); + + if (hasStatusChanged) + Controller.getInstance().updateSysTray(); + + try { + SynchronizationResult syncResult = Synchronizer.getInstance().synchronize(peer, force); + switch (syncResult) { + case GENESIS_ONLY: + case NO_COMMON_BLOCK: + case TOO_DIVERGENT: + case INVALID_DATA: { + // These are more serious results that warrant a cool-off + LOGGER.info(String.format("Failed to synchronize with peer %s (%s) - cooling off", peer, syncResult.name())); + + // Don't use this peer again for a while + Network.getInstance().peerMisbehaved(peer); + break; + } + + case INFERIOR_CHAIN: { + // Update our list of inferior chain tips + ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature()); + if (!inferiorChainSignatures.contains(inferiorChainSignature)) + inferiorChainSignatures.add(inferiorChainSignature); + + // These are minor failure results so fine to try again + LOGGER.debug(() -> String.format("Refused to synchronize with peer %s (%s)", peer, syncResult.name())); + + // Notify peer of our superior chain + if (!peer.sendMessage(Network.getInstance().buildHeightMessage(peer, priorChainTip))) + peer.disconnect("failed to notify peer of our superior chain"); + break; + } + + case NO_REPLY: + case NO_BLOCKCHAIN_LOCK: + case REPOSITORY_ISSUE: + // These are minor failure results so fine to try again + LOGGER.debug(() -> String.format("Failed to synchronize with peer %s (%s)", peer, syncResult.name())); + break; + + case SHUTTING_DOWN: + // Just quietly exit + break; + + case OK: + // fall-through... + case NOTHING_TO_DO: { + // Update our list of inferior chain tips + ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature()); + if (!inferiorChainSignatures.contains(inferiorChainSignature)) + inferiorChainSignatures.add(inferiorChainSignature); + + LOGGER.debug(() -> String.format("Synchronized with peer %s (%s)", peer, syncResult.name())); + break; + } + } + + // Has our chain tip changed? + BlockData newChainTip; + + try (final Repository repository = RepositoryManager.getRepository()) { + newChainTip = repository.getBlockRepository().getLastBlock(); + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue when trying to fetch post-synchronization chain tip: %s", e.getMessage())); + return syncResult; + } + + if (!Arrays.equals(newChainTip.getSignature(), priorChainTip.getSignature())) { + // Reset our cache of inferior chains + inferiorChainSignatures.clear(); + + Network network = Network.getInstance(); + network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip)); + } + + return syncResult; + } finally { + this.isSynchronizing = false; + peer.setSyncInProgress(false); + } + } + + private boolean checkRecoveryModeForPeers(List qualifiedPeers) { + List handshakedPeers = Network.getInstance().getHandshakedPeers(); + + if (handshakedPeers.size() > 0) { + // There is at least one handshaked peer + if (qualifiedPeers.isEmpty()) { + // There are no 'qualified' peers - i.e. peers that have a recent block we can sync to + boolean werePeersAvailable = peersAvailable; + peersAvailable = false; + + // If peers only just became unavailable, update our record of the time they were last available + if (werePeersAvailable) + timePeersLastAvailable = NTP.getTime(); + + // If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint + if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) { + if (recoveryMode == false) { + LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000)); + recoveryMode = true; + } + } + } else { + // We now have at least one peer with a recent block, so we can exit recovery mode and sync normally + peersAvailable = true; + if (recoveryMode) { + LOGGER.info("Peers have become available again. Exiting recovery mode..."); + recoveryMode = false; + } + } + } + return recoveryMode; + } + + public void addInferiorChainSignature(byte[] inferiorSignature) { + // Update our list of inferior chain tips + ByteArray inferiorChainSignature = new ByteArray(inferiorSignature); + if (!inferiorChainSignatures.contains(inferiorChainSignature)) + inferiorChainSignatures.add(inferiorChainSignature); + } + + /** * Iterate through a list of supplied peers, and attempt to find our common block with each. * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later. @@ -279,7 +555,7 @@ public class Synchronizer { // We have already determined that the correct chain diverged from a lower height. We are safe to skip these peers. for (Peer peer : peersSharingCommonBlock) { LOGGER.debug(String.format("Peer %s has common block at height %d but the superior chain is at height %d. Removing it from this round.", peer, commonBlockSummary.getHeight(), dropPeersAfterCommonBlockHeight)); - Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature()); + this.addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature()); } continue; } diff --git a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java index 3b92db51..54fba699 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java @@ -3,6 +3,7 @@ package org.qortal.controller.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -47,7 +48,7 @@ public class AtStatesPruner implements Runnable { continue; // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages - if (Controller.getInstance().isSynchronizing()) + if (Synchronizer.getInstance().isSynchronizing()) continue; // Prune AT states for all blocks up until our latest minus pruneBlockLimit diff --git a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java index 98a1a889..d3bdc345 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java @@ -3,6 +3,7 @@ package org.qortal.controller.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -34,7 +35,7 @@ public class AtStatesTrimmer implements Runnable { continue; // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages - if (Controller.getInstance().isSynchronizing()) + if (Synchronizer.getInstance().isSynchronizing()) continue; long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime(); diff --git a/src/main/java/org/qortal/controller/repository/BlockArchiver.java b/src/main/java/org/qortal/controller/repository/BlockArchiver.java index a329e912..ef26610c 100644 --- a/src/main/java/org/qortal/controller/repository/BlockArchiver.java +++ b/src/main/java/org/qortal/controller/repository/BlockArchiver.java @@ -3,6 +3,7 @@ package org.qortal.controller.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.data.block.BlockData; import org.qortal.repository.*; import org.qortal.settings.Settings; @@ -51,7 +52,7 @@ public class BlockArchiver implements Runnable { } // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages - if (Controller.getInstance().isSynchronizing()) { + if (Synchronizer.getInstance().isSynchronizing()) { continue; } diff --git a/src/main/java/org/qortal/controller/repository/BlockPruner.java b/src/main/java/org/qortal/controller/repository/BlockPruner.java index 1258ee38..03fb38b9 100644 --- a/src/main/java/org/qortal/controller/repository/BlockPruner.java +++ b/src/main/java/org/qortal/controller/repository/BlockPruner.java @@ -3,6 +3,7 @@ package org.qortal.controller.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -51,7 +52,7 @@ public class BlockPruner implements Runnable { continue; // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages - if (Controller.getInstance().isSynchronizing()) { + if (Synchronizer.getInstance().isSynchronizing()) { continue; } diff --git a/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java index c7f248d5..dfd9d45e 100644 --- a/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.block.BlockChain; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.data.block.BlockData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -36,7 +37,7 @@ public class OnlineAccountsSignaturesTrimmer implements Runnable { continue; // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages - if (Controller.getInstance().isSynchronizing()) + if (Synchronizer.getInstance().isSynchronizing()) continue; // Trim blockchain by removing 'old' online accounts signatures diff --git a/src/main/java/org/qortal/repository/BlockArchiveWriter.java b/src/main/java/org/qortal/repository/BlockArchiveWriter.java index 39c28fd6..5127bf9b 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveWriter.java +++ b/src/main/java/org/qortal/repository/BlockArchiveWriter.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.block.Block; import org.qortal.controller.Controller; +import org.qortal.controller.Synchronizer; import org.qortal.data.block.BlockArchiveData; import org.qortal.data.block.BlockData; import org.qortal.settings.Settings; @@ -100,7 +101,7 @@ public class BlockArchiveWriter { if (Controller.isStopping()) { return BlockArchiveWriteResult.STOPPING; } - if (Controller.getInstance().isSynchronizing()) { + if (Synchronizer.getInstance().isSynchronizing()) { continue; } From 8a1fb6fe4e5e0b8afabb45bb7f54c074a48a06e1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:09:09 +0000 Subject: [PATCH 040/151] If a lock can't be obtained when synchronizing, automatically request the sync again. --- .../org/qortal/controller/Synchronizer.java | 22 ++++++++++++++----- 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 75bf0691..0cb0fdda 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -116,7 +116,11 @@ public class Synchronizer extends Thread { if (requestSync) { requestSync = false; - Synchronizer.getInstance().potentiallySynchronize(); + boolean success = Synchronizer.getInstance().potentiallySynchronize(); + if (!success) { + // Something went wrong, so try again next time + requestSync = true; + } } } } catch (InterruptedException e) { @@ -156,10 +160,10 @@ public class Synchronizer extends Thread { } - public void potentiallySynchronize() throws InterruptedException { + public boolean potentiallySynchronize() throws InterruptedException { // Already synchronizing via another thread? if (this.isSynchronizing) - return; + return true; List peers = Network.getInstance().getHandshakedPeers(); @@ -185,7 +189,7 @@ public class Synchronizer extends Thread { // Check we have enough peers to potentially synchronize if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) - return; + return true; // Disregard peers that have no block signature or the same block signature as us peers.removeIf(Controller.hasNoOrSameBlock); @@ -209,7 +213,7 @@ public class Synchronizer extends Thread { LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size())); if (peers.isEmpty()) - return; + return true; if (peers.size() > 1) { StringBuilder finalPeersString = new StringBuilder(); @@ -222,7 +226,13 @@ public class Synchronizer extends Thread { int index = new SecureRandom().nextInt(peers.size()); Peer peer = peers.get(index); - actuallySynchronize(peer, false); + SynchronizationResult syncResult = actuallySynchronize(peer, false); + if (syncResult == SynchronizationResult.NO_BLOCKCHAIN_LOCK) { + // No blockchain lock - force a retry by returning false + return false; + } + + return true; } public SynchronizationResult actuallySynchronize(Peer peer, boolean force) throws InterruptedException { From 5700369935ec8be759a363eeff7f553b7b598345 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:09:35 +0000 Subject: [PATCH 041/151] Prioritize syncing over transaction importing. --- src/main/java/org/qortal/controller/Controller.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index f22a3259..3bb3052b 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1279,6 +1279,11 @@ public class Controller extends Thread { return; } + if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) { + // Prioritize syncing, and don't attempt to lock + return; + } + try { ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); if (!blockchainLock.tryLock(2, TimeUnit.SECONDS)) { From f005a0975d646d9656a304978af6fe8c6ce4f8da Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:14:11 +0000 Subject: [PATCH 042/151] Shut down the synchronizer as soon as the controller is stopping, if we are able to. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 0cb0fdda..1b3285c2 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -111,7 +111,7 @@ public class Synchronizer extends Thread { @Override public void run() { try { - while (running) { + while (running && !Controller.isStopping()) { Thread.sleep(1000); if (requestSync) { From e2e87766faa0676a6bd334572743f2951918a379 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:15:43 +0000 Subject: [PATCH 043/151] Moved log from INFO to DEBUG, as now the synchronizer is on its own thread it can occur more often than before. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 1b3285c2..14d1a071 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -859,7 +859,7 @@ public class Synchronizer extends Thread { ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); if (!blockchainLock.tryLock()) { // Wasn't peer's fault we couldn't sync - LOGGER.info("Synchronizer couldn't acquire blockchain lock"); + LOGGER.debug("Synchronizer couldn't acquire blockchain lock"); return SynchronizationResult.NO_BLOCKCHAIN_LOCK; } From b198a8ea07249e04262a2564d1032dfb44500adf Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 31 Jan 2022 21:38:51 +0000 Subject: [PATCH 044/151] Fixed issue in ArbitraryDataFile.chunkExists() due to it not checking for the metadata file. --- src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 1eaeda3c..cc667cda 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -439,6 +439,11 @@ public class ArbitraryDataFile { return chunk.exists(); } } + if (Arrays.equals(hash, this.metadataHash)) { + if (this.metadataFile != null) { + return this.metadataFile.exists(); + } + } if (Arrays.equals(this.getHash(), hash)) { return this.exists(); } From 01e4bf3a772a6e60c6cd765eb4186c9fd603eeec Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 31 Jan 2022 21:39:20 +0000 Subject: [PATCH 045/151] Try to speed up the shutdown process of the cleanup manager. --- .../controller/arbitrary/ArbitraryDataCleanupManager.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 8c263568..e1eaa491 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -108,6 +108,10 @@ public class ArbitraryDataCleanupManager extends Thread { try (final Repository repository = RepositoryManager.getRepository()) { List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true); // LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit); + if (isStopping) { + return; + } + if (signatures == null || signatures.isEmpty()) { offset = 0; continue; @@ -117,6 +121,10 @@ public class ArbitraryDataCleanupManager extends Thread { // Loop through the signatures in this batch for (int i=0; i Date: Mon, 31 Jan 2022 21:39:49 +0000 Subject: [PATCH 046/151] Fixed occasional ConcurrentModificationException in the block archive reader. --- .../qortal/repository/BlockArchiveReader.java | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/qortal/repository/BlockArchiveReader.java b/src/main/java/org/qortal/repository/BlockArchiveReader.java index cff272a8..b6d7cdd6 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveReader.java +++ b/src/main/java/org/qortal/repository/BlockArchiveReader.java @@ -145,20 +145,22 @@ public class BlockArchiveReader { } private String getFilenameForHeight(int height) { - Iterator it = this.fileListCache.entrySet().iterator(); - while (it.hasNext()) { - Map.Entry pair = (Map.Entry)it.next(); - if (pair == null && pair.getKey() == null && pair.getValue() == null) { - continue; - } - Triple heightInfo = (Triple) pair.getValue(); - Integer startHeight = heightInfo.getA(); - Integer endHeight = heightInfo.getB(); + synchronized (this.fileListCache) { + Iterator it = this.fileListCache.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry) it.next(); + if (pair == null && pair.getKey() == null && pair.getValue() == null) { + continue; + } + Triple heightInfo = (Triple) pair.getValue(); + Integer startHeight = heightInfo.getA(); + Integer endHeight = heightInfo.getB(); - if (height >= startHeight && height <= endHeight) { - // Found the correct file - String filename = (String) pair.getKey(); - return filename; + if (height >= startHeight && height <= endHeight) { + // Found the correct file + String filename = (String) pair.getKey(); + return filename; + } } } From 40a8cdc71f976c31fbe19da8f8c4e993b6c24c02 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 31 Jan 2022 22:35:12 +0000 Subject: [PATCH 047/151] Improved logging when fetching data files --- .../arbitrary/ArbitraryDataFileManager.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 8eeda508..6c5376dd 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -119,13 +119,16 @@ public class ArbitraryDataFileManager { if (!arbitraryDataFile.chunkExists(hash)) { // Only request the file if we aren't already requesting it from someone else if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) { + LOGGER.debug("Requesting data file {} from peer {}", Base58.encode(hash), peer); + Long startTime = NTP.getTime(); ArbitraryDataFileMessage receivedArbitraryDataFileMessage = fetchArbitraryDataFile(peer, null, signature, hash, null); + Long endTime = NTP.getTime(); if (receivedArbitraryDataFileMessage != null) { - LOGGER.debug("Received data file {} from peer {}", receivedArbitraryDataFileMessage.getArbitraryDataFile().getHash58(), peer); + LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFileMessage.getArbitraryDataFile().getHash58(), peer, (endTime-startTime)); receivedAtLeastOneFile = true; } else { - LOGGER.debug("Peer {} didn't respond with data file {} for signature {}", peer, Base58.encode(hash), Base58.encode(signature)); + LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime)); } } else { @@ -171,11 +174,11 @@ public class ArbitraryDataFileManager { private ArbitraryDataFileMessage fetchArbitraryDataFile(Peer peer, Peer requestingPeer, byte[] signature, byte[] hash, Message originalMessage) throws DataException { ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature); boolean fileAlreadyExists = existingFile.exists(); + String hash58 = Base58.encode(hash); Message message = null; // Fetch the file if it doesn't exist locally if (!fileAlreadyExists) { - String hash58 = Base58.encode(hash); LOGGER.debug(String.format("Fetching data file %.8s from peer %s", hash58, peer)); arbitraryDataFileRequests.put(hash58, NTP.getTime()); Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash); @@ -195,6 +198,9 @@ public class ArbitraryDataFileManager { return null; } } + else { + LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58)); + } ArbitraryDataFileMessage arbitraryDataFileMessage = (ArbitraryDataFileMessage) message; // We might want to forward the request to the peer that originally requested it From 33731b969a588d3156882e27bf21caba405c82b8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 1 Feb 2022 09:04:31 +0000 Subject: [PATCH 048/151] Direct peer connections now send a file list request to the peer, rather than individually requesting every chunk for a transaction. --- .../ArbitraryDataFileListManager.java | 54 +++++++++++++++++++ .../arbitrary/ArbitraryDataFileManager.java | 33 ------------ src/main/java/org/qortal/network/Network.java | 9 +--- 3 files changed, 56 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index a1dc5d21..f4af43ef 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -304,6 +304,60 @@ public class ArbitraryDataFileListManager { return true; } + public boolean fetchArbitraryDataFileList(Peer peer, byte[] signature) { + String signature58 = Base58.encode(signature); + + // Require an NTP sync + Long now = NTP.getTime(); + if (now == null) { + return false; + } + + LOGGER.debug(String.format("Sending data file list request for signature %s to peer %s...", signature58, peer)); + + // Build request + // Use a time in the past, so that the recipient peer doesn't try and relay it + long timestamp = now - 60000L; + Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, timestamp, 0); + + // Save our request into requests map + Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); + + // Assign random ID to this message + int id; + do { + id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1; + + // Put queue into map (keyed by message ID) so we can poll for a response + // If putIfAbsent() doesn't return null, then this ID is already taken + } while (arbitraryDataFileListRequests.put(id, requestEntry) != null); + getArbitraryDataFileListMessage.setId(id); + + // Send the request + peer.sendMessage(getArbitraryDataFileListMessage); + + // Poll to see if data has arrived + final long singleWait = 100; + long totalWait = 0; + while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) { + try { + Thread.sleep(singleWait); + } catch (InterruptedException e) { + break; + } + + requestEntry = arbitraryDataFileListRequests.get(id); + if (requestEntry == null) + return false; + + if (requestEntry.getA() == null) + break; + + totalWait += singleWait; + } + return true; + } + public void deleteFileListRequestsForSignature(byte[] signature) { String signature58 = Base58.encode(signature); for (Iterator>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 6c5376dd..339d9123 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -69,22 +69,6 @@ public class ArbitraryDataFileManager { // Fetch data files by hash - public boolean fetchAllArbitraryDataFiles(Repository repository, Peer peer, byte[] signature) { - try { - TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); - if (!(transactionData instanceof ArbitraryTransactionData)) - return false; - - ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - - // We use null to represent all hashes associated with this transaction - return this.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, null); - - } catch (DataException e) {} - - return false; - } - public boolean fetchArbitraryDataFiles(Repository repository, Peer peer, byte[] signature, @@ -95,23 +79,6 @@ public class ArbitraryDataFileManager { ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature); byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); arbitraryDataFile.setMetadataHash(metadataHash); - - // If hashes are null, we will treat this to mean all data hashes associated with this file - if (hashes == null) { - if (metadataHash == null) { - // This transaction has no metadata/chunks, so use the main file hash - hashes = Arrays.asList(arbitraryDataFile.getHash()); - } - else if (!arbitraryDataFile.getMetadataFile().exists()) { - // We don't have the metadata file yet, so request it - hashes = Arrays.asList(arbitraryDataFile.getMetadataFile().getHash()); - } - else { - // Add the chunk hashes - hashes = arbitraryDataFile.getChunkHashes(); - } - } - boolean receivedAtLeastOneFile = false; // Now fetch actual data from this peer diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index dde82112..2f5d9bd9 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -6,7 +6,7 @@ import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters; import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters; import org.qortal.block.BlockChain; import org.qortal.controller.Controller; -import org.qortal.controller.arbitrary.ArbitraryDataFileManager; +import org.qortal.controller.arbitrary.ArbitraryDataFileListManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.crypto.Crypto; import org.qortal.data.block.BlockData; @@ -307,12 +307,7 @@ public class Network { return false; } - try (final Repository repository = RepositoryManager.getRepository()) { - return ArbitraryDataFileManager.getInstance().fetchAllArbitraryDataFiles(repository, connectedPeer, signature); - } catch (DataException e) { - LOGGER.info("Unable to fetch arbitrary data files"); - } - return false; + return ArbitraryDataFileListManager.getInstance().fetchArbitraryDataFileList(connectedPeer, signature); } /** From 45f2d7ab704dad0319650ece4ba96fe6867af163 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 1 Feb 2022 21:55:41 +0000 Subject: [PATCH 049/151] Revert "We (currently) can't filter unconfirmed transactions by address, because only the public key is stored in the database until it is confirmed (at which point there is an entry in the TransactionParticipants table which contains the address). Given that this isn't a simple problem to solve, for now it makes sense to reject this combination if requested via the /transactions/search API." This reverts commit 7aed0354f1e2926baa4257d4fbb2a04d14e195c2. --- .../java/org/qortal/api/resource/TransactionsResource.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/TransactionsResource.java b/src/main/java/org/qortal/api/resource/TransactionsResource.java index 62e3bdc6..9bc6d497 100644 --- a/src/main/java/org/qortal/api/resource/TransactionsResource.java +++ b/src/main/java/org/qortal/api/resource/TransactionsResource.java @@ -349,10 +349,6 @@ public class TransactionsResource { if (confirmationStatus != ConfirmationStatus.CONFIRMED && (startBlock != null || blockLimit != null)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - // You can't ask for unconfirmed and filter by address (due to only public key being stored when unconfirmed) - if (confirmationStatus != ConfirmationStatus.CONFIRMED && address != null && !address.isEmpty()) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); - try (final Repository repository = RepositoryManager.getRepository()) { List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(startBlock, blockLimit, txGroupId, txTypes, null, null, address, confirmationStatus, limit, offset, reverse); From 82fa6a4fd856c4755e55209478b78c4fdbc93df5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 1 Feb 2022 22:02:14 +0000 Subject: [PATCH 050/151] Include "localChunkCount" and "totalChunkCount" in the GET /arbitrary/resource/status/* API responses. These values are left out of other API endpoints where multiple resources are returned, because calculating the chunk counts is too time consuming. --- .../api/gateway/resource/GatewayResource.java | 2 +- .../api/resource/ArbitraryResource.java | 4 +- .../arbitrary/ArbitraryDataResource.java | 73 ++++++++++++++----- .../arbitrary/ArbitraryResourceStatus.java | 10 ++- .../utils/ArbitraryTransactionUtils.java | 40 +++++++++- 5 files changed, 102 insertions(+), 27 deletions(-) diff --git a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java index cee1613f..a73de1fb 100644 --- a/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java +++ b/src/main/java/org/qortal/api/gateway/resource/GatewayResource.java @@ -65,7 +65,7 @@ public class GatewayResource { } ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); - return resource.getStatus(); + return resource.getStatus(false); } diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index d542b89c..c49969c5 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1097,7 +1097,7 @@ public class ArbitraryResource { } ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); - return resource.getStatus(); + return resource.getStatus(false); } private List addStatusToResources(List resources) { @@ -1106,7 +1106,7 @@ public class ArbitraryResource { for (ArbitraryResourceInfo resourceInfo : resources) { ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ResourceIdType.NAME, resourceInfo.service, resourceInfo.identifier); - ArbitraryResourceStatus status = resource.getStatus(); + ArbitraryResourceStatus status = resource.getStatus(true); if (status != null) { resourceInfo.status = status; } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 7e00e0d0..36bd8f4c 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -38,6 +38,8 @@ public class ArbitraryDataResource { private List transactions; private ArbitraryTransactionData latestPutTransaction; private int layerCount; + private Integer localChunkCount = null; + private Integer totalChunkCount = null; public ArbitraryDataResource(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) { this.resourceId = resourceId.toLowerCase(); @@ -51,50 +53,56 @@ public class ArbitraryDataResource { this.identifier = identifier; } - public ArbitraryResourceStatus getStatus() { + public ArbitraryResourceStatus getStatus(boolean quick) { + // Calculate the chunk counts + // Avoid this for "quick" statuses, to speed things up + if (!quick) { + this.calculateChunkCounts(); + } + if (resourceIdType != ResourceIdType.NAME) { // We only support statuses for resources with a name - return new ArbitraryResourceStatus(Status.UNSUPPORTED); + return new ArbitraryResourceStatus(Status.UNSUPPORTED, this.localChunkCount, this.totalChunkCount); } // Check if the name is blocked if (ResourceListManager.getInstance() .listContains("blockedNames", this.resourceId, false)) { - return new ArbitraryResourceStatus(Status.BLOCKED); + return new ArbitraryResourceStatus(Status.BLOCKED, this.localChunkCount, this.totalChunkCount); + } + + // Check if a build has failed + ArbitraryDataBuildQueueItem queueItem = + new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier); + if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) { + return new ArbitraryResourceStatus(Status.BUILD_FAILED, this.localChunkCount, this.totalChunkCount); } // Firstly check the cache to see if it's already built ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader( resourceId, resourceIdType, service, identifier); if (arbitraryDataReader.isCachedDataAvailable()) { - return new ArbitraryResourceStatus(Status.READY); - } - - // Next check if there's a build in progress - ArbitraryDataBuildQueueItem queueItem = - new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier); - if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { - return new ArbitraryResourceStatus(Status.BUILDING); - } - - // Check if a build has failed - if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) { - return new ArbitraryResourceStatus(Status.BUILD_FAILED); + return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount); } // Check if we have all data locally for this resource if (!this.allFilesDownloaded()) { if (this.isDownloading()) { - return new ArbitraryResourceStatus(Status.DOWNLOADING); + return new ArbitraryResourceStatus(Status.DOWNLOADING, this.localChunkCount, this.totalChunkCount); } else if (this.isDataPotentiallyAvailable()) { - return new ArbitraryResourceStatus(Status.PUBLISHED); + return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount); } - return new ArbitraryResourceStatus(Status.MISSING_DATA); + return new ArbitraryResourceStatus(Status.MISSING_DATA, this.localChunkCount, this.totalChunkCount); + } + + // Check if there's a build in progress + if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { + return new ArbitraryResourceStatus(Status.BUILDING, this.localChunkCount, this.totalChunkCount); } // We have all data locally - return new ArbitraryResourceStatus(Status.DOWNLOADED); + return new ArbitraryResourceStatus(Status.DOWNLOADED, this.localChunkCount, this.totalChunkCount); } public boolean delete() { @@ -147,6 +155,12 @@ public class ArbitraryDataResource { } private boolean allFilesDownloaded() { + // Use chunk counts to speed things up if we can + if (this.localChunkCount != null && this.totalChunkCount != null && + this.localChunkCount >= this.totalChunkCount) { + return true; + } + try { this.fetchTransactions(); @@ -165,6 +179,25 @@ public class ArbitraryDataResource { } } + private void calculateChunkCounts() { + try { + this.fetchTransactions(); + + List transactionDataList = new ArrayList<>(this.transactions); + int localChunkCount = 0; + int totalChunkCount = 0; + + for (ArbitraryTransactionData transactionData : transactionDataList) { + localChunkCount += ArbitraryTransactionUtils.ourChunkCount(transactionData); + totalChunkCount += ArbitraryTransactionUtils.totalChunkCount(transactionData); + } + + this.localChunkCount = localChunkCount; + this.totalChunkCount = totalChunkCount; + + } catch (DataException e) {} + } + private boolean isRateLimited() { try { this.fetchTransactions(); diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java index 35b83507..5e6ac055 100644 --- a/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java @@ -30,13 +30,21 @@ public class ArbitraryResourceStatus { private String title; private String description; + private Integer localChunkCount; + private Integer totalChunkCount; + public ArbitraryResourceStatus() { } - public ArbitraryResourceStatus(Status status) { + public ArbitraryResourceStatus(Status status, Integer localChunkCount, Integer totalChunkCount) { this.id = status.toString(); this.title = status.title; this.description = status.description; + this.localChunkCount = localChunkCount; + this.totalChunkCount = totalChunkCount; } + public ArbitraryResourceStatus(Status status) { + this(status, null, null); + } } diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index 3221c87b..9b81bd68 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -1,5 +1,6 @@ package org.qortal.utils; +import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile; @@ -147,16 +148,49 @@ public class ArbitraryTransactionUtils { byte[] metadataHash = transactionData.getMetadataHash(); byte[] signature = transactionData.getSignature(); - if (metadataHash == null) { - // This file doesn't have any metadata, therefore it has no chunks + ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); + arbitraryDataFile.setMetadataHash(metadataHash); + + // Find the folder containing the files + Path parentPath = arbitraryDataFile.getFilePath().getParent(); + String[] files = parentPath.toFile().list(); + if (files == null) { return 0; } + // Remove the original copy indicator file if it exists + files = ArrayUtils.removeElement(files, ".original"); + + int count = files.length; + + // If the complete file exists (and this transaction has chunks), subtract it from the count + if (arbitraryDataFile.chunkCount() > 0 && arbitraryDataFile.exists()) { + // We are only measuring the individual chunks, not the joined file + count -= 1; + } + + return count; + } + + public static int totalChunkCount(ArbitraryTransactionData transactionData) throws DataException { + if (transactionData == null) { + return 0; + } + + byte[] digest = transactionData.getData(); + byte[] metadataHash = transactionData.getMetadataHash(); + byte[] signature = transactionData.getSignature(); + + if (metadataHash == null) { + // This file doesn't have any metadata, therefore it has a single (complete) chunk + return 1; + } + // Load complete file and chunks ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); arbitraryDataFile.setMetadataHash(metadataHash); - return arbitraryDataFile.chunkCount(); + return arbitraryDataFile.chunkCount() + 1; // +1 for the metadata file } public static boolean isFileRecent(Path filePath, long now, long cleanupAfter) { From 6c9600cda0aebe485f3ebb45b298c00feee0e943 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 1 Feb 2022 22:02:41 +0000 Subject: [PATCH 051/151] Added API key header to GET /arbitrary/resource/status/* endpoints --- .../qortal/api/resource/ArbitraryResource.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index c49969c5..f588e9c9 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -232,9 +232,10 @@ public class ArbitraryResource { } ) @SecurityRequirement(name = "apiKey") - public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service, - @PathParam("name") String name, - @QueryParam("build") Boolean build) { + public ArbitraryResourceStatus getDefaultResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") Service service, + @PathParam("name") String name, + @QueryParam("build") Boolean build) { Security.requirePriorAuthorizationOrApiKey(request, name, service, null); return this.getStatus(service, name, null, build); @@ -252,10 +253,11 @@ public class ArbitraryResource { } ) @SecurityRequirement(name = "apiKey") - public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service, - @PathParam("name") String name, - @PathParam("identifier") String identifier, - @QueryParam("build") Boolean build) { + public ArbitraryResourceStatus getResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") Service service, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @QueryParam("build") Boolean build) { Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier); return this.getStatus(service, name, identifier, build); From 5a8b89547578aaf964505adb9e9e279364159767 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 1 Feb 2022 22:03:17 +0000 Subject: [PATCH 052/151] Fixed bug in loading screen, which prevented the DOWNLOADING status from showing. --- src/main/resources/loading/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/loading/index.html b/src/main/resources/loading/index.html index 6f234c45..807fa17f 100644 --- a/src/main/resources/loading/index.html +++ b/src/main/resources/loading/index.html @@ -82,7 +82,7 @@ textStatus = status.description; retryInterval = 1000; } - else if (status.status == "DOWNLOADING") { + else if (status.id == "DOWNLOADING") { textStatus = status.description; retryInterval = 1000; } From 97199d9b91ba8026369348847804bb4dfcfe9212 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 1 Feb 2022 22:03:52 +0000 Subject: [PATCH 053/151] Display the local and total chunk counts on the loading screen. --- src/main/resources/loading/index.html | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/resources/loading/index.html b/src/main/resources/loading/index.html index 807fa17f..fc50c72d 100644 --- a/src/main/resources/loading/index.html +++ b/src/main/resources/loading/index.html @@ -46,6 +46,7 @@ var url = host + '/arbitrary/resource/status/' + service + '/' + name + '?build=true'; var textStatus = "Loading..."; + var textProgress = ""; var retryInterval = 2500; const response = await fetch(url, { @@ -96,7 +97,12 @@ textStatus = status.description; } + if (status.localChunkCount != null && status.totalChunkCount != null) { + textProgress = "Files downloaded: " + status.localChunkCount + " / " + status.totalChunkCount; + } + document.getElementById("status").innerHTML = textStatus; + document.getElementById("progress").innerHTML = textProgress; setTimeout(checkStatus, retryInterval); } @@ -260,6 +266,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI This page will refresh automatically when the content becomes available.

Loading...

+

From 8ccb15824198750ce951d8f94a10209c6b1be1d6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 1 Feb 2022 22:04:12 +0000 Subject: [PATCH 054/151] Reduce log spam when in DEBUG mode. --- src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index a6fad12d..619e5330 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -363,7 +363,7 @@ public class ArbitraryDataReader { } // Throw a missing data exception, which allows subsequent layers to fetch data - LOGGER.debug(message); + LOGGER.trace(message); throw new MissingDataException(message); } } From 710befec0c1ad3a6ad3fff3d198c28d561da2ec3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 1 Feb 2022 22:04:53 +0000 Subject: [PATCH 055/151] Increased ARBITRARY_RELAY_TIMEOUT from 30 to 60 seconds, so that relay peers remember their mappings for longer. --- .../org/qortal/controller/arbitrary/ArbitraryDataManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index 20b4885a..f07f0669 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -41,7 +41,7 @@ public class ArbitraryDataManager extends Thread { public static final long ARBITRARY_REQUEST_TIMEOUT = 10 * 1000L; // ms /** Maximum time to hold information about an in-progress relay */ - public static final long ARBITRARY_RELAY_TIMEOUT = 30 * 1000L; // ms + public static final long ARBITRARY_RELAY_TIMEOUT = 60 * 1000L; // ms /** Maximum number of hops that an arbitrary signatures request is allowed to make */ private static int ARBITRARY_SIGNATURES_REQUEST_MAX_HOPS = 3; From c9d5d996e56951dd2f01fe60a588829940acafe6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 1 Feb 2022 22:06:01 +0000 Subject: [PATCH 056/151] Increased accuracy of the block weights in the synchronizer logs, as the extra precision is now needed for debugging. --- .../java/org/qortal/controller/Synchronizer.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 14d1a071..4e8aa33c 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -545,6 +545,8 @@ public class Synchronizer extends Thread { // Create a placeholder to track of common blocks that we can discard due to being inferior chains int dropPeersAfterCommonBlockHeight = 0; + NumberFormat accurateFormatter = new DecimalFormat("0.################E0"); + // Remove peers with no common block data Iterator iterator = peers.iterator(); while (iterator.hasNext()) { @@ -667,9 +669,7 @@ public class Synchronizer extends Thread { if (ourBlockSummaries.size() > 0) ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, maxHeightForChainWeightComparisons); - NumberFormat formatter = new DecimalFormat("0.###E0"); - NumberFormat accurateFormatter = new DecimalFormat("0.################E0"); - LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", (usingSameLengthChainWeight ? minChainLength : ourBlockSummaries.size()), formatter.format(ourChainWeight))); + LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", (usingSameLengthChainWeight ? minChainLength : ourBlockSummaries.size()), accurateFormatter.format(ourChainWeight))); LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature()))); for (Peer peer : peersSharingCommonBlock) { @@ -691,7 +691,7 @@ public class Synchronizer extends Thread { LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock)); BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, maxHeightForChainWeightComparisons); peer.getCommonBlockData().setChainWeight(peerChainWeight); - LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight))); + LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), accurateFormatter.format(peerChainWeight))); // Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group) if (ourChainWeight.compareTo(peerChainWeight) > 0) { @@ -1141,8 +1141,9 @@ public class Synchronizer extends Thread { BigInteger ourChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight); BigInteger peerChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, peerBlockSummaries, mutualHeight); - NumberFormat formatter = new DecimalFormat("0.###E0"); - LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", formatter.format(ourChainWeight), formatter.format(peerChainWeight))); + NumberFormat accurateFormatter = new DecimalFormat("0.################E0"); + LOGGER.debug(String.format("commonBlockHeight: %d, commonBlockSig: %.8s, ourBlockSummaries.size(): %d, peerBlockSummaries.size(): %d", commonBlockHeight, Base58.encode(commonBlockSig), ourBlockSummaries.size(), peerBlockSummaries.size())); + LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", accurateFormatter.format(ourChainWeight), accurateFormatter.format(peerChainWeight))); // If our blockchain has greater weight then don't synchronize with peer if (ourChainWeight.compareTo(peerChainWeight) >= 0) { From 640bcdd504791ffdb406d7077956e50efb702927 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 2 Feb 2022 09:13:49 +0000 Subject: [PATCH 057/151] Shutdown/interrupt the synchronizer as early as possible --- src/main/java/org/qortal/controller/Controller.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 3bb3052b..1f44d622 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -826,6 +826,9 @@ public class Controller extends Thread { if (!isStopping) { isStopping = true; + LOGGER.info("Shutting down synchronizer"); + Synchronizer.getInstance().shutdown(); + LOGGER.info("Shutting down API"); ApiService.getInstance().stop(); @@ -852,9 +855,6 @@ public class Controller extends Thread { } } - LOGGER.info("Shutting down synchronizer"); - Synchronizer.getInstance().shutdown(); - // Export local data LOGGER.info("Backing up local data"); this.exportRepositoryData(); From 7338f5f98546a62684beaccf2510c0a816508377 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 2 Feb 2022 09:17:24 +0000 Subject: [PATCH 058/151] Attempt to acquire a blockchain lock (for up to 5 seconds) before shutting down the repository. This should fix conflicts caused by the synchronizer and controller now being on separate threads. It may also reduce the chances of the database corrupting on shutdown, but this remains to be seen. --- .../java/org/qortal/controller/Controller.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 1f44d622..4d3ad391 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -870,6 +870,17 @@ public class Controller extends Thread { // We were interrupted while waiting for thread to join } + // Make sure we're the only thread modifying the blockchain when shutting down the repository + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + try { + if (!blockchainLock.tryLock(5, TimeUnit.SECONDS)) { + LOGGER.debug("Couldn't acquire blockchain lock even after waiting 5 seconds"); + // Proceed anyway, as we have to shut down + } + } catch (InterruptedException e) { + LOGGER.info("Interrupted when waiting for blockchain lock"); + } + try { LOGGER.info("Shutting down repository"); RepositoryManager.closeRepositoryFactory(); @@ -877,6 +888,11 @@ public class Controller extends Thread { LOGGER.error("Error occurred while shutting down repository", e); } + // Release the lock if we acquired it + if (blockchainLock.isHeldByCurrentThread()) { + blockchainLock.unlock(); + } + LOGGER.info("Shutting down NTP"); NTP.shutdownNow(); From 0430fc8a478f0309e788a582a3db0cb4d0e22508 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 2 Feb 2022 09:18:46 +0000 Subject: [PATCH 059/151] Give up immediately after a synchronization if we are shutting down the synchronizer --- src/main/java/org/qortal/controller/Synchronizer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 4e8aa33c..e9090cf0 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -307,6 +307,11 @@ public class Synchronizer extends Thread { } } + if (!running) { + // We've stopped + return SynchronizationResult.SHUTTING_DOWN; + } + // Has our chain tip changed? BlockData newChainTip; From 6d06953a0e87df657239dee0d3c2c446e082bbec Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 2 Feb 2022 22:31:40 +0000 Subject: [PATCH 060/151] Increased RELAY_REQUEST_MAX_HOPS from 3 to 4, in an attempt to reach more peers. --- .../controller/arbitrary/ArbitraryDataFileListManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index f4af43ef..008217b0 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -59,7 +59,7 @@ public class ArbitraryDataFileListManager { /** Maximum number of seconds that a file list relay request is able to exist on the network */ private static long RELAY_REQUEST_MAX_DURATION = 5000L; /** Maximum number of hops that a file list relay request is allowed to make */ - private static int RELAY_REQUEST_MAX_HOPS = 3; + private static int RELAY_REQUEST_MAX_HOPS = 4; private ArbitraryDataFileListManager() { From 114b1aac76ac3322a28290e1b881634b11e443d0 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 19:51:02 +0000 Subject: [PATCH 061/151] Added arbitrary data file manager thread, which will ensure that all file list responses are tried until we receive the files. Previously we would only try the first response and then discard the others due to being duplicates. They are now added to a queue and retried by the dedicated thread (up to the 60 second timeout). --- .../org/qortal/controller/Controller.java | 2 + .../ArbitraryDataFileListManager.java | 8 ++ .../arbitrary/ArbitraryDataFileManager.java | 84 ++++++++++++++++++- 3 files changed, 91 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 4d3ad391..c4429346 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -452,6 +452,7 @@ public class Controller extends Thread { // Arbitrary data controllers LOGGER.info("Starting arbitrary-transaction controllers"); ArbitraryDataManager.getInstance().start(); + ArbitraryDataFileManager.getInstance().start(); ArbitraryDataBuildManager.getInstance().start(); ArbitraryDataCleanupManager.getInstance().start(); ArbitraryDataStorageManager.getInstance().start(); @@ -840,6 +841,7 @@ public class Controller extends Thread { // Arbitrary data controllers LOGGER.info("Shutting down arbitrary-transaction controllers"); ArbitraryDataManager.getInstance().shutdown(); + ArbitraryDataFileManager.getInstance().shutdown(); ArbitraryDataBuildManager.getInstance().shutdown(); ArbitraryDataCleanupManager.getInstance().shutdown(); ArbitraryDataStorageManager.getInstance().shutdown(); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 008217b0..6e20499d 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -431,6 +431,14 @@ public class ArbitraryDataFileListManager { // } if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) { + // Keep track of the hashes this peer reports to have access to + Long now = NTP.getTime(); + for (byte[] hash : hashes) { + String hash58 = Base58.encode(hash); + String sig58 = Base58.encode(signature); + ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.put(hash58, new Triple<>(peer, sig58, now)); + } + // Go and fetch the actual data, since this isn't a relay request arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, hashes); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 339d9123..5c20585d 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -7,7 +7,6 @@ import org.qortal.controller.Controller; import org.qortal.data.network.ArbitraryPeerData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.ArbitraryTransactionData; -import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; import org.qortal.network.Peer; import org.qortal.network.message.*; @@ -24,11 +23,12 @@ import java.security.SecureRandom; import java.util.*; import java.util.stream.Collectors; -public class ArbitraryDataFileManager { +public class ArbitraryDataFileManager extends Thread { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileManager.class); private static ArbitraryDataFileManager instance; + private volatile boolean isStopping = false; /** @@ -42,6 +42,13 @@ public class ArbitraryDataFileManager { */ public Map> arbitraryRelayMap = Collections.synchronizedMap(new HashMap<>()); + /** + * Map to keep track of any arbitrary data file hash responses + * Key: string - the hash encoded in base58 + * Value: Triple + */ + public Map> arbitraryDataFileHashResponses = Collections.synchronizedMap(new HashMap<>()); + private ArbitraryDataFileManager() { } @@ -53,6 +60,65 @@ public class ArbitraryDataFileManager { return instance; } + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data File Manager"); + + try { + while (!isStopping) { + Thread.sleep(1000); + + Long now = NTP.getTime(); + this.processFileHashes(now); + } + } catch (InterruptedException e) { + // Fall-through to exit thread... + } + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + } + + private void processFileHashes(Long now) { + try (final Repository repository = RepositoryManager.getRepository()) { + + for (String hash58 : arbitraryDataFileHashResponses.keySet()) { + if (isStopping) { + return; + } + + Triple value = arbitraryDataFileHashResponses.get(hash58); + if (value != null) { + Peer peer = value.getA(); + String signature58 = value.getB(); + Long timestamp = value.getC(); + + if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { + // Ignore - to be deleted + continue; + } + + byte[] hash = Base58.decode(hash58); + byte[] signature = Base58.decode(signature58); + + // Fetch the transaction data + ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); + if (arbitraryTransactionData == null) { + continue; + } + + LOGGER.debug("Fetching file {} from peer {} via response queue...", hash58, peer); + this.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash)); + } + } + + } catch (DataException e) { + LOGGER.info("Unable to process file hashes: {}", e.getMessage()); + } + } + public void cleanupRequestCache(Long now) { if (now == null) { @@ -63,6 +129,7 @@ public class ArbitraryDataFileManager { final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT; arbitraryRelayMap.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp); + arbitraryDataFileHashResponses.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp); } @@ -83,10 +150,14 @@ public class ArbitraryDataFileManager { // Now fetch actual data from this peer for (byte[] hash : hashes) { + if (isStopping) { + return false; + } + String hash58 = Base58.encode(hash); if (!arbitraryDataFile.chunkExists(hash)) { // Only request the file if we aren't already requesting it from someone else if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) { - LOGGER.debug("Requesting data file {} from peer {}", Base58.encode(hash), peer); + LOGGER.debug("Requesting data file {} from peer {}", hash58, peer); Long startTime = NTP.getTime(); ArbitraryDataFileMessage receivedArbitraryDataFileMessage = fetchArbitraryDataFile(peer, null, signature, hash, null); Long endTime = NTP.getTime(); @@ -97,11 +168,18 @@ public class ArbitraryDataFileManager { else { LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime)); } + + // Remove this hash from arbitraryDataFileHashResponses now that we have tried to request it + arbitraryDataFileHashResponses.remove(hash58); } else { LOGGER.trace("Already requesting data file {} for signature {}", arbitraryDataFile, Base58.encode(signature)); } } + else { + // Remove this hash from arbitraryDataFileHashResponses because we have a local copy + arbitraryDataFileHashResponses.remove(hash58); + } } if (receivedAtLeastOneFile) { From 0b7a7ed0f1df38815dde4b8c3a383d009bc31163 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 19:52:28 +0000 Subject: [PATCH 062/151] Added some more debug logging relating to file responses. --- .../controller/arbitrary/ArbitraryDataFileManager.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 5c20585d..342b142d 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -239,7 +239,12 @@ public class ArbitraryDataFileManager extends Thread { // We may need to remove the file list request, if we have all the files for this transaction this.handleFileListRequests(signature); - if (message == null || message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) { + if (message == null) { + LOGGER.debug("Received null message from peer {}", peer); + return null; + } + if (message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) { + LOGGER.debug("Received message with invalid type: {} from peer {}", message.getType(), peer); return null; } } From 1064b1a08b6e263c4be8537ba74920bafbb5252c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 19:53:29 +0000 Subject: [PATCH 063/151] Removed duplicate metadata file checks. --- src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index cc667cda..1307eab7 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -460,9 +460,6 @@ public class ArbitraryDataFile { if (this.metadataFile == null) { this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature); - if (!metadataFile.exists()) { - return false; - } } // If the metadata file doesn't exist, we can't check if we have the chunks @@ -501,9 +498,6 @@ public class ArbitraryDataFile { if (this.metadataFile == null) { this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature); - if (!metadataFile.exists()) { - return false; - } } // If the metadata file doesn't exist, we can't check if we have any chunks From d98df3e47db27afed485bd600a12751450937ed3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 19:55:22 +0000 Subject: [PATCH 064/151] Added support for file hashes to optionally be included in GetArbitraryDataFileListMessage. Needs testing on testnet, as the majority of nodes need to update before the hashes can be used. --- .../GetArbitraryDataFileListMessage.java | 45 ++++++++++++++++--- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java index e19bbb25..af19eec1 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java @@ -3,11 +3,14 @@ package org.qortal.network.message; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; import org.qortal.transform.Transformer; +import org.qortal.transform.transaction.TransactionTransformer; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import static org.qortal.transform.Transformer.INT_LENGTH; import static org.qortal.transform.Transformer.LONG_LENGTH; @@ -15,19 +18,22 @@ import static org.qortal.transform.Transformer.LONG_LENGTH; public class GetArbitraryDataFileListMessage extends Message { private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; private final byte[] signature; + private List hashes; private final long requestTime; private int requestHops; - public GetArbitraryDataFileListMessage(byte[] signature, long requestTime, int requestHops) { - this(-1, signature, requestTime, requestHops); + public GetArbitraryDataFileListMessage(byte[] signature, List hashes, long requestTime, int requestHops) { + this(-1, signature, hashes, requestTime, requestHops); } - private GetArbitraryDataFileListMessage(int id, byte[] signature, long requestTime, int requestHops) { + private GetArbitraryDataFileListMessage(int id, byte[] signature, List hashes, long requestTime, int requestHops) { super(id, MessageType.GET_ARBITRARY_DATA_FILE_LIST); this.signature = signature; + this.hashes = hashes; this.requestTime = requestTime; this.requestHops = requestHops; } @@ -36,10 +42,11 @@ public class GetArbitraryDataFileListMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != SIGNATURE_LENGTH + LONG_LENGTH + INT_LENGTH) - return null; + public List getHashes() { + return this.hashes; + } + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { byte[] signature = new byte[SIGNATURE_LENGTH]; bytes.get(signature); @@ -48,7 +55,23 @@ public class GetArbitraryDataFileListMessage extends Message { int requestHops = bytes.getInt(); - return new GetArbitraryDataFileListMessage(id, signature, requestTime, requestHops); + List hashes = null; + if (bytes.hasRemaining()) { + int hashCount = bytes.getInt(); + + if (bytes.remaining() != hashCount * HASH_LENGTH) { + return null; + } + + hashes = new ArrayList<>(); + for (int i = 0; i < hashCount; ++i) { + byte[] hash = new byte[HASH_LENGTH]; + bytes.get(hash); + hashes.add(hash); + } + } + + return new GetArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops); } @Override @@ -62,6 +85,14 @@ public class GetArbitraryDataFileListMessage extends Message { bytes.write(Ints.toByteArray(this.requestHops)); + if (this.hashes != null) { + bytes.write(Ints.toByteArray(this.hashes.size())); + + for (byte[] hash : this.hashes) { + bytes.write(hash); + } + } + return bytes.toByteArray(); } catch (IOException e) { return null; From 7994fc6407cdf7843a1f7ae301f7b74c1205ad35 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 19:58:50 +0000 Subject: [PATCH 065/151] Rework of onNetworkGetArbitraryDataFileListMessage() to support custom hashes to be optionally supplied. Also simplified the existing logic, to make the code more readable. --- .../ArbitraryDataFileListManager.java | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 6e20499d..3147f9cb 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -484,6 +484,7 @@ public class ArbitraryDataFileListManager { GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message; byte[] signature = getArbitraryDataFileListMessage.getSignature(); String signature58 = Base58.encode(signature); + List requestedHashes = getArbitraryDataFileListMessage.getHashes(); Long now = NTP.getTime(); Triple newEntry = new Triple<>(signature58, peer, now); @@ -513,36 +514,37 @@ public class ArbitraryDataFileListManager { // Load file(s) and add any that exist to the list of hashes ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); - if (metadataHash != null) { - arbitraryDataFile.setMetadataHash(metadataHash); + arbitraryDataFile.setMetadataHash(metadataHash); - // Assume all chunks exists, unless one can't be found below - allChunksExist = true; + // If the peer didn't supply a hash list, we need to return all hashes for this transaction + if (requestedHashes == null || requestedHashes.isEmpty()) { + requestedHashes = new ArrayList<>(); - // If we have the metadata file, add its hash - if (arbitraryDataFile.getMetadataFile().exists()) { - hashes.add(arbitraryDataFile.getMetadataHash()); + // Add the metadata file + if (arbitraryDataFile.getMetadataHash() != null) { + requestedHashes.add(arbitraryDataFile.getMetadataHash()); } + + // Add the chunk hashes + if (arbitraryDataFile.getChunkHashes().size() > 0) { + requestedHashes.addAll(arbitraryDataFile.getChunkHashes()); + } + // Add complete file if there are no hashes else { - allChunksExist = false; + requestedHashes.add(arbitraryDataFile.getHash()); } + } - for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) { - if (chunk.exists()) { - hashes.add(chunk.getHash()); - //LOGGER.trace("Added hash {}", chunk.getHash58()); - } else { - LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58()); - allChunksExist = false; - } - } - } else { - // This transaction has no chunks, so include the complete file if we have it - if (arbitraryDataFile.exists()) { - hashes.add(arbitraryDataFile.getHash()); - allChunksExist = true; - } - else { + // Assume all chunks exists, unless one can't be found below + allChunksExist = true; + + for (byte[] requestedHash : requestedHashes) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature); + if (chunk.exists()) { + hashes.add(chunk.getHash()); + //LOGGER.trace("Added hash {}", chunk.getHash58()); + } else { + LOGGER.trace("Couldn't add hash {} because it doesn't exist", chunk.getHash58()); allChunksExist = false; } } From 077165b80731e6d714c5fbabf9a7a401b6563c71 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 20:01:44 +0000 Subject: [PATCH 066/151] Modified fetchArbitraryDataFileList() to support requesting only the missing hashes, but it is not yet used. Once GetArbitraryDataFileListMessage is rolled out to the network we can uncomment this code. Needs testnet testing prior to that. --- .../qortal/arbitrary/ArbitraryDataFile.java | 44 +++++++++++++++++++ .../ArbitraryDataFileListManager.java | 30 ++++++++++--- 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 1307eab7..2d7346ea 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -539,6 +539,50 @@ public class ArbitraryDataFile { return false; } + /** + * Retrieve a list of file hashes for this transaction that we do not hold locally + * + * @return a List of chunk hashes, or null if we are unable to determine what is missing + */ + public List missingHashes() { + List missingHashes = new ArrayList<>(); + try { + if (this.metadataHash == null) { + // We don't have any metadata so can't check if we have the chunks + // Even if this transaction has no chunks, we don't have the file either (already checked above) + return null; + } + + if (this.metadataFile == null) { + this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature); + } + + // If the metadata file doesn't exist, we can't check if we have the chunks + if (!metadataFile.getFilePath().toFile().exists()) { + return null; + } + + if (this.metadata == null) { + this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath())); + } + + // Read the metadata + List chunks = metadata.getChunks(); + for (byte[] chunkHash : chunks) { + ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature); + if (!chunk.exists()) { + missingHashes.add(chunkHash); + } + } + + return missingHashes; + + } catch (DataException e) { + // Something went wrong, so we can't make a sensible decision + return null; + } + } + public boolean containsChunk(byte[] hash) { for (ArbitraryDataFileChunk chunk : this.chunks) { if (Arrays.equals(hash, chunk.getHash())) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 3147f9cb..6337fc7c 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -236,9 +236,11 @@ public class ArbitraryDataFileListManager { } - // Lookup file lists by signature + // Lookup file lists by signature (and optionally hashes) public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) { + byte[] digest = arbitraryTransactionData.getData(); + byte[] metadataHash = arbitraryTransactionData.getMetadataHash(); byte[] signature = arbitraryTransactionData.getSignature(); String signature58 = Base58.encode(signature); @@ -261,10 +263,24 @@ public class ArbitraryDataFileListManager { this.addToSignatureRequests(signature58, true, false); List handshakedPeers = Network.getInstance().getHandshakedPeers(); - LOGGER.debug(String.format("Sending data file list request for signature %s to %d peers...", signature58, handshakedPeers.size())); + List missingHashes = null; + +// // TODO: uncomment after GetArbitraryDataFileListMessage updates are deployed +// // Find hashes that we are missing +// try { +// ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); +// arbitraryDataFile.setMetadataHash(metadataHash); +// missingHashes = arbitraryDataFile.missingHashes(); +// } catch (DataException e) { +// // Leave missingHashes as null, so that all hashes are requested +// } +// int hashCount = missingHashes != null ? missingHashes.size() : 0; + + int hashCount = 0; + LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size())); // Build request - Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, now, 0); + Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, missingHashes, now, 0); // Save our request into requests map Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); @@ -313,12 +329,16 @@ public class ArbitraryDataFileListManager { return false; } - LOGGER.debug(String.format("Sending data file list request for signature %s to peer %s...", signature58, peer)); + int hashCount = 0; + LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to peer %s...", signature58, hashCount, peer)); // Build request // Use a time in the past, so that the recipient peer doesn't try and relay it + // Also, set hashes to null since it's easier to request all hashes than it is to determine which ones we need + // This could be optimized in the future long timestamp = now - 60000L; - Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, timestamp, 0); + List hashes = null; + Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, timestamp, 0); // Save our request into requests map Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); From 892612c084ac34b0904d6deb72d5750d55e5abb7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 20:22:25 +0000 Subject: [PATCH 067/151] Calculate wallet balances from the transactions (ElectrumX) rather than using bitcoinj. Hopeful fix for incorrect balances in wallets with large numbers of transactions. At the very least, this gives us control of the code that calculates the balance. --- .../api/resource/CrossChainBitcoinResource.java | 13 +++++++++---- .../api/resource/CrossChainDogecoinResource.java | 13 +++++++++---- .../api/resource/CrossChainLitecoinResource.java | 13 +++++++++---- src/main/java/org/qortal/crosschain/Bitcoiny.java | 15 +++++++++++++++ 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java index 9bbf0e43..834c7b81 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java @@ -67,11 +67,16 @@ public class CrossChainBitcoinResource { if (!bitcoin.isValidDeterministicKey(key58)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - Long balance = bitcoin.getWalletBalance(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + try { + Long balance = bitcoin.getWalletBalanceFromTransactions(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - return balance.toString(); + return balance.toString(); + + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } } @POST diff --git a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java index bb2dcbbc..189a53d3 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java @@ -65,11 +65,16 @@ public class CrossChainDogecoinResource { if (!dogecoin.isValidDeterministicKey(key58)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - Long balance = dogecoin.getWalletBalance(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + try { + Long balance = dogecoin.getWalletBalanceFromTransactions(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - return balance.toString(); + return balance.toString(); + + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } } @POST diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java index 8f6fa582..627c00c7 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java @@ -67,11 +67,16 @@ public class CrossChainLitecoinResource { if (!litecoin.isValidDeterministicKey(key58)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - Long balance = litecoin.getWalletBalance(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + try { + Long balance = litecoin.getWalletBalanceFromTransactions(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - return balance.toString(); + return balance.toString(); + + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } } @POST diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 3665f4ba..b4cc389b 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -337,6 +337,21 @@ public abstract class Bitcoiny implements ForeignBlockchain { return balance.value; } + public Long getWalletBalanceFromTransactions(String key58) throws ForeignBlockchainException { + long balance = 0; + Comparator oldestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp); + List transactions = getWalletTransactions(key58).stream().sorted(oldestTimestampFirstComparator).collect(Collectors.toList()); + for (SimpleTransaction transaction : transactions) { + balance += transaction.getTotalAmount(); + + if (transaction.getTotalAmount() < 0) { + // Outgoing transaction - so this wallet paid the fee + balance -= transaction.getFeeAmount(); + } + } + return balance; + } + public List getWalletTransactions(String key58) throws ForeignBlockchainException { Context.propagate(bitcoinjContext); From 9224ffbf73a7ac060e6c3142decdb9711c7b5511 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 21:04:49 +0000 Subject: [PATCH 068/151] Cache transaction list for 2 minutes, and synchronize, to prevent the balance and transactions APIs both requesting at once. This ensures that only a single round of requests (per coin) is used for the wallettransactions and balance APIs. It also speeds up loading on subsequent requests. The 2 minute cache isn't much longer than the foreign block times, so shouldn't cause values to be too out of date. --- .../java/org/qortal/crosschain/Bitcoiny.java | 134 +++++++++++------- 1 file changed, 79 insertions(+), 55 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index b4cc389b..27c89e1f 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -39,6 +39,7 @@ import org.qortal.utils.Amounts; import org.qortal.utils.BitTwiddling; import com.google.common.hash.HashCode; +import org.qortal.utils.NTP; /** Bitcoin-like (Bitcoin, Litecoin, etc.) support */ public abstract class Bitcoiny implements ForeignBlockchain { @@ -53,6 +54,11 @@ public abstract class Bitcoiny implements ForeignBlockchain { protected final NetworkParameters params; + /** Cache recent transactions to speed up subsequent lookups */ + protected List transactionsCache; + protected Long transactionsCacheTimestamp; + protected static long TRANSACTIONS_CACHE_TIMEOUT = 2 * 60 * 1000L; // 2 minutes + /** Keys that have been previously marked as fully spent,
* i.e. keys with transactions but with no unspent outputs. */ protected final Set spentKeys = Collections.synchronizedSet(new HashSet<>()); @@ -353,69 +359,87 @@ public abstract class Bitcoiny implements ForeignBlockchain { } public List getWalletTransactions(String key58) throws ForeignBlockchainException { - Context.propagate(bitcoinjContext); - - Wallet wallet = walletFromDeterministicKey58(key58); - DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); - - keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); - keyChain.maybeLookAhead(); - - List keys = new ArrayList<>(keyChain.getLeafKeys()); - - Set walletTransactions = new HashSet<>(); - Set keySet = new HashSet<>(); - - // Set the number of consecutive empty batches required before giving up - final int numberOfAdditionalBatchesToSearch = 5; - - int unusedCounter = 0; - int ki = 0; - do { - boolean areAllKeysUnused = true; - - for (; ki < keys.size(); ++ki) { - DeterministicKey dKey = keys.get(ki); - - // Check for transactions - Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); - keySet.add(address.toString()); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - - // Ask for transaction history - if it's empty then key has never been used - List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); - - if (!historicTransactionHashes.isEmpty()) { - areAllKeysUnused = false; - - for (TransactionHash transactionHash : historicTransactionHashes) - walletTransactions.add(this.getTransaction(transactionHash.txHash)); + synchronized (this) { + // Serve from the cache if it's recent + if (transactionsCache != null && transactionsCacheTimestamp != null) { + Long now = NTP.getTime(); + boolean isCacheStale = (now != null && now - transactionsCacheTimestamp >= TRANSACTIONS_CACHE_TIMEOUT); + if (!isCacheStale) { + LOGGER.info("Serving transactions from cache"); + return transactionsCache; } } + LOGGER.info("Fetching transactions from ElectrumX"); - if (areAllKeysUnused) { - // No transactions - if (unusedCounter >= numberOfAdditionalBatchesToSearch) { - // ... and we've hit our search limit - break; + Context.propagate(bitcoinjContext); + + Wallet wallet = walletFromDeterministicKey58(key58); + DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); + + keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + keyChain.maybeLookAhead(); + + List keys = new ArrayList<>(keyChain.getLeafKeys()); + + Set walletTransactions = new HashSet<>(); + Set keySet = new HashSet<>(); + + // Set the number of consecutive empty batches required before giving up + final int numberOfAdditionalBatchesToSearch = 5; + + int unusedCounter = 0; + int ki = 0; + do { + boolean areAllKeysUnused = true; + + for (; ki < keys.size(); ++ki) { + DeterministicKey dKey = keys.get(ki); + + // Check for transactions + Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); + keySet.add(address.toString()); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); + + if (!historicTransactionHashes.isEmpty()) { + areAllKeysUnused = false; + + for (TransactionHash transactionHash : historicTransactionHashes) + walletTransactions.add(this.getTransaction(transactionHash.txHash)); + } } - // We haven't hit our search limit yet so increment the counter and keep looking - unusedCounter++; - } - else { - // Some keys in this batch were used, so reset the counter - unusedCounter = 0; - } - // Generate some more keys - keys.addAll(generateMoreKeys(keyChain)); + if (areAllKeysUnused) { + // No transactions + if (unusedCounter >= numberOfAdditionalBatchesToSearch) { + // ... and we've hit our search limit + break; + } + // We haven't hit our search limit yet so increment the counter and keep looking + unusedCounter++; + } else { + // Some keys in this batch were used, so reset the counter + unusedCounter = 0; + } - // Process new keys - } while (true); + // Generate some more keys + keys.addAll(generateMoreKeys(keyChain)); - Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); + // Process new keys + } while (true); - return walletTransactions.stream().map(t -> convertToSimpleTransaction(t, keySet)).sorted(newestTimestampFirstComparator).collect(Collectors.toList()); + Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); + + // Update cache and return + transactionsCacheTimestamp = NTP.getTime(); + transactionsCache = walletTransactions.stream() + .map(t -> convertToSimpleTransaction(t, keySet)) + .sorted(newestTimestampFirstComparator).collect(Collectors.toList()); + + return transactionsCache; + } } protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set keySet) { From 618aaaf243c1a5974126ad12045611b89df715a9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 21:06:36 +0000 Subject: [PATCH 069/151] Removed logs used for debugging only --- src/main/java/org/qortal/crosschain/Bitcoiny.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 27c89e1f..8039d54c 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -365,11 +365,9 @@ public abstract class Bitcoiny implements ForeignBlockchain { Long now = NTP.getTime(); boolean isCacheStale = (now != null && now - transactionsCacheTimestamp >= TRANSACTIONS_CACHE_TIMEOUT); if (!isCacheStale) { - LOGGER.info("Serving transactions from cache"); return transactionsCache; } } - LOGGER.info("Fetching transactions from ElectrumX"); Context.propagate(bitcoinjContext); From 694ea689c8284754160ea2d2ddf3c1088560d726 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 3 Feb 2022 21:14:41 +0000 Subject: [PATCH 070/151] Synchronize the loop and break out of it before fetching arbitrary data files. Hopeful fix for ConcurrentModificationException, and maybe a potential deadlock. --- .../arbitrary/ArbitraryDataFileManager.java | 66 ++++++++++++------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 342b142d..1b544434 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -84,36 +84,58 @@ public class ArbitraryDataFileManager extends Thread { private void processFileHashes(Long now) { try (final Repository repository = RepositoryManager.getRepository()) { - for (String hash58 : arbitraryDataFileHashResponses.keySet()) { - if (isStopping) { - return; - } + ArbitraryTransactionData arbitraryTransactionData = null; + byte[] signature = null; + byte[] hash = null; + Peer peer = null; + boolean shouldProcess = false; - Triple value = arbitraryDataFileHashResponses.get(hash58); - if (value != null) { - Peer peer = value.getA(); - String signature58 = value.getB(); - Long timestamp = value.getC(); - - if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { - // Ignore - to be deleted - continue; + synchronized (arbitraryDataFileHashResponses) { + for (String hash58 : arbitraryDataFileHashResponses.keySet()) { + if (isStopping) { + return; } - byte[] hash = Base58.decode(hash58); - byte[] signature = Base58.decode(signature58); + Triple value = arbitraryDataFileHashResponses.get(hash58); + if (value != null) { + peer = value.getA(); + String signature58 = value.getB(); + Long timestamp = value.getC(); - // Fetch the transaction data - ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); - if (arbitraryTransactionData == null) { - continue; + if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { + // Ignore - to be deleted + continue; + } + + hash = Base58.decode(hash58); + signature = Base58.decode(signature58); + + // Fetch the transaction data + arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); + if (arbitraryTransactionData == null) { + continue; + } + + // We want to process this file + shouldProcess = true; + break; } - - LOGGER.debug("Fetching file {} from peer {} via response queue...", hash58, peer); - this.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash)); } } + if (!shouldProcess) { + // Nothing to do + return; + } + + if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) { + return; + } + + String hash58 = Base58.encode(hash); + LOGGER.debug("Fetching file {} from peer {} via response queue...", hash58, peer); + this.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash)); + } catch (DataException e) { LOGGER.info("Unable to process file hashes: {}", e.getMessage()); } From 98a2dd04b8d6184f8bda141d19be34cdad913982 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Feb 2022 10:28:26 +0000 Subject: [PATCH 071/151] Fixed bug caused by improper handling of UPDATE_NAME transactions, similar to commit d16663f. --- .../controller/repository/NamesDatabaseIntegrityCheck.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index 0b941c0c..c8f65aa5 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -338,8 +338,9 @@ public class NamesDatabaseIntegrityCheck { } if ((transactionData instanceof UpdateNameTransactionData)) { UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData; + boolean hasReducedNewName = updateNameTransactionData.getReducedNewName() == null && !updateNameTransactionData.getReducedNewName().isEmpty(); if (Objects.equals(updateNameTransactionData.getName(), name) || - Objects.equals(updateNameTransactionData.getReducedNewName(), reducedName)) { + (hasReducedNewName && Objects.equals(updateNameTransactionData.getReducedNewName(), reducedName))) { transactions.add(transactionData); } } From c2bf37b8781c78639ae7f41fe4d3abf6a2c81791 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Feb 2022 11:37:41 +0000 Subject: [PATCH 072/151] Added transactionV5Timestamp featureTrigger to unit tests --- src/test/resources/test-chain-v2-founder-rewards.json | 3 ++- src/test/resources/test-chain-v2-leftover-reward.json | 3 ++- src/test/resources/test-chain-v2-minting.json | 3 ++- src/test/resources/test-chain-v2-qora-holder-extremes.json | 3 ++- src/test/resources/test-chain-v2-qora-holder.json | 3 ++- src/test/resources/test-chain-v2-reward-levels.json | 3 ++- src/test/resources/test-chain-v2-reward-scaling.json | 3 ++- src/test/resources/test-chain-v2.json | 3 ++- 8 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index 2b96da55..5a2ac599 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -48,7 +48,8 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, - "calcChainWeightTimestamp": 0 + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index 3ff0c8e7..f0ff5985 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -48,7 +48,8 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, - "calcChainWeightTimestamp": 0 + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index 94014868..b83789cb 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -48,7 +48,8 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, - "calcChainWeightTimestamp": 0 + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index 308461c1..d85484fb 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -48,7 +48,8 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, - "calcChainWeightTimestamp": 0 + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index 99adf1be..32bc9c57 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -48,7 +48,8 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, - "calcChainWeightTimestamp": 0 + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-reward-levels.json b/src/test/resources/test-chain-v2-reward-levels.json index a078119a..06fbffa9 100644 --- a/src/test/resources/test-chain-v2-reward-levels.json +++ b/src/test/resources/test-chain-v2-reward-levels.json @@ -48,7 +48,8 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 6, - "calcChainWeightTimestamp": 0 + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index e0faeec2..66bac366 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -48,7 +48,8 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, - "calcChainWeightTimestamp": 0 + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index e7347246..03e70a3b 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -48,7 +48,8 @@ "atFindNextTransactionFix": 0, "newBlockSigHeight": 999999, "shareBinFix": 999999, - "calcChainWeightTimestamp": 0 + "calcChainWeightTimestamp": 0, + "transactionV5Timestamp": 0 }, "genesisInfo": { "version": 4, From f52530b8482acd9708bb43da9e7bd37ca947ccd6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Feb 2022 11:58:43 +0000 Subject: [PATCH 073/151] More thorough approach to fetchAllTransactionsInvolvingName(), to fix an issue found in unit testing. --- .../controller/repository/NamesDatabaseIntegrityCheck.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index c8f65aa5..272bd5ba 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -332,7 +332,8 @@ public class NamesDatabaseIntegrityCheck { if ((transactionData instanceof RegisterNameTransactionData)) { RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData; - if (Objects.equals(registerNameTransactionData.getReducedName(), reducedName)) { + if (Objects.equals(registerNameTransactionData.getName(), name) || + Objects.equals(registerNameTransactionData.getReducedName(), reducedName)) { transactions.add(transactionData); } } @@ -340,7 +341,8 @@ public class NamesDatabaseIntegrityCheck { UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData; boolean hasReducedNewName = updateNameTransactionData.getReducedNewName() == null && !updateNameTransactionData.getReducedNewName().isEmpty(); if (Objects.equals(updateNameTransactionData.getName(), name) || - (hasReducedNewName && Objects.equals(updateNameTransactionData.getReducedNewName(), reducedName))) { + (hasReducedNewName && Objects.equals(updateNameTransactionData.getReducedNewName(), reducedName)) || + Objects.equals(updateNameTransactionData.getNewName(), name)) { transactions.add(transactionData); } } From 756d5e685aca289be8823a908dd3992f48244a28 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Feb 2022 12:50:05 +0000 Subject: [PATCH 074/151] Added naming tests for blank new names --- .../NamesDatabaseIntegrityCheck.java | 2 +- .../qortal/test/naming/IntegrityTests.java | 59 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index 272bd5ba..4214eccf 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -319,7 +319,7 @@ public class NamesDatabaseIntegrityCheck { this.nameTransactions = nameTransactions; } - private List fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException { + public List fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException { List transactions = new ArrayList<>(); String reducedName = Unicode.sanitize(name); diff --git a/src/test/java/org/qortal/test/naming/IntegrityTests.java b/src/test/java/org/qortal/test/naming/IntegrityTests.java index d278cf3a..419f6870 100644 --- a/src/test/java/org/qortal/test/naming/IntegrityTests.java +++ b/src/test/java/org/qortal/test/naming/IntegrityTests.java @@ -13,6 +13,8 @@ import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; import org.qortal.transaction.Transaction; +import java.util.List; + import static org.junit.Assert.*; public class IntegrityTests extends Common { @@ -45,6 +47,63 @@ public class IntegrityTests extends Common { } } + @Test + public void testBlankReducedName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "\uD83E\uDD73"; // Translates to a reducedName of "" + String data = "\uD83E\uDD73"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Ensure the name exists and the data is correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + + // Ensure the reducedName is blank + assertEquals("", repository.getNameRepository().fromName(name).getReducedName()); + + // Run the database integrity check for this name + NamesDatabaseIntegrityCheck integrityCheck = new NamesDatabaseIntegrityCheck(); + assertEquals(1, integrityCheck.rebuildName(name, repository)); + + // Ensure the name still exists and the data is still correct + assertEquals(data, repository.getNameRepository().fromName(name).getData()); + assertEquals("", repository.getNameRepository().fromName(name).getReducedName()); + } + } + + @Test + public void testUpdateWithBlankNewName() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name to Alice + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "initial_name"; + String data = "initial_data"; + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Update the name, but keep the new name blank + String newName = ""; + String newData = "updated_data"; + UpdateNameTransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), name, newName, newData); + TransactionUtils.signAndMint(repository, updateTransactionData, alice); + + // Ensure the original name exists and the data is correct + assertEquals(name, repository.getNameRepository().fromName(name).getName()); + assertEquals(newData, repository.getNameRepository().fromName(name).getData()); + + // Run the database integrity check for this name + NamesDatabaseIntegrityCheck integrityCheck = new NamesDatabaseIntegrityCheck(); + assertEquals(2, integrityCheck.rebuildName(name, repository)); + + // Ensure the name still exists and the data is still correct + assertEquals(name, repository.getNameRepository().fromName(name).getName()); + assertEquals(newData, repository.getNameRepository().fromName(name).getData()); + } + } + @Test public void testMissingName() throws DataException { try (final Repository repository = RepositoryManager.getRepository()) { From 4e59eb8958a20c22208a487456da97b69d6cee31 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Feb 2022 12:51:22 +0000 Subject: [PATCH 075/151] Added unit test to simulate the false association between previous a UPDATE_NAME transaction, and the emoji name with a blank reducedName. --- .../qortal/test/naming/IntegrityTests.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/test/java/org/qortal/test/naming/IntegrityTests.java b/src/test/java/org/qortal/test/naming/IntegrityTests.java index 419f6870..1a8af3ad 100644 --- a/src/test/java/org/qortal/test/naming/IntegrityTests.java +++ b/src/test/java/org/qortal/test/naming/IntegrityTests.java @@ -104,6 +104,37 @@ public class IntegrityTests extends Common { } } + @Test + public void testUpdateWithBlankNewNameAndBlankEmojiName() throws DataException { + // Attempt to simulate a real world problem where an emoji with blank reducedName + // confused the integrity check by associating it with previous UPDATE_NAME transactions + // due to them also having a blank "newReducedName" + + try (final Repository repository = RepositoryManager.getRepository()) { + // Register-name to Alice + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "initial_name"; + String data = "initial_data"; + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Update the name, but keep the new name blank + String newName = ""; + String newData = "updated_data"; + UpdateNameTransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), name, newName, newData); + TransactionUtils.signAndMint(repository, updateTransactionData, alice); + + // Register emoji name + PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); + String emojiName = "\uD83E\uDD73"; // Translates to a reducedName of "" + + // Ensure that the initial_name isn't associated with the emoji name + NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck(); + List transactions = namesDatabaseIntegrityCheck.fetchAllTransactionsInvolvingName(emojiName, repository); + assertEquals(0, transactions.size()); + } + } + @Test public void testMissingName() throws DataException { try (final Repository repository = RepositoryManager.getRepository()) { From 6dec65c5d959abf7554166d1c4662363bb7fa7d6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Feb 2022 13:58:05 +0000 Subject: [PATCH 076/151] Rewrite of fetchAllTransactionsInvolvingName() to avoid having to load all name transactions into memory. --- .../NamesDatabaseIntegrityCheck.java | 41 +-------- .../repository/TransactionRepository.java | 11 +++ .../HSQLDBTransactionRepository.java | 83 +++++++++++++++++++ .../qortal/test/naming/IntegrityTests.java | 1 - 4 files changed, 95 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index 4214eccf..0e65e746 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -320,46 +320,7 @@ public class NamesDatabaseIntegrityCheck { } public List fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException { - List transactions = new ArrayList<>(); - String reducedName = Unicode.sanitize(name); - - // Fetch all the confirmed name-modification transactions - if (this.nameTransactions.isEmpty()) { - this.fetchAllNameTransactions(repository); - } - - for (TransactionData transactionData : this.nameTransactions) { - - if ((transactionData instanceof RegisterNameTransactionData)) { - RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData; - if (Objects.equals(registerNameTransactionData.getName(), name) || - Objects.equals(registerNameTransactionData.getReducedName(), reducedName)) { - transactions.add(transactionData); - } - } - if ((transactionData instanceof UpdateNameTransactionData)) { - UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData; - boolean hasReducedNewName = updateNameTransactionData.getReducedNewName() == null && !updateNameTransactionData.getReducedNewName().isEmpty(); - if (Objects.equals(updateNameTransactionData.getName(), name) || - (hasReducedNewName && Objects.equals(updateNameTransactionData.getReducedNewName(), reducedName)) || - Objects.equals(updateNameTransactionData.getNewName(), name)) { - transactions.add(transactionData); - } - } - if ((transactionData instanceof BuyNameTransactionData)) { - BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData; - if (Objects.equals(buyNameTransactionData.getName(), name)) { - transactions.add(transactionData); - } - } - if ((transactionData instanceof SellNameTransactionData)) { - SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData; - if (Objects.equals(sellNameTransactionData.getName(), name)) { - transactions.add(transactionData); - } - } - } - return transactions; + return repository.getTransactionRepository().getTransactionsInvolvingName(name, ConfirmationStatus.CONFIRMED); } private TransactionData fetchLatestModificationTransactionInvolvingName(String registeredName, Repository repository) throws DataException { diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index b0e3a864..643de60a 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -125,6 +125,17 @@ public interface TransactionRepository { */ public byte[] getLatestAutoUpdateTransaction(TransactionType txType, int txGroupId, Integer service) throws DataException; + /** + * Returns signatures for all name-registration related transactions relating to supplied name. + * Note: this does not currently include ARBITRARY data relating to the name. + * + * @param name + * @param confirmationStatus + * @return + * @throws DataException + */ + public List getTransactionsInvolvingName(String name, ConfirmationStatus confirmationStatus) throws DataException; + /** * Returns list of transactions relating to specific asset ID. * diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index e326b498..141d5c1f 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -30,6 +30,7 @@ import org.qortal.repository.hsqldb.HSQLDBSaver; import org.qortal.transaction.Transaction.ApprovalStatus; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.utils.Base58; +import org.qortal.utils.Unicode; public class HSQLDBTransactionRepository implements TransactionRepository { @@ -700,6 +701,88 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } } + @Override + public List getTransactionsInvolvingName(String name, ConfirmationStatus confirmationStatus) throws DataException { + TransactionType[] transactionTypes = new TransactionType[] { + REGISTER_NAME, UPDATE_NAME, BUY_NAME, SELL_NAME + }; // TODO: CancelSellNameTransaction? + + String reducedName = Unicode.sanitize(name); + + StringBuilder sql = new StringBuilder(1024); + List bindParams = new ArrayList<>(); + sql.append("SELECT Transactions.signature FROM Transactions"); + + for (int ti = 0; ti < transactionTypes.length; ++ti) { + sql.append(" LEFT OUTER JOIN "); + sql.append(transactionTypes[ti].className); + sql.append("Transactions USING (signature)"); + } + + sql.append(" WHERE Transactions.type IN ("); + for (int ti = 0; ti < transactionTypes.length; ++ti) { + if (ti != 0) + sql.append(", "); + + sql.append(transactionTypes[ti].value); + } + sql.append(")"); + + // Confirmation status + switch (confirmationStatus) { + case BOTH: + break; + + case CONFIRMED: + sql.append(" AND Transactions.block_height IS NOT NULL"); + break; + + case UNCONFIRMED: + sql.append(" AND Transactions.block_height IS NULL"); + break; + } + + sql.append(" AND (RegisterNameTransactions.name = ?"); + bindParams.add(name); + sql.append(" OR RegisterNameTransactions.reduced_name = ?"); + bindParams.add(reducedName); + sql.append(" OR UpdateNameTransactions.name = ?"); + bindParams.add(name); + sql.append(" OR (UpdateNameTransactions.reduced_new_name != '' AND UpdateNameTransactions.reduced_new_name = ?)"); + bindParams.add(reducedName); + sql.append(" OR UpdateNameTransactions.new_name = ?"); + bindParams.add(name); + sql.append(" OR SellNameTransactions.name = ?"); + bindParams.add(name); + sql.append(" OR BuyNameTransactions.name = ?"); + bindParams.add(name); + + sql.append(") GROUP BY Transactions.signature, Transactions.created_when ORDER BY Transactions.created_when"); + + List transactions = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return transactions; + + do { + byte[] signature = resultSet.getBytes(1); + + TransactionData transactionData = this.fromSignature(signature); + + if (transactionData == null) + // Something inconsistent with the repository + throw new DataException("Unable to fetch name-related transaction from repository?"); + + transactions.add(transactionData); + } while (resultSet.next()); + + return transactions; + } catch (SQLException | DataException e) { + throw new DataException("Unable to fetch name-related transactions from repository", e); + } + } + @Override public List getAssetTransactions(long assetId, ConfirmationStatus confirmationStatus, Integer limit, Integer offset, Boolean reverse) throws DataException { diff --git a/src/test/java/org/qortal/test/naming/IntegrityTests.java b/src/test/java/org/qortal/test/naming/IntegrityTests.java index 1a8af3ad..7531bea6 100644 --- a/src/test/java/org/qortal/test/naming/IntegrityTests.java +++ b/src/test/java/org/qortal/test/naming/IntegrityTests.java @@ -125,7 +125,6 @@ public class IntegrityTests extends Common { TransactionUtils.signAndMint(repository, updateTransactionData, alice); // Register emoji name - PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); String emojiName = "\uD83E\uDD73"; // Translates to a reducedName of "" // Ensure that the initial_name isn't associated with the emoji name From 23bafb6233e620c6db751f963062dac2890a16bd Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Feb 2022 14:00:44 +0000 Subject: [PATCH 077/151] Removed unused methods --- .../NamesDatabaseIntegrityCheck.java | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index 0e65e746..97b6ff9d 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -11,7 +11,6 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.transaction.Transaction.TransactionType; -import org.qortal.utils.Unicode; import java.util.*; @@ -268,42 +267,6 @@ public class NamesDatabaseIntegrityCheck { return registerNameTransactions; } - private List fetchUpdateNameTransactions() { - List updateNameTransactions = new ArrayList<>(); - - for (TransactionData transactionData : this.nameTransactions) { - if (transactionData.getType() == TransactionType.UPDATE_NAME) { - UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData; - updateNameTransactions.add(updateNameTransactionData); - } - } - return updateNameTransactions; - } - - private List fetchSellNameTransactions() { - List sellNameTransactions = new ArrayList<>(); - - for (TransactionData transactionData : this.nameTransactions) { - if (transactionData.getType() == TransactionType.SELL_NAME) { - SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData; - sellNameTransactions.add(sellNameTransactionData); - } - } - return sellNameTransactions; - } - - private List fetchBuyNameTransactions() { - List buyNameTransactions = new ArrayList<>(); - - for (TransactionData transactionData : this.nameTransactions) { - if (transactionData.getType() == TransactionType.BUY_NAME) { - BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData; - buyNameTransactions.add(buyNameTransactionData); - } - } - return buyNameTransactions; - } - private void fetchAllNameTransactions(Repository repository) throws DataException { List nameTransactions = new ArrayList<>(); From 9e571b87e8f821c5b215951e57a78c6d603eaddb Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Feb 2022 16:17:31 +0000 Subject: [PATCH 078/151] Yet another rewrite of fetchAllTransactionsInvolvingName() - this time over 1000x faster since it doesn't involve joining the Transactions table. --- .../NamesDatabaseIntegrityCheck.java | 31 ++++++++++++++- .../repository/TransactionRepository.java | 17 +++++++++ .../HSQLDBTransactionRepository.java | 38 +++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index 97b6ff9d..235732ee 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -283,7 +283,36 @@ public class NamesDatabaseIntegrityCheck { } public List fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException { - return repository.getTransactionRepository().getTransactionsInvolvingName(name, ConfirmationStatus.CONFIRMED); + List signatures = new ArrayList<>(); + String reducedName = Unicode.sanitize(name); + + List registerNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria( + TransactionType.REGISTER_NAME, Arrays.asList("(name = ? OR reduced_name = ?)"), Arrays.asList(name, reducedName)); + signatures.addAll(registerNameTransactions); + + List updateNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria( + TransactionType.UPDATE_NAME, + Arrays.asList("(name = ? OR new_name = ? OR (reduced_new_name != '' AND reduced_new_name = ?))"), + Arrays.asList(name, name, reducedName)); + signatures.addAll(updateNameTransactions); + + List sellNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria( + TransactionType.SELL_NAME, Arrays.asList("name = ?"), Arrays.asList(name)); + signatures.addAll(sellNameTransactions); + + List buyNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria( + TransactionType.BUY_NAME, Arrays.asList("name = ?"), Arrays.asList(name)); + signatures.addAll(buyNameTransactions); + + List transactions = new ArrayList<>(); + for (byte[] signature : signatures) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + // Filter out any unconfirmed transactions + if (transactionData.getBlockHeight() != null && transactionData.getBlockHeight() > 0) { + transactions.add(transactionData); + } + } + return transactions; } private TransactionData fetchLatestModificationTransactionInvolvingName(String registeredName, Repository repository) throws DataException { diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index 643de60a..20096eb8 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -108,6 +108,23 @@ public interface TransactionRepository { public List getSignaturesMatchingCriteria(TransactionType txType, byte[] publicKey, Integer minBlockHeight, Integer maxBlockHeight) throws DataException; + /** + * Returns signatures for transactions that match search criteria. + *

+ * Alternate version that allows for custom where clauses and bind params. + * Only use for very specific use cases, such as the names integrity check. + * Not advised to be used otherwise, given that it could be possible for + * unsanitized inputs to be passed in if not careful. + * + * @param txType + * @param whereClauses + * @param bindParams + * @return + * @throws DataException + */ + public List getSignaturesMatchingCustomCriteria(TransactionType txType, List whereClauses, + List bindParams) throws DataException; + /** * Returns signature for latest auto-update transaction. *

diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index 141d5c1f..f228944e 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -656,6 +656,44 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } } + + public List getSignaturesMatchingCustomCriteria(TransactionType txType, List whereClauses, + List bindParams) throws DataException { + List signatures = new ArrayList<>(); + + StringBuilder sql = new StringBuilder(1024); + sql.append(String.format("SELECT signature FROM %sTransactions", txType.className)); + + if (!whereClauses.isEmpty()) { + sql.append(" WHERE "); + + final int whereClausesSize = whereClauses.size(); + for (int wci = 0; wci < whereClausesSize; ++wci) { + if (wci != 0) + sql.append(" AND "); + + sql.append(whereClauses.get(wci)); + } + } + + LOGGER.trace(() -> String.format("Transaction search SQL: %s", sql)); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { + if (resultSet == null) + return signatures; + + do { + byte[] signature = resultSet.getBytes(1); + + signatures.add(signature); + } while (resultSet.next()); + + return signatures; + } catch (SQLException e) { + throw new DataException("Unable to fetch matching transaction signatures from repository", e); + } + } + @Override public byte[] getLatestAutoUpdateTransaction(TransactionType txType, int txGroupId, Integer service) throws DataException { StringBuilder sql = new StringBuilder(1024); From 0cf2f7f2546975e259c6aaa5c489ab2ac7c28f4c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Feb 2022 16:18:00 +0000 Subject: [PATCH 079/151] Missing import from last commit --- .../controller/repository/NamesDatabaseIntegrityCheck.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index 235732ee..5e54b905 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -11,6 +11,7 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.utils.Unicode; import java.util.*; From 3fbb86fded85264d8eccc354952880d6ad985f80 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 4 Feb 2022 16:27:31 +0000 Subject: [PATCH 080/151] Added indexes, to make looking up name transactions by name around 5x faster. --- .../repository/hsqldb/HSQLDBDatabaseUpdates.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 065cfd0d..9588aaa6 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -945,6 +945,20 @@ public class HSQLDBDatabaseUpdates { stmt.execute("CREATE INDEX ArbitraryPeersHashIndex ON ArbitraryPeers (hash)"); break; + case 40: + // For looking up name registration transactions based on name or reduced name + stmt.execute("CREATE INDEX RegisterNameNameIndex ON RegisterNameTransactions (name)"); + stmt.execute("CREATE INDEX RegisterNameReducedNameIndex ON RegisterNameTransactions (reduced_name)"); + // For looking up update name transactions based on name, new name, or new reduced name + stmt.execute("CREATE INDEX UpdateNameNameIndex ON UpdateNameTransactions (name)"); + stmt.execute("CREATE INDEX UpdateNameNewNameIndex ON UpdateNameTransactions (new_name)"); + stmt.execute("CREATE INDEX UpdateNameReducedNewNameIndex ON UpdateNameTransactions (reduced_new_name)"); + // For looking up buy name transactions based on name + stmt.execute("CREATE INDEX BuyNameNameIndex ON BuyNameTransactions (name)"); + // For looking up sell name transactions based on name + stmt.execute("CREATE INDEX SellNameNameIndex ON SellNameTransactions (name)"); + break; + default: // nothing to do return false; From 8937b3ec86c276057444033d13bbfed0c86fdd84 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 5 Feb 2022 10:19:26 +0000 Subject: [PATCH 081/151] Don't allow duplicate transaction in the incoming transactions queue. This should reduce database load slightly, as it won't have to check the same transaction multiple times in each batch. --- src/main/java/org/qortal/controller/Controller.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index c4429346..51f91970 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1287,7 +1287,9 @@ public class Controller extends Thread { TransactionMessage transactionMessage = (TransactionMessage) message; TransactionData transactionData = transactionMessage.getTransactionData(); if (this.incomingTransactions.size() < MAX_INCOMING_TRANSACTIONS) { - this.incomingTransactions.add(transactionData); + if (!this.incomingTransactions.contains(transactionData)) { + this.incomingTransactions.add(transactionData); + } } } From 775e3c065e88f30839a2f94cdf56793e63767484 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 5 Feb 2022 10:23:25 +0000 Subject: [PATCH 082/151] Invalidate ElectrumX transactions cache when switching accounts. --- .../java/org/qortal/crosschain/Bitcoiny.java | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 8039d54c..616f713a 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -1,12 +1,6 @@ package org.qortal.crosschain; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; @@ -57,6 +51,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { /** Cache recent transactions to speed up subsequent lookups */ protected List transactionsCache; protected Long transactionsCacheTimestamp; + protected String transactionsCacheXpub; protected static long TRANSACTIONS_CACHE_TIMEOUT = 2 * 60 * 1000L; // 2 minutes /** Keys that have been previously marked as fully spent,
@@ -360,12 +355,14 @@ public abstract class Bitcoiny implements ForeignBlockchain { public List getWalletTransactions(String key58) throws ForeignBlockchainException { synchronized (this) { - // Serve from the cache if it's recent - if (transactionsCache != null && transactionsCacheTimestamp != null) { - Long now = NTP.getTime(); - boolean isCacheStale = (now != null && now - transactionsCacheTimestamp >= TRANSACTIONS_CACHE_TIMEOUT); - if (!isCacheStale) { - return transactionsCache; + // Serve from the cache if it's recent, and matches this xpub + if (Objects.equals(transactionsCacheXpub, key58)) { + if (transactionsCache != null && transactionsCacheTimestamp != null) { + Long now = NTP.getTime(); + boolean isCacheStale = (now != null && now - transactionsCacheTimestamp >= TRANSACTIONS_CACHE_TIMEOUT); + if (!isCacheStale) { + return transactionsCache; + } } } @@ -432,6 +429,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { // Update cache and return transactionsCacheTimestamp = NTP.getTime(); + transactionsCacheXpub = key58; transactionsCache = walletTransactions.stream() .map(t -> convertToSimpleTransaction(t, keySet)) .sorted(newestTimestampFirstComparator).collect(Collectors.toList()); From c6405340bcce284672f857cb2539489beeaee599 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 5 Feb 2022 11:33:28 +0000 Subject: [PATCH 083/151] minAccountLevelForBlockSubmissions reduced from 6 to 5 --- src/main/resources/blockchain.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 4df3f09a..3d0e5559 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -9,7 +9,7 @@ "defaultGroupId": 0, "oneNamePerAccount": true, "minAccountLevelToMint": 1, - "minAccountLevelForBlockSubmissions": 6, + "minAccountLevelForBlockSubmissions": 5, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 6, "founderEffectiveMintingLevel": 10, From 76df332b578f42c256c007405443f09d0ebcf98b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 5 Feb 2022 11:51:54 +0000 Subject: [PATCH 084/151] Check for null IP address before notifying of an external IP update. --- src/main/java/org/qortal/network/Network.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index 2f5d9bd9..2a25864a 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -1164,11 +1164,13 @@ public class Network { if (consecutiveReadings >= consecutiveReadingsRequired) { // Last 10 readings were the same - i.e. more than one peer agreed on the new IP address... String ip = ipAddressHistory.get(size - 1); - if (!Objects.equals(ip, this.ourExternalIpAddress)) { - // ... and the readings were different to our current recorded value, so - // update our external IP address value - this.ourExternalIpAddress = ip; - this.onExternalIpUpdate(ip); + if (ip != null && !Objects.equals(ip, "null")) { + if (!Objects.equals(ip, this.ourExternalIpAddress)) { + // ... and the readings were different to our current recorded value, so + // update our external IP address value + this.ourExternalIpAddress = ip; + this.onExternalIpUpdate(ip); + } } } } From 9692539a3fce6945ec3d373433a9cb2255ac39d2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 5 Feb 2022 17:24:33 +0000 Subject: [PATCH 085/151] Don't include fee in balance calculation (it looks like it could be double counting at the moment). --- src/main/java/org/qortal/crosschain/Bitcoiny.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 616f713a..18ef860a 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -344,11 +344,6 @@ public abstract class Bitcoiny implements ForeignBlockchain { List transactions = getWalletTransactions(key58).stream().sorted(oldestTimestampFirstComparator).collect(Collectors.toList()); for (SimpleTransaction transaction : transactions) { balance += transaction.getTotalAmount(); - - if (transaction.getTotalAmount() < 0) { - // Outgoing transaction - so this wallet paid the fee - balance -= transaction.getFeeAmount(); - } } return balance; } From 98831a9449b76fc36c6f8cc16746085299f08004 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 5 Feb 2022 17:53:49 +0000 Subject: [PATCH 086/151] Break out of the various loops in the cleanup manager if the thread is stopping. --- .../ArbitraryDataCleanupManager.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index e1eaa491..64916df5 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -239,6 +239,9 @@ public class ArbitraryDataCleanupManager extends Thread { // Delete random data associated with name if we're over our storage limit for this name // Use the DELETION_THRESHOLD, for the same reasons as above for (String followedName : storageManager.followedNames()) { + if (isStopping) { + return; + } if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) { this.storageLimitReachedForName(repository, followedName); } @@ -261,6 +264,9 @@ public class ArbitraryDataCleanupManager extends Thread { // Loop through each path and find those without matching signatures for (Path path : allPaths) { + if (isStopping) { + break; + } try { String[] contents = path.toFile().list(); if (contents == null || contents.length == 0) { @@ -287,6 +293,9 @@ public class ArbitraryDataCleanupManager extends Thread { private void checkForExpiredTransactions(Repository repository) { List expiredPaths = this.findPathsWithNoAssociatedTransaction(repository); for (Path expiredPath : expiredPaths) { + if (isStopping) { + return; + } LOGGER.info("Found path with no associated transaction: {}", expiredPath.toString()); this.safeDeleteDirectory(expiredPath.toFile(), "no matching transaction"); } @@ -308,6 +317,9 @@ public class ArbitraryDataCleanupManager extends Thread { // when they reach their storage limit Path dataPath = Paths.get(Settings.getInstance().getDataPath()); for (int i=0; i Date: Wed, 26 Jan 2022 22:03:33 +0000 Subject: [PATCH 087/151] Initial attempt to avoid an unnecessary block submission if one of our peers already has a higher weight chain. --- .../org/qortal/controller/BlockMinter.java | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 616fd611..4ed2a27f 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -20,6 +20,7 @@ import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; +import org.qortal.data.block.CommonBlockData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; import org.qortal.network.Peer; @@ -295,6 +296,20 @@ public class BlockMinter extends Thread { } } + try { + if (this.higherWeightChainExists(repository, bestWeight)) { + LOGGER.info("Higher weight chain found in peers, so not signing a block this round"); + repository.discardChanges(); // Need to remove blockchain lock so that it can actually sync + Thread.sleep(10 * 1000L); // Allow some time for syncing to occur + continue; + } + else { + LOGGER.debug("No higher weight chain found in peers"); + } + } catch (DataException e) { + LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway..."); + } + // Add unconfirmed transactions addUnconfirmedTransactions(repository, newBlock); @@ -462,6 +477,51 @@ public class BlockMinter extends Thread { } } + private BigInteger getOurChainWeightSinceBlock(Repository repository, BlockSummaryData commonBlock, List peerBlockSummaries) throws DataException { + final int commonBlockHeight = commonBlock.getHeight(); + final byte[] commonBlockSig = commonBlock.getSignature(); + int mutualHeight = commonBlockHeight; + + // Fetch our corresponding block summaries + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + List ourBlockSummaries = repository.getBlockRepository() + .getBlockSummaries(commonBlockHeight + 1, ourLatestBlockData.getHeight()); + if (!ourBlockSummaries.isEmpty()) { + Synchronizer.getInstance().populateBlockSummariesMinterLevels(repository, ourBlockSummaries); + } + + if (ourBlockSummaries != null && peerBlockSummaries != null) { + mutualHeight += Math.min(ourBlockSummaries.size(), peerBlockSummaries.size()); + } + return Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight); + } + + private boolean higherWeightChainExists(Repository repository, BigInteger blockCandidateWeight) throws DataException { + if (blockCandidateWeight == null) { + // Can't make decisions without knowing the block candidate weight + return false; + } + List peers = Network.getInstance().getHandshakedPeers(); + // Loop through handshaked peers and check for any new block candidates + for (Peer peer : peers) { + if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) { + // This peer has common block data + CommonBlockData commonBlockData = peer.getCommonBlockData(); + BlockSummaryData commonBlockSummaryData = commonBlockData.getCommonBlockSummary(); + if (commonBlockData.getChainWeight() != null) { + // The synchronizer has calculated this peer's chain weight + BigInteger ourChainWeightSinceCommonBlock = this.getOurChainWeightSinceBlock(repository, commonBlockSummaryData, commonBlockData.getBlockSummariesAfterCommonBlock()); + BigInteger ourChainWeight = ourChainWeightSinceCommonBlock.add(blockCandidateWeight); + BigInteger peerChainWeight = commonBlockData.getChainWeight(); + if (peerChainWeight.compareTo(ourChainWeight) >= 0) { + // This peer has a higher weight chain than ours (including our block candidate) + return true; + } + } + } + } + return false; + } private static void moderatedLog(Runnable logFunction) { // We only log if logging at TRACE or previous log timeout has expired if (!LOGGER.isTraceEnabled() && lastLogTimestamp != null && lastLogTimestamp + logTimeout > System.currentTimeMillis()) From cbf03d58c875e680c94c02853f6f5706ba6c0dca Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 26 Jan 2022 22:04:09 +0000 Subject: [PATCH 088/151] Made synchronizer method public as it is now also used by the block minter. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index e9090cf0..41ed06a8 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -1516,7 +1516,7 @@ public class Synchronizer extends Thread { return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates()); } - private void populateBlockSummariesMinterLevels(Repository repository, List blockSummaries) throws DataException { + public void populateBlockSummariesMinterLevels(Repository repository, List blockSummaries) throws DataException { final int firstBlockHeight = blockSummaries.get(0).getHeight(); for (int i = 0; i < blockSummaries.size(); ++i) { From 472e1da792d0d14edaf66be53fe523b276cbffe9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 26 Jan 2022 22:34:15 +0000 Subject: [PATCH 089/151] Added debug level logging in higherWeightChainExists() for better visibility. --- .../java/org/qortal/controller/BlockMinter.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 4ed2a27f..f4222750 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -1,6 +1,8 @@ package org.qortal.controller; import java.math.BigInteger; +import java.text.DecimalFormat; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -501,6 +503,8 @@ public class BlockMinter extends Thread { // Can't make decisions without knowing the block candidate weight return false; } + NumberFormat formatter = new DecimalFormat("0.###E0"); + List peers = Network.getInstance().getHandshakedPeers(); // Loop through handshaked peers and check for any new block candidates for (Peer peer : peers) { @@ -514,10 +518,18 @@ public class BlockMinter extends Thread { BigInteger ourChainWeight = ourChainWeightSinceCommonBlock.add(blockCandidateWeight); BigInteger peerChainWeight = commonBlockData.getChainWeight(); if (peerChainWeight.compareTo(ourChainWeight) >= 0) { - // This peer has a higher weight chain than ours (including our block candidate) + // This peer has a higher weight chain than ours + LOGGER.debug("Peer {} is on a higher weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight)); return true; + + } else { + LOGGER.debug("Peer {} is on a lower weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight)); } + } else { + LOGGER.debug("Peer {} has no chain weight", peer); } + } else { + LOGGER.debug("Peer {} has no common block data", peer); } } return false; From 170244e679ae7c56c201cdbb5c23543fb4b69304 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 17:04:45 +0000 Subject: [PATCH 090/151] More work on higher weight chain detection in the block minter. Added 30 second timeout, so that any errors should self correct. --- .../org/qortal/controller/BlockMinter.java | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index f4222750..ddbf1929 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -78,6 +78,10 @@ public class BlockMinter extends Thread { BlockRepository blockRepository = repository.getBlockRepository(); BlockData previousBlockData = null; + // Vars to keep track of blocks that were skipped due to chain weight + byte[] parentSignatureForLastLowWeightBlock = null; + Long timeOfLastLowWeightBlock = null; + List newBlocks = new ArrayList<>(); // Flags for tracking change in whether minting is possible, @@ -114,6 +118,14 @@ public class BlockMinter extends Thread { if (mintingAccountsData.isEmpty()) continue; + if (parentSignatureForLastLowWeightBlock != null) { + // The last iteration found a higher weight block in the network, so sleep for a while + // to allow is to sync the higher weight chain. We are sleeping here rather than when + // detected as we don't want to hold the blockchain lock open. + LOGGER.info("Sleeping for 10 seconds..."); + Thread.sleep(10 * 1000L); + } + // Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level // Note that minting accounts are actually reward-shares in Qortal Iterator madi = mintingAccountsData.iterator(); @@ -300,10 +312,26 @@ public class BlockMinter extends Thread { try { if (this.higherWeightChainExists(repository, bestWeight)) { - LOGGER.info("Higher weight chain found in peers, so not signing a block this round"); - repository.discardChanges(); // Need to remove blockchain lock so that it can actually sync - Thread.sleep(10 * 1000L); // Allow some time for syncing to occur - continue; + + // Check if the base block has updated since the last time we were here + if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null || + !Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) { + // We've switched to a different chain, so reset the timer + timeOfLastLowWeightBlock = NTP.getTime(); + } + parentSignatureForLastLowWeightBlock = previousBlockData.getSignature(); + + // If less than 30 seconds has passed since first detection the higher weight chain, + // we should skip our block submission to give us the opportunity to sync to the better chain + if (NTP.getTime() - timeOfLastLowWeightBlock < 30*1000L) { + LOGGER.info("Higher weight chain found in peers, so not signing a block this round"); + LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock); + continue; + } + else { + // More than 30 seconds have passed, so we should submit our block candidate anyway. + LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate..."); + } } else { LOGGER.debug("No higher weight chain found in peers"); @@ -312,6 +340,11 @@ public class BlockMinter extends Thread { LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway..."); } + // Clear variables that track low weight blocks + parentSignatureForLastLowWeightBlock = null; + timeOfLastLowWeightBlock = null; + + // Add unconfirmed transactions addUnconfirmedTransactions(repository, newBlock); From 60d71863dc5c75f37a38ab0ace41fed75f56fe47 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 17:11:30 +0000 Subject: [PATCH 091/151] Allow 3 seconds for the synchronizer to obtain a blockchain lock, to reduce missed attempts. --- src/main/java/org/qortal/controller/Synchronizer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 41ed06a8..25c10fd6 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -5,6 +5,7 @@ import java.security.SecureRandom; import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -862,7 +863,7 @@ public class Synchronizer extends Thread { // Make sure we're the only thread modifying the blockchain // If we're already synchronizing with another peer then this will also return fast ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock()) { + if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) // Wasn't peer's fault we couldn't sync LOGGER.debug("Synchronizer couldn't acquire blockchain lock"); return SynchronizationResult.NO_BLOCKCHAIN_LOCK; From 483e7549f8c88f1275e09534094304e1eb942c76 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:16:52 +0000 Subject: [PATCH 092/151] Revert "Moved log from INFO to DEBUG, as now the synchronizer is on its own thread it can occur more often than before." This reverts commit e2e87766faa0676a6bd334572743f2951918a379. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 25c10fd6..d5c7d1db 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -865,7 +865,7 @@ public class Synchronizer extends Thread { ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) // Wasn't peer's fault we couldn't sync - LOGGER.debug("Synchronizer couldn't acquire blockchain lock"); + LOGGER.info("Synchronizer couldn't acquire blockchain lock"); return SynchronizationResult.NO_BLOCKCHAIN_LOCK; } From 816b01c1fc6c5902fd835c9bab905a482f1cf2b2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:17:46 +0000 Subject: [PATCH 093/151] Fixed issue in rebase --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index d5c7d1db..6a5a5aad 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -863,7 +863,7 @@ public class Synchronizer extends Thread { // Make sure we're the only thread modifying the blockchain // If we're already synchronizing with another peer then this will also return fast ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) + if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) { // Wasn't peer's fault we couldn't sync LOGGER.info("Synchronizer couldn't acquire blockchain lock"); return SynchronizationResult.NO_BLOCKCHAIN_LOCK; From a4cbbb38683557f88f3fc4539500140361a814a8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 20:37:26 +0000 Subject: [PATCH 094/151] Moved block minter sleep to later in the process, otherwise it can remain there for longer than expected. --- .../java/org/qortal/controller/BlockMinter.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index ddbf1929..a6aa21a8 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -118,14 +118,6 @@ public class BlockMinter extends Thread { if (mintingAccountsData.isEmpty()) continue; - if (parentSignatureForLastLowWeightBlock != null) { - // The last iteration found a higher weight block in the network, so sleep for a while - // to allow is to sync the higher weight chain. We are sleeping here rather than when - // detected as we don't want to hold the blockchain lock open. - LOGGER.info("Sleeping for 10 seconds..."); - Thread.sleep(10 * 1000L); - } - // Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level // Note that minting accounts are actually reward-shares in Qortal Iterator madi = mintingAccountsData.iterator(); @@ -219,6 +211,14 @@ public class BlockMinter extends Thread { continue; } + if (parentSignatureForLastLowWeightBlock != null) { + // The last iteration found a higher weight block in the network, so sleep for a while + // to allow is to sync the higher weight chain. We are sleeping here rather than when + // detected as we don't want to hold the blockchain lock open. + LOGGER.info("Sleeping for 10 seconds..."); + Thread.sleep(10 * 1000L); + } + for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) { // First block does the AT heavy-lifting if (newBlocks.isEmpty()) { From 55b57021584e817e499a5289902083b64130f1a6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 30 Jan 2022 21:00:27 +0000 Subject: [PATCH 095/151] Invalidate last low weight block signature whenever the previous block data changes. --- src/main/java/org/qortal/controller/BlockMinter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index a6aa21a8..154f7e25 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -195,6 +195,9 @@ public class BlockMinter extends Thread { // Reduce log timeout logTimeout = 10 * 1000L; + + // Last low weight block is no longer valid + parentSignatureForLastLowWeightBlock = null; } // Discard accounts we have already built blocks with From 0fe2f226bca844ea757867c56d50ba38f0dfd60b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 11:23:28 +0000 Subject: [PATCH 096/151] Added invalidUnconfirmedTransactions map An incoming invalid unconfirmed transaction will be added to this map if its timestamp is more than 30 minutes old. This should allow enough time and opportunities for it to be imported and included in a block (allowing for re-orgs which could switch its status from invalid to valid). Once added, it will be removed after an hour to allow for another chance to be requested from any peers that still have it. If invalid again, it's added back to the map for another hour. This fixes a 24 hour long loop, where invalid transactions are requested over and over from peers that have already imported them. It could be improved further by periodically removing invalid unconfirmed transactions from the database, but this will be a higher risk. The results of this feature should be less network traffic, and less blockchain locks (which should ultimately increase the responsiveness of the synchronizer). --- .../org/qortal/controller/Controller.java | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 51f91970..fdb513ae 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -11,18 +11,7 @@ import java.security.Security; import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Deque; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Random; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -101,6 +90,11 @@ public class Controller extends Thread { private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms private static final int MAX_INCOMING_TRANSACTIONS = 5000; + /** Minimum time before considering an invalid unconfirmed transaction as "stale" */ + public static final long INVALID_TRANSACTION_STALE_TIMEOUT = 30 * 60 * 1000L; // ms + /** Minimum frequency to re-request stale unconfirmed transactions from peers, to recheck validity */ + public static final long INVALID_TRANSACTION_RECHECK_INTERVAL = 60 * 60 * 1000L; // ms + // To do with online accounts list private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 1 * 60 * 1000L; // ms @@ -147,6 +141,9 @@ public class Controller extends Thread { /** List of incoming transaction that are in the import queue */ private List incomingTransactions = Collections.synchronizedList(new ArrayList<>()); + /** List of recent invalid unconfirmed transactions */ + private Map invalidUnconfirmedTransactions = Collections.synchronizedMap(new HashMap<>()); + /** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly minted block. */ private final ReentrantLock blockchainLock = new ReentrantLock(); @@ -557,6 +554,8 @@ public class Controller extends Thread { // Process incoming transactions queue processIncomingTransactionsQueue(); + // Clean up invalid incoming transactions list + cleanupInvalidTransactionsList(now); // Clean up arbitrary data request cache ArbitraryDataManager.getInstance().cleanupRequestCache(now); @@ -1351,6 +1350,12 @@ public class Controller extends Thread { if (validationResult != ValidationResult.OK) { LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + Long now = NTP.getTime(); + if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { + LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", Base58.encode(transactionData.getSignature())); + // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it + invalidUnconfirmedTransactions.put(transactionData.getSignature(), NTP.getTime()); + } iterator.remove(); continue; } @@ -1366,6 +1371,15 @@ public class Controller extends Thread { } } + private void cleanupInvalidTransactionsList(Long now) { + if (now == null) { + return; + } + // Periodically remove invalid unconfirmed transactions from the list, so that they can be fetched again + final long minimumTimestamp = now - INVALID_TRANSACTION_RECHECK_INTERVAL; + invalidUnconfirmedTransactions.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < minimumTimestamp); + } + private void onNetworkGetBlockSummariesMessage(Peer peer, Message message) { GetBlockSummariesMessage getBlockSummariesMessage = (GetBlockSummariesMessage) message; final byte[] parentSignature = getBlockSummariesMessage.getParentSignature(); @@ -1561,6 +1575,12 @@ public class Controller extends Thread { try (final Repository repository = RepositoryManager.getRepository()) { for (byte[] signature : signatures) { + if (invalidUnconfirmedTransactions.get(signature) != null) { + // Previously invalid transaction - don't keep requesting it + // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks + continue; + } + // Do we have it already? (Before requesting transaction data itself) if (repository.getTransactionRepository().exists(signature)) { LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer)); From 8c03164ea5fed5165a70a1c1ee42e85c723ce68d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 11:55:07 +0000 Subject: [PATCH 097/151] Don't add expired transactions to invalidUnconfirmedTransactions, as there is no need to keep track of these. --- src/main/java/org/qortal/controller/Controller.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index fdb513ae..816458a5 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1350,11 +1350,13 @@ public class Controller extends Thread { if (validationResult != ValidationResult.OK) { LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); - Long now = NTP.getTime(); - if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { - LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", Base58.encode(transactionData.getSignature())); - // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it - invalidUnconfirmedTransactions.put(transactionData.getSignature(), NTP.getTime()); + if (validationResult != ValidationResult.TIMESTAMP_TOO_OLD) { + Long now = NTP.getTime(); + if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { + LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", Base58.encode(transactionData.getSignature())); + // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it + invalidUnconfirmedTransactions.put(transactionData.getSignature(), NTP.getTime()); + } } iterator.remove(); continue; From 08e06ba11ae89c75d74b3a4e2a15fee9d6746067 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 12:09:44 +0000 Subject: [PATCH 098/151] Fixed bugs preventing invalidUnconfirmedTransactions from working as intended. --- src/main/java/org/qortal/controller/Controller.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 816458a5..c01b1e48 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -142,7 +142,7 @@ public class Controller extends Thread { private List incomingTransactions = Collections.synchronizedList(new ArrayList<>()); /** List of recent invalid unconfirmed transactions */ - private Map invalidUnconfirmedTransactions = Collections.synchronizedMap(new HashMap<>()); + private Map invalidUnconfirmedTransactions = Collections.synchronizedMap(new HashMap<>()); /** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly minted block. */ private final ReentrantLock blockchainLock = new ReentrantLock(); @@ -1349,13 +1349,14 @@ public class Controller extends Thread { } if (validationResult != ValidationResult.OK) { - LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + final String signature58 = Base58.encode(transactionData.getSignature()); + LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58)); if (validationResult != ValidationResult.TIMESTAMP_TOO_OLD) { Long now = NTP.getTime(); if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { - LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", Base58.encode(transactionData.getSignature())); + LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58); // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it - invalidUnconfirmedTransactions.put(transactionData.getSignature(), NTP.getTime()); + invalidUnconfirmedTransactions.put(signature58, NTP.getTime()); } } iterator.remove(); @@ -1577,7 +1578,8 @@ public class Controller extends Thread { try (final Repository repository = RepositoryManager.getRepository()) { for (byte[] signature : signatures) { - if (invalidUnconfirmedTransactions.get(signature) != null) { + String signature58 = Base58.encode(signature); + if (invalidUnconfirmedTransactions.containsKey(signature58)) { // Previously invalid transaction - don't keep requesting it // It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks continue; From cfe0414d96d08259d27fee618becbf5189ea1576 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 12:35:41 +0000 Subject: [PATCH 099/151] Small rework of invalidUnconfirmedTransactions to specify the expiry time instead of the time added. This allows TIMESTAMP_TOO_OLD transactions to be tracked for a shorter time (10 minutes) than the other invalid transactions (60 minutes). Should reduce network traffic and db load around the time that transactions are expiring, as there is a lag before they are noticed and removed from each node. Due to the variance, it could cause other peers to request them again after deleting. They are now ignored for 10 minutes to avoid request spam. --- .../org/qortal/controller/Controller.java | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index c01b1e48..e69dc558 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -93,7 +93,10 @@ public class Controller extends Thread { /** Minimum time before considering an invalid unconfirmed transaction as "stale" */ public static final long INVALID_TRANSACTION_STALE_TIMEOUT = 30 * 60 * 1000L; // ms /** Minimum frequency to re-request stale unconfirmed transactions from peers, to recheck validity */ - public static final long INVALID_TRANSACTION_RECHECK_INTERVAL = 60 * 60 * 1000L; // ms + public static final long INVALID_TRANSACTION_RECHECK_INTERVAL = 60 * 60 * 1000L; // ms\ + /** Minimum frequency to re-request expired unconfirmed transactions from peers, to recheck validity + * This mainly exists to stop expired transactions from bloating the list */ + public static final long EXPIRED_TRANSACTION_RECHECK_INTERVAL = 10 * 60 * 1000L; // ms // To do with online accounts list private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms @@ -1351,13 +1354,17 @@ public class Controller extends Thread { if (validationResult != ValidationResult.OK) { final String signature58 = Base58.encode(transactionData.getSignature()); LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58)); - if (validationResult != ValidationResult.TIMESTAMP_TOO_OLD) { - Long now = NTP.getTime(); - if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { - LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58); - // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it - invalidUnconfirmedTransactions.put(signature58, NTP.getTime()); + Long now = NTP.getTime(); + if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { + Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL; + if (validationResult == ValidationResult.TIMESTAMP_TOO_OLD) { + // Use shorter recheck interval for expired transactions + expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL; } + Long expiry = now + expiryLength; + LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58); + // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it + invalidUnconfirmedTransactions.put(signature58, expiry); } iterator.remove(); continue; @@ -1379,8 +1386,7 @@ public class Controller extends Thread { return; } // Periodically remove invalid unconfirmed transactions from the list, so that they can be fetched again - final long minimumTimestamp = now - INVALID_TRANSACTION_RECHECK_INTERVAL; - invalidUnconfirmedTransactions.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < minimumTimestamp); + invalidUnconfirmedTransactions.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < now); } private void onNetworkGetBlockSummariesMessage(Peer peer, Message message) { From 3c526db52e1b4503d8066564429509a7fc716190 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 13:03:01 +0000 Subject: [PATCH 100/151] Fixed bug in build manager which would prevent future builds until the core was restarted. --- .../qortal/controller/arbitrary/ArbitraryDataBuilderThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index da7c7293..781c0a84 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -39,7 +39,7 @@ public class ArbitraryDataBuilderThread implements Runnable { Map.Entry next = buildManager.arbitraryDataBuildQueue .entrySet().stream() .filter(e -> e.getValue().isQueued()) - .findFirst().get(); + .findFirst().orElse(null); if (next == null) { continue; From 2740543abfec2973a6618f91ec4b5ace377f2ff8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 13:04:58 +0000 Subject: [PATCH 101/151] Added "async" and "attempts" parameters to GET /arbitrary/{service}/{name}* endpoints. async = fail immediately with 404 if missing, and request in the background attempts = the number of times to request the data (synchronous mode only for now) --- .../api/resource/ArbitraryResource.java | 51 +++++++++++++------ .../qortal/arbitrary/ArbitraryDataReader.java | 12 ++++- .../arbitrary/ArbitraryDataRenderer.java | 2 +- 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index f588e9c9..84e53200 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -576,14 +576,16 @@ public class ArbitraryResource { @PathParam("service") Service service, @PathParam("name") String name, @QueryParam("filepath") String filepath, - @QueryParam("rebuild") boolean rebuild) { + @QueryParam("rebuild") boolean rebuild, + @QueryParam("async") boolean async, + @QueryParam("attempts") Integer attempts) { // Authentication can be bypassed in the settings, for those running public QDN nodes if (!Settings.getInstance().isQDNAuthBypassEnabled()) { Security.checkApiCallAllowed(request); } - return this.download(service, name, null, filepath, rebuild); + return this.download(service, name, null, filepath, rebuild, async, attempts); } @GET @@ -609,14 +611,16 @@ public class ArbitraryResource { @PathParam("name") String name, @PathParam("identifier") String identifier, @QueryParam("filepath") String filepath, - @QueryParam("rebuild") boolean rebuild) { + @QueryParam("rebuild") boolean rebuild, + @QueryParam("async") boolean async, + @QueryParam("attempts") Integer attempts) { // Authentication can be bypassed in the settings, for those running public QDN nodes if (!Settings.getInstance().isQDNAuthBypassEnabled()) { Security.checkApiCallAllowed(request); } - return this.download(service, name, identifier, filepath, rebuild); + return this.download(service, name, identifier, filepath, rebuild, async, attempts); } @@ -1027,30 +1031,45 @@ public class ArbitraryResource { } } - private HttpServletResponse download(Service service, String name, String identifier, String filepath, boolean rebuild) { + private HttpServletResponse download(Service service, String name, String identifier, String filepath, boolean rebuild, boolean async, Integer maxAttempts) { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); try { int attempts = 0; + if (maxAttempts == null) { + maxAttempts = 5; + } // Loop until we have data - while (!Controller.isStopping()) { - attempts++; - if (!arbitraryDataReader.isBuilding()) { - try { - arbitraryDataReader.loadSynchronously(rebuild); - break; - } catch (MissingDataException e) { - if (attempts > 5) { - // Give up after 5 attempts - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); + if (async) { + // Asynchronous + arbitraryDataReader.loadAsynchronously(false); + } + else { + // Synchronous + while (!Controller.isStopping()) { + attempts++; + if (!arbitraryDataReader.isBuilding()) { + try { + arbitraryDataReader.loadSynchronously(rebuild); + break; + } catch (MissingDataException e) { + if (attempts > maxAttempts) { + // Give up after 5 attempts + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); + } } } + Thread.sleep(3000L); } - Thread.sleep(3000L); } + java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); + if (outputPath == null) { + // Assume the resource doesn't exist + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); + } if (filepath == null || filepath.isEmpty()) { // No file path supplied - so check if this is a single file resource diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 619e5330..bb5641c2 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -122,9 +122,19 @@ public class ArbitraryDataReader { * This adds the build task to a queue, and the result will be cached when complete * To check the status of the build, periodically call isCachedDataAvailable() * Once it returns true, you can then use getFilePath() to access the data itself. + * + * @param overwrite - set to true to force rebuild an existing cache * @return true if added or already present in queue; false if not */ - public boolean loadAsynchronously() { + public boolean loadAsynchronously(boolean overwrite) { + ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, + this.resourceId, this.resourceIdType, this.service, this.identifier); + if (cache.isCachedDataAvailable()) { + // Use cached data + this.filePath = this.uncompressedPath; + return true; + } + return ArbitraryDataBuildManager.getInstance().addToBuildQueue(this.createQueueItem()); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java index 445ff2f6..e4d90b79 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java @@ -76,7 +76,7 @@ public class ArbitraryDataRenderer { if (!arbitraryDataReader.isCachedDataAvailable()) { // If async is requested, show a loading screen whilst build is in progress if (async) { - arbitraryDataReader.loadAsynchronously(); + arbitraryDataReader.loadAsynchronously(false); return this.getLoadingResponse(service, resourceId); } From b8aaf14cdc9187f9702bc035a34d96ba99e67a8a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 15:34:06 +0000 Subject: [PATCH 102/151] Introduced ArbitraryDataFileRequestThread to allow for multiple concurrent file requests. This is likely a short term solution (to allow existing code to be repurposed) until replaced with a task-based approach, as this will allow for a much greater number of threads. --- .../arbitrary/ArbitraryDataFileManager.java | 75 ++--------- .../ArbitraryDataFileRequestThread.java | 117 ++++++++++++++++++ 2 files changed, 128 insertions(+), 64 deletions(-) create mode 100644 src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 1b544434..27433180 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -21,6 +21,8 @@ import org.qortal.utils.Triple; import java.security.SecureRandom; import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.stream.Collectors; public class ArbitraryDataFileManager extends Thread { @@ -65,11 +67,16 @@ public class ArbitraryDataFileManager extends Thread { Thread.currentThread().setName("Arbitrary Data File Manager"); try { - while (!isStopping) { - Thread.sleep(1000); + // Use a fixed thread pool to execute the arbitrary data file requests + int threadCount = 10; + ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount); + for (int i = 0; i < threadCount; i++) { + arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread()); + } - Long now = NTP.getTime(); - this.processFileHashes(now); + while (!isStopping) { + // Nothing to do yet + Thread.sleep(1000); } } catch (InterruptedException e) { // Fall-through to exit thread... @@ -81,66 +88,6 @@ public class ArbitraryDataFileManager extends Thread { this.interrupt(); } - private void processFileHashes(Long now) { - try (final Repository repository = RepositoryManager.getRepository()) { - - ArbitraryTransactionData arbitraryTransactionData = null; - byte[] signature = null; - byte[] hash = null; - Peer peer = null; - boolean shouldProcess = false; - - synchronized (arbitraryDataFileHashResponses) { - for (String hash58 : arbitraryDataFileHashResponses.keySet()) { - if (isStopping) { - return; - } - - Triple value = arbitraryDataFileHashResponses.get(hash58); - if (value != null) { - peer = value.getA(); - String signature58 = value.getB(); - Long timestamp = value.getC(); - - if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { - // Ignore - to be deleted - continue; - } - - hash = Base58.decode(hash58); - signature = Base58.decode(signature58); - - // Fetch the transaction data - arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); - if (arbitraryTransactionData == null) { - continue; - } - - // We want to process this file - shouldProcess = true; - break; - } - } - } - - if (!shouldProcess) { - // Nothing to do - return; - } - - if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) { - return; - } - - String hash58 = Base58.encode(hash); - LOGGER.debug("Fetching file {} from peer {} via response queue...", hash58, peer); - this.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash)); - - } catch (DataException e) { - LOGGER.info("Unable to process file hashes: {}", e.getMessage()); - } - } - public void cleanupRequestCache(Long now) { if (now == null) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java new file mode 100644 index 00000000..97704ae5 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -0,0 +1,117 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.network.Peer; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.ArbitraryTransactionUtils; +import org.qortal.utils.Base58; +import org.qortal.utils.NTP; +import org.qortal.utils.Triple; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; + +public class ArbitraryDataFileRequestThread implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class); + + public ArbitraryDataFileRequestThread() { + + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data File Request Thread"); + + try { + while (!Controller.isStopping()) { + Thread.sleep(1000); + + Long now = NTP.getTime(); + this.processFileHashes(now); + } + } catch (InterruptedException e) { + // Fall-through to exit thread... + } + } + + private void processFileHashes(Long now) { + try (final Repository repository = RepositoryManager.getRepository()) { + ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance(); + + ArbitraryTransactionData arbitraryTransactionData = null; + byte[] signature = null; + byte[] hash = null; + Peer peer = null; + boolean shouldProcess = false; + + synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) { + Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator(); + while (iterator.hasNext()) { + if (Controller.isStopping()) { + return; + } + + Map.Entry entry = (Map.Entry) iterator.next(); + if (entry == null || entry.getKey() == null || entry.getValue() == null) { + iterator.remove(); + continue; + } + + String hash58 = (String) entry.getKey(); + Triple value = (Triple) entry.getValue(); + if (value == null) { + iterator.remove(); + continue; + } + + peer = value.getA(); + String signature58 = value.getB(); + Long timestamp = value.getC(); + + if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { + // Ignore - to be deleted + iterator.remove(); + continue; + } + + hash = Base58.decode(hash58); + signature = Base58.decode(signature58); + + // We want to process this file + shouldProcess = true; + iterator.remove(); + break; + } + } + + if (!shouldProcess) { + // Nothing to do + return; + } + + // Fetch the transaction data + arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); + if (arbitraryTransactionData == null) { + return; + } + + if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) { + return; + } + + String hash58 = Base58.encode(hash); + LOGGER.debug("Fetching file {} from peer {} via request thread...", hash58, peer); + arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash)); + + } catch (DataException e) { + LOGGER.debug("Unable to process file hashes: {}", e.getMessage()); + } + } +} From ef838627c408e3b5138bf3bb147dc351053df17c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 15:37:08 +0000 Subject: [PATCH 103/151] Stop asking for hashes from a peer if one fails. This fixes the request looping that occurs on when a peer is unable to serve files. --- .../arbitrary/ArbitraryDataFileManager.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 27433180..44411e92 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -133,13 +133,19 @@ public class ArbitraryDataFileManager extends Thread { if (receivedArbitraryDataFileMessage != null) { LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFileMessage.getArbitraryDataFile().getHash58(), peer, (endTime-startTime)); receivedAtLeastOneFile = true; + + // Remove this hash from arbitraryDataFileHashResponses now that we have received it + arbitraryDataFileHashResponses.remove(hash58); } else { LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime)); - } - // Remove this hash from arbitraryDataFileHashResponses now that we have tried to request it - arbitraryDataFileHashResponses.remove(hash58); + // Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it + arbitraryDataFileHashResponses.remove(hash58); + + // Stop asking for files from this peer + break; + } } else { LOGGER.trace("Already requesting data file {} for signature {}", arbitraryDataFile, Base58.encode(signature)); From fa447ccded27f32ab067ec7c6932068fec6c200f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 15:37:21 +0000 Subject: [PATCH 104/151] Builder thread updates. --- .../controller/arbitrary/ArbitraryDataBuilderThread.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index 781c0a84..8da18a2b 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -20,8 +20,9 @@ public class ArbitraryDataBuilderThread implements Runnable { } + @Override public void run() { - Thread.currentThread().setName("Arbitrary Data Build Manager"); + Thread.currentThread().setName("Arbitrary Data Builder Thread"); ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance(); while (!Controller.isStopping()) { From 9ec4e24ef68150b72ce89d12dacec90b5bf1066c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 15:45:40 +0000 Subject: [PATCH 105/151] Slightly optimized logic in fetchArbitraryDataFiles() --- .../arbitrary/ArbitraryDataFileManager.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 44411e92..c7326c96 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -170,22 +170,23 @@ public class ArbitraryDataFileManager extends Thread { // Invalidate the hosted transactions cache as we are now hosting something new ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache(); - } - // Check if we have all the files we need for this transaction - if (arbitraryDataFile.allFilesExist()) { + // Check if we have all the files we need for this transaction + if (arbitraryDataFile.allFilesExist()) { - // We have all the chunks for this transaction, so we should invalidate the transaction's name's - // data cache so that it is rebuilt the next time we serve it - ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData); + // We have all the chunks for this transaction, so we should invalidate the transaction's name's + // data cache so that it is rebuilt the next time we serve it + ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData); - // We may also need to broadcast to the network that we are now hosting files for this transaction, - // but only if these files are in accordance with our storage policy - if (ArbitraryDataStorageManager.getInstance().canStoreData(arbitraryTransactionData)) { - // Use a null peer address to indicate our own - Message newArbitrarySignatureMessage = new ArbitrarySignaturesMessage(null, 0, Arrays.asList(signature)); - Network.getInstance().broadcast(broadcastPeer -> newArbitrarySignatureMessage); + // We may also need to broadcast to the network that we are now hosting files for this transaction, + // but only if these files are in accordance with our storage policy + if (ArbitraryDataStorageManager.getInstance().canStoreData(arbitraryTransactionData)) { + // Use a null peer address to indicate our own + Message newArbitrarySignatureMessage = new ArbitrarySignaturesMessage(null, 0, Arrays.asList(signature)); + Network.getInstance().broadcast(broadcastPeer -> newArbitrarySignatureMessage); + } } + } return receivedAtLeastOneFile; From cd5ce6dd5e00cad775812528015eb8517c967e62 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 16:04:54 +0000 Subject: [PATCH 106/151] Don't remove from the relay map after a file is requested, as it may be needed by other peers. It will be cleaned up automatically after 60 seconds, so it is best to keep the data intact until then. --- .../qortal/controller/arbitrary/ArbitraryDataFileManager.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index c7326c96..cab4a93e 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -432,9 +432,6 @@ public class ArbitraryDataFileManager extends Thread { // Forward the message to this peer LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58); this.fetchArbitraryDataFile(peerToAsk, peer, signature, hash, message); - - // Remove from the map regardless of outcome, as the relay attempt is now considered complete - arbitraryRelayMap.remove(hash58); } else { LOGGER.debug("Peer {} not found in relay info", peer); From 84e4f9a1c1bd9cda93bee555554fba3c2e11c845 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 17:20:01 +0000 Subject: [PATCH 107/151] Rework of arbitraryRelayMap to keep track of multiple responses. Previously, only one peer's response for a hash would be remembered, even if multiple others reported back too. This would cause useful mapping to be lost. --- .../ArbitraryDataFileListManager.java | 7 +-- .../arbitrary/ArbitraryDataFileManager.java | 54 +++++++++++++++-- .../data/arbitrary/ArbitraryRelayInfo.java | 60 +++++++++++++++++++ 3 files changed, 111 insertions(+), 10 deletions(-) create mode 100644 src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index 6337fc7c..46c2ff15 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataFileChunk; import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryRelayInfo; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; @@ -477,10 +478,8 @@ public class ArbitraryDataFileListManager { Long now = NTP.getTime(); for (byte[] hash : hashes) { String hash58 = Base58.encode(hash); - Triple value = new Triple<>(signature58, peer, now); - if (arbitraryDataFileManager.arbitraryRelayMap.putIfAbsent(hash58, value) == null) { - LOGGER.debug("Added {} to relay map: {}, {}, {}", hash58, signature58, peer, now); - } + ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now); + ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap); } // Forward to requesting peer diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index cab4a93e..8461448e 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.controller.Controller; +import org.qortal.data.arbitrary.ArbitraryRelayInfo; import org.qortal.data.network.ArbitraryPeerData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.ArbitraryTransactionData; @@ -39,10 +40,9 @@ public class ArbitraryDataFileManager extends Thread { private Map arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>()); /** - * Map to keep track of hashes that we might need to relay, keyed by the hash of the file (base58 encoded). - * Value is comprised of the base58-encoded signature, the peer that is hosting it, and the timestamp that it was added + * Map to keep track of hashes that we might need to relay */ - public Map> arbitraryRelayMap = Collections.synchronizedMap(new HashMap<>()); + public List arbitraryRelayMap = Collections.synchronizedList(new ArrayList<>()); /** * Map to keep track of any arbitrary data file hash responses @@ -97,7 +97,7 @@ public class ArbitraryDataFileManager extends Thread { arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < requestMinimumTimestamp); final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT; - arbitraryRelayMap.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp); + arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp); arbitraryDataFileHashResponses.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp); } @@ -391,6 +391,48 @@ public class ArbitraryDataFileManager extends Thread { } + // Relays + + private List getRelayInfoListForHash(String hash58) { + synchronized (arbitraryRelayMap) { + return arbitraryRelayMap.stream() + .filter(relayInfo -> Objects.equals(relayInfo.getHash58(), hash58)) + .collect(Collectors.toList()); + } + } + + private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) { + LOGGER.info("Fetching random relay info for hash: {}", hash58); + List relayInfoList = this.getRelayInfoListForHash(hash58); + if (relayInfoList != null && !relayInfoList.isEmpty()) { + + // Pick random item + int index = new SecureRandom().nextInt(relayInfoList.size()); + LOGGER.info("Returning random relay info for hash: {} (index {})", hash58, index); + return relayInfoList.get(index); + } + LOGGER.info("No relay info exists for hash: {}", hash58); + return null; + } + + public void addToRelayMap(ArbitraryRelayInfo newEntry) { + if (newEntry == null || !newEntry.isValid()) { + return; + } + + // Remove existing entry for this peer if it exists, to renew the timestamp + this.removeFromRelayMap(newEntry); + + // Re-add + arbitraryRelayMap.add(newEntry); + LOGGER.debug("Added entry to relay map: {}", newEntry); + } + + private void removeFromRelayMap(ArbitraryRelayInfo entry) { + arbitraryRelayMap.removeIf(relayInfo -> relayInfo.equals(entry)); + } + + // Network handlers public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) { @@ -409,7 +451,7 @@ public class ArbitraryDataFileManager extends Thread { try { ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature); - Triple relayInfo = this.arbitraryRelayMap.get(hash58); + ArbitraryRelayInfo relayInfo = this.getRandomRelayInfoEntryForHash(hash58); if (arbitraryDataFile.exists()) { LOGGER.trace("Hash {} exists", hash58); @@ -426,7 +468,7 @@ public class ArbitraryDataFileManager extends Thread { else if (relayInfo != null) { LOGGER.debug("We have relay info for hash {}", Base58.encode(hash)); // We need to ask this peer for the file - Peer peerToAsk = relayInfo.getB(); + Peer peerToAsk = relayInfo.getPeer(); if (peerToAsk != null) { // Forward the message to this peer diff --git a/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java b/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java new file mode 100644 index 00000000..94f41d18 --- /dev/null +++ b/src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java @@ -0,0 +1,60 @@ +package org.qortal.data.arbitrary; + +import org.qortal.network.Peer; +import java.util.Objects; + +public class ArbitraryRelayInfo { + + private final String hash58; + private final String signature58; + private final Peer peer; + private final Long timestamp; + + public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp) { + this.hash58 = hash58; + this.signature58 = signature58; + this.peer = peer; + this.timestamp = timestamp; + } + + public boolean isValid() { + return this.getHash58() != null && this.getSignature58() != null + && this.getPeer() != null && this.getTimestamp() != null; + } + + public String getHash58() { + return this.hash58; + } + + public String getSignature58() { + return signature58; + } + + public Peer getPeer() { + return peer; + } + + public Long getTimestamp() { + return timestamp; + } + + @Override + public String toString() { + return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp); + } + + @Override + public boolean equals(Object other) { + if (other == this) + return true; + + if (!(other instanceof ArbitraryRelayInfo)) + return false; + + ArbitraryRelayInfo otherRelayInfo = (ArbitraryRelayInfo) other; + + return this.peer == otherRelayInfo.getPeer() + && Objects.equals(this.hash58, otherRelayInfo.getHash58()) + && Objects.equals(this.signature58, otherRelayInfo.getSignature58()); + } +} From 3e0306f6467d51fd4ede6dca8c056c843b273ab4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 17:29:00 +0000 Subject: [PATCH 108/151] Increased minPeerConnectionTime and maxPeerConnectionTime to reduce the chances of forced connections during relays. An alternate option would be to avoid force disconnecting while relays are in progress, but some nodes could have active relays 100% of the time and therefore would never recycle their peers. So it is simpler to just increase the average peer connection time for everyone. --- src/main/java/org/qortal/settings/Settings.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 41b69114..dd62189f 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -202,9 +202,9 @@ public class Settings { private boolean allowConnectionsWithOlderPeerVersions = true; /** Minimum time (in seconds) that we should attempt to remain connected to a peer for */ - private int minPeerConnectionTime = 2 * 60; // seconds + private int minPeerConnectionTime = 5 * 60; // seconds /** Maximum time (in seconds) that we should attempt to remain connected to a peer for */ - private int maxPeerConnectionTime = 20 * 60; // seconds + private int maxPeerConnectionTime = 60 * 60; // seconds /** Whether to sync multiple blocks at once in normal operation */ private boolean fastSyncEnabled = true; From 99f6bb5ac6b17f8e15782d5a2c70883e0f6606cc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 6 Feb 2022 18:32:43 +0000 Subject: [PATCH 109/151] Reorganized some controller methods. --- .../org/qortal/controller/Controller.java | 339 +++++++++--------- 1 file changed, 174 insertions(+), 165 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index e69dc558..7c3caad5 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -822,6 +822,103 @@ public class Controller extends Thread { } } + // Incoming transactions queue + + private void processIncomingTransactionsQueue() { + if (this.incomingTransactions.size() == 0) { + // Don't bother locking if there are no new transactions to process + return; + } + + if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) { + // Prioritize syncing, and don't attempt to lock + return; + } + + try { + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + if (!blockchainLock.tryLock(2, TimeUnit.SECONDS)) { + LOGGER.trace(() -> String.format("Too busy to process incoming transactions queue")); + return; + } + } catch (InterruptedException e) { + LOGGER.info("Interrupted when trying to acquire blockchain lock"); + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + + // Iterate through incoming transactions list + synchronized (this.incomingTransactions) { // Required in order to safely iterate a synchronizedList() + Iterator iterator = this.incomingTransactions.iterator(); + while (iterator.hasNext()) { + if (isStopping) { + return; + } + + TransactionData transactionData = (TransactionData) iterator.next(); + Transaction transaction = Transaction.fromData(repository, transactionData); + + // Check signature + if (!transaction.isSignatureValid()) { + LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + iterator.remove(); + continue; + } + + ValidationResult validationResult = transaction.importAsUnconfirmed(); + + if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) { + LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature()))); + iterator.remove(); + continue; + } + + if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) { + LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction", Base58.encode(transactionData.getSignature()))); + iterator.remove(); + continue; + } + + if (validationResult != ValidationResult.OK) { + final String signature58 = Base58.encode(transactionData.getSignature()); + LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58)); + Long now = NTP.getTime(); + if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { + Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL; + if (validationResult == ValidationResult.TIMESTAMP_TOO_OLD) { + // Use shorter recheck interval for expired transactions + expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL; + } + Long expiry = now + expiryLength; + LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58); + // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it + invalidUnconfirmedTransactions.put(signature58, expiry); + } + iterator.remove(); + continue; + } + + LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + iterator.remove(); + } + } + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while processing incoming transactions", e)); + } finally { + blockchainLock.unlock(); + } + } + + private void cleanupInvalidTransactionsList(Long now) { + if (now == null) { + return; + } + // Periodically remove invalid unconfirmed transactions from the list, so that they can be fetched again + invalidUnconfirmedTransactions.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < now); + } + + // Shutdown public void shutdown() { @@ -1295,100 +1392,6 @@ public class Controller extends Thread { } } - private void processIncomingTransactionsQueue() { - if (this.incomingTransactions.size() == 0) { - // Don't bother locking if there are no new transactions to process - return; - } - - if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) { - // Prioritize syncing, and don't attempt to lock - return; - } - - try { - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - if (!blockchainLock.tryLock(2, TimeUnit.SECONDS)) { - LOGGER.trace(() -> String.format("Too busy to process incoming transactions queue")); - return; - } - } catch (InterruptedException e) { - LOGGER.info("Interrupted when trying to acquire blockchain lock"); - return; - } - - try (final Repository repository = RepositoryManager.getRepository()) { - - // Iterate through incoming transactions list - synchronized (this.incomingTransactions) { // Required in order to safely iterate a synchronizedList() - Iterator iterator = this.incomingTransactions.iterator(); - while (iterator.hasNext()) { - if (isStopping) { - return; - } - - TransactionData transactionData = (TransactionData) iterator.next(); - Transaction transaction = Transaction.fromData(repository, transactionData); - - // Check signature - if (!transaction.isSignatureValid()) { - LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); - iterator.remove(); - continue; - } - - ValidationResult validationResult = transaction.importAsUnconfirmed(); - - if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) { - LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature()))); - iterator.remove(); - continue; - } - - if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) { - LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction", Base58.encode(transactionData.getSignature()))); - iterator.remove(); - continue; - } - - if (validationResult != ValidationResult.OK) { - final String signature58 = Base58.encode(transactionData.getSignature()); - LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58)); - Long now = NTP.getTime(); - if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { - Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL; - if (validationResult == ValidationResult.TIMESTAMP_TOO_OLD) { - // Use shorter recheck interval for expired transactions - expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL; - } - Long expiry = now + expiryLength; - LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58); - // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it - invalidUnconfirmedTransactions.put(signature58, expiry); - } - iterator.remove(); - continue; - } - - LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); - iterator.remove(); - } - } - } catch (DataException e) { - LOGGER.error(String.format("Repository issue while processing incoming transactions", e)); - } finally { - blockchainLock.unlock(); - } - } - - private void cleanupInvalidTransactionsList(Long now) { - if (now == null) { - return; - } - // Periodically remove invalid unconfirmed transactions from the list, so that they can be fetched again - invalidUnconfirmedTransactions.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < now); - } - private void onNetworkGetBlockSummariesMessage(Peer peer, Message message) { GetBlockSummariesMessage getBlockSummariesMessage = (GetBlockSummariesMessage) message; final byte[] parentSignature = getBlockSummariesMessage.getParentSignature(); @@ -1784,88 +1787,94 @@ public class Controller extends Thread { private void sendOurOnlineAccountsInfo() { final Long now = NTP.getTime(); - if (now == null) - return; + if (now != null) { - List mintingAccounts; - try (final Repository repository = RepositoryManager.getRepository()) { - mintingAccounts = repository.getAccountRepository().getMintingAccounts(); + List mintingAccounts; + try (final Repository repository = RepositoryManager.getRepository()) { + mintingAccounts = repository.getAccountRepository().getMintingAccounts(); - // We have no accounts, but don't reset timestamp - if (mintingAccounts.isEmpty()) - return; + // We have no accounts, but don't reset timestamp + if (mintingAccounts.isEmpty()) + return; - // Only reward-share accounts allowed - Iterator iterator = mintingAccounts.iterator(); - while (iterator.hasNext()) { - MintingAccountData mintingAccountData = iterator.next(); - - RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey()); - if (rewardShareData == null) { - // Reward-share doesn't even exist - probably not a good sign - iterator.remove(); - continue; - } - - Account mintingAccount = new Account(repository, rewardShareData.getMinter()); - if (!mintingAccount.canMint()) { - // Minting-account component of reward-share can no longer mint - disregard - iterator.remove(); - continue; - } - } - } catch (DataException e) { - LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage())); - return; - } - - // 'current' timestamp - final long onlineAccountsTimestamp = Controller.toOnlineAccountTimestamp(now); - boolean hasInfoChanged = false; - - byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); - List ourOnlineAccounts = new ArrayList<>(); - - MINTING_ACCOUNTS: - for (MintingAccountData mintingAccountData : mintingAccounts) { - PrivateKeyAccount mintingAccount = new PrivateKeyAccount(null, mintingAccountData.getPrivateKey()); - - byte[] signature = mintingAccount.sign(timestampBytes); - byte[] publicKey = mintingAccount.getPublicKey(); - - // Our account is online - OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); - synchronized (this.onlineAccounts) { - Iterator iterator = this.onlineAccounts.iterator(); + // Only reward-share accounts allowed + Iterator iterator = mintingAccounts.iterator(); + int i = 0; while (iterator.hasNext()) { - OnlineAccountData existingOnlineAccountData = iterator.next(); + MintingAccountData mintingAccountData = iterator.next(); - if (Arrays.equals(existingOnlineAccountData.getPublicKey(), ourOnlineAccountData.getPublicKey())) { - // If our online account is already present, with same timestamp, then move on to next mintingAccount - if (existingOnlineAccountData.getTimestamp() == onlineAccountsTimestamp) - continue MINTING_ACCOUNTS; - - // If our online account is already present, but with older timestamp, then remove it + RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey()); + if (rewardShareData == null) { + // Reward-share doesn't even exist - probably not a good sign iterator.remove(); - break; + continue; + } + + Account mintingAccount = new Account(repository, rewardShareData.getMinter()); + if (!mintingAccount.canMint()) { + // Minting-account component of reward-share can no longer mint - disregard + iterator.remove(); + continue; + } + + if (++i > 2) { + iterator.remove(); + continue; } } - - this.onlineAccounts.add(ourOnlineAccountData); + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage())); + return; } - LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", mintingAccount.getAddress(), onlineAccountsTimestamp)); - ourOnlineAccounts.add(ourOnlineAccountData); - hasInfoChanged = true; + // 'current' timestamp + final long onlineAccountsTimestamp = Controller.toOnlineAccountTimestamp(now); + boolean hasInfoChanged = false; + + byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); + List ourOnlineAccounts = new ArrayList<>(); + + MINTING_ACCOUNTS: + for (MintingAccountData mintingAccountData : mintingAccounts) { + PrivateKeyAccount mintingAccount = new PrivateKeyAccount(null, mintingAccountData.getPrivateKey()); + + byte[] signature = mintingAccount.sign(timestampBytes); + byte[] publicKey = mintingAccount.getPublicKey(); + + // Our account is online + OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); + synchronized (this.onlineAccounts) { + Iterator iterator = this.onlineAccounts.iterator(); + while (iterator.hasNext()) { + OnlineAccountData existingOnlineAccountData = iterator.next(); + + if (Arrays.equals(existingOnlineAccountData.getPublicKey(), ourOnlineAccountData.getPublicKey())) { + // If our online account is already present, with same timestamp, then move on to next mintingAccount + if (existingOnlineAccountData.getTimestamp() == onlineAccountsTimestamp) + continue MINTING_ACCOUNTS; + + // If our online account is already present, but with older timestamp, then remove it + iterator.remove(); + break; + } + } + + this.onlineAccounts.add(ourOnlineAccountData); + } + + LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", mintingAccount.getAddress(), onlineAccountsTimestamp)); + ourOnlineAccounts.add(ourOnlineAccountData); + hasInfoChanged = true; + } + + if (!hasInfoChanged) + return; + + Message message = new OnlineAccountsMessage(ourOnlineAccounts); + Network.getInstance().broadcast(peer -> message); + + LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp)); } - - if (!hasInfoChanged) - return; - - Message message = new OnlineAccountsMessage(ourOnlineAccounts); - Network.getInstance().broadcast(peer -> message); - - LOGGER.trace(()-> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp)); } public static long toOnlineAccountTimestamp(long timestamp) { From f8ffb1a179cfc8943df367c6263bf9a81ae2539b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 7 Feb 2022 22:03:26 +0000 Subject: [PATCH 110/151] Updated thread names --- src/main/java/org/qortal/controller/Synchronizer.java | 2 ++ .../qortal/controller/arbitrary/ArbitraryDataBuildManager.java | 2 ++ .../qortal/controller/arbitrary/ArbitraryDataRenderManager.java | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index e9090cf0..07c4abba 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -110,6 +110,8 @@ public class Synchronizer extends Thread { @Override public void run() { + Thread.currentThread().setName("Synchronizer"); + try { while (running && !Controller.isStopping()) { Thread.sleep(1000); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java index 3df82d66..d607047e 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -37,6 +37,8 @@ public class ArbitraryDataBuildManager extends Thread { @Override public void run() { + Thread.currentThread().setName("Arbitrary Data Build Manager"); + try { // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) // This can be expanded to have multiple threads processing the build queue when needed diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java index 483ab92f..2844cef8 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java @@ -32,7 +32,7 @@ public class ArbitraryDataRenderManager extends Thread { @Override public void run() { - Thread.currentThread().setName("Arbitrary Data Manager"); + Thread.currentThread().setName("Arbitrary Data Render Manager"); try { while (!isStopping) { From 133943cd4ed4e23785a0e68393125e5dbfa70cd1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 7 Feb 2022 22:03:41 +0000 Subject: [PATCH 111/151] Reduce log spam --- .../controller/arbitrary/ArbitraryDataFileManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 8461448e..ff502ff4 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -402,16 +402,16 @@ public class ArbitraryDataFileManager extends Thread { } private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) { - LOGGER.info("Fetching random relay info for hash: {}", hash58); + LOGGER.trace("Fetching random relay info for hash: {}", hash58); List relayInfoList = this.getRelayInfoListForHash(hash58); if (relayInfoList != null && !relayInfoList.isEmpty()) { // Pick random item int index = new SecureRandom().nextInt(relayInfoList.size()); - LOGGER.info("Returning random relay info for hash: {} (index {})", hash58, index); + LOGGER.trace("Returning random relay info for hash: {} (index {})", hash58, index); return relayInfoList.get(index); } - LOGGER.info("No relay info exists for hash: {}", hash58); + LOGGER.trace("No relay info exists for hash: {}", hash58); return null; } From b6d633ab24b2fd0fcc31cfd79e91ba5a663db3f7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 7 Feb 2022 22:05:13 +0000 Subject: [PATCH 112/151] Break out of incoming transactions processing loop if we need to sync. --- src/main/java/org/qortal/controller/Controller.java | 5 +++++ src/main/java/org/qortal/controller/Synchronizer.java | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 7c3caad5..e61ee259 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -856,6 +856,11 @@ public class Controller extends Thread { return; } + if (Synchronizer.getInstance().isSyncRequestPending()) { + LOGGER.debug("Breaking out of transaction processing loop with {} remaining, because a sync request is pending", this.incomingTransactions.size()); + return; + } + TransactionData transactionData = (TransactionData) iterator.next(); Transaction transaction = Transaction.fromData(repository, transactionData); diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 07c4abba..1d604edd 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -82,6 +82,7 @@ public class Synchronizer extends Thread { private volatile int syncPercent = 0; private static volatile boolean requestSync = false; + private boolean syncRequestPending = false; // Keep track of invalid blocks so that we don't keep trying to sync them private Map invalidBlockSignatures = Collections.synchronizedMap(new HashMap<>()); @@ -123,6 +124,8 @@ public class Synchronizer extends Thread { // Something went wrong, so try again next time requestSync = true; } + // Remember that we have a pending sync request if this attempt failed + syncRequestPending = !success; } } } catch (InterruptedException e) { @@ -143,6 +146,10 @@ public class Synchronizer extends Thread { return this.isSynchronizing; } + public boolean isSyncRequestPending() { + return this.syncRequestPending; + } + public Integer getSyncPercent() { synchronized (this.syncLock) { return this.isSynchronizing ? this.syncPercent : null; From a49218a840f2e604b9f1dc77e3ac725c8c47bd9f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 7 Feb 2022 22:06:45 +0000 Subject: [PATCH 113/151] Optimized ArbitraryDataFileRequestThread - only start a database transaction when there's something to process. --- .../ArbitraryDataFileRequestThread.java | 100 +++++++++--------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index 97704ae5..62c22513 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -42,61 +42,61 @@ public class ArbitraryDataFileRequestThread implements Runnable { } private void processFileHashes(Long now) { - try (final Repository repository = RepositoryManager.getRepository()) { - ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance(); + ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance(); - ArbitraryTransactionData arbitraryTransactionData = null; - byte[] signature = null; - byte[] hash = null; - Peer peer = null; - boolean shouldProcess = false; + ArbitraryTransactionData arbitraryTransactionData = null; + byte[] signature = null; + byte[] hash = null; + Peer peer = null; + boolean shouldProcess = false; - synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) { - Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator(); - while (iterator.hasNext()) { - if (Controller.isStopping()) { - return; - } - - Map.Entry entry = (Map.Entry) iterator.next(); - if (entry == null || entry.getKey() == null || entry.getValue() == null) { - iterator.remove(); - continue; - } - - String hash58 = (String) entry.getKey(); - Triple value = (Triple) entry.getValue(); - if (value == null) { - iterator.remove(); - continue; - } - - peer = value.getA(); - String signature58 = value.getB(); - Long timestamp = value.getC(); - - if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { - // Ignore - to be deleted - iterator.remove(); - continue; - } - - hash = Base58.decode(hash58); - signature = Base58.decode(signature58); - - // We want to process this file - shouldProcess = true; - iterator.remove(); - break; + synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) { + Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator(); + while (iterator.hasNext()) { + if (Controller.isStopping()) { + return; } - } - if (!shouldProcess) { - // Nothing to do - return; - } + Map.Entry entry = (Map.Entry) iterator.next(); + if (entry == null || entry.getKey() == null || entry.getValue() == null) { + iterator.remove(); + continue; + } - // Fetch the transaction data + String hash58 = (String) entry.getKey(); + Triple value = (Triple) entry.getValue(); + if (value == null) { + iterator.remove(); + continue; + } + + peer = value.getA(); + String signature58 = value.getB(); + Long timestamp = value.getC(); + + if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { + // Ignore - to be deleted + iterator.remove(); + continue; + } + + hash = Base58.decode(hash58); + signature = Base58.decode(signature58); + + // We want to process this file + shouldProcess = true; + iterator.remove(); + break; + } + } + + if (!shouldProcess) { + // Nothing to do + return; + } + + // Fetch the transaction data + try (final Repository repository = RepositoryManager.getRepository()) { arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); if (arbitraryTransactionData == null) { return; From ab4ba9bb17540accb75cd5eb5fd41c34de61e938 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 8 Feb 2022 08:36:45 +0000 Subject: [PATCH 114/151] Don't re-fetch unconfirmed transactions that are already in the queue --- src/main/java/org/qortal/controller/Controller.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index e61ee259..b8fd2aa8 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -824,6 +824,12 @@ public class Controller extends Thread { // Incoming transactions queue + private boolean incomingTransactionQueueContains(byte[] signature) { + synchronized (incomingTransactions) { + return incomingTransactions.stream().anyMatch(t -> Arrays.equals(t.getSignature(), signature)); + } + } + private void processIncomingTransactionsQueue() { if (this.incomingTransactions.size() == 0) { // Don't bother locking if there are no new transactions to process @@ -1599,6 +1605,12 @@ public class Controller extends Thread { continue; } + // Ignore if this transaction is in the queue + if (incomingTransactionQueueContains(signature)) { + LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peer)); + continue; + } + // Do we have it already? (Before requesting transaction data itself) if (repository.getTransactionRepository().exists(signature)) { LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer)); From 0a88a0c95ea083460eb98eebafd279baeaf04da5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 8 Feb 2022 08:45:58 +0000 Subject: [PATCH 115/151] Perform the base58 decoding outside of the arbitraryDataFileHashResponses lock, to reduce the amount of waiting around by other threads. --- .../arbitrary/ArbitraryDataFileRequestThread.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index 62c22513..7e510407 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -45,8 +45,8 @@ public class ArbitraryDataFileRequestThread implements Runnable { ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance(); ArbitraryTransactionData arbitraryTransactionData = null; - byte[] signature = null; - byte[] hash = null; + String signature58 = null; + String hash58 = null; Peer peer = null; boolean shouldProcess = false; @@ -63,7 +63,7 @@ public class ArbitraryDataFileRequestThread implements Runnable { continue; } - String hash58 = (String) entry.getKey(); + hash58 = (String) entry.getKey(); Triple value = (Triple) entry.getValue(); if (value == null) { iterator.remove(); @@ -71,7 +71,7 @@ public class ArbitraryDataFileRequestThread implements Runnable { } peer = value.getA(); - String signature58 = value.getB(); + signature58 = value.getB(); Long timestamp = value.getC(); if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) { @@ -80,9 +80,6 @@ public class ArbitraryDataFileRequestThread implements Runnable { continue; } - hash = Base58.decode(hash58); - signature = Base58.decode(signature58); - // We want to process this file shouldProcess = true; iterator.remove(); @@ -95,6 +92,9 @@ public class ArbitraryDataFileRequestThread implements Runnable { return; } + byte[] hash = Base58.decode(hash58); + byte[] signature = Base58.decode(signature58); + // Fetch the transaction data try (final Repository repository = RepositoryManager.getRepository()) { arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature); @@ -106,7 +106,6 @@ public class ArbitraryDataFileRequestThread implements Runnable { return; } - String hash58 = Base58.encode(hash); LOGGER.debug("Fetching file {} from peer {} via request thread...", hash58, peer); arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash)); From b72153f62b9be303a55b5e7147c34e4828d1b14e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 8 Feb 2022 09:02:20 +0000 Subject: [PATCH 116/151] Renamed main thread from "Controller" to "Qortal" --- src/main/java/org/qortal/controller/Controller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index b8fd2aa8..0c9e30f6 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -515,7 +515,7 @@ public class Controller extends Thread { @Override public void run() { - Thread.currentThread().setName("Controller"); + Thread.currentThread().setName("Qortal"); final long repositoryBackupInterval = Settings.getInstance().getRepositoryBackupInterval(); final long repositoryCheckpointInterval = Settings.getInstance().getRepositoryCheckpointInterval(); From 9630625449c443a612097f8683fc9f3a5cde5478 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 8 Feb 2022 09:18:14 +0000 Subject: [PATCH 117/151] Rework of processIncomingTransactionsQueue() so that it no longer holds the lock while processing. This should fix an issue where network threads could be blocked when new transactions arrived, due to waiting for the incomingTransactions lock to free up. --- .../org/qortal/controller/Controller.java | 121 +++++++++--------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 0c9e30f6..0026be41 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -830,6 +830,10 @@ public class Controller extends Thread { } } + private void removeIncomingTransaction(byte[] signature) { + incomingTransactions.removeIf(t -> Arrays.equals(t.getSignature(), signature)); + } + private void processIncomingTransactionsQueue() { if (this.incomingTransactions.size() == 0) { // Don't bother locking if there are no new transactions to process @@ -853,70 +857,73 @@ public class Controller extends Thread { } try (final Repository repository = RepositoryManager.getRepository()) { + LOGGER.debug("Processing incoming transactions queue (size {})...", this.incomingTransactions.size()); + + // Take a copy of incomingTransactions so we can release the lock + ListincomingTransactionsCopy = new ArrayList<>(this.incomingTransactions); // Iterate through incoming transactions list - synchronized (this.incomingTransactions) { // Required in order to safely iterate a synchronizedList() - Iterator iterator = this.incomingTransactions.iterator(); - while (iterator.hasNext()) { - if (isStopping) { - return; - } - - if (Synchronizer.getInstance().isSyncRequestPending()) { - LOGGER.debug("Breaking out of transaction processing loop with {} remaining, because a sync request is pending", this.incomingTransactions.size()); - return; - } - - TransactionData transactionData = (TransactionData) iterator.next(); - Transaction transaction = Transaction.fromData(repository, transactionData); - - // Check signature - if (!transaction.isSignatureValid()) { - LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); - iterator.remove(); - continue; - } - - ValidationResult validationResult = transaction.importAsUnconfirmed(); - - if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) { - LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature()))); - iterator.remove(); - continue; - } - - if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) { - LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction", Base58.encode(transactionData.getSignature()))); - iterator.remove(); - continue; - } - - if (validationResult != ValidationResult.OK) { - final String signature58 = Base58.encode(transactionData.getSignature()); - LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58)); - Long now = NTP.getTime(); - if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { - Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL; - if (validationResult == ValidationResult.TIMESTAMP_TOO_OLD) { - // Use shorter recheck interval for expired transactions - expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL; - } - Long expiry = now + expiryLength; - LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58); - // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it - invalidUnconfirmedTransactions.put(signature58, expiry); - } - iterator.remove(); - continue; - } - - LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); - iterator.remove(); + Iterator iterator = incomingTransactionsCopy.iterator(); + while (iterator.hasNext()) { + if (isStopping) { + return; } + + if (Synchronizer.getInstance().isSyncRequestPending()) { + LOGGER.debug("Breaking out of transaction processing loop with {} remaining, because a sync request is pending", incomingTransactionsCopy.size()); + return; + } + + TransactionData transactionData = (TransactionData) iterator.next(); + Transaction transaction = Transaction.fromData(repository, transactionData); + + // Check signature + if (!transaction.isSignatureValid()) { + LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + removeIncomingTransaction(transactionData.getSignature()); + continue; + } + + ValidationResult validationResult = transaction.importAsUnconfirmed(); + + if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) { + LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature()))); + removeIncomingTransaction(transactionData.getSignature()); + continue; + } + + if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) { + LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction", Base58.encode(transactionData.getSignature()))); + removeIncomingTransaction(transactionData.getSignature()); + continue; + } + + if (validationResult != ValidationResult.OK) { + final String signature58 = Base58.encode(transactionData.getSignature()); + LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58)); + Long now = NTP.getTime(); + if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) { + Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL; + if (validationResult == ValidationResult.TIMESTAMP_TOO_OLD) { + // Use shorter recheck interval for expired transactions + expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL; + } + Long expiry = now + expiryLength; + LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58); + // Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it + invalidUnconfirmedTransactions.put(signature58, expiry); + } + removeIncomingTransaction(transactionData.getSignature()); + continue; + } + + LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()))); + removeIncomingTransaction(transactionData.getSignature()); } } catch (DataException e) { LOGGER.error(String.format("Repository issue while processing incoming transactions", e)); } finally { + LOGGER.debug("Finished processing incoming transactions queue"); blockchainLock.unlock(); } } From d1f24d45da79b233d7d6057a617fc5df2b601a77 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 8 Feb 2022 18:24:42 +0000 Subject: [PATCH 118/151] Added defensiveness in convertToSimpleTransaction() --- .../java/org/qortal/crosschain/Bitcoiny.java | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 18ef860a..2678a08e 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -447,13 +447,15 @@ public abstract class Bitcoiny implements ForeignBlockchain { List senders = t2.outputs.get(input.outputVout).addresses; long inputAmount = t2.outputs.get(input.outputVout).value; totalInputAmount += inputAmount; - for (String sender : senders) { - boolean addressInWallet = false; - if (keySet.contains(sender)) { - total += inputAmount; - addressInWallet = true; + if (senders != null) { + for (String sender : senders) { + boolean addressInWallet = false; + if (keySet.contains(sender)) { + total += inputAmount; + addressInWallet = true; + } + inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet)); } - inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet)); } } catch (ForeignBlockchainException e) { LOGGER.trace("Failed to retrieve transaction information {}", input.outputTxHash); @@ -461,17 +463,19 @@ public abstract class Bitcoiny implements ForeignBlockchain { } if (t.outputs != null && !t.outputs.isEmpty()) { for (BitcoinyTransaction.Output output : t.outputs) { - for (String address : output.addresses) { - boolean addressInWallet = false; - if (keySet.contains(address)) { - if (total > 0L) { - amount -= (total - output.value); - } else { - amount += output.value; + if (output.addresses != null) { + for (String address : output.addresses) { + boolean addressInWallet = false; + if (keySet.contains(address)) { + if (total > 0L) { + amount -= (total - output.value); + } else { + amount += output.value; + } + addressInWallet = true; } - addressInWallet = true; + outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet)); } - outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet)); } totalOutputAmount += output.value; } From 9804eccbf0aa1b5051afdf2e18d7b0554f931a71 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 8 Feb 2022 18:26:15 +0000 Subject: [PATCH 119/151] Removed transaction caching. Can be reintroduced later. --- .../java/org/qortal/crosschain/Bitcoiny.java | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 2678a08e..bf5bd230 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -33,7 +33,6 @@ import org.qortal.utils.Amounts; import org.qortal.utils.BitTwiddling; import com.google.common.hash.HashCode; -import org.qortal.utils.NTP; /** Bitcoin-like (Bitcoin, Litecoin, etc.) support */ public abstract class Bitcoiny implements ForeignBlockchain { @@ -48,12 +47,6 @@ public abstract class Bitcoiny implements ForeignBlockchain { protected final NetworkParameters params; - /** Cache recent transactions to speed up subsequent lookups */ - protected List transactionsCache; - protected Long transactionsCacheTimestamp; - protected String transactionsCacheXpub; - protected static long TRANSACTIONS_CACHE_TIMEOUT = 2 * 60 * 1000L; // 2 minutes - /** Keys that have been previously marked as fully spent,
* i.e. keys with transactions but with no unspent outputs. */ protected final Set spentKeys = Collections.synchronizedSet(new HashSet<>()); @@ -350,17 +343,6 @@ public abstract class Bitcoiny implements ForeignBlockchain { public List getWalletTransactions(String key58) throws ForeignBlockchainException { synchronized (this) { - // Serve from the cache if it's recent, and matches this xpub - if (Objects.equals(transactionsCacheXpub, key58)) { - if (transactionsCache != null && transactionsCacheTimestamp != null) { - Long now = NTP.getTime(); - boolean isCacheStale = (now != null && now - transactionsCacheTimestamp >= TRANSACTIONS_CACHE_TIMEOUT); - if (!isCacheStale) { - return transactionsCache; - } - } - } - Context.propagate(bitcoinjContext); Wallet wallet = walletFromDeterministicKey58(key58); @@ -423,13 +405,9 @@ public abstract class Bitcoiny implements ForeignBlockchain { Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); // Update cache and return - transactionsCacheTimestamp = NTP.getTime(); - transactionsCacheXpub = key58; - transactionsCache = walletTransactions.stream() + return walletTransactions.stream() .map(t -> convertToSimpleTransaction(t, keySet)) .sorted(newestTimestampFirstComparator).collect(Collectors.toList()); - - return transactionsCache; } } From 70c864bc2fda7f84a2015ef93ee0f451ee07d305 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 8 Feb 2022 18:27:08 +0000 Subject: [PATCH 120/151] Removed getWalletTransactions() synchronization. Again, can be re-added later. --- .../java/org/qortal/crosschain/Bitcoiny.java | 96 +++++++++---------- 1 file changed, 47 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index bf5bd230..fad05f6d 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -342,73 +342,71 @@ public abstract class Bitcoiny implements ForeignBlockchain { } public List getWalletTransactions(String key58) throws ForeignBlockchainException { - synchronized (this) { - Context.propagate(bitcoinjContext); + Context.propagate(bitcoinjContext); - Wallet wallet = walletFromDeterministicKey58(key58); - DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); + Wallet wallet = walletFromDeterministicKey58(key58); + DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); - keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); - keyChain.maybeLookAhead(); + keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + keyChain.maybeLookAhead(); - List keys = new ArrayList<>(keyChain.getLeafKeys()); + List keys = new ArrayList<>(keyChain.getLeafKeys()); - Set walletTransactions = new HashSet<>(); - Set keySet = new HashSet<>(); + Set walletTransactions = new HashSet<>(); + Set keySet = new HashSet<>(); - // Set the number of consecutive empty batches required before giving up - final int numberOfAdditionalBatchesToSearch = 5; + // Set the number of consecutive empty batches required before giving up + final int numberOfAdditionalBatchesToSearch = 5; - int unusedCounter = 0; - int ki = 0; - do { - boolean areAllKeysUnused = true; + int unusedCounter = 0; + int ki = 0; + do { + boolean areAllKeysUnused = true; - for (; ki < keys.size(); ++ki) { - DeterministicKey dKey = keys.get(ki); + for (; ki < keys.size(); ++ki) { + DeterministicKey dKey = keys.get(ki); - // Check for transactions - Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); - keySet.add(address.toString()); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + // Check for transactions + Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); + keySet.add(address.toString()); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - // Ask for transaction history - if it's empty then key has never been used - List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); - if (!historicTransactionHashes.isEmpty()) { - areAllKeysUnused = false; + if (!historicTransactionHashes.isEmpty()) { + areAllKeysUnused = false; - for (TransactionHash transactionHash : historicTransactionHashes) - walletTransactions.add(this.getTransaction(transactionHash.txHash)); - } + for (TransactionHash transactionHash : historicTransactionHashes) + walletTransactions.add(this.getTransaction(transactionHash.txHash)); } + } - if (areAllKeysUnused) { - // No transactions - if (unusedCounter >= numberOfAdditionalBatchesToSearch) { - // ... and we've hit our search limit - break; - } - // We haven't hit our search limit yet so increment the counter and keep looking - unusedCounter++; - } else { - // Some keys in this batch were used, so reset the counter - unusedCounter = 0; + if (areAllKeysUnused) { + // No transactions + if (unusedCounter >= numberOfAdditionalBatchesToSearch) { + // ... and we've hit our search limit + break; } + // We haven't hit our search limit yet so increment the counter and keep looking + unusedCounter++; + } else { + // Some keys in this batch were used, so reset the counter + unusedCounter = 0; + } - // Generate some more keys - keys.addAll(generateMoreKeys(keyChain)); + // Generate some more keys + keys.addAll(generateMoreKeys(keyChain)); - // Process new keys - } while (true); + // Process new keys + } while (true); - Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); + Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); - // Update cache and return - return walletTransactions.stream() - .map(t -> convertToSimpleTransaction(t, keySet)) - .sorted(newestTimestampFirstComparator).collect(Collectors.toList()); - } + // Update cache and return + return walletTransactions.stream() + .map(t -> convertToSimpleTransaction(t, keySet)) + .sorted(newestTimestampFirstComparator).collect(Collectors.toList()); } protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set keySet) { From d7658ee9f9bc8d7edef9e9189af1d0a129286a67 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 8 Feb 2022 18:27:44 +0000 Subject: [PATCH 121/151] Try a lookahead size of 20 (instead of 3) when asking Bitcoinj for the balance. --- src/main/java/org/qortal/crosschain/Bitcoiny.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index fad05f6d..bd2e165b 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -51,9 +51,14 @@ public abstract class Bitcoiny implements ForeignBlockchain { * i.e. keys with transactions but with no unspent outputs. */ protected final Set spentKeys = Collections.synchronizedSet(new HashSet<>()); - /** How many bitcoinj wallet keys to generate in each batch. */ + /** How many wallet keys to generate in each batch. */ private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3; + /** How many wallet keys to generate in each batch when using bitcoinj as the data provider. + * We must use a higher value here since we are unable to request multiple batches of keys. + * Without this, the bitcoinj balance (or other data) can be missing transactions. */ + private static final int WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ = 20; + /** Byte offset into raw block headers to block timestamp. */ private static final int TIMESTAMP_OFFSET = 4 + 32 + 32; @@ -549,7 +554,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { this.keyChain = this.wallet.getActiveKeyChain(); // Set up wallet's key chain - this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ); this.keyChain.maybeLookAhead(); } From 214f49e356ae0a28289736b731a1d0582dc6fb28 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 8 Feb 2022 18:29:32 +0000 Subject: [PATCH 122/151] Revert "Calculate wallet balances from the transactions (ElectrumX) rather than using bitcoinj." This reverts commit 892612c084ac34b0904d6deb72d5750d55e5abb7. # Conflicts: # src/main/java/org/qortal/crosschain/Bitcoiny.java --- .../api/resource/CrossChainBitcoinResource.java | 13 ++++--------- .../api/resource/CrossChainDogecoinResource.java | 13 ++++--------- .../api/resource/CrossChainLitecoinResource.java | 13 ++++--------- src/main/java/org/qortal/crosschain/Bitcoiny.java | 10 ---------- 4 files changed, 12 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java index 834c7b81..9bbf0e43 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java @@ -67,16 +67,11 @@ public class CrossChainBitcoinResource { if (!bitcoin.isValidDeterministicKey(key58)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - try { - Long balance = bitcoin.getWalletBalanceFromTransactions(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - - return balance.toString(); - - } catch (ForeignBlockchainException e) { + Long balance = bitcoin.getWalletBalance(key58); + if (balance == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - } + + return balance.toString(); } @POST diff --git a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java index 189a53d3..bb2dcbbc 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java @@ -65,16 +65,11 @@ public class CrossChainDogecoinResource { if (!dogecoin.isValidDeterministicKey(key58)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - try { - Long balance = dogecoin.getWalletBalanceFromTransactions(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - - return balance.toString(); - - } catch (ForeignBlockchainException e) { + Long balance = dogecoin.getWalletBalance(key58); + if (balance == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - } + + return balance.toString(); } @POST diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java index 627c00c7..8f6fa582 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java @@ -67,16 +67,11 @@ public class CrossChainLitecoinResource { if (!litecoin.isValidDeterministicKey(key58)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - try { - Long balance = litecoin.getWalletBalanceFromTransactions(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - - return balance.toString(); - - } catch (ForeignBlockchainException e) { + Long balance = litecoin.getWalletBalance(key58); + if (balance == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - } + + return balance.toString(); } @POST diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index bd2e165b..b9ed1951 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -336,16 +336,6 @@ public abstract class Bitcoiny implements ForeignBlockchain { return balance.value; } - public Long getWalletBalanceFromTransactions(String key58) throws ForeignBlockchainException { - long balance = 0; - Comparator oldestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp); - List transactions = getWalletTransactions(key58).stream().sorted(oldestTimestampFirstComparator).collect(Collectors.toList()); - for (SimpleTransaction transaction : transactions) { - balance += transaction.getTotalAmount(); - } - return balance; - } - public List getWalletTransactions(String key58) throws ForeignBlockchainException { Context.propagate(bitcoinjContext); From de5f31ac58a28fa706e885bd258e53db96fb5db5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 9 Feb 2022 19:40:20 +0000 Subject: [PATCH 123/151] Don't process file hashes if we're stopping --- .../controller/arbitrary/ArbitraryDataFileRequestThread.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java index 97704ae5..f1015916 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java @@ -42,6 +42,10 @@ public class ArbitraryDataFileRequestThread implements Runnable { } private void processFileHashes(Long now) { + if (Controller.isStopping()) { + return; + } + try (final Repository repository = RepositoryManager.getRepository()) { ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance(); From b782679d1f2acf9c2d097f9788baaabc306178f2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 9 Feb 2022 19:46:06 +0000 Subject: [PATCH 124/151] Revert "Revert "Calculate wallet balances from the transactions (ElectrumX) rather than using bitcoinj."" This reverts commit 214f49e356ae0a28289736b731a1d0582dc6fb28. --- .../api/resource/CrossChainBitcoinResource.java | 13 +++++++++---- .../api/resource/CrossChainDogecoinResource.java | 13 +++++++++---- .../api/resource/CrossChainLitecoinResource.java | 13 +++++++++---- src/main/java/org/qortal/crosschain/Bitcoiny.java | 10 ++++++++++ 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java index 9bbf0e43..834c7b81 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java @@ -67,11 +67,16 @@ public class CrossChainBitcoinResource { if (!bitcoin.isValidDeterministicKey(key58)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - Long balance = bitcoin.getWalletBalance(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + try { + Long balance = bitcoin.getWalletBalanceFromTransactions(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - return balance.toString(); + return balance.toString(); + + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } } @POST diff --git a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java index bb2dcbbc..189a53d3 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java @@ -65,11 +65,16 @@ public class CrossChainDogecoinResource { if (!dogecoin.isValidDeterministicKey(key58)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - Long balance = dogecoin.getWalletBalance(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + try { + Long balance = dogecoin.getWalletBalanceFromTransactions(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - return balance.toString(); + return balance.toString(); + + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } } @POST diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java index 8f6fa582..627c00c7 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java @@ -67,11 +67,16 @@ public class CrossChainLitecoinResource { if (!litecoin.isValidDeterministicKey(key58)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); - Long balance = litecoin.getWalletBalance(key58); - if (balance == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + try { + Long balance = litecoin.getWalletBalanceFromTransactions(key58); + if (balance == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); - return balance.toString(); + return balance.toString(); + + } catch (ForeignBlockchainException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); + } } @POST diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index b9ed1951..bd2e165b 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -336,6 +336,16 @@ public abstract class Bitcoiny implements ForeignBlockchain { return balance.value; } + public Long getWalletBalanceFromTransactions(String key58) throws ForeignBlockchainException { + long balance = 0; + Comparator oldestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp); + List transactions = getWalletTransactions(key58).stream().sorted(oldestTimestampFirstComparator).collect(Collectors.toList()); + for (SimpleTransaction transaction : transactions) { + balance += transaction.getTotalAmount(); + } + return balance; + } + public List getWalletTransactions(String key58) throws ForeignBlockchainException { Context.propagate(bitcoinjContext); From dda2316884aff930abf24e6722c518d4bb507412 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 9 Feb 2022 19:46:10 +0000 Subject: [PATCH 125/151] Revert "Try a lookahead size of 20 (instead of 3) when asking Bitcoinj for the balance." This reverts commit d7658ee9f9bc8d7edef9e9189af1d0a129286a67. --- src/main/java/org/qortal/crosschain/Bitcoiny.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index bd2e165b..fad05f6d 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -51,14 +51,9 @@ public abstract class Bitcoiny implements ForeignBlockchain { * i.e. keys with transactions but with no unspent outputs. */ protected final Set spentKeys = Collections.synchronizedSet(new HashSet<>()); - /** How many wallet keys to generate in each batch. */ + /** How many bitcoinj wallet keys to generate in each batch. */ private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3; - /** How many wallet keys to generate in each batch when using bitcoinj as the data provider. - * We must use a higher value here since we are unable to request multiple batches of keys. - * Without this, the bitcoinj balance (or other data) can be missing transactions. */ - private static final int WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ = 20; - /** Byte offset into raw block headers to block timestamp. */ private static final int TIMESTAMP_OFFSET = 4 + 32 + 32; @@ -554,7 +549,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { this.keyChain = this.wallet.getActiveKeyChain(); // Set up wallet's key chain - this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ); + this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); this.keyChain.maybeLookAhead(); } From b7b66f6cba577e8edb36b25d769256db4483bba5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 9 Feb 2022 19:46:16 +0000 Subject: [PATCH 126/151] Revert "Removed getWalletTransactions() synchronization. Again, can be re-added later." This reverts commit 70c864bc2fda7f84a2015ef93ee0f451ee07d305. --- .../java/org/qortal/crosschain/Bitcoiny.java | 96 ++++++++++--------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index fad05f6d..bf5bd230 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -342,71 +342,73 @@ public abstract class Bitcoiny implements ForeignBlockchain { } public List getWalletTransactions(String key58) throws ForeignBlockchainException { - Context.propagate(bitcoinjContext); + synchronized (this) { + Context.propagate(bitcoinjContext); - Wallet wallet = walletFromDeterministicKey58(key58); - DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); + Wallet wallet = walletFromDeterministicKey58(key58); + DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); - keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); - keyChain.maybeLookAhead(); + keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + keyChain.maybeLookAhead(); - List keys = new ArrayList<>(keyChain.getLeafKeys()); + List keys = new ArrayList<>(keyChain.getLeafKeys()); - Set walletTransactions = new HashSet<>(); - Set keySet = new HashSet<>(); + Set walletTransactions = new HashSet<>(); + Set keySet = new HashSet<>(); - // Set the number of consecutive empty batches required before giving up - final int numberOfAdditionalBatchesToSearch = 5; + // Set the number of consecutive empty batches required before giving up + final int numberOfAdditionalBatchesToSearch = 5; - int unusedCounter = 0; - int ki = 0; - do { - boolean areAllKeysUnused = true; + int unusedCounter = 0; + int ki = 0; + do { + boolean areAllKeysUnused = true; - for (; ki < keys.size(); ++ki) { - DeterministicKey dKey = keys.get(ki); + for (; ki < keys.size(); ++ki) { + DeterministicKey dKey = keys.get(ki); - // Check for transactions - Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); - keySet.add(address.toString()); - byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + // Check for transactions + Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); + keySet.add(address.toString()); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); - // Ask for transaction history - if it's empty then key has never been used - List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); - if (!historicTransactionHashes.isEmpty()) { - areAllKeysUnused = false; + if (!historicTransactionHashes.isEmpty()) { + areAllKeysUnused = false; - for (TransactionHash transactionHash : historicTransactionHashes) - walletTransactions.add(this.getTransaction(transactionHash.txHash)); + for (TransactionHash transactionHash : historicTransactionHashes) + walletTransactions.add(this.getTransaction(transactionHash.txHash)); + } } - } - if (areAllKeysUnused) { - // No transactions - if (unusedCounter >= numberOfAdditionalBatchesToSearch) { - // ... and we've hit our search limit - break; + if (areAllKeysUnused) { + // No transactions + if (unusedCounter >= numberOfAdditionalBatchesToSearch) { + // ... and we've hit our search limit + break; + } + // We haven't hit our search limit yet so increment the counter and keep looking + unusedCounter++; + } else { + // Some keys in this batch were used, so reset the counter + unusedCounter = 0; } - // We haven't hit our search limit yet so increment the counter and keep looking - unusedCounter++; - } else { - // Some keys in this batch were used, so reset the counter - unusedCounter = 0; - } - // Generate some more keys - keys.addAll(generateMoreKeys(keyChain)); + // Generate some more keys + keys.addAll(generateMoreKeys(keyChain)); - // Process new keys - } while (true); + // Process new keys + } while (true); - Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); + Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); - // Update cache and return - return walletTransactions.stream() - .map(t -> convertToSimpleTransaction(t, keySet)) - .sorted(newestTimestampFirstComparator).collect(Collectors.toList()); + // Update cache and return + return walletTransactions.stream() + .map(t -> convertToSimpleTransaction(t, keySet)) + .sorted(newestTimestampFirstComparator).collect(Collectors.toList()); + } } protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set keySet) { From 61f58173cba7cf297b609f894a7711cd9a6b2efb Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 9 Feb 2022 19:46:20 +0000 Subject: [PATCH 127/151] Revert "Removed transaction caching. Can be reintroduced later." This reverts commit 9804eccbf0aa1b5051afdf2e18d7b0554f931a71. --- .../java/org/qortal/crosschain/Bitcoiny.java | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index bf5bd230..2678a08e 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -33,6 +33,7 @@ import org.qortal.utils.Amounts; import org.qortal.utils.BitTwiddling; import com.google.common.hash.HashCode; +import org.qortal.utils.NTP; /** Bitcoin-like (Bitcoin, Litecoin, etc.) support */ public abstract class Bitcoiny implements ForeignBlockchain { @@ -47,6 +48,12 @@ public abstract class Bitcoiny implements ForeignBlockchain { protected final NetworkParameters params; + /** Cache recent transactions to speed up subsequent lookups */ + protected List transactionsCache; + protected Long transactionsCacheTimestamp; + protected String transactionsCacheXpub; + protected static long TRANSACTIONS_CACHE_TIMEOUT = 2 * 60 * 1000L; // 2 minutes + /** Keys that have been previously marked as fully spent,
* i.e. keys with transactions but with no unspent outputs. */ protected final Set spentKeys = Collections.synchronizedSet(new HashSet<>()); @@ -343,6 +350,17 @@ public abstract class Bitcoiny implements ForeignBlockchain { public List getWalletTransactions(String key58) throws ForeignBlockchainException { synchronized (this) { + // Serve from the cache if it's recent, and matches this xpub + if (Objects.equals(transactionsCacheXpub, key58)) { + if (transactionsCache != null && transactionsCacheTimestamp != null) { + Long now = NTP.getTime(); + boolean isCacheStale = (now != null && now - transactionsCacheTimestamp >= TRANSACTIONS_CACHE_TIMEOUT); + if (!isCacheStale) { + return transactionsCache; + } + } + } + Context.propagate(bitcoinjContext); Wallet wallet = walletFromDeterministicKey58(key58); @@ -405,9 +423,13 @@ public abstract class Bitcoiny implements ForeignBlockchain { Comparator newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed(); // Update cache and return - return walletTransactions.stream() + transactionsCacheTimestamp = NTP.getTime(); + transactionsCacheXpub = key58; + transactionsCache = walletTransactions.stream() .map(t -> convertToSimpleTransaction(t, keySet)) .sorted(newestTimestampFirstComparator).collect(Collectors.toList()); + + return transactionsCache; } } From 06b5b8f793b4dba01c746a937240c7e6037b93ee Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 9 Feb 2022 20:17:56 +0000 Subject: [PATCH 128/151] Reduced time between processing build tasks, to prevent builds with invalid criteria from holding up legitimate builds too much. --- .../qortal/controller/arbitrary/ArbitraryDataBuilderThread.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index 8da18a2b..f6c14669 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -27,7 +27,7 @@ public class ArbitraryDataBuilderThread implements Runnable { while (!Controller.isStopping()) { try { - Thread.sleep(1000); + Thread.sleep(100); if (buildManager.arbitraryDataBuildQueue == null) { continue; From 2637311ef59a5f53c8f19c1b565d73f2b063b276 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 9 Feb 2022 20:20:30 +0000 Subject: [PATCH 129/151] Prevent potential ConcurrentModificationException in the build queue --- .../arbitrary/ArbitraryDataBuilderThread.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index f6c14669..1c03daed 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -36,11 +36,15 @@ public class ArbitraryDataBuilderThread implements Runnable { continue; } + Map.Entry next = null; + // Find resources that are queued for building - Map.Entry next = buildManager.arbitraryDataBuildQueue - .entrySet().stream() - .filter(e -> e.getValue().isQueued()) - .findFirst().orElse(null); + synchronized (buildManager.arbitraryDataBuildQueue) { + next = buildManager.arbitraryDataBuildQueue + .entrySet().stream() + .filter(e -> e.getValue().isQueued()) + .findFirst().orElse(null); + } if (next == null) { continue; From a8c79b807baa3497e22a91bb9d6b04f40a72fc13 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 10 Feb 2022 08:16:30 +0000 Subject: [PATCH 130/151] Discard any uncommitted changes as a result of the higher weight chain detection --- src/main/java/org/qortal/controller/BlockMinter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 154f7e25..d7d1dd48 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -343,6 +343,9 @@ public class BlockMinter extends Thread { LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway..."); } + // Discard any uncommitted changes as a result of the higher weight chain detection + repository.discardChanges(); + // Clear variables that track low weight blocks parentSignatureForLastLowWeightBlock = null; timeOfLastLowWeightBlock = null; From 9332d7207ead46c2b0a126e65f03d20965fffec6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 10 Feb 2022 09:22:54 +0000 Subject: [PATCH 131/151] Fixed bug in cache clearing logic, which was often preventing resource updates from being detected. --- .../org/qortal/arbitrary/ArbitraryDataCache.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java index cfe445e2..accd808d 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java @@ -61,6 +61,9 @@ public class ArbitraryDataCache { } // No need to invalidate the cache + // Remember that it's up to date, so that we won't check again for a while + ArbitraryDataManager.getInstance().addResourceToCache(this.getArbitraryDataResource()); + return false; } @@ -84,14 +87,7 @@ public class ArbitraryDataCache { // If the state's sig doesn't match the latest transaction's sig, we need to invalidate // This means that an updated layer is available - if (this.shouldInvalidateDueToSignatureMismatch()) { - - // Add to the in-memory cache first, so that we won't check again for a while - ArbitraryDataManager.getInstance().addResourceToCache(this.getArbitraryDataResource()); - return true; - } - - return false; + return this.shouldInvalidateDueToSignatureMismatch(); } /** From c0c50f2e189f6e3fde96b8441a04077057f82726 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 13:33:25 +0000 Subject: [PATCH 132/151] Updated bootstrap hosts --- src/main/java/org/qortal/settings/Settings.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index dd62189f..45f89697 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -243,7 +243,8 @@ public class Settings { private String[] bootstrapHosts = new String[] { "http://bootstrap.qortal.org", "http://bootstrap2.qortal.org", - "http://cinfu1.crowetic.com" + "http://81.169.136.59", + "http://62.171.190.193" }; // Auto-update sources From fc82f0b62222ba4ae80f0e8a8930f4de1b9914a9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 13:58:45 +0000 Subject: [PATCH 133/151] Use 5 builder threads, so that one slow resource (e.g. a thumbnail) doesn't hold up the other queued build items. This can be replaced with a task-based approach longer term. --- .../ArbitraryDataBuildQueueItem.java | 9 +++- .../arbitrary/ArbitraryDataBuildManager.java | 7 ++- .../arbitrary/ArbitraryDataBuilderThread.java | 50 ++++++++++--------- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index ffbf8fe3..ddbf9f24 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -27,13 +27,20 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource { this.creationTimestamp = NTP.getTime(); } + public void prepareForBuild() { + this.buildStartTimestamp = NTP.getTime(); + } + public void build() throws IOException, DataException, MissingDataException { Long now = NTP.getTime(); if (now == null) { + this.buildStartTimestamp = null; throw new DataException("NTP time hasn't synced yet"); } - this.buildStartTimestamp = now; + if (this.buildStartTimestamp == null) { + this.buildStartTimestamp = now; + } ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java index d607047e..0054356e 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -42,8 +42,11 @@ public class ArbitraryDataBuildManager extends Thread { try { // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) // This can be expanded to have multiple threads processing the build queue when needed - ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); - arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread()); + int threadCount = 5; + ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(threadCount); + for (int i = 0; i < threadCount; i++) { + arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread()); + } while (!isStopping) { // Nothing to do yet diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index 1c03daed..17808daa 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -36,37 +36,41 @@ public class ArbitraryDataBuilderThread implements Runnable { continue; } - Map.Entry next = null; - - // Find resources that are queued for building - synchronized (buildManager.arbitraryDataBuildQueue) { - next = buildManager.arbitraryDataBuildQueue - .entrySet().stream() - .filter(e -> e.getValue().isQueued()) - .findFirst().orElse(null); - } - - if (next == null) { - continue; - } - Long now = NTP.getTime(); if (now == null) { continue; } - ArbitraryDataBuildQueueItem queueItem = next.getValue(); + ArbitraryDataBuildQueueItem queueItem = null; - if (queueItem == null) { - this.removeFromQueue(queueItem); + // Find resources that are queued for building + synchronized (buildManager.arbitraryDataBuildQueue) { + Map.Entry next = buildManager.arbitraryDataBuildQueue + .entrySet().stream() + .filter(e -> e.getValue().isQueued()) + .findFirst().orElse(null); + + if (next == null) { + continue; + } + + queueItem = next.getValue(); + + if (queueItem == null) { + this.removeFromQueue(queueItem); + continue; + } + + // Ignore builds that have failed recently + if (buildManager.isInFailedBuildsList(queueItem)) { + this.removeFromQueue(queueItem); + continue; + } + + // Set the start timestamp, to prevent other threads from building it at the same time + queueItem.prepareForBuild(); } - // Ignore builds that have failed recently - if (buildManager.isInFailedBuildsList(queueItem)) { - continue; - } - - try { // Perform the build LOGGER.info("Building {}...", queueItem); From 2343e739d10febb3d65dfa207c2b9a11e8ac5859 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 14:35:46 +0000 Subject: [PATCH 134/151] Handle case where a file cannot be unzipped. --- .../java/org/qortal/arbitrary/ArbitraryDataReader.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index bb5641c2..3a3f84eb 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -468,12 +468,18 @@ public class ArbitraryDataReader { throw new DataException(String.format("Unable to unzip file: %s", e.getMessage())); } - // Replace filePath pointer with the uncompressed file path + if (!this.uncompressedPath.toFile().exists()) { + throw new DataException(String.format("Unable to unzip file: %s", this.filePath)); + } + + // Delete original compressed file if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) { if (Files.exists(this.filePath)) { Files.delete(this.filePath); } } + + // Replace filePath pointer with the uncompressed file path this.filePath = this.uncompressedPath; } From 6932fb9935d51c71781f02c82dcdd07277cd5e07 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 15:08:12 +0000 Subject: [PATCH 135/151] Added "priority" property to build queue items. /render APIs use priority 10, whereas /arbitrary use priority 0, to prevent thumbnail downloads from holding up website loading. The priorities can be adjusted later, with maybe some service types being given higher priority than others. --- .../java/org/qortal/api/resource/ArbitraryResource.java | 2 +- .../qortal/arbitrary/ArbitraryDataBuildQueueItem.java | 9 +++++++++ .../java/org/qortal/arbitrary/ArbitraryDataReader.java | 6 ++++-- .../java/org/qortal/arbitrary/ArbitraryDataRenderer.java | 2 +- .../controller/arbitrary/ArbitraryDataBuilderThread.java | 6 ++++-- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 84e53200..430ff83f 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1044,7 +1044,7 @@ public class ArbitraryResource { // Loop until we have data if (async) { // Asynchronous - arbitraryDataReader.loadAsynchronously(false); + arbitraryDataReader.loadAsynchronously(false, 0); } else { // Synchronous diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index ddbf9f24..78f3f00a 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -13,6 +13,7 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource { private final Long creationTimestamp; private Long buildStartTimestamp = null; private Long buildEndTimestamp = null; + private Integer priority = 0; private boolean failed = false; /* The maximum amount of time to spend on a single build */ @@ -77,6 +78,14 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource { return this.buildStartTimestamp; } + public Integer getPriority() { + return this.priority; + } + + public void setPriority(Integer priority) { + this.priority = priority; + } + public void setFailed(boolean failed) { this.failed = failed; } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 3a3f84eb..568549d8 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -126,7 +126,7 @@ public class ArbitraryDataReader { * @param overwrite - set to true to force rebuild an existing cache * @return true if added or already present in queue; false if not */ - public boolean loadAsynchronously(boolean overwrite) { + public boolean loadAsynchronously(boolean overwrite, int priority) { ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, this.resourceId, this.resourceIdType, this.service, this.identifier); if (cache.isCachedDataAvailable()) { @@ -135,7 +135,9 @@ public class ArbitraryDataReader { return true; } - return ArbitraryDataBuildManager.getInstance().addToBuildQueue(this.createQueueItem()); + ArbitraryDataBuildQueueItem item = this.createQueueItem(); + item.setPriority(priority); + return ArbitraryDataBuildManager.getInstance().addToBuildQueue(item); } /** diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java index e4d90b79..3bd47b26 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java @@ -76,7 +76,7 @@ public class ArbitraryDataRenderer { if (!arbitraryDataReader.isCachedDataAvailable()) { // If async is requested, show a loading screen whilst build is in progress if (async) { - arbitraryDataReader.loadAsynchronously(false); + arbitraryDataReader.loadAsynchronously(false, 10); return this.getLoadingResponse(service, resourceId); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index 17808daa..0f6f5ef7 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -9,6 +9,7 @@ import org.qortal.repository.DataException; import org.qortal.utils.NTP; import java.io.IOException; +import java.util.Comparator; import java.util.Map; @@ -43,12 +44,13 @@ public class ArbitraryDataBuilderThread implements Runnable { ArbitraryDataBuildQueueItem queueItem = null; - // Find resources that are queued for building + // Find resources that are queued for building (sorted by highest priority first) synchronized (buildManager.arbitraryDataBuildQueue) { Map.Entry next = buildManager.arbitraryDataBuildQueue .entrySet().stream() .filter(e -> e.getValue().isQueued()) - .findFirst().orElse(null); + .sorted(Comparator.comparing(item -> item.getValue().getPriority())) + .reduce((first, second) -> second).orElse(null); if (next == null) { continue; From f7341cd9abbaf3560e10eb0ab2248648a3d01e4c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 15:13:53 +0000 Subject: [PATCH 136/151] Increased /arbitrary priority to 1 --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 430ff83f..8031bf83 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1044,7 +1044,7 @@ public class ArbitraryResource { // Loop until we have data if (async) { // Asynchronous - arbitraryDataReader.loadAsynchronously(false, 0); + arbitraryDataReader.loadAsynchronously(false, 1); } else { // Synchronous From 49b307db6001df8912b31e571126a92b29b80055 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 15:17:02 +0000 Subject: [PATCH 137/151] Treat a null priority as 0 --- .../org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index 78f3f00a..00999ee3 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -79,7 +79,10 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource { } public Integer getPriority() { - return this.priority; + if (this.priority != null) { + return this.priority; + } + return 0; } public void setPriority(Integer priority) { From a6aabaa7f0e1a818fb15e95e110a01517c2a5488 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 15:28:41 +0000 Subject: [PATCH 138/151] Reduce build queue log spam by only logging high priority items (5 and above). --- .../ArbitraryDataBuildQueueItem.java | 19 +++++++++++++++++++ .../arbitrary/ArbitraryDataBuildManager.java | 8 ++------ .../arbitrary/ArbitraryDataBuilderThread.java | 12 ++++-------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index 00999ee3..bad50f7e 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -1,5 +1,7 @@ package org.qortal.arbitrary; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.arbitrary.misc.Service; @@ -10,12 +12,16 @@ import java.io.IOException; public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource { + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildQueueItem.class); + private final Long creationTimestamp; private Long buildStartTimestamp = null; private Long buildEndTimestamp = null; private Integer priority = 0; private boolean failed = false; + private static int HIGH_PRIORITY_THRESHOLD = 5; + /* The maximum amount of time to spend on a single build */ // TODO: interrupt an in-progress build public static long BUILD_TIMEOUT = 60*1000L; // 60 seconds @@ -89,6 +95,19 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource { this.priority = priority; } + public boolean isHighPriority() { + return this.priority >= HIGH_PRIORITY_THRESHOLD; + } + + public void log(String message) { + if (this.isHighPriority()) { + LOGGER.info(message); + } + else { + LOGGER.debug(message); + } + } + public void setFailed(boolean failed) { this.failed = failed; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java index 0054356e..3bbdc61e 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -1,7 +1,5 @@ package org.qortal.controller.arbitrary; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; import org.qortal.utils.NTP; @@ -13,8 +11,6 @@ import java.util.concurrent.Executors; public class ArbitraryDataBuildManager extends Thread { - private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class); - private static ArbitraryDataBuildManager instance; private volatile boolean isStopping = false; @@ -106,7 +102,7 @@ public class ArbitraryDataBuildManager extends Thread { return true; } - LOGGER.info("Added {} to build queue", queueItem); + queueItem.log(String.format("Added %s to build queue", queueItem)); // Added to queue return true; @@ -154,7 +150,7 @@ public class ArbitraryDataBuildManager extends Thread { return true; } - LOGGER.info("Added {} to failed builds list", queueItem); + queueItem.log(String.format("Added %s to failed builds list", queueItem)); // Added to queue return true; diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index 0f6f5ef7..c8f91bb5 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -1,7 +1,5 @@ package org.qortal.controller.arbitrary; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.controller.Controller; @@ -15,8 +13,6 @@ import java.util.Map; public class ArbitraryDataBuilderThread implements Runnable { - private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuilderThread.class); - public ArbitraryDataBuilderThread() { } @@ -75,19 +71,19 @@ public class ArbitraryDataBuilderThread implements Runnable { try { // Perform the build - LOGGER.info("Building {}...", queueItem); + queueItem.log(String.format("Building %s... priority: %d", queueItem, queueItem.getPriority())); queueItem.build(); this.removeFromQueue(queueItem); - LOGGER.info("Finished building {}", queueItem); + queueItem.log(String.format("Finished building %s", queueItem)); } catch (MissingDataException e) { - LOGGER.info("Missing data for {}: {}", queueItem, e.getMessage()); + queueItem.log(String.format("Missing data for %s: %s", queueItem, e.getMessage())); queueItem.setFailed(true); this.removeFromQueue(queueItem); // Don't add to the failed builds list, as we may want to retry sooner } catch (IOException | DataException | RuntimeException e) { - LOGGER.info("Error building {}: {}", queueItem, e.getMessage()); + queueItem.log(String.format("Error building %s: %s", queueItem, e.getMessage())); // Something went wrong - so remove it from the queue, and add to failed builds list queueItem.setFailed(true); buildManager.addToFailedBuildsList(queueItem); From ee1f0720565c141e85c20529701be06e121914da Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 15:34:31 +0000 Subject: [PATCH 139/151] Improvement to last commit, so that caller class names are preserved. --- .../ArbitraryDataBuildQueueItem.java | 13 ---------- .../arbitrary/ArbitraryDataBuildManager.java | 21 ++++++++++++++-- .../arbitrary/ArbitraryDataBuilderThread.java | 25 ++++++++++++++++--- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java index bad50f7e..4a02f092 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -1,7 +1,5 @@ package org.qortal.arbitrary; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.arbitrary.misc.Service; @@ -12,8 +10,6 @@ import java.io.IOException; public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource { - private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildQueueItem.class); - private final Long creationTimestamp; private Long buildStartTimestamp = null; private Long buildEndTimestamp = null; @@ -99,15 +95,6 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource { return this.priority >= HIGH_PRIORITY_THRESHOLD; } - public void log(String message) { - if (this.isHighPriority()) { - LOGGER.info(message); - } - else { - LOGGER.debug(message); - } - } - public void setFailed(boolean failed) { this.failed = failed; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java index 3bbdc61e..ebff6913 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -1,5 +1,7 @@ package org.qortal.controller.arbitrary; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; import org.qortal.utils.NTP; @@ -11,6 +13,8 @@ import java.util.concurrent.Executors; public class ArbitraryDataBuildManager extends Thread { + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class); + private static ArbitraryDataBuildManager instance; private volatile boolean isStopping = false; @@ -102,7 +106,7 @@ public class ArbitraryDataBuildManager extends Thread { return true; } - queueItem.log(String.format("Added %s to build queue", queueItem)); + log(queueItem, String.format("Added %s to build queue", queueItem)); // Added to queue return true; @@ -150,7 +154,7 @@ public class ArbitraryDataBuildManager extends Thread { return true; } - queueItem.log(String.format("Added %s to failed builds list", queueItem)); + log(queueItem, String.format("Added %s to failed builds list", queueItem)); // Added to queue return true; @@ -183,4 +187,17 @@ public class ArbitraryDataBuildManager extends Thread { public boolean getBuildInProgress() { return this.buildInProgress; } + + private void log(ArbitraryDataBuildQueueItem queueItem, String message) { + if (queueItem == null) { + return; + } + + if (queueItem.isHighPriority()) { + LOGGER.info(message); + } + else { + LOGGER.debug(message); + } + } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java index c8f91bb5..0fb685a3 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java @@ -1,5 +1,7 @@ package org.qortal.controller.arbitrary; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.controller.Controller; @@ -13,6 +15,8 @@ import java.util.Map; public class ArbitraryDataBuilderThread implements Runnable { + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuilderThread.class); + public ArbitraryDataBuilderThread() { } @@ -71,19 +75,19 @@ public class ArbitraryDataBuilderThread implements Runnable { try { // Perform the build - queueItem.log(String.format("Building %s... priority: %d", queueItem, queueItem.getPriority())); + log(queueItem, String.format("Building %s... priority: %d", queueItem, queueItem.getPriority())); queueItem.build(); this.removeFromQueue(queueItem); - queueItem.log(String.format("Finished building %s", queueItem)); + log(queueItem, String.format("Finished building %s", queueItem)); } catch (MissingDataException e) { - queueItem.log(String.format("Missing data for %s: %s", queueItem, e.getMessage())); + log(queueItem, String.format("Missing data for %s: %s", queueItem, e.getMessage())); queueItem.setFailed(true); this.removeFromQueue(queueItem); // Don't add to the failed builds list, as we may want to retry sooner } catch (IOException | DataException | RuntimeException e) { - queueItem.log(String.format("Error building %s: %s", queueItem, e.getMessage())); + log(queueItem, String.format("Error building %s: %s", queueItem, e.getMessage())); // Something went wrong - so remove it from the queue, and add to failed builds list queueItem.setFailed(true); buildManager.addToFailedBuildsList(queueItem); @@ -102,4 +106,17 @@ public class ArbitraryDataBuilderThread implements Runnable { } ArbitraryDataBuildManager.getInstance().arbitraryDataBuildQueue.remove(queueItem.getUniqueKey()); } + + private void log(ArbitraryDataBuildQueueItem queueItem, String message) { + if (queueItem == null) { + return; + } + + if (queueItem.isHighPriority()) { + LOGGER.info(message); + } + else { + LOGGER.debug(message); + } + } } From a664a6a79081b486a41067854ff7ce4598068948 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 16:44:34 +0000 Subject: [PATCH 140/151] Added more LTC Electrum peers from https://1209k.com/bitcoin-eye/ele.php?chain=ltc --- src/main/java/org/qortal/crosschain/Litecoin.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java index 42ee70de..e5c748e8 100644 --- a/src/main/java/org/qortal/crosschain/Litecoin.java +++ b/src/main/java/org/qortal/crosschain/Litecoin.java @@ -50,6 +50,12 @@ public class Litecoin extends Bitcoiny { new Server("electrum.ltc.xurious.com", Server.ConnectionType.TCP, 50001), new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002), new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 50002), + new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum3.cipig.net", ConnectionType.TCP, 10063), + new Server("electrum2.cipig.net", Server.ConnectionType.TCP, 10063), + new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20063), + new Server("electrum1.cipig.net", Server.ConnectionType.TCP, 10063), new Server("ltc.rentonisk.com", Server.ConnectionType.TCP, 50001), new Server("ltc.rentonisk.com", Server.ConnectionType.SSL, 50002), new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002), From dbacfb964b899462c3750dafc55ad82ec96cbee4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 16:55:29 +0000 Subject: [PATCH 141/151] Increased TX_CACHE_SIZE from 200 to 1000, to speed up loading times on large wallets. --- src/main/java/org/qortal/crosschain/ElectrumX.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 4ab7e0b1..0708dfc1 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -103,7 +103,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { private Scanner scanner; private int nextId = 1; - private static final int TX_CACHE_SIZE = 200; + private static final int TX_CACHE_SIZE = 1000; @SuppressWarnings("serial") private final Map transactionCache = Collections.synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) { // This method is called just after a new entry has been added From 9b43e4ea3d59b57f94baa9d0cef70904c71a7614 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 18:02:56 +0000 Subject: [PATCH 142/151] Time electrum requests, and move on to another server if one takes more than 1000ms on average to respond (measured over the last 5 requests). --- .../java/org/qortal/crosschain/ElectrumX.java | 58 ++++++++++++++----- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 0708dfc1..87641607 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -5,19 +5,7 @@ import java.math.BigDecimal; import java.net.InetSocketAddress; import java.net.Socket; import java.net.SocketAddress; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Random; -import java.util.Scanner; -import java.util.Set; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -50,6 +38,9 @@ public class ElectrumX extends BitcoinyBlockchainProvider { /** Error message sent by some ElectrumX servers when they don't support returning verbose transactions. */ private static final String VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE = "verbose transactions are currently unsupported"; + private static final int RESPONSE_TIME_READINGS = 5; + private static final long MAX_AVG_RESPONSE_TIME = 1000L; // ms + public static class Server { String hostname; @@ -57,6 +48,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { ConnectionType connectionType; int port; + private List responseTimes = new ArrayList<>(); public Server(String hostname, ConnectionType connectionType, int port) { this.hostname = hostname; @@ -64,6 +56,25 @@ public class ElectrumX extends BitcoinyBlockchainProvider { this.port = port; } + public void addResponseTime(long responseTime) { + while (this.responseTimes.size() > RESPONSE_TIME_READINGS) { + this.responseTimes.remove(0); + } + this.responseTimes.add(responseTime); + } + + public long averageResponseTime() { + if (this.responseTimes.size() < RESPONSE_TIME_READINGS) { + // Not enough readings yet + return 0L; + } + OptionalDouble average = this.responseTimes.stream().mapToDouble(a -> a).average(); + if (average.isPresent()) { + return Double.valueOf(average.getAsDouble()).longValue(); + } + return 0L; + } + @Override public boolean equals(Object other) { if (other == this) @@ -539,6 +550,17 @@ public class ElectrumX extends BitcoinyBlockchainProvider { while (haveConnection()) { Object response = connectedRpc(method, params); + + // If we have more servers and this one replied slowly, try another + if (!this.remainingServers.isEmpty()) { + long averageResponseTime = this.currentServer.averageResponseTime(); + if (averageResponseTime > MAX_AVG_RESPONSE_TIME) { + LOGGER.info("Slow average response time {}ms from {} - trying another server...", this.currentServer.hostname, averageResponseTime); + this.closeServer(); + break; + } + } + if (response != null) return response; @@ -628,6 +650,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { String request = requestJson.toJSONString() + "\n"; LOGGER.trace(() -> String.format("Request: %s", request)); + long startTime = System.currentTimeMillis(); final String response; try { @@ -638,7 +661,11 @@ public class ElectrumX extends BitcoinyBlockchainProvider { return null; } + long endTime = System.currentTimeMillis(); + long responseTime = endTime-startTime; + LOGGER.trace(() -> String.format("Response: %s", response)); + LOGGER.info(() -> String.format("Time taken: %dms", endTime-startTime)); if (response.isEmpty()) // Empty response - try another server? @@ -649,6 +676,11 @@ public class ElectrumX extends BitcoinyBlockchainProvider { // Unexpected response - try another server? return null; + // Keep track of response times + if (this.currentServer != null) { + this.currentServer.addResponseTime(responseTime); + } + JSONObject responseJson = (JSONObject) responseObj; Object errorObj = responseJson.get("error"); From 8ac298e07df2972320f77da5ecbc776170f7e31f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 18:11:00 +0000 Subject: [PATCH 143/151] Allow 3 retries for getTransaction() and getAddressTransactions() requests --- .../java/org/qortal/crosschain/Bitcoiny.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 2678a08e..f9951aab 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -229,6 +229,25 @@ public abstract class Bitcoiny implements ForeignBlockchain { return transaction.getOutputs(); } + /** + * Returns transactions for passed script + *

+ * @throws ForeignBlockchainException if error occurs + */ + public List getAddressTransactions(byte[] scriptPubKey, boolean includeUnconfirmed) throws ForeignBlockchainException { + int retries = 0; + ForeignBlockchainException e2 = null; + while (retries <= 3) { + try { + return this.blockchain.getAddressTransactions(scriptPubKey, includeUnconfirmed); + } catch (ForeignBlockchainException e) { + e2 = e; + retries++; + } + } + throw(e2); + } + /** * Returns list of transaction hashes pertaining to passed address. *

@@ -263,7 +282,17 @@ public abstract class Bitcoiny implements ForeignBlockchain { * @throws ForeignBlockchainException if error occurs */ public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException { - return this.blockchain.getTransaction(txHash); + int retries = 0; + ForeignBlockchainException e2 = null; + while (retries <= 3) { + try { + return this.blockchain.getTransaction(txHash); + } catch (ForeignBlockchainException e) { + e2 = e; + retries++; + } + } + throw(e2); } /** From 19c83cc54d58e049671dc194025073d45f6e7aad Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 18:12:34 +0000 Subject: [PATCH 144/151] MAX_AVG_RESPONSE_TIME reduced to 500, as one peer regularly takes around 600ms to reply. --- src/main/java/org/qortal/crosschain/ElectrumX.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 87641607..21a478ac 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -39,7 +39,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { private static final String VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE = "verbose transactions are currently unsupported"; private static final int RESPONSE_TIME_READINGS = 5; - private static final long MAX_AVG_RESPONSE_TIME = 1000L; // ms + private static final long MAX_AVG_RESPONSE_TIME = 500L; // ms public static class Server { String hostname; From 3ae2f0086e7de3f3451af2b7de8c92b43d31d005 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 18:13:45 +0000 Subject: [PATCH 145/151] Removed unusably slow electrum peer --- src/main/java/org/qortal/crosschain/Litecoin.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Litecoin.java b/src/main/java/org/qortal/crosschain/Litecoin.java index e5c748e8..21ecd1db 100644 --- a/src/main/java/org/qortal/crosschain/Litecoin.java +++ b/src/main/java/org/qortal/crosschain/Litecoin.java @@ -56,8 +56,6 @@ public class Litecoin extends Bitcoiny { new Server("electrum2.cipig.net", Server.ConnectionType.TCP, 10063), new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20063), new Server("electrum1.cipig.net", Server.ConnectionType.TCP, 10063), - new Server("ltc.rentonisk.com", Server.ConnectionType.TCP, 50001), - new Server("ltc.rentonisk.com", Server.ConnectionType.SSL, 50002), new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002), new Server("ltc.litepay.ch", Server.ConnectionType.SSL, 50022), new Server("electrum-ltc-bysh.me", Server.ConnectionType.TCP, 50002), From 58a690e2c3cd097bdf07288b271e7d1d90b8ad3e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 11 Feb 2022 18:15:27 +0000 Subject: [PATCH 146/151] Route through new getAddressTransactions() wrapper. --- src/main/java/org/qortal/crosschain/Bitcoiny.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index f9951aab..a5a1ab12 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -420,7 +420,7 @@ public abstract class Bitcoiny implements ForeignBlockchain { byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); // Ask for transaction history - if it's empty then key has never been used - List historicTransactionHashes = this.blockchain.getAddressTransactions(script, false); + List historicTransactionHashes = this.getAddressTransactions(script, false); if (!historicTransactionHashes.isEmpty()) { areAllKeysUnused = false; From ea42a5617fc7087492af34959cdceaf23eeebdbf Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 13 Feb 2022 10:58:45 +0000 Subject: [PATCH 147/151] Fixed ElectrumX log spam and errors --- src/main/java/org/qortal/crosschain/ElectrumX.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/ElectrumX.java b/src/main/java/org/qortal/crosschain/ElectrumX.java index 21a478ac..7f1eb4c4 100644 --- a/src/main/java/org/qortal/crosschain/ElectrumX.java +++ b/src/main/java/org/qortal/crosschain/ElectrumX.java @@ -555,7 +555,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { if (!this.remainingServers.isEmpty()) { long averageResponseTime = this.currentServer.averageResponseTime(); if (averageResponseTime > MAX_AVG_RESPONSE_TIME) { - LOGGER.info("Slow average response time {}ms from {} - trying another server...", this.currentServer.hostname, averageResponseTime); + LOGGER.info("Slow average response time {}ms from {} - trying another server...", averageResponseTime, this.currentServer.hostname); this.closeServer(); break; } @@ -665,7 +665,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider { long responseTime = endTime-startTime; LOGGER.trace(() -> String.format("Response: %s", response)); - LOGGER.info(() -> String.format("Time taken: %dms", endTime-startTime)); + LOGGER.trace(() -> String.format("Time taken: %dms", endTime-startTime)); if (response.isEmpty()) // Empty response - try another server? From 07122590577653b365f37ddf82b936a5f0238233 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 13 Feb 2022 13:45:48 +0000 Subject: [PATCH 148/151] Implemented REGISTER_NAME transaction fee increase from 0.001 to 5 QORT (average value based on community vote). --- .../java/org/qortal/block/BlockChain.java | 15 ++++ .../transaction/RegisterNameTransaction.java | 9 +++ src/main/resources/blockchain.json | 2 + .../org/qortal/test/naming/BuySellTests.java | 3 + .../qortal/test/naming/IntegrityTests.java | 17 +++++ .../org/qortal/test/naming/MiscTests.java | 69 +++++++++++++++++++ .../org/qortal/test/naming/UpdateTests.java | 9 +++ .../test-chain-v2-founder-rewards.json | 1 + .../test-chain-v2-leftover-reward.json | 1 + src/test/resources/test-chain-v2-minting.json | 1 + .../test-chain-v2-qora-holder-extremes.json | 1 + .../resources/test-chain-v2-qora-holder.json | 1 + .../test-chain-v2-reward-levels.json | 1 + .../test-chain-v2-reward-scaling.json | 1 + src/test/resources/test-chain-v2.json | 1 + 15 files changed, 132 insertions(+) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index defa9120..69779d96 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -72,6 +72,11 @@ public class BlockChain { transactionV5Timestamp; } + // Custom transaction fees + @XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class) + private long nameRegistrationUnitFee; + private long nameRegistrationUnitFeeTimestamp; + /** Map of which blockchain features are enabled when (height/timestamp) */ @XmlJavaTypeAdapter(StringLongMapXmlAdapter.class) private Map featureTriggers; @@ -301,6 +306,16 @@ public class BlockChain { return this.maxBlockSize; } + // Custom transaction fees + public long getNameRegistrationUnitFee() { + return this.nameRegistrationUnitFee; + } + + public long getNameRegistrationUnitFeeTimestamp() { + // FUTURE: we could use a separate structure to indicate fee adjustments for different transaction types + return this.nameRegistrationUnitFeeTimestamp; + } + /** Returns true if approval-needing transaction types require a txGroupId other than NO_GROUP. */ public boolean getRequireGroupForApproval() { return this.requireGroupForApproval; diff --git a/src/main/java/org/qortal/transaction/RegisterNameTransaction.java b/src/main/java/org/qortal/transaction/RegisterNameTransaction.java index d0a2f49c..1ababa88 100644 --- a/src/main/java/org/qortal/transaction/RegisterNameTransaction.java +++ b/src/main/java/org/qortal/transaction/RegisterNameTransaction.java @@ -37,6 +37,15 @@ public class RegisterNameTransaction extends Transaction { return Collections.emptyList(); } + @Override + public long getUnitFee(Long timestamp) { + // Use a higher unit fee after the fee increase timestamp + if (timestamp > BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp()) { + return BlockChain.getInstance().getNameRegistrationUnitFee(); + } + return BlockChain.getInstance().getUnitFee(); + } + // Navigation public Account getRegistrant() { diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 3d0e5559..df29cd95 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -4,6 +4,8 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.001", + "nameRegistrationUnitFee": "5", + "nameRegistrationUnitFeeTimestamp": 9999999999999, "useBrokenMD160ForAddresses": false, "requireGroupForApproval": false, "defaultGroupId": 0, diff --git a/src/test/java/org/qortal/test/naming/BuySellTests.java b/src/test/java/org/qortal/test/naming/BuySellTests.java index f0320da5..872c3e2c 100644 --- a/src/test/java/org/qortal/test/naming/BuySellTests.java +++ b/src/test/java/org/qortal/test/naming/BuySellTests.java @@ -20,7 +20,9 @@ import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; import org.qortal.utils.Amounts; +import org.qortal.utils.NTP; public class BuySellTests extends Common { @@ -62,6 +64,7 @@ public class BuySellTests extends Common { public void testRegisterName() throws DataException { // Register-name RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); String name = transactionData.getName(); diff --git a/src/test/java/org/qortal/test/naming/IntegrityTests.java b/src/test/java/org/qortal/test/naming/IntegrityTests.java index 7531bea6..85b22c21 100644 --- a/src/test/java/org/qortal/test/naming/IntegrityTests.java +++ b/src/test/java/org/qortal/test/naming/IntegrityTests.java @@ -11,7 +11,9 @@ import org.qortal.repository.RepositoryManager; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; import org.qortal.transaction.Transaction; +import org.qortal.utils.NTP; import java.util.List; @@ -33,6 +35,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -56,6 +59,7 @@ public class IntegrityTests extends Common { String data = "\uD83E\uDD73"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -82,6 +86,7 @@ public class IntegrityTests extends Common { String name = "initial_name"; String data = "initial_data"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Update the name, but keep the new name blank @@ -116,6 +121,7 @@ public class IntegrityTests extends Common { String name = "initial_name"; String data = "initial_data"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Update the name, but keep the new name blank @@ -143,6 +149,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -172,6 +179,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -210,6 +218,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -235,6 +244,7 @@ public class IntegrityTests extends Common { // Attempt to register the new name transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), newName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -254,6 +264,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -268,6 +279,7 @@ public class IntegrityTests extends Common { // Attempt to register the name again String duplicateName = "TEST-nÁme"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), duplicateName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -287,6 +299,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -320,6 +333,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -329,6 +343,7 @@ public class IntegrityTests extends Common { String secondName = "new-missing-name"; String secondNameData = "{\"data2\":true}"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), secondName, secondNameData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the second name exists and the data is correct @@ -362,6 +377,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -393,6 +409,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct diff --git a/src/test/java/org/qortal/test/naming/MiscTests.java b/src/test/java/org/qortal/test/naming/MiscTests.java index 84fe3351..09713733 100644 --- a/src/test/java/org/qortal/test/naming/MiscTests.java +++ b/src/test/java/org/qortal/test/naming/MiscTests.java @@ -3,20 +3,26 @@ package org.qortal.test.naming; import static org.junit.Assert.*; import java.util.List; +import java.util.Optional; +import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; +import org.qortal.block.BlockChain; import org.qortal.controller.BlockMinter; import org.qortal.data.transaction.*; import org.qortal.naming.Name; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.test.common.*; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.ValidationResult; +import org.qortal.utils.NTP; public class MiscTests extends Common { @@ -34,6 +40,7 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); List recentNames = repository.getNameRepository().getRecentNames(0L); @@ -53,11 +60,13 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // duplicate String duplicateName = "TEST-nÁme"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), duplicateName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -76,12 +85,14 @@ public class MiscTests extends Common { String data = "{}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // duplicate (this time registered by Bob) PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); String duplicateName = "TEST-nÁme"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), duplicateName, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -100,12 +111,14 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Register another name that we will later attempt to rename to first name (above) String otherName = "new-name"; String otherData = ""; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), otherName, otherData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // we shouldn't be able to update name to existing name @@ -129,6 +142,7 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -147,6 +161,7 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // we shouldn't be able to update name to an address @@ -175,6 +190,7 @@ public class MiscTests extends Common { // Register the name RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -201,6 +217,7 @@ public class MiscTests extends Common { // Register the name RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -252,6 +269,7 @@ public class MiscTests extends Common { // Register the name RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -283,6 +301,7 @@ public class MiscTests extends Common { PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); // Ensure the name doesn't exist assertNull(repository.getNameRepository().fromName(name)); @@ -304,4 +323,54 @@ public class MiscTests extends Common { } } + // test name registration fee increase + @Test + public void testRegisterNameFeeIncrease() throws DataException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Set nameRegistrationUnitFeeTimestamp to a time far in the future + long futureTimestamp = 9999999999999L; // 20 Nov 2286 + FieldUtils.writeField(BlockChain.getInstance(), "nameRegistrationUnitFeeTimestamp", futureTimestamp, true); + assertEquals(futureTimestamp, BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp()); + + // Validate unit fees pre and post timestamp + assertEquals(10000000, BlockChain.getInstance().getUnitFee()); // 0.1 QORT + assertEquals(500000000, BlockChain.getInstance().getNameRegistrationUnitFee()); // 5 QORT + + // Register-name + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String name = "test-name"; + String data = "{\"age\":30}"; + + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + assertEquals(10000000L, transactionData.getFee().longValue()); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Set nameRegistrationUnitFeeTimestamp to a time in the past + Long now = NTP.getTime(); + FieldUtils.writeField(BlockChain.getInstance(), "nameRegistrationUnitFeeTimestamp", now - 1000L, true); + assertEquals(now - 1000L, BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp()); + + // Register a different name + // First try with the default unit fee + String name2 = "test-name-2"; + transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, data); + assertEquals(10000000L, transactionData.getFee().longValue()); + Transaction transaction = Transaction.fromData(repository, transactionData); + transaction.sign(alice); + ValidationResult result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be invalid", ValidationResult.INSUFFICIENT_FEE == result); + + // Now try using correct fee (this is specified by the UI, via the /transaction/unitfee API endpoint) + transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, data); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + assertEquals(500000000L, transactionData.getFee().longValue()); + transaction = Transaction.fromData(repository, transactionData); + transaction.sign(alice); + result = transaction.importAsUnconfirmed(); + assertTrue("Transaction should be valid", ValidationResult.OK == result); + } + } + } diff --git a/src/test/java/org/qortal/test/naming/UpdateTests.java b/src/test/java/org/qortal/test/naming/UpdateTests.java index a13b3138..0311b0a7 100644 --- a/src/test/java/org/qortal/test/naming/UpdateTests.java +++ b/src/test/java/org/qortal/test/naming/UpdateTests.java @@ -16,6 +16,8 @@ import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.utils.NTP; public class UpdateTests extends Common { @@ -34,6 +36,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, initialTransactionData, alice); // Check name, reduced name, and data exist @@ -100,6 +103,7 @@ public class UpdateTests extends Common { String constantReducedName = "initia1-name"; TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, initialTransactionData, alice); // Check initial name exists @@ -147,6 +151,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, initialTransactionData, alice); // Check initial name exists @@ -225,6 +230,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Check initial name exists @@ -282,6 +288,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Check initial name exists @@ -323,6 +330,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Check initial name exists @@ -385,6 +393,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Check initial name exists diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index 5a2ac599..c2a61503 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index f0ff5985..be04d7a2 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index b83789cb..d79c8e98 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index d85484fb..08c6fab3 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index 32bc9c57..804087b7 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, diff --git a/src/test/resources/test-chain-v2-reward-levels.json b/src/test/resources/test-chain-v2-reward-levels.json index 06fbffa9..2eae612d 100644 --- a/src/test/resources/test-chain-v2-reward-levels.json +++ b/src/test/resources/test-chain-v2-reward-levels.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index 66bac366..6842a727 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 03e70a3b..8fc7f957 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -5,6 +5,7 @@ "maxBlockSize": 2097152, "maxBytesPerUnitFee": 1024, "unitFee": "0.1", + "nameRegistrationUnitFee": "5", "requireGroupForApproval": false, "minAccountLevelToRewardShare": 5, "maxRewardSharesPerMintingAccount": 20, From c1598d20b5f48475d465a15ca8d53d34296b7e9f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 13 Feb 2022 13:47:00 +0000 Subject: [PATCH 149/151] Name registration fee increase timestamp set to Sunday, 20 February 2022 16:00:00 UTC --- src/main/resources/blockchain.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index df29cd95..17858d8d 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -5,7 +5,7 @@ "maxBytesPerUnitFee": 1024, "unitFee": "0.001", "nameRegistrationUnitFee": "5", - "nameRegistrationUnitFeeTimestamp": 9999999999999, + "nameRegistrationUnitFeeTimestamp": 1645372800000, "useBrokenMD160ForAddresses": false, "requireGroupForApproval": false, "defaultGroupId": 0, From 265ae195917c51cd640c71e930b8351d95bef66c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 13 Feb 2022 14:31:21 +0000 Subject: [PATCH 150/151] Fixed other failing tests due to increased REGISTER_NAME transaction fee. At some point we should determine the correct fee inside of generateBase(), but setting it explicitly adds confidence in testing for now. --- .../org/qortal/test/api/NamesApiTests.java | 5 ++ .../ArbitraryDataStorageCapacityTests.java | 3 + .../ArbitraryDataStoragePolicyTests.java | 72 +++++++++++-------- .../test/arbitrary/ArbitraryDataTests.java | 11 +++ .../ArbitraryTransactionMetadataTests.java | 3 + .../arbitrary/ArbitraryTransactionTests.java | 3 + 6 files changed, 67 insertions(+), 30 deletions(-) diff --git a/src/test/java/org/qortal/test/api/NamesApiTests.java b/src/test/java/org/qortal/test/api/NamesApiTests.java index 2d31c8c2..36b17a08 100644 --- a/src/test/java/org/qortal/test/api/NamesApiTests.java +++ b/src/test/java/org/qortal/test/api/NamesApiTests.java @@ -16,6 +16,8 @@ import org.qortal.test.common.ApiCommon; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; +import org.qortal.utils.NTP; public class NamesApiTests extends ApiCommon { @@ -47,6 +49,7 @@ public class NamesApiTests extends ApiCommon { String name = "test-name"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); assertNotNull(this.namesResource.getNamesByAddress(alice.getAddress(), null, null, null)); @@ -62,6 +65,7 @@ public class NamesApiTests extends ApiCommon { String name = "test-name"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); assertNotNull(this.namesResource.getName(name)); @@ -77,6 +81,7 @@ public class NamesApiTests extends ApiCommon { long price = 1_23456789L; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Sell-name diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java index c38327c3..74d6417b 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java @@ -22,6 +22,7 @@ import org.qortal.test.common.ArbitraryUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; import org.qortal.utils.Base58; import org.qortal.utils.NTP; @@ -153,6 +154,7 @@ public class ArbitraryDataStorageCapacityTests extends Common { PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String aliceName = "alice"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength); ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); @@ -161,6 +163,7 @@ public class ArbitraryDataStorageCapacityTests extends Common { PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); String bobName = "bob"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, bob); Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength); ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize); diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java index 30abc9f5..6c58c4c1 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java @@ -21,7 +21,9 @@ import org.qortal.settings.Settings; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; import org.qortal.utils.Base58; +import org.qortal.utils.NTP; import java.io.IOException; import java.nio.file.Path; @@ -59,25 +61,27 @@ public class ArbitraryDataStoragePolicyTests extends Common { String name = "Test"; // Register the name to Alice - TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice); + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + TransactionUtils.signAndMint(repository, transactionData, alice); // Create transaction - ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); + ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name); // Add name to followed list assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy()); - assertTrue(storageManager.canStoreData(transactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store but not pre-fetch data for this transaction - assertTrue(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); } } @@ -92,25 +96,27 @@ public class ArbitraryDataStoragePolicyTests extends Common { FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "FOLLOWED", true); // Register the name to Alice - TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice); + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + TransactionUtils.signAndMint(repository, transactionData, alice); // Create transaction - ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); + ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name); // Add name to followed list assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy()); - assertTrue(storageManager.canStoreData(transactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We shouldn't store or pre-fetch data for this transaction - assertFalse(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); + assertFalse(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); } } @@ -125,25 +131,27 @@ public class ArbitraryDataStoragePolicyTests extends Common { FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "VIEWED", true); // Register the name to Alice - TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice); + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + TransactionUtils.signAndMint(repository, transactionData, alice); // Create transaction - ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); + ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name); // Add name to followed list assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); // We should store but not pre-fetch data for this transaction assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy()); - assertTrue(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store but not pre-fetch data for this transaction - assertTrue(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); } } @@ -158,25 +166,27 @@ public class ArbitraryDataStoragePolicyTests extends Common { FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "ALL", true); // Register the name to Alice - TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice); + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + TransactionUtils.signAndMint(repository, transactionData, alice); // Create transaction - ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); + ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name); // Add name to followed list assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); // We should store and pre-fetch data for this transaction assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy()); - assertTrue(storageManager.canStoreData(transactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We should store and pre-fetch data for this transaction - assertTrue(storageManager.canStoreData(transactionData)); - assertTrue(storageManager.shouldPreFetchData(repository, transactionData)); + assertTrue(storageManager.canStoreData(arbitraryTransactionData)); + assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); } } @@ -191,25 +201,27 @@ public class ArbitraryDataStoragePolicyTests extends Common { FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "NONE", true); // Register the name to Alice - TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice); + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + TransactionUtils.signAndMint(repository, transactionData, alice); // Create transaction - ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name); + ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name); // Add name to followed list assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false)); // We shouldn't store or pre-fetch data for this transaction assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy()); - assertFalse(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); + assertFalse(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); // Now unfollow the name assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false)); // We shouldn't store or pre-fetch data for this transaction - assertFalse(storageManager.canStoreData(transactionData)); - assertFalse(storageManager.shouldPreFetchData(repository, transactionData)); + assertFalse(storageManager.canStoreData(arbitraryTransactionData)); + assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData)); } } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java index e8e4a288..c461f798 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java @@ -23,7 +23,9 @@ import org.qortal.test.common.ArbitraryUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; import org.qortal.utils.Base58; +import org.qortal.utils.NTP; import java.io.IOException; import java.nio.file.Files; @@ -55,6 +57,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -149,6 +152,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -181,6 +185,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -226,6 +231,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -294,6 +300,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -343,6 +350,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -380,6 +388,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -409,6 +418,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -435,6 +445,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java index 5f76c9c0..305cff9c 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java @@ -20,7 +20,9 @@ import org.qortal.test.common.ArbitraryUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; +import org.qortal.transaction.RegisterNameTransaction; import org.qortal.utils.Base58; +import org.qortal.utils.NTP; import java.io.IOException; import java.nio.file.Path; @@ -50,6 +52,7 @@ public class ArbitraryTransactionMetadataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java index 150038ca..5535c5ed 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java @@ -19,7 +19,9 @@ import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; import org.qortal.test.common.transaction.TestTransaction; import org.qortal.transaction.ArbitraryTransaction; +import org.qortal.transaction.RegisterNameTransaction; import org.qortal.utils.Base58; +import org.qortal.utils.NTP; import java.io.IOException; import java.nio.file.Path; @@ -46,6 +48,7 @@ public class ArbitraryTransactionTests extends Common { // Register the name to Alice RegisterNameTransactionData registerNameTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + registerNameTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); TransactionUtils.signAndMint(repository, registerNameTransactionData, alice); // Set difficulty to 1 From 83213800b9ef11ce502c5dd3987608b9d3e0f63c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 13 Feb 2022 15:05:28 +0000 Subject: [PATCH 151/151] Use the timestamp from the registerNameTransactionData in unit tests, rather than the current time. --- .../org/qortal/test/api/NamesApiTests.java | 6 ++-- .../ArbitraryDataStorageCapacityTests.java | 4 +-- .../ArbitraryDataStoragePolicyTests.java | 10 +++---- .../test/arbitrary/ArbitraryDataTests.java | 18 +++++------ .../ArbitraryTransactionMetadataTests.java | 2 +- .../arbitrary/ArbitraryTransactionTests.java | 2 +- .../org/qortal/test/naming/BuySellTests.java | 2 +- .../qortal/test/naming/IntegrityTests.java | 30 +++++++++---------- .../org/qortal/test/naming/MiscTests.java | 30 +++++++++---------- .../org/qortal/test/naming/UpdateTests.java | 14 ++++----- 10 files changed, 59 insertions(+), 59 deletions(-) diff --git a/src/test/java/org/qortal/test/api/NamesApiTests.java b/src/test/java/org/qortal/test/api/NamesApiTests.java index 36b17a08..962f1b92 100644 --- a/src/test/java/org/qortal/test/api/NamesApiTests.java +++ b/src/test/java/org/qortal/test/api/NamesApiTests.java @@ -49,7 +49,7 @@ public class NamesApiTests extends ApiCommon { String name = "test-name"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); assertNotNull(this.namesResource.getNamesByAddress(alice.getAddress(), null, null, null)); @@ -65,7 +65,7 @@ public class NamesApiTests extends ApiCommon { String name = "test-name"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); assertNotNull(this.namesResource.getName(name)); @@ -81,7 +81,7 @@ public class NamesApiTests extends ApiCommon { long price = 1_23456789L; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Sell-name diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java index 74d6417b..0523963e 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java @@ -154,7 +154,7 @@ public class ArbitraryDataStorageCapacityTests extends Common { PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String aliceName = "alice"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength); ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize); @@ -163,7 +163,7 @@ public class ArbitraryDataStorageCapacityTests extends Common { PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); String bobName = "bob"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, bob); Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength); ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize); diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java index 6c58c4c1..dd132eb0 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java @@ -62,7 +62,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create transaction @@ -97,7 +97,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create transaction @@ -132,7 +132,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create transaction @@ -167,7 +167,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create transaction @@ -202,7 +202,7 @@ public class ArbitraryDataStoragePolicyTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create transaction diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java index c461f798..3a31e652 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java @@ -57,7 +57,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -152,7 +152,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -185,7 +185,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -231,7 +231,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -300,7 +300,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -350,7 +350,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -388,7 +388,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -418,7 +418,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction @@ -445,7 +445,7 @@ public class ArbitraryDataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java index 305cff9c..4d82f1f2 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java @@ -52,7 +52,7 @@ public class ArbitraryTransactionMetadataTests extends Common { // Register the name to Alice RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Create PUT transaction diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java index 5535c5ed..866aa338 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java @@ -48,7 +48,7 @@ public class ArbitraryTransactionTests extends Common { // Register the name to Alice RegisterNameTransactionData registerNameTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); - registerNameTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + registerNameTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(registerNameTransactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, registerNameTransactionData, alice); // Set difficulty to 1 diff --git a/src/test/java/org/qortal/test/naming/BuySellTests.java b/src/test/java/org/qortal/test/naming/BuySellTests.java index 872c3e2c..cc014c25 100644 --- a/src/test/java/org/qortal/test/naming/BuySellTests.java +++ b/src/test/java/org/qortal/test/naming/BuySellTests.java @@ -64,7 +64,7 @@ public class BuySellTests extends Common { public void testRegisterName() throws DataException { // Register-name RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}"); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); String name = transactionData.getName(); diff --git a/src/test/java/org/qortal/test/naming/IntegrityTests.java b/src/test/java/org/qortal/test/naming/IntegrityTests.java index 85b22c21..c2232ec3 100644 --- a/src/test/java/org/qortal/test/naming/IntegrityTests.java +++ b/src/test/java/org/qortal/test/naming/IntegrityTests.java @@ -35,7 +35,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp())); TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -59,7 +59,7 @@ public class IntegrityTests extends Common { String data = "\uD83E\uDD73"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -86,7 +86,7 @@ public class IntegrityTests extends Common { String name = "initial_name"; String data = "initial_data"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Update the name, but keep the new name blank @@ -121,7 +121,7 @@ public class IntegrityTests extends Common { String name = "initial_name"; String data = "initial_data"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Update the name, but keep the new name blank @@ -149,7 +149,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -179,7 +179,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -218,7 +218,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -244,7 +244,7 @@ public class IntegrityTests extends Common { // Attempt to register the new name transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), newName, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -264,7 +264,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -279,7 +279,7 @@ public class IntegrityTests extends Common { // Attempt to register the name again String duplicateName = "TEST-nÁme"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), duplicateName, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -299,7 +299,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -333,7 +333,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -343,7 +343,7 @@ public class IntegrityTests extends Common { String secondName = "new-missing-name"; String secondNameData = "{\"data2\":true}"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), secondName, secondNameData); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the second name exists and the data is correct @@ -377,7 +377,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -409,7 +409,7 @@ public class IntegrityTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct diff --git a/src/test/java/org/qortal/test/naming/MiscTests.java b/src/test/java/org/qortal/test/naming/MiscTests.java index 09713733..bdf3df9f 100644 --- a/src/test/java/org/qortal/test/naming/MiscTests.java +++ b/src/test/java/org/qortal/test/naming/MiscTests.java @@ -40,7 +40,7 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); List recentNames = repository.getNameRepository().getRecentNames(0L); @@ -60,13 +60,13 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // duplicate String duplicateName = "TEST-nÁme"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), duplicateName, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -85,14 +85,14 @@ public class MiscTests extends Common { String data = "{}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // duplicate (this time registered by Bob) PrivateKeyAccount bob = Common.getTestAccount(repository, "bob"); String duplicateName = "TEST-nÁme"; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), duplicateName, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -111,14 +111,14 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Register another name that we will later attempt to rename to first name (above) String otherName = "new-name"; String otherData = ""; transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), otherName, otherData); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // we shouldn't be able to update name to existing name @@ -142,7 +142,7 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); @@ -161,7 +161,7 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // we shouldn't be able to update name to an address @@ -190,7 +190,7 @@ public class MiscTests extends Common { // Register the name RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -217,7 +217,7 @@ public class MiscTests extends Common { // Register the name RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -269,7 +269,7 @@ public class MiscTests extends Common { // Register the name RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Ensure the name exists and the data is correct @@ -301,7 +301,7 @@ public class MiscTests extends Common { PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; // Ensure the name doesn't exist assertNull(repository.getNameRepository().fromName(name)); @@ -343,7 +343,7 @@ public class MiscTests extends Common { String data = "{\"age\":30}"; RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; assertEquals(10000000L, transactionData.getFee().longValue()); TransactionUtils.signAndMint(repository, transactionData, alice); @@ -364,7 +364,7 @@ public class MiscTests extends Common { // Now try using correct fee (this is specified by the UI, via the /transaction/unitfee API endpoint) transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, data); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; assertEquals(500000000L, transactionData.getFee().longValue()); transaction = Transaction.fromData(repository, transactionData); transaction.sign(alice); diff --git a/src/test/java/org/qortal/test/naming/UpdateTests.java b/src/test/java/org/qortal/test/naming/UpdateTests.java index 0311b0a7..d591b3f3 100644 --- a/src/test/java/org/qortal/test/naming/UpdateTests.java +++ b/src/test/java/org/qortal/test/naming/UpdateTests.java @@ -36,7 +36,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); - initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, initialTransactionData, alice); // Check name, reduced name, and data exist @@ -103,7 +103,7 @@ public class UpdateTests extends Common { String constantReducedName = "initia1-name"; TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); - initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, initialTransactionData, alice); // Check initial name exists @@ -151,7 +151,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); - initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, initialTransactionData, alice); // Check initial name exists @@ -230,7 +230,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Check initial name exists @@ -288,7 +288,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Check initial name exists @@ -330,7 +330,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Check initial name exists @@ -393,7 +393,7 @@ public class UpdateTests extends Common { String initialData = "{\"age\":30}"; TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData); - transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(NTP.getTime())); + transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));; TransactionUtils.signAndMint(repository, transactionData, alice); // Check initial name exists