From 8e97c05b56d215a4f217c74a275881a763f92d31 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 13 Jan 2023 19:25:06 +0000 Subject: [PATCH 01/25] Added missing feature trigger from unit tests. --- src/test/resources/test-chain-v2-block-timestamps.json | 1 + src/test/resources/test-chain-v2-disable-reference.json | 1 + src/test/resources/test-chain-v2-founder-rewards.json | 1 + src/test/resources/test-chain-v2-leftover-reward.json | 1 + src/test/resources/test-chain-v2-minting.json | 1 + src/test/resources/test-chain-v2-qora-holder-extremes.json | 1 + src/test/resources/test-chain-v2-qora-holder-reduction.json | 1 + src/test/resources/test-chain-v2-qora-holder.json | 1 + src/test/resources/test-chain-v2-reward-levels.json | 1 + src/test/resources/test-chain-v2-reward-scaling.json | 1 + src/test/resources/test-chain-v2-reward-shares.json | 1 + src/test/resources/test-chain-v2-self-sponsorship-algo.json | 1 + src/test/resources/test-chain-v2.json | 1 + 13 files changed, 13 insertions(+) diff --git a/src/test/resources/test-chain-v2-block-timestamps.json b/src/test/resources/test-chain-v2-block-timestamps.json index 0a479a75..8c2e0503 100644 --- a/src/test/resources/test-chain-v2-block-timestamps.json +++ b/src/test/resources/test-chain-v2-block-timestamps.json @@ -74,6 +74,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-disable-reference.json b/src/test/resources/test-chain-v2-disable-reference.json index 15c4bedd..f7f8e7d8 100644 --- a/src/test/resources/test-chain-v2-disable-reference.json +++ b/src/test/resources/test-chain-v2-disable-reference.json @@ -77,6 +77,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index e17b6687..20d10233 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -78,6 +78,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index abb78528..e71ebab6 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -78,6 +78,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index 31f89916..2a388e1f 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -78,6 +78,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index 8d4351eb..cface0e7 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -78,6 +78,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-qora-holder-reduction.json b/src/test/resources/test-chain-v2-qora-holder-reduction.json index 20bd27c5..f233680b 100644 --- a/src/test/resources/test-chain-v2-qora-holder-reduction.json +++ b/src/test/resources/test-chain-v2-qora-holder-reduction.json @@ -79,6 +79,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index b638e759..4ea82290 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -78,6 +78,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-reward-levels.json b/src/test/resources/test-chain-v2-reward-levels.json index 7ba5c8b6..5de8d9ff 100644 --- a/src/test/resources/test-chain-v2-reward-levels.json +++ b/src/test/resources/test-chain-v2-reward-levels.json @@ -78,6 +78,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index 5aa9084f..c008ed42 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -78,6 +78,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-reward-shares.json b/src/test/resources/test-chain-v2-reward-shares.json index 70b746a8..2fc0151f 100644 --- a/src/test/resources/test-chain-v2-reward-shares.json +++ b/src/test/resources/test-chain-v2-reward-shares.json @@ -78,6 +78,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2-self-sponsorship-algo.json b/src/test/resources/test-chain-v2-self-sponsorship-algo.json index 36df9a62..c13d55da 100644 --- a/src/test/resources/test-chain-v2-self-sponsorship-algo.json +++ b/src/test/resources/test-chain-v2-self-sponsorship-algo.json @@ -77,6 +77,7 @@ "disableReferenceTimestamp": 9999999999999, "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, + "feeValidationFixTimestamp": 0, "selfSponsorshipAlgoV1Height": 20 }, "genesisInfo": { diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index cd28d214..63abc695 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -78,6 +78,7 @@ "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, "selfSponsorshipAlgoV1Height": 999999999, + "feeValidationFixTimestamp": 0, "chatReferenceTimestamp": 0 }, "genesisInfo": { From ba95f8376f19cc76791aba451920bdabff0a8fb3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 13 Jan 2023 19:27:02 +0000 Subject: [PATCH 02/25] Increase CHAT transaction data limits to the maximum (4000 bytes) to allow for upcoming UI features. --- src/main/java/org/qortal/transaction/ChatTransaction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/transaction/ChatTransaction.java b/src/main/java/org/qortal/transaction/ChatTransaction.java index a248268c..5ed96494 100644 --- a/src/main/java/org/qortal/transaction/ChatTransaction.java +++ b/src/main/java/org/qortal/transaction/ChatTransaction.java @@ -30,7 +30,7 @@ public class ChatTransaction extends Transaction { private ChatTransactionData chatTransactionData; // Other useful constants - public static final int MAX_DATA_SIZE = 1024; + public static final int MAX_DATA_SIZE = 4000; public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes public static final int POW_DIFFICULTY_ABOVE_QORT_THRESHOLD = 8; // leading zero bits public static final int POW_DIFFICULTY_BELOW_QORT_THRESHOLD = 18; // leading zero bits From 41f88be55eedae8b575f0be2ea220601f9d44819 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 13 Jan 2023 19:27:38 +0000 Subject: [PATCH 03/25] Test serialization of CHAT transactions --- .../org/qortal/test/SerializationTests.java | 1 - .../transaction/ChatTestTransaction.java | 40 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 src/test/java/org/qortal/test/common/transaction/ChatTestTransaction.java diff --git a/src/test/java/org/qortal/test/SerializationTests.java b/src/test/java/org/qortal/test/SerializationTests.java index d9fe978c..8422bd9c 100644 --- a/src/test/java/org/qortal/test/SerializationTests.java +++ b/src/test/java/org/qortal/test/SerializationTests.java @@ -47,7 +47,6 @@ public class SerializationTests extends Common { switch (txType) { case GENESIS: case ACCOUNT_FLAGS: - case CHAT: case PUBLICIZE: case AIRDROP: case ENABLE_FORGING: diff --git a/src/test/java/org/qortal/test/common/transaction/ChatTestTransaction.java b/src/test/java/org/qortal/test/common/transaction/ChatTestTransaction.java new file mode 100644 index 00000000..bab1f1a0 --- /dev/null +++ b/src/test/java/org/qortal/test/common/transaction/ChatTestTransaction.java @@ -0,0 +1,40 @@ +package org.qortal.test.common.transaction; + +import org.qortal.account.PrivateKeyAccount; +import org.qortal.crypto.Crypto; +import org.qortal.data.transaction.ChatTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; + +import java.util.Random; + +public class ChatTestTransaction extends TestTransaction { + + public static TransactionData randomTransaction(Repository repository, PrivateKeyAccount account, boolean wantValid) throws DataException { + Random random = new Random(); + byte[] orderId = new byte[64]; + random.nextBytes(orderId); + + String sender = Crypto.toAddress(account.getPublicKey()); + int nonce = 1234567; + + // Generate random recipient + byte[] randomPrivateKey = new byte[32]; + random.nextBytes(randomPrivateKey); + PrivateKeyAccount recipientAccount = new PrivateKeyAccount(repository, randomPrivateKey); + String recipient = Crypto.toAddress(recipientAccount.getPublicKey()); + + byte[] chatReference = new byte[64]; + random.nextBytes(chatReference); + + byte[] data = new byte[4000]; + random.nextBytes(data); + + boolean isText = true; + boolean isEncrypted = true; + + return new ChatTransactionData(generateBase(account), sender, nonce, recipient, chatReference, data, isText, isEncrypted); + } + +} From 6284a4691caa2aa21b47b0431cbdf168b5bb888b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 13 Jan 2023 19:28:44 +0000 Subject: [PATCH 04/25] Import test transactions as part of the serialization tests, to catch any issues with db schema data lengths. --- src/test/java/org/qortal/test/SerializationTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/qortal/test/SerializationTests.java b/src/test/java/org/qortal/test/SerializationTests.java index 8422bd9c..d5c12c00 100644 --- a/src/test/java/org/qortal/test/SerializationTests.java +++ b/src/test/java/org/qortal/test/SerializationTests.java @@ -59,6 +59,7 @@ public class SerializationTests extends Common { TransactionData transactionData = TransactionUtils.randomTransaction(repository, signingAccount, txType, true); Transaction transaction = Transaction.fromData(repository, transactionData); transaction.sign(signingAccount); + transaction.importAsUnconfirmed(); final int claimedLength = TransactionTransformer.getDataLength(transactionData); byte[] serializedTransaction = TransactionTransformer.toBytes(transactionData); From 745cfe8ea15f31bde71a4f591065907f74d1aa00 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 13 Jan 2023 19:45:38 +0000 Subject: [PATCH 05/25] chatReferenceTimestamp set to 1674316800000 (Sat, 21 Jan 2023 16:00:00 GMT) --- src/main/resources/blockchain.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 4ac40f62..aa6cd73b 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -85,7 +85,7 @@ "onlineAccountMinterLevelValidationHeight": 1092000, "selfSponsorshipAlgoV1Height": 1092400, "feeValidationFixTimestamp": 1671918000000, - "chatReferenceTimestamp": 9999999999999 + "chatReferenceTimestamp": 1674316800000 }, "genesisInfo": { "version": 4, From 4dc0033a5a76d6ba54cf96df12ce4c302eba8740 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 13 Jan 2023 19:45:52 +0000 Subject: [PATCH 06/25] Added missing chatReferenceTimestamp in unit tests. --- src/test/resources/test-chain-v2-self-sponsorship-algo.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/resources/test-chain-v2-self-sponsorship-algo.json b/src/test/resources/test-chain-v2-self-sponsorship-algo.json index c13d55da..68b33cc3 100644 --- a/src/test/resources/test-chain-v2-self-sponsorship-algo.json +++ b/src/test/resources/test-chain-v2-self-sponsorship-algo.json @@ -77,8 +77,9 @@ "disableReferenceTimestamp": 9999999999999, "increaseOnlineAccountsDifficultyTimestamp": 9999999999999, "onlineAccountMinterLevelValidationHeight": 0, + "selfSponsorshipAlgoV1Height": 20, "feeValidationFixTimestamp": 0, - "selfSponsorshipAlgoV1Height": 20 + "chatReferenceTimestamp": 0 }, "genesisInfo": { "version": 4, From 0ad9e2f65bc4aedb26f63ea8144058f395dce800 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 13 Jan 2023 20:08:47 +0000 Subject: [PATCH 07/25] Added QCHAT_ATTACHMENT service, with custom validation function. --- .../org/qortal/arbitrary/misc/Service.java | 32 ++++++- .../test/arbitrary/ArbitraryServiceTests.java | 91 ++++++++++++++++++- 2 files changed, 118 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java index 5dd8d94e..dc2deaeb 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Service.java +++ b/src/main/java/org/qortal/arbitrary/misc/Service.java @@ -10,9 +10,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; import static java.util.Arrays.stream; import static java.util.stream.Collectors.toMap; @@ -20,6 +18,31 @@ import static java.util.stream.Collectors.toMap; public enum Service { AUTO_UPDATE(1, false, null, null), ARBITRARY_DATA(100, false, null, null), + QCHAT_ATTACHMENT(120, true, 1024*1024L, null) { + @Override + public ValidationResult validate(Path path) { + // Custom validation function to require a single file, with a whitelisted extension + int fileCount = 0; + File[] files = path.toFile().listFiles(); + if (files != null) { + for (File file : files) { + if (file.isDirectory()) { + return ValidationResult.DIRECTORIES_NOT_ALLOWED; + } + final String extension = FilenameUtils.getExtension(file.getName()).toLowerCase(); + final List allowedExtensions = Arrays.asList("zip", "pdf", "txt", "odt", "ods", "doc", "docx", "xls", "xlsx", "ppt", "pptx"); + if (extension == null || !allowedExtensions.contains(extension)) { + return ValidationResult.INVALID_FILE_EXTENSION; + } + fileCount++; + } + } + if (fileCount != 1) { + return ValidationResult.INVALID_FILE_COUNT; + } + return ValidationResult.OK; + } + }, WEBSITE(200, true, null, null) { @Override public ValidationResult validate(Path path) { @@ -143,7 +166,8 @@ public enum Service { MISSING_INDEX_FILE(4), DIRECTORIES_NOT_ALLOWED(5), INVALID_FILE_EXTENSION(6), - MISSING_DATA(7); + MISSING_DATA(7), + INVALID_FILE_COUNT(8); public final int value; diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java index e6a51776..f7738c45 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java @@ -175,4 +175,93 @@ public class ArbitraryServiceTests extends Common { assertEquals(ValidationResult.INVALID_FILE_EXTENSION, service.validate(path)); } -} + @Test + public void testValidateQChatAttachment() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateQChatAttachment"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "document.pdf"), data, StandardOpenOption.CREATE); + + Service service = Service.QCHAT_ATTACHMENT; + assertTrue(service.isValidationRequired()); + + // There is an index file in the root + assertEquals(ValidationResult.OK, service.validate(path)); + } + + @Test + public void testValidateInvalidQChatAttachmentFileExtension() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateInvalidQChatAttachmentFileExtension"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "application.exe"), data, StandardOpenOption.CREATE); + + Service service = Service.QCHAT_ATTACHMENT; + assertTrue(service.isValidationRequired()); + + // There is an index file in the root + assertEquals(ValidationResult.INVALID_FILE_EXTENSION, service.validate(path)); + } + + @Test + public void testValidateEmptyQChatAttachment() throws IOException { + Path path = Files.createTempDirectory("testValidateEmptyQChatAttachment"); + + Service service = Service.QCHAT_ATTACHMENT; + assertTrue(service.isValidationRequired()); + + // There is an index file in the root + assertEquals(ValidationResult.INVALID_FILE_COUNT, service.validate(path)); + } + + @Test + public void testValidateMultiLayerQChatAttachment() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateMultiLayerQChatAttachment"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "file1.txt"), data, StandardOpenOption.CREATE); + + Path subdirectory = Paths.get(path.toString(), "subdirectory"); + Files.createDirectories(subdirectory); + Files.write(Paths.get(subdirectory.toString(), "file2.txt"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(subdirectory.toString(), "file3.txt"), data, StandardOpenOption.CREATE); + + Service service = Service.QCHAT_ATTACHMENT; + assertTrue(service.isValidationRequired()); + + // There is an index file in the root + assertEquals(ValidationResult.DIRECTORIES_NOT_ALLOWED, service.validate(path)); + } + + @Test + public void testValidateMultiFileQChatAttachment() throws IOException { + // Generate some random data + byte[] data = new byte[1024]; + new Random().nextBytes(data); + + // Write the data to several files in a temp path + Path path = Files.createTempDirectory("testValidateMultiFileQChatAttachment"); + path.toFile().deleteOnExit(); + Files.write(Paths.get(path.toString(), "file1.txt"), data, StandardOpenOption.CREATE); + Files.write(Paths.get(path.toString(), "file2.txt"), data, StandardOpenOption.CREATE); + + Service service = Service.QCHAT_ATTACHMENT; + assertTrue(service.isValidationRequired()); + + // There is an index file in the root + assertEquals(ValidationResult.INVALID_FILE_COUNT, service.validate(path)); + } + +} \ No newline at end of file From 02d5043ef7900166af851d07bae76d00ed0d43db Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 13 Jan 2023 20:17:27 +0000 Subject: [PATCH 08/25] Added missing calls to electrumX.setBlockchain(instance); for DGB and RVN. Thanks to @QuickMythril for noticing this. --- src/main/java/org/qortal/crosschain/Digibyte.java | 2 ++ src/main/java/org/qortal/crosschain/Ravencoin.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/crosschain/Digibyte.java b/src/main/java/org/qortal/crosschain/Digibyte.java index 3ab5e78e..4358b3b3 100644 --- a/src/main/java/org/qortal/crosschain/Digibyte.java +++ b/src/main/java/org/qortal/crosschain/Digibyte.java @@ -134,6 +134,8 @@ public class Digibyte extends Bitcoiny { Context bitcoinjContext = new Context(digibyteNet.getParams()); instance = new Digibyte(digibyteNet, electrumX, bitcoinjContext, CURRENCY_CODE); + + electrumX.setBlockchain(instance); } return instance; diff --git a/src/main/java/org/qortal/crosschain/Ravencoin.java b/src/main/java/org/qortal/crosschain/Ravencoin.java index d65c0a13..7bf5b20f 100644 --- a/src/main/java/org/qortal/crosschain/Ravencoin.java +++ b/src/main/java/org/qortal/crosschain/Ravencoin.java @@ -138,6 +138,8 @@ public class Ravencoin extends Bitcoiny { Context bitcoinjContext = new Context(ravencoinNet.getParams()); instance = new Ravencoin(ravencoinNet, electrumX, bitcoinjContext, CURRENCY_CODE); + + electrumX.setBlockchain(instance); } return instance; From 476fdcb31d442e49c5093911f71f0c44fff69edf Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Jan 2023 10:38:50 +0000 Subject: [PATCH 09/25] Added serialization tests for chatReference, and grouped with other serialization tests into a single package. --- .../data/transaction/ChatTransactionData.java | 4 + .../AtSerializationTests.java | 2 +- .../serialization/ChatSerializationTests.java | 102 ++++++++++++++++++ .../SerializationTests.java | 2 +- 4 files changed, 108 insertions(+), 2 deletions(-) rename src/test/java/org/qortal/test/{at => serialization}/AtSerializationTests.java (99%) create mode 100644 src/test/java/org/qortal/test/serialization/ChatSerializationTests.java rename src/test/java/org/qortal/test/{ => serialization}/SerializationTests.java (99%) diff --git a/src/main/java/org/qortal/data/transaction/ChatTransactionData.java b/src/main/java/org/qortal/data/transaction/ChatTransactionData.java index 81bdb2b7..5a6adf7f 100644 --- a/src/main/java/org/qortal/data/transaction/ChatTransactionData.java +++ b/src/main/java/org/qortal/data/transaction/ChatTransactionData.java @@ -85,6 +85,10 @@ public class ChatTransactionData extends TransactionData { return this.chatReference; } + public void setChatReference(byte[] chatReference) { + this.chatReference = chatReference; + } + public byte[] getData() { return this.data; } diff --git a/src/test/java/org/qortal/test/at/AtSerializationTests.java b/src/test/java/org/qortal/test/serialization/AtSerializationTests.java similarity index 99% rename from src/test/java/org/qortal/test/at/AtSerializationTests.java rename to src/test/java/org/qortal/test/serialization/AtSerializationTests.java index 3953bcdf..ea8d6bcd 100644 --- a/src/test/java/org/qortal/test/at/AtSerializationTests.java +++ b/src/test/java/org/qortal/test/serialization/AtSerializationTests.java @@ -1,4 +1,4 @@ -package org.qortal.test.at; +package org.qortal.test.serialization; import com.google.common.hash.HashCode; import org.junit.After; diff --git a/src/test/java/org/qortal/test/serialization/ChatSerializationTests.java b/src/test/java/org/qortal/test/serialization/ChatSerializationTests.java new file mode 100644 index 00000000..983896db --- /dev/null +++ b/src/test/java/org/qortal/test/serialization/ChatSerializationTests.java @@ -0,0 +1,102 @@ +package org.qortal.test.serialization; + +import com.google.common.hash.HashCode; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.data.transaction.ChatTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.Common; +import org.qortal.test.common.transaction.ChatTestTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.transform.TransformationException; +import org.qortal.transform.transaction.TransactionTransformer; +import org.qortal.utils.Base58; + +import static org.junit.Assert.*; + +public class ChatSerializationTests { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + + @Test + public void testChatSerializationWithChatReference() throws DataException, TransformationException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Build MESSAGE-type AT transaction with chatReference + PrivateKeyAccount signingAccount = Common.getTestAccount(repository, "alice"); + ChatTransactionData transactionData = (ChatTransactionData) ChatTestTransaction.randomTransaction(repository, signingAccount, true); + Transaction transaction = Transaction.fromData(repository, transactionData); + transaction.sign(signingAccount); + + assertNotNull(transactionData.getChatReference()); + + final int claimedLength = TransactionTransformer.getDataLength(transactionData); + byte[] serializedTransaction = TransactionTransformer.toBytes(transactionData); + assertEquals("Serialized CHAT transaction length differs from declared length", claimedLength, serializedTransaction.length); + + TransactionData deserializedTransactionData = TransactionTransformer.fromBytes(serializedTransaction); + // Re-sign + Transaction deserializedTransaction = Transaction.fromData(repository, deserializedTransactionData); + deserializedTransaction.sign(signingAccount); + assertEquals("Deserialized CHAT transaction signature differs", Base58.encode(transactionData.getSignature()), Base58.encode(deserializedTransactionData.getSignature())); + + // Re-serialize to check new length and bytes + final int reclaimedLength = TransactionTransformer.getDataLength(deserializedTransactionData); + assertEquals("Reserialized CHAT transaction declared length differs", claimedLength, reclaimedLength); + + byte[] reserializedTransaction = TransactionTransformer.toBytes(deserializedTransactionData); + assertEquals("Reserialized CHAT transaction bytes differ", HashCode.fromBytes(serializedTransaction).toString(), HashCode.fromBytes(reserializedTransaction).toString()); + + // Deserialized chat reference must match initial chat reference + ChatTransactionData deserializedChatTransactionData = (ChatTransactionData) deserializedTransactionData; + assertNotNull(deserializedChatTransactionData.getChatReference()); + assertArrayEquals(deserializedChatTransactionData.getChatReference(), transactionData.getChatReference()); + } + } + + @Test + public void testChatSerializationWithoutChatReference() throws DataException, TransformationException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Build MESSAGE-type AT transaction without chatReference + PrivateKeyAccount signingAccount = Common.getTestAccount(repository, "alice"); + ChatTransactionData transactionData = (ChatTransactionData) ChatTestTransaction.randomTransaction(repository, signingAccount, true); + transactionData.setChatReference(null); + Transaction transaction = Transaction.fromData(repository, transactionData); + transaction.sign(signingAccount); + + assertNull(transactionData.getChatReference()); + + final int claimedLength = TransactionTransformer.getDataLength(transactionData); + byte[] serializedTransaction = TransactionTransformer.toBytes(transactionData); + assertEquals("Serialized CHAT transaction length differs from declared length", claimedLength, serializedTransaction.length); + + TransactionData deserializedTransactionData = TransactionTransformer.fromBytes(serializedTransaction); + // Re-sign + Transaction deserializedTransaction = Transaction.fromData(repository, deserializedTransactionData); + deserializedTransaction.sign(signingAccount); + assertEquals("Deserialized CHAT transaction signature differs", Base58.encode(transactionData.getSignature()), Base58.encode(deserializedTransactionData.getSignature())); + + // Re-serialize to check new length and bytes + final int reclaimedLength = TransactionTransformer.getDataLength(deserializedTransactionData); + assertEquals("Reserialized CHAT transaction declared length differs", claimedLength, reclaimedLength); + + byte[] reserializedTransaction = TransactionTransformer.toBytes(deserializedTransactionData); + assertEquals("Reserialized CHAT transaction bytes differ", HashCode.fromBytes(serializedTransaction).toString(), HashCode.fromBytes(reserializedTransaction).toString()); + + // Deserialized chat reference must match initial chat reference + ChatTransactionData deserializedChatTransactionData = (ChatTransactionData) deserializedTransactionData; + assertNull(deserializedChatTransactionData.getChatReference()); + assertArrayEquals(deserializedChatTransactionData.getChatReference(), transactionData.getChatReference()); + } + } + +} diff --git a/src/test/java/org/qortal/test/SerializationTests.java b/src/test/java/org/qortal/test/serialization/SerializationTests.java similarity index 99% rename from src/test/java/org/qortal/test/SerializationTests.java rename to src/test/java/org/qortal/test/serialization/SerializationTests.java index d5c12c00..e9767909 100644 --- a/src/test/java/org/qortal/test/SerializationTests.java +++ b/src/test/java/org/qortal/test/serialization/SerializationTests.java @@ -1,4 +1,4 @@ -package org.qortal.test; +package org.qortal.test.serialization; import org.junit.Ignore; import org.junit.Test; From f78101e9cc68ab8ec199f1269f282ee5dc484d37 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Jan 2023 11:07:54 +0000 Subject: [PATCH 10/25] Updated a default bootstrap host to use a domain instead of its IP. --- src/main/java/org/qortal/settings/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 0423f855..546bd936 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -273,7 +273,7 @@ public class Settings { private String[] bootstrapHosts = new String[] { "http://bootstrap.qortal.org", "http://bootstrap2.qortal.org", - "http://62.171.190.193" + "http://bootstrap.qortal.online" }; // Auto-update sources From c62c59b44571d54410109fa756dfa50a9972e3ce Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Jan 2023 12:57:44 +0000 Subject: [PATCH 11/25] Use correct timeout (12s) when sending arbitrary data to a peer, and improved logging. --- .../arbitrary/ArbitraryDataFileManager.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 30b0fcca..807704dd 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -288,7 +288,7 @@ public class ArbitraryDataFileManager extends Thread { // The ID needs to match that of the original request message.setId(originalMessage.getId()); - if (!requestingPeer.sendMessage(message)) { + if (!requestingPeer.sendMessageWithTimeout(message, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT)) { LOGGER.debug("Failed to forward arbitrary data file to peer {}", requestingPeer); requestingPeer.disconnect("failed to forward arbitrary data file"); } @@ -564,13 +564,16 @@ public class ArbitraryDataFileManager extends Thread { LOGGER.trace("Hash {} exists", hash58); // We can serve the file directly as we already have it + LOGGER.debug("Sending file {}...", arbitraryDataFile); ArbitraryDataFileMessage arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, arbitraryDataFile); arbitraryDataFileMessage.setId(message.getId()); - if (!peer.sendMessage(arbitraryDataFileMessage)) { - LOGGER.debug("Couldn't sent file"); + if (!peer.sendMessageWithTimeout(arbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT)) { + LOGGER.debug("Couldn't send file {}", arbitraryDataFile); peer.disconnect("failed to send file"); } - LOGGER.debug("Sent file {}", arbitraryDataFile); + else { + LOGGER.debug("Sent file {}", arbitraryDataFile); + } } else if (relayInfo != null) { LOGGER.debug("We have relay info for hash {}", Base58.encode(hash)); From 0596a07c7de7cfe36e9b1aade9f07b149c3fec28 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Jan 2023 12:58:35 +0000 Subject: [PATCH 12/25] Reduced ArbitraryDataFileRequestThread count from 10 to 5, to reduce network flooding. --- .../qortal/controller/arbitrary/ArbitraryDataFileManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 807704dd..e2de1ae0 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -82,7 +82,7 @@ public class ArbitraryDataFileManager extends Thread { try { // Use a fixed thread pool to execute the arbitrary data file requests - int threadCount = 10; + int threadCount = 5; ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount); for (int i = 0; i < threadCount; i++) { arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread()); From 016191bdb0887c20df91f859e6821b0342503772 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Jan 2023 15:15:48 +0000 Subject: [PATCH 13/25] Reduce log spam when a QDN resource can't be found due to it not being published. --- .../arbitrary/ArbitraryDataBuilder.java | 3 ++- .../qortal/arbitrary/ArbitraryDataReader.java | 11 +++++++++- .../arbitrary/ArbitraryDataResource.java | 3 ++- .../exception/DataNotPublishedException.java | 22 +++++++++++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/qortal/arbitrary/exception/DataNotPublishedException.java diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index 4f0e3835..b6b17ea5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -2,6 +2,7 @@ package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.exception.DataNotPublishedException; import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; import org.qortal.arbitrary.misc.Service; @@ -88,7 +89,7 @@ public class ArbitraryDataBuilder { if (latestPut == null) { String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s", this.name, this.service, this.identifierString()); - throw new DataException(message); + throw new DataNotPublishedException(message); } this.latestPutTransaction = latestPut; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 5d4b015c..d1a8b4f5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.exception.DataNotPublishedException; import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; @@ -169,10 +170,18 @@ public class ArbitraryDataReader { this.uncompress(); this.validate(); + } catch (DataNotPublishedException e) { + if (e.getMessage() != null) { + // Log the message only, to avoid spamming the logs with a full stack trace + LOGGER.debug("DataNotPublishedException when trying to load QDN resource: {}", e.getMessage()); + } + this.deleteWorkingDirectory(); + throw e; + } catch (DataException e) { LOGGER.info("DataException when trying to load QDN resource", e); this.deleteWorkingDirectory(); - throw new DataException(e.getMessage()); + throw e; } finally { this.postExecute(); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java index 616c9b03..2720e4b2 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -3,6 +3,7 @@ package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; +import org.qortal.arbitrary.exception.DataNotPublishedException; import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; @@ -325,7 +326,7 @@ public class ArbitraryDataResource { if (latestPut == null) { String message = String.format("Couldn't find PUT transaction for name %s, service %s and identifier %s", this.resourceId, this.service, this.identifierString()); - throw new DataException(message); + throw new DataNotPublishedException(message); } this.latestPutTransaction = latestPut; diff --git a/src/main/java/org/qortal/arbitrary/exception/DataNotPublishedException.java b/src/main/java/org/qortal/arbitrary/exception/DataNotPublishedException.java new file mode 100644 index 00000000..4782826b --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/exception/DataNotPublishedException.java @@ -0,0 +1,22 @@ +package org.qortal.arbitrary.exception; + +import org.qortal.repository.DataException; + +public class DataNotPublishedException extends DataException { + + public DataNotPublishedException() { + } + + public DataNotPublishedException(String message) { + super(message); + } + + public DataNotPublishedException(String message, Throwable cause) { + super(message, cause); + } + + public DataNotPublishedException(Throwable cause) { + super(cause); + } + +} From 39e59cbcf812ded42cc4d144997cfec153194bb1 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 14 Jan 2023 18:47:46 +0000 Subject: [PATCH 14/25] Bump version to 3.8.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b66f016f..7a82ad37 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.8.2 + 3.8.3 jar true From 2a55eba1f7b695f34c82bf52fd4407d7c387325f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Jan 2023 11:28:37 +0000 Subject: [PATCH 15/25] Updated AdvancedInstaller project for v3.8.3 --- WindowsInstaller/Qortal.aip | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip index 1f579a9c..7af02485 100755 --- a/WindowsInstaller/Qortal.aip +++ b/WindowsInstaller/Qortal.aip @@ -17,10 +17,10 @@ - + - + @@ -212,7 +212,7 @@ - + From e91e612b55e5cbbbf781bad317e2f2244952d13c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Jan 2023 11:33:16 +0000 Subject: [PATCH 16/25] Added checkpoint lookup on startup. Currently enabled for topOnly nodes only. This will detect if the node is on a divergent chain, and will force a bootstrap or resync (depending on settings) in order to rejoin the main chain. --- .../java/org/qortal/block/BlockChain.java | 60 +++++++++++++++---- src/main/resources/blockchain.json | 3 + 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index bacd7825..437a48ab 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -100,6 +100,13 @@ public class BlockChain { /** Whether only one registered name is allowed per account. */ private boolean oneNamePerAccount = false; + /** Checkpoints */ + public static class Checkpoint { + public int height; + public String signature; + } + private List checkpoints; + /** Block rewards by block height */ public static class RewardByHeight { public int height; @@ -381,6 +388,10 @@ public class BlockChain { return this.oneNamePerAccount; } + public List getCheckpoints() { + return this.checkpoints; + } + public List getBlockRewardsByHeight() { return this.rewardsByHeight; } @@ -679,6 +690,7 @@ public class BlockChain { boolean isTopOnly = Settings.getInstance().isTopOnly(); boolean archiveEnabled = Settings.getInstance().isArchiveEnabled(); + boolean isLite = Settings.getInstance().isLite(); boolean canBootstrap = Settings.getInstance().getBootstrap(); boolean needsArchiveRebuild = false; BlockData chainTip; @@ -699,22 +711,44 @@ public class BlockChain { } } } + + // Validate checkpoints + // Limited to topOnly nodes for now, in order to reduce risk, and to solve a real-world problem with divergent topOnly nodes + // TODO: remove the isTopOnly conditional below once this feature has had more testing time + if (isTopOnly && !isLite) { + List checkpoints = BlockChain.getInstance().getCheckpoints(); + for (Checkpoint checkpoint : checkpoints) { + BlockData blockData = repository.getBlockRepository().fromHeight(checkpoint.height); + if (blockData == null) { + // Try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(checkpoint.height); + } + if (blockData == null) { + LOGGER.trace("Couldn't find block for height {}", checkpoint.height); + // This is likely due to the block being pruned, so is safe to ignore. + // Continue, as there might be other blocks we can check more definitively. + continue; + } + + byte[] signature = Base58.decode(checkpoint.signature); + if (!Arrays.equals(signature, blockData.getSignature())) { + LOGGER.info("Error: block at height {} with signature {} doesn't match checkpoint sig: {}. Bootstrapping...", checkpoint.height, Base58.encode(blockData.getSignature()), checkpoint.signature); + needsArchiveRebuild = true; + break; + } + LOGGER.info("Block at height {} matches checkpoint signature", blockData.getHeight()); + } + } + } - boolean hasBlocks = (chainTip != null && chainTip.getHeight() > 1); + // Check first block is Genesis Block + if (!isGenesisBlockValid() || needsArchiveRebuild) { + try { + rebuildBlockchain(); - if (isTopOnly && hasBlocks) { - // Top-only mode is enabled and we have blocks, so it's possible that the genesis block has been pruned - // It's best not to validate it, and there's no real need to - } else { - // Check first block is Genesis Block - if (!isGenesisBlockValid() || needsArchiveRebuild) { - try { - rebuildBlockchain(); - - } catch (InterruptedException e) { - throw new DataException(String.format("Interrupted when trying to rebuild blockchain: %s", e.getMessage())); - } + } catch (InterruptedException e) { + throw new DataException(String.format("Interrupted when trying to rebuild blockchain: %s", e.getMessage())); } } diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index aa6cd73b..f48958eb 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -87,6 +87,9 @@ "feeValidationFixTimestamp": 1671918000000, "chatReferenceTimestamp": 1674316800000 }, + "checkpoints": [ + { "height": 1131800, "signature": "EpRam4PLdKzULMp7xNU7XG964AKfioG3g1k7cxwxWXnXspPwnjfF6UncEz4feuSA9mr1vW5d3YQPGruXYjj4vciSh4SPj5iWRxkHRWFeRpQnmVUyaVumuBTwM8nnLKJTdtkZnd6d8Mc5mVFdHs6EwLBTY4HECoRcbo4e4FwkfqVon4M" } + ], "genesisInfo": { "version": 4, "timestamp": "1593450000000", From 30105199a2de58b949b3bdac97bde8d5a83ee5a2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Jan 2023 12:00:32 +0000 Subject: [PATCH 17/25] Default pruneBlockLimit increased from 1450 to 6000 (approx 5 days), to be more similar to the AT states retention time of full nodes. --- src/main/java/org/qortal/block/BlockChain.java | 4 +--- src/main/java/org/qortal/settings/Settings.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 437a48ab..b96350e6 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -757,9 +757,7 @@ public class BlockChain { try (final Repository repository = RepositoryManager.getRepository()) { repository.checkConsistency(); - // Set the number of blocks to validate based on the pruned state of the chain - // If pruned, subtract an extra 10 to allow room for error - int blocksToValidate = (isTopOnly || archiveEnabled) ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440; + int blocksToValidate = Math.min(Settings.getInstance().getPruneBlockLimit() - 10, 1440); int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - blocksToValidate, 1); BlockData detachedBlockData = repository.getBlockRepository().getDetachedBlockSignature(startHeight); diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 546bd936..d51737a3 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -159,7 +159,7 @@ public class Settings { * This prevents the node from being able to serve older blocks */ private boolean topOnly = false; /** The amount of recent blocks we should keep when pruning */ - private int pruneBlockLimit = 1450; + private int pruneBlockLimit = 6000; /** How often to attempt AT state pruning (ms). */ private long atStatesPruneInterval = 3219L; // milliseconds From dfe3754afc3d3ede3e7f4722a1baae9f4432c324 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Jan 2023 12:07:27 +0000 Subject: [PATCH 18/25] Block connections with peers older than 3.8.2, as those versions are nonfunctional due to recent feature triggers. --- src/main/java/org/qortal/network/Handshake.java | 2 +- src/main/java/org/qortal/settings/Settings.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index b2e5f829..47752767 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -265,7 +265,7 @@ public enum Handshake { private static final long PEER_VERSION_131 = 0x0100030001L; /** Minimum peer version that we are allowed to communicate with */ - private static final String MIN_PEER_VERSION = "3.7.0"; + private static final String MIN_PEER_VERSION = "3.8.2"; private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes private static final int POW_DIFFICULTY_PRE_131 = 8; // leading zero bits diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index d51737a3..5799bd26 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -215,7 +215,7 @@ public class Settings { public long recoveryModeTimeout = 10 * 60 * 1000L; /** Minimum peer version number required in order to sync with them */ - private String minPeerVersion = "3.8.0"; + private String minPeerVersion = "3.8.2"; /** Whether to allow connections with peers below minPeerVersion * If true, we won't sync with them but they can still sync with us, and will show in the peers list * If false, sync will be blocked both ways, and they will not appear in the peers list */ From c03f271825595cc1350b2a2274047a548cc52a73 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Jan 2023 12:44:19 +0000 Subject: [PATCH 19/25] Keep track of peers which are too divergent, and return an `isTooDivergent` boolean in /peers APIs. isTooDivergent will be true or false if a definitive decision has been made, or missing from the response if not yet known. Therefore it should be safe to treat `"isTooDivergent": false` as a peer that is on the same chain. --- .../java/org/qortal/api/model/ConnectedPeer.java | 7 +++++++ src/main/java/org/qortal/controller/Controller.java | 10 ++++++++++ .../java/org/qortal/controller/Synchronizer.java | 4 ++++ src/main/java/org/qortal/network/Peer.java | 13 +++++++++++++ 4 files changed, 34 insertions(+) diff --git a/src/main/java/org/qortal/api/model/ConnectedPeer.java b/src/main/java/org/qortal/api/model/ConnectedPeer.java index 3d383321..c4198654 100644 --- a/src/main/java/org/qortal/api/model/ConnectedPeer.java +++ b/src/main/java/org/qortal/api/model/ConnectedPeer.java @@ -1,6 +1,7 @@ package org.qortal.api.model; import io.swagger.v3.oas.annotations.media.Schema; +import org.qortal.controller.Controller; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.network.PeerData; import org.qortal.network.Handshake; @@ -36,6 +37,7 @@ public class ConnectedPeer { public Long lastBlockTimestamp; public UUID connectionId; public String age; + public Boolean isTooDivergent; protected ConnectedPeer() { } @@ -69,6 +71,11 @@ public class ConnectedPeer { this.lastBlockSignature = peerChainTipData.getSignature(); this.lastBlockTimestamp = peerChainTipData.getTimestamp(); } + + // Only include isTooDivergent decision if we've had the opportunity to request block summaries this peer + if (peer.getLastTooDivergentTime() != null) { + this.isTooDivergent = Controller.wasRecentlyTooDivergent.test(peer); + } } } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 0a323cb2..e9e1fcc2 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -769,6 +769,16 @@ public class Controller extends Thread { } }; + public static final Predicate wasRecentlyTooDivergent = peer -> { + Long now = NTP.getTime(); + Long peerLastTooDivergentTime = peer.getLastTooDivergentTime(); + if (now == null || peerLastTooDivergentTime == null) + return false; + + // Exclude any peers that were TOO_DIVERGENT in the last 5 mins + return (now - peerLastTooDivergentTime < 5 * 60 * 1000L); + }; + private long getRandomRepositoryMaintenanceInterval() { final long minInterval = Settings.getInstance().getRepositoryMaintenanceMinInterval(); final long maxInterval = Settings.getInstance().getRepositoryMaintenanceMaxInterval(); diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index e3ace9ed..2dad62e7 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -1121,6 +1121,7 @@ public class Synchronizer extends Thread { // If common block is too far behind us then we're on massively different forks so give up. if (!force && testHeight < ourHeight - MAXIMUM_COMMON_DELTA) { LOGGER.info(String.format("Blockchain too divergent with peer %s", peer)); + peer.setLastTooDivergentTime(NTP.getTime()); return SynchronizationResult.TOO_DIVERGENT; } @@ -1130,6 +1131,9 @@ public class Synchronizer extends Thread { testHeight = Math.max(testHeight - step, 1); } + // Peer not considered too divergent + peer.setLastTooDivergentTime(0L); + // Prepend test block's summary as first block summary, as summaries returned are *after* test block BlockSummaryData testBlockSummary = new BlockSummaryData(testBlockData); blockSummariesFromCommon.add(0, testBlockSummary); diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index a187d29b..4c05d5b9 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -155,6 +155,11 @@ public class Peer { */ private CommonBlockData commonBlockData; + /** + * Last time we detected this peer as TOO_DIVERGENT + */ + private Long lastTooDivergentTime; + // Message stats private static class MessageStats { @@ -383,6 +388,14 @@ public class Peer { this.commonBlockData = commonBlockData; } + public Long getLastTooDivergentTime() { + return this.lastTooDivergentTime; + } + + public void setLastTooDivergentTime(Long lastTooDivergentTime) { + this.lastTooDivergentTime = lastTooDivergentTime; + } + public boolean isSyncInProgress() { return this.syncInProgress; } From 4c52d6f0fcf1205c7bc7a47faca83845bbf4216c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Jan 2023 15:51:10 +0000 Subject: [PATCH 20/25] Fixed bug causing initial latestATStates data to be discarded. --- .../java/org/qortal/controller/repository/AtStatesPruner.java | 1 + .../java/org/qortal/controller/repository/AtStatesTrimmer.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java index bd12f784..064fe0ea 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java @@ -42,6 +42,7 @@ public class AtStatesPruner implements Runnable { repository.discardChanges(); repository.getATRepository().rebuildLatestAtStates(); + repository.saveChanges(); while (!Controller.isStopping()) { repository.discardChanges(); diff --git a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java index 69fa347c..6c026385 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java @@ -29,6 +29,7 @@ public class AtStatesTrimmer implements Runnable { repository.discardChanges(); repository.getATRepository().rebuildLatestAtStates(); + repository.saveChanges(); while (!Controller.isStopping()) { repository.discardChanges(); From 81cf46f5dd3102c1159717a381d2ff42ee44a993 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 16 Jan 2023 20:18:23 +0000 Subject: [PATCH 21/25] Disable block signing on topOnly nodes. Minting rewards are still earned on topOnly for now. --- src/main/java/org/qortal/controller/BlockMinter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index e2d01147..185dd7cd 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -63,8 +63,8 @@ public class BlockMinter extends Thread { public void run() { Thread.currentThread().setName("BlockMinter"); - if (Settings.getInstance().isLite()) { - // Lite nodes do not mint + if (Settings.getInstance().isTopOnly() || Settings.getInstance().isLite()) { + // Top only and lite nodes do not sign blocks return; } if (Settings.getInstance().getWipeUnconfirmedOnStart()) { From 688acd466c902a219afd92eb78b941a94a0acec6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 16 Jan 2023 20:23:43 +0000 Subject: [PATCH 22/25] Set checkpoint to block 1136300 --- src/main/resources/blockchain.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index f48958eb..46b4b4f9 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -88,7 +88,7 @@ "chatReferenceTimestamp": 1674316800000 }, "checkpoints": [ - { "height": 1131800, "signature": "EpRam4PLdKzULMp7xNU7XG964AKfioG3g1k7cxwxWXnXspPwnjfF6UncEz4feuSA9mr1vW5d3YQPGruXYjj4vciSh4SPj5iWRxkHRWFeRpQnmVUyaVumuBTwM8nnLKJTdtkZnd6d8Mc5mVFdHs6EwLBTY4HECoRcbo4e4FwkfqVon4M" } + { "height": 1136300, "signature": "3BbwawEF2uN8Ni5ofpJXkukoU8ctAPxYoFB7whq9pKfBnjfZcpfEJT4R95NvBDoTP8WDyWvsUvbfHbcr9qSZuYpSKZjUQTvdFf6eqznHGEwhZApWfvXu6zjGCxYCp65F4jsVYYJjkzbjmkCg5WAwN5voudngA23kMK6PpTNygapCzXt" } ], "genesisInfo": { "version": 4, From 9d81ea7744c2edbf527b5db9a59039977dc9fc9a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 16 Jan 2023 20:26:00 +0000 Subject: [PATCH 23/25] Bump version to 3.8.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7a82ad37..12f8472c 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 3.8.3 + 3.8.4 jar true From 64529e8abfb6a60125a634e8829fb74eab412c65 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Jan 2023 19:04:54 +0000 Subject: [PATCH 24/25] Added "reverse" and "includeOnlineSignatures" params to `GET /blocks/range/{height}` endpoint. --- .../org/qortal/api/resource/BlocksResource.java | 17 +++++++++++++---- .../java/org/qortal/test/api/BlockApiTests.java | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index 195b2ca4..15541802 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -634,13 +634,16 @@ public class BlocksResource { @ApiErrors({ ApiError.REPOSITORY_ISSUE }) - public List getBlockRange(@PathParam("height") int height, @Parameter( - ref = "count" - ) @QueryParam("count") int count) { + public List getBlockRange(@PathParam("height") int height, + @Parameter(ref = "count") @QueryParam("count") int count, + @Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse, + @QueryParam("includeOnlineSignatures") Boolean includeOnlineSignatures) { try (final Repository repository = RepositoryManager.getRepository()) { List blocks = new ArrayList<>(); + boolean shouldReverse = (reverse != null && reverse == true); - for (/* count already set */; count > 0; --count, ++height) { + int i = 0; + while (i < count) { BlockData blockData = repository.getBlockRepository().fromHeight(height); if (blockData == null) { // Not found - try the archive @@ -650,8 +653,14 @@ public class BlocksResource { break; } } + if (includeOnlineSignatures == null || includeOnlineSignatures == false) { + blockData.setOnlineAccountsSignatures(null); + } blocks.add(blockData); + + height = shouldReverse ? height - 1 : height + 1; + i++; } return blocks; diff --git a/src/test/java/org/qortal/test/api/BlockApiTests.java b/src/test/java/org/qortal/test/api/BlockApiTests.java index 47d5318a..23e7b007 100644 --- a/src/test/java/org/qortal/test/api/BlockApiTests.java +++ b/src/test/java/org/qortal/test/api/BlockApiTests.java @@ -84,7 +84,7 @@ public class BlockApiTests extends ApiCommon { @Test public void testGetBlockRange() { - assertNotNull(this.blocksResource.getBlockRange(1, 1)); + assertNotNull(this.blocksResource.getBlockRange(1, 1, false, false)); List testValues = Arrays.asList(null, Integer.valueOf(1)); From 2f7912abce09763f3dd1600828f31bcbecb31909 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Jan 2023 19:30:43 +0000 Subject: [PATCH 25/25] Compute balances for Bitcoin-like coins using unspent outputs. Should fix occasional incorrect balance issue, and speed up loading time. --- .../resource/CrossChainBitcoinResource.java | 2 +- .../resource/CrossChainDigibyteResource.java | 2 +- .../resource/CrossChainDogecoinResource.java | 2 +- .../resource/CrossChainLitecoinResource.java | 2 +- .../resource/CrossChainRavencoinResource.java | 2 +- .../java/org/qortal/crosschain/Bitcoiny.java | 96 ++++++++++++++++--- 6 files changed, 89 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java index 80d19804..dd967451 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java @@ -68,7 +68,7 @@ public class CrossChainBitcoinResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); try { - Long balance = bitcoin.getWalletBalanceFromTransactions(key58); + Long balance = bitcoin.getWalletBalance(key58); if (balance == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); diff --git a/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java b/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java index 57049639..31d51c73 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainDigibyteResource.java @@ -68,7 +68,7 @@ public class CrossChainDigibyteResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); try { - Long balance = digibyte.getWalletBalanceFromTransactions(key58); + Long balance = digibyte.getWalletBalance(key58); if (balance == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); diff --git a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java index 189a53d3..28bebfb8 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java @@ -66,7 +66,7 @@ public class CrossChainDogecoinResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); try { - Long balance = dogecoin.getWalletBalanceFromTransactions(key58); + Long balance = dogecoin.getWalletBalance(key58); if (balance == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); diff --git a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java index 8ac0f9a0..d12dd94c 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java @@ -68,7 +68,7 @@ public class CrossChainLitecoinResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); try { - Long balance = litecoin.getWalletBalanceFromTransactions(key58); + Long balance = litecoin.getWalletBalance(key58); if (balance == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); diff --git a/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java b/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java index 756b0bb5..97550392 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainRavencoinResource.java @@ -68,7 +68,7 @@ public class CrossChainRavencoinResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY); try { - Long balance = ravencoin.getWalletBalanceFromTransactions(key58); + Long balance = ravencoin.getWalletBalance(key58); if (balance == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE); diff --git a/src/main/java/org/qortal/crosschain/Bitcoiny.java b/src/main/java/org/qortal/crosschain/Bitcoiny.java index 350779bc..c08bd91e 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoiny.java +++ b/src/main/java/org/qortal/crosschain/Bitcoiny.java @@ -357,19 +357,33 @@ public abstract class Bitcoiny implements ForeignBlockchain { * @return unspent BTC balance, or null if unable to determine balance */ public Long getWalletBalance(String key58) throws ForeignBlockchainException { - // It's more accurate to calculate the balance from the transactions, rather than asking Bitcoinj - return this.getWalletBalanceFromTransactions(key58); + Long balance = 0L; -// Context.propagate(bitcoinjContext); -// -// Wallet wallet = walletFromDeterministicKey58(key58); -// wallet.setUTXOProvider(new WalletAwareUTXOProvider(this, wallet)); -// -// Coin balance = wallet.getBalance(); -// if (balance == null) -// return null; -// -// return balance.value; + List allUnspentOutputs = new ArrayList<>(); + Set walletAddresses = this.getWalletAddresses(key58); + for (String address : walletAddresses) { + allUnspentOutputs.addAll(this.getUnspentOutputs(address)); + } + for (TransactionOutput output : allUnspentOutputs) { + if (!output.isAvailableForSpending()) { + continue; + } + balance += output.getValue().value; + } + return balance; + } + + public Long getWalletBalanceFromBitcoinj(String key58) { + Context.propagate(bitcoinjContext); + + Wallet wallet = walletFromDeterministicKey58(key58); + wallet.setUTXOProvider(new WalletAwareUTXOProvider(this, wallet)); + + Coin balance = wallet.getBalance(); + if (balance == null) + return null; + + return balance.value; } public Long getWalletBalanceFromTransactions(String key58) throws ForeignBlockchainException { @@ -464,6 +478,64 @@ public abstract class Bitcoiny implements ForeignBlockchain { } } + public Set getWalletAddresses(String key58) throws ForeignBlockchainException { + synchronized (this) { + Context.propagate(bitcoinjContext); + + Wallet wallet = walletFromDeterministicKey58(key58); + DeterministicKeyChain keyChain = wallet.getActiveKeyChain(); + + keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT); + keyChain.maybeLookAhead(); + + List keys = new ArrayList<>(keyChain.getLeafKeys()); + + Set keySet = new HashSet<>(); + + int unusedCounter = 0; + int ki = 0; + do { + boolean areAllKeysUnused = true; + + for (; ki < keys.size(); ++ki) { + DeterministicKey dKey = keys.get(ki); + + // Check for transactions + Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH); + keySet.add(address.toString()); + byte[] script = ScriptBuilder.createOutputScript(address).getProgram(); + + // Ask for transaction history - if it's empty then key has never been used + List historicTransactionHashes = this.getAddressTransactions(script, false); + + if (!historicTransactionHashes.isEmpty()) { + areAllKeysUnused = false; + } + } + + if (areAllKeysUnused) { + // No transactions + if (unusedCounter >= Settings.getInstance().getGapLimit()) { + // ... and we've hit our search limit + break; + } + // We haven't hit our search limit yet so increment the counter and keep looking + unusedCounter += WALLET_KEY_LOOKAHEAD_INCREMENT; + } else { + // Some keys in this batch were used, so reset the counter + unusedCounter = 0; + } + + // Generate some more keys + keys.addAll(generateMoreKeys(keyChain)); + + // Process new keys + } while (true); + + return keySet; + } + } + protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set keySet) { long amount = 0; long total = 0L;