From 3acc0babb7abe149f0d7c6d9aa28f8fd5a27ba47 Mon Sep 17 00:00:00 2001 From: catbref Date: Wed, 29 Jul 2020 14:25:00 +0100 Subject: [PATCH 01/57] More chain-weight tests --- .../org/qortal/test/ChainWeightTests.java | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/src/test/java/org/qortal/test/ChainWeightTests.java b/src/test/java/org/qortal/test/ChainWeightTests.java index b02c155e..d133dad1 100644 --- a/src/test/java/org/qortal/test/ChainWeightTests.java +++ b/src/test/java/org/qortal/test/ChainWeightTests.java @@ -3,6 +3,8 @@ package org.qortal.test; import static org.junit.Assert.*; import java.math.BigInteger; +import java.text.DecimalFormat; +import java.text.NumberFormat; import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -23,6 +25,7 @@ import org.junit.Test; public class ChainWeightTests extends Common { private static final Random RANDOM = new Random(); + private static final NumberFormat FORMATTER = new DecimalFormat("0.###E0"); @Before public void beforeTest() throws DataException { @@ -89,6 +92,96 @@ public class ChainWeightTests extends Common { } } + // Demonstrates that typical key distance ranges from roughly 1E75 to 1E77 + @Test + public void testKeyDistances() { + byte[] parentMinterKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + byte[] testKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + + for (int i = 0; i < 50; ++i) { + int parentHeight = RANDOM.nextInt(50000); + RANDOM.nextBytes(parentMinterKey); + RANDOM.nextBytes(testKey); + int minterLevel = RANDOM.nextInt(10) + 1; + + BigInteger keyDistance = Block.calcKeyDistance(parentHeight, parentMinterKey, testKey, minterLevel); + + System.out.println(String.format("Parent height: %d, minter level: %d, distance: %s", + parentHeight, + minterLevel, + FORMATTER.format(keyDistance))); + } + } + + // If typical key distance ranges from 1E75 to 1E77 + // then we want lots of online accounts to push a 1E75 distance + // towards 1E77 so that it competes with a 1E77 key that has hardly any online accounts + // 1E75 is approx. 2**249 so maybe that's a good value for Block.ACCOUNTS_COUNT_SHIFT + @Test + public void testMoreAccountsVersusKeyDistance() throws DataException { + BigInteger minimumBetterKeyDistance = BigInteger.TEN.pow(77); + BigInteger maximumWorseKeyDistance = BigInteger.TEN.pow(75); + + try (final Repository repository = RepositoryManager.getRepository()) { + final byte[] parentMinterKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + + TestAccount betterAccount = Common.getTestAccount(repository, "bob-reward-share"); + byte[] betterKey = betterAccount.getPublicKey(); + int betterMinterLevel = Account.getRewardShareEffectiveMintingLevel(repository, betterKey); + + TestAccount worseAccount = Common.getTestAccount(repository, "dilbert-reward-share"); + byte[] worseKey = worseAccount.getPublicKey(); + int worseMinterLevel = Account.getRewardShareEffectiveMintingLevel(repository, worseKey); + + // This is to check that the hard-coded keys ARE actually better/worse as expected, before moving on testing more online accounts + BigInteger betterKeyDistance; + BigInteger worseKeyDistance; + + int parentHeight = 0; + do { + ++parentHeight; + betterKeyDistance = Block.calcKeyDistance(parentHeight, parentMinterKey, betterKey, betterMinterLevel); + worseKeyDistance = Block.calcKeyDistance(parentHeight, parentMinterKey, worseKey, worseMinterLevel); + } while (betterKeyDistance.compareTo(minimumBetterKeyDistance) < 0 || worseKeyDistance.compareTo(maximumWorseKeyDistance) > 0); + + System.out.println(String.format("Parent height: %d, better key distance: %s, worse key distance: %s", + parentHeight, + FORMATTER.format(betterKeyDistance), + FORMATTER.format(worseKeyDistance))); + + for (int accountsCountShift = 244; accountsCountShift <= 256; accountsCountShift += 2) { + for (int worseAccountsCount = 1; worseAccountsCount <= 101; worseAccountsCount += 25) { + for (int betterAccountsCount = 1; betterAccountsCount <= 1001; betterAccountsCount += 250) { + BlockSummaryData worseKeyBlockSummary = new BlockSummaryData(parentHeight + 1, null, worseKey, betterAccountsCount); + BlockSummaryData betterKeyBlockSummary = new BlockSummaryData(parentHeight + 1, null, betterKey, worseAccountsCount); + + populateBlockSummaryMinterLevel(repository, worseKeyBlockSummary); + populateBlockSummaryMinterLevel(repository, betterKeyBlockSummary); + + BigInteger worseKeyBlockWeight = calcBlockWeight(parentHeight, parentMinterKey, worseKeyBlockSummary, accountsCountShift); + BigInteger betterKeyBlockWeight = calcBlockWeight(parentHeight, parentMinterKey, betterKeyBlockSummary, accountsCountShift); + + System.out.println(String.format("Shift: %d, worse key: %d accounts, %s diff; better key: %d accounts: %s diff; winner: %s", + accountsCountShift, + betterAccountsCount, // used with worseKey + FORMATTER.format(worseKeyBlockWeight), + worseAccountsCount, // used with betterKey + FORMATTER.format(betterKeyBlockWeight), + worseKeyBlockWeight.compareTo(betterKeyBlockWeight) > 0 ? "worse key/better accounts" : "better key/worse accounts" + )); + } + } + + System.out.println(); + } + } + } + + private static BigInteger calcBlockWeight(int parentHeight, byte[] parentBlockSignature, BlockSummaryData blockSummaryData, int accountsCountShift) { + BigInteger keyDistance = Block.calcKeyDistance(parentHeight, parentBlockSignature, blockSummaryData.getMinterPublicKey(), blockSummaryData.getMinterLevel()); + return BigInteger.valueOf(blockSummaryData.getOnlineAccountsCount()).shiftLeft(accountsCountShift).add(keyDistance); + } + // Check that a longer chain beats a shorter chain @Test public void testLongerChain() throws DataException { From 9b0e88ca8753db1e16599c2ca388c587a01c494a Mon Sep 17 00:00:00 2001 From: catbref Date: Sat, 6 Feb 2021 11:40:29 +0000 Subject: [PATCH 02/57] Only compare same number of blocks when comparing peer chains --- src/main/java/org/qortal/block/Block.java | 5 ++-- .../java/org/qortal/block/BlockChain.java | 7 +++++- src/main/resources/blockchain.json | 3 ++- .../org/qortal/test/ChainWeightTests.java | 23 ++++++++++++++----- .../test-chain-v2-founder-rewards.json | 4 +++- .../test-chain-v2-leftover-reward.json | 4 +++- src/test/resources/test-chain-v2-minting.json | 4 +++- .../test-chain-v2-qora-holder-extremes.json | 4 +++- .../resources/test-chain-v2-qora-holder.json | 4 +++- .../test-chain-v2-reward-scaling.json | 4 +++- src/test/resources/test-chain-v2.json | 4 +++- 11 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 7a2be548..ce7d220b 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -822,10 +822,9 @@ public class Block { parentHeight = blockSummaryData.getHeight(); parentBlockSignature = blockSummaryData.getSignature(); - /* Potential future consensus change: only comparing the same number of blocks. - if (parentHeight >= maxHeight) + // After this timestamp, we only compare the same number of blocks + if (NTP.getTime() >= BlockChain.getInstance().getCalcChainWeightTimestamp() && parentHeight >= maxHeight) break; - */ } return cumulativeWeight; diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index ad9140e3..43b19468 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -70,7 +70,8 @@ public class BlockChain { private GenesisBlock.GenesisInfo genesisInfo; public enum FeatureTrigger { - atFindNextTransactionFix; + atFindNextTransactionFix, + calcChainWeightTimestamp; } /** Map of which blockchain features are enabled when (height/timestamp) */ @@ -376,6 +377,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.atFindNextTransactionFix.name()).intValue(); } + public long getCalcChainWeightTimestamp() { + return this.featureTriggers.get(FeatureTrigger.calcChainWeightTimestamp.name()).longValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 5b9a6202..17ffcf01 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -48,7 +48,8 @@ "minutesPerBlock": 1 }, "featureTriggers": { - "atFindNextTransactionFix": 275000 + "atFindNextTransactionFix": 275000, + "calcChainWeightTimestamp": 1616000000000, }, "genesisInfo": { "version": 4, diff --git a/src/test/java/org/qortal/test/ChainWeightTests.java b/src/test/java/org/qortal/test/ChainWeightTests.java index d133dad1..e53c4c8e 100644 --- a/src/test/java/org/qortal/test/ChainWeightTests.java +++ b/src/test/java/org/qortal/test/ChainWeightTests.java @@ -11,6 +11,7 @@ import java.util.Random; import org.qortal.account.Account; import org.qortal.block.Block; +import org.qortal.block.BlockChain; import org.qortal.data.block.BlockSummaryData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; @@ -19,7 +20,9 @@ import org.qortal.test.common.Common; import org.qortal.test.common.TestAccount; import org.qortal.transform.Transformer; import org.qortal.transform.block.BlockTransformer; +import org.qortal.utils.NTP; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; public class ChainWeightTests extends Common { @@ -27,6 +30,12 @@ public class ChainWeightTests extends Common { private static final Random RANDOM = new Random(); private static final NumberFormat FORMATTER = new DecimalFormat("0.###E0"); + @BeforeClass + public static void beforeClass() { + // We need this so that NTP.getTime() in Block.calcChainWeight() doesn't return null, causing NPE + NTP.setFixedOffset(0L); + } + @Before public void beforeTest() throws DataException { Common.useSettings("test-settings-v2-minting.json"); @@ -182,7 +191,7 @@ public class ChainWeightTests extends Common { return BigInteger.valueOf(blockSummaryData.getOnlineAccountsCount()).shiftLeft(accountsCountShift).add(keyDistance); } - // Check that a longer chain beats a shorter chain + // Check that a longer chain has same weight as shorter/truncated chain @Test public void testLongerChain() throws DataException { try (final Repository repository = RepositoryManager.getRepository()) { @@ -190,18 +199,20 @@ public class ChainWeightTests extends Common { BlockSummaryData commonBlockSummary = genBlockSummary(repository, commonBlockHeight); byte[] commonBlockGeneratorKey = commonBlockSummary.getMinterPublicKey(); - List shorterChain = genBlockSummaries(repository, 3, commonBlockSummary); - List longerChain = genBlockSummaries(repository, shorterChain.size() + 1, commonBlockSummary); - - populateBlockSummariesMinterLevels(repository, shorterChain); + List longerChain = genBlockSummaries(repository, 6, commonBlockSummary); populateBlockSummariesMinterLevels(repository, longerChain); + List shorterChain = longerChain.subList(0, longerChain.size() / 2); + final int mutualHeight = commonBlockHeight - 1 + Math.min(shorterChain.size(), longerChain.size()); BigInteger shorterChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockGeneratorKey, shorterChain, mutualHeight); BigInteger longerChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockGeneratorKey, longerChain, mutualHeight); - assertEquals("longer chain should have greater weight", 1, longerChainWeight.compareTo(shorterChainWeight)); + if (NTP.getTime() >= BlockChain.getInstance().getCalcChainWeightTimestamp()) + assertEquals("longer chain should have same weight", 0, longerChainWeight.compareTo(shorterChainWeight)); + else + assertEquals("longer chain should have greater weight", 1, longerChainWeight.compareTo(shorterChainWeight)); } } diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index 4ad21f35..6ffe946a 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -44,7 +44,9 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "calcChainWeightTimestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index d402aa95..2cf6f2ab 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -44,7 +44,9 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "calcChainWeightTimestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index 02b31ef9..4370f52b 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -44,7 +44,9 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "calcChainWeightTimestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index 2962f7a7..1c0f0a7b 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -44,7 +44,9 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "calcChainWeightTimestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index 11ccb0b0..ddb3cac9 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -44,7 +44,9 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "calcChainWeightTimestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index e454d8e7..c588bc9f 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -44,7 +44,9 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "calcChainWeightTimestamp": 0 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index 1939f357..e4f20209 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -44,7 +44,9 @@ "powfixTimestamp": 0, "qortalTimestamp": 0, "newAssetPricingTimestamp": 0, - "groupApprovalTimestamp": 0 + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "calcChainWeightTimestamp": 0 }, "genesisInfo": { "version": 4, From 1e6e5e66da9b6a87f44741294b60828459be3725 Mon Sep 17 00:00:00 2001 From: catbref Date: Sat, 6 Feb 2021 12:09:24 +0000 Subject: [PATCH 03/57] Fix trailing comma on blockchain.json! --- src/main/resources/blockchain.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 17ffcf01..29bb6d1a 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -49,7 +49,7 @@ }, "featureTriggers": { "atFindNextTransactionFix": 275000, - "calcChainWeightTimestamp": 1616000000000, + "calcChainWeightTimestamp": 1616000000000 }, "genesisInfo": { "version": 4, From 847e81e95cda304b29521b092755398e762614f0 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 12 Mar 2021 19:48:49 +0000 Subject: [PATCH 04/57] Fixed a mapping issue in Block->getShareBins(), to take effect at some future (undecided) height. Post trigger, account levels will map correctly to share bins, subtracting 1 to account for the 0th element of the shareBinsByLevel array. Pre-trigger, the legacy mapping will remain in effect. --- src/main/java/org/qortal/block/Block.java | 17 ++++++++++++----- src/main/java/org/qortal/block/BlockChain.java | 7 ++++++- src/main/resources/blockchain.json | 3 ++- .../test-chain-v2-founder-rewards.json | 3 ++- .../test-chain-v2-leftover-reward.json | 3 ++- src/test/resources/test-chain-v2-minting.json | 3 ++- .../test-chain-v2-qora-holder-extremes.json | 3 ++- .../resources/test-chain-v2-qora-holder.json | 3 ++- .../resources/test-chain-v2-reward-scaling.json | 3 ++- src/test/resources/test-chain-v2.json | 3 ++- 10 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 8551e4e7..34a87e9a 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -176,19 +176,26 @@ public class Block { * * @return account-level share "bin" from blockchain config, or null if founder / none found */ - public AccountLevelShareBin getShareBin() { + public AccountLevelShareBin getShareBin(int blockHeight) { if (this.isMinterFounder) return null; final int accountLevel = this.mintingAccountData.getLevel(); if (accountLevel <= 0) - return null; + return null; // level 0 isn't included in any share bins - final AccountLevelShareBin[] shareBinsByLevel = BlockChain.getInstance().getShareBinsByAccountLevel(); + final BlockChain blockChain = BlockChain.getInstance(); + final AccountLevelShareBin[] shareBinsByLevel = blockChain.getShareBinsByAccountLevel(); if (accountLevel > shareBinsByLevel.length) return null; - return shareBinsByLevel[accountLevel]; + if (blockHeight < blockChain.getShareBinFixHeight()) + // Off-by-one bug still in effect + return shareBinsByLevel[accountLevel]; + + // level 1 stored at index 0, level 2 stored at index 1, etc. + return shareBinsByLevel[accountLevel-1]; + } public long distribute(long accountAmount, Map balanceChanges) { @@ -1783,7 +1790,7 @@ public class Block { // Find all accounts in share bin. getShareBin() returns null for minter accounts that are also founders, so they are effectively filtered out. AccountLevelShareBin accountLevelShareBin = accountLevelShareBins.get(binIndex); // Object reference compare is OK as all references are read-only from blockchain config. - List binnedAccounts = expandedAccounts.stream().filter(accountInfo -> accountInfo.getShareBin() == accountLevelShareBin).collect(Collectors.toList()); + List binnedAccounts = expandedAccounts.stream().filter(accountInfo -> accountInfo.getShareBin(this.blockData.getHeight()) == accountLevelShareBin).collect(Collectors.toList()); // No online accounts in this bin? Skip to next one if (binnedAccounts.isEmpty()) diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index b3221619..a91a33d1 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -71,7 +71,8 @@ public class BlockChain { public enum FeatureTrigger { atFindNextTransactionFix, - newBlockSigHeight; + newBlockSigHeight, + shareBinFix; } /** Map of which blockchain features are enabled when (height/timestamp) */ @@ -381,6 +382,10 @@ public class BlockChain { return this.featureTriggers.get(FeatureTrigger.newBlockSigHeight.name()).intValue(); } + public int getShareBinFixHeight() { + return this.featureTriggers.get(FeatureTrigger.shareBinFix.name()).intValue(); + } + // More complex getters for aspects that change by height or timestamp public long getRewardAtHeight(int ourHeight) { diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 92d4ad86..363b80cb 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -49,7 +49,8 @@ }, "featureTriggers": { "atFindNextTransactionFix": 275000, - "newBlockSigHeight": 320000 + "newBlockSigHeight": 320000, + "shareBinFix": 999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-founder-rewards.json b/src/test/resources/test-chain-v2-founder-rewards.json index 6f3c5fff..ab082d9a 100644 --- a/src/test/resources/test-chain-v2-founder-rewards.json +++ b/src/test/resources/test-chain-v2-founder-rewards.json @@ -46,7 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, - "newBlockSigHeight": 999999 + "newBlockSigHeight": 999999, + "shareBinFix": 999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-leftover-reward.json b/src/test/resources/test-chain-v2-leftover-reward.json index 53e13915..2db28f99 100644 --- a/src/test/resources/test-chain-v2-leftover-reward.json +++ b/src/test/resources/test-chain-v2-leftover-reward.json @@ -46,7 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, - "newBlockSigHeight": 999999 + "newBlockSigHeight": 999999, + "shareBinFix": 999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-minting.json b/src/test/resources/test-chain-v2-minting.json index a5c841a0..13ed7009 100644 --- a/src/test/resources/test-chain-v2-minting.json +++ b/src/test/resources/test-chain-v2-minting.json @@ -46,7 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, - "newBlockSigHeight": 999999 + "newBlockSigHeight": 999999, + "shareBinFix": 999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder-extremes.json b/src/test/resources/test-chain-v2-qora-holder-extremes.json index 883e36fe..1e6948ba 100644 --- a/src/test/resources/test-chain-v2-qora-holder-extremes.json +++ b/src/test/resources/test-chain-v2-qora-holder-extremes.json @@ -46,7 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, - "newBlockSigHeight": 999999 + "newBlockSigHeight": 999999, + "shareBinFix": 999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-qora-holder.json b/src/test/resources/test-chain-v2-qora-holder.json index a06dda7f..89c461ba 100644 --- a/src/test/resources/test-chain-v2-qora-holder.json +++ b/src/test/resources/test-chain-v2-qora-holder.json @@ -46,7 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, - "newBlockSigHeight": 999999 + "newBlockSigHeight": 999999, + "shareBinFix": 999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2-reward-scaling.json b/src/test/resources/test-chain-v2-reward-scaling.json index 91d6a36b..f7c47c78 100644 --- a/src/test/resources/test-chain-v2-reward-scaling.json +++ b/src/test/resources/test-chain-v2-reward-scaling.json @@ -46,7 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, - "newBlockSigHeight": 999999 + "newBlockSigHeight": 999999, + "shareBinFix": 999999 }, "genesisInfo": { "version": 4, diff --git a/src/test/resources/test-chain-v2.json b/src/test/resources/test-chain-v2.json index dd8377be..b5e9f3bf 100644 --- a/src/test/resources/test-chain-v2.json +++ b/src/test/resources/test-chain-v2.json @@ -46,7 +46,8 @@ "newAssetPricingTimestamp": 0, "groupApprovalTimestamp": 0, "atFindNextTransactionFix": 0, - "newBlockSigHeight": 999999 + "newBlockSigHeight": 999999, + "shareBinFix": 999999 }, "genesisInfo": { "version": 4, From 22e3140ff0241b07cb9aabcbe3e6cfb03ef8efdd Mon Sep 17 00:00:00 2001 From: QuickMythril Date: Tue, 16 Mar 2021 03:00:55 -0400 Subject: [PATCH 05/57] add version on tooltip add Version Number on Qortal Core tooltip. https://i.imgur.com/eLnLnQ5.png --- src/main/java/org/qortal/controller/Controller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index b2c6c182..ce270ac1 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -775,7 +775,7 @@ public class Controller extends Thread { actionText = Translator.INSTANCE.translate("SysTray", "MINTING_DISABLED"); } - String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height); + String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height) + "\n" + String.format("Build version: %s", this.buildVersion); SysTray.getInstance().setToolTipText(tooltip); this.callbackExecutor.execute(() -> { From fde68dc598eec98fd8262d60ec43217beb602705 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 16 Mar 2021 09:11:49 +0000 Subject: [PATCH 06/57] Added unit test to test level 1 and 2 rewards. 1. Assign 3 minters (one founder, one level 1, one level 2) 2. Mint a block after the shareBinFix, ensuring that level 1 and 2 are being rewarded evenly from the same share bin. 3. Orphan the block and ensure the rewards are reversed. 4. Orphan two more blocks, each time checking that the balances are being reduced in accordance with the pre-shareBinFix mapping. --- .../org/qortal/test/minting/RewardTests.java | 127 ++++++++++++++++++ .../test-chain-v2-reward-levels.json | 74 ++++++++++ .../test-settings-v2-reward-levels.json | 7 + 3 files changed, 208 insertions(+) create mode 100644 src/test/resources/test-chain-v2-reward-levels.json create mode 100644 src/test/resources/test-settings-v2-reward-levels.json diff --git a/src/test/java/org/qortal/test/minting/RewardTests.java b/src/test/java/org/qortal/test/minting/RewardTests.java index 4d098f67..813c55cb 100644 --- a/src/test/java/org/qortal/test/minting/RewardTests.java +++ b/src/test/java/org/qortal/test/minting/RewardTests.java @@ -336,4 +336,131 @@ public class RewardTests extends Common { } } + /** Test rewards for level 1 and 2 accounts both pre and post the shareBinFix, including orphaning back through the feature trigger block */ + @Test + public void testLevel1And2Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share NOT online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint a couple of blocks so that we are able to orphan them later + for (int i=0; i<2; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(1, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(2, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Ensure that only Alice is a founder + assertEquals(1, getFlags(repository, "alice")); + assertEquals(0, getFlags(repository, "bob")); + assertEquals(0, getFlags(repository, "chloe")); + assertEquals(0, getFlags(repository, "dilbert")); + + // Now that everyone is at level 1, we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are at the correct height and block reward value + assertEquals(6, (int) repository.getBlockRepository().getLastBlock().getHeight()); + assertEquals(10000000000L, blockReward); + + /* + * Alice, Chloe, and Dilbert are 'online'. Bob is offline. + * Chloe is level 1, Dilbert is level 2. + * One founder online (Alice, who is also level 1). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 5% block reward for Level 1 and 2 + */ + + // We are after the shareBinFix feature trigger, so we expect level 1 and 2 to share the same reward (5%) + final int level1And2SharePercent = 5_00; // 5% + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long expectedReward = level1And2ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level1And2ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + assertEquals(500000000, level1And2ShareAmount); + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedReward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedReward); + + // Now orphan the latest block. This brings us to the threshold of the shareBinFix feature trigger. + BlockUtils.orphanBlocks(repository, 1); + assertEquals(5, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Ensure the latest post-fix block rewards have been subtracted and they have returned to their initial values + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance); + + // Orphan another block. This time, the block that was orphaned was prior to the shareBinFix feature trigger. + BlockUtils.orphanBlocks(repository, 1); + assertEquals(4, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Prior to the fix, the levels were incorrectly grouped + // Chloe should receive 100% of the level 1 reward, and Dilbert should receive 100% of the level 2+3 reward + final int level1SharePercent = 5_00; // 5% + final int level2And3SharePercent = 10_00; // 10% + final long level1ShareAmountBeforeFix = (blockReward * level1SharePercent) / 100L / 100L; + final long level2And3ShareAmountBeforeFix = (blockReward * level2And3SharePercent) / 100L / 100L; + final long expectedFounderRewardBeforeFix = blockReward - level1ShareAmountBeforeFix - level2And3ShareAmountBeforeFix; // Alice should receive the remainder + + // Validate the share amounts and balances + assertEquals(500000000, level1ShareAmountBeforeFix); + assertEquals(1000000000, level2And3ShareAmountBeforeFix); + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance-expectedFounderRewardBeforeFix); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance-level1ShareAmountBeforeFix); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance-level2And3ShareAmountBeforeFix); + + // Orphan the latest block one last time + BlockUtils.orphanBlocks(repository, 1); + assertEquals(3, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Validate balances + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance-(expectedFounderRewardBeforeFix*2)); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance-(level1ShareAmountBeforeFix*2)); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance-(level2And3ShareAmountBeforeFix*2)); + + } + } + + + private int getFlags(Repository repository, String name) throws DataException { + TestAccount testAccount = Common.getTestAccount(repository, name); + return repository.getAccountRepository().getAccount(testAccount.getAddress()).getFlags(); + } + } \ No newline at end of file diff --git a/src/test/resources/test-chain-v2-reward-levels.json b/src/test/resources/test-chain-v2-reward-levels.json new file mode 100644 index 00000000..2f0dbd4c --- /dev/null +++ b/src/test/resources/test-chain-v2-reward-levels.json @@ -0,0 +1,74 @@ +{ + "isTestChain": true, + "blockTimestampMargin": 500, + "transactionExpiryPeriod": 86400000, + "maxBlockSize": 2097152, + "maxBytesPerUnitFee": 1024, + "unitFee": "0.1", + "requireGroupForApproval": false, + "minAccountLevelToRewardShare": 5, + "maxRewardSharesPerMintingAccount": 20, + "founderEffectiveMintingLevel": 10, + "onlineAccountSignaturesMinLifetime": 3600000, + "onlineAccountSignaturesMaxLifetime": 86400000, + "rewardsByHeight": [ + { "height": 1, "reward": 100 }, + { "height": 11, "reward": 10 }, + { "height": 21, "reward": 1 } + ], + "sharesByLevel": [ + { "levels": [ 1, 2 ], "share": 0.05 }, + { "levels": [ 3, 4 ], "share": 0.10 }, + { "levels": [ 5, 6 ], "share": 0.15 }, + { "levels": [ 7, 8 ], "share": 0.20 }, + { "levels": [ 9, 10 ], "share": 0.25 } + ], + "qoraHoldersShare": 0.20, + "qoraPerQortReward": 250, + "blocksNeededByLevel": [ 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 ], + "blockTimingsByHeight": [ + { "height": 1, "target": 60000, "deviation": 30000, "power": 0.2 } + ], + "ciyamAtSettings": { + "feePerStep": "0.0001", + "maxStepsPerRound": 500, + "stepsPerFunctionCall": 10, + "minutesPerBlock": 1 + }, + "featureTriggers": { + "messageHeight": 0, + "atHeight": 0, + "assetsTimestamp": 0, + "votingTimestamp": 0, + "arbitraryTimestamp": 0, + "powfixTimestamp": 0, + "qortalTimestamp": 0, + "newAssetPricingTimestamp": 0, + "groupApprovalTimestamp": 0, + "atFindNextTransactionFix": 0, + "newBlockSigHeight": 999999, + "shareBinFix": 6 + }, + "genesisInfo": { + "version": 4, + "timestamp": 0, + "transactions": [ + { "type": "ISSUE_ASSET", "assetName": "QORT", "description": "QORT native coin", "data": "", "quantity": 0, "isDivisible": true, "fee": 0 }, + { "type": "ISSUE_ASSET", "assetName": "Legacy-QORA", "description": "Representative legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, + { "type": "ISSUE_ASSET", "assetName": "QORT-from-QORA", "description": "QORT gained from holding legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true }, + + { "type": "GENESIS", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "amount": "1000000000" }, + { "type": "GENESIS", "recipient": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "amount": "1000000" }, + { "type": "GENESIS", "recipient": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "amount": "1000000" }, + { "type": "GENESIS", "recipient": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "amount": "1000000" }, + + { "type": "ACCOUNT_FLAGS", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "andMask": -1, "orMask": 1, "xorMask": 0 }, + { "type": "REWARD_SHARE", "minterPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "rewardSharePublicKey": "7PpfnvLSG7y4HPh8hE7KoqAjLCkv7Ui6xw4mKAkbZtox", "sharePercent": 100 }, + + { "type": "ACCOUNT_LEVEL", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "level": 1 }, + { "type": "ACCOUNT_LEVEL", "target": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "level": 2 } + ] + } +} diff --git a/src/test/resources/test-settings-v2-reward-levels.json b/src/test/resources/test-settings-v2-reward-levels.json new file mode 100644 index 00000000..1c6862ad --- /dev/null +++ b/src/test/resources/test-settings-v2-reward-levels.json @@ -0,0 +1,7 @@ +{ + "restrictedApi": false, + "blockchainConfig": "src/test/resources/test-chain-v2-reward-levels.json", + "wipeUnconfirmedOnStart": false, + "testNtpOffset": 0, + "minPeers": 0 +} From 16453ed6022c51fef41b8e96a6c232d75c4e0b86 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 17 Mar 2021 08:50:53 +0000 Subject: [PATCH 07/57] Added unit tests for level 3+4, 5+6, 7+8, and 9+10 rewards. These are simpler than the level 1+2 tests; they only test that the rewards are correct for each level post-shareBinFix. I don't think we need multiple instances of the pre-shareBinFix or block orphaning tests. There are a few subtle differences between each test, such as the online status of Bob, in order the make the tests slightly more comprehensive. --- .../org/qortal/test/minting/RewardTests.java | 328 +++++++++++++++++- 1 file changed, 327 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/qortal/test/minting/RewardTests.java b/src/test/java/org/qortal/test/minting/RewardTests.java index 813c55cb..6c03662c 100644 --- a/src/test/java/org/qortal/test/minting/RewardTests.java +++ b/src/test/java/org/qortal/test/minting/RewardTests.java @@ -377,7 +377,7 @@ public class RewardTests extends Common { assertEquals(0, getFlags(repository, "chloe")); assertEquals(0, getFlags(repository, "dilbert")); - // Now that everyone is at level 1, we can capture initial balances + // Now that everyone is at level 1 or 2, we can capture initial balances Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); @@ -399,6 +399,7 @@ public class RewardTests extends Common { * No legacy QORA holders. * * Chloe and Dilbert should receive equal shares of the 5% block reward for Level 1 and 2 + * Alice should receive the remainder (95%) */ // We are after the shareBinFix feature trigger, so we expect level 1 and 2 to share the same reward (5%) @@ -457,6 +458,331 @@ public class RewardTests extends Common { } } + /** Test rewards for level 3 and 4 accounts */ + @Test + public void testLevel3And4Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share online + byte[] bobRewardSharePrivateKey = AccountUtils.rewardShare(repository, "bob", "bob", 0); + PrivateKeyAccount bobRewardShareAccount = new PrivateKeyAccount(repository, bobRewardSharePrivateKey); + mintingAndOnlineAccounts.add(bobRewardShareAccount); + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 3 and 4 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(4) - 20; // 20 blocks before level 4, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(3, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(3, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(3, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(4, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 3 or 4, we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Bob, Chloe, and Dilbert are 'online'. + * Bob and Chloe are level 3; Dilbert is level 4. + * One founder online (Alice, who is also level 3). + * No legacy QORA holders. + * + * Chloe, Bob and Dilbert should receive equal shares of the 10% block reward for level 3 and 4 + * Alice should receive the remainder (90%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 3 and 4 to share the same reward (10%) + final int level3And4SharePercent = 10_00; // 10% + final long level3And4ShareAmount = (blockReward * level3And4SharePercent) / 100L / 100L; + final long expectedReward = level3And4ShareAmount / 3; // The reward is split between Bob, Chloe, and Dilbert + final long expectedFounderReward = blockReward - level3And4ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance+expectedReward); + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedReward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedReward); + + } + } + + /** Test rewards for level 5 and 6 accounts */ + @Test + public void testLevel5And6Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share not initially online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 5 and 6 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(6) - 20; // 20 blocks before level 6, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Bob self-share now comes online + byte[] bobRewardSharePrivateKey = AccountUtils.rewardShare(repository, "bob", "bob", 0); + PrivateKeyAccount bobRewardShareAccount = new PrivateKeyAccount(repository, bobRewardSharePrivateKey); + mintingAndOnlineAccounts.add(bobRewardShareAccount); + + // Ensure that the levels are as we expect + assertEquals(5, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(5, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(6, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 5 or 6 (except Bob who has only just started minting, so is at level 1), we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Bob, Chloe, and Dilbert are 'online'. + * Bob is level 1; Chloe is level 5; Dilbert is level 6. + * One founder online (Alice, who is also level 5). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 15% block reward for level 5 and 6 + * Bob should receive all of the level 1 and 2 reward (5%) + * Alice should receive the remainder (80%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 5 and 6 to share the same reward (15%) + final int level1And2SharePercent = 5_00; // 5% + final int level5And6SharePercent = 15_00; // 10% + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long level5And6ShareAmount = (blockReward * level5And6SharePercent) / 100L / 100L; + final long expectedLevel1And2Reward = level1And2ShareAmount; // The reward is given entirely to Bob + final long expectedLevel5And6Reward = level5And6ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level1And2ShareAmount - level5And6ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance+expectedLevel1And2Reward); + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedLevel5And6Reward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel5And6Reward); + + } + } + + /** Test rewards for level 7 and 8 accounts */ + @Test + public void testLevel7And8Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share NOT online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 7 and 8 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(8) - 20; // 20 blocks before level 8, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure that the levels are as we expect + assertEquals(7, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(7, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(8, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 7 or 8 (except Bob who has only just started minting, so is at level 1), we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Chloe, and Dilbert are 'online'. + * Chloe is level 7; Dilbert is level 8. + * One founder online (Alice, who is also level 7). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 20% block reward for level 7 and 8 + * Alice should receive the remainder (80%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 7 and 8 to share the same reward (20%) + final int level7And8SharePercent = 20_00; // 20% + final long level7And8ShareAmount = (blockReward * level7And8SharePercent) / 100L / 100L; + final long expectedLevel7And8Reward = level7And8ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level7And8ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance); // Bob not online so his balance remains the same + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedLevel7And8Reward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel7And8Reward); + + } + } + + /** Test rewards for level 9 and 10 accounts */ + @Test + public void testLevel9And10Rewards() throws DataException { + Common.useSettings("test-settings-v2-reward-levels.json"); + + try (final Repository repository = RepositoryManager.getRepository()) { + + List cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel(); + List mintingAndOnlineAccounts = new ArrayList<>(); + + // Alice self share online + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Bob self-share not initially online + + // Chloe self share online + byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0); + PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey); + mintingAndOnlineAccounts.add(chloeRewardShareAccount); + + // Dilbert self share online + byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0); + PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey); + mintingAndOnlineAccounts.add(dilbertRewardShareAccount); + + // Mint enough blocks to bump testAccount levels to 9 and 10 + final int minterBlocksNeeded = cumulativeBlocksByLevel.get(10) - 20; // 20 blocks before level 10, so that the test accounts reach the correct levels + for (int bc = 0; bc < minterBlocksNeeded; ++bc) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Bob self-share now comes online + byte[] bobRewardSharePrivateKey = AccountUtils.rewardShare(repository, "bob", "bob", 0); + PrivateKeyAccount bobRewardShareAccount = new PrivateKeyAccount(repository, bobRewardSharePrivateKey); + mintingAndOnlineAccounts.add(bobRewardShareAccount); + + // Ensure that the levels are as we expect + assertEquals(9, (int) Common.getTestAccount(repository, "alice").getLevel()); + assertEquals(1, (int) Common.getTestAccount(repository, "bob").getLevel()); + assertEquals(9, (int) Common.getTestAccount(repository, "chloe").getLevel()); + assertEquals(10, (int) Common.getTestAccount(repository, "dilbert").getLevel()); + + // Now that everyone is at level 7 or 8 (except Bob who has only just started minting, so is at level 1), we can capture initial balances + Map> initialBalances = AccountUtils.getBalances(repository, Asset.QORT, Asset.LEGACY_QORA, Asset.QORT_FROM_QORA); + final long aliceInitialBalance = initialBalances.get("alice").get(Asset.QORT); + final long bobInitialBalance = initialBalances.get("bob").get(Asset.QORT); + final long chloeInitialBalance = initialBalances.get("chloe").get(Asset.QORT); + final long dilbertInitialBalance = initialBalances.get("dilbert").get(Asset.QORT); + + // Mint a block + final long blockReward = BlockUtils.getNextBlockReward(repository); + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Ensure we are using the correct block reward value + assertEquals(100000000L, blockReward); + + /* + * Alice, Bob, Chloe, and Dilbert are 'online'. + * Bob is level 1; Chloe is level 9; Dilbert is level 10. + * One founder online (Alice, who is also level 9). + * No legacy QORA holders. + * + * Chloe and Dilbert should receive equal shares of the 25% block reward for level 9 and 10 + * Bob should receive all of the level 1 and 2 reward (5%) + * Alice should receive the remainder (70%) + */ + + // We are after the shareBinFix feature trigger, so we expect level 9 and 10 to share the same reward (25%) + final int level1And2SharePercent = 5_00; // 5% + final int level9And10SharePercent = 25_00; // 25% + final long level1And2ShareAmount = (blockReward * level1And2SharePercent) / 100L / 100L; + final long level9And10ShareAmount = (blockReward * level9And10SharePercent) / 100L / 100L; + final long expectedLevel1And2Reward = level1And2ShareAmount; // The reward is given entirely to Bob + final long expectedLevel9And10Reward = level9And10ShareAmount / 2; // The reward is split between Chloe and Dilbert + final long expectedFounderReward = blockReward - level1And2ShareAmount - level9And10ShareAmount; // Alice should receive the remainder + + // Validate the balances to ensure that the correct post-shareBinFix distribution is being applied + AccountUtils.assertBalance(repository, "alice", Asset.QORT, aliceInitialBalance+expectedFounderReward); + AccountUtils.assertBalance(repository, "bob", Asset.QORT, bobInitialBalance+expectedLevel1And2Reward); + AccountUtils.assertBalance(repository, "chloe", Asset.QORT, chloeInitialBalance+expectedLevel9And10Reward); + AccountUtils.assertBalance(repository, "dilbert", Asset.QORT, dilbertInitialBalance+expectedLevel9And10Reward); + + } + } + private int getFlags(Repository repository, String name) throws DataException { TestAccount testAccount = Common.getTestAccount(repository, name); From e89d31eb5a7c7c8b97c8fd2ee37228712490bc54 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 30 Mar 2021 12:29:27 +0100 Subject: [PATCH 08/57] Rewrite of Synchronizer.syncToPeerChain(), this time borrowing ideas from Synchronizer.applyNewBlocks(). Main differences / improvements: - Only request a single batch of signatures upfront, instead of the entire peer's chain. There is no point in requesting them all, as the later ones may not be valid by the time we have finished requesting all the blocks before them. - If we fail to fetch a block, clear any queued signatures that are in memory and re-fetch signatures after the last block received. This allows us to cope with peers that re-org whilst we are syncing with them. - If we can't find any more block signatures, or the peer fails to respond to a block, apply our progress anyway. This should reduce wasted work and network congestion, and helps cope with larger peer re-orgs. - The retry mechanism remains in place, but instead of fetching the same incorrect block over and over, it will attempt to locate a new block signature each time, as described above. To help reduce code complexity, block signature requests are no longer retried. --- .../org/qortal/controller/Synchronizer.java | 130 ++++++++++-------- 1 file changed, 73 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 0804c2df..90b1ab28 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -354,45 +354,89 @@ public class Synchronizer { final byte[] commonBlockSig = commonBlockData.getSignature(); String commonBlockSig58 = Base58.encode(commonBlockSig); + byte[] latestPeerSignature = commonBlockSig; + int height = commonBlockHeight; + LOGGER.debug(() -> String.format("Fetching peer %s chain from height %d, sig %.8s", peer, commonBlockHeight, commonBlockSig58)); - int ourHeight = ourInitialHeight; // Overall plan: fetch peer's blocks first, then orphan, then apply // Convert any leftover (post-common) block summaries into signatures to request from peer List peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList()); + // Keep a list of blocks received so far + List peerBlocks = new ArrayList<>(); + // Calculate the total number of additional blocks this peer has beyond the common block int additionalPeerBlocksAfterCommonBlock = peerHeight - commonBlockHeight; // Subtract the number of signatures that we already have, as we don't need to request them again int numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size(); - // Fetch remaining block signatures, if needed int retryCount = 0; - while (numberSignaturesRequired > 0) { - byte[] latestPeerSignature = peerBlockSignatures.isEmpty() ? commonBlockSig : peerBlockSignatures.get(peerBlockSignatures.size() - 1); - int lastPeerHeight = commonBlockHeight + peerBlockSignatures.size(); - int numberOfSignaturesToRequest = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE); + while (height < peerHeight) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; - LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s", - numberOfSignaturesToRequest, (numberOfSignaturesToRequest != 1 ? "s": ""), lastPeerHeight, Base58.encode(latestPeerSignature))); + // Ensure we don't request more than MAXIMUM_REQUEST_SIZE + int numberRequested = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE); - List moreBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberOfSignaturesToRequest); + // Do we need more signatures? + if (peerBlockSignatures.isEmpty() && numberRequested > 0) { + LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s", + numberRequested, (numberRequested != 1 ? "s" : ""), height, Base58.encode(latestPeerSignature))); - if (moreBlockSignatures == null || moreBlockSignatures.isEmpty()) { - LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, - lastPeerHeight, Base58.encode(latestPeerSignature))); + peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested); + + if (peerBlockSignatures == null || peerBlockSignatures.isEmpty()) { + LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, + height, Base58.encode(latestPeerSignature))); + + // If we have already received blocks from this peer, go ahead and apply them + if (peerBlocks.size() > 0) { + break; + } + // Otherwise, give up and move on to the next peer + return SynchronizationResult.NO_REPLY; + } + + numberSignaturesRequired = peerHeight - height - peerBlockSignatures.size(); + LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : ""))); + } + + if (peerBlockSignatures.isEmpty()) { + LOGGER.trace(String.format("No more signatures or blocks to request from peer %s", peer)); + break; + } + + byte[] nextPeerSignature = peerBlockSignatures.get(0); + int nextHeight = height + 1; + + LOGGER.trace(String.format("Fetching block %d, sig %.8s from %s", nextHeight, Base58.encode(nextPeerSignature), peer)); + Block newBlock = this.fetchBlock(repository, peer, nextPeerSignature); + + if (newBlock == null) { + LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, + nextHeight, Base58.encode(nextPeerSignature))); if (retryCount >= MAXIMUM_RETRIES) { - // Give up with this peer + + // If we have already received blocks from this peer, go ahead and apply them + if (peerBlocks.size() > 0) { + break; + } + // Otherwise, give up and move on to the next peer return SynchronizationResult.NO_REPLY; - } - else { + + } else { + // Re-fetch signatures, in case the peer is now on a different fork + peerBlockSignatures.clear(); + numberSignaturesRequired = peerHeight - height; + // Retry until retryCount reaches MAXIMUM_RETRIES retryCount++; int triesRemaining = MAXIMUM_RETRIES - retryCount; - LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s": ""))); + LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s" : ""))); continue; } } @@ -400,62 +444,31 @@ public class Synchronizer { // Reset retryCount because the last request succeeded retryCount = 0; - LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : ""))); - - peerBlockSignatures.addAll(moreBlockSignatures); - numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size(); - } - - // Fetch blocks using signatures - LOGGER.debug(String.format("Fetching new blocks from peer %s after height %d", peer, commonBlockHeight)); - List peerBlocks = new ArrayList<>(); - - retryCount = 0; - while (peerBlocks.size() < peerBlockSignatures.size()) { - byte[] blockSignature = peerBlockSignatures.get(peerBlocks.size()); - - LOGGER.debug(String.format("Fetching block with signature %.8s", Base58.encode(blockSignature))); - int blockHeightToRequest = commonBlockHeight + peerBlocks.size() + 1; // +1 because we are requesting the next block, beyond what we already have in the peerBlocks array - Block newBlock = this.fetchBlock(repository, peer, blockSignature); - - if (newBlock == null) { - LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, blockHeightToRequest, Base58.encode(blockSignature))); - - if (retryCount >= MAXIMUM_RETRIES) { - // Give up with this peer - return SynchronizationResult.NO_REPLY; - } - else { - // Retry until retryCount reaches MAXIMUM_RETRIES - retryCount++; - int triesRemaining = MAXIMUM_RETRIES - retryCount; - LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s": ""))); - continue; - } - } + LOGGER.trace(String.format("Fetched block %d, sig %.8s from %s", nextHeight, Base58.encode(latestPeerSignature), peer)); if (!newBlock.isSignatureValid()) { LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer, - blockHeightToRequest, Base58.encode(blockSignature))); + nextHeight, Base58.encode(latestPeerSignature))); return SynchronizationResult.INVALID_DATA; } - // Reset retryCount because the last request succeeded - retryCount = 0; - - LOGGER.debug(String.format("Received block with height %d, sig: %.8s", newBlock.getBlockData().getHeight(), Base58.encode(blockSignature))); - // Transactions are transmitted without approval status so determine that now for (Transaction transaction : newBlock.getTransactions()) transaction.setInitialApprovalStatus(); peerBlocks.add(newBlock); + + // Now that we've received this block, we can increase our height and move on to the next one + latestPeerSignature = nextPeerSignature; + peerBlockSignatures.remove(0); + ++height; } // Unwind to common block (unless common block is our latest block) - LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s", commonBlockHeight, commonBlockSig58)); + int ourHeight = ourInitialHeight; + LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s. Our height: %d", commonBlockHeight, commonBlockSig58, ourHeight)); - BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourHeight); + BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourInitialHeight); while (ourHeight > commonBlockHeight) { if (Controller.isStopping()) return SynchronizationResult.SHUTTING_DOWN; @@ -477,6 +490,9 @@ public class Synchronizer { LOGGER.debug(String.format("Orphaned blocks back to height %d, sig %.8s - applying new blocks from peer %s", commonBlockHeight, commonBlockSig58, peer)); for (Block newBlock : peerBlocks) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + ValidationResult blockResult = newBlock.isValid(); if (blockResult != ValidationResult.OK) { LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer, From c3e5298ecda67b62a15589ce3a2e5bbc7ada2ebe Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 30 Mar 2021 13:05:43 +0100 Subject: [PATCH 09/57] Added a few checks for Controller.isStopping() in synchronizer loops, to try and speed up the shutdown time. --- .../java/org/qortal/controller/Synchronizer.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 90b1ab28..7f3b9861 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -257,9 +257,13 @@ public class Synchronizer { // Currently we work forward from common block until we hit a block we don't have // TODO: rewrite as modified binary search! int i; - for (i = 1; i < blockSummariesFromCommon.size(); ++i) + for (i = 1; i < blockSummariesFromCommon.size(); ++i) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + if (!repository.getBlockRepository().exists(blockSummariesFromCommon.get(i).getSignature())) break; + } // Note: index i - 1 isn't cleared: List.subList is fromIndex inclusive to toIndex exclusive blockSummariesFromCommon.subList(0, i - 1).clear(); @@ -308,6 +312,9 @@ public class Synchronizer { // Check peer sent valid heights for (int i = 0; i < moreBlockSummaries.size(); ++i) { + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + ++lastSummaryHeight; BlockSummaryData blockSummary = moreBlockSummaries.get(i); @@ -645,6 +652,9 @@ public class Synchronizer { final int firstBlockHeight = blockSummaries.get(0).getHeight(); for (int i = 0; i < blockSummaries.size(); ++i) { + if (Controller.isStopping()) + return; + BlockSummaryData blockSummary = blockSummaries.get(i); // Qortal: minter is always a reward-share, so find actual minter and get their effective minting level From 8d613a647200211c64d73bc1155db66f8af144c9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 30 Mar 2021 13:07:34 +0100 Subject: [PATCH 10/57] MAXIMUM_RETRIES reduced from 3 to 1 --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 7f3b9861..c35f37a3 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -54,7 +54,7 @@ public class Synchronizer { private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? /** Number of retry attempts if a peer fails to respond with the requested data */ - private static final int MAXIMUM_RETRIES = 3; // XXX move to Settings? + private static final int MAXIMUM_RETRIES = 1; // XXX move to Settings? private static Synchronizer instance; From 98308ecf986190f17c293a72daaae15fe749254c Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 1 Apr 2021 08:09:50 +0100 Subject: [PATCH 11/57] Bump version to 1.4.6 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3eda336f..e7dcc009 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 1.4.5 + 1.4.6 jar true From 44ec4470147875efe04cca9c7c743db4d3d9cfa3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 1 Apr 2021 08:27:56 +0100 Subject: [PATCH 12/57] Show an error in publish-auto-update.pl if both sha256sum and sha256 aren't found in PATH. --- tools/publish-auto-update.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/publish-auto-update.pl b/tools/publish-auto-update.pl index 0c7a47d6..ad43b2f4 100755 --- a/tools/publish-auto-update.pl +++ b/tools/publish-auto-update.pl @@ -58,6 +58,7 @@ $timestamp *= 1000; # Convert to milliseconds # locate sha256 utility my $SHA256 = `which sha256sum || which sha256`; chomp $SHA256; +die("Can't find sha256sum or sha256\n") unless length($SHA256) > 0; # SHA256 of actual update file my $sha256 = `git show auto-update-${commit_hash}:${project}.update | ${SHA256} | head -c 64`; From 41505dae1121305eb965ca0e5ea3f8f4015537f5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 16 Apr 2021 09:40:22 +0100 Subject: [PATCH 13/57] Treat two block summaries as equal if they have matching signatures --- .../qortal/data/block/BlockSummaryData.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/main/java/org/qortal/data/block/BlockSummaryData.java b/src/main/java/org/qortal/data/block/BlockSummaryData.java index a0c39f75..2167f0f0 100644 --- a/src/main/java/org/qortal/data/block/BlockSummaryData.java +++ b/src/main/java/org/qortal/data/block/BlockSummaryData.java @@ -2,6 +2,7 @@ package org.qortal.data.block; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import java.util.Arrays; @XmlAccessorType(XmlAccessType.FIELD) public class BlockSummaryData { @@ -84,4 +85,21 @@ public class BlockSummaryData { this.minterLevel = minterLevel; } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + BlockSummaryData otherBlockSummary = (BlockSummaryData) o; + if (this.getSignature() == null || otherBlockSummary.getSignature() == null) + return false; + + // Treat two block summaries as equal if they have matching signatures + return Arrays.equals(this.getSignature(), otherBlockSummary.getSignature()); + } + } From 2efc9218dfa0921f8c4601dabb3dcf79a88af13d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Apr 2021 12:52:19 +0100 Subject: [PATCH 14/57] Improved the process of selecting the next peer to sync with Added a new step, which attempts to filter out peers that are on inferior chains, by comparing them against each other and our chain. The basic logic is as follows: 1. Take the list of peers that we'd previously have chosen from randomly. 2. Figure out our common block with each of those peers (if its within 240 blocks), using cached data if possible. 3. Remove peers with no common block. 4. Find the earliest common block, and compare all peers with that common block against each other (and against our chain) using the chain weight method. This involves fetching (up to 200) summaries from each peer after the common block, and (up to 200) summaries from our own chain after the common block. 5. If our chain was superior, remove all peers with this common block, then move up to the next common block (in ascending order), and repeat from step 4. 6. If our chain was inferior, remove any peers with lower weights, then remove all peers with higher common blocks. 7. We end up with a reduced list of peers, that should in theory be on superior or equal chains to us. Pick one of those at random and sync to it. This is a high risk feature - we don't yet know the impact on network load. Nor do we know whether it will cause issues due to prioritising longer chains, since the chain weight algorithm currently prefers them. --- .../org/qortal/controller/Controller.java | 22 ++ .../org/qortal/controller/Synchronizer.java | 358 ++++++++++++++++++ .../qortal/data/block/CommonBlockData.java | 56 +++ src/main/java/org/qortal/network/Peer.java | 21 +- 4 files changed, 453 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/qortal/data/block/CommonBlockData.java diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index a0ca1d05..f57915b8 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -639,6 +639,21 @@ public class Controller extends Thread { // Disregard peers that are on the same block as last sync attempt and we didn't like their chain peers.removeIf(hasInferiorChainTip); + final int peersBeforeComparison = peers.size(); + + // Request recent block summaries from the remaining peers, and locate our common block with each + Synchronizer.getInstance().findCommonBlocksWithPeers(peers); + + // Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks + peers = Synchronizer.getInstance().comparePeers(peers); + + // We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains + peers.removeIf(hasInferiorChainTip); + + final int peersRemoved = peersBeforeComparison - peers.size(); + if (peersRemoved > 0) + LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size())); + if (peers.isEmpty()) return; @@ -744,6 +759,13 @@ public class Controller extends Thread { } } + public void addInferiorChainSignature(byte[] inferiorSignature) { + // Update our list of inferior chain tips + ByteArray inferiorChainSignature = new ByteArray(inferiorSignature); + if (!inferiorChainSignatures.contains(inferiorChainSignature)) + inferiorChainSignatures.add(inferiorChainSignature); + } + public static class StatusChangeEvent implements Event { public StatusChangeEvent() { } diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index c35f37a3..54d05a06 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -8,6 +8,7 @@ import java.util.Arrays; import java.util.List; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; +import java.util.Iterator; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -17,6 +18,7 @@ import org.qortal.block.Block; import org.qortal.block.Block.ValidationResult; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; +import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.transaction.RewardShareTransactionData; import org.qortal.data.transaction.TransactionData; @@ -75,6 +77,362 @@ public class Synchronizer { return instance; } + + /** + * Iterate through a list of supplied peers, and attempt to find our common block with each. + * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later. + *

+ * Will return SynchronizationResult.OK on success. + *

+ * @param peers + * @return SynchronizationResult.OK if the process completed successfully, or a different SynchronizationResult if something went wrong. + * @throws InterruptedException + */ + public SynchronizationResult findCommonBlocksWithPeers(List peers) throws InterruptedException { + try (final Repository repository = RepositoryManager.getRepository()) { + try { + + if (peers.size() == 0) + return SynchronizationResult.NOTHING_TO_DO; + + // If our latest block is very old, it's best that we don't try and determine the best peers to sync to. + // This is because it can involve very large chain comparisons, which is too intensive. + // In reality, most forking problems occur near the chain tips, so we will reserve this functionality for those situations. + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null) + return SynchronizationResult.REPOSITORY_ISSUE; + + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + LOGGER.debug(String.format("Our latest block is very old, so we won't collect common block info from peers")); + return SynchronizationResult.NOTHING_TO_DO; + } + + LOGGER.debug(String.format("Searching for common blocks with %d peers...", peers.size())); + final long startTime = System.currentTimeMillis(); + + for (Peer peer : peers) { + // Are we shutting down? + if (Controller.isStopping()) + return SynchronizationResult.SHUTTING_DOWN; + + // Check if we can use the cached common block data, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block + PeerChainTipData peerChainTipData = peer.getChainTipData(); + CommonBlockData commonBlockData = peer.getCommonBlockData(); + + if (peerChainTipData != null && commonBlockData != null) { + PeerChainTipData commonBlockChainTipData = commonBlockData.getChainTipData(); + if (peerChainTipData.getLastBlockSignature() != null && commonBlockChainTipData != null && commonBlockChainTipData.getLastBlockSignature() != null) { + if (Arrays.equals(peerChainTipData.getLastBlockSignature(), commonBlockChainTipData.getLastBlockSignature())) { + LOGGER.debug(String.format("Skipping peer %s because we already have the latest common block data in our cache. Cached common block sig is %.08s", peer, Base58.encode(commonBlockData.getCommonBlockSummary().getSignature()))); + continue; + } + } + } + + // Cached data is stale, so clear it and repopulate + peer.setCommonBlockData(null); + + // Search for the common block + Synchronizer.getInstance().findCommonBlockWithPeer(peer, repository); + } + + final long totalTimeTaken = System.currentTimeMillis() - startTime; + LOGGER.info(String.format("Finished searching for common blocks with %d peer%s. Total time taken: %d ms", peers.size(), (peers.size() != 1 ? "s" : ""), totalTimeTaken)); + + return SynchronizationResult.OK; + } finally { + repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong + } + } catch (DataException e) { + LOGGER.error("Repository issue during synchronization with peer", e); + return SynchronizationResult.REPOSITORY_ISSUE; + } + } + + /** + * Attempt to find the find our common block with supplied peer. + * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later. + *

+ * Will return SynchronizationResult.OK on success. + *

+ * @param peer + * @param repository + * @return SynchronizationResult.OK if the process completed successfully, or a different SynchronizationResult if something went wrong. + * @throws InterruptedException + */ + public SynchronizationResult findCommonBlockWithPeer(Peer peer, Repository repository) throws InterruptedException { + try { + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + final int ourInitialHeight = ourLatestBlockData.getHeight(); + + PeerChainTipData peerChainTipData = peer.getChainTipData(); + int peerHeight = peerChainTipData.getLastHeight(); + byte[] peersLastBlockSignature = peerChainTipData.getLastBlockSignature(); + + byte[] ourLastBlockSignature = ourLatestBlockData.getSignature(); + LOGGER.debug(String.format("Fetching summaries from peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer, + peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(), + ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp())); + + List peerBlockSummaries = new ArrayList<>(); + SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, false, peerBlockSummaries); + if (findCommonBlockResult != SynchronizationResult.OK) { + // Logging performed by fetchSummariesFromCommonBlock() above + peer.setCommonBlockData(null); + return findCommonBlockResult; + } + + // First summary is common block + final BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature()); + final BlockSummaryData commonBlockSummary = new BlockSummaryData(commonBlockData); + final int commonBlockHeight = commonBlockData.getHeight(); + final byte[] commonBlockSig = commonBlockData.getSignature(); + final String commonBlockSig58 = Base58.encode(commonBlockSig); + LOGGER.debug(String.format("Common block with peer %s is at height %d, sig %.8s, ts %d", peer, + commonBlockHeight, commonBlockSig58, commonBlockData.getTimestamp())); + peerBlockSummaries.remove(0); + + // Store the common block summary against the peer, and the current chain tip (for caching) + peer.setCommonBlockData(new CommonBlockData(commonBlockSummary, peerChainTipData)); + + return SynchronizationResult.OK; + } catch (DataException e) { + LOGGER.error("Repository issue during synchronization with peer", e); + return SynchronizationResult.REPOSITORY_ISSUE; + } + } + + + /** + * Compare a list of peers to determine the best peer(s) to sync to next. + *

+ * Will return a filtered list of peers on success, or an identical list of peers on failure. + * This allows us to fall back to legacy behaviour (random selection from the entire list of peers), if we are unable to make the comparison. + *

+ * @param peers + * @return a list of peers, possibly filtered. + * @throws InterruptedException + */ + public List comparePeers(List peers) throws InterruptedException { + try (final Repository repository = RepositoryManager.getRepository()) { + try { + + // If our latest block is very old, it's best that we don't try and determine the best peers to sync to. + // This is because it can involve very large chain comparisons, which is too intensive. + // In reality, most forking problems occur near the chain tips, so we will reserve this functionality for those situations. + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null) + return peers; + + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + LOGGER.debug(String.format("Our latest block is very old, so we won't filter the peers list")); + return peers; + } + + // Retrieve a list of unique common blocks from this list of peers + List commonBlocks = this.uniqueCommonBlocks(peers); + + // Order common blocks by height, in ascending order + // This is essential for the logic below to make the correct decisions when discarding chains - do not remove + commonBlocks.sort((b1, b2) -> Integer.valueOf(b1.getHeight()).compareTo(Integer.valueOf(b2.getHeight()))); + + // Get our latest height + final int ourHeight = ourLatestBlockData.getHeight(); + + // Create a placeholder to track of common blocks that we can discard due to being inferior chains + int dropPeersAfterCommonBlockHeight = 0; + + // Remove peers with no common block data + Iterator iterator = peers.iterator(); + while (iterator.hasNext()) { + Peer peer = (Peer) iterator.next(); + if (peer.getCommonBlockData() == null) { + LOGGER.debug(String.format("Removed peer %s because it has no common block data")); + iterator.remove(); + } + } + + // Loop through each group of common blocks + for (BlockSummaryData commonBlockSummary : commonBlocks) { + List peersSharingCommonBlock = peers.stream().filter(peer -> peer.getCommonBlockData().getCommonBlockSummary().equals(commonBlockSummary)).collect(Collectors.toList()); + + // Check if we need to discard this group of peers + if (dropPeersAfterCommonBlockHeight > 0) { + if (commonBlockSummary.getHeight() > dropPeersAfterCommonBlockHeight) { + // We have already determined that the correct chain diverged from a lower height. We are safe to skip these peers. + for (Peer peer : peersSharingCommonBlock) { + LOGGER.debug(String.format("Peer %s has common block at height %d but the superior chain is at height %d. Removing it from this round.", peer, commonBlockSummary.getHeight(), dropPeersAfterCommonBlockHeight)); + Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature()); + } + continue; + } + } + + // Calculate the length of the shortest peer chain sharing this common block, including our chain + final int ourAdditionalBlocksAfterCommonBlock = ourHeight - commonBlockSummary.getHeight(); + int minChainLength = this.calculateMinChainLength(commonBlockSummary, ourAdditionalBlocksAfterCommonBlock, peersSharingCommonBlock); + + // Fetch block summaries from each peer + for (Peer peer : peersSharingCommonBlock) { + + // Count the number of blocks this peer has beyond our common block + final int peerHeight = peer.getChainTipData().getLastHeight(); + final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); + // Limit the number of blocks we are comparing. FUTURE: we could request more in batches, but there may not be a case when this is needed + final int summariesRequired = Math.min(peerAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE); + + if (summariesRequired > 0) { + LOGGER.trace(String.format("Requesting %d block summar%s from peer %s after common block %.8s. Peer height: %d", summariesRequired, (summariesRequired != 1 ? "ies" : "y"), peer, Base58.encode(commonBlockSummary.getSignature()), peerHeight)); + + List blockSummaries = this.getBlockSummaries(peer, commonBlockSummary.getSignature(), summariesRequired); + peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(blockSummaries); + + if (blockSummaries != null) { + LOGGER.trace(String.format("Peer %s returned %d block summar%s", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"))); + + // We need to adjust minChainLength if peers fail to return all expected block summaries + if (blockSummaries.size() < summariesRequired) { + // This could mean that the peer has re-orged. But we still have the same common block, so it's safe to proceed with this set of signatures instead. + LOGGER.debug(String.format("Peer %s returned %d block summar%s instead of expected %d", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"), summariesRequired)); + + // Update minChainLength if we have at least 1 block for this peer. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength + if (blockSummaries.size() > 0) + minChainLength = blockSummaries.size(); + } + } + } + else { + // There are no block summaries after this common block + peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(null); + } + } + + // Fetch our corresponding block summaries. Limit to MAXIMUM_REQUEST_SIZE, in order to make the comparison fairer, as peers have been limited too + final int ourSummariesRequired = Math.min(ourAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE); + LOGGER.trace(String.format("About to fetch our block summaries from %d to %d. Our height: %d", commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired, ourHeight)); + List ourBlockSummaries = repository.getBlockRepository().getBlockSummaries(commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired); + if (ourBlockSummaries.isEmpty()) + LOGGER.debug(String.format("We don't have any block summaries so can't compare our chain against peers with this common block. We can still compare them against each other.")); + else + populateBlockSummariesMinterLevels(repository, ourBlockSummaries); + + // Create array to hold peers for comparison + List superiorPeersForComparison = new ArrayList<>(); + + // Calculate our chain weight + BigInteger ourChainWeight = BigInteger.valueOf(0); + if (ourBlockSummaries.size() > 0) + ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, minChainLength); + + NumberFormat formatter = new DecimalFormat("0.###E0"); + LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", ourBlockSummaries.size(), formatter.format(ourChainWeight))); + + LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature()))); + iterator = peersSharingCommonBlock.iterator(); + while (iterator.hasNext()) { + Peer peer = (Peer)iterator.next(); + + final int peerHeight = peer.getChainTipData().getLastHeight(); + final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); + final CommonBlockData peerCommonBlockData = peer.getCommonBlockData(); + + if (peerCommonBlockData == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock() == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock().isEmpty()) { + // No response - remove this peer for now + LOGGER.debug(String.format("Peer %s doesn't have any block summaries - removing it from this round", peer)); + iterator.remove(); + continue; + } + + final List peerBlockSummariesAfterCommonBlock = peerCommonBlockData.getBlockSummariesAfterCommonBlock(); + populateBlockSummariesMinterLevels(repository, peerBlockSummariesAfterCommonBlock); + + // Calculate cumulative chain weight of this blockchain subset, from common block to highest mutual block held by all peers in this group. + LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", peerBlockSummariesAfterCommonBlock.size(), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock)); + BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, minChainLength); + peer.getCommonBlockData().setChainWeight(peerChainWeight); + LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, peerBlockSummariesAfterCommonBlock.size(), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight))); + + // Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group) + if (ourChainWeight.compareTo(peerChainWeight) > 0) { + // This peer is on an inferior chain - remove it + LOGGER.debug(String.format("Peer %s is on an inferior chain to us - added %.8s to inferior chain signatures list", peer, Base58.encode(peer.getChainTipData().getLastBlockSignature()))); + Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature()); + } + else { + // Our chain is inferior + LOGGER.debug(String.format("Peer %s is on a better chain to us. We will compare the other peers sharing this common block against each other, and drop all peers sharing higher common blocks.", peer)); + dropPeersAfterCommonBlockHeight = commonBlockSummary.getHeight(); + superiorPeersForComparison.add(peer); + } + } + + // Now that we have selected the best peers, compare them against each other and remove any with lower weights + if (superiorPeersForComparison.size() > 0) { + BigInteger bestChainWeight = null; + for (Peer peer : superiorPeersForComparison) { + // Increase bestChainWeight if needed + if (bestChainWeight == null || peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) >= 0) + bestChainWeight = peer.getCommonBlockData().getChainWeight(); + } + for (Peer peer : superiorPeersForComparison) { + // Check if we should discard an inferior peer + if (peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) < 0) { + LOGGER.debug(String.format("Peer %s has a lower chain weight than other peer(s) in this group - removing it from this round.", peer)); + Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature()); + } + } + } + } + + return peers; + } finally { + repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong + } + } catch (DataException e) { + LOGGER.error("Repository issue during peer comparison", e); + return peers; + } + } + + private List uniqueCommonBlocks(List peers) { + List commonBlocks = new ArrayList<>(); + + Iterator iterator = peers.iterator(); + while (iterator.hasNext()) { + Peer peer = iterator.next(); + + if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) { + LOGGER.debug(String.format("Peer %s has common block %.8s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature()))); + + BlockSummaryData commonBlockSummary = peer.getCommonBlockData().getCommonBlockSummary(); + if (!commonBlocks.contains(commonBlockSummary)) + commonBlocks.add(commonBlockSummary); + } + else { + LOGGER.debug(String.format("Peer %s has no common block data. Skipping...", peer)); + iterator.remove(); + } + } + + return commonBlocks; + } + + private int calculateMinChainLength(BlockSummaryData commonBlockSummary, int ourAdditionalBlocksAfterCommonBlock, List peersSharingCommonBlock) { + // Calculate the length of the shortest peer chain sharing this common block, including our chain + int minChainLength = ourAdditionalBlocksAfterCommonBlock; + for (Peer peer : peersSharingCommonBlock) { + final int peerHeight = peer.getChainTipData().getLastHeight(); + final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); + + if (peerAdditionalBlocksAfterCommonBlock < minChainLength) + minChainLength = peerAdditionalBlocksAfterCommonBlock; + } + return minChainLength; + } + + /** * Attempt to synchronize blockchain with peer. *

diff --git a/src/main/java/org/qortal/data/block/CommonBlockData.java b/src/main/java/org/qortal/data/block/CommonBlockData.java new file mode 100644 index 00000000..dd502df7 --- /dev/null +++ b/src/main/java/org/qortal/data/block/CommonBlockData.java @@ -0,0 +1,56 @@ +package org.qortal.data.block; + +import org.qortal.data.network.PeerChainTipData; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.math.BigInteger; +import java.util.List; + +@XmlAccessorType(XmlAccessType.FIELD) +public class CommonBlockData { + + // Properties + private BlockSummaryData commonBlockSummary = null; + private List blockSummariesAfterCommonBlock = null; + private BigInteger chainWeight = null; + private PeerChainTipData chainTipData = null; + + // Constructors + + protected CommonBlockData() { + } + + public CommonBlockData(BlockSummaryData commonBlockSummary, PeerChainTipData chainTipData) { + this.commonBlockSummary = commonBlockSummary; + this.chainTipData = chainTipData; + } + + + // Getters / setters + + public BlockSummaryData getCommonBlockSummary() { + return this.commonBlockSummary; + } + + public List getBlockSummariesAfterCommonBlock() { + return this.blockSummariesAfterCommonBlock; + } + + public void setBlockSummariesAfterCommonBlock(List blockSummariesAfterCommonBlock) { + this.blockSummariesAfterCommonBlock = blockSummariesAfterCommonBlock; + } + + public BigInteger getChainWeight() { + return this.chainWeight; + } + + public void setChainWeight(BigInteger chainWeight) { + this.chainWeight = chainWeight; + } + + public PeerChainTipData getChainTipData() { + return this.chainTipData; + } + +} diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index f619111a..3061431e 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -11,10 +11,7 @@ import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.nio.channels.SocketChannel; import java.security.SecureRandom; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; +import java.util.*; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -22,6 +19,7 @@ import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.network.message.ChallengeMessage; @@ -106,6 +104,9 @@ public class Peer { /** Latest block info as reported by peer. */ private PeerChainTipData peersChainTipData; + /** Our common block with this peer */ + private CommonBlockData commonBlockData; + // Constructors /** Construct unconnected, outbound Peer using socket address in peer data */ @@ -272,6 +273,18 @@ public class Peer { } } + public CommonBlockData getCommonBlockData() { + synchronized (this.peerInfoLock) { + return this.commonBlockData; + } + } + + public void setCommonBlockData(CommonBlockData commonBlockData) { + synchronized (this.peerInfoLock) { + this.commonBlockData = commonBlockData; + } + } + /*package*/ void queueMessage(Message message) { if (!this.pendingMessages.offer(message)) LOGGER.info(() -> String.format("No room to queue message from peer %s - discarding", this)); From 08dacab05c6af9f2ecdd743d48dbc93953c46eda Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Apr 2021 12:57:28 +0100 Subject: [PATCH 15/57] Make sure to give up if we are requesting block summaries when the core needs to shut down. --- src/main/java/org/qortal/controller/Synchronizer.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 54d05a06..94a3a644 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -277,6 +277,10 @@ public class Synchronizer { // Fetch block summaries from each peer for (Peer peer : peersSharingCommonBlock) { + // If we're shutting down, just return the latest peer list + if (Controller.isStopping()) + return peers; + // Count the number of blocks this peer has beyond our common block final int peerHeight = peer.getChainTipData().getLastHeight(); final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); From c919797553754f6d7277123323d981919c96c98f Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Apr 2021 13:09:52 +0100 Subject: [PATCH 16/57] When syncing to a peer on a different fork, ensure that all blocks are obtained before applying them. In version 1.4.6, we would still sync with a peer even if we only received a partial number of the requested blocks/summaries. This could create a new problem, because the BlockMinter would often try and make up the difference by minting a new fork of up to 5 blocks in quick succession. This could have added to network confusion. Longer term we may want to adjust the BlockMinter code to prevent this from taking place altogether, but in the short term I will revert this change from 1.4.6 until we have a better way. --- .../java/org/qortal/controller/Synchronizer.java | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 94a3a644..b5aa31d3 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -761,11 +761,7 @@ public class Synchronizer { LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, height, Base58.encode(latestPeerSignature))); - // If we have already received blocks from this peer, go ahead and apply them - if (peerBlocks.size() > 0) { - break; - } - // Otherwise, give up and move on to the next peer + // We need to fully synchronize, so give up and move on to the next peer return SynchronizationResult.NO_REPLY; } @@ -790,11 +786,7 @@ public class Synchronizer { if (retryCount >= MAXIMUM_RETRIES) { - // If we have already received blocks from this peer, go ahead and apply them - if (peerBlocks.size() > 0) { - break; - } - // Otherwise, give up and move on to the next peer + // We need to fully synchronize, so give up and move on to the next peer return SynchronizationResult.NO_REPLY; } else { From 2c0e099d1cf7cdec345cb4e76d8f5f5faa3af523 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Apr 2021 14:36:24 +0100 Subject: [PATCH 17/57] Removed wildcard import that was automatically introduced by Intellij. --- src/main/java/org/qortal/network/Peer.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index 3061431e..ef6fb88a 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -11,7 +11,10 @@ import java.nio.channels.SelectionKey; import java.nio.channels.Selector; import java.nio.channels.SocketChannel; import java.security.SecureRandom; -import java.util.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; From 4312ebfcc35803a5be5e8b2fea772e0fdbba458b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Apr 2021 20:44:57 +0100 Subject: [PATCH 18/57] Adapted the HSQLDBRepository.exportNodeLocalData() method It now has a new parameter - keepArchivedCopy - which when set to true will cause it to rename an existing TradeBotStates.script to TradeBotStates-archive-.script before creating a new backup. This should avoid keys being lost if a new backup is taken after replacing the db. In a future version we can improve this in such a way that it combines existing and new backups into a single file. This is just a "quick fix" to increase the chances of keys being recoverable after accidentally bootstrapping without a backup. --- .../qortal/api/resource/AdminResource.java | 2 +- .../org/qortal/repository/Repository.java | 2 +- .../repository/hsqldb/HSQLDBRepository.java | 53 +++++++++++++++---- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index 8069a0d5..c295b90b 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -547,7 +547,7 @@ public class AdminResource { blockchainLock.lockInterruptibly(); try { - repository.exportNodeLocalData(); + repository.exportNodeLocalData(true); return "true"; } finally { blockchainLock.unlock(); diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index 656e6e1e..5438f1d9 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -49,7 +49,7 @@ public interface Repository extends AutoCloseable { public void performPeriodicMaintenance() throws DataException; - public void exportNodeLocalData() throws DataException; + public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException; public void importDataFromFile(String filename) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 7c514d73..5557c13e 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -52,6 +52,7 @@ import org.qortal.repository.TransactionRepository; import org.qortal.repository.VotingRepository; import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository; import org.qortal.settings.Settings; +import org.qortal.utils.NTP; public class HSQLDBRepository implements Repository { @@ -459,10 +460,44 @@ public class HSQLDBRepository implements Repository { } @Override - public void exportNodeLocalData() throws DataException { + public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException { + + // Create the qortal-backup folder if it doesn't exist + Path backupPath = Paths.get("qortal-backup"); + try { + Files.createDirectories(backupPath); + } catch (IOException e) { + LOGGER.info("Unable to create backup folder"); + throw new DataException("Unable to create backup folder"); + } + + // We need to rename or delete an existing TradeBotStates backup before creating a new one + File tradeBotStatesBackupFile = new File("qortal-backup/TradeBotStates.script"); + if (tradeBotStatesBackupFile.exists()) { + if (keepArchivedCopy) { + // Rename existing TradeBotStates backup, to make sure that we're not overwriting any keys + File archivedBackupFile = new File(String.format("qortal-backup/TradeBotStates-archive-%d.script", NTP.getTime())); + if (tradeBotStatesBackupFile.renameTo(archivedBackupFile)) + LOGGER.info(String.format("Moved existing TradeBotStates backup file to %s", archivedBackupFile.getPath())); + else + throw new DataException("Unable to rename existing TradeBotStates backup"); + } else { + // Delete existing copy + LOGGER.info("Deleting existing TradeBotStates backup because it is being replaced with a new one"); + tradeBotStatesBackupFile.delete(); + } + } + + // There's currently no need to take an archived copy of the MintingAccounts data - just delete the old one if it exists + File mintingAccountsBackupFile = new File("qortal-backup/MintingAccounts.script"); + if (mintingAccountsBackupFile.exists()) { + LOGGER.info("Deleting existing MintingAccounts backup because it is being replaced with a new one"); + mintingAccountsBackupFile.delete(); + } + try (Statement stmt = this.connection.createStatement()) { - stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'MintingAccounts.script'"); - stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'TradeBotStates.script'"); + stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'qortal-backup/MintingAccounts.script'"); + stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'qortal-backup/TradeBotStates.script'"); LOGGER.info("Exported sensitive/node-local data: minting keys and trade bot states"); } catch (SQLException e) { throw new DataException("Unable to export sensitive/node-local data from repository"); @@ -475,12 +510,12 @@ public class HSQLDBRepository implements Repository { LOGGER.info(() -> String.format("Importing data into repository from %s", filename)); String escapedFilename = stmt.enquoteLiteral(filename); - stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " STOP ON ERROR"); + stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " CONTINUE ON ERROR"); LOGGER.info(() -> String.format("Imported data into repository from %s", filename)); } catch (SQLException e) { LOGGER.info(() -> String.format("Failed to import data into repository from %s: %s", filename, e.getMessage())); - throw new DataException("Unable to export sensitive/node-local data from repository: " + e.getMessage()); + throw new DataException("Unable to import sensitive/node-local data to repository: " + e.getMessage()); } } @@ -681,7 +716,7 @@ public class HSQLDBRepository implements Repository { /** * Execute PreparedStatement and return changed row count. * - * @param preparedStatement + * @param sql * @param objects * @return number of changed rows * @throws SQLException @@ -693,8 +728,8 @@ public class HSQLDBRepository implements Repository { /** * Execute batched PreparedStatement * - * @param preparedStatement - * @param objects + * @param sql + * @param batchedObjects * @return number of changed rows * @throws SQLException */ @@ -818,7 +853,7 @@ public class HSQLDBRepository implements Repository { * * @param tableName * @param whereClause - * @param objects + * @param batchedObjects * @throws SQLException */ public int deleteBatch(String tableName, String whereClause, List batchedObjects) throws SQLException { From e9b4a3f6b357d1c367ecb03d5c473894931c1853 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Apr 2021 20:45:35 +0100 Subject: [PATCH 19/57] Automatically backup trade bot data when starting a new trade (from either side). --- .../tradebot/LitecoinACCTv1TradeBot.java | 6 ++++++ .../qortal/controller/tradebot/TradeBot.java | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java index 15764199..286cbf74 100644 --- a/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java @@ -211,6 +211,9 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot { TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress)); + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository); + // Return to user for signing and broadcast as we don't have their Qortal private key try { return DeployAtTransactionTransformer.toBytes(deployAtTransactionData); @@ -283,6 +286,9 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot { tradeForeignPublicKey, tradeForeignPublicKeyHash, crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash); + // Attempt to backup the trade bot data + TradeBot.backupTradeBotData(repository); + // Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount long p2shFee; try { diff --git a/src/main/java/org/qortal/controller/tradebot/TradeBot.java b/src/main/java/org/qortal/controller/tradebot/TradeBot.java index 84e32125..94c7cefb 100644 --- a/src/main/java/org/qortal/controller/tradebot/TradeBot.java +++ b/src/main/java/org/qortal/controller/tradebot/TradeBot.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.concurrent.locks.ReentrantLock; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -267,6 +268,22 @@ public class TradeBot implements Listener { return secret; } + /*package*/ static void backupTradeBotData(Repository repository) { + // Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure + try { + LOGGER.info("About to backup trade bot data..."); + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + blockchainLock.lockInterruptibly(); + try { + repository.exportNodeLocalData(true); + } finally { + blockchainLock.unlock(); + } + } catch (InterruptedException | DataException e) { + LOGGER.info(String.format("Failed to obtain blockchain lock when exporting trade bot data: %s", e.getMessage())); + } + } + /** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */ /*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData, String newState, int newStateValue, Supplier logMessageSupplier) throws DataException { From 3071ef2f36b81cb428afffa6bac2907fa38b7956 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 17 Apr 2021 20:55:30 +0100 Subject: [PATCH 20/57] Removed redundant uiLocalServers --- src/main/java/org/qortal/settings/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index fb068b8d..cf1830cf 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -52,7 +52,7 @@ public class Settings { // UI servers private int uiPort = 12388; private String[] uiLocalServers = new String[] { - "localhost", "127.0.0.1", "172.24.1.1", "qor.tal" + "localhost", "127.0.0.1" }; private String[] uiRemoteServers = new String[] { "node1.qortal.org", "node2.qortal.org", "node3.qortal.org", "node4.qortal.org", "node5.qortal.org", From 3c22a12cbbb274f924df01edf848bf589d3788a2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 15 Apr 2021 08:58:43 +0100 Subject: [PATCH 21/57] Experimental idea to prevent a single node signing more than one block in a row. This could drastically reduce the number of forks being created. Currently, if a node is having problems syncing, it will continue adding to its own fork, which adds confusion to the network. With this new idea, the node would be prevented from adding to its own chain and is instead forced to wait until it has retrieved the next block from the network. We will need to test this on the testnet very carefully. My worry is that, because all minters submit blocks, it could create a situation where the first block is submitted by everyone, and the second block is submitted by no-one, until a different candidate for the first block has been obtained from a peer. This may not be a problem at all, and could actually improve stability in a huge way, but at the same time it has the potential to introduce serious network problems if we are not careful. --- src/main/java/org/qortal/controller/BlockMinter.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index f804456f..8e6f51fa 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -165,6 +165,14 @@ public class BlockMinter extends Thread { // Do we need to build any potential new blocks? List newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList()); + // We might need to sit the next block out, if one of our minting accounts signed the previous one + final byte[] previousBlockMinter = previousBlockData.getMinterPublicKey(); + final boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter)); + if (mintedLastBlock) { + LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one")); + continue; + } + for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) { // First block does the AT heavy-lifting if (newBlocks.isEmpty()) { From 2d2bfc0a4c654692bf47002316a6e61da03c6eda Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 18 Apr 2021 13:02:38 +0100 Subject: [PATCH 22/57] Log the number of common blocks found in each search. --- src/main/java/org/qortal/controller/Synchronizer.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index b5aa31d3..1f5b1c2a 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -110,6 +110,7 @@ public class Synchronizer { LOGGER.debug(String.format("Searching for common blocks with %d peers...", peers.size())); final long startTime = System.currentTimeMillis(); + int commonBlocksFound = 0; for (Peer peer : peers) { // Are we shutting down? @@ -135,10 +136,12 @@ public class Synchronizer { // Search for the common block Synchronizer.getInstance().findCommonBlockWithPeer(peer, repository); + if (peer.getCommonBlockData() != null) + commonBlocksFound++; } final long totalTimeTaken = System.currentTimeMillis() - startTime; - LOGGER.info(String.format("Finished searching for common blocks with %d peer%s. Total time taken: %d ms", peers.size(), (peers.size() != 1 ? "s" : ""), totalTimeTaken)); + LOGGER.info(String.format("Finished searching for common blocks with %d peer%s. Found: %d. Total time taken: %d ms", peers.size(), (peers.size() != 1 ? "s" : ""), commonBlocksFound, totalTimeTaken)); return SynchronizationResult.OK; } finally { From 02ace065264cd8e0961b4af4045f83dd185068b7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 18 Apr 2021 13:03:04 +0100 Subject: [PATCH 23/57] Revert "When syncing to a peer on a different fork, ensure that all blocks are obtained before applying them." This reverts commit c919797553754f6d7277123323d981919c96c98f. --- .../java/org/qortal/controller/Synchronizer.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 1f5b1c2a..1c4d6e0d 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -764,7 +764,11 @@ public class Synchronizer { LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, height, Base58.encode(latestPeerSignature))); - // We need to fully synchronize, so give up and move on to the next peer + // If we have already received blocks from this peer, go ahead and apply them + if (peerBlocks.size() > 0) { + break; + } + // Otherwise, give up and move on to the next peer return SynchronizationResult.NO_REPLY; } @@ -789,7 +793,11 @@ public class Synchronizer { if (retryCount >= MAXIMUM_RETRIES) { - // We need to fully synchronize, so give up and move on to the next peer + // If we have already received blocks from this peer, go ahead and apply them + if (peerBlocks.size() > 0) { + break; + } + // Otherwise, give up and move on to the next peer return SynchronizationResult.NO_REPLY; } else { From dbf1ed40b36458c7481196b077c81a8ad9660ed5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 19 Apr 2021 09:33:24 +0100 Subject: [PATCH 24/57] Log the parent block's signature when minting a new block, to help us keep track of the chain it's being minted on. --- src/main/java/org/qortal/controller/BlockMinter.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 8e6f51fa..05c33217 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -290,15 +290,17 @@ public class BlockMinter extends Thread { RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey()); if (rewardShareData != null) { - LOGGER.info(String.format("Minted block %d, sig %.8s by %s on behalf of %s", + LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s", newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()), + Base58.encode(newBlock.getParent().getSignature()), rewardShareData.getMinter(), rewardShareData.getRecipient())); } else { - LOGGER.info(String.format("Minted block %d, sig %.8s by %s", + LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s", newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()), + Base58.encode(newBlock.getParent().getSignature()), newBlock.getMinter().getAddress())); } From bdddb526da01eab36cb7487fdb7f18fcb5afccb7 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 23 Apr 2021 09:21:15 +0100 Subject: [PATCH 25/57] Added recovery mode, which is designed to automatically bring back a stalled network. The existing system was unable to resume without manual intervention if it stalled for more than 7.5 minutes. After this time, no peers would have "recent' blocks, which are prerequisites for synchronization and minting. This new code monitors for such a situation, and enters "recovery mode" if there are no peers with recent blocks for at least 10 minutes. It also requires that there is at least one connected peer, to reduce false positives due to bad network connectivity. Once in recovery mode, peers with no recent blocks are added back into the pool of available peers to sync with, and restrictions on minting are lifted. This should allow for peers to collaborate to bring the chain back to a "recent" block height. Once we have a peer with a recent block, the node will exit recovery mode and sync as normal. Previously, lifting minting restrictions could have increased the risk of extra forks, however it is much less risky now that nodes no longer mint multiple blocks in a row. In all cases, minBlockchainPeers is used, so a minimum number of connected peers is required for syncing and minting in recovery mode, too. --- .../org/qortal/controller/BlockMinter.java | 11 ++-- .../org/qortal/controller/Controller.java | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 05c33217..8b6563f2 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -135,16 +135,19 @@ public class BlockMinter extends Thread { // Disregard peers that have "misbehaved" recently peers.removeIf(Controller.hasMisbehaved); - // Disregard peers that don't have a recent block - peers.removeIf(Controller.hasNoRecentBlock); + // Disregard peers that don't have a recent block, but only if we're not in recovery mode. + // In that mode, we want to allow minting on top of older blocks, to recover stalled networks. + if (Controller.getInstance().getRecoveryMode() == false) + peers.removeIf(Controller.hasNoRecentBlock); // Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from? if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) continue; - // If our latest block isn't recent then we need to synchronize instead of minting. + // If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode. if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp) - continue; + if (Controller.getInstance().getRecoveryMode() == false) + continue; // There are enough peers with a recent block and our latest block is recent // so go ahead and mint a block if possible. diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index b24661f2..7d01ad32 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -121,6 +121,7 @@ public class Controller extends Thread { private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms + private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000; // To do with online accounts list private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms @@ -175,6 +176,11 @@ public class Controller extends Thread { /** Latest block signatures from other peers that we know are on inferior chains. */ List inferiorChainSignatures = new ArrayList<>(); + /** Recovery mode, which is used to bring back a stalled network */ + private boolean recoveryMode = false; + private boolean peersAvailable = true; // peersAvailable must default to true + private long timePeersLastAvailable = 0; + /** * Map of recent requests for ARBITRARY transaction data payloads. *

@@ -358,6 +364,10 @@ public class Controller extends Thread { } } + public boolean getRecoveryMode() { + return this.recoveryMode; + } + // Entry point public static void main(String[] args) { @@ -629,6 +639,13 @@ public class Controller extends Thread { // Disregard peers that don't have a recent block peers.removeIf(hasNoRecentBlock); + checkRecoveryModeForPeers(peers); + if (recoveryMode) { + peers = Network.getInstance().getHandshakedPeers(); + peers.removeIf(hasOnlyGenesisBlock); + peers.removeIf(hasMisbehaved); + } + // Check we have enough peers to potentially synchronize if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) return; @@ -759,6 +776,39 @@ public class Controller extends Thread { } } + public boolean checkRecoveryModeForPeers(List qualifiedPeers) { + List handshakedPeers = Network.getInstance().getHandshakedPeers(); + + if (handshakedPeers.size() > 0) { + // There is at least one handshaked peer + if (qualifiedPeers.isEmpty()) { + // There are no 'qualified' peers - i.e. peers that have a recent block we can sync to + boolean werePeersAvailable = peersAvailable; + peersAvailable = false; + + // If peers only just became unavailable, update our record of the time they were last available + if (werePeersAvailable) + timePeersLastAvailable = NTP.getTime(); + + // If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint + if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) { + if (recoveryMode == false) { + LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000)); + recoveryMode = true; + } + } + } else { + // We now have at least one peer with a recent block, so we can exit recovery mode and sync normally + peersAvailable = true; + if (recoveryMode) { + LOGGER.info("Peers have become available again. Exiting recovery mode..."); + recoveryMode = false; + } + } + } + return recoveryMode; + } + public void addInferiorChainSignature(byte[] inferiorSignature) { // Update our list of inferior chain tips ByteArray inferiorChainSignature = new ByteArray(inferiorSignature); From 423142d730d9d811eee71bf134d2ae08fab46294 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Apr 2021 10:35:01 +0100 Subject: [PATCH 26/57] Tidied up RECOVERY_MODE_TIMEOUT constant, and made checkRecoveryModeForPeers() private. --- src/main/java/org/qortal/controller/Controller.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 7d01ad32..0d28ccc7 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -121,7 +121,7 @@ public class Controller extends Thread { private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms - private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000; + private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms // To do with online accounts list private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms @@ -776,7 +776,7 @@ public class Controller extends Thread { } } - public boolean checkRecoveryModeForPeers(List qualifiedPeers) { + private boolean checkRecoveryModeForPeers(List qualifiedPeers) { List handshakedPeers = Network.getInstance().getHandshakedPeers(); if (handshakedPeers.size() > 0) { From ec2af62b4d649bcc0e17751108fd80d52f5b47a4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Apr 2021 15:21:30 +0100 Subject: [PATCH 27/57] Fix for bug which failed to remove peers without block summaries. The iterator was removing the peer from the "peersSharingCommonBlock" array, when it should have been removing it from the "peers" array. The result was that the bad peer would end up in the final list of good peers, and we could then sync with it when we shouldn't have. --- src/main/java/org/qortal/controller/Synchronizer.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 1c4d6e0d..babc2ee3 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -337,10 +337,7 @@ public class Synchronizer { LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", ourBlockSummaries.size(), formatter.format(ourChainWeight))); LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature()))); - iterator = peersSharingCommonBlock.iterator(); - while (iterator.hasNext()) { - Peer peer = (Peer)iterator.next(); - + for (Peer peer : peersSharingCommonBlock) { final int peerHeight = peer.getChainTipData().getLastHeight(); final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); final CommonBlockData peerCommonBlockData = peer.getCommonBlockData(); @@ -348,7 +345,7 @@ public class Synchronizer { if (peerCommonBlockData == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock() == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock().isEmpty()) { // No response - remove this peer for now LOGGER.debug(String.format("Peer %s doesn't have any block summaries - removing it from this round", peer)); - iterator.remove(); + peers.remove(peer); continue; } From f532dbe7b4808c769297f64e9a12719c0f7efa52 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Apr 2021 15:22:29 +0100 Subject: [PATCH 28/57] Optimized code in Synchronizer.uniqueCommonBlocks() --- src/main/java/org/qortal/controller/Synchronizer.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index babc2ee3..4580a919 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -403,10 +403,7 @@ public class Synchronizer { private List uniqueCommonBlocks(List peers) { List commonBlocks = new ArrayList<>(); - Iterator iterator = peers.iterator(); - while (iterator.hasNext()) { - Peer peer = iterator.next(); - + for (Peer peer : peers) { if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) { LOGGER.debug(String.format("Peer %s has common block %.8s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature()))); @@ -416,7 +413,6 @@ public class Synchronizer { } else { LOGGER.debug(String.format("Peer %s has no common block data. Skipping...", peer)); - iterator.remove(); } } From 5643e57edeefc5d23dbcff73b4028e5f8895ab4e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Apr 2021 16:21:04 +0100 Subject: [PATCH 29/57] Fixed string formatting error. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 4580a919..313b37b4 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -252,7 +252,7 @@ public class Synchronizer { while (iterator.hasNext()) { Peer peer = (Peer) iterator.next(); if (peer.getCommonBlockData() == null) { - LOGGER.debug(String.format("Removed peer %s because it has no common block data")); + LOGGER.debug(String.format("Removed peer %s because it has no common block data", peer)); iterator.remove(); } } From 3146da6aec4f6fd4d23db1f5c568a02af757f1ff Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Apr 2021 16:43:29 +0100 Subject: [PATCH 30/57] Don't add to the inferior chain signatures list when comparing peers against each other. In these comparisons it's easy to incorrectly identify a bad chain, as we aren't comparing the same number of blocks. It's quite common for one peer to fail to return all blocks and be marked as an inferior chain, yet we have other "good" peers on that exact same chain. In those cases we would have stopped talking to the good peers again until they received another block. Instead of complicating the logic and keeping track of the various good chain tip signatures, it is simpler to just remove the inferior peers from this round of syncing, and re-test them in the next round, in case they are in fact superior or equal. --- src/main/java/org/qortal/controller/Synchronizer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 313b37b4..41b0cda7 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -361,8 +361,8 @@ public class Synchronizer { // Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group) if (ourChainWeight.compareTo(peerChainWeight) > 0) { // This peer is on an inferior chain - remove it - LOGGER.debug(String.format("Peer %s is on an inferior chain to us - added %.8s to inferior chain signatures list", peer, Base58.encode(peer.getChainTipData().getLastBlockSignature()))); - Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature()); + LOGGER.debug(String.format("Peer %s is on an inferior chain to us - removing it from this round", peer)); + peers.remove(peer); } else { // Our chain is inferior @@ -384,7 +384,7 @@ public class Synchronizer { // Check if we should discard an inferior peer if (peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) < 0) { LOGGER.debug(String.format("Peer %s has a lower chain weight than other peer(s) in this group - removing it from this round.", peer)); - Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature()); + peers.remove(peer); } } } From ba6397b9638f4741d6955b9c19e4e3ce6b0f5284 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Apr 2021 19:23:09 +0100 Subject: [PATCH 31/57] Improved logging, to give a clearer picture of the peer selection decisions. --- src/main/java/org/qortal/controller/Controller.java | 9 ++++++++- src/main/java/org/qortal/controller/Synchronizer.java | 4 +++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 0d28ccc7..a7d028bc 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -669,11 +669,18 @@ public class Controller extends Thread { final int peersRemoved = peersBeforeComparison - peers.size(); if (peersRemoved > 0) - LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size())); + LOGGER.info(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size())); if (peers.isEmpty()) return; + if (peers.size() > 1) { + StringBuilder finalPeersString = new StringBuilder(); + for (Peer peer : peers) + finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer); + LOGGER.info(String.format("Choosing random peer from: [%s]", finalPeersString.toString())); + } + // Pick random peer to sync with int index = new SecureRandom().nextInt(peers.size()); Peer peer = peers.get(index); diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 41b0cda7..0cbb9e30 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -334,6 +334,7 @@ public class Synchronizer { ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, minChainLength); NumberFormat formatter = new DecimalFormat("0.###E0"); + NumberFormat accurateFormatter = new DecimalFormat("0.################E0"); LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", ourBlockSummaries.size(), formatter.format(ourChainWeight))); LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature()))); @@ -383,7 +384,8 @@ public class Synchronizer { for (Peer peer : superiorPeersForComparison) { // Check if we should discard an inferior peer if (peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) < 0) { - LOGGER.debug(String.format("Peer %s has a lower chain weight than other peer(s) in this group - removing it from this round.", peer)); + BigInteger difference = bestChainWeight.subtract(peer.getCommonBlockData().getChainWeight()); + LOGGER.debug(String.format("Peer %s has a lower chain weight (difference: %s) than other peer(s) in this group - removing it from this round.", peer, accurateFormatter.format(difference))); peers.remove(peer); } } From 1e491dd8fbae1052f5ca6222688cb74242874233 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Apr 2021 19:45:53 +0100 Subject: [PATCH 32/57] MAXIMUM_RETRIES increased from 1 to 3. Now that we are spending a lot of time to carefully select a peer to sync with, it makes sense to retry a couple more times before giving up and starting the peer selection process all over again. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 0cbb9e30..fd95006c 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -56,7 +56,7 @@ public class Synchronizer { private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? /** Number of retry attempts if a peer fails to respond with the requested data */ - private static final int MAXIMUM_RETRIES = 1; // XXX move to Settings? + private static final int MAXIMUM_RETRIES = 3; // XXX move to Settings? private static Synchronizer instance; From 476731a2c314ccc82205b4f41a63613c10752773 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Apr 2021 20:12:11 +0100 Subject: [PATCH 33/57] In syncToPeerChain(), only apply a partial set of peer's blocks if they are recent. If a peer fails to reply with all requested blocks, we will now only apply the blocks we have received so far if at least one of them is recent. This should prevent or greatly reduce the scenario where our chain is taken from a recent to an outdated state due to only partially syncing with a peer. It is best to keep our chain "recent" if possible, as this ensures that the peer selection code always runs, and therefore avoids unnecessarily syncing to a random peer on an inferior chain. --- .../org/qortal/controller/Synchronizer.java | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index fd95006c..1609979e 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -759,11 +759,17 @@ public class Synchronizer { LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, height, Base58.encode(latestPeerSignature))); - // If we have already received blocks from this peer, go ahead and apply them + // If we have already received RECENT blocks from this peer, go ahead and apply them if (peerBlocks.size() > 0) { - break; + final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (peerLatestBlock != null && minLatestBlockTimestamp != null + && peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) { + LOGGER.debug("Newly received blocks are recent, so we will apply them"); + break; + } } - // Otherwise, give up and move on to the next peer + // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state return SynchronizationResult.NO_REPLY; } @@ -788,11 +794,17 @@ public class Synchronizer { if (retryCount >= MAXIMUM_RETRIES) { - // If we have already received blocks from this peer, go ahead and apply them + // If we have already received RECENT blocks from this peer, go ahead and apply them if (peerBlocks.size() > 0) { - break; + final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (peerLatestBlock != null && minLatestBlockTimestamp != null + && peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) { + LOGGER.debug("Newly received blocks are recent, so we will apply them"); + break; + } } - // Otherwise, give up and move on to the next peer + // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state return SynchronizationResult.NO_REPLY; } else { From d599146c3ae72227b4cb28046ca95c3f4e92e37b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 24 Apr 2021 22:10:40 +0100 Subject: [PATCH 34/57] Cache peer block summaries to avoid duplicate requests when comparing peers. --- .../org/qortal/controller/Synchronizer.java | 68 ++++++++++--------- src/main/java/org/qortal/network/Peer.java | 20 ++++++ 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 1609979e..dd279a3c 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -118,17 +118,9 @@ public class Synchronizer { return SynchronizationResult.SHUTTING_DOWN; // Check if we can use the cached common block data, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block - PeerChainTipData peerChainTipData = peer.getChainTipData(); - CommonBlockData commonBlockData = peer.getCommonBlockData(); - - if (peerChainTipData != null && commonBlockData != null) { - PeerChainTipData commonBlockChainTipData = commonBlockData.getChainTipData(); - if (peerChainTipData.getLastBlockSignature() != null && commonBlockChainTipData != null && commonBlockChainTipData.getLastBlockSignature() != null) { - if (Arrays.equals(peerChainTipData.getLastBlockSignature(), commonBlockChainTipData.getLastBlockSignature())) { - LOGGER.debug(String.format("Skipping peer %s because we already have the latest common block data in our cache. Cached common block sig is %.08s", peer, Base58.encode(commonBlockData.getCommonBlockSummary().getSignature()))); - continue; - } - } + if (peer.canUseCachedCommonBlockData()) { + LOGGER.debug(String.format("Skipping peer %s because we already have the latest common block data in our cache. Cached common block sig is %.08s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature()))); + continue; } // Cached data is stale, so clear it and repopulate @@ -288,31 +280,43 @@ public class Synchronizer { final int peerHeight = peer.getChainTipData().getLastHeight(); final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); // Limit the number of blocks we are comparing. FUTURE: we could request more in batches, but there may not be a case when this is needed - final int summariesRequired = Math.min(peerAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE); + int summariesRequired = Math.min(peerAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE); - if (summariesRequired > 0) { - LOGGER.trace(String.format("Requesting %d block summar%s from peer %s after common block %.8s. Peer height: %d", summariesRequired, (summariesRequired != 1 ? "ies" : "y"), peer, Base58.encode(commonBlockSummary.getSignature()), peerHeight)); - - List blockSummaries = this.getBlockSummaries(peer, commonBlockSummary.getSignature(), summariesRequired); - peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(blockSummaries); - - if (blockSummaries != null) { - LOGGER.trace(String.format("Peer %s returned %d block summar%s", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"))); - - // We need to adjust minChainLength if peers fail to return all expected block summaries - if (blockSummaries.size() < summariesRequired) { - // This could mean that the peer has re-orged. But we still have the same common block, so it's safe to proceed with this set of signatures instead. - LOGGER.debug(String.format("Peer %s returned %d block summar%s instead of expected %d", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"), summariesRequired)); - - // Update minChainLength if we have at least 1 block for this peer. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength - if (blockSummaries.size() > 0) - minChainLength = blockSummaries.size(); + // Check if we can use the cached common block summaries, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block + boolean useCachedSummaries = false; + if (peer.canUseCachedCommonBlockData()) { + if (peer.getCommonBlockData().getBlockSummariesAfterCommonBlock() != null) { + if (peer.getCommonBlockData().getBlockSummariesAfterCommonBlock().size() == summariesRequired) { + LOGGER.debug(String.format("Using cached block summaries for peer %s", peer)); + useCachedSummaries = true; } } } - else { - // There are no block summaries after this common block - peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(null); + + if (useCachedSummaries == false) { + if (summariesRequired > 0) { + LOGGER.trace(String.format("Requesting %d block summar%s from peer %s after common block %.8s. Peer height: %d", summariesRequired, (summariesRequired != 1 ? "ies" : "y"), peer, Base58.encode(commonBlockSummary.getSignature()), peerHeight)); + + List blockSummaries = this.getBlockSummaries(peer, commonBlockSummary.getSignature(), summariesRequired); + peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(blockSummaries); + + if (blockSummaries != null) { + LOGGER.trace(String.format("Peer %s returned %d block summar%s", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"))); + + // We need to adjust minChainLength if peers fail to return all expected block summaries + if (blockSummaries.size() < summariesRequired) { + // This could mean that the peer has re-orged. But we still have the same common block, so it's safe to proceed with this set of signatures instead. + LOGGER.debug(String.format("Peer %s returned %d block summar%s instead of expected %d", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"), summariesRequired)); + + // Update minChainLength if we have at least 1 block for this peer. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength + if (blockSummaries.size() > 0) + minChainLength = blockSummaries.size(); + } + } + } else { + // There are no block summaries after this common block + peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(null); + } } } diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index ef6fb88a..08db0dd9 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -15,6 +15,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Random; +import java.util.Arrays; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -632,6 +633,25 @@ public class Peer { } } + + // Common block data + + public boolean canUseCachedCommonBlockData() { + PeerChainTipData peerChainTipData = this.getChainTipData(); + CommonBlockData commonBlockData = this.getCommonBlockData(); + + if (peerChainTipData != null && commonBlockData != null) { + PeerChainTipData commonBlockChainTipData = commonBlockData.getChainTipData(); + if (peerChainTipData.getLastBlockSignature() != null && commonBlockChainTipData != null && commonBlockChainTipData.getLastBlockSignature() != null) { + if (Arrays.equals(peerChainTipData.getLastBlockSignature(), commonBlockChainTipData.getLastBlockSignature())) { + return true; + } + } + } + return false; + } + + // Utility methods /** Returns true if ports and addresses (or hostnames) match */ From e12b99d17e23a59198c9246664838fef45bd28b2 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 25 Apr 2021 09:37:32 +0100 Subject: [PATCH 35/57] Invalidate our common block cache for a peer if we can't find a common block when synchronizing. --- src/main/java/org/qortal/controller/Synchronizer.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index dd279a3c..fccf5ba3 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -474,9 +474,12 @@ public class Synchronizer { List peerBlockSummaries = new ArrayList<>(); SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, force, peerBlockSummaries); - if (findCommonBlockResult != SynchronizationResult.OK) + if (findCommonBlockResult != SynchronizationResult.OK) { // Logging performed by fetchSummariesFromCommonBlock() above + // Clear our common block cache for this peer + peer.setCommonBlockData(null); return findCommonBlockResult; + } // First summary is common block final BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature()); From 0c0c5ff077b6bd2ef97a3a3930854ec887ea1c26 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 25 Apr 2021 12:50:40 +0100 Subject: [PATCH 36/57] Invalidate our block summaries cache for a peer if it fails to respond with signatures when synchronizing. --- src/main/java/org/qortal/controller/Synchronizer.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index fccf5ba3..8e917dca 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -766,6 +766,11 @@ public class Synchronizer { LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, height, Base58.encode(latestPeerSignature))); + // Clear our cache of common block summaries for this peer, as they are likely to be invalid + CommonBlockData cachedCommonBlockData = peer.getCommonBlockData(); + if (cachedCommonBlockData != null) + cachedCommonBlockData.setBlockSummariesAfterCommonBlock(null); + // If we have already received RECENT blocks from this peer, go ahead and apply them if (peerBlocks.size() > 0) { final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); From b37f2c7d7fa225b65651614b59a1b1dd5884e1b9 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 26 Apr 2021 17:08:21 +0100 Subject: [PATCH 37/57] MAXIMUM_RETRIES set to 2, as 3 retries may have been slightly too many. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 8e917dca..bad364ee 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -56,7 +56,7 @@ public class Synchronizer { private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? /** Number of retry attempts if a peer fails to respond with the requested data */ - private static final int MAXIMUM_RETRIES = 3; // XXX move to Settings? + private static final int MAXIMUM_RETRIES = 2; // XXX move to Settings? private static Synchronizer instance; From a9a0e69ec0c3d8af752b3cda182e1cab32fc2a66 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 26 Apr 2021 17:19:39 +0100 Subject: [PATCH 38/57] Set go-live block height for share bin fix: block 399000 --- src/main/resources/blockchain.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 363b80cb..e54676b2 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -50,7 +50,7 @@ "featureTriggers": { "atFindNextTransactionFix": 275000, "newBlockSigHeight": 320000, - "shareBinFix": 999999 + "shareBinFix": 399000 }, "genesisInfo": { "version": 4, From c17a481b742cb26f120ef1728aa1ea45aaa6f709 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 26 Apr 2021 18:34:01 +0100 Subject: [PATCH 39/57] Bump version to 1.5.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e7dcc009..6697cc81 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 1.4.6 + 1.5.0 jar true From c0c5bf1591923d8eb7c729c2d69e244f582061ad Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 28 Apr 2021 22:03:13 +0100 Subject: [PATCH 40/57] Apply blocks in syncToPeerChain() if the latest received block is newer than our latest, and we started from an out of date chain. This solves a common problem that is mostly seen when starting a node that has been switched off for some time, or when starting from a bootstrap. In these cases, it can be difficult get synced to the latest if you are starting from a small fork. This is because it required that the node was brought up to date via a single peer, and there wasn't much room for error if it failed to retrieve a block a couple of times. This generally caused the blocks to be thrown away and it would try the same process over and over. The solution is to apply new blocks if the most recently received block is newer than our current latest block. This gets the node back on to the main fork where it can then sync using the regular applyNewBlocks() method. --- .../org/qortal/controller/Synchronizer.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index bad364ee..e1be8427 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -771,15 +771,33 @@ public class Synchronizer { if (cachedCommonBlockData != null) cachedCommonBlockData.setBlockSummariesAfterCommonBlock(null); - // If we have already received RECENT blocks from this peer, go ahead and apply them + // If we have already received recent or newer blocks from this peer, go ahead and apply them if (peerBlocks.size() > 0) { final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + + // If we have received at least one recent block, we can apply them if (peerLatestBlock != null && minLatestBlockTimestamp != null && peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) { LOGGER.debug("Newly received blocks are recent, so we will apply them"); break; } + + // If our latest block is very old.... + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + if (minLatestBlockTimestamp != null && ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + // ... and we have received a block that is more recent than our latest block ... + if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) { + // ... then apply the blocks, as it takes us a step forward. + // This is particularly useful when starting up a node that was on a small fork when it was last shut down. + // In these cases, we now allow the node to sync forward, and get onto the main chain again. + // Without this, we would require that the node syncs ENTIRELY with this peer, + // and any problems downloading a block would cause all progress to be lost. + LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp())); + break; + } + } + } // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state return SynchronizationResult.NO_REPLY; From 26d8ed783aae8333f955702feb3529dcc4a5de50 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 29 Apr 2021 08:55:16 +0100 Subject: [PATCH 41/57] Same as commit c0c5bf1, but for blocks as well as block summaries. --- .../org/qortal/controller/Synchronizer.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index e1be8427..a4d60e29 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -824,15 +824,32 @@ public class Synchronizer { if (retryCount >= MAXIMUM_RETRIES) { - // If we have already received RECENT blocks from this peer, go ahead and apply them + // If we have already received recent or newer blocks from this peer, go ahead and apply them if (peerBlocks.size() > 0) { final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + + // If we have received at least one recent block, we can apply them if (peerLatestBlock != null && minLatestBlockTimestamp != null && peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) { LOGGER.debug("Newly received blocks are recent, so we will apply them"); break; } + + // If our latest block is very old.... + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); + if (minLatestBlockTimestamp != null && ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + // ... and we have received a block that is more recent than our latest block ... + if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) { + // ... then apply the blocks, as it takes us a step forward. + // This is particularly useful when starting up a node that was on a small fork when it was last shut down. + // In these cases, we now allow the node to sync forward, and get onto the main chain again. + // Without this, we would require that the node syncs ENTIRELY with this peer, + // and any problems downloading a block would cause all progress to be lost. + LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp())); + break; + } + } } // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state return SynchronizationResult.NO_REPLY; From 5fd8528c49ffdad72b8344a06c8a759c64f615a4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 29 Apr 2021 09:04:59 +0100 Subject: [PATCH 42/57] Small refactor for code readability, and added some defensiveness to avoid possible NPEs. --- .../org/qortal/controller/Synchronizer.java | 75 ++++++++++--------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index a4d60e29..21376838 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -773,31 +773,31 @@ public class Synchronizer { // If we have already received recent or newer blocks from this peer, go ahead and apply them if (peerBlocks.size() > 0) { + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (ourLatestBlockData != null && peerLatestBlock != null && minLatestBlockTimestamp != null) { - // If we have received at least one recent block, we can apply them - if (peerLatestBlock != null && minLatestBlockTimestamp != null - && peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) { - LOGGER.debug("Newly received blocks are recent, so we will apply them"); - break; - } - - // If our latest block is very old.... - final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); - if (minLatestBlockTimestamp != null && ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { - // ... and we have received a block that is more recent than our latest block ... - if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) { - // ... then apply the blocks, as it takes us a step forward. - // This is particularly useful when starting up a node that was on a small fork when it was last shut down. - // In these cases, we now allow the node to sync forward, and get onto the main chain again. - // Without this, we would require that the node syncs ENTIRELY with this peer, - // and any problems downloading a block would cause all progress to be lost. - LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp())); + // If we have received at least one recent block, we can apply them + if (peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) { + LOGGER.debug("Newly received blocks are recent, so we will apply them"); break; } - } + // If our latest block is very old.... + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + // ... and we have received a block that is more recent than our latest block ... + if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) { + // ... then apply the blocks, as it takes us a step forward. + // This is particularly useful when starting up a node that was on a small fork when it was last shut down. + // In these cases, we now allow the node to sync forward, and get onto the main chain again. + // Without this, we would require that the node syncs ENTIRELY with this peer, + // and any problems downloading a block would cause all progress to be lost. + LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp())); + break; + } + } + } } // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state return SynchronizationResult.NO_REPLY; @@ -826,29 +826,30 @@ public class Synchronizer { // If we have already received recent or newer blocks from this peer, go ahead and apply them if (peerBlocks.size() > 0) { + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1); final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (ourLatestBlockData != null && peerLatestBlock != null && minLatestBlockTimestamp != null) { - // If we have received at least one recent block, we can apply them - if (peerLatestBlock != null && minLatestBlockTimestamp != null - && peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) { - LOGGER.debug("Newly received blocks are recent, so we will apply them"); - break; - } - - // If our latest block is very old.... - final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); - if (minLatestBlockTimestamp != null && ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { - // ... and we have received a block that is more recent than our latest block ... - if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) { - // ... then apply the blocks, as it takes us a step forward. - // This is particularly useful when starting up a node that was on a small fork when it was last shut down. - // In these cases, we now allow the node to sync forward, and get onto the main chain again. - // Without this, we would require that the node syncs ENTIRELY with this peer, - // and any problems downloading a block would cause all progress to be lost. - LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp())); + // If we have received at least one recent block, we can apply them + if (peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) { + LOGGER.debug("Newly received blocks are recent, so we will apply them"); break; } + + // If our latest block is very old.... + if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { + // ... and we have received a block that is more recent than our latest block ... + if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) { + // ... then apply the blocks, as it takes us a step forward. + // This is particularly useful when starting up a node that was on a small fork when it was last shut down. + // In these cases, we now allow the node to sync forward, and get onto the main chain again. + // Without this, we would require that the node syncs ENTIRELY with this peer, + // and any problems downloading a block would cause all progress to be lost. + LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp())); + break; + } + } } } // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state From 55ff1e2bb1f0aec35e5d96da1816f6cced3a1415 Mon Sep 17 00:00:00 2001 From: QuickMythril Date: Sat, 1 May 2021 04:18:46 -0400 Subject: [PATCH 43/57] updated and tested BTC electrum servers (#36) * updated electrum servers mainnet list: https://1209k.com/bitcoin-eye/ele.php?chain=btc testnet list: https://1209k.com/bitcoin-eye/ele.php?chain=tbtc * removed servers tested each mainnet server individually and removed those that did not respond --- .../java/org/qortal/crosschain/Bitcoin.java | 57 +++++++++---------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/qortal/crosschain/Bitcoin.java b/src/main/java/org/qortal/crosschain/Bitcoin.java index a8c6469a..28275d6a 100644 --- a/src/main/java/org/qortal/crosschain/Bitcoin.java +++ b/src/main/java/org/qortal/crosschain/Bitcoin.java @@ -42,35 +42,32 @@ public class Bitcoin extends Bitcoiny { public Collection getServers() { return Arrays.asList( // Servers chosen on NO BASIS WHATSOEVER from various sources! - new Server("enode.duckdns.org", Server.ConnectionType.SSL, 50002), - new Server("electrumx.ml", Server.ConnectionType.SSL, 50002), - new Server("electrum.bitkoins.nl", Server.ConnectionType.SSL, 50512), - new Server("btc.electroncash.dk", Server.ConnectionType.SSL, 60002), - new Server("electrumx.electricnewyear.net", Server.ConnectionType.SSL, 50002), - new Server("dxm.no-ip.biz", Server.ConnectionType.TCP, 50001), - new Server("kirsche.emzy.de", Server.ConnectionType.TCP, 50001), - new Server("2AZZARITA.hopto.org", Server.ConnectionType.TCP, 50001), - new Server("xtrum.com", Server.ConnectionType.TCP, 50001), - new Server("electrum.srvmin.network", Server.ConnectionType.TCP, 50001), - new Server("electrumx.alexridevski.net", Server.ConnectionType.TCP, 50001), - new Server("bitcoin.lukechilds.co", Server.ConnectionType.TCP, 50001), - new Server("electrum.poiuty.com", Server.ConnectionType.TCP, 50001), - new Server("horsey.cryptocowboys.net", Server.ConnectionType.TCP, 50001), + new Server("128.0.190.26", Server.ConnectionType.SSL, 50002), + new Server("hodlers.beer", Server.ConnectionType.SSL, 50002), + new Server("electrumx.erbium.eu", Server.ConnectionType.TCP, 50001), + new Server("electrumx.erbium.eu", Server.ConnectionType.SSL, 50002), + new Server("btc.lastingcoin.net", Server.ConnectionType.SSL, 50002), + new Server("electrum.bitaroo.net", Server.ConnectionType.SSL, 50002), + new Server("bitcoin.grey.pw", Server.ConnectionType.SSL, 50002), + new Server("2electrumx.hopto.me", Server.ConnectionType.SSL, 56022), + new Server("185.64.116.15", Server.ConnectionType.SSL, 50002), + new Server("kirsche.emzy.de", Server.ConnectionType.SSL, 50002), + new Server("alviss.coinjoined.com", Server.ConnectionType.SSL, 50002), + new Server("electrum.emzy.de", Server.ConnectionType.SSL, 50002), new Server("electrum.emzy.de", Server.ConnectionType.TCP, 50001), - new Server("electrum-server.ninja", Server.ConnectionType.TCP, 50081), - new Server("bitcoin.electrumx.multicoin.co", Server.ConnectionType.TCP, 50001), - new Server("esx.geekhosters.com", Server.ConnectionType.TCP, 50001), - new Server("bitcoin.grey.pw", Server.ConnectionType.TCP, 50003), - new Server("exs.ignorelist.com", Server.ConnectionType.TCP, 50001), - new Server("electrum.coinext.com.br", Server.ConnectionType.TCP, 50001), - new Server("bitcoin.aranguren.org", Server.ConnectionType.TCP, 50001), - new Server("skbxmit.coinjoined.com", Server.ConnectionType.TCP, 50001), - new Server("alviss.coinjoined.com", Server.ConnectionType.TCP, 50001), - new Server("electrum2.privateservers.network", Server.ConnectionType.TCP, 50001), - new Server("electrumx.schulzemic.net", Server.ConnectionType.TCP, 50001), - new Server("bitcoins.sk", Server.ConnectionType.TCP, 56001), - new Server("node.mendonca.xyz", Server.ConnectionType.TCP, 50001), - new Server("bitcoin.aranguren.org", Server.ConnectionType.TCP, 50001)); + new Server("vmd71287.contaboserver.net", Server.ConnectionType.SSL, 50002), + new Server("btc.litepay.ch", Server.ConnectionType.SSL, 50002), + new Server("electrum.stippy.com", Server.ConnectionType.SSL, 50002), + new Server("xtrum.com", Server.ConnectionType.SSL, 50002), + new Server("electrum.acinq.co", Server.ConnectionType.SSL, 50002), + new Server("electrum2.taborsky.cz", Server.ConnectionType.SSL, 50002), + new Server("vmd63185.contaboserver.net", Server.ConnectionType.SSL, 50002), + new Server("electrum2.privateservers.network", Server.ConnectionType.SSL, 50002), + new Server("electrumx.alexridevski.net", Server.ConnectionType.SSL, 50002), + new Server("192.166.219.200", Server.ConnectionType.SSL, 50002), + new Server("2ex.digitaleveryware.com", Server.ConnectionType.SSL, 50002), + new Server("dxm.no-ip.biz", Server.ConnectionType.SSL, 50002), + new Server("caleb.vegas", Server.ConnectionType.SSL, 50002)); } @Override @@ -96,10 +93,8 @@ public class Bitcoin extends Bitcoiny { @Override public Collection getServers() { return Arrays.asList( - new Server("electrum.blockstream.info", Server.ConnectionType.TCP, 60001), - new Server("electrum.blockstream.info", Server.ConnectionType.SSL, 60002), + new Server("tn.not.fyi", Server.ConnectionType.SSL, 55002), new Server("electrumx-test.1209k.com", Server.ConnectionType.SSL, 50002), - new Server("testnet.qtornado.com", Server.ConnectionType.TCP, 51001), new Server("testnet.qtornado.com", Server.ConnectionType.SSL, 51002), new Server("testnet.aranguren.org", Server.ConnectionType.TCP, 51001), new Server("testnet.aranguren.org", Server.ConnectionType.SSL, 51002), From 1da8994be72a7f25799fa0ee0396b51cfd377593 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 1 May 2021 10:24:50 +0100 Subject: [PATCH 44/57] Log the block timestamp, minter level, online accounts, key distance, and weight, when orphaning or processing. This gives an insight into the contents of each chain when doing a re-org. To enable this logging, add the following to log4j2.properties: logger.block.name = org.qortal.block.Block logger.block.level = debug --- src/main/java/org/qortal/block/Block.java | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 34a87e9a..ee41bc07 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1335,6 +1335,9 @@ public class Block { // Give Controller our cached, valid online accounts data (if any) to help reduce CPU load for next block Controller.getInstance().pushLatestBlocksOnlineAccounts(this.cachedValidOnlineAccounts); + + // Log some debugging info relating to the block weight calculation + this.logDebugInfo(); } protected void increaseAccountLevels() throws DataException { @@ -1516,6 +1519,9 @@ public class Block { public void orphan() throws DataException { LOGGER.trace(() -> String.format("Orphaning block %d", this.blockData.getHeight())); + // Log some debugging info relating to the block weight calculation + this.logDebugInfo(); + // Return AT fees and delete AT states from repository orphanAtFeesAndStates(); @@ -1988,4 +1994,33 @@ public class Block { this.repository.getAccountRepository().tidy(); } + private void logDebugInfo() { + try { + if (this.repository == null || this.getMinter() == null || this.getBlockData() == null) + return; + + int minterLevel = Account.getRewardShareEffectiveMintingLevel(this.repository, this.getMinter().getPublicKey()); + + LOGGER.debug(String.format("======= BLOCK %d (%.8s) =======", this.getBlockData().getHeight(), Base58.encode(this.getSignature()))); + LOGGER.debug(String.format("Timestamp: %d", this.getBlockData().getTimestamp())); + LOGGER.debug(String.format("Minter level: %d", minterLevel)); + LOGGER.debug(String.format("Online accounts: %d", this.getBlockData().getOnlineAccountsCount())); + + BlockSummaryData blockSummaryData = new BlockSummaryData(this.getBlockData()); + if (this.getParent() == null || this.getParent().getSignature() == null || blockSummaryData == null) + return; + + blockSummaryData.setMinterLevel(minterLevel); + BigInteger blockWeight = calcBlockWeight(this.getParent().getHeight(), this.getParent().getSignature(), blockSummaryData); + BigInteger keyDistance = calcKeyDistance(this.getParent().getHeight(), this.getParent().getSignature(), blockSummaryData.getMinterPublicKey(), blockSummaryData.getMinterLevel()); + NumberFormat formatter = new DecimalFormat("0.###E0"); + + LOGGER.debug(String.format("Key distance: %s", formatter.format(keyDistance))); + LOGGER.debug(String.format("Weight: %s", formatter.format(blockWeight))); + + } catch (DataException e) { + LOGGER.info(() -> String.format("Unable to log block debugging info: %s", e.getMessage())); + } + } + } From b4395fdad1579f0db774405268d597e95d8eb066 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 1 May 2021 10:57:24 +0100 Subject: [PATCH 45/57] Fixed bug which could cause minChainLength to report a higher value. This wouldn't have affected anything in 1.5.0, but it will become more significant if we switch to same-length chain weight comparisons. --- src/main/java/org/qortal/controller/Synchronizer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 21376838..05db6c54 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -308,9 +308,10 @@ public class Synchronizer { // This could mean that the peer has re-orged. But we still have the same common block, so it's safe to proceed with this set of signatures instead. LOGGER.debug(String.format("Peer %s returned %d block summar%s instead of expected %d", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"), summariesRequired)); - // Update minChainLength if we have at least 1 block for this peer. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength + // Reduce minChainLength if we have at least 1 block for this peer. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength if (blockSummaries.size() > 0) - minChainLength = blockSummaries.size(); + if (blockSummaries.size() < minChainLength) + minChainLength = blockSummaries.size(); } } } else { From 50244c1c40a115fc75c163cf7a6c2a7e309566dd Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 1 May 2021 13:32:16 +0100 Subject: [PATCH 46/57] Fixed bug which would cause other peers to not be compared against each other, if we had no blocks ourselves. Again, this wouldn't have affected anything in 1.5.0 or before, but it will become more significant if we switch to same-length chain weight comparisons. --- .../org/qortal/controller/Synchronizer.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 05db6c54..a6131840 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -267,7 +267,7 @@ public class Synchronizer { // Calculate the length of the shortest peer chain sharing this common block, including our chain final int ourAdditionalBlocksAfterCommonBlock = ourHeight - commonBlockSummary.getHeight(); - int minChainLength = this.calculateMinChainLength(commonBlockSummary, ourAdditionalBlocksAfterCommonBlock, peersSharingCommonBlock); + int minChainLength = this.calculateMinChainLengthOfPeers(peersSharingCommonBlock, commonBlockSummary); // Fetch block summaries from each peer for (Peer peer : peersSharingCommonBlock) { @@ -325,10 +325,15 @@ public class Synchronizer { final int ourSummariesRequired = Math.min(ourAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE); LOGGER.trace(String.format("About to fetch our block summaries from %d to %d. Our height: %d", commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired, ourHeight)); List ourBlockSummaries = repository.getBlockRepository().getBlockSummaries(commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired); - if (ourBlockSummaries.isEmpty()) + if (ourBlockSummaries.isEmpty()) { LOGGER.debug(String.format("We don't have any block summaries so can't compare our chain against peers with this common block. We can still compare them against each other.")); - else + } + else { populateBlockSummariesMinterLevels(repository, ourBlockSummaries); + // Reduce minChainLength if we have less summaries + if (ourBlockSummaries.size() < minChainLength) + minChainLength = ourBlockSummaries.size(); + } // Create array to hold peers for comparison List superiorPeersForComparison = new ArrayList<>(); @@ -426,14 +431,14 @@ public class Synchronizer { return commonBlocks; } - private int calculateMinChainLength(BlockSummaryData commonBlockSummary, int ourAdditionalBlocksAfterCommonBlock, List peersSharingCommonBlock) { + private int calculateMinChainLengthOfPeers(List peersSharingCommonBlock, BlockSummaryData commonBlockSummary) { // Calculate the length of the shortest peer chain sharing this common block, including our chain - int minChainLength = ourAdditionalBlocksAfterCommonBlock; + int minChainLength = 0; for (Peer peer : peersSharingCommonBlock) { final int peerHeight = peer.getChainTipData().getLastHeight(); final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight(); - if (peerAdditionalBlocksAfterCommonBlock < minChainLength) + if (peerAdditionalBlocksAfterCommonBlock < minChainLength || minChainLength == 0) minChainLength = peerAdditionalBlocksAfterCommonBlock; } return minChainLength; From 9ebcd55ff5bcaae8fd1b45ec96ae5e925507ca48 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 1 May 2021 13:34:13 +0100 Subject: [PATCH 47/57] Fixed calculation error in existing chain weight code, which would have caused the last block to be missed out of the comparison after switching to same-length chain comparisons. --- src/main/java/org/qortal/controller/Synchronizer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index a6131840..fba3dddb 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -709,7 +709,7 @@ public class Synchronizer { populateBlockSummariesMinterLevels(repository, ourBlockSummaries); populateBlockSummariesMinterLevels(repository, peerBlockSummaries); - final int mutualHeight = commonBlockHeight - 1 + Math.min(ourBlockSummaries.size(), peerBlockSummaries.size()); + final int mutualHeight = commonBlockHeight + Math.min(ourBlockSummaries.size(), peerBlockSummaries.size()); // Calculate cumulative chain weights of both blockchain subsets, from common block to highest mutual block. BigInteger ourChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight); From fac02dbc7db5ea32b96211ad9923e9939c73f82a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 2 May 2021 15:56:13 +0100 Subject: [PATCH 48/57] Fixed bug in maxHeight parameter passed to Block.calcChainWeight() Like the others, this one is only relevant after switching to same-length chain weight comparisons. --- src/main/java/org/qortal/controller/Synchronizer.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index fba3dddb..2d4391b1 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -338,10 +338,13 @@ public class Synchronizer { // Create array to hold peers for comparison List superiorPeersForComparison = new ArrayList<>(); + // Calculate max height for chain weight comparisons + int maxHeightForChainWeightComparisons = commonBlockSummary.getHeight() + minChainLength; + // Calculate our chain weight BigInteger ourChainWeight = BigInteger.valueOf(0); if (ourBlockSummaries.size() > 0) - ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, minChainLength); + ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, maxHeightForChainWeightComparisons); NumberFormat formatter = new DecimalFormat("0.###E0"); NumberFormat accurateFormatter = new DecimalFormat("0.################E0"); @@ -365,7 +368,7 @@ public class Synchronizer { // Calculate cumulative chain weight of this blockchain subset, from common block to highest mutual block held by all peers in this group. LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", peerBlockSummariesAfterCommonBlock.size(), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock)); - BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, minChainLength); + BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, maxHeightForChainWeightComparisons); peer.getCommonBlockData().setChainWeight(peerChainWeight); LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, peerBlockSummariesAfterCommonBlock.size(), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight))); From db7710805452d5433fefad6352073a3f019d84df Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 2 May 2021 19:59:32 +0100 Subject: [PATCH 49/57] Log the number of blocks used in Block.calcChainWeight() This makes it easier to check that the new consensus code is being used, and that it is working correctly. --- src/main/java/org/qortal/block/Block.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 47d82043..fda06d73 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -796,7 +796,9 @@ public class Block { NumberFormat formatter = new DecimalFormat("0.###E0"); boolean isLogging = LOGGER.getLevel().isLessSpecificThan(Level.TRACE); + int blockCount = 0; for (BlockSummaryData blockSummaryData : blockSummaries) { + blockCount++; StringBuilder stringBuilder = isLogging ? new StringBuilder(512) : null; if (isLogging) @@ -829,6 +831,7 @@ public class Block { if (NTP.getTime() >= BlockChain.getInstance().getCalcChainWeightTimestamp() && parentHeight >= maxHeight) break; } + LOGGER.debug(String.format("Chain weight calculation was based on %d blocks", blockCount)); return cumulativeWeight; } From 2eb677196341625eefa7d171e98d44dfbe067c44 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 2 May 2021 20:26:51 +0100 Subject: [PATCH 50/57] Adapted logging in comparePeers() to report correct values for both chain weight algorithms. --- .../java/org/qortal/controller/Synchronizer.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 2d4391b1..bc527784 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -16,6 +16,7 @@ import org.qortal.account.Account; import org.qortal.account.PublicKeyAccount; import org.qortal.block.Block; import org.qortal.block.Block.ValidationResult; +import org.qortal.block.BlockChain; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.block.CommonBlockData; @@ -36,6 +37,7 @@ import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.transaction.Transaction; import org.qortal.utils.Base58; +import org.qortal.utils.NTP; public class Synchronizer { @@ -226,6 +228,10 @@ public class Synchronizer { return peers; } + // We will switch to a new chain weight consensus algorithm at a hard fork, so determine if this has happened yet + boolean usingSameLengthChainWeight = (NTP.getTime() >= BlockChain.getInstance().getCalcChainWeightTimestamp()); + LOGGER.debug(String.format("Using %s chain weight consensus algorithm", (usingSameLengthChainWeight ? "same-length" : "variable-length"))); + // Retrieve a list of unique common blocks from this list of peers List commonBlocks = this.uniqueCommonBlocks(peers); @@ -348,7 +354,7 @@ public class Synchronizer { NumberFormat formatter = new DecimalFormat("0.###E0"); NumberFormat accurateFormatter = new DecimalFormat("0.################E0"); - LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", ourBlockSummaries.size(), formatter.format(ourChainWeight))); + LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", (usingSameLengthChainWeight ? minChainLength : ourBlockSummaries.size()), formatter.format(ourChainWeight))); LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature()))); for (Peer peer : peersSharingCommonBlock) { @@ -367,10 +373,10 @@ public class Synchronizer { populateBlockSummariesMinterLevels(repository, peerBlockSummariesAfterCommonBlock); // Calculate cumulative chain weight of this blockchain subset, from common block to highest mutual block held by all peers in this group. - LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", peerBlockSummariesAfterCommonBlock.size(), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock)); + LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock)); BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, maxHeightForChainWeightComparisons); peer.getCommonBlockData().setChainWeight(peerChainWeight); - LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, peerBlockSummariesAfterCommonBlock.size(), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight))); + LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight))); // Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group) if (ourChainWeight.compareTo(peerChainWeight) > 0) { From 8e244fd956a4bf4d912ff4eee1af300744be5254 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 2 May 2021 20:45:20 +0100 Subject: [PATCH 51/57] Fixed yet another bug with minChainLength. --- .../org/qortal/controller/Synchronizer.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index bc527784..a07303e9 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -309,22 +309,21 @@ public class Synchronizer { if (blockSummaries != null) { LOGGER.trace(String.format("Peer %s returned %d block summar%s", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"))); - // We need to adjust minChainLength if peers fail to return all expected block summaries - if (blockSummaries.size() < summariesRequired) { + if (blockSummaries.size() < summariesRequired) // This could mean that the peer has re-orged. But we still have the same common block, so it's safe to proceed with this set of signatures instead. LOGGER.debug(String.format("Peer %s returned %d block summar%s instead of expected %d", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"), summariesRequired)); - - // Reduce minChainLength if we have at least 1 block for this peer. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength - if (blockSummaries.size() > 0) - if (blockSummaries.size() < minChainLength) - minChainLength = blockSummaries.size(); - } } } else { // There are no block summaries after this common block peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(null); } } + + // Reduce minChainLength if needed. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength + List peerBlockSummaries = peer.getCommonBlockData().getBlockSummariesAfterCommonBlock(); + if (peerBlockSummaries != null && peerBlockSummaries.size() > 0) + if (peerBlockSummaries.size() < minChainLength) + minChainLength = peerBlockSummaries.size(); } // Fetch our corresponding block summaries. Limit to MAXIMUM_REQUEST_SIZE, in order to make the comparison fairer, as peers have been limited too @@ -441,7 +440,7 @@ public class Synchronizer { } private int calculateMinChainLengthOfPeers(List peersSharingCommonBlock, BlockSummaryData commonBlockSummary) { - // Calculate the length of the shortest peer chain sharing this common block, including our chain + // Calculate the length of the shortest peer chain sharing this common block int minChainLength = 0; for (Peer peer : peersSharingCommonBlock) { final int peerHeight = peer.getChainTipData().getLastHeight(); From 6e9a61c4e5389705183cc3070a2bb642de642066 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 2 May 2021 20:51:53 +0100 Subject: [PATCH 52/57] Fixed logging issue where it would underreport the number of common blocks found when loading some from the cache. --- src/main/java/org/qortal/controller/Synchronizer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index a07303e9..84f210d0 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -122,6 +122,7 @@ public class Synchronizer { // Check if we can use the cached common block data, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block if (peer.canUseCachedCommonBlockData()) { LOGGER.debug(String.format("Skipping peer %s because we already have the latest common block data in our cache. Cached common block sig is %.08s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature()))); + commonBlocksFound++; continue; } From 6109bdeafeccd05bbd7f7d263995092054674933 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 5 May 2021 18:40:07 +0100 Subject: [PATCH 53/57] Set go-live timestamp for same-length chain weight consensus: 1620579600000 --- src/main/resources/blockchain.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index 8cc06651..d0ac9ffb 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -51,7 +51,7 @@ "atFindNextTransactionFix": 275000, "newBlockSigHeight": 320000, "shareBinFix": 399000, - "calcChainWeightTimestamp": 1616000000000 + "calcChainWeightTimestamp": 1620579600000 }, "genesisInfo": { "version": 4, From 0c3597f75714b7e96962e4345d62529480bc6a9b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 5 May 2021 18:41:05 +0100 Subject: [PATCH 54/57] Bump version to 1.5.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6697cc81..43e9eae1 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 1.5.0 + 1.5.1 jar true From fe4ae615527b8e7526b1c98a9d44144dc4f568c3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Thu, 6 May 2021 17:49:45 +0100 Subject: [PATCH 55/57] Added "maxRetries" setting. This controls the maximum number of retry attempts if a peer fails to respond with the requested data. --- src/main/java/org/qortal/controller/Synchronizer.java | 11 +++++------ src/main/java/org/qortal/settings/Settings.java | 4 ++++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 84f210d0..42b46a31 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -35,6 +35,7 @@ import org.qortal.network.message.Message.MessageType; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.transaction.Transaction; import org.qortal.utils.Base58; import org.qortal.utils.NTP; @@ -57,9 +58,6 @@ public class Synchronizer { /** Maximum number of block signatures we ask from peer in one go */ private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings? - /** Number of retry attempts if a peer fails to respond with the requested data */ - private static final int MAXIMUM_RETRIES = 2; // XXX move to Settings? - private static Synchronizer instance; @@ -748,6 +746,7 @@ public class Synchronizer { LOGGER.debug(() -> String.format("Fetching peer %s chain from height %d, sig %.8s", peer, commonBlockHeight, commonBlockSig58)); + final int maxRetries = Settings.getInstance().getMaxRetries(); // Overall plan: fetch peer's blocks first, then orphan, then apply @@ -837,7 +836,7 @@ public class Synchronizer { LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, nextHeight, Base58.encode(nextPeerSignature))); - if (retryCount >= MAXIMUM_RETRIES) { + if (retryCount >= maxRetries) { // If we have already received recent or newer blocks from this peer, go ahead and apply them if (peerBlocks.size() > 0) { @@ -875,9 +874,9 @@ public class Synchronizer { peerBlockSignatures.clear(); numberSignaturesRequired = peerHeight - height; - // Retry until retryCount reaches MAXIMUM_RETRIES + // Retry until retryCount reaches maxRetries retryCount++; - int triesRemaining = MAXIMUM_RETRIES - retryCount; + int triesRemaining = maxRetries - retryCount; LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s" : ""))); continue; } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index cf1830cf..ba9678f2 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -122,6 +122,8 @@ public class Settings { private int maxNetworkThreadPoolSize = 20; /** Maximum number of threads for network proof-of-work compute, used during handshaking. */ private int networkPoWComputePoolSize = 2; + /** Maximum number of retry attempts if a peer fails to respond with the requested data */ + private int maxRetries = 2; // Which blockchains this node is running private String blockchainConfig = null; // use default from resources @@ -408,6 +410,8 @@ public class Settings { return this.networkPoWComputePoolSize; } + public int getMaxRetries() { return this.maxRetries; } + public String getBlockchainConfig() { return this.blockchainConfig; } From f8dac390762bd0ce2ac542e8dbc128e8289f420e Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 7 May 2021 09:40:38 +0100 Subject: [PATCH 56/57] Updated AdvancedInstaller project for v1.5.0 This includes updating AdoptOpenJDK to version 11.0.11.9, because 11.0.6.10 is no longer recommended or available in their archive. It also looks like I am using a newer version of AdvancedInstaller itself. --- WindowsInstaller/Qortal.aip | 1465 +++++++++++++++++++++-------------- 1 file changed, 881 insertions(+), 584 deletions(-) diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip index e19f3664..6d5b2d96 100755 --- a/WindowsInstaller/Qortal.aip +++ b/WindowsInstaller/Qortal.aip @@ -1,10 +1,8 @@ - - - - - + + + @@ -19,10 +17,10 @@ - + - + @@ -35,8 +33,8 @@ - - + + @@ -48,10 +46,12 @@ - + + + @@ -76,226 +76,296 @@ + + + + + + + + + + + + + + + + + + + - + - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - + + + + + + + + + + + + @@ -305,335 +375,506 @@ - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -648,7 +889,7 @@ - + @@ -659,9 +900,9 @@ - - - + + + @@ -680,10 +921,14 @@ + + + + @@ -882,13 +1127,17 @@ + + + - + + - + @@ -904,9 +1153,8 @@ - - + @@ -949,8 +1197,8 @@ - + @@ -982,6 +1230,7 @@ + @@ -990,12 +1239,14 @@ + - + + @@ -1005,21 +1256,43 @@ + + + + + + + + + + + + + + + + + + + + + + @@ -1032,10 +1305,21 @@ + - + + + + + + + + + + + @@ -1051,74 +1335,88 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -1142,7 +1440,7 @@ - + @@ -1154,9 +1452,6 @@ - - - @@ -1175,6 +1470,8 @@ + + @@ -1197,13 +1494,13 @@ + - - + @@ -1229,7 +1526,7 @@ - + @@ -1241,8 +1538,8 @@ - - + + @@ -1266,7 +1563,7 @@ - - + + From a170668d9ddd92c05fc207369fea1086684a3a07 Mon Sep 17 00:00:00 2001 From: Tom Date: Fri, 7 May 2021 09:58:15 +0100 Subject: [PATCH 57/57] Updated AdvancedInstaller project for v1.5.1 --- WindowsInstaller/Qortal.aip | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/WindowsInstaller/Qortal.aip b/WindowsInstaller/Qortal.aip index 6d5b2d96..443e483f 100755 --- a/WindowsInstaller/Qortal.aip +++ b/WindowsInstaller/Qortal.aip @@ -17,10 +17,10 @@ - + - + @@ -212,7 +212,7 @@ - +