From 6f27d3798c6cc9f1071d981f969517bdd0cae356 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Oct 2022 19:18:41 +0100 Subject: [PATCH 1/5] Improved online accounts processing, to avoid creating keys in the map before validation. --- .../java/org/qortal/controller/OnlineAccountsManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 7b60f0d9..fe6e3078 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -192,8 +192,8 @@ public class OnlineAccountsManager { return; // Skip this account if it's already validated - Set onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountData.getTimestamp(), k -> ConcurrentHashMap.newKeySet()); - if (onlineAccounts.contains(onlineAccountData)) { + Set onlineAccounts = this.currentOnlineAccounts.get(onlineAccountData.getTimestamp()); + if (onlineAccounts != null && onlineAccounts.contains(onlineAccountData)) { // We have already validated this online account onlineAccountsImportQueue.remove(onlineAccountData); continue; From 72985b1fc6a11f2a15d36861af5eaa85cf78a9f6 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Oct 2022 19:24:54 +0100 Subject: [PATCH 2/5] Reduce log spam, especially around the time of node startup before online accounts have been retrieved. We expect a "Couldn't build a to-be-minted block" log on every startup due to trying to mint before having any accounts. This one has moved from error to info level because error logs can be quite intrusive when using an IDE. --- src/main/java/org/qortal/controller/BlockMinter.java | 2 +- src/main/java/org/qortal/controller/OnlineAccountsManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 100e74db..0734d4e9 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -244,7 +244,7 @@ public class BlockMinter extends Thread { Block newBlock = Block.mint(repository, previousBlockData, mintingAccount); if (newBlock == null) { // For some reason we can't mint right now - moderatedLog(() -> LOGGER.error("Couldn't build a to-be-minted block")); + moderatedLog(() -> LOGGER.info("Couldn't build a to-be-minted block")); continue; } diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index fe6e3078..c4eadd44 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -702,7 +702,7 @@ public class OnlineAccountsManager { */ // Block::mint() - only wants online accounts with (online) timestamp that matches block's (online) timestamp so they can be added to new block public List getOnlineAccounts(long onlineTimestamp) { - LOGGER.info(String.format("caller's timestamp: %d, our timestamps: %s", onlineTimestamp, String.join(", ", this.currentOnlineAccounts.keySet().stream().map(l -> Long.toString(l)).collect(Collectors.joining(", "))))); + LOGGER.debug(String.format("caller's timestamp: %d, our timestamps: %s", onlineTimestamp, String.join(", ", this.currentOnlineAccounts.keySet().stream().map(l -> Long.toString(l)).collect(Collectors.joining(", "))))); return new ArrayList<>(Set.copyOf(this.currentOnlineAccounts.getOrDefault(onlineTimestamp, Collections.emptySet()))); } From e45ad37eb57c26445478937444833f2bd22b0b60 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Oct 2022 19:30:08 +0100 Subject: [PATCH 3/5] Fixed bug which could prevent invalid accounts being removed from the queue until the next valid one is added. --- src/main/java/org/qortal/controller/OnlineAccountsManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index c4eadd44..487a5253 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -214,8 +214,8 @@ public class OnlineAccountsManager { if (!onlineAccountsToAdd.isEmpty()) { LOGGER.debug("Merging {} validated online accounts from import queue", onlineAccountsToAdd.size()); addAccounts(onlineAccountsToAdd); - onlineAccountsImportQueue.removeAll(onlineAccountsToRemove); } + onlineAccountsImportQueue.removeAll(onlineAccountsToRemove); } } From b37aa749c6bc518e33d7a2917244c5ef2d4b3f99 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Oct 2022 19:34:24 +0100 Subject: [PATCH 4/5] Removed onlineAccountsMemPoWEnabled setting as it's no longer needed. --- .../java/org/qortal/controller/OnlineAccountsManager.java | 4 ++-- src/main/java/org/qortal/settings/Settings.java | 8 -------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 487a5253..45b47f5d 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -600,7 +600,7 @@ public class OnlineAccountsManager { // MemoryPoW private boolean isMemoryPoWActive(Long timestamp) { - if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp() || Settings.getInstance().isOnlineAccountsMemPoWEnabled()) { + if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) { return true; } return false; @@ -617,7 +617,7 @@ public class OnlineAccountsManager { private Integer computeMemoryPoW(byte[] bytes, byte[] publicKey, long onlineAccountsTimestamp) throws TimeoutException { if (!isMemoryPoWActive(NTP.getTime())) { - LOGGER.info("Mempow start timestamp not yet reached, and onlineAccountsMemPoWEnabled not enabled in settings"); + LOGGER.info("Mempow start timestamp not yet reached"); return null; } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 40b2a247..2e57142e 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -290,10 +290,6 @@ public class Settings { /** Additional offset added to values returned by NTP.getTime() */ private Long testNtpOffset = null; - // Online accounts - - /** Whether to opt-in to mempow computations for online accounts, ahead of general release */ - private boolean onlineAccountsMemPoWEnabled = false; /* Foreign chains */ @@ -800,10 +796,6 @@ public class Settings { return this.testNtpOffset; } - public boolean isOnlineAccountsMemPoWEnabled() { - return this.onlineAccountsMemPoWEnabled; - } - public long getRepositoryBackupInterval() { return this.repositoryBackupInterval; } From 1d5497e484849cf63f7662d5cc129ac4b23f4469 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 23 Oct 2022 14:13:38 +0100 Subject: [PATCH 5/5] Modifications to support a single node testnet: - Added "singleNodeTestnet" setting, allowing for fast and consecutive block minting, and no requirement for a minimum number of peers. - Added "recoveryModeTimeout" setting (previously hardcoded in Synchronizer). - Updated testnets documentation to include new settings and a quick start guide. - Added "generic" minting account that can be used in testnets (not functional on mainnet), to simplify the process for new devs. --- TestNets.md | 44 +++++++++++++++---- .../org/qortal/controller/BlockMinter.java | 10 +++-- .../org/qortal/controller/Controller.java | 4 ++ .../org/qortal/controller/Synchronizer.java | 7 ++- .../java/org/qortal/settings/Settings.java | 18 +++++++- 5 files changed, 67 insertions(+), 16 deletions(-) diff --git a/TestNets.md b/TestNets.md index e475e593..b4b9feed 100644 --- a/TestNets.md +++ b/TestNets.md @@ -52,14 +52,13 @@ ## Single-node testnet -A single-node testnet is possible with code modifications, for basic testing, or to more easily start a new testnet. -To do so, follow these steps: -- Comment out the `if (mintedLastBlock) { }` conditional in BlockMinter.java -- Comment out the `minBlockchainPeers` validation in Settings.validate() -- Set `minBlockchainPeers` to 0 in settings.json -- Set `Synchronizer.RECOVERY_MODE_TIMEOUT` to `0` -- All other steps should remain the same. Only a single reward share key is needed. -- Remember to put these values back after introducing other nodes +A single-node testnet is possible with an additional settings, or to more easily start a new testnet. +Just add this setting: +``` +"singleNodeTestnet": true +``` +This will automatically allow multiple consecutive blocks to be minted, as well as setting minBlockchainPeers to 0. +Remember to put these values back after introducing other nodes ## Fixed network @@ -93,3 +92,32 @@ Your options are: - `qort` tool, but prepend with one-time shell variable: `BASE_URL=some-node-hostname-or-ip:port qort ......` - `peer-heights`, but use `-t` option, or `BASE_URL` shell variable as above +## Example settings-test.json +``` +{ + "isTestNet": true, + "bitcoinNet": "TEST3", + "repositoryPath": "db-testnet", + "blockchainConfig": "testchain.json", + "minBlockchainPeers": 1, + "apiDocumentationEnabled": true, + "apiRestricted": false, + "bootstrap": false, + "maxPeerConnectionTime": 999999999, + "localAuthBypassEnabled": true, + "singleNodeTestnet": true, + "recoveryModeTimeout": 0 +} +``` + +## Quick start +Here are some steps to quickly get a single node testnet up and running with a generic minting account: +1. Start with template `settings-test.json`, and create a `testchain.json` based on mainnet's blockchain.json (or obtain one from Qortal developers). These should be in the same directory as the jar. +2. Make sure feature triggers and other timestamp/height activations are correctly set. Generally these would be `0` so that they are enabled from the start. +3. Set a recent genesis `timestamp` in testchain.json, and add this reward share entry: +`{ "type": "REWARD_SHARE", "minterPublicKey": "DwcUnhxjamqppgfXCLgbYRx8H9XFPUc2qYRy3CEvQWEw", "recipient": "QbTDMss7NtRxxQaSqBZtSLSNdSYgvGaqFf", "rewardSharePublicKey": "CRvQXxFfUMfr4q3o1PcUZPA4aPCiubBsXkk47GzRo754", "sharePercent": 0 },` +4. Start the node, passing in settings-test.json, e.g: `java -jar qortal.jar settings-test.json` +5. Once started, add the corresponding minting key to the node: +`curl -X POST "http://localhost:62391/admin/mintingaccounts" -d "F48mYJycFgRdqtc58kiovwbcJgVukjzRE4qRRtRsK9ix"` +6. Alternatively you can use your own minting account instead of the generic one above. +7. After a short while, blocks should be minted from the genesis timestamp until the current time. \ No newline at end of file diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 0734d4e9..7e3b4b9e 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -93,6 +93,8 @@ public class BlockMinter extends Thread { List newBlocks = new ArrayList<>(); + final boolean isSingleNodeTestnet = Settings.getInstance().isSingleNodeTestnet(); + try (final Repository repository = RepositoryManager.getRepository()) { // Going to need this a lot... BlockRepository blockRepository = repository.getBlockRepository(); @@ -111,8 +113,9 @@ public class BlockMinter extends Thread { // Free up any repository locks repository.discardChanges(); - // Sleep for a while - Thread.sleep(1000); + // Sleep for a while. + // It's faster on single node testnets, to allow lots of blocks to be minted quickly. + Thread.sleep(isSingleNodeTestnet ? 50 : 1000); isMintingPossible = false; @@ -223,9 +226,10 @@ public class BlockMinter extends Thread { List newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList()); // We might need to sit the next block out, if one of our minting accounts signed the previous one + // Skip this check for single node testnets, since they definitely need to mint every block byte[] previousBlockMinter = previousBlockData.getMinterPublicKey(); boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter)); - if (mintedLastBlock) { + if (mintedLastBlock && !isSingleNodeTestnet) { LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one")); continue; } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 12ad11a1..6fe6a159 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1872,6 +1872,10 @@ public class Controller extends Thread { if (latestBlockData == null || latestBlockData.getTimestamp() < minLatestBlockTimestamp) return false; + if (Settings.getInstance().isSingleNodeTestnet()) + // Single node testnets won't have peers, so we can assume up to date from this point + return true; + // Needs a mutable copy of the unmodifiableList List peers = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers()); if (peers == null) diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index a7dd38ff..6f2a0fe1 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -56,8 +56,6 @@ public class Synchronizer extends Thread { /** Maximum number of consecutive failed sync attempts before marking peer as misbehaved */ private static final int MAX_CONSECUTIVE_FAILED_SYNC_ATTEMPTS = 3; - private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms - private boolean running; @@ -399,9 +397,10 @@ public class Synchronizer extends Thread { timePeersLastAvailable = NTP.getTime(); // If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint - if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) { + long recoveryModeTimeout = Settings.getInstance().getRecoveryModeTimeout(); + if (NTP.getTime() - timePeersLastAvailable > recoveryModeTimeout) { if (recoveryMode == false) { - LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000)); + LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", recoveryModeTimeout/60/1000)); recoveryMode = true; } } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 2e57142e..acfd0e78 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -184,6 +184,8 @@ public class Settings { // Peer-to-peer related private boolean isTestNet = false; + /** Single node testnet mode */ + private boolean singleNodeTestnet = false; /** Port number for inbound peer-to-peer connections. */ private Integer listenPort; /** Whether to attempt to open the listen port via UPnP */ @@ -203,6 +205,9 @@ public class Settings { /** Maximum number of retry attempts if a peer fails to respond with the requested data */ private int maxRetries = 2; + /** The number of seconds of no activity before recovery mode begins */ + public long recoveryModeTimeout = 10 * 60 * 1000L; + /** Minimum peer version number required in order to sync with them */ private String minPeerVersion = "3.6.3"; /** Whether to allow connections with peers below minPeerVersion @@ -486,7 +491,7 @@ public class Settings { private void validate() { // Validation goes here - if (this.minBlockchainPeers < 1) + if (this.minBlockchainPeers < 1 && !singleNodeTestnet) throwValidationError("minBlockchainPeers must be at least 1"); if (this.apiKey != null && this.apiKey.trim().length() < 8) @@ -643,6 +648,10 @@ public class Settings { return this.isTestNet; } + public boolean isSingleNodeTestnet() { + return this.singleNodeTestnet; + } + public int getListenPort() { if (this.listenPort != null) return this.listenPort; @@ -663,6 +672,9 @@ public class Settings { } public int getMinBlockchainPeers() { + if (singleNodeTestnet) + return 0; + return this.minBlockchainPeers; } @@ -688,6 +700,10 @@ public class Settings { public int getMaxRetries() { return this.maxRetries; } + public long getRecoveryModeTimeout() { + return recoveryModeTimeout; + } + public String getMinPeerVersion() { return this.minPeerVersion; } public boolean getAllowConnectionsWithOlderPeerVersions() { return this.allowConnectionsWithOlderPeerVersions; }