mirror of
https://github.com/Qortal/qortal.git
synced 2025-07-30 13:41:23 +00:00
Compare commits
55 Commits
block-rewa
...
prioritize
Author | SHA1 | Date | |
---|---|---|---|
|
b37f2c7d7f | ||
|
0c0c5ff077 | ||
|
e12b99d17e | ||
|
d599146c3a | ||
|
476731a2c3 | ||
|
1e491dd8fb | ||
|
ba6397b963 | ||
|
3146da6aec | ||
|
5643e57ede | ||
|
f532dbe7b4 | ||
|
ec2af62b4d | ||
|
423142d730 | ||
|
bdddb526da | ||
|
dbf1ed40b3 | ||
|
02ace06526 | ||
|
2d2bfc0a4c | ||
|
3c22a12cbb | ||
|
3071ef2f36 | ||
|
3022cb22d6 | ||
|
e9b4a3f6b3 | ||
|
4312ebfcc3 | ||
|
2c0e099d1c | ||
|
b1eb02eb1d | ||
|
c919797553 | ||
|
08dacab05c | ||
|
2efc9218df | ||
|
41505dae11 | ||
|
44ec447014 | ||
|
98308ecf98 | ||
|
8d613a6472 | ||
|
c3e5298ecd | ||
|
e89d31eb5a | ||
|
30160e2843 | ||
|
503d22e4d0 | ||
|
b9a0d489d7 | ||
|
d9d4c4c302 | ||
|
81c6d75d62 | ||
|
d1419bdfbd | ||
|
8566d9b7e5 | ||
|
b319d6db6b | ||
|
35fd1d8455 | ||
|
be21771e49 | ||
|
745528a9b1 | ||
|
f1422af95b | ||
|
f92f4dc1e2 | ||
|
019cfdc1db | ||
|
e694a51cdd | ||
|
22e3140ff0 | ||
|
4824c4198b | ||
|
ec7d4f4498 | ||
|
d635de44a8 | ||
|
bce66bf57f | ||
|
0fc5153f9b | ||
|
0398c2fae1 | ||
|
5fc495eb6a |
@@ -19,10 +19,10 @@
|
||||
<ROW Property="Manufacturer" Value="Qortal"/>
|
||||
<ROW Property="MsiLogging" MultiBuildValue="DefaultBuild:vp"/>
|
||||
<ROW Property="NTP_GOOD" Value="false"/>
|
||||
<ROW Property="ProductCode" Value="1033:{9EA6B58D-641E-442E-8F16-5D35B92B9F9B} 1049:{B16722F6-C2FA-418D-A9DA-69707FE2034B} 2052:{459AC873-98DC-43AC-8787-B23BAF976FF5} 2057:{CD923B63-65A0-4F2A-93B6-6362AAC8608E} " Type="16"/>
|
||||
<ROW Property="ProductCode" Value="1033:{DE0C17EF-F9D2-48A3-A0D7-FF7BB01698A2} 1049:{89129011-9CE7-44AD-97EC-152371A87D58} 2052:{C93D5260-C6BD-4341-A96D-7D878EE6D59B} 2057:{AE6EDB44-3120-4588-8C9A-558F1BE42D73} " Type="16"/>
|
||||
<ROW Property="ProductLanguage" Value="2057"/>
|
||||
<ROW Property="ProductName" Value="Qortal"/>
|
||||
<ROW Property="ProductVersion" Value="1.4.2" Type="32"/>
|
||||
<ROW Property="ProductVersion" Value="1.4.5" Type="32"/>
|
||||
<ROW Property="RECONFIG_NTP" Value="true"/>
|
||||
<ROW Property="REMOVE_BLOCKCHAIN" Value="YES" Type="4"/>
|
||||
<ROW Property="REPAIR_BLOCKCHAIN" Value="YES" Type="4"/>
|
||||
@@ -174,7 +174,7 @@
|
||||
<ROW Component="ADDITIONAL_LICENSE_INFO_97" ComponentId="{D5544706-E2A7-424F-AEA5-3963E355AA29}" Directory_="jdk.crypto.mscapi_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_97" Type="0"/>
|
||||
<ROW Component="ADDITIONAL_LICENSE_INFO_98" ComponentId="{104DBCE8-A458-4B3E-9EFA-2D8613561619}" Directory_="jdk.dynalink_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_98" Type="0"/>
|
||||
<ROW Component="ADDITIONAL_LICENSE_INFO_99" ComponentId="{D02E3C37-E81A-48FA-9E28-B26B728AECD9}" Directory_="jdk.httpserver_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_99" Type="0"/>
|
||||
<ROW Component="AI_CustomARPName" ComponentId="{F9375D31-26C0-4E23-948D-3570B43B7FA2}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
|
||||
<ROW Component="AI_CustomARPName" ComponentId="{CFB6FB43-13A6-4A4F-BCF9-7698463CE032}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
|
||||
<ROW Component="AI_ExePath" ComponentId="{3644948D-AE0B-41BB-9FAF-A79E70490A08}" Directory_="APPDIR" Attributes="260" KeyPath="AI_ExePath"/>
|
||||
<ROW Component="APPDIR" ComponentId="{680DFDDE-3FB4-47A5-8FF5-934F576C6F91}" Directory_="APPDIR" Attributes="0"/>
|
||||
<ROW Component="DATA_PATH" ComponentId="{EE0B6107-E244-4CDB-B195-E9038D2F1E0E}" Directory_="DATA_PATH" Attributes="0"/>
|
||||
|
2
pom.xml
2
pom.xml
@@ -3,7 +3,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>org.qortal</groupId>
|
||||
<artifactId>qortal</artifactId>
|
||||
<version>1.4.3</version>
|
||||
<version>1.4.6</version>
|
||||
<packaging>jar</packaging>
|
||||
<properties>
|
||||
<skipTests>true</skipTests>
|
||||
|
@@ -547,7 +547,7 @@ public class AdminResource {
|
||||
blockchainLock.lockInterruptibly();
|
||||
|
||||
try {
|
||||
repository.exportNodeLocalData();
|
||||
repository.exportNodeLocalData(true);
|
||||
return "true";
|
||||
} finally {
|
||||
blockchainLock.unlock();
|
||||
|
@@ -135,16 +135,19 @@ public class BlockMinter extends Thread {
|
||||
// Disregard peers that have "misbehaved" recently
|
||||
peers.removeIf(Controller.hasMisbehaved);
|
||||
|
||||
// Disregard peers that don't have a recent block
|
||||
peers.removeIf(Controller.hasNoRecentBlock);
|
||||
// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
|
||||
// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
|
||||
if (Controller.getInstance().getRecoveryMode() == false)
|
||||
peers.removeIf(Controller.hasNoRecentBlock);
|
||||
|
||||
// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
|
||||
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
|
||||
continue;
|
||||
|
||||
// If our latest block isn't recent then we need to synchronize instead of minting.
|
||||
// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
|
||||
if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
|
||||
continue;
|
||||
if (Controller.getInstance().getRecoveryMode() == false)
|
||||
continue;
|
||||
|
||||
// There are enough peers with a recent block and our latest block is recent
|
||||
// so go ahead and mint a block if possible.
|
||||
@@ -165,6 +168,14 @@ public class BlockMinter extends Thread {
|
||||
// Do we need to build any potential new blocks?
|
||||
List<PrivateKeyAccount> newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
|
||||
|
||||
// We might need to sit the next block out, if one of our minting accounts signed the previous one
|
||||
final byte[] previousBlockMinter = previousBlockData.getMinterPublicKey();
|
||||
final boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter));
|
||||
if (mintedLastBlock) {
|
||||
LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one"));
|
||||
continue;
|
||||
}
|
||||
|
||||
for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
|
||||
// First block does the AT heavy-lifting
|
||||
if (newBlocks.isEmpty()) {
|
||||
@@ -282,15 +293,17 @@ public class BlockMinter extends Thread {
|
||||
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey());
|
||||
|
||||
if (rewardShareData != null) {
|
||||
LOGGER.info(String.format("Minted block %d, sig %.8s by %s on behalf of %s",
|
||||
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s",
|
||||
newBlock.getBlockData().getHeight(),
|
||||
Base58.encode(newBlock.getBlockData().getSignature()),
|
||||
Base58.encode(newBlock.getParent().getSignature()),
|
||||
rewardShareData.getMinter(),
|
||||
rewardShareData.getRecipient()));
|
||||
} else {
|
||||
LOGGER.info(String.format("Minted block %d, sig %.8s by %s",
|
||||
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s",
|
||||
newBlock.getBlockData().getHeight(),
|
||||
Base58.encode(newBlock.getBlockData().getSignature()),
|
||||
Base58.encode(newBlock.getParent().getSignature()),
|
||||
newBlock.getMinter().getAddress()));
|
||||
}
|
||||
|
||||
|
@@ -67,8 +67,8 @@ import org.qortal.gui.SysTray;
|
||||
import org.qortal.network.Network;
|
||||
import org.qortal.network.Peer;
|
||||
import org.qortal.network.message.ArbitraryDataMessage;
|
||||
import org.qortal.network.message.BlockMessage;
|
||||
import org.qortal.network.message.BlockSummariesMessage;
|
||||
import org.qortal.network.message.CachedBlockMessage;
|
||||
import org.qortal.network.message.GetArbitraryDataMessage;
|
||||
import org.qortal.network.message.GetBlockMessage;
|
||||
import org.qortal.network.message.GetBlockSummariesMessage;
|
||||
@@ -121,6 +121,7 @@ public class Controller extends Thread {
|
||||
private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms
|
||||
private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms
|
||||
private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms
|
||||
private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms
|
||||
|
||||
// To do with online accounts list
|
||||
private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms
|
||||
@@ -148,9 +149,9 @@ public class Controller extends Thread {
|
||||
|
||||
/** Cache of BlockMessages, indexed by block signature */
|
||||
@SuppressWarnings("serial")
|
||||
private final LinkedHashMap<ByteArray, BlockMessage> blockMessageCache = new LinkedHashMap<>() {
|
||||
private final LinkedHashMap<ByteArray, CachedBlockMessage> blockMessageCache = new LinkedHashMap<>() {
|
||||
@Override
|
||||
protected boolean removeEldestEntry(Map.Entry<ByteArray, BlockMessage> eldest) {
|
||||
protected boolean removeEldestEntry(Map.Entry<ByteArray, CachedBlockMessage> eldest) {
|
||||
return this.size() > Settings.getInstance().getBlockCacheSize();
|
||||
}
|
||||
};
|
||||
@@ -175,6 +176,11 @@ public class Controller extends Thread {
|
||||
/** Latest block signatures from other peers that we know are on inferior chains. */
|
||||
List<ByteArray> inferiorChainSignatures = new ArrayList<>();
|
||||
|
||||
/** Recovery mode, which is used to bring back a stalled network */
|
||||
private boolean recoveryMode = false;
|
||||
private boolean peersAvailable = true; // peersAvailable must default to true
|
||||
private long timePeersLastAvailable = 0;
|
||||
|
||||
/**
|
||||
* Map of recent requests for ARBITRARY transaction data payloads.
|
||||
* <p>
|
||||
@@ -358,6 +364,10 @@ public class Controller extends Thread {
|
||||
}
|
||||
}
|
||||
|
||||
public boolean getRecoveryMode() {
|
||||
return this.recoveryMode;
|
||||
}
|
||||
|
||||
// Entry point
|
||||
|
||||
public static void main(String[] args) {
|
||||
@@ -629,6 +639,13 @@ public class Controller extends Thread {
|
||||
// Disregard peers that don't have a recent block
|
||||
peers.removeIf(hasNoRecentBlock);
|
||||
|
||||
checkRecoveryModeForPeers(peers);
|
||||
if (recoveryMode) {
|
||||
peers = Network.getInstance().getHandshakedPeers();
|
||||
peers.removeIf(hasOnlyGenesisBlock);
|
||||
peers.removeIf(hasMisbehaved);
|
||||
}
|
||||
|
||||
// Check we have enough peers to potentially synchronize
|
||||
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
|
||||
return;
|
||||
@@ -639,9 +656,31 @@ public class Controller extends Thread {
|
||||
// Disregard peers that are on the same block as last sync attempt and we didn't like their chain
|
||||
peers.removeIf(hasInferiorChainTip);
|
||||
|
||||
final int peersBeforeComparison = peers.size();
|
||||
|
||||
// Request recent block summaries from the remaining peers, and locate our common block with each
|
||||
Synchronizer.getInstance().findCommonBlocksWithPeers(peers);
|
||||
|
||||
// Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks
|
||||
peers = Synchronizer.getInstance().comparePeers(peers);
|
||||
|
||||
// We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains
|
||||
peers.removeIf(hasInferiorChainTip);
|
||||
|
||||
final int peersRemoved = peersBeforeComparison - peers.size();
|
||||
if (peersRemoved > 0)
|
||||
LOGGER.info(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size()));
|
||||
|
||||
if (peers.isEmpty())
|
||||
return;
|
||||
|
||||
if (peers.size() > 1) {
|
||||
StringBuilder finalPeersString = new StringBuilder();
|
||||
for (Peer peer : peers)
|
||||
finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer);
|
||||
LOGGER.info(String.format("Choosing random peer from: [%s]", finalPeersString.toString()));
|
||||
}
|
||||
|
||||
// Pick random peer to sync with
|
||||
int index = new SecureRandom().nextInt(peers.size());
|
||||
Peer peer = peers.get(index);
|
||||
@@ -744,6 +783,46 @@ public class Controller extends Thread {
|
||||
}
|
||||
}
|
||||
|
||||
private boolean checkRecoveryModeForPeers(List<Peer> qualifiedPeers) {
|
||||
List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
|
||||
|
||||
if (handshakedPeers.size() > 0) {
|
||||
// There is at least one handshaked peer
|
||||
if (qualifiedPeers.isEmpty()) {
|
||||
// There are no 'qualified' peers - i.e. peers that have a recent block we can sync to
|
||||
boolean werePeersAvailable = peersAvailable;
|
||||
peersAvailable = false;
|
||||
|
||||
// If peers only just became unavailable, update our record of the time they were last available
|
||||
if (werePeersAvailable)
|
||||
timePeersLastAvailable = NTP.getTime();
|
||||
|
||||
// If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint
|
||||
if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) {
|
||||
if (recoveryMode == false) {
|
||||
LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000));
|
||||
recoveryMode = true;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// We now have at least one peer with a recent block, so we can exit recovery mode and sync normally
|
||||
peersAvailable = true;
|
||||
if (recoveryMode) {
|
||||
LOGGER.info("Peers have become available again. Exiting recovery mode...");
|
||||
recoveryMode = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
return recoveryMode;
|
||||
}
|
||||
|
||||
public void addInferiorChainSignature(byte[] inferiorSignature) {
|
||||
// Update our list of inferior chain tips
|
||||
ByteArray inferiorChainSignature = new ByteArray(inferiorSignature);
|
||||
if (!inferiorChainSignatures.contains(inferiorChainSignature))
|
||||
inferiorChainSignatures.add(inferiorChainSignature);
|
||||
}
|
||||
|
||||
public static class StatusChangeEvent implements Event {
|
||||
public StatusChangeEvent() {
|
||||
}
|
||||
@@ -775,7 +854,7 @@ public class Controller extends Thread {
|
||||
actionText = Translator.INSTANCE.translate("SysTray", "MINTING_DISABLED");
|
||||
}
|
||||
|
||||
String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height);
|
||||
String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height) + "\n" + String.format("Build version: %s", this.buildVersion);
|
||||
SysTray.getInstance().setToolTipText(tooltip);
|
||||
|
||||
this.callbackExecutor.execute(() -> {
|
||||
@@ -1151,7 +1230,7 @@ public class Controller extends Thread {
|
||||
|
||||
ByteArray signatureAsByteArray = new ByteArray(signature);
|
||||
|
||||
BlockMessage cachedBlockMessage = this.blockMessageCache.get(signatureAsByteArray);
|
||||
CachedBlockMessage cachedBlockMessage = this.blockMessageCache.get(signatureAsByteArray);
|
||||
int blockCacheSize = Settings.getInstance().getBlockCacheSize();
|
||||
|
||||
// Check cached latest block message
|
||||
@@ -1159,7 +1238,7 @@ public class Controller extends Thread {
|
||||
this.stats.getBlockMessageStats.cacheHits.incrementAndGet();
|
||||
|
||||
// We need to duplicate it to prevent multiple threads setting ID on the same message
|
||||
BlockMessage clonedBlockMessage = cachedBlockMessage.cloneWithNewId(message.getId());
|
||||
CachedBlockMessage clonedBlockMessage = cachedBlockMessage.cloneWithNewId(message.getId());
|
||||
|
||||
if (!peer.sendMessage(clonedBlockMessage))
|
||||
peer.disconnect("failed to send block");
|
||||
@@ -1187,12 +1266,15 @@ public class Controller extends Thread {
|
||||
|
||||
Block block = new Block(repository, blockData);
|
||||
|
||||
BlockMessage blockMessage = new BlockMessage(block);
|
||||
CachedBlockMessage blockMessage = new CachedBlockMessage(block);
|
||||
blockMessage.setId(message.getId());
|
||||
|
||||
// This call also causes the other needed data to be pulled in from repository
|
||||
if (!peer.sendMessage(blockMessage))
|
||||
if (!peer.sendMessage(blockMessage)) {
|
||||
peer.disconnect("failed to send block");
|
||||
// Don't fall-through to caching because failure to send might be from failure to build message
|
||||
return;
|
||||
}
|
||||
|
||||
// If request is for a recent block, cache it
|
||||
if (getChainHeight() - blockData.getHeight() <= blockCacheSize) {
|
||||
@@ -1209,6 +1291,18 @@ public class Controller extends Thread {
|
||||
TransactionMessage transactionMessage = (TransactionMessage) message;
|
||||
TransactionData transactionData = transactionMessage.getTransactionData();
|
||||
|
||||
/*
|
||||
* If we can't obtain blockchain lock immediately,
|
||||
* e.g. Synchronizer is active, or another transaction is taking a while to validate,
|
||||
* then we're using up a network thread for ages and clogging things up
|
||||
* so bail out early
|
||||
*/
|
||||
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
|
||||
if (!blockchainLock.tryLock()) {
|
||||
LOGGER.trace(() -> String.format("Too busy to import %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer));
|
||||
return;
|
||||
}
|
||||
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
Transaction transaction = Transaction.fromData(repository, transactionData);
|
||||
|
||||
@@ -1238,6 +1332,8 @@ public class Controller extends Thread {
|
||||
LOGGER.debug(() -> String.format("Imported %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer));
|
||||
} catch (DataException e) {
|
||||
LOGGER.error(String.format("Repository issue while processing transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer), e);
|
||||
} finally {
|
||||
blockchainLock.unlock();
|
||||
}
|
||||
}
|
||||
|
||||
|
@@ -8,6 +8,7 @@ import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.Iterator;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
@@ -17,6 +18,7 @@ import org.qortal.block.Block;
|
||||
import org.qortal.block.Block.ValidationResult;
|
||||
import org.qortal.data.block.BlockData;
|
||||
import org.qortal.data.block.BlockSummaryData;
|
||||
import org.qortal.data.block.CommonBlockData;
|
||||
import org.qortal.data.network.PeerChainTipData;
|
||||
import org.qortal.data.transaction.RewardShareTransactionData;
|
||||
import org.qortal.data.transaction.TransactionData;
|
||||
@@ -39,10 +41,23 @@ public class Synchronizer {
|
||||
|
||||
private static final Logger LOGGER = LogManager.getLogger(Synchronizer.class);
|
||||
|
||||
/** Max number of new blocks we aim to add to chain tip in each sync round */
|
||||
private static final int SYNC_BATCH_SIZE = 200; // XXX move to Settings?
|
||||
|
||||
/** Initial jump back of block height when searching for common block with peer */
|
||||
private static final int INITIAL_BLOCK_STEP = 8;
|
||||
private static final int MAXIMUM_BLOCK_STEP = 500;
|
||||
/** Maximum jump back of block height when searching for common block with peer */
|
||||
private static final int MAXIMUM_BLOCK_STEP = 128;
|
||||
|
||||
/** Maximum difference in block height between tip and peer's common block before peer is considered TOO DIVERGENT */
|
||||
private static final int MAXIMUM_COMMON_DELTA = 240; // XXX move to Settings?
|
||||
private static final int SYNC_BATCH_SIZE = 200;
|
||||
|
||||
/** Maximum number of block signatures we ask from peer in one go */
|
||||
private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings?
|
||||
|
||||
/** Number of retry attempts if a peer fails to respond with the requested data */
|
||||
private static final int MAXIMUM_RETRIES = 2; // XXX move to Settings?
|
||||
|
||||
|
||||
private static Synchronizer instance;
|
||||
|
||||
@@ -62,6 +77,368 @@ public class Synchronizer {
|
||||
return instance;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Iterate through a list of supplied peers, and attempt to find our common block with each.
|
||||
* If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later.
|
||||
* <p>
|
||||
* Will return <tt>SynchronizationResult.OK</tt> on success.
|
||||
* <p>
|
||||
* @param peers
|
||||
* @return SynchronizationResult.OK if the process completed successfully, or a different SynchronizationResult if something went wrong.
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public SynchronizationResult findCommonBlocksWithPeers(List<Peer> peers) throws InterruptedException {
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
try {
|
||||
|
||||
if (peers.size() == 0)
|
||||
return SynchronizationResult.NOTHING_TO_DO;
|
||||
|
||||
// If our latest block is very old, it's best that we don't try and determine the best peers to sync to.
|
||||
// This is because it can involve very large chain comparisons, which is too intensive.
|
||||
// In reality, most forking problems occur near the chain tips, so we will reserve this functionality for those situations.
|
||||
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
|
||||
if (minLatestBlockTimestamp == null)
|
||||
return SynchronizationResult.REPOSITORY_ISSUE;
|
||||
|
||||
final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
|
||||
if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
|
||||
LOGGER.debug(String.format("Our latest block is very old, so we won't collect common block info from peers"));
|
||||
return SynchronizationResult.NOTHING_TO_DO;
|
||||
}
|
||||
|
||||
LOGGER.debug(String.format("Searching for common blocks with %d peers...", peers.size()));
|
||||
final long startTime = System.currentTimeMillis();
|
||||
int commonBlocksFound = 0;
|
||||
|
||||
for (Peer peer : peers) {
|
||||
// Are we shutting down?
|
||||
if (Controller.isStopping())
|
||||
return SynchronizationResult.SHUTTING_DOWN;
|
||||
|
||||
// Check if we can use the cached common block data, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block
|
||||
if (peer.canUseCachedCommonBlockData()) {
|
||||
LOGGER.debug(String.format("Skipping peer %s because we already have the latest common block data in our cache. Cached common block sig is %.08s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature())));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Cached data is stale, so clear it and repopulate
|
||||
peer.setCommonBlockData(null);
|
||||
|
||||
// Search for the common block
|
||||
Synchronizer.getInstance().findCommonBlockWithPeer(peer, repository);
|
||||
if (peer.getCommonBlockData() != null)
|
||||
commonBlocksFound++;
|
||||
}
|
||||
|
||||
final long totalTimeTaken = System.currentTimeMillis() - startTime;
|
||||
LOGGER.info(String.format("Finished searching for common blocks with %d peer%s. Found: %d. Total time taken: %d ms", peers.size(), (peers.size() != 1 ? "s" : ""), commonBlocksFound, totalTimeTaken));
|
||||
|
||||
return SynchronizationResult.OK;
|
||||
} finally {
|
||||
repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong
|
||||
}
|
||||
} catch (DataException e) {
|
||||
LOGGER.error("Repository issue during synchronization with peer", e);
|
||||
return SynchronizationResult.REPOSITORY_ISSUE;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to find the find our common block with supplied peer.
|
||||
* If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later.
|
||||
* <p>
|
||||
* Will return <tt>SynchronizationResult.OK</tt> on success.
|
||||
* <p>
|
||||
* @param peer
|
||||
* @param repository
|
||||
* @return SynchronizationResult.OK if the process completed successfully, or a different SynchronizationResult if something went wrong.
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public SynchronizationResult findCommonBlockWithPeer(Peer peer, Repository repository) throws InterruptedException {
|
||||
try {
|
||||
final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
|
||||
final int ourInitialHeight = ourLatestBlockData.getHeight();
|
||||
|
||||
PeerChainTipData peerChainTipData = peer.getChainTipData();
|
||||
int peerHeight = peerChainTipData.getLastHeight();
|
||||
byte[] peersLastBlockSignature = peerChainTipData.getLastBlockSignature();
|
||||
|
||||
byte[] ourLastBlockSignature = ourLatestBlockData.getSignature();
|
||||
LOGGER.debug(String.format("Fetching summaries from peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer,
|
||||
peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(),
|
||||
ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp()));
|
||||
|
||||
List<BlockSummaryData> peerBlockSummaries = new ArrayList<>();
|
||||
SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, false, peerBlockSummaries);
|
||||
if (findCommonBlockResult != SynchronizationResult.OK) {
|
||||
// Logging performed by fetchSummariesFromCommonBlock() above
|
||||
peer.setCommonBlockData(null);
|
||||
return findCommonBlockResult;
|
||||
}
|
||||
|
||||
// First summary is common block
|
||||
final BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature());
|
||||
final BlockSummaryData commonBlockSummary = new BlockSummaryData(commonBlockData);
|
||||
final int commonBlockHeight = commonBlockData.getHeight();
|
||||
final byte[] commonBlockSig = commonBlockData.getSignature();
|
||||
final String commonBlockSig58 = Base58.encode(commonBlockSig);
|
||||
LOGGER.debug(String.format("Common block with peer %s is at height %d, sig %.8s, ts %d", peer,
|
||||
commonBlockHeight, commonBlockSig58, commonBlockData.getTimestamp()));
|
||||
peerBlockSummaries.remove(0);
|
||||
|
||||
// Store the common block summary against the peer, and the current chain tip (for caching)
|
||||
peer.setCommonBlockData(new CommonBlockData(commonBlockSummary, peerChainTipData));
|
||||
|
||||
return SynchronizationResult.OK;
|
||||
} catch (DataException e) {
|
||||
LOGGER.error("Repository issue during synchronization with peer", e);
|
||||
return SynchronizationResult.REPOSITORY_ISSUE;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Compare a list of peers to determine the best peer(s) to sync to next.
|
||||
* <p>
|
||||
* Will return a filtered list of peers on success, or an identical list of peers on failure.
|
||||
* This allows us to fall back to legacy behaviour (random selection from the entire list of peers), if we are unable to make the comparison.
|
||||
* <p>
|
||||
* @param peers
|
||||
* @return a list of peers, possibly filtered.
|
||||
* @throws InterruptedException
|
||||
*/
|
||||
public List<Peer> comparePeers(List<Peer> peers) throws InterruptedException {
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
try {
|
||||
|
||||
// If our latest block is very old, it's best that we don't try and determine the best peers to sync to.
|
||||
// This is because it can involve very large chain comparisons, which is too intensive.
|
||||
// In reality, most forking problems occur near the chain tips, so we will reserve this functionality for those situations.
|
||||
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
|
||||
if (minLatestBlockTimestamp == null)
|
||||
return peers;
|
||||
|
||||
final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
|
||||
if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
|
||||
LOGGER.debug(String.format("Our latest block is very old, so we won't filter the peers list"));
|
||||
return peers;
|
||||
}
|
||||
|
||||
// Retrieve a list of unique common blocks from this list of peers
|
||||
List<BlockSummaryData> commonBlocks = this.uniqueCommonBlocks(peers);
|
||||
|
||||
// Order common blocks by height, in ascending order
|
||||
// This is essential for the logic below to make the correct decisions when discarding chains - do not remove
|
||||
commonBlocks.sort((b1, b2) -> Integer.valueOf(b1.getHeight()).compareTo(Integer.valueOf(b2.getHeight())));
|
||||
|
||||
// Get our latest height
|
||||
final int ourHeight = ourLatestBlockData.getHeight();
|
||||
|
||||
// Create a placeholder to track of common blocks that we can discard due to being inferior chains
|
||||
int dropPeersAfterCommonBlockHeight = 0;
|
||||
|
||||
// Remove peers with no common block data
|
||||
Iterator iterator = peers.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
Peer peer = (Peer) iterator.next();
|
||||
if (peer.getCommonBlockData() == null) {
|
||||
LOGGER.debug(String.format("Removed peer %s because it has no common block data", peer));
|
||||
iterator.remove();
|
||||
}
|
||||
}
|
||||
|
||||
// Loop through each group of common blocks
|
||||
for (BlockSummaryData commonBlockSummary : commonBlocks) {
|
||||
List<Peer> peersSharingCommonBlock = peers.stream().filter(peer -> peer.getCommonBlockData().getCommonBlockSummary().equals(commonBlockSummary)).collect(Collectors.toList());
|
||||
|
||||
// Check if we need to discard this group of peers
|
||||
if (dropPeersAfterCommonBlockHeight > 0) {
|
||||
if (commonBlockSummary.getHeight() > dropPeersAfterCommonBlockHeight) {
|
||||
// We have already determined that the correct chain diverged from a lower height. We are safe to skip these peers.
|
||||
for (Peer peer : peersSharingCommonBlock) {
|
||||
LOGGER.debug(String.format("Peer %s has common block at height %d but the superior chain is at height %d. Removing it from this round.", peer, commonBlockSummary.getHeight(), dropPeersAfterCommonBlockHeight));
|
||||
Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate the length of the shortest peer chain sharing this common block, including our chain
|
||||
final int ourAdditionalBlocksAfterCommonBlock = ourHeight - commonBlockSummary.getHeight();
|
||||
int minChainLength = this.calculateMinChainLength(commonBlockSummary, ourAdditionalBlocksAfterCommonBlock, peersSharingCommonBlock);
|
||||
|
||||
// Fetch block summaries from each peer
|
||||
for (Peer peer : peersSharingCommonBlock) {
|
||||
|
||||
// If we're shutting down, just return the latest peer list
|
||||
if (Controller.isStopping())
|
||||
return peers;
|
||||
|
||||
// Count the number of blocks this peer has beyond our common block
|
||||
final int peerHeight = peer.getChainTipData().getLastHeight();
|
||||
final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight();
|
||||
// Limit the number of blocks we are comparing. FUTURE: we could request more in batches, but there may not be a case when this is needed
|
||||
int summariesRequired = Math.min(peerAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE);
|
||||
|
||||
// Check if we can use the cached common block summaries, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block
|
||||
boolean useCachedSummaries = false;
|
||||
if (peer.canUseCachedCommonBlockData()) {
|
||||
if (peer.getCommonBlockData().getBlockSummariesAfterCommonBlock() != null) {
|
||||
if (peer.getCommonBlockData().getBlockSummariesAfterCommonBlock().size() == summariesRequired) {
|
||||
LOGGER.debug(String.format("Using cached block summaries for peer %s", peer));
|
||||
useCachedSummaries = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (useCachedSummaries == false) {
|
||||
if (summariesRequired > 0) {
|
||||
LOGGER.trace(String.format("Requesting %d block summar%s from peer %s after common block %.8s. Peer height: %d", summariesRequired, (summariesRequired != 1 ? "ies" : "y"), peer, Base58.encode(commonBlockSummary.getSignature()), peerHeight));
|
||||
|
||||
List<BlockSummaryData> blockSummaries = this.getBlockSummaries(peer, commonBlockSummary.getSignature(), summariesRequired);
|
||||
peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(blockSummaries);
|
||||
|
||||
if (blockSummaries != null) {
|
||||
LOGGER.trace(String.format("Peer %s returned %d block summar%s", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y")));
|
||||
|
||||
// We need to adjust minChainLength if peers fail to return all expected block summaries
|
||||
if (blockSummaries.size() < summariesRequired) {
|
||||
// This could mean that the peer has re-orged. But we still have the same common block, so it's safe to proceed with this set of signatures instead.
|
||||
LOGGER.debug(String.format("Peer %s returned %d block summar%s instead of expected %d", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"), summariesRequired));
|
||||
|
||||
// Update minChainLength if we have at least 1 block for this peer. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength
|
||||
if (blockSummaries.size() > 0)
|
||||
minChainLength = blockSummaries.size();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// There are no block summaries after this common block
|
||||
peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch our corresponding block summaries. Limit to MAXIMUM_REQUEST_SIZE, in order to make the comparison fairer, as peers have been limited too
|
||||
final int ourSummariesRequired = Math.min(ourAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE);
|
||||
LOGGER.trace(String.format("About to fetch our block summaries from %d to %d. Our height: %d", commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired, ourHeight));
|
||||
List<BlockSummaryData> ourBlockSummaries = repository.getBlockRepository().getBlockSummaries(commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired);
|
||||
if (ourBlockSummaries.isEmpty())
|
||||
LOGGER.debug(String.format("We don't have any block summaries so can't compare our chain against peers with this common block. We can still compare them against each other."));
|
||||
else
|
||||
populateBlockSummariesMinterLevels(repository, ourBlockSummaries);
|
||||
|
||||
// Create array to hold peers for comparison
|
||||
List<Peer> superiorPeersForComparison = new ArrayList<>();
|
||||
|
||||
// Calculate our chain weight
|
||||
BigInteger ourChainWeight = BigInteger.valueOf(0);
|
||||
if (ourBlockSummaries.size() > 0)
|
||||
ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, minChainLength);
|
||||
|
||||
NumberFormat formatter = new DecimalFormat("0.###E0");
|
||||
NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
|
||||
LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", ourBlockSummaries.size(), formatter.format(ourChainWeight)));
|
||||
|
||||
LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature())));
|
||||
for (Peer peer : peersSharingCommonBlock) {
|
||||
final int peerHeight = peer.getChainTipData().getLastHeight();
|
||||
final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight();
|
||||
final CommonBlockData peerCommonBlockData = peer.getCommonBlockData();
|
||||
|
||||
if (peerCommonBlockData == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock() == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock().isEmpty()) {
|
||||
// No response - remove this peer for now
|
||||
LOGGER.debug(String.format("Peer %s doesn't have any block summaries - removing it from this round", peer));
|
||||
peers.remove(peer);
|
||||
continue;
|
||||
}
|
||||
|
||||
final List<BlockSummaryData> peerBlockSummariesAfterCommonBlock = peerCommonBlockData.getBlockSummariesAfterCommonBlock();
|
||||
populateBlockSummariesMinterLevels(repository, peerBlockSummariesAfterCommonBlock);
|
||||
|
||||
// Calculate cumulative chain weight of this blockchain subset, from common block to highest mutual block held by all peers in this group.
|
||||
LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", peerBlockSummariesAfterCommonBlock.size(), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock));
|
||||
BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, minChainLength);
|
||||
peer.getCommonBlockData().setChainWeight(peerChainWeight);
|
||||
LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, peerBlockSummariesAfterCommonBlock.size(), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight)));
|
||||
|
||||
// Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group)
|
||||
if (ourChainWeight.compareTo(peerChainWeight) > 0) {
|
||||
// This peer is on an inferior chain - remove it
|
||||
LOGGER.debug(String.format("Peer %s is on an inferior chain to us - removing it from this round", peer));
|
||||
peers.remove(peer);
|
||||
}
|
||||
else {
|
||||
// Our chain is inferior
|
||||
LOGGER.debug(String.format("Peer %s is on a better chain to us. We will compare the other peers sharing this common block against each other, and drop all peers sharing higher common blocks.", peer));
|
||||
dropPeersAfterCommonBlockHeight = commonBlockSummary.getHeight();
|
||||
superiorPeersForComparison.add(peer);
|
||||
}
|
||||
}
|
||||
|
||||
// Now that we have selected the best peers, compare them against each other and remove any with lower weights
|
||||
if (superiorPeersForComparison.size() > 0) {
|
||||
BigInteger bestChainWeight = null;
|
||||
for (Peer peer : superiorPeersForComparison) {
|
||||
// Increase bestChainWeight if needed
|
||||
if (bestChainWeight == null || peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) >= 0)
|
||||
bestChainWeight = peer.getCommonBlockData().getChainWeight();
|
||||
}
|
||||
for (Peer peer : superiorPeersForComparison) {
|
||||
// Check if we should discard an inferior peer
|
||||
if (peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) < 0) {
|
||||
BigInteger difference = bestChainWeight.subtract(peer.getCommonBlockData().getChainWeight());
|
||||
LOGGER.debug(String.format("Peer %s has a lower chain weight (difference: %s) than other peer(s) in this group - removing it from this round.", peer, accurateFormatter.format(difference)));
|
||||
peers.remove(peer);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return peers;
|
||||
} finally {
|
||||
repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong
|
||||
}
|
||||
} catch (DataException e) {
|
||||
LOGGER.error("Repository issue during peer comparison", e);
|
||||
return peers;
|
||||
}
|
||||
}
|
||||
|
||||
private List<BlockSummaryData> uniqueCommonBlocks(List<Peer> peers) {
|
||||
List<BlockSummaryData> commonBlocks = new ArrayList<>();
|
||||
|
||||
for (Peer peer : peers) {
|
||||
if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) {
|
||||
LOGGER.debug(String.format("Peer %s has common block %.8s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature())));
|
||||
|
||||
BlockSummaryData commonBlockSummary = peer.getCommonBlockData().getCommonBlockSummary();
|
||||
if (!commonBlocks.contains(commonBlockSummary))
|
||||
commonBlocks.add(commonBlockSummary);
|
||||
}
|
||||
else {
|
||||
LOGGER.debug(String.format("Peer %s has no common block data. Skipping...", peer));
|
||||
}
|
||||
}
|
||||
|
||||
return commonBlocks;
|
||||
}
|
||||
|
||||
private int calculateMinChainLength(BlockSummaryData commonBlockSummary, int ourAdditionalBlocksAfterCommonBlock, List<Peer> peersSharingCommonBlock) {
|
||||
// Calculate the length of the shortest peer chain sharing this common block, including our chain
|
||||
int minChainLength = ourAdditionalBlocksAfterCommonBlock;
|
||||
for (Peer peer : peersSharingCommonBlock) {
|
||||
final int peerHeight = peer.getChainTipData().getLastHeight();
|
||||
final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight();
|
||||
|
||||
if (peerAdditionalBlocksAfterCommonBlock < minChainLength)
|
||||
minChainLength = peerAdditionalBlocksAfterCommonBlock;
|
||||
}
|
||||
return minChainLength;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Attempt to synchronize blockchain with peer.
|
||||
* <p>
|
||||
@@ -97,9 +474,12 @@ public class Synchronizer {
|
||||
|
||||
List<BlockSummaryData> peerBlockSummaries = new ArrayList<>();
|
||||
SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, force, peerBlockSummaries);
|
||||
if (findCommonBlockResult != SynchronizationResult.OK)
|
||||
if (findCommonBlockResult != SynchronizationResult.OK) {
|
||||
// Logging performed by fetchSummariesFromCommonBlock() above
|
||||
// Clear our common block cache for this peer
|
||||
peer.setCommonBlockData(null);
|
||||
return findCommonBlockResult;
|
||||
}
|
||||
|
||||
// First summary is common block
|
||||
final BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature());
|
||||
@@ -244,9 +624,13 @@ public class Synchronizer {
|
||||
// Currently we work forward from common block until we hit a block we don't have
|
||||
// TODO: rewrite as modified binary search!
|
||||
int i;
|
||||
for (i = 1; i < blockSummariesFromCommon.size(); ++i)
|
||||
for (i = 1; i < blockSummariesFromCommon.size(); ++i) {
|
||||
if (Controller.isStopping())
|
||||
return SynchronizationResult.SHUTTING_DOWN;
|
||||
|
||||
if (!repository.getBlockRepository().exists(blockSummariesFromCommon.get(i).getSignature()))
|
||||
break;
|
||||
}
|
||||
|
||||
// Note: index i - 1 isn't cleared: List.subList is fromIndex inclusive to toIndex exclusive
|
||||
blockSummariesFromCommon.subList(0, i - 1).clear();
|
||||
@@ -295,6 +679,9 @@ public class Synchronizer {
|
||||
|
||||
// Check peer sent valid heights
|
||||
for (int i = 0; i < moreBlockSummaries.size(); ++i) {
|
||||
if (Controller.isStopping())
|
||||
return SynchronizationResult.SHUTTING_DOWN;
|
||||
|
||||
++lastSummaryHeight;
|
||||
|
||||
BlockSummaryData blockSummary = moreBlockSummaries.get(i);
|
||||
@@ -341,52 +728,118 @@ public class Synchronizer {
|
||||
final byte[] commonBlockSig = commonBlockData.getSignature();
|
||||
String commonBlockSig58 = Base58.encode(commonBlockSig);
|
||||
|
||||
byte[] latestPeerSignature = commonBlockSig;
|
||||
int height = commonBlockHeight;
|
||||
|
||||
LOGGER.debug(() -> String.format("Fetching peer %s chain from height %d, sig %.8s", peer, commonBlockHeight, commonBlockSig58));
|
||||
|
||||
int ourHeight = ourInitialHeight;
|
||||
|
||||
// Overall plan: fetch peer's blocks first, then orphan, then apply
|
||||
|
||||
// Convert any leftover (post-common) block summaries into signatures to request from peer
|
||||
List<byte[]> peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList());
|
||||
|
||||
// Fetch remaining block signatures, if needed
|
||||
int numberSignaturesRequired = peerBlockSignatures.size() - (peerHeight - commonBlockHeight);
|
||||
if (numberSignaturesRequired > 0) {
|
||||
byte[] latestPeerSignature = peerBlockSignatures.isEmpty() ? commonBlockSig : peerBlockSignatures.get(peerBlockSignatures.size() - 1);
|
||||
|
||||
LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s",
|
||||
numberSignaturesRequired, (numberSignaturesRequired != 1 ? "s": ""), ourHeight, Base58.encode(latestPeerSignature)));
|
||||
|
||||
List<byte[]> moreBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberSignaturesRequired);
|
||||
|
||||
if (moreBlockSignatures == null || moreBlockSignatures.isEmpty()) {
|
||||
LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer,
|
||||
ourHeight, Base58.encode(latestPeerSignature)));
|
||||
return SynchronizationResult.NO_REPLY;
|
||||
}
|
||||
|
||||
LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : "")));
|
||||
|
||||
peerBlockSignatures.addAll(moreBlockSignatures);
|
||||
}
|
||||
|
||||
// Fetch blocks using signatures
|
||||
LOGGER.debug(String.format("Fetching new blocks from peer %s", peer));
|
||||
// Keep a list of blocks received so far
|
||||
List<Block> peerBlocks = new ArrayList<>();
|
||||
|
||||
for (byte[] blockSignature : peerBlockSignatures) {
|
||||
Block newBlock = this.fetchBlock(repository, peer, blockSignature);
|
||||
// Calculate the total number of additional blocks this peer has beyond the common block
|
||||
int additionalPeerBlocksAfterCommonBlock = peerHeight - commonBlockHeight;
|
||||
// Subtract the number of signatures that we already have, as we don't need to request them again
|
||||
int numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size();
|
||||
|
||||
int retryCount = 0;
|
||||
while (height < peerHeight) {
|
||||
if (Controller.isStopping())
|
||||
return SynchronizationResult.SHUTTING_DOWN;
|
||||
|
||||
// Ensure we don't request more than MAXIMUM_REQUEST_SIZE
|
||||
int numberRequested = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE);
|
||||
|
||||
// Do we need more signatures?
|
||||
if (peerBlockSignatures.isEmpty() && numberRequested > 0) {
|
||||
LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s",
|
||||
numberRequested, (numberRequested != 1 ? "s" : ""), height, Base58.encode(latestPeerSignature)));
|
||||
|
||||
peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested);
|
||||
|
||||
if (peerBlockSignatures == null || peerBlockSignatures.isEmpty()) {
|
||||
LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer,
|
||||
height, Base58.encode(latestPeerSignature)));
|
||||
|
||||
// Clear our cache of common block summaries for this peer, as they are likely to be invalid
|
||||
CommonBlockData cachedCommonBlockData = peer.getCommonBlockData();
|
||||
if (cachedCommonBlockData != null)
|
||||
cachedCommonBlockData.setBlockSummariesAfterCommonBlock(null);
|
||||
|
||||
// If we have already received RECENT blocks from this peer, go ahead and apply them
|
||||
if (peerBlocks.size() > 0) {
|
||||
final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1);
|
||||
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
|
||||
if (peerLatestBlock != null && minLatestBlockTimestamp != null
|
||||
&& peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) {
|
||||
LOGGER.debug("Newly received blocks are recent, so we will apply them");
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state
|
||||
return SynchronizationResult.NO_REPLY;
|
||||
}
|
||||
|
||||
numberSignaturesRequired = peerHeight - height - peerBlockSignatures.size();
|
||||
LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : "")));
|
||||
}
|
||||
|
||||
if (peerBlockSignatures.isEmpty()) {
|
||||
LOGGER.trace(String.format("No more signatures or blocks to request from peer %s", peer));
|
||||
break;
|
||||
}
|
||||
|
||||
byte[] nextPeerSignature = peerBlockSignatures.get(0);
|
||||
int nextHeight = height + 1;
|
||||
|
||||
LOGGER.trace(String.format("Fetching block %d, sig %.8s from %s", nextHeight, Base58.encode(nextPeerSignature), peer));
|
||||
Block newBlock = this.fetchBlock(repository, peer, nextPeerSignature);
|
||||
|
||||
if (newBlock == null) {
|
||||
LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer,
|
||||
ourHeight, Base58.encode(blockSignature)));
|
||||
return SynchronizationResult.NO_REPLY;
|
||||
nextHeight, Base58.encode(nextPeerSignature)));
|
||||
|
||||
if (retryCount >= MAXIMUM_RETRIES) {
|
||||
|
||||
// If we have already received RECENT blocks from this peer, go ahead and apply them
|
||||
if (peerBlocks.size() > 0) {
|
||||
final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1);
|
||||
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
|
||||
if (peerLatestBlock != null && minLatestBlockTimestamp != null
|
||||
&& peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) {
|
||||
LOGGER.debug("Newly received blocks are recent, so we will apply them");
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state
|
||||
return SynchronizationResult.NO_REPLY;
|
||||
|
||||
} else {
|
||||
// Re-fetch signatures, in case the peer is now on a different fork
|
||||
peerBlockSignatures.clear();
|
||||
numberSignaturesRequired = peerHeight - height;
|
||||
|
||||
// Retry until retryCount reaches MAXIMUM_RETRIES
|
||||
retryCount++;
|
||||
int triesRemaining = MAXIMUM_RETRIES - retryCount;
|
||||
LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s" : "")));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Reset retryCount because the last request succeeded
|
||||
retryCount = 0;
|
||||
|
||||
LOGGER.trace(String.format("Fetched block %d, sig %.8s from %s", nextHeight, Base58.encode(latestPeerSignature), peer));
|
||||
|
||||
if (!newBlock.isSignatureValid()) {
|
||||
LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer,
|
||||
ourHeight, Base58.encode(blockSignature)));
|
||||
nextHeight, Base58.encode(latestPeerSignature)));
|
||||
return SynchronizationResult.INVALID_DATA;
|
||||
}
|
||||
|
||||
@@ -395,12 +848,18 @@ public class Synchronizer {
|
||||
transaction.setInitialApprovalStatus();
|
||||
|
||||
peerBlocks.add(newBlock);
|
||||
|
||||
// Now that we've received this block, we can increase our height and move on to the next one
|
||||
latestPeerSignature = nextPeerSignature;
|
||||
peerBlockSignatures.remove(0);
|
||||
++height;
|
||||
}
|
||||
|
||||
// Unwind to common block (unless common block is our latest block)
|
||||
LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s", commonBlockHeight, commonBlockSig58));
|
||||
int ourHeight = ourInitialHeight;
|
||||
LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s. Our height: %d", commonBlockHeight, commonBlockSig58, ourHeight));
|
||||
|
||||
BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourHeight);
|
||||
BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourInitialHeight);
|
||||
while (ourHeight > commonBlockHeight) {
|
||||
if (Controller.isStopping())
|
||||
return SynchronizationResult.SHUTTING_DOWN;
|
||||
@@ -422,10 +881,13 @@ public class Synchronizer {
|
||||
LOGGER.debug(String.format("Orphaned blocks back to height %d, sig %.8s - applying new blocks from peer %s", commonBlockHeight, commonBlockSig58, peer));
|
||||
|
||||
for (Block newBlock : peerBlocks) {
|
||||
if (Controller.isStopping())
|
||||
return SynchronizationResult.SHUTTING_DOWN;
|
||||
|
||||
ValidationResult blockResult = newBlock.isValid();
|
||||
if (blockResult != ValidationResult.OK) {
|
||||
LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer,
|
||||
ourHeight, Base58.encode(newBlock.getSignature()), blockResult.name()));
|
||||
newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getSignature()), blockResult.name()));
|
||||
return SynchronizationResult.INVALID_DATA;
|
||||
}
|
||||
|
||||
@@ -469,7 +931,8 @@ public class Synchronizer {
|
||||
|
||||
// Do we need more signatures?
|
||||
if (peerBlockSignatures.isEmpty()) {
|
||||
int numberRequested = maxBatchHeight - ourHeight;
|
||||
int numberRequested = Math.min(maxBatchHeight - ourHeight, MAXIMUM_REQUEST_SIZE);
|
||||
|
||||
LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s",
|
||||
numberRequested, (numberRequested != 1 ? "s": ""), ourHeight, Base58.encode(latestPeerSignature)));
|
||||
|
||||
@@ -573,6 +1036,9 @@ public class Synchronizer {
|
||||
final int firstBlockHeight = blockSummaries.get(0).getHeight();
|
||||
|
||||
for (int i = 0; i < blockSummaries.size(); ++i) {
|
||||
if (Controller.isStopping())
|
||||
return;
|
||||
|
||||
BlockSummaryData blockSummary = blockSummaries.get(i);
|
||||
|
||||
// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level
|
||||
|
@@ -386,7 +386,7 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot {
|
||||
|
||||
// If it has been over 24 hours since we last updated this trade-bot entry then assume AT is never coming back
|
||||
// and so wipe the trade-bot entry
|
||||
if (tradeBotData.getTimestamp() + MAX_AT_CONFIRMATION_PERIOD > NTP.getTime()) {
|
||||
if (tradeBotData.getTimestamp() + MAX_AT_CONFIRMATION_PERIOD < NTP.getTime()) {
|
||||
LOGGER.info(() -> String.format("AT %s has been gone for too long - deleting trade-bot entry", tradeBotData.getAtAddress()));
|
||||
repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
|
||||
repository.saveChanges();
|
||||
|
@@ -211,6 +211,9 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
|
||||
|
||||
TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress));
|
||||
|
||||
// Attempt to backup the trade bot data
|
||||
TradeBot.backupTradeBotData(repository);
|
||||
|
||||
// Return to user for signing and broadcast as we don't have their Qortal private key
|
||||
try {
|
||||
return DeployAtTransactionTransformer.toBytes(deployAtTransactionData);
|
||||
@@ -283,6 +286,9 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
|
||||
tradeForeignPublicKey, tradeForeignPublicKeyHash,
|
||||
crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash);
|
||||
|
||||
// Attempt to backup the trade bot data
|
||||
TradeBot.backupTradeBotData(repository);
|
||||
|
||||
// Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount
|
||||
long p2shFee;
|
||||
try {
|
||||
@@ -384,7 +390,7 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
|
||||
|
||||
// If it has been over 24 hours since we last updated this trade-bot entry then assume AT is never coming back
|
||||
// and so wipe the trade-bot entry
|
||||
if (tradeBotData.getTimestamp() + MAX_AT_CONFIRMATION_PERIOD > NTP.getTime()) {
|
||||
if (tradeBotData.getTimestamp() + MAX_AT_CONFIRMATION_PERIOD < NTP.getTime()) {
|
||||
LOGGER.info(() -> String.format("AT %s has been gone for too long - deleting trade-bot entry", tradeBotData.getAtAddress()));
|
||||
repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
|
||||
repository.saveChanges();
|
||||
|
@@ -7,6 +7,7 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.locks.ReentrantLock;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
@@ -267,6 +268,22 @@ public class TradeBot implements Listener {
|
||||
return secret;
|
||||
}
|
||||
|
||||
/*package*/ static void backupTradeBotData(Repository repository) {
|
||||
// Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure
|
||||
try {
|
||||
LOGGER.info("About to backup trade bot data...");
|
||||
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
|
||||
blockchainLock.lockInterruptibly();
|
||||
try {
|
||||
repository.exportNodeLocalData(true);
|
||||
} finally {
|
||||
blockchainLock.unlock();
|
||||
}
|
||||
} catch (InterruptedException | DataException e) {
|
||||
LOGGER.info(String.format("Failed to obtain blockchain lock when exporting trade bot data: %s", e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
/** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */
|
||||
/*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData,
|
||||
String newState, int newStateValue, Supplier<String> logMessageSupplier) throws DataException {
|
||||
|
@@ -2,6 +2,7 @@ package org.qortal.data.block;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import java.util.Arrays;
|
||||
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class BlockSummaryData {
|
||||
@@ -84,4 +85,21 @@ public class BlockSummaryData {
|
||||
this.minterLevel = minterLevel;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o)
|
||||
return true;
|
||||
|
||||
if (o == null || getClass() != o.getClass())
|
||||
return false;
|
||||
|
||||
BlockSummaryData otherBlockSummary = (BlockSummaryData) o;
|
||||
if (this.getSignature() == null || otherBlockSummary.getSignature() == null)
|
||||
return false;
|
||||
|
||||
// Treat two block summaries as equal if they have matching signatures
|
||||
return Arrays.equals(this.getSignature(), otherBlockSummary.getSignature());
|
||||
}
|
||||
|
||||
}
|
||||
|
56
src/main/java/org/qortal/data/block/CommonBlockData.java
Normal file
56
src/main/java/org/qortal/data/block/CommonBlockData.java
Normal file
@@ -0,0 +1,56 @@
|
||||
package org.qortal.data.block;
|
||||
|
||||
import org.qortal.data.network.PeerChainTipData;
|
||||
|
||||
import javax.xml.bind.annotation.XmlAccessType;
|
||||
import javax.xml.bind.annotation.XmlAccessorType;
|
||||
import java.math.BigInteger;
|
||||
import java.util.List;
|
||||
|
||||
@XmlAccessorType(XmlAccessType.FIELD)
|
||||
public class CommonBlockData {
|
||||
|
||||
// Properties
|
||||
private BlockSummaryData commonBlockSummary = null;
|
||||
private List<BlockSummaryData> blockSummariesAfterCommonBlock = null;
|
||||
private BigInteger chainWeight = null;
|
||||
private PeerChainTipData chainTipData = null;
|
||||
|
||||
// Constructors
|
||||
|
||||
protected CommonBlockData() {
|
||||
}
|
||||
|
||||
public CommonBlockData(BlockSummaryData commonBlockSummary, PeerChainTipData chainTipData) {
|
||||
this.commonBlockSummary = commonBlockSummary;
|
||||
this.chainTipData = chainTipData;
|
||||
}
|
||||
|
||||
|
||||
// Getters / setters
|
||||
|
||||
public BlockSummaryData getCommonBlockSummary() {
|
||||
return this.commonBlockSummary;
|
||||
}
|
||||
|
||||
public List<BlockSummaryData> getBlockSummariesAfterCommonBlock() {
|
||||
return this.blockSummariesAfterCommonBlock;
|
||||
}
|
||||
|
||||
public void setBlockSummariesAfterCommonBlock(List<BlockSummaryData> blockSummariesAfterCommonBlock) {
|
||||
this.blockSummariesAfterCommonBlock = blockSummariesAfterCommonBlock;
|
||||
}
|
||||
|
||||
public BigInteger getChainWeight() {
|
||||
return this.chainWeight;
|
||||
}
|
||||
|
||||
public void setChainWeight(BigInteger chainWeight) {
|
||||
this.chainWeight = chainWeight;
|
||||
}
|
||||
|
||||
public PeerChainTipData getChainTipData() {
|
||||
return this.chainTipData;
|
||||
}
|
||||
|
||||
}
|
@@ -15,6 +15,7 @@ import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.Arrays;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
@@ -22,6 +23,7 @@ import java.util.concurrent.TimeUnit;
|
||||
|
||||
import org.apache.logging.log4j.LogManager;
|
||||
import org.apache.logging.log4j.Logger;
|
||||
import org.qortal.data.block.CommonBlockData;
|
||||
import org.qortal.data.network.PeerChainTipData;
|
||||
import org.qortal.data.network.PeerData;
|
||||
import org.qortal.network.message.ChallengeMessage;
|
||||
@@ -46,7 +48,7 @@ public class Peer {
|
||||
private static final int CONNECT_TIMEOUT = 2000; // ms
|
||||
|
||||
/** Maximum time to wait for a message reply to arrive from peer. (ms) */
|
||||
private static final int RESPONSE_TIMEOUT = 2000; // ms
|
||||
private static final int RESPONSE_TIMEOUT = 3000; // ms
|
||||
|
||||
/**
|
||||
* Interval between PING messages to a peer. (ms)
|
||||
@@ -106,6 +108,9 @@ public class Peer {
|
||||
/** Latest block info as reported by peer. */
|
||||
private PeerChainTipData peersChainTipData;
|
||||
|
||||
/** Our common block with this peer */
|
||||
private CommonBlockData commonBlockData;
|
||||
|
||||
// Constructors
|
||||
|
||||
/** Construct unconnected, outbound Peer using socket address in peer data */
|
||||
@@ -272,6 +277,18 @@ public class Peer {
|
||||
}
|
||||
}
|
||||
|
||||
public CommonBlockData getCommonBlockData() {
|
||||
synchronized (this.peerInfoLock) {
|
||||
return this.commonBlockData;
|
||||
}
|
||||
}
|
||||
|
||||
public void setCommonBlockData(CommonBlockData commonBlockData) {
|
||||
synchronized (this.peerInfoLock) {
|
||||
this.commonBlockData = commonBlockData;
|
||||
}
|
||||
}
|
||||
|
||||
/*package*/ void queueMessage(Message message) {
|
||||
if (!this.pendingMessages.offer(message))
|
||||
LOGGER.info(() -> String.format("No room to queue message from peer %s - discarding", this));
|
||||
@@ -507,6 +524,7 @@ public class Peer {
|
||||
}
|
||||
} catch (MessageException e) {
|
||||
LOGGER.warn(String.format("Failed to send %s message with ID %d to peer %s: %s", message.getType().name(), message.getId(), this, e.getMessage()));
|
||||
return false;
|
||||
} catch (IOException e) {
|
||||
// Send failure
|
||||
return false;
|
||||
@@ -615,6 +633,25 @@ public class Peer {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Common block data
|
||||
|
||||
public boolean canUseCachedCommonBlockData() {
|
||||
PeerChainTipData peerChainTipData = this.getChainTipData();
|
||||
CommonBlockData commonBlockData = this.getCommonBlockData();
|
||||
|
||||
if (peerChainTipData != null && commonBlockData != null) {
|
||||
PeerChainTipData commonBlockChainTipData = commonBlockData.getChainTipData();
|
||||
if (peerChainTipData.getLastBlockSignature() != null && commonBlockChainTipData != null && commonBlockChainTipData.getLastBlockSignature() != null) {
|
||||
if (Arrays.equals(peerChainTipData.getLastBlockSignature(), commonBlockChainTipData.getLastBlockSignature())) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
// Utility methods
|
||||
|
||||
/** Returns true if ports and addresses (or hostnames) match */
|
||||
|
@@ -0,0 +1,70 @@
|
||||
package org.qortal.network.message;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
import org.qortal.block.Block;
|
||||
import org.qortal.transform.TransformationException;
|
||||
import org.qortal.transform.block.BlockTransformer;
|
||||
|
||||
import com.google.common.primitives.Ints;
|
||||
|
||||
// This is an OUTGOING-only Message which more readily lends itself to being cached
|
||||
public class CachedBlockMessage extends Message {
|
||||
|
||||
private Block block = null;
|
||||
private byte[] cachedBytes = null;
|
||||
|
||||
public CachedBlockMessage(Block block) {
|
||||
super(MessageType.BLOCK);
|
||||
|
||||
this.block = block;
|
||||
}
|
||||
|
||||
private CachedBlockMessage(byte[] cachedBytes) {
|
||||
super(MessageType.BLOCK);
|
||||
|
||||
this.block = null;
|
||||
this.cachedBytes = cachedBytes;
|
||||
}
|
||||
|
||||
public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException {
|
||||
throw new UnsupportedOperationException("CachedBlockMessage is for outgoing messages only");
|
||||
}
|
||||
|
||||
@Override
|
||||
protected byte[] toData() {
|
||||
// Already serialized?
|
||||
if (this.cachedBytes != null)
|
||||
return cachedBytes;
|
||||
|
||||
if (this.block == null)
|
||||
return null;
|
||||
|
||||
try {
|
||||
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
|
||||
|
||||
bytes.write(Ints.toByteArray(this.block.getBlockData().getHeight()));
|
||||
|
||||
bytes.write(BlockTransformer.toBytes(this.block));
|
||||
|
||||
this.cachedBytes = bytes.toByteArray();
|
||||
// We no longer need source Block
|
||||
// and Block contains repository handle which is highly likely to be invalid after this call
|
||||
this.block = null;
|
||||
|
||||
return this.cachedBytes;
|
||||
} catch (TransformationException | IOException e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public CachedBlockMessage cloneWithNewId(int newId) {
|
||||
CachedBlockMessage clone = new CachedBlockMessage(this.cachedBytes);
|
||||
clone.setId(newId);
|
||||
return clone;
|
||||
}
|
||||
|
||||
}
|
@@ -49,7 +49,7 @@ public interface Repository extends AutoCloseable {
|
||||
|
||||
public void performPeriodicMaintenance() throws DataException;
|
||||
|
||||
public void exportNodeLocalData() throws DataException;
|
||||
public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException;
|
||||
|
||||
public void importDataFromFile(String filename) throws DataException;
|
||||
|
||||
|
@@ -52,6 +52,7 @@ import org.qortal.repository.TransactionRepository;
|
||||
import org.qortal.repository.VotingRepository;
|
||||
import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository;
|
||||
import org.qortal.settings.Settings;
|
||||
import org.qortal.utils.NTP;
|
||||
|
||||
public class HSQLDBRepository implements Repository {
|
||||
|
||||
@@ -459,10 +460,44 @@ public class HSQLDBRepository implements Repository {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void exportNodeLocalData() throws DataException {
|
||||
public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException {
|
||||
|
||||
// Create the qortal-backup folder if it doesn't exist
|
||||
Path backupPath = Paths.get("qortal-backup");
|
||||
try {
|
||||
Files.createDirectories(backupPath);
|
||||
} catch (IOException e) {
|
||||
LOGGER.info("Unable to create backup folder");
|
||||
throw new DataException("Unable to create backup folder");
|
||||
}
|
||||
|
||||
// We need to rename or delete an existing TradeBotStates backup before creating a new one
|
||||
File tradeBotStatesBackupFile = new File("qortal-backup/TradeBotStates.script");
|
||||
if (tradeBotStatesBackupFile.exists()) {
|
||||
if (keepArchivedCopy) {
|
||||
// Rename existing TradeBotStates backup, to make sure that we're not overwriting any keys
|
||||
File archivedBackupFile = new File(String.format("qortal-backup/TradeBotStates-archive-%d.script", NTP.getTime()));
|
||||
if (tradeBotStatesBackupFile.renameTo(archivedBackupFile))
|
||||
LOGGER.info(String.format("Moved existing TradeBotStates backup file to %s", archivedBackupFile.getPath()));
|
||||
else
|
||||
throw new DataException("Unable to rename existing TradeBotStates backup");
|
||||
} else {
|
||||
// Delete existing copy
|
||||
LOGGER.info("Deleting existing TradeBotStates backup because it is being replaced with a new one");
|
||||
tradeBotStatesBackupFile.delete();
|
||||
}
|
||||
}
|
||||
|
||||
// There's currently no need to take an archived copy of the MintingAccounts data - just delete the old one if it exists
|
||||
File mintingAccountsBackupFile = new File("qortal-backup/MintingAccounts.script");
|
||||
if (mintingAccountsBackupFile.exists()) {
|
||||
LOGGER.info("Deleting existing MintingAccounts backup because it is being replaced with a new one");
|
||||
mintingAccountsBackupFile.delete();
|
||||
}
|
||||
|
||||
try (Statement stmt = this.connection.createStatement()) {
|
||||
stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'MintingAccounts.script'");
|
||||
stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'TradeBotStates.script'");
|
||||
stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'qortal-backup/MintingAccounts.script'");
|
||||
stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'qortal-backup/TradeBotStates.script'");
|
||||
LOGGER.info("Exported sensitive/node-local data: minting keys and trade bot states");
|
||||
} catch (SQLException e) {
|
||||
throw new DataException("Unable to export sensitive/node-local data from repository");
|
||||
@@ -475,12 +510,12 @@ public class HSQLDBRepository implements Repository {
|
||||
LOGGER.info(() -> String.format("Importing data into repository from %s", filename));
|
||||
|
||||
String escapedFilename = stmt.enquoteLiteral(filename);
|
||||
stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " STOP ON ERROR");
|
||||
stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " CONTINUE ON ERROR");
|
||||
|
||||
LOGGER.info(() -> String.format("Imported data into repository from %s", filename));
|
||||
} catch (SQLException e) {
|
||||
LOGGER.info(() -> String.format("Failed to import data into repository from %s: %s", filename, e.getMessage()));
|
||||
throw new DataException("Unable to export sensitive/node-local data from repository: " + e.getMessage());
|
||||
throw new DataException("Unable to import sensitive/node-local data to repository: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -681,7 +716,7 @@ public class HSQLDBRepository implements Repository {
|
||||
/**
|
||||
* Execute PreparedStatement and return changed row count.
|
||||
*
|
||||
* @param preparedStatement
|
||||
* @param sql
|
||||
* @param objects
|
||||
* @return number of changed rows
|
||||
* @throws SQLException
|
||||
@@ -693,8 +728,8 @@ public class HSQLDBRepository implements Repository {
|
||||
/**
|
||||
* Execute batched PreparedStatement
|
||||
*
|
||||
* @param preparedStatement
|
||||
* @param objects
|
||||
* @param sql
|
||||
* @param batchedObjects
|
||||
* @return number of changed rows
|
||||
* @throws SQLException
|
||||
*/
|
||||
@@ -818,7 +853,7 @@ public class HSQLDBRepository implements Repository {
|
||||
*
|
||||
* @param tableName
|
||||
* @param whereClause
|
||||
* @param objects
|
||||
* @param batchedObjects
|
||||
* @throws SQLException
|
||||
*/
|
||||
public int deleteBatch(String tableName, String whereClause, List<Object[]> batchedObjects) throws SQLException {
|
||||
@@ -931,6 +966,8 @@ public class HSQLDBRepository implements Repository {
|
||||
|
||||
/** Logs other HSQLDB sessions then returns passed exception */
|
||||
public SQLException examineException(SQLException e) {
|
||||
// TODO: could log at DEBUG for deadlocks by checking RepositoryManager.isDeadlockRelated(e)?
|
||||
|
||||
LOGGER.error(() -> String.format("[Session %d] HSQLDB error: %s", this.sessionId, e.getMessage()), e);
|
||||
|
||||
logStatements();
|
||||
|
@@ -14,11 +14,11 @@ import org.hsqldb.jdbc.HSQLDBPool;
|
||||
import org.qortal.repository.DataException;
|
||||
import org.qortal.repository.Repository;
|
||||
import org.qortal.repository.RepositoryFactory;
|
||||
import org.qortal.settings.Settings;
|
||||
|
||||
public class HSQLDBRepositoryFactory implements RepositoryFactory {
|
||||
|
||||
private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepositoryFactory.class);
|
||||
private static final int POOL_SIZE = 100;
|
||||
|
||||
/** Log getConnection() calls that take longer than this. (ms) */
|
||||
private static final long SLOW_CONNECTION_THRESHOLD = 1000L;
|
||||
@@ -57,7 +57,7 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory {
|
||||
HSQLDBRepository.attemptRecovery(connectionUrl);
|
||||
}
|
||||
|
||||
this.connectionPool = new HSQLDBPool(POOL_SIZE);
|
||||
this.connectionPool = new HSQLDBPool(Settings.getInstance().getRepositoryConnectionPoolSize());
|
||||
this.connectionPool.setUrl(this.connectionUrl);
|
||||
|
||||
Properties properties = new Properties();
|
||||
|
@@ -52,7 +52,7 @@ public class Settings {
|
||||
// UI servers
|
||||
private int uiPort = 12388;
|
||||
private String[] uiLocalServers = new String[] {
|
||||
"localhost", "127.0.0.1", "172.24.1.1", "qor.tal"
|
||||
"localhost", "127.0.0.1"
|
||||
};
|
||||
private String[] uiRemoteServers = new String[] {
|
||||
"node1.qortal.org", "node2.qortal.org", "node3.qortal.org", "node4.qortal.org", "node5.qortal.org",
|
||||
@@ -136,6 +136,8 @@ public class Settings {
|
||||
private Long slowQueryThreshold = null;
|
||||
/** Repository storage path. */
|
||||
private String repositoryPath = "db";
|
||||
/** Repository connection pool size. Needs to be a bit bigger than maxNetworkThreadPoolSize */
|
||||
private int repositoryConnectionPoolSize = 100;
|
||||
|
||||
// Auto-update sources
|
||||
private String[] autoUpdateRepos = new String[] {
|
||||
@@ -430,6 +432,10 @@ public class Settings {
|
||||
return this.repositoryPath;
|
||||
}
|
||||
|
||||
public int getRepositoryConnectionPoolSize() {
|
||||
return this.repositoryConnectionPoolSize;
|
||||
}
|
||||
|
||||
public boolean isAutoUpdateEnabled() {
|
||||
return this.autoUpdateEnabled;
|
||||
}
|
||||
|
@@ -57,9 +57,11 @@ $timestamp *= 1000; # Convert to milliseconds
|
||||
|
||||
# locate sha256 utility
|
||||
my $SHA256 = `which sha256sum || which sha256`;
|
||||
chomp $SHA256;
|
||||
die("Can't find sha256sum or sha256\n") unless length($SHA256) > 0;
|
||||
|
||||
# SHA256 of actual update file
|
||||
my $sha256 = `git show auto-update-${commit_hash}:${project}.update | ${SHA256}`;
|
||||
my $sha256 = `git show auto-update-${commit_hash}:${project}.update | ${SHA256} | head -c 64`;
|
||||
die("Can't calculate SHA256 of ${project}.update\n") unless $sha256 =~ m/(\S{64})/;
|
||||
chomp $sha256;
|
||||
|
||||
|
Reference in New Issue
Block a user