diff --git a/src/main/java/org/qora/api/model/ApiOnlineAccount.java b/src/main/java/org/qora/api/model/ApiOnlineAccount.java new file mode 100644 index 00000000..2f8a7ca1 --- /dev/null +++ b/src/main/java/org/qora/api/model/ApiOnlineAccount.java @@ -0,0 +1,50 @@ +package org.qora.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +// All properties to be converted to JSON via JAXB +@XmlAccessorType(XmlAccessType.FIELD) +public class ApiOnlineAccount { + + protected long timestamp; + protected byte[] signature; + protected byte[] publicKey; + protected String generatorAddress; + protected String recipientAddress; + + // Constructors + + // necessary for JAXB serialization + protected ApiOnlineAccount() { + } + + public ApiOnlineAccount(long timestamp, byte[] signature, byte[] publicKey, String generatorAddress, String recipientAddress) { + this.timestamp = timestamp; + this.signature = signature; + this.publicKey = publicKey; + this.generatorAddress = generatorAddress; + this.recipientAddress = recipientAddress; + } + + public long getTimestamp() { + return this.timestamp; + } + + public byte[] getSignature() { + return this.signature; + } + + public byte[] getPublicKey() { + return this.publicKey; + } + + public String getGeneratorAddress() { + return this.generatorAddress; + } + + public String getRecipientAddress() { + return this.recipientAddress; + } + +} diff --git a/src/main/java/org/qora/api/model/ConnectedPeer.java b/src/main/java/org/qora/api/model/ConnectedPeer.java index 09cb0c01..6234cd54 100644 --- a/src/main/java/org/qora/api/model/ConnectedPeer.java +++ b/src/main/java/org/qora/api/model/ConnectedPeer.java @@ -3,6 +3,7 @@ package org.qora.api.model; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; +import org.qora.data.network.PeerChainTipData; import org.qora.data.network.PeerData; import org.qora.network.Handshake; import org.qora.network.Peer; @@ -49,9 +50,12 @@ public class ConnectedPeer { this.buildTimestamp = peer.getVersionMessage().getBuildTimestamp(); } - this.lastHeight = peer.getLastHeight(); - this.lastBlockSignature = peer.getLastBlockSignature(); - this.lastBlockTimestamp = peer.getLastBlockTimestamp(); + PeerChainTipData peerChainTipData = peer.getChainTipData(); + if (peerChainTipData != null) { + this.lastHeight = peerChainTipData.getLastHeight(); + this.lastBlockSignature = peerChainTipData.getLastBlockSignature(); + this.lastBlockTimestamp = peerChainTipData.getLastBlockTimestamp(); + } } } diff --git a/src/main/java/org/qora/api/resource/AddressesResource.java b/src/main/java/org/qora/api/resource/AddressesResource.java index de9a3c2c..70111508 100644 --- a/src/main/java/org/qora/api/resource/AddressesResource.java +++ b/src/main/java/org/qora/api/resource/AddressesResource.java @@ -10,6 +10,7 @@ import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; import java.math.BigDecimal; +import java.util.ArrayList; import java.util.List; import javax.servlet.http.HttpServletRequest; @@ -27,6 +28,7 @@ import org.qora.api.ApiError; import org.qora.api.ApiErrors; import org.qora.api.ApiException; import org.qora.api.ApiExceptionFactory; +import org.qora.api.model.ApiOnlineAccount; import org.qora.api.model.ProxyKeyRequest; import org.qora.api.resource.TransactionsResource; import org.qora.asset.Asset; @@ -34,7 +36,7 @@ import org.qora.controller.Controller; import org.qora.crypto.Crypto; import org.qora.data.account.AccountData; import org.qora.data.account.ProxyForgerData; -import org.qora.data.network.OnlineAccount; +import org.qora.data.network.OnlineAccountData; import org.qora.data.transaction.ProxyForgingTransactionData; import org.qora.repository.DataException; import org.qora.repository.Repository; @@ -159,12 +161,32 @@ public class AddressesResource { responses = { @ApiResponse( description = "online accounts", - content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = OnlineAccount.class))) + content = @Content(mediaType = MediaType.APPLICATION_JSON, array = @ArraySchema(schema = @Schema(implementation = ApiOnlineAccount.class))) ) } ) - public List getOnlineAccounts() { - return Controller.getInstance().getOnlineAccounts(); + @ApiErrors({ApiError.PUBLIC_KEY_NOT_FOUND, ApiError.REPOSITORY_ISSUE}) + public List getOnlineAccounts() { + List onlineAccounts = Controller.getInstance().getOnlineAccounts(); + + // Map OnlineAccountData entries to OnlineAccount via proxy-relationship data + try (final Repository repository = RepositoryManager.getRepository()) { + List apiOnlineAccounts = new ArrayList<>(); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + ProxyForgerData proxyForgerData = repository.getAccountRepository().getProxyForgeData(onlineAccountData.getPublicKey()); + if (proxyForgerData == null) + // This shouldn't happen? + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.PUBLIC_KEY_NOT_FOUND); + + apiOnlineAccounts.add(new ApiOnlineAccount(onlineAccountData.getTimestamp(), onlineAccountData.getSignature(), onlineAccountData.getPublicKey(), + proxyForgerData.getForger(), proxyForgerData.getRecipient())); + } + + return apiOnlineAccounts; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } } @GET diff --git a/src/main/java/org/qora/api/resource/BlocksResource.java b/src/main/java/org/qora/api/resource/BlocksResource.java index 675eb4d1..1193ecae 100644 --- a/src/main/java/org/qora/api/resource/BlocksResource.java +++ b/src/main/java/org/qora/api/resource/BlocksResource.java @@ -320,58 +320,6 @@ public class BlocksResource { } } - @GET - @Path("/time") - @Operation( - summary = "Estimated time to forge next block", - description = "Calculates the time it should take for the network to generate the next block", - responses = { - @ApiResponse( - description = "the time in seconds", - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "number" - ) - ) - ) - } - ) - @ApiErrors({ - ApiError.REPOSITORY_ISSUE - }) - public long getTimePerBlock() { - try (final Repository repository = RepositoryManager.getRepository()) { - BlockData blockData = repository.getBlockRepository().getLastBlock(); - return Block.calcForgingDelay(blockData.getGeneratingBalance()); - } catch (ApiException e) { - throw e; - } catch (DataException e) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); - } - } - - @GET - @Path("/time/{generatingbalance}") - @Operation( - summary = "Estimated time to forge block given generating balance", - description = "Calculates the time it should take for the network to generate blocks based on specified generating balance", - responses = { - @ApiResponse( - description = "the time", // in seconds? - content = @Content( - mediaType = MediaType.TEXT_PLAIN, - schema = @Schema( - type = "number" - ) - ) - ) - } - ) - public long getTimePerBlock(@PathParam("generatingbalance") BigDecimal generatingbalance) { - return Block.calcForgingDelay(generatingbalance); - } - @GET @Path("/height") @Operation( diff --git a/src/main/java/org/qora/block/Block.java b/src/main/java/org/qora/block/Block.java index 2be25a1a..bbe0d582 100644 --- a/src/main/java/org/qora/block/Block.java +++ b/src/main/java/org/qora/block/Block.java @@ -29,8 +29,9 @@ import org.qora.data.account.ProxyForgerData; import org.qora.data.at.ATData; import org.qora.data.at.ATStateData; import org.qora.data.block.BlockData; +import org.qora.data.block.BlockSummaryData; import org.qora.data.block.BlockTransactionData; -import org.qora.data.network.OnlineAccount; +import org.qora.data.network.OnlineAccountData; import org.qora.data.transaction.TransactionData; import org.qora.repository.ATRepository; import org.qora.repository.BlockRepository; @@ -128,6 +129,11 @@ public class Block { // Other properties private static final Logger LOGGER = LogManager.getLogger(Block.class); + /** Number of left-shifts to apply to block's online accounts count when calculating block's weight. */ + private static final int ACCOUNTS_COUNT_SHIFT = Transformer.PUBLIC_KEY_LENGTH * 8; + /** Number of left-shifts to apply to previous block's weight when calculating a chain's weight. */ + private static final int CHAIN_WEIGHT_SHIFT = 8; + /** Sorted list of transactions attached to this block */ protected List transactions; @@ -228,26 +234,26 @@ public class Block { BigDecimal generatingBalance = parentBlock.calcNextBlockGeneratingBalance(); // Fetch our list of online accounts - List onlineAccounts = Controller.getInstance().getOnlineAccounts(); + List onlineAccounts = Controller.getInstance().getOnlineAccounts(); if (onlineAccounts.isEmpty()) throw new IllegalStateException("No online accounts - not even our own?"); // Find newest online accounts timestamp long onlineAccountsTimestamp = 0; - for (OnlineAccount onlineAccount : onlineAccounts) { - if (onlineAccount.getTimestamp() > onlineAccountsTimestamp) - onlineAccountsTimestamp = onlineAccount.getTimestamp(); + for (OnlineAccountData onlineAccountData : onlineAccounts) { + if (onlineAccountData.getTimestamp() > onlineAccountsTimestamp) + onlineAccountsTimestamp = onlineAccountData.getTimestamp(); } // Map using account index (in list of proxy forger accounts) - Map indexedOnlineAccounts = new HashMap<>(); - for (OnlineAccount onlineAccount : onlineAccounts) { + Map indexedOnlineAccounts = new HashMap<>(); + for (OnlineAccountData onlineAccountData : onlineAccounts) { // Disregard online accounts with different timestamps - if (onlineAccount.getTimestamp() != onlineAccountsTimestamp) + if (onlineAccountData.getTimestamp() != onlineAccountsTimestamp) continue; - int accountIndex = repository.getAccountRepository().getProxyAccountIndex(onlineAccount.getPublicKey()); - indexedOnlineAccounts.put(accountIndex, onlineAccount); + int accountIndex = repository.getAccountRepository().getProxyAccountIndex(onlineAccountData.getPublicKey()); + indexedOnlineAccounts.put(accountIndex, onlineAccountData); } List accountIndexes = new ArrayList<>(indexedOnlineAccounts.keySet()); accountIndexes.sort(null); @@ -262,8 +268,8 @@ public class Block { byte[] onlineAccountsSignatures = new byte[onlineAccountsCount * Transformer.SIGNATURE_LENGTH]; for (int i = 0; i < onlineAccountsCount; ++i) { Integer accountIndex = accountIndexes.get(i); - OnlineAccount onlineAccount = indexedOnlineAccounts.get(accountIndex); - System.arraycopy(onlineAccount.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH); + OnlineAccountData onlineAccountData = indexedOnlineAccounts.get(accountIndex); + System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH); } byte[] generatorSignature; @@ -421,8 +427,10 @@ public class Block { if (this.blockData.getHeight() == null) throw new IllegalStateException("Can't calculate next block's generating balance as this block's height is unset"); + final int blockDifficultyInterval = BlockChain.getInstance().getBlockDifficultyInterval(); + // This block not at the start of an interval? - if (this.blockData.getHeight() % BlockChain.getInstance().getBlockDifficultyInterval() != 0) + if (this.blockData.getHeight() % blockDifficultyInterval != 0) return this.blockData.getGeneratingBalance(); // Return cached calculation if we have one @@ -437,7 +445,7 @@ public class Block { BlockData firstBlock = this.blockData; try { - for (int i = 1; firstBlock != null && i < BlockChain.getInstance().getBlockDifficultyInterval(); ++i) + for (int i = 1; firstBlock != null && i < blockDifficultyInterval; ++i) firstBlock = blockRepo.fromSignature(firstBlock.getReference()); } catch (DataException e) { firstBlock = null; @@ -451,8 +459,7 @@ public class Block { long previousGeneratingTime = this.blockData.getTimestamp() - firstBlock.getTimestamp(); // Calculate expected forging time (in ms) for a whole interval based on this block's generating balance. - long expectedGeneratingTime = Block.calcForgingDelay(this.blockData.getGeneratingBalance()) * BlockChain.getInstance().getBlockDifficultyInterval() - * 1000; + long expectedGeneratingTime = Block.calcForgingDelay(this.blockData.getGeneratingBalance(), this.blockData.getHeight()) * blockDifficultyInterval; // Finally, scale generating balance such that faster than expected previous intervals produce larger generating balances. // NOTE: we have to use doubles and longs here to keep compatibility with Qora v1 results @@ -464,20 +471,17 @@ public class Block { return this.cachedNextGeneratingBalance; } - public static long calcBaseTarget(BigDecimal generatingBalance) { - generatingBalance = Block.minMaxBalance(generatingBalance); - return generatingBalance.longValue() * calcForgingDelay(generatingBalance); - } - /** * Return expected forging delay, in seconds, since previous block based on passed generating balance. */ - public static long calcForgingDelay(BigDecimal generatingBalance) { + public static long calcForgingDelay(BigDecimal generatingBalance, int previousBlockHeight) { generatingBalance = Block.minMaxBalance(generatingBalance); double percentageOfTotal = generatingBalance.divide(BlockChain.getInstance().getMaxBalance()).doubleValue(); - long actualBlockTime = (long) (BlockChain.getInstance().getMinBlockTime() - + ((BlockChain.getInstance().getMaxBlockTime() - BlockChain.getInstance().getMinBlockTime()) * (1 - percentageOfTotal))); + + BlockTimingByHeight blockTiming = BlockChain.getInstance().getBlockTimingByHeight(previousBlockHeight + 1); + + long actualBlockTime = (long) (blockTiming.target + (blockTiming.deviation * (1 - (2 * percentageOfTotal)))); return actualBlockTime; } @@ -723,14 +727,32 @@ public class Block { return Crypto.digest(Bytes.concat(Longs.toByteArray(height), publicKey)); } - public static BigInteger calcGeneratorDistance(BlockData parentBlockData, byte[] generatorPublicKey) { - final int parentHeight = parentBlockData.getHeight(); - final int thisHeight = parentHeight + 1; + public static BigInteger calcKeyDistance(int parentHeight, byte[] parentBlockSignature, byte[] publicKey) { + byte[] idealKey = calcIdealGeneratorPublicKey(parentHeight, parentBlockSignature); + byte[] perturbedKey = calcHeightPerturbedPublicKey(parentHeight + 1, publicKey); - // Convert all bits into unsigned BigInteger - BigInteger idealBI = new BigInteger(1, calcIdealGeneratorPublicKey(parentHeight, parentBlockData.getSignature())); - BigInteger generatorBI = new BigInteger(1, calcHeightPerturbedPublicKey(thisHeight, generatorPublicKey)); - return idealBI.subtract(generatorBI).abs(); + BigInteger keyDistance = MAX_DISTANCE.subtract(new BigInteger(idealKey).subtract(new BigInteger(perturbedKey)).abs()); + return keyDistance; + } + + public static BigInteger calcBlockWeight(int parentHeight, byte[] parentBlockSignature, BlockSummaryData blockSummaryData) { + BigInteger keyDistance = calcKeyDistance(parentHeight, parentBlockSignature, blockSummaryData.getGeneratorPublicKey()); + BigInteger weight = BigInteger.valueOf(blockSummaryData.getOnlineAccountsCount()).shiftLeft(ACCOUNTS_COUNT_SHIFT).add(keyDistance); + return weight; + } + + public static BigInteger calcChainWeight(int commonBlockHeight, byte[] commonBlockSignature, List blockSummaries) { + BigInteger cumulativeWeight = BigInteger.ZERO; + int parentHeight = commonBlockHeight; + byte[] parentBlockSignature = commonBlockSignature; + + for (BlockSummaryData blockSummaryData : blockSummaries) { + cumulativeWeight = cumulativeWeight.shiftLeft(CHAIN_WEIGHT_SHIFT).add(calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData)); + parentHeight = blockSummaryData.getHeight(); + parentBlockSignature = blockSummaryData.getSignature(); + } + + return cumulativeWeight; } /** @@ -746,7 +768,7 @@ public class Block { * So this block's timestamp is previous block's timestamp + 30s + 12s. */ public static long calcTimestamp(BlockData parentBlockData, byte[] generatorPublicKey) { - BigInteger distance = calcGeneratorDistance(parentBlockData, generatorPublicKey); + BigInteger distance = calcKeyDistance(parentBlockData.getHeight(), parentBlockData.getSignature(), generatorPublicKey); final int thisHeight = parentBlockData.getHeight() + 1; BlockTimingByHeight blockTiming = BlockChain.getInstance().getBlockTimingByHeight(thisHeight); diff --git a/src/main/java/org/qora/block/BlockChain.java b/src/main/java/org/qora/block/BlockChain.java index 792374f7..6a8b4d21 100644 --- a/src/main/java/org/qora/block/BlockChain.java +++ b/src/main/java/org/qora/block/BlockChain.java @@ -61,10 +61,6 @@ public class BlockChain { /** Number of blocks between recalculating block's generating balance. */ private int blockDifficultyInterval; - /** Minimum target time between blocks, in seconds. */ - private long minBlockTime; - /** Maximum target time between blocks, in seconds. */ - private long maxBlockTime; /** Maximum acceptable timestamp disagreement offset in milliseconds. */ private long blockTimestampMargin; /** Maximum block size, in bytes. */ @@ -253,14 +249,6 @@ public class BlockChain { return this.blockDifficultyInterval; } - public long getMinBlockTime() { - return this.minBlockTime; - } - - public long getMaxBlockTime() { - return this.maxBlockTime; - } - public long getBlockTimestampMargin() { return this.blockTimestampMargin; } @@ -361,22 +349,25 @@ public class BlockChain { /** Validate blockchain config read from JSON */ private void validateConfig() { - if (this.genesisInfo == null) { - LOGGER.error("No \"genesisInfo\" entry found in blockchain config"); - throw new RuntimeException("No \"genesisInfo\" entry found in blockchain config"); - } + if (this.genesisInfo == null) + Settings.throwValidationError("No \"genesisInfo\" entry found in blockchain config"); - if (this.featureTriggers == null) { - LOGGER.error("No \"featureTriggers\" entry found in blockchain config"); - throw new RuntimeException("No \"featureTriggers\" entry found in blockchain config"); - } + if (this.featureTriggers == null) + Settings.throwValidationError("No \"featureTriggers\" entry found in blockchain config"); + + if (this.blockTimestampMargin <= 0) + Settings.throwValidationError("Invalid \"blockTimestampMargin\" in blockchain config"); + + if (this.transactionExpiryPeriod <= 0) + Settings.throwValidationError("Invalid \"transactionExpiryPeriod\" in blockchain config"); + + if (this.maxBlockSize <= 0) + Settings.throwValidationError("Invalid \"maxBlockSize\" in blockchain config"); // Check all featureTriggers are present for (FeatureTrigger featureTrigger : FeatureTrigger.values()) - if (!this.featureTriggers.containsKey(featureTrigger.name())) { - LOGGER.error(String.format("Missing feature trigger \"%s\" in blockchain config", featureTrigger.name())); - throw new RuntimeException("Missing feature trigger in blockchain config"); - } + if (!this.featureTriggers.containsKey(featureTrigger.name())) + Settings.throwValidationError(String.format("Missing feature trigger \"%s\" in blockchain config", featureTrigger.name())); } /** diff --git a/src/main/java/org/qora/block/BlockGenerator.java b/src/main/java/org/qora/block/BlockGenerator.java index 7727358c..17b08655 100644 --- a/src/main/java/org/qora/block/BlockGenerator.java +++ b/src/main/java/org/qora/block/BlockGenerator.java @@ -113,14 +113,14 @@ public class BlockGenerator extends Thread { BlockData lastBlockData = blockRepository.getLastBlock(); // Disregard peers that have "misbehaved" recently - peers.removeIf(Controller.hasPeerMisbehaved); + peers.removeIf(Controller.hasMisbehaved); // Don't generate if we don't have enough connected peers as where would the transactions/consensus come from? if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) continue; // Disregard peers that don't have a recent block - peers.removeIf(peer -> peer.getLastBlockTimestamp() == null || peer.getLastBlockTimestamp() < minLatestBlockTimestamp); + peers.removeIf(Controller.hasNoRecentBlock); // If we have any peers with a recent block, but our latest block isn't recent // then we need to synchronize instead of generating. diff --git a/src/main/java/org/qora/controller/Controller.java b/src/main/java/org/qora/controller/Controller.java index 1fe1f731..4329b33f 100644 --- a/src/main/java/org/qora/controller/Controller.java +++ b/src/main/java/org/qora/controller/Controller.java @@ -30,13 +30,15 @@ import org.qora.account.PublicKeyAccount; import org.qora.api.ApiService; import org.qora.block.Block; import org.qora.block.BlockChain; +import org.qora.block.BlockChain.BlockTimingByHeight; import org.qora.block.BlockGenerator; import org.qora.controller.Synchronizer.SynchronizationResult; import org.qora.crypto.Crypto; import org.qora.data.account.ForgingAccountData; import org.qora.data.block.BlockData; import org.qora.data.block.BlockSummaryData; -import org.qora.data.network.OnlineAccount; +import org.qora.data.network.OnlineAccountData; +import org.qora.data.network.PeerChainTipData; import org.qora.data.network.PeerData; import org.qora.data.transaction.ArbitraryTransactionData; import org.qora.data.transaction.ArbitraryTransactionData.DataType; @@ -77,6 +79,7 @@ import org.qora.transaction.Transaction.TransactionType; import org.qora.transaction.Transaction.ValidationResult; import org.qora.ui.UiService; import org.qora.utils.Base58; +import org.qora.utils.ByteArray; import org.qora.utils.NTP; import org.qora.utils.Triple; @@ -129,10 +132,8 @@ public class Controller extends Thread { /** Whether we can generate new blocks, as reported by BlockGenerator. */ private volatile boolean isGenerationPossible = false; - /** Signature of peer's latest block that will result in no sync action needed (e.g. INFERIOR_CHAIN, NOTHING_TO_DO, OK). */ - private byte[] noSyncPeerBlockSignature = null; - /** Signature of our latest block that will result in no sync action needed (e.g. INFERIOR_CHAIN, NOTHING_TO_DO, OK). */ - private byte[] noSyncOurBlockSignature = null; + /** Latest block signatures from other peers that we know are on inferior chains. */ + List inferiorChainSignatures = new ArrayList<>(); /** * Map of recent requests for ARBITRARY transaction data payloads. @@ -157,7 +158,7 @@ public class Controller extends Thread { private final ReentrantLock blockchainLock = new ReentrantLock(); /** Cache of 'online accounts' */ - List onlineAccounts = new ArrayList<>(); + List onlineAccounts = new ArrayList<>(); // Constructors @@ -205,7 +206,7 @@ public class Controller extends Thread { return this.buildVersion; } - /** Returns current blockchain height, or 0 if there's a repository issue */ + /** Returns current blockchain height, or 0 if it's not available. */ public int getChainHeight() { BlockData blockData = this.chainTip.get(); if (blockData == null) @@ -214,7 +215,7 @@ public class Controller extends Thread { return blockData.getHeight(); } - /** Returns highest block, or null if there's a repository issue */ + /** Returns highest block, or null if it's not available. */ public BlockData getChainTip() { return this.chainTip.get(); } @@ -240,10 +241,15 @@ public class Controller extends Thread { Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); // Load/check settings, which potentially sets up blockchain config, etc. - if (args.length > 0) - Settings.fileInstance(args[0]); - else - Settings.getInstance(); + try { + if (args.length > 0) + Settings.fileInstance(args[0]); + else + Settings.getInstance(); + } catch (Throwable t) { + Gui.getInstance().fatalError("Settings file", t.getMessage()); + return; // Not System.exit() so that GUI can display error + } LOGGER.info("Starting NTP"); NTP.start(); @@ -406,54 +412,72 @@ public class Controller extends Thread { } } - private void potentiallySynchronize() throws InterruptedException { - final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); - if (minLatestBlockTimestamp == null) - return; + public static final Predicate hasMisbehaved = peer -> { + final Long lastMisbehaved = peer.getPeerData().getLastMisbehaved(); + return lastMisbehaved != null && lastMisbehaved > NTP.getTime() - MISBEHAVIOUR_COOLOFF; + }; + public static final Predicate hasNoRecentBlock = peer -> { + final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); + final PeerChainTipData peerChainTipData = peer.getChainTipData(); + return peerChainTipData == null || peerChainTipData.getLastBlockTimestamp() == null || peerChainTipData.getLastBlockTimestamp() < minLatestBlockTimestamp; + }; + + public static final Predicate hasNoOrSameBlock = peer -> { + final BlockData latestBlockData = getInstance().getChainTip(); + final PeerChainTipData peerChainTipData = peer.getChainTipData(); + return peerChainTipData == null || peerChainTipData.getLastBlockSignature() == null || Arrays.equals(latestBlockData.getSignature(), peerChainTipData.getLastBlockSignature()); + }; + + public static final Predicate hasOnlyGenesisBlock = peer -> { + final PeerChainTipData peerChainTipData = peer.getChainTipData(); + return peerChainTipData == null || peerChainTipData.getLastHeight() == null || peerChainTipData.getLastHeight() == 1; + }; + + public static final Predicate hasInferiorChainTip = peer -> { + final PeerChainTipData peerChainTipData = peer.getChainTipData(); + final List inferiorChainTips = getInstance().inferiorChainSignatures; + return peerChainTipData == null || peerChainTipData.getLastBlockSignature() == null || inferiorChainTips.contains(new ByteArray(peerChainTipData.getLastBlockSignature())); + }; + + private void potentiallySynchronize() throws InterruptedException { List peers = Network.getInstance().getUniqueHandshakedPeers(); // Disregard peers that have "misbehaved" recently - peers.removeIf(hasPeerMisbehaved); + peers.removeIf(hasMisbehaved); + + // Disregard peers that only have genesis block + peers.removeIf(hasOnlyGenesisBlock); + + // Disregard peers that don't have a recent block + peers.removeIf(hasNoRecentBlock); + + // Disregard peers that have no block signature or the same block signature as us + peers.removeIf(hasNoOrSameBlock); + + // Disregard peers that are on the same block as last sync attempt and we didn't like their chain + peers.removeIf(hasInferiorChainTip); // Check we have enough peers to potentially synchronize if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) return; - // Disregard peers that don't have a recent block - peers.removeIf(peer -> peer.getLastBlockTimestamp() == null || peer.getLastBlockTimestamp() < minLatestBlockTimestamp); + // Pick random peer to sync with + int index = new SecureRandom().nextInt(peers.size()); + Peer peer = peers.get(index); - BlockData latestBlockData = getChainTip(); - - // Disregard peers that have no block signature or the same block signature as us - peers.removeIf(peer -> peer.getLastBlockSignature() == null || Arrays.equals(latestBlockData.getSignature(), peer.getLastBlockSignature())); - - // Disregard peers that are on the same block as last sync attempt and we didn't like their chain - if (noSyncOurBlockSignature != null && Arrays.equals(noSyncOurBlockSignature, latestBlockData.getSignature())) - peers.removeIf(peer -> Arrays.equals(noSyncPeerBlockSignature, peer.getLastBlockSignature())); - - if (!peers.isEmpty()) { - // Pick random peer to sync with - int index = new SecureRandom().nextInt(peers.size()); - Peer peer = peers.get(index); - - actuallySynchronize(peer, false); - } + actuallySynchronize(peer, false); } public SynchronizationResult actuallySynchronize(Peer peer, boolean force) throws InterruptedException { BlockData latestBlockData = getChainTip(); - noSyncOurBlockSignature = null; - noSyncPeerBlockSignature = null; - SynchronizationResult syncResult = Synchronizer.getInstance().synchronize(peer, force); switch (syncResult) { case GENESIS_ONLY: case NO_COMMON_BLOCK: - case TOO_FAR_BEHIND: case TOO_DIVERGENT: - case INVALID_DATA: + case INVALID_DATA: { // These are more serious results that warrant a cool-off LOGGER.info(String.format("Failed to synchronize with peer %s (%s) - cooling off", peer, syncResult.name())); @@ -470,13 +494,22 @@ public class Controller extends Thread { LOGGER.warn("Repository issue while updating peer synchronization info", e); } break; + } + + case INFERIOR_CHAIN: { + // Update our list of inferior chain tips + ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature()); + if (!inferiorChainSignatures.contains(inferiorChainSignature)) + inferiorChainSignatures.add(inferiorChainSignature); - case INFERIOR_CHAIN: - noSyncOurBlockSignature = latestBlockData.getSignature(); - noSyncPeerBlockSignature = peer.getLastBlockSignature(); // These are minor failure results so fine to try again LOGGER.debug(() -> String.format("Refused to synchronize with peer %s (%s)", peer, syncResult.name())); + + // Notify peer of our superior chain + if (!peer.sendMessage(Network.getInstance().buildHeightMessage(peer, latestBlockData))) + peer.disconnect("failed to notify peer of our superior chain"); break; + } case NO_REPLY: case NO_BLOCKCHAIN_LOCK: @@ -488,14 +521,18 @@ public class Controller extends Thread { case OK: requestSysTrayUpdate = true; // fall-through... - case NOTHING_TO_DO: - noSyncOurBlockSignature = latestBlockData.getSignature(); - noSyncPeerBlockSignature = peer.getLastBlockSignature(); + case NOTHING_TO_DO: { + // Update our list of inferior chain tips + ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature()); + if (!inferiorChainSignatures.contains(inferiorChainSignature)) + inferiorChainSignatures.add(inferiorChainSignature); + LOGGER.debug(() -> String.format("Synchronized with peer %s (%s)", peer, syncResult.name())); break; + } } - // Broadcast our new chain tip (if changed) + // Has our chain tip changed? BlockData newLatestBlockData; try (final Repository repository = RepositoryManager.getRepository()) { @@ -506,9 +543,14 @@ public class Controller extends Thread { return syncResult; } - if (!Arrays.equals(newLatestBlockData.getSignature(), latestBlockData.getSignature())) + if (!Arrays.equals(newLatestBlockData.getSignature(), latestBlockData.getSignature())) { + // Broadcast our new chain tip Network.getInstance().broadcast(recipientPeer -> Network.getInstance().buildHeightMessage(recipientPeer, newLatestBlockData)); + // Reset our cache of inferior chains + inferiorChainSignatures.clear(); + } + return syncResult; } @@ -725,17 +767,9 @@ public class Controller extends Thread { if (connectedPeer.getPeerId() == null || !Arrays.equals(connectedPeer.getPeerId(), peer.getPeerId())) continue; - // We want to update atomically so use lock - ReentrantLock peerLock = connectedPeer.getPeerDataLock(); - peerLock.lock(); - try { - connectedPeer.setLastHeight(blockData.getHeight()); - connectedPeer.setLastBlockSignature(blockData.getSignature()); - connectedPeer.setLastBlockTimestamp(blockData.getTimestamp()); - connectedPeer.setLastBlockGenerator(blockData.getGeneratorPublicKey()); - } finally { - peerLock.unlock(); - } + // Update peer chain tip data + PeerChainTipData newChainTipData = new PeerChainTipData(blockData.getHeight(), blockData.getSignature(), blockData.getTimestamp(), blockData.getGeneratorPublicKey()); + connectedPeer.setChainTipData(newChainTipData); } // Potentially synchronize @@ -754,7 +788,9 @@ public class Controller extends Thread { if (connectedPeer.getPeerId() == null || !Arrays.equals(connectedPeer.getPeerId(), peer.getPeerId())) continue; - connectedPeer.setLastHeight(heightMessage.getHeight()); + // Update peer chain tip data + PeerChainTipData newChainTipData = new PeerChainTipData(heightMessage.getHeight(), null, null, null); + connectedPeer.setChainTipData(newChainTipData); } // Potentially synchronize @@ -769,7 +805,7 @@ public class Controller extends Thread { // If peer is inbound and we've not updated their height // then this is probably their initial HEIGHT_V2 message // so they need a corresponding HEIGHT_V2 message from us - if (!peer.isOutbound() && peer.getLastHeight() == null) + if (!peer.isOutbound() && (peer.getChainTipData() == null || peer.getChainTipData().getLastHeight() == null)) peer.sendMessage(Network.getInstance().buildHeightMessage(peer, getChainTip())); // Update all peers with same ID @@ -780,17 +816,9 @@ public class Controller extends Thread { if (connectedPeer.getPeerId() == null || !Arrays.equals(connectedPeer.getPeerId(), peer.getPeerId())) continue; - // We want to update atomically so use lock - ReentrantLock peerLock = connectedPeer.getPeerDataLock(); - peerLock.lock(); - try { - connectedPeer.setLastHeight(heightV2Message.getHeight()); - connectedPeer.setLastBlockSignature(heightV2Message.getSignature()); - connectedPeer.setLastBlockTimestamp(heightV2Message.getTimestamp()); - connectedPeer.setLastBlockGenerator(heightV2Message.getGenerator()); - } finally { - peerLock.unlock(); - } + // Update peer chain tip data + PeerChainTipData newChainTipData = new PeerChainTipData(heightV2Message.getHeight(), heightV2Message.getSignature(), heightV2Message.getTimestamp(), heightV2Message.getGenerator()); + connectedPeer.setChainTipData(newChainTipData); } // Potentially synchronize @@ -1168,24 +1196,24 @@ public class Controller extends Thread { case GET_ONLINE_ACCOUNTS: { GetOnlineAccountsMessage getOnlineAccountsMessage = (GetOnlineAccountsMessage) message; - List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); + List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts - List accountsToSend; + List accountsToSend; synchronized (this.onlineAccounts) { accountsToSend = new ArrayList<>(this.onlineAccounts); } - Iterator iterator = accountsToSend.iterator(); + Iterator iterator = accountsToSend.iterator(); SEND_ITERATOR: while (iterator.hasNext()) { - OnlineAccount onlineAccount = iterator.next(); + OnlineAccountData onlineAccountData = iterator.next(); for (int i = 0; i < excludeAccounts.size(); ++i) { - OnlineAccount excludeAccount = excludeAccounts.get(i); + OnlineAccountData excludeAccountData = excludeAccounts.get(i); - if (onlineAccount.getTimestamp() == excludeAccount.getTimestamp() && Arrays.equals(onlineAccount.getPublicKey(), excludeAccount.getPublicKey())) { + if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) { iterator.remove(); continue SEND_ITERATOR; } @@ -1203,11 +1231,11 @@ public class Controller extends Thread { case ONLINE_ACCOUNTS: { OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message; - List onlineAccounts = onlineAccountsMessage.getOnlineAccounts(); + List onlineAccounts = onlineAccountsMessage.getOnlineAccounts(); LOGGER.trace(() -> String.format("Received %d online accounts from %s", onlineAccounts.size(), peer)); - for (OnlineAccount onlineAccount : onlineAccounts) - this.verifyAndAddAccount(onlineAccount); + for (OnlineAccountData onlineAccountData : onlineAccounts) + this.verifyAndAddAccount(onlineAccountData); break; } @@ -1220,35 +1248,35 @@ public class Controller extends Thread { // Utilities - private void verifyAndAddAccount(OnlineAccount onlineAccount) { + private void verifyAndAddAccount(OnlineAccountData onlineAccountData) { // We would check timestamp is 'recent' here // Verify - byte[] data = Longs.toByteArray(onlineAccount.getTimestamp()); - PublicKeyAccount otherAccount = new PublicKeyAccount(null, onlineAccount.getPublicKey()); - if (!otherAccount.verify(onlineAccount.getSignature(), data)) { + byte[] data = Longs.toByteArray(onlineAccountData.getTimestamp()); + PublicKeyAccount otherAccount = new PublicKeyAccount(null, onlineAccountData.getPublicKey()); + if (!otherAccount.verify(onlineAccountData.getSignature(), data)) { LOGGER.trace(() -> String.format("Rejecting invalid online account %s", otherAccount.getAddress())); return; } synchronized (this.onlineAccounts) { - OnlineAccount existingAccount = this.onlineAccounts.stream().filter(account -> Arrays.equals(account.getPublicKey(), onlineAccount.getPublicKey())).findFirst().orElse(null); + OnlineAccountData existingAccountData = this.onlineAccounts.stream().filter(account -> Arrays.equals(account.getPublicKey(), onlineAccountData.getPublicKey())).findFirst().orElse(null); - if (existingAccount != null) { - if (existingAccount.getTimestamp() < onlineAccount.getTimestamp()) { - this.onlineAccounts.remove(existingAccount); + if (existingAccountData != null) { + if (existingAccountData.getTimestamp() < onlineAccountData.getTimestamp()) { + this.onlineAccounts.remove(existingAccountData); - LOGGER.trace(() -> String.format("Updated online account %s with timestamp %d (was %d)", otherAccount.getAddress(), onlineAccount.getTimestamp(), existingAccount.getTimestamp())); + LOGGER.trace(() -> String.format("Updated online account %s with timestamp %d (was %d)", otherAccount.getAddress(), onlineAccountData.getTimestamp(), existingAccountData.getTimestamp())); } else { LOGGER.trace(() -> String.format("Not updating existing online account %s", otherAccount.getAddress())); return; } } else { - LOGGER.trace(() -> String.format("Added online account %s with timestamp %d", otherAccount.getAddress(), onlineAccount.getTimestamp())); + LOGGER.trace(() -> String.format("Added online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp())); } - this.onlineAccounts.add(onlineAccount); + this.onlineAccounts.add(onlineAccountData); } } @@ -1260,16 +1288,16 @@ public class Controller extends Thread { // Expire old entries final long cutoffThreshold = now - LAST_SEEN_EXPIRY_PERIOD; synchronized (this.onlineAccounts) { - Iterator iterator = this.onlineAccounts.iterator(); + Iterator iterator = this.onlineAccounts.iterator(); while (iterator.hasNext()) { - OnlineAccount onlineAccount = iterator.next(); + OnlineAccountData onlineAccountData = iterator.next(); - if (onlineAccount.getTimestamp() < cutoffThreshold) { + if (onlineAccountData.getTimestamp() < cutoffThreshold) { iterator.remove(); LOGGER.trace(() -> { - PublicKeyAccount otherAccount = new PublicKeyAccount(null, onlineAccount.getPublicKey()); - return String.format("Removed expired online account %s with timestamp %d", otherAccount.getAddress(), onlineAccount.getTimestamp()); + PublicKeyAccount otherAccount = new PublicKeyAccount(null, onlineAccountData.getPublicKey()); + return String.format("Removed expired online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp()); }); } } @@ -1322,7 +1350,7 @@ public class Controller extends Thread { boolean hasInfoChanged = false; byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp); - List ourOnlineAccounts = new ArrayList<>(); + List ourOnlineAccounts = new ArrayList<>(); FORGING_ACCOUNTS: for (ForgingAccountData forgingAccountData : forgingAccounts) { @@ -1332,28 +1360,28 @@ public class Controller extends Thread { byte[] publicKey = forgingAccount.getPublicKey(); // Our account is online - OnlineAccount onlineAccount = new OnlineAccount(onlineAccountsTimestamp, signature, publicKey); + OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey); synchronized (this.onlineAccounts) { - Iterator iterator = this.onlineAccounts.iterator(); + Iterator iterator = this.onlineAccounts.iterator(); while (iterator.hasNext()) { - OnlineAccount account = iterator.next(); + OnlineAccountData existingOnlineAccountData = iterator.next(); - if (Arrays.equals(account.getPublicKey(), forgingAccount.getPublicKey())) { - // If onlineAccount is already present, with same timestamp, then move on to next forgingAccount - if (account.getTimestamp() == onlineAccountsTimestamp) + if (Arrays.equals(existingOnlineAccountData.getPublicKey(), ourOnlineAccountData.getPublicKey())) { + // If our online account is already present, with same timestamp, then move on to next forgingAccount + if (existingOnlineAccountData.getTimestamp() == onlineAccountsTimestamp) continue FORGING_ACCOUNTS; - // If onlineAccount is already present, but with older timestamp, then remove it + // If our online account is already present, but with older timestamp, then remove it iterator.remove(); break; } } - this.onlineAccounts.add(onlineAccount); + this.onlineAccounts.add(ourOnlineAccountData); } LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", forgingAccount.getAddress(), onlineAccountsTimestamp)); - ourOnlineAccounts.add(onlineAccount); + ourOnlineAccounts.add(ourOnlineAccountData); hasInfoChanged = true; } @@ -1370,7 +1398,7 @@ public class Controller extends Thread { return (timestamp / ONLINE_TIMESTAMP_MODULUS) * ONLINE_TIMESTAMP_MODULUS; } - public List getOnlineAccounts() { + public List getOnlineAccounts() { final long onlineTimestamp = Controller.toOnlineAccountTimestamp(NTP.getTime()); synchronized (this.onlineAccounts) { @@ -1423,45 +1451,93 @@ public class Controller extends Thread { } } - public static final Predicate hasPeerMisbehaved = peer -> { - Long lastMisbehaved = peer.getPeerData().getLastMisbehaved(); - return lastMisbehaved != null && lastMisbehaved > NTP.getTime() - MISBEHAVIOUR_COOLOFF; - }; + /** Returns a list of peers that are not misbehaving, and have a recent block. */ + public List getRecentBehavingPeers() { + final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null) + return null; + + List peers = Network.getInstance().getUniqueHandshakedPeers(); + + // Filter out unsuitable peers + Iterator iterator = peers.iterator(); + while (iterator.hasNext()) { + final Peer peer = iterator.next(); + + final PeerData peerData = peer.getPeerData(); + if (peerData == null) { + iterator.remove(); + continue; + } + + // Disregard peers that have "misbehaved" recently + if (hasMisbehaved.test(peer)) { + iterator.remove(); + continue; + } + + final PeerChainTipData peerChainTipData = peer.getChainTipData(); + if (peerChainTipData == null) { + iterator.remove(); + continue; + } + + // Disregard peers that don't have a recent block + if (peerChainTipData.getLastBlockTimestamp() == null || peerChainTipData.getLastBlockTimestamp() < minLatestBlockTimestamp) { + iterator.remove(); + continue; + } + } + + return peers; + } /** Returns whether we think our node has up-to-date blockchain based on our info about other peers. */ public boolean isUpToDate() { + // Do we even have a vaguely recent block? final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp(); if (minLatestBlockTimestamp == null) return false; - BlockData latestBlockData = getChainTip(); - - // Is our blockchain too old? - if (latestBlockData.getTimestamp() < minLatestBlockTimestamp) + final BlockData latestBlockData = getChainTip(); + if (latestBlockData == null || latestBlockData.getTimestamp() < minLatestBlockTimestamp) return false; List peers = Network.getInstance().getUniqueHandshakedPeers(); - - // Disregard peers that have "misbehaved" recently - peers.removeIf(hasPeerMisbehaved); - - // Check we have enough peers to potentially synchronize/generator - if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) + if (peers == null) return false; + // Disregard peers that have "misbehaved" recently + peers.removeIf(hasMisbehaved); + // Disregard peers that don't have a recent block - peers.removeIf(peer -> peer.getLastBlockTimestamp() == null || peer.getLastBlockTimestamp() < minLatestBlockTimestamp); + peers.removeIf(hasNoRecentBlock); + + // Check we have enough peers to potentially synchronize/generate + if (peers.size() < Settings.getInstance().getMinBlockchainPeers()) + return false; // If we don't have any peers left then can't synchronize, therefore consider ourself not up to date return !peers.isEmpty(); } + /** Returns minimum block timestamp for block to be considered 'recent', or null if NTP not synced. */ public static Long getMinimumLatestBlockTimestamp() { Long now = NTP.getTime(); if (now == null) return null; - return now - BlockChain.getInstance().getMaxBlockTime() * 1000L * MAX_BLOCKCHAIN_TIP_AGE; + int height = getInstance().getChainHeight(); + if (height == 0) + return null; + + long offset = 0; + for (int ai = 0; height >= 1 && ai < MAX_BLOCKCHAIN_TIP_AGE; ++ai, --height) { + BlockTimingByHeight blockTiming = BlockChain.getInstance().getBlockTimingByHeight(height); + offset += blockTiming.target + blockTiming.deviation; + } + + return now - offset; } } diff --git a/src/main/java/org/qora/controller/Synchronizer.java b/src/main/java/org/qora/controller/Synchronizer.java index c7720f7d..64085852 100644 --- a/src/main/java/org/qora/controller/Synchronizer.java +++ b/src/main/java/org/qora/controller/Synchronizer.java @@ -1,8 +1,12 @@ package org.qora.controller; +import java.math.BigInteger; +import java.text.DecimalFormat; +import java.text.NumberFormat; import java.util.Collections; import java.util.List; import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -10,6 +14,7 @@ import org.qora.block.Block; import org.qora.block.Block.ValidationResult; import org.qora.data.block.BlockData; import org.qora.data.block.BlockSummaryData; +import org.qora.data.network.PeerChainTipData; import org.qora.data.transaction.TransactionData; import org.qora.network.Peer; import org.qora.network.message.BlockMessage; @@ -33,16 +38,13 @@ public class Synchronizer { private static final int INITIAL_BLOCK_STEP = 8; private static final int MAXIMUM_BLOCK_STEP = 500; - private static final int MAXIMUM_HEIGHT_DELTA = 300; // XXX move to blockchain config? - private static final int MAXIMUM_COMMON_DELTA = 60; // XXX move to blockchain config? + private static final int MAXIMUM_COMMON_DELTA = 1440; // XXX move to Settings? private static final int SYNC_BATCH_SIZE = 200; private static Synchronizer instance; - private Repository repository; - public enum SynchronizationResult { - OK, NOTHING_TO_DO, GENESIS_ONLY, NO_COMMON_BLOCK, TOO_FAR_BEHIND, TOO_DIVERGENT, NO_REPLY, INFERIOR_CHAIN, INVALID_DATA, NO_BLOCKCHAIN_LOCK, REPOSITORY_ISSUE; + OK, NOTHING_TO_DO, GENESIS_ONLY, NO_COMMON_BLOCK, TOO_DIVERGENT, NO_REPLY, INFERIOR_CHAIN, INVALID_DATA, NO_BLOCKCHAIN_LOCK, REPOSITORY_ISSUE; } // Constructors @@ -78,55 +80,34 @@ public class Synchronizer { try { try (final Repository repository = RepositoryManager.getRepository()) { try { - this.repository = repository; - final BlockData ourLatestBlockData = this.repository.getBlockRepository().getLastBlock(); + final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock(); final int ourInitialHeight = ourLatestBlockData.getHeight(); - int ourHeight = ourInitialHeight; - int peerHeight; - byte[] peersLastBlockSignature; - - ReentrantLock peerLock = peer.getPeerDataLock(); - peerLock.lockInterruptibly(); - try { - peerHeight = peer.getLastHeight(); - peersLastBlockSignature = peer.getLastBlockSignature(); - } finally { - peerLock.unlock(); - } - - // If peer is at genesis block then peer has no blocks so ignore them for a while - if (peerHeight == 1) - return SynchronizationResult.GENESIS_ONLY; - - // If peer is too far behind us then don't them. - int minHeight = ourHeight - MAXIMUM_HEIGHT_DELTA; - if (!force && peerHeight < minHeight) { - LOGGER.info(String.format("Peer %s height %d is too far behind our height %d", peer, peerHeight, ourHeight)); - return SynchronizationResult.TOO_FAR_BEHIND; - } + PeerChainTipData peerChainTipData = peer.getChainTipData(); + int peerHeight = peerChainTipData.getLastHeight(); + byte[] peersLastBlockSignature = peerChainTipData.getLastBlockSignature(); byte[] ourLastBlockSignature = ourLatestBlockData.getSignature(); LOGGER.debug(String.format("Synchronizing with peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer, - peerHeight, Base58.encode(peersLastBlockSignature), peer.getLastBlockTimestamp(), - ourHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp())); + peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(), + ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp())); - List signatures = findSignaturesFromCommonBlock(peer, ourHeight); - if (signatures == null) { + List peerBlockSummaries = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight); + if (peerBlockSummaries == null) { LOGGER.info(String.format("Error while trying to find common block with peer %s", peer)); return SynchronizationResult.NO_REPLY; } - if (signatures.isEmpty()) { + if (peerBlockSummaries.isEmpty()) { LOGGER.info(String.format("Failure to find common block with peer %s", peer)); return SynchronizationResult.NO_COMMON_BLOCK; } - // First signature is common block - BlockData commonBlockData = this.repository.getBlockRepository().fromSignature(signatures.get(0)); + // First summary is common block + BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature()); final int commonBlockHeight = commonBlockData.getHeight(); LOGGER.debug(String.format("Common block with peer %s is at height %d, sig %.8s, ts %d", peer, commonBlockHeight, Base58.encode(commonBlockData.getSignature()), commonBlockData.getTimestamp())); - signatures.remove(0); + peerBlockSummaries.remove(0); // If common block height is higher than peer's last reported height // then peer must have a very recent sync. Update our idea of peer's height. @@ -137,7 +118,7 @@ public class Synchronizer { // If common block is peer's latest block then we simply have the same, or longer, chain to peer, so exit now if (commonBlockHeight == peerHeight) { - if (peerHeight == ourHeight) + if (peerHeight == ourInitialHeight) LOGGER.debug(String.format("We have the same blockchain as peer %s", peer)); else LOGGER.debug(String.format("We have the same blockchain as peer %s, but longer", peer)); @@ -146,14 +127,13 @@ public class Synchronizer { } // If common block is too far behind us then we're on massively different forks so give up. - int minCommonHeight = ourHeight - MAXIMUM_COMMON_DELTA; + int minCommonHeight = ourInitialHeight - MAXIMUM_COMMON_DELTA; if (!force && commonBlockHeight < minCommonHeight) { LOGGER.info(String.format("Blockchain too divergent with peer %s", peer)); return SynchronizationResult.TOO_DIVERGENT; } - // If we both have blocks after common block then decide whether we want to sync - int highestMutualHeight = Math.min(peerHeight, ourHeight); + // At this point, we both have blocks after common block // If our latest block is very old, we're very behind and should ditch our fork. final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); @@ -162,45 +142,63 @@ public class Synchronizer { if (ourInitialHeight > commonBlockHeight && ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) { LOGGER.info(String.format("Ditching our chain after height %d as our latest block is very old", commonBlockHeight)); - highestMutualHeight = commonBlockHeight; - } + } else if (!force) { + // Compare chain weights - for (int height = commonBlockHeight + 1; height <= highestMutualHeight; ++height) { - int sigIndex = height - commonBlockHeight - 1; + LOGGER.debug(String.format("Comparing chains from block %d with peer %s", commonBlockHeight + 1, peer)); - // Do we need more signatures? - if (signatures.size() - 1 < sigIndex) { - // Grab more signatures - byte[] previousSignature = sigIndex == 0 ? commonBlockData.getSignature() : signatures.get(sigIndex - 1); - List moreSignatures = this.getBlockSignatures(peer, previousSignature, MAXIMUM_BLOCK_STEP); - if (moreSignatures == null || moreSignatures.isEmpty()) { - LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, - height - 1, Base58.encode(previousSignature))); + // Fetch remaining peer's block summaries (which we also use to fill signatures list) + int peerBlockCount = peerHeight - commonBlockHeight; + byte[] previousSignature; + if (peerBlockSummaries.isEmpty()) + previousSignature = commonBlockData.getSignature(); + else + previousSignature = peerBlockSummaries.get(peerBlockSummaries.size() - 1).getSignature(); + + while (peerBlockSummaries.size() < peerBlockCount) { + int lastSummaryHeight = commonBlockHeight + peerBlockSummaries.size(); + + List moreBlockSummaries = this.getBlockSummaries(peer, previousSignature, peerBlockCount - peerBlockSummaries.size()); + + if (moreBlockSummaries == null || moreBlockSummaries.isEmpty()) { + LOGGER.info(String.format("Peer %s failed to respond with block summaries after height %d, sig %.8s", peer, + lastSummaryHeight, Base58.encode(previousSignature))); return SynchronizationResult.NO_REPLY; } - signatures.addAll(moreSignatures); - } + // Check peer sent valid heights + for (int i = 0; i < moreBlockSummaries.size(); ++i) { + ++lastSummaryHeight; - byte[] ourSignature = this.repository.getBlockRepository().fromHeight(height).getSignature(); - byte[] peerSignature = signatures.get(sigIndex); + BlockSummaryData blockSummary = moreBlockSummaries.get(i); - for (int i = 0; i < ourSignature.length; ++i) { - /* - * If our byte is lower, we don't synchronize with this peer, - * if their byte is lower, check next height, - * (if bytes are equal, try next byte). - */ - if (ourSignature[i] < peerSignature[i]) { - LOGGER.info(String.format("Not synchronizing with peer %s as we have better block at height %d", peer, height)); - return SynchronizationResult.INFERIOR_CHAIN; + if (blockSummary.getHeight() != lastSummaryHeight) { + LOGGER.info(String.format("Peer %s responded with invalid block summary for height %d, sig %.8s", peer, + lastSummaryHeight, Base58.encode(blockSummary.getSignature()))); + return SynchronizationResult.NO_REPLY; + } } - if (peerSignature[i] < ourSignature[i]) - break; + peerBlockSummaries.addAll(moreBlockSummaries); + } + + // Fetch our corresponding block summaries + List ourBlockSummaries = repository.getBlockRepository().getBlockSummaries(commonBlockHeight + 1, ourInitialHeight); + + // Calculate cumulative chain weights of both blockchain subsets, from common block to highest mutual block. + BigInteger ourChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockData.getSignature(), ourBlockSummaries); + BigInteger peerChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockData.getSignature(), peerBlockSummaries); + + // If our blockchain has greater weight then don't synchronize with peer + if (ourChainWeight.compareTo(peerChainWeight) >= 0) { + LOGGER.debug(String.format("Not synchronizing with peer %s as we have better blockchain", peer)); + NumberFormat formatter = new DecimalFormat("0.###E0"); + LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", formatter.format(ourChainWeight), formatter.format(peerChainWeight))); + return SynchronizationResult.INFERIOR_CHAIN; } } + int ourHeight = ourInitialHeight; if (ourHeight > commonBlockHeight) { // Unwind to common block (unless common block is our latest block) LOGGER.debug(String.format("Orphaning blocks back to height %d", commonBlockHeight)); @@ -219,40 +217,44 @@ public class Synchronizer { } // Fetch, and apply, blocks from peer - byte[] signature = commonBlockData.getSignature(); + byte[] latestPeerSignature = commonBlockData.getSignature(); int maxBatchHeight = commonBlockHeight + SYNC_BATCH_SIZE; + + // Convert any block summaries from above into signatures to request from peer + List peerBlockSignatures = peerBlockSummaries.stream().map(blockSummaryData -> blockSummaryData.getSignature()).collect(Collectors.toList()); + while (ourHeight < peerHeight && ourHeight < maxBatchHeight) { // Do we need more signatures? - if (signatures.isEmpty()) { + if (peerBlockSummaries.isEmpty()) { int numberRequested = maxBatchHeight - ourHeight; LOGGER.trace(String.format("Requesting %d signature%s after height %d", numberRequested, (numberRequested != 1 ? "s": ""), ourHeight)); - signatures = this.getBlockSignatures(peer, signature, numberRequested); + peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested); - if (signatures == null || signatures.isEmpty()) { + if (peerBlockSummaries == null || peerBlockSummaries.isEmpty()) { LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer, - ourHeight, Base58.encode(signature))); + ourHeight, Base58.encode(latestPeerSignature))); return SynchronizationResult.NO_REPLY; } - LOGGER.trace(String.format("Received %s signature%s", signatures.size(), (signatures.size() != 1 ? "s" : ""))); + LOGGER.trace(String.format("Received %s signature%s", peerBlockSummaries.size(), (peerBlockSummaries.size() != 1 ? "s" : ""))); } - signature = signatures.get(0); - signatures.remove(0); + latestPeerSignature = peerBlockSignatures.get(0); + peerBlockSignatures.remove(0); ++ourHeight; - Block newBlock = this.fetchBlock(repository, peer, signature); + Block newBlock = this.fetchBlock(repository, peer, latestPeerSignature); if (newBlock == null) { LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, - ourHeight, Base58.encode(signature))); + ourHeight, Base58.encode(latestPeerSignature))); return SynchronizationResult.NO_REPLY; } if (!newBlock.isSignatureValid()) { LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer, - ourHeight, Base58.encode(signature))); + ourHeight, Base58.encode(latestPeerSignature))); return SynchronizationResult.INVALID_DATA; } @@ -263,7 +265,7 @@ public class Synchronizer { ValidationResult blockResult = newBlock.isValid(); if (blockResult != ValidationResult.OK) { LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer, - ourHeight, Base58.encode(signature), blockResult.name())); + ourHeight, Base58.encode(latestPeerSignature), blockResult.name())); return SynchronizationResult.INVALID_DATA; } @@ -283,7 +285,7 @@ public class Synchronizer { // Commit repository.saveChanges(); - final BlockData newLatestBlockData = this.repository.getBlockRepository().getLastBlock(); + final BlockData newLatestBlockData = repository.getBlockRepository().getLastBlock(); LOGGER.info(String.format("Synchronized with peer %s to height %d, sig %.8s, ts: %d", peer, newLatestBlockData.getHeight(), Base58.encode(newLatestBlockData.getSignature()), newLatestBlockData.getTimestamp())); @@ -291,7 +293,6 @@ public class Synchronizer { return SynchronizationResult.OK; } finally { repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong - this.repository = null; } } catch (DataException e) { LOGGER.error("Repository issue during synchronization with peer", e); @@ -303,45 +304,45 @@ public class Synchronizer { } /** - * Returns list of peer's block signatures starting with common block with peer. + * Returns list of peer's block summaries starting with common block with peer. * * @param peer - * @return block signatures, or empty list if no common block, or null if there was an issue + * @return block summaries, or empty list if no common block, or null if there was an issue * @throws DataException * @throws InterruptedException */ - private List findSignaturesFromCommonBlock(Peer peer, int ourHeight) throws DataException, InterruptedException { + private List fetchSummariesFromCommonBlock(Repository repository, Peer peer, int ourHeight) throws DataException, InterruptedException { // Start by asking for a few recent block hashes as this will cover a majority of reorgs // Failing that, back off exponentially int step = INITIAL_BLOCK_STEP; - List blockSignatures = null; + List blockSummaries = null; + int testHeight = Math.max(ourHeight - step, 1); - byte[] testSignature = null; + BlockData testBlockData = null; while (testHeight >= 1) { // Fetch our block signature at this height - BlockData testBlockData = this.repository.getBlockRepository().fromHeight(testHeight); + testBlockData = repository.getBlockRepository().fromHeight(testHeight); if (testBlockData == null) { // Not found? But we've locked the blockchain and height is below blockchain's tip! LOGGER.error("Failed to get block at height lower than blockchain tip during synchronization?"); return null; } - testSignature = testBlockData.getSignature(); - // Ask for block signatures since test block's signature - LOGGER.trace(String.format("Requesting %d signature%s after height %d", step, (step != 1 ? "s": ""), testHeight)); - blockSignatures = this.getBlockSignatures(peer, testSignature, step); + byte[] testSignature = testBlockData.getSignature(); + LOGGER.trace(String.format("Requesting %d summar%s after height %d", step, (step != 1 ? "ies": "y"), testHeight)); + blockSummaries = this.getBlockSummaries(peer, testSignature, step); - if (blockSignatures == null) + if (blockSummaries == null) // No response - give up this time return null; - LOGGER.trace(String.format("Received %s signature%s", blockSignatures.size(), (blockSignatures.size() != 1 ? "s" : ""))); + LOGGER.trace(String.format("Received %s summar%s", blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"))); // Empty list means remote peer is unaware of test signature OR has no new blocks after test signature - if (!blockSignatures.isEmpty()) + if (!blockSummaries.isEmpty()) // We have entries so we have found a common block break; @@ -361,22 +362,22 @@ public class Synchronizer { testHeight = Math.max(testHeight - step, 1); } - // Prepend common block's signature as first block sig - blockSignatures.add(0, testSignature); + // Prepend test block's summary as first block summary, as summaries returned are *after* test block + BlockSummaryData testBlockSummary = new BlockSummaryData(testBlockData); + blockSummaries.add(0, testBlockSummary); - // Work through returned signatures to get closer common block - // Do this by trimming all-but-one leading known signatures - for (int i = blockSignatures.size() - 1; i > 0; --i) { - BlockData blockData = this.repository.getBlockRepository().fromSignature(blockSignatures.get(i)); - - if (blockData != null) { + // Trim summaries so that first summary is common block. + // Currently we work back from the end until we hit a block we also have. + // TODO: rewrite as modified binary search! + for (int i = blockSummaries.size() - 1; i > 0; --i) { + if (repository.getBlockRepository().exists(blockSummaries.get(i).getSignature())) { // Note: index i isn't cleared: List.subList is fromIndex inclusive to toIndex exclusive - blockSignatures.subList(0, i).clear(); + blockSummaries.subList(0, i).clear(); break; } } - return blockSignatures; + return blockSummaries; } private List getBlockSummaries(Peer peer, byte[] parentSignature, int numberRequested) throws InterruptedException { diff --git a/src/main/java/org/qora/data/network/OnlineAccount.java b/src/main/java/org/qora/data/network/OnlineAccountData.java similarity index 85% rename from src/main/java/org/qora/data/network/OnlineAccount.java rename to src/main/java/org/qora/data/network/OnlineAccountData.java index daf6a3b5..d7375760 100644 --- a/src/main/java/org/qora/data/network/OnlineAccount.java +++ b/src/main/java/org/qora/data/network/OnlineAccountData.java @@ -8,7 +8,7 @@ import org.qora.account.PublicKeyAccount; // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) -public class OnlineAccount { +public class OnlineAccountData { protected long timestamp; protected byte[] signature; @@ -17,10 +17,10 @@ public class OnlineAccount { // Constructors // necessary for JAXB serialization - protected OnlineAccount() { + protected OnlineAccountData() { } - public OnlineAccount(long timestamp, byte[] signature, byte[] publicKey) { + public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey) { this.timestamp = timestamp; this.signature = signature; this.publicKey = publicKey; diff --git a/src/main/java/org/qora/data/network/PeerChainTipData.java b/src/main/java/org/qora/data/network/PeerChainTipData.java new file mode 100644 index 00000000..d4910aa2 --- /dev/null +++ b/src/main/java/org/qora/data/network/PeerChainTipData.java @@ -0,0 +1,37 @@ +package org.qora.data.network; + +public class PeerChainTipData { + + /** Latest block height as reported by peer. */ + private Integer lastHeight; + /** Latest block signature as reported by peer. */ + private byte[] lastBlockSignature; + /** Latest block timestamp as reported by peer. */ + private Long lastBlockTimestamp; + /** Latest block generator public key as reported by peer. */ + private byte[] lastBlockGenerator; + + public PeerChainTipData(Integer lastHeight, byte[] lastBlockSignature, Long lastBlockTimestamp, byte[] lastBlockGenerator) { + this.lastHeight = lastHeight; + this.lastBlockSignature = lastBlockSignature; + this.lastBlockTimestamp = lastBlockTimestamp; + this.lastBlockGenerator = lastBlockGenerator; + } + + public Integer getLastHeight() { + return this.lastHeight; + } + + public byte[] getLastBlockSignature() { + return this.lastBlockSignature; + } + + public Long getLastBlockTimestamp() { + return this.lastBlockTimestamp; + } + + public byte[] getLastBlockGenerator() { + return this.lastBlockGenerator; + } + +} diff --git a/src/main/java/org/qora/network/Peer.java b/src/main/java/org/qora/network/Peer.java index a7a2692b..6b9904da 100644 --- a/src/main/java/org/qora/network/Peer.java +++ b/src/main/java/org/qora/network/Peer.java @@ -17,11 +17,11 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReentrantLock; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qora.controller.Controller; +import org.qora.data.network.PeerChainTipData; import org.qora.data.network.PeerData; import org.qora.network.message.Message; import org.qora.network.message.Message.MessageException; @@ -77,7 +77,6 @@ public class Peer { private volatile byte[] verificationCodeExpected; private volatile PeerData peerData = null; - private final ReentrantLock peerDataLock = new ReentrantLock(); /** Timestamp of when socket was accepted, or connected. */ private volatile Long connectionTimestamp = null; @@ -93,17 +92,8 @@ public class Peer { /** When last PING message was sent, or null if pings not started yet. */ private volatile Long lastPingSent; - /** Latest block height as reported by peer. */ - private volatile Integer lastHeight; - - /** Latest block signature as reported by peer. */ - private volatile byte[] lastBlockSignature; - - /** Latest block timestamp as reported by peer. */ - private volatile Long lastBlockTimestamp; - - /** Latest block generator public key as reported by peer. */ - private volatile byte[] lastBlockGenerator; + /** Latest block info as reported by peer. */ + private volatile PeerChainTipData chainTipData; // Constructors @@ -231,41 +221,12 @@ public class Peer { this.verificationCodeExpected = expected; } - public Integer getLastHeight() { - return this.lastHeight; + public PeerChainTipData getChainTipData() { + return this.chainTipData; } - public void setLastHeight(Integer lastHeight) { - this.lastHeight = lastHeight; - } - - public byte[] getLastBlockSignature() { - return lastBlockSignature; - } - - public void setLastBlockSignature(byte[] lastBlockSignature) { - this.lastBlockSignature = lastBlockSignature; - } - - public Long getLastBlockTimestamp() { - return lastBlockTimestamp; - } - - public void setLastBlockTimestamp(Long lastBlockTimestamp) { - this.lastBlockTimestamp = lastBlockTimestamp; - } - - public byte[] getLastBlockGenerator() { - return lastBlockGenerator; - } - - public void setLastBlockGenerator(byte[] lastBlockGenerator) { - this.lastBlockGenerator = lastBlockGenerator; - } - - /** Returns the lock used for synchronizing access to peer info. */ - public ReentrantLock getPeerDataLock() { - return this.peerDataLock; + public void setChainTipData(PeerChainTipData chainTipData) { + this.chainTipData = chainTipData; } /* package */ void queueMessage(Message message) { @@ -334,13 +295,14 @@ public class Peer { if (!this.socketChannel.isOpen() || this.socketChannel.socket().isClosed()) return; - int bytesRead = this.socketChannel.read(this.byteBuffer); + final int bytesRead = this.socketChannel.read(this.byteBuffer); if (bytesRead == -1) { this.disconnect("EOF"); return; } LOGGER.trace(() -> String.format("Received %d bytes from peer %s", bytesRead, this)); + final boolean wasByteBufferFull = !this.byteBuffer.hasRemaining(); while (true) { final Message message; @@ -354,8 +316,8 @@ public class Peer { return; } - if (message == null && bytesRead == 0) - // No complete message in buffer and no more bytes to read from socket + if (message == null && bytesRead == 0 && !wasByteBufferFull) + // No complete message in buffer, no more bytes to read from socket and there was room to read bytes return; if (message == null) @@ -380,7 +342,8 @@ public class Peer { return; } - // Prematurely end any blocking channel select so that new messages can be processed + // Prematurely end any blocking channel select so that new messages can be processed. + // This might cause this.socketChannel.read() above to return zero into bytesRead. Network.getInstance().wakeupChannelSelector(); } } diff --git a/src/main/java/org/qora/network/message/GetOnlineAccountsMessage.java b/src/main/java/org/qora/network/message/GetOnlineAccountsMessage.java index e22f671d..e6e76113 100644 --- a/src/main/java/org/qora/network/message/GetOnlineAccountsMessage.java +++ b/src/main/java/org/qora/network/message/GetOnlineAccountsMessage.java @@ -7,7 +7,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import org.qora.data.network.OnlineAccount; +import org.qora.data.network.OnlineAccountData; import org.qora.transform.Transformer; import com.google.common.primitives.Ints; @@ -16,19 +16,19 @@ import com.google.common.primitives.Longs; public class GetOnlineAccountsMessage extends Message { private static final int MAX_ACCOUNT_COUNT = 1000; - private List onlineAccounts; + private List onlineAccounts; - public GetOnlineAccountsMessage(List onlineAccounts) { + public GetOnlineAccountsMessage(List onlineAccounts) { this(-1, onlineAccounts); } - private GetOnlineAccountsMessage(int id, List onlineAccounts) { + private GetOnlineAccountsMessage(int id, List onlineAccounts) { super(id, MessageType.GET_ONLINE_ACCOUNTS); this.onlineAccounts = onlineAccounts; } - public List getOnlineAccounts() { + public List getOnlineAccounts() { return this.onlineAccounts; } @@ -38,7 +38,7 @@ public class GetOnlineAccountsMessage extends Message { if (accountCount > MAX_ACCOUNT_COUNT) return null; - List onlineAccounts = new ArrayList<>(accountCount); + List onlineAccounts = new ArrayList<>(accountCount); for (int i = 0; i < accountCount; ++i) { long timestamp = bytes.getLong(); @@ -46,7 +46,7 @@ public class GetOnlineAccountsMessage extends Message { byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; bytes.get(publicKey); - onlineAccounts.add(new OnlineAccount(timestamp, null, publicKey)); + onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey)); } return new GetOnlineAccountsMessage(id, onlineAccounts); @@ -60,10 +60,10 @@ public class GetOnlineAccountsMessage extends Message { bytes.write(Ints.toByteArray(this.onlineAccounts.size())); for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccount onlineAccount = this.onlineAccounts.get(i); - bytes.write(Longs.toByteArray(onlineAccount.getTimestamp())); + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); + bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - bytes.write(onlineAccount.getPublicKey()); + bytes.write(onlineAccountData.getPublicKey()); } return bytes.toByteArray(); diff --git a/src/main/java/org/qora/network/message/OnlineAccountsMessage.java b/src/main/java/org/qora/network/message/OnlineAccountsMessage.java index b7322d8f..9c7d410d 100644 --- a/src/main/java/org/qora/network/message/OnlineAccountsMessage.java +++ b/src/main/java/org/qora/network/message/OnlineAccountsMessage.java @@ -7,7 +7,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import org.qora.data.network.OnlineAccount; +import org.qora.data.network.OnlineAccountData; import org.qora.transform.Transformer; import com.google.common.primitives.Ints; @@ -16,19 +16,19 @@ import com.google.common.primitives.Longs; public class OnlineAccountsMessage extends Message { private static final int MAX_ACCOUNT_COUNT = 1000; - private List onlineAccounts; + private List onlineAccounts; - public OnlineAccountsMessage(List onlineAccounts) { + public OnlineAccountsMessage(List onlineAccounts) { this(-1, onlineAccounts); } - private OnlineAccountsMessage(int id, List onlineAccounts) { + private OnlineAccountsMessage(int id, List onlineAccounts) { super(id, MessageType.ONLINE_ACCOUNTS); this.onlineAccounts = onlineAccounts; } - public List getOnlineAccounts() { + public List getOnlineAccounts() { return this.onlineAccounts; } @@ -38,7 +38,7 @@ public class OnlineAccountsMessage extends Message { if (accountCount > MAX_ACCOUNT_COUNT) return null; - List onlineAccounts = new ArrayList<>(accountCount); + List onlineAccounts = new ArrayList<>(accountCount); for (int i = 0; i < accountCount; ++i) { long timestamp = bytes.getLong(); @@ -49,8 +49,8 @@ public class OnlineAccountsMessage extends Message { byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; bytes.get(publicKey); - OnlineAccount onlineAccount = new OnlineAccount(timestamp, signature, publicKey); - onlineAccounts.add(onlineAccount); + OnlineAccountData onlineAccountData = new OnlineAccountData(timestamp, signature, publicKey); + onlineAccounts.add(onlineAccountData); } return new OnlineAccountsMessage(id, onlineAccounts); @@ -64,13 +64,13 @@ public class OnlineAccountsMessage extends Message { bytes.write(Ints.toByteArray(this.onlineAccounts.size())); for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccount onlineAccount = this.onlineAccounts.get(i); + OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); - bytes.write(Longs.toByteArray(onlineAccount.getTimestamp())); + bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - bytes.write(onlineAccount.getSignature()); + bytes.write(onlineAccountData.getSignature()); - bytes.write(onlineAccount.getPublicKey()); + bytes.write(onlineAccountData.getPublicKey()); } return bytes.toByteArray(); diff --git a/src/main/java/org/qora/settings/Settings.java b/src/main/java/org/qora/settings/Settings.java index 3f56c47a..883156fa 100644 --- a/src/main/java/org/qora/settings/Settings.java +++ b/src/main/java/org/qora/settings/Settings.java @@ -76,7 +76,7 @@ public class Settings { /** Port number for inbound peer-to-peer connections. */ private Integer listenPort; /** Minimum number of peers to allow block generation / synchronization. */ - private int minBlockchainPeers = 3; + private int minBlockchainPeers = 5; /** Target number of outbound connections to peers we should make. */ private int minOutboundPeers = 20; /** Maximum number of peer connections we allow. */ @@ -128,6 +128,21 @@ public class Settings { return instance; } + /** + * Parse settings from given file. + *

+ * Throws RuntimeException with UnmarshalException as cause if settings file could not be parsed. + *

+ * We use RuntimeException because it can be caught first caller of {@link #getInstance()} above, + * but it's not necessary to surround later {@link #getInstance()} calls + * with try-catch as they should be read-only. + * + * @param filename + * @throws RuntimeException with UnmarshalException as cause if settings file could not be parsed + * @throws RuntimeException with FileNotFoundException as cause if settings file could not be found/opened + * @throws RuntimeException with JAXBException as cause if some unexpected JAXB-related error occurred + * @throws RuntimeException with IOException as cause if some unexpected I/O-related error occurred + */ public static void fileInstance(String filename) { JAXBContext jc; Unmarshaller unmarshaller; @@ -147,8 +162,9 @@ public class Settings { // Tell unmarshaller that there's no JSON root element in the JSON input unmarshaller.setProperty(UnmarshallerProperties.JSON_INCLUDE_ROOT, false); } catch (JAXBException e) { - LOGGER.error("Unable to process settings file", e); - throw new RuntimeException("Unable to process settings file", e); + String message = "Failed to setup unmarshaller to process settings file"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); } Settings settings = null; @@ -164,24 +180,28 @@ public class Settings { // Attempt to unmarshal JSON stream to Settings settings = unmarshaller.unmarshal(json, Settings.class).getValue(); } catch (FileNotFoundException e) { - LOGGER.error("Settings file not found: " + path + filename); - throw new RuntimeException("Settings file not found: " + path + filename); + String message = "Settings file not found: " + path + filename; + LOGGER.error(message, e); + throw new RuntimeException(message, e); } catch (UnmarshalException e) { Throwable linkedException = e.getLinkedException(); if (linkedException instanceof XMLMarshalException) { String message = ((XMLMarshalException) linkedException).getInternalException().getLocalizedMessage(); LOGGER.error(message); - throw new RuntimeException(message); + throw new RuntimeException(message, e); } - LOGGER.error("Unable to process settings file", e); - throw new RuntimeException("Unable to process settings file", e); + String message = "Failed to parse settings file"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); } catch (JAXBException e) { - LOGGER.error("Unable to process settings file", e); - throw new RuntimeException("Unable to process settings file", e); + String message = "Unexpected JAXB issue while processing settings file"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); } catch (IOException e) { - LOGGER.error("Unable to process settings file", e); - throw new RuntimeException("Unable to process settings file", e); + String message = "Unexpected I/O issue while processing settings file"; + LOGGER.error(message, e); + throw new RuntimeException(message, e); } if (settings.userPath != null) { @@ -207,8 +227,14 @@ public class Settings { BlockChain.fileInstance(settings.getUserPath(), settings.getBlockchainConfig()); } + public static void throwValidationError(String message) { + throw new RuntimeException(message, new UnmarshalException(message)); + } + private void validate() { // Validation goes here + if (this.minBlockchainPeers < 1) + throwValidationError("minBlockchainPeers must be at least 1"); } // Getters / setters diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index f828afea..30c94617 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -4,6 +4,8 @@ "minBlockTime": 60, "maxBlockTime": 300, "blockTimestampMargin": 2000, + "transactionExpiryPeriod": 86400000, + "maxBlockSize": 1048576, "maxBytesPerUnitFee": 1024, "unitFee": "1.0", "useBrokenMD160ForAddresses": false, @@ -15,7 +17,7 @@ "onlineAccountSignaturesMaxLifetime": 3196800000, "genesisInfo": { "version": 4, - "timestamp": "1568720000000", + "timestamp": "1569510000000", "generatingBalance": "100000", "transactions": [ { "type": "ISSUE_ASSET", "owner": "QUwGVHPPxJNJ2dq95abQNe79EyBN2K26zM", "assetName": "QORT", "description": "QORTAL coin", "quantity": 10000000, "isDivisible": true, "fee": 0, "reference": "28u54WRcMfGujtQMZ9dNKFXVqucY7XfPihXAqPFsnx853NPUwfDJy1sMH5boCkahFgjUNYqc5fkduxdBhQTKgUsC", "data": "{}" }, diff --git a/src/test/java/org/qora/test/ChainWeightTests.java b/src/test/java/org/qora/test/ChainWeightTests.java new file mode 100644 index 00000000..2e56c52f --- /dev/null +++ b/src/test/java/org/qora/test/ChainWeightTests.java @@ -0,0 +1,134 @@ +package org.qora.test; + +import static org.junit.Assert.*; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.qora.crypto.Crypto; +import org.qora.data.block.BlockSummaryData; +import org.qora.transform.Transformer; +import org.qora.transform.block.BlockTransformer; +import org.junit.Test; + +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Longs; + +public class ChainWeightTests { + + private static final int ACCOUNTS_COUNT_SHIFT = Transformer.PUBLIC_KEY_LENGTH * 8; + private static final int CHAIN_WEIGHT_SHIFT = 8; + private static final Random RANDOM = new Random(); + + private static final BigInteger MAX_DISTANCE; + static { + byte[] maxValue = new byte[Transformer.PUBLIC_KEY_LENGTH]; + Arrays.fill(maxValue, (byte) 0xFF); + MAX_DISTANCE = new BigInteger(1, maxValue); + } + + + private static byte[] perturbPublicKey(int height, byte[] publicKey) { + return Crypto.digest(Bytes.concat(Longs.toByteArray(height), publicKey)); + } + + private static BigInteger calcKeyDistance(int parentHeight, byte[] parentGeneratorKey, byte[] publicKey) { + byte[] idealKey = perturbPublicKey(parentHeight, parentGeneratorKey); + byte[] perturbedKey = perturbPublicKey(parentHeight + 1, publicKey); + + BigInteger keyDistance = MAX_DISTANCE.subtract(new BigInteger(idealKey).subtract(new BigInteger(perturbedKey)).abs()); + return keyDistance; + } + + private static BigInteger calcBlockWeight(int parentHeight, byte[] parentGeneratorKey, BlockSummaryData blockSummaryData) { + BigInteger keyDistance = calcKeyDistance(parentHeight, parentGeneratorKey, blockSummaryData.getGeneratorPublicKey()); + BigInteger weight = BigInteger.valueOf(blockSummaryData.getOnlineAccountsCount()).shiftLeft(ACCOUNTS_COUNT_SHIFT).add(keyDistance); + return weight; + } + + private static BigInteger calcChainWeight(int commonBlockHeight, byte[] commonBlockGeneratorKey, List blockSummaries) { + BigInteger cumulativeWeight = BigInteger.ZERO; + int parentHeight = commonBlockHeight; + byte[] parentGeneratorKey = commonBlockGeneratorKey; + + for (BlockSummaryData blockSummaryData : blockSummaries) { + cumulativeWeight = cumulativeWeight.shiftLeft(CHAIN_WEIGHT_SHIFT).add(calcBlockWeight(parentHeight, parentGeneratorKey, blockSummaryData)); + parentHeight = blockSummaryData.getHeight(); + parentGeneratorKey = blockSummaryData.getGeneratorPublicKey(); + } + + return cumulativeWeight; + } + + private static BlockSummaryData genBlockSummary(int height) { + byte[] generatorPublicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + RANDOM.nextBytes(generatorPublicKey); + + byte[] signature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; + RANDOM.nextBytes(signature); + + int onlineAccountsCount = RANDOM.nextInt(1000); + + return new BlockSummaryData(height, signature, generatorPublicKey, onlineAccountsCount); + } + + private static List genBlockSummaries(int count, BlockSummaryData commonBlockSummary) { + List blockSummaries = new ArrayList<>(); + blockSummaries.add(commonBlockSummary); + + final int commonBlockHeight = commonBlockSummary.getHeight(); + + for (int i = 1; i <= count; ++i) + blockSummaries.add(genBlockSummary(commonBlockHeight + i)); + + return blockSummaries; + } + + // Check that more online accounts beats a better key + @Test + public void testMoreAccountsBlock() { + final int parentHeight = 1; + final byte[] parentGeneratorKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + + int betterAccountsCount = 100; + int worseAccountsCount = 20; + + byte[] betterKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + betterKey[0] = 0x41; + + byte[] worseKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; + worseKey[0] = 0x23; + + BigInteger betterKeyDistance = calcKeyDistance(parentHeight, parentGeneratorKey, betterKey); + BigInteger worseKeyDistance = calcKeyDistance(parentHeight, parentGeneratorKey, worseKey); + assertEquals("hard-coded keys are wrong", 1, betterKeyDistance.compareTo(worseKeyDistance)); + + BlockSummaryData betterBlockSummary = new BlockSummaryData(parentHeight + 1, null, worseKey, betterAccountsCount); + BlockSummaryData worseBlockSummary = new BlockSummaryData(parentHeight + 1, null, betterKey, worseAccountsCount); + + BigInteger betterBlockWeight = calcBlockWeight(parentHeight, parentGeneratorKey, betterBlockSummary); + BigInteger worseBlockWeight = calcBlockWeight(parentHeight, parentGeneratorKey, worseBlockSummary); + + assertEquals("block weights are wrong", 1, betterBlockWeight.compareTo(worseBlockWeight)); + } + + // Check that a longer chain beats a shorter chain + @Test + public void testLongerChain() { + final int commonBlockHeight = 1; + BlockSummaryData commonBlockSummary = genBlockSummary(commonBlockHeight); + byte[] commonBlockGeneratorKey = commonBlockSummary.getGeneratorPublicKey(); + + List shorterChain = genBlockSummaries(3, commonBlockSummary); + List longerChain = genBlockSummaries(shorterChain.size() + 1, commonBlockSummary); + + BigInteger shorterChainWeight = calcChainWeight(commonBlockHeight, commonBlockGeneratorKey, shorterChain); + BigInteger longerChainWeight = calcChainWeight(commonBlockHeight, commonBlockGeneratorKey, longerChain); + + assertEquals("longer chain should have greater weight", 1, longerChainWeight.compareTo(shorterChainWeight)); + } + +} diff --git a/src/test/java/org/qora/test/OnlineTests.java b/src/test/java/org/qora/test/OnlineTests.java index ef3bcfb7..3ab55307 100644 --- a/src/test/java/org/qora/test/OnlineTests.java +++ b/src/test/java/org/qora/test/OnlineTests.java @@ -13,7 +13,7 @@ import org.junit.Before; import org.junit.Test; import org.qora.account.PrivateKeyAccount; import org.qora.account.PublicKeyAccount; -import org.qora.data.network.OnlineAccount; +import org.qora.data.network.OnlineAccountData; import org.qora.network.message.GetOnlineAccountsMessage; import org.qora.network.message.Message; import org.qora.network.message.OnlineAccountsMessage; @@ -48,7 +48,7 @@ public class OnlineTests extends Common { private final PrivateKeyAccount account; - private List onlineAccounts; + private List onlineAccounts; private long nextOnlineRefresh = 0; public OnlinePeer(int id, PrivateKeyAccount account) { @@ -65,22 +65,22 @@ public class OnlineTests extends Common { case GET_ONLINE_ACCOUNTS: { GetOnlineAccountsMessage getOnlineAccountsMessage = (GetOnlineAccountsMessage) message; - List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); + List excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts(); // Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts - List accountsToSend; + List accountsToSend; synchronized (this.onlineAccounts) { accountsToSend = new ArrayList<>(this.onlineAccounts); } - Iterator iterator = accountsToSend.iterator(); + Iterator iterator = accountsToSend.iterator(); SEND_ITERATOR: while (iterator.hasNext()) { - OnlineAccount onlineAccount = iterator.next(); + OnlineAccountData onlineAccount = iterator.next(); for (int i = 0; i < excludeAccounts.size(); ++i) { - OnlineAccount excludeAccount = excludeAccounts.get(i); + OnlineAccountData excludeAccount = excludeAccounts.get(i); if (onlineAccount.getTimestamp() == excludeAccount.getTimestamp() && Arrays.equals(onlineAccount.getPublicKey(), excludeAccount.getPublicKey())) { iterator.remove(); @@ -101,12 +101,12 @@ public class OnlineTests extends Common { case ONLINE_ACCOUNTS: { OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message; - List onlineAccounts = onlineAccountsMessage.getOnlineAccounts(); + List onlineAccounts = onlineAccountsMessage.getOnlineAccounts(); if (LOG_ACCOUNT_CHANGES) System.out.println(String.format("[%d] received %d online accounts from %d", this.getId(), onlineAccounts.size(), peer.getId())); - for (OnlineAccount onlineAccount : onlineAccounts) + for (OnlineAccountData onlineAccount : onlineAccounts) verifyAndAddAccount(onlineAccount); break; @@ -117,7 +117,7 @@ public class OnlineTests extends Common { } } - private void verifyAndAddAccount(OnlineAccount onlineAccount) { + private void verifyAndAddAccount(OnlineAccountData onlineAccount) { // we would check timestamp is 'recent' here // Verify @@ -131,7 +131,7 @@ public class OnlineTests extends Common { ByteArray publicKeyBA = new ByteArray(onlineAccount.getPublicKey()); synchronized (this.onlineAccounts) { - OnlineAccount existingAccount = this.onlineAccounts.stream().filter(account -> new ByteArray(account.getPublicKey()).equals(publicKeyBA)).findFirst().orElse(null); + OnlineAccountData existingAccount = this.onlineAccounts.stream().filter(account -> new ByteArray(account.getPublicKey()).equals(publicKeyBA)).findFirst().orElse(null); if (existingAccount != null) { if (existingAccount.getTimestamp() < onlineAccount.getTimestamp()) { @@ -161,9 +161,9 @@ public class OnlineTests extends Common { // Expire old entries final long cutoffThreshold = now - LAST_SEEN_EXPIRY_PERIOD; synchronized (this.onlineAccounts) { - Iterator iterator = this.onlineAccounts.iterator(); + Iterator iterator = this.onlineAccounts.iterator(); while (iterator.hasNext()) { - OnlineAccount onlineAccount = iterator.next(); + OnlineAccountData onlineAccount = iterator.next(); if (onlineAccount.getTimestamp() < cutoffThreshold) { iterator.remove(); @@ -211,7 +211,7 @@ public class OnlineTests extends Common { byte[] publicKey = this.account.getPublicKey(); // Our account is online - OnlineAccount onlineAccount = new OnlineAccount(timestamp, signature, publicKey); + OnlineAccountData onlineAccount = new OnlineAccountData(timestamp, signature, publicKey); synchronized (this.onlineAccounts) { this.onlineAccounts.removeIf(account -> account.getPublicKey() == this.account.getPublicKey()); this.onlineAccounts.add(onlineAccount);