diff --git a/src/main/java/org/qortal/api/model/BlockMintingInfo.java b/src/main/java/org/qortal/api/model/BlockMintingInfo.java new file mode 100644 index 00000000..f84e179e --- /dev/null +++ b/src/main/java/org/qortal/api/model/BlockMintingInfo.java @@ -0,0 +1,23 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.math.BigDecimal; +import java.math.BigInteger; + +@XmlAccessorType(XmlAccessType.FIELD) +public class BlockMintingInfo { + + public byte[] minterPublicKey; + public int minterLevel; + public int onlineAccountsCount; + public BigDecimal maxDistance; + public BigInteger keyDistance; + public double keyDistanceRatio; + public long timestamp; + public long timeDelta; + + public BlockMintingInfo() { + } + +} diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index 30cc477e..b2f29305 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -8,6 +8,9 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.List; @@ -20,10 +23,13 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import org.qortal.account.Account; import org.qortal.api.ApiError; import org.qortal.api.ApiErrors; import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.model.BlockMintingInfo; import org.qortal.api.model.BlockSignerSummary; +import org.qortal.block.Block; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountData; import org.qortal.data.block.BlockData; @@ -328,6 +334,59 @@ public class BlocksResource { } } + @GET + @Path("/byheight/{height}/mintinginfo") + @Operation( + summary = "Fetch block minter info using block height", + description = "Returns the minter info for the block with given height", + responses = { + @ApiResponse( + description = "the block", + content = @Content( + schema = @Schema( + implementation = BlockData.class + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE + }) + public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) { + try (final Repository repository = RepositoryManager.getRepository()) { + BlockData blockData = repository.getBlockRepository().fromHeight(height); + if (blockData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + + Block block = new Block(repository, blockData); + BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference()); + int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey()); + if (minterLevel == 0) + // This may be unavailable when requesting a trimmed block + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + BigInteger distance = block.calcKeyDistance(parentBlockData.getHeight(), parentBlockData.getSignature(), blockData.getMinterPublicKey(), minterLevel); + double ratio = new BigDecimal(distance).divide(new BigDecimal(block.MAX_DISTANCE), 40, RoundingMode.DOWN).doubleValue(); + long timestamp = block.calcTimestamp(parentBlockData, blockData.getMinterPublicKey(), minterLevel); + long timeDelta = timestamp - parentBlockData.getTimestamp(); + + BlockMintingInfo blockMintingInfo = new BlockMintingInfo(); + blockMintingInfo.minterPublicKey = blockData.getMinterPublicKey(); + blockMintingInfo.minterLevel = minterLevel; + blockMintingInfo.onlineAccountsCount = blockData.getOnlineAccountsCount(); + blockMintingInfo.maxDistance = new BigDecimal(block.MAX_DISTANCE); + blockMintingInfo.keyDistance = distance; + blockMintingInfo.keyDistanceRatio = ratio; + blockMintingInfo.timestamp = timestamp; + blockMintingInfo.timeDelta = timeDelta; + + return blockMintingInfo; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + @GET @Path("/timestamp/{timestamp}") @Operation( diff --git a/src/main/java/org/qortal/api/resource/CrossChainResource.java b/src/main/java/org/qortal/api/resource/CrossChainResource.java index 7a6c2c96..d1692b71 100644 --- a/src/main/java/org/qortal/api/resource/CrossChainResource.java +++ b/src/main/java/org/qortal/api/resource/CrossChainResource.java @@ -262,6 +262,7 @@ public class CrossChainResource { // We want both a minimum of 5 trades and enough trades to span at least 4 hours int minimumCount = 5; + int maximumCount = 10; long minimumPeriod = 4 * 60 * 60 * 1000L; // ms Boolean isFinished = Boolean.TRUE; @@ -276,7 +277,7 @@ public class CrossChainResource { ACCT acct = acctInfo.getValue().get(); List atStates = repository.getATRepository().getMatchingFinalATStatesQuorum(codeHash, - isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumCount, minimumPeriod); + isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumCount, maximumCount, minimumPeriod); for (ATStateData atState : atStates) { CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState); diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 1f00afc2..25ca97a7 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -232,7 +232,7 @@ public class Block { // Other useful constants - private static final BigInteger MAX_DISTANCE; + public static final BigInteger MAX_DISTANCE; static { byte[] maxValue = new byte[Transformer.PUBLIC_KEY_LENGTH]; Arrays.fill(maxValue, (byte) 0xFF); diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 4fcabe55..e5f38347 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -637,6 +637,11 @@ public class Controller extends Thread { return peerChainTipData == null || peerChainTipData.getLastBlockSignature() == null || inferiorChainTips.contains(new ByteArray(peerChainTipData.getLastBlockSignature())); }; + public static final Predicate hasOldVersion = peer -> { + final String minPeerVersion = Settings.getInstance().getMinPeerVersion(); + return peer.isAtLeastVersion(minPeerVersion) == false; + }; + private void potentiallySynchronize() throws InterruptedException { // Already synchronizing via another thread? if (this.isSynchronizing) @@ -653,11 +658,15 @@ public class Controller extends Thread { // Disregard peers that don't have a recent block peers.removeIf(hasNoRecentBlock); + // Disregard peers that are on an old version + peers.removeIf(hasOldVersion); + checkRecoveryModeForPeers(peers); if (recoveryMode) { peers = Network.getInstance().getHandshakedPeers(); peers.removeIf(hasOnlyGenesisBlock); peers.removeIf(hasMisbehaved); + peers.removeIf(hasOldVersion); } // Check we have enough peers to potentially synchronize diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index 3c033e88..8bee63a2 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -4,7 +4,6 @@ import java.util.Arrays; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -51,7 +50,7 @@ public enum Handshake { String versionString = helloMessage.getVersionString(); - Matcher matcher = VERSION_PATTERN.matcher(versionString); + Matcher matcher = peer.VERSION_PATTERN.matcher(versionString); if (!matcher.lookingAt()) { LOGGER.debug(() -> String.format("Peer %s sent invalid HELLO version string '%s'", peer, versionString)); return null; @@ -72,6 +71,15 @@ public enum Handshake { peer.setPeersConnectionTimestamp(peersConnectionTimestamp); peer.setPeersVersion(versionString, version); + if (Settings.getInstance().getAllowConnectionsWithOlderPeerVersions() == false) { + // Ensure the peer is running at least the minimum version allowed for connections + final String minPeerVersion = Settings.getInstance().getMinPeerVersion(); + if (peer.isAtLeastVersion(minPeerVersion) == false) { + LOGGER.info(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString)); + return null; + } + } + return CHALLENGE; } @@ -244,8 +252,6 @@ public enum Handshake { /** Maximum allowed difference between peer's reported timestamp and when they connected, in milliseconds. */ private static final long MAX_TIMESTAMP_DELTA = 30 * 1000L; // ms - private static final Pattern VERSION_PATTERN = Pattern.compile(Controller.VERSION_PREFIX + "(\\d{1,3})\\.(\\d{1,5})\\.(\\d{1,5})"); - private static final long PEER_VERSION_131 = 0x0100030001L; private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index e5bd369d..79495277 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -20,9 +20,12 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; import org.qortal.data.block.CommonBlockData; import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; @@ -87,6 +90,9 @@ public class Peer { byte[] ourChallenge; + // Versioning + public static final Pattern VERSION_PATTERN = Pattern.compile(Controller.VERSION_PREFIX + "(\\d{1,3})\\.(\\d{1,5})\\.(\\d{1,5})"); + // Peer info private final Object peerInfoLock = new Object(); @@ -651,6 +657,35 @@ public class Peer { } + // Minimum version + + public boolean isAtLeastVersion(String minVersionString) { + if (minVersionString == null) + return false; + + // Add the version prefix + minVersionString = Controller.VERSION_PREFIX + minVersionString; + + Matcher matcher = VERSION_PATTERN.matcher(minVersionString); + if (!matcher.lookingAt()) + return false; + + // We're expecting 3 positive shorts, so we can convert 1.2.3 into 0x0100020003 + long minVersion = 0; + for (int g = 1; g <= 3; ++g) { + long value = Long.parseLong(matcher.group(g)); + + if (value < 0 || value > Short.MAX_VALUE) + return false; + + minVersion <<= 16; + minVersion |= value; + } + + return this.getPeersVersion() >= minVersion; + } + + // Common block data public boolean canUseCachedCommonBlockData() { diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 0854a21c..5516ac28 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -98,7 +98,7 @@ public interface ATRepository { */ public List getMatchingFinalATStatesQuorum(byte[] codeHash, Boolean isFinished, Integer dataByteOffset, Long expectedValue, - int minimumCount, long minimumPeriod) throws DataException; + int minimumCount, int maximumCount, long minimumPeriod) throws DataException; /** * Returns all ATStateData for a given block height. diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index f82e4e62..8193c5d2 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -454,7 +454,7 @@ public class HSQLDBATRepository implements ATRepository { @Override public List getMatchingFinalATStatesQuorum(byte[] codeHash, Boolean isFinished, Integer dataByteOffset, Long expectedValue, - int minimumCount, long minimumPeriod) throws DataException { + int minimumCount, int maximumCount, long minimumPeriod) throws DataException { // We need most recent entry first so we can use its timestamp to slice further results List mostRecentStates = this.getMatchingFinalATStates(codeHash, isFinished, dataByteOffset, expectedValue, null, @@ -510,7 +510,8 @@ public class HSQLDBATRepository implements ATRepository { bindParams.add(minimumHeight); bindParams.add(minimumCount); - sql.append("ORDER BY FinalATStates.height DESC"); + sql.append("ORDER BY FinalATStates.height DESC LIMIT ?"); + bindParams.add(maximumCount); List atStates = new ArrayList<>(); diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 8b0eefcc..39365f5c 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -125,6 +125,13 @@ public class Settings { /** Maximum number of retry attempts if a peer fails to respond with the requested data */ private int maxRetries = 2; + /** Minimum peer version number required in order to sync with them */ + private String minPeerVersion = "1.5.0"; + /** Whether to allow connections with peers below minPeerVersion + * If true, we won't sync with them but they can still sync with us, and will show in the peers list + * If false, sync will be blocked both ways, and they will not appear in the peers list */ + private boolean allowConnectionsWithOlderPeerVersions = true; + /** Whether to sync multiple blocks at once in normal operation */ private boolean fastSyncEnabled = false; /** Whether to sync multiple blocks at once when the peer has a different chain */ @@ -421,6 +428,10 @@ public class Settings { public int getMaxRetries() { return this.maxRetries; } + public String getMinPeerVersion() { return this.minPeerVersion; } + + public boolean getAllowConnectionsWithOlderPeerVersions() { return this.allowConnectionsWithOlderPeerVersions; } + public String getBlockchainConfig() { return this.blockchainConfig; } diff --git a/tools/block-timings.sh b/tools/block-timings.sh new file mode 100755 index 00000000..5324209b --- /dev/null +++ b/tools/block-timings.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash + +start_height=$1 +count=$2 +target=$3 +deviation=$4 +power=$5 + +if [ -z "${start_height}" ]; then + echo + echo "Error: missing start height." + echo + echo "Usage:" + echo "block-timings.sh [count] [target] [deviation] [power]" + echo + echo "startheight: a block height, preferably within the untrimmed range, to avoid data gaps" + echo "count: the number of blocks to request and analyse after the start height. Default: 100" + echo "target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000" + echo "deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000" + echo "power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2" + echo + exit +fi + +count=${count:=100} +target=${target:=60000} +deviation=${deviation:=30000} +power=${power:=0.2} + +finish_height=$((start_height + count - 1)) +height=$start_height + +echo "Settings:" +echo "Target time offset: ${target}" +echo "Deviation: ${deviation}" +echo "Power transform: ${power}" +echo + +function calculate_time_offset { + local key_distance_ratio=$1 + local transformed=$( echo "" | awk "END {print ${key_distance_ratio} ^ ${power}}") + local time_offset=$(echo "${deviation}*2*${transformed}" | bc) + time_offset=${time_offset%.*} + echo $time_offset +} + + +function fetch_and_process_blocks { + + echo "Fetching blocks from height ${start_height} to ${finish_height}..." + echo + + total_time_offset=0 + errors=0 + + while [ "${height}" -le "${finish_height}" ]; do + block_minting_info=$(curl -s "http://localhost:12391/blocks/byheight/${height}/mintinginfo") + error=$(echo "${block_minting_info}" | jq -r .error) + if [ "${error}" != "null" ]; then + echo "Error fetching minting info for block ${height}" + echo + errors=$((errors+1)) + height=$((height+1)) + continue; + fi + + # Parse minting info + minter_level=$(echo "${block_minting_info}" | jq -r .minterLevel) + online_accounts_count=$(echo "${block_minting_info}" | jq -r .onlineAccountsCount) + key_distance_ratio=$(echo "${block_minting_info}" | jq -r .keyDistanceRatio) + time_delta=$(echo "${block_minting_info}" | jq -r .timeDelta) + + time_offset=$(calculate_time_offset "${key_distance_ratio}") + block_time=$((target-deviation+time_offset)) + + echo "=== BLOCK ${height} ===" + echo "Minter level: ${minter_level}" + echo "Online accounts: ${online_accounts_count}" + echo "Key distance ratio: ${key_distance_ratio}" + echo "Time offset: ${time_offset}" + echo "Block time (real): ${time_delta}" + echo "Block time (calculated): ${block_time}" + + if [ "${time_delta}" -ne "${block_time}" ]; then + echo "WARNING: Block time mismatch. This is to be expected when using custom settings." + fi + echo + + total_time_offset=$((total_time_offset+block_time)) + + height=$((height+1)) + done + + adjusted_count=$((count-errors)) + if [ "${adjusted_count}" -eq 0 ]; then + echo "No blocks were retrieved." + echo + exit; + fi + + mean_time_offset=$((total_time_offset/adjusted_count)) + time_offset_diff=$((mean_time_offset-target)) + + echo "===================" + echo "===== SUMMARY =====" + echo "===================" + echo "Total blocks retrieved: ${adjusted_count}" + echo "Total blocks failed: ${errors}" + echo "Mean time offset: ${mean_time_offset}ms" + echo "Target time offset: ${target}ms" + echo "Difference from target: ${time_offset_diff}ms" + echo + +} + +function estimate_key_distance_ratio_for_level { + local level=$1 + local example_key_distance="0.5" + echo "(${example_key_distance}/${level})" +} + +function estimate_block_timestamps { + min_block_time=9999999 + max_block_time=0 + + echo "===== BLOCK TIME ESTIMATES =====" + + for level in {1..10}; do + example_key_distance_ratio=$(estimate_key_distance_ratio_for_level "${level}") + time_offset=$(calculate_time_offset "${example_key_distance_ratio}") + block_time=$((target-deviation+time_offset)) + + if [ "${block_time}" -gt "${max_block_time}" ]; then + max_block_time=${block_time} + fi + if [ "${block_time}" -lt "${min_block_time}" ]; then + min_block_time=${block_time} + fi + + echo "Level: ${level}, time offset: ${time_offset}, block time: ${block_time}" + done + block_time_range=$((max_block_time-min_block_time)) + echo "Range: ${block_time_range}" + echo +} + +fetch_and_process_blocks +estimate_block_timestamps