From 641a6580593b761a007ffa3e2e0eb5d2ebb47350 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 17:49:04 +0100 Subject: [PATCH 1/4] Added /blocks/byheight/{height}/mintinginfo API, which returns info on the minter level, key distance, and block timings. --- .../qortal/api/model/BlockMintingInfo.java | 22 +++++++ .../qortal/api/resource/BlocksResource.java | 58 +++++++++++++++++++ src/main/java/org/qortal/block/Block.java | 2 +- 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/qortal/api/model/BlockMintingInfo.java diff --git a/src/main/java/org/qortal/api/model/BlockMintingInfo.java b/src/main/java/org/qortal/api/model/BlockMintingInfo.java new file mode 100644 index 00000000..e71c918b --- /dev/null +++ b/src/main/java/org/qortal/api/model/BlockMintingInfo.java @@ -0,0 +1,22 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import java.math.BigDecimal; +import java.math.BigInteger; + +@XmlAccessorType(XmlAccessType.FIELD) +public class BlockMintingInfo { + + public byte[] minterPublicKey; + public int minterLevel; + public BigDecimal maxDistance; + public BigInteger keyDistance; + public double keyDistanceRatio; + public long timestamp; + public long timeDelta; + + public BlockMintingInfo() { + } + +} diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index 30cc477e..1b160b91 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -8,6 +8,9 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; import java.util.ArrayList; import java.util.List; @@ -20,10 +23,13 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import org.qortal.account.Account; import org.qortal.api.ApiError; import org.qortal.api.ApiErrors; import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.model.BlockMintingInfo; import org.qortal.api.model.BlockSignerSummary; +import org.qortal.block.Block; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountData; import org.qortal.data.block.BlockData; @@ -328,6 +334,58 @@ public class BlocksResource { } } + @GET + @Path("/byheight/{height}/mintinginfo") + @Operation( + summary = "Fetch block minter info using block height", + description = "Returns the minter info for the block with given height", + responses = { + @ApiResponse( + description = "the block", + content = @Content( + schema = @Schema( + implementation = BlockData.class + ) + ) + ) + } + ) + @ApiErrors({ + ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE + }) + public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) { + try (final Repository repository = RepositoryManager.getRepository()) { + BlockData blockData = repository.getBlockRepository().fromHeight(height); + if (blockData == null) + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + + Block block = new Block(repository, blockData); + BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference()); + int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey()); + if (minterLevel == 0) + // This may be unavailable when requesting a trimmed block + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + + BigInteger distance = block.calcKeyDistance(parentBlockData.getHeight(), parentBlockData.getSignature(), blockData.getMinterPublicKey(), minterLevel); + double ratio = new BigDecimal(distance).divide(new BigDecimal(block.MAX_DISTANCE), 40, RoundingMode.DOWN).doubleValue(); + long timestamp = block.calcTimestamp(parentBlockData, blockData.getMinterPublicKey(), minterLevel); + long timeDelta = timestamp - parentBlockData.getTimestamp(); + + BlockMintingInfo blockMintingInfo = new BlockMintingInfo(); + blockMintingInfo.minterPublicKey = blockData.getMinterPublicKey(); + blockMintingInfo.minterLevel = minterLevel; + blockMintingInfo.maxDistance = new BigDecimal(block.MAX_DISTANCE); + blockMintingInfo.keyDistance = distance; + blockMintingInfo.keyDistanceRatio = ratio; + blockMintingInfo.timestamp = timestamp; + blockMintingInfo.timeDelta = timeDelta; + + return blockMintingInfo; + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); + } + } + @GET @Path("/timestamp/{timestamp}") @Operation( diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 8551e4e7..74fe059c 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -225,7 +225,7 @@ public class Block { // Other useful constants - private static final BigInteger MAX_DISTANCE; + public static final BigInteger MAX_DISTANCE; static { byte[] maxValue = new byte[Transformer.PUBLIC_KEY_LENGTH]; Arrays.fill(maxValue, (byte) 0xFF); From a1a1b8e94a1236b29f8b59625872077bb77589f8 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 17:57:28 +0100 Subject: [PATCH 2/4] Added tools/block-timings-sh which can be used to test out new block timings (specified in blockchain.json). The script will fetch a set of blocks and then backtest the specified blockTimings settings (target, deviation, and power) against those real life blocks. This allows configurations to be fine tuned to tighten up block times, and to adjust the timestamp variance between levels. Usage: block-timings.sh [target] [deviation] [power] startheight: a block height, preferably within the untrimmed range, to avoid data gaps count: the number of blocks to request and analyse after the start height. Default: 100 target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000 deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000 power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2 --- tools/block-timings.sh | 147 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100755 tools/block-timings.sh diff --git a/tools/block-timings.sh b/tools/block-timings.sh new file mode 100755 index 00000000..43f2f466 --- /dev/null +++ b/tools/block-timings.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash + +start_height=$1 +count=$2 + +if [ -z "${start_height}" ]; then + echo + echo "Error: missing start height." + echo + echo "Usage:" + echo "block-timings.sh [target] [deviation] [power]" + echo + echo "startheight: a block height, preferably within the untrimmed range, to avoid data gaps" + echo "count: the number of blocks to request and analyse after the start height. Default: 100" + echo "target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000" + echo "deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000" + echo "power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2" + echo + exit +fi + +target=$3 +deviation=$4 +power=$5 + +count=${count:=100} +target=${target:=60000} +deviation=${deviation:=30000} +power=${power:=0.2} + +finish_height=$((start_height + count - 1)) +height=$start_height + +echo "Settings:" +echo "Target time offset: ${target}" +echo "Deviation: ${deviation}" +echo "Power transform: ${power}" +echo + +function calculate_time_offset { + local key_distance_ratio=$1 + local transformed=$( echo "" | awk "END {print ${key_distance_ratio} ^ ${power}}") + local time_offset=$(echo "${deviation}*2*${transformed}" | bc) + time_offset=${time_offset%.*} + echo $time_offset +} + + +function fetch_and_process_blocks { + + echo "Fetching blocks from height ${start_height} to ${finish_height}..." + echo + + total_time_offset=0 + errors=0 + + while [ "${height}" -le "${finish_height}" ]; do + block_minting_info=$(curl -s "http://localhost:12391/blocks/byheight/${height}/mintinginfo") + error=$(echo "${block_minting_info}" | jq -r .error) + if [ "${error}" != "null" ]; then + echo "Error fetching minting info for block ${height}" + echo + errors=$((errors+1)) + height=$((height+1)) + continue; + fi + + # Parse minting info + minter_level=$(echo "${block_minting_info}" | jq -r .minterLevel) + key_distance_ratio=$(echo "${block_minting_info}" | jq -r .keyDistanceRatio) + time_delta=$(echo "${block_minting_info}" | jq -r .timeDelta) + + time_offset=$(calculate_time_offset "${key_distance_ratio}") + block_time=$((target-deviation+time_offset)) + + echo "=== BLOCK ${height} ===" + echo "Minter level: ${minter_level}" + echo "Key distance ratio: ${key_distance_ratio}" + echo "Time offset: ${time_offset}" + echo "Block time (real): ${time_delta}" + echo "Block time (calculated): ${block_time}" + + if [ "${time_delta}" -ne "${block_time}" ]; then + echo "WARNING: Block time mismatch. This is to be expected when using custom settings." + fi + echo + + total_time_offset=$((total_time_offset+block_time)) + + height=$((height+1)) + done + + adjusted_count=$((count-errors)) + if [ "${adjusted_count}" -eq 0 ]; then + echo "No blocks were retrieved." + echo + exit; + fi + + mean_time_offset=$((total_time_offset/adjusted_count)) + time_offset_diff=$((mean_time_offset-target)) + + echo "===================" + echo "===== SUMMARY =====" + echo "===================" + echo "Total blocks retrieved: ${adjusted_count}" + echo "Total blocks failed: ${errors}" + echo "Mean time offset: ${mean_time_offset}ms" + echo "Target time offset: ${target}ms" + echo "Difference from target: ${time_offset_diff}ms" + echo + +} + +function estimate_key_distance_ratio_for_level { + local level=$1 + local example_key_distance="0.5" + echo "(${example_key_distance}/${level})" +} + +function estimate_block_timestamps { + min_block_time=9999999 + max_block_time=0 + + echo "===== BLOCK TIME ESTIMATES =====" + + for level in {1..10}; do + example_key_distance_ratio=$(estimate_key_distance_ratio_for_level "${level}") + time_offset=$(calculate_time_offset "${example_key_distance_ratio}") + block_time=$((target-deviation+time_offset)) + + if [ "${block_time}" -gt "${max_block_time}" ]; then + max_block_time=${block_time} + fi + if [ "${block_time}" -lt "${min_block_time}" ]; then + min_block_time=${block_time} + fi + + echo "Level: ${level}, time offset: ${time_offset}, block time: ${block_time}" + done + block_time_range=$((max_block_time-min_block_time)) + echo "Range: ${block_time_range}" + echo +} + +fetch_and_process_blocks +estimate_block_timestamps From 78cac7f0e64bb78b472aad339d1c060517f55835 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 18:12:09 +0100 Subject: [PATCH 3/4] Updated usage info to reflect the fact that the "count" parameter is optional. Usage: block-timings.sh [count] [target] [deviation] [power] --- tools/block-timings.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/block-timings.sh b/tools/block-timings.sh index 43f2f466..514168dd 100755 --- a/tools/block-timings.sh +++ b/tools/block-timings.sh @@ -8,7 +8,7 @@ if [ -z "${start_height}" ]; then echo "Error: missing start height." echo echo "Usage:" - echo "block-timings.sh [target] [deviation] [power]" + echo "block-timings.sh [count] [target] [deviation] [power]" echo echo "startheight: a block height, preferably within the untrimmed range, to avoid data gaps" echo "count: the number of blocks to request and analyse after the start height. Default: 100" From 45efe7cd5689dde7e010de4187b59da549f3ebfe Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 10 Apr 2021 18:24:33 +0100 Subject: [PATCH 4/4] Slight reordering of vars. --- tools/block-timings.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/block-timings.sh b/tools/block-timings.sh index 514168dd..5ca4f08a 100755 --- a/tools/block-timings.sh +++ b/tools/block-timings.sh @@ -2,6 +2,9 @@ start_height=$1 count=$2 +target=$3 +deviation=$4 +power=$5 if [ -z "${start_height}" ]; then echo @@ -19,10 +22,6 @@ if [ -z "${start_height}" ]; then exit fi -target=$3 -deviation=$4 -power=$5 - count=${count:=100} target=${target:=60000} deviation=${deviation:=30000}