crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Merge branch 'master' into sync-multiple-blocks

Browse Source 
# Conflicts:
#	src/main/java/org/qortal/settings/Settings.java
This commit is contained in:
CalDescent 2021-05-10 09:09:41 +01:00
commit 4ac3984b7c
11 changed files with 302 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/main/java/org/qortal/api/model/BlockMintingInfo.java Normal file
View File

@ -0,0 +1,23 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.math.BigDecimal;
import java.math.BigInteger;
@XmlAccessorType(XmlAccessType.FIELD)
public class BlockMintingInfo {
public byte[] minterPublicKey;
public int minterLevel;
public int onlineAccountsCount;
public BigDecimal maxDistance;
public BigInteger keyDistance;
public double keyDistanceRatio;
public long timestamp;
public long timeDelta;
public BlockMintingInfo() {
}
}

59
src/main/java/org/qortal/api/resource/BlocksResource.java
View File

@ -8,6 +8,9 @@ import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
import java.util.ArrayList;
import java.util.List;
@ -20,10 +23,13 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import org.qortal.account.Account;
import org.qortal.api.ApiError;
import org.qortal.api.ApiErrors;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.model.BlockMintingInfo;
import org.qortal.api.model.BlockSignerSummary;
import org.qortal.block.Block;
import org.qortal.crypto.Crypto;
import org.qortal.data.account.AccountData;
import org.qortal.data.block.BlockData;
@ -328,6 +334,59 @@ public class BlocksResource {
}
}
@GET
@Path("/byheight/{height}/mintinginfo")
@Operation(
summary = "Fetch block minter info using block height",
description = "Returns the minter info for the block with given height",
responses = {
@ApiResponse(
description = "the block",
content = @Content(
schema = @Schema(
implementation = BlockData.class
)
)
)
}
)
@ApiErrors({
ApiError.BLOCK_UNKNOWN, ApiError.REPOSITORY_ISSUE
})
public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) {
try (final Repository repository = RepositoryManager.getRepository()) {
BlockData blockData = repository.getBlockRepository().fromHeight(height);
if (blockData == null)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
Block block = new Block(repository, blockData);
BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference());
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
if (minterLevel == 0)
// This may be unavailable when requesting a trimmed block
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
BigInteger distance = block.calcKeyDistance(parentBlockData.getHeight(), parentBlockData.getSignature(), blockData.getMinterPublicKey(), minterLevel);
double ratio = new BigDecimal(distance).divide(new BigDecimal(block.MAX_DISTANCE), 40, RoundingMode.DOWN).doubleValue();
long timestamp = block.calcTimestamp(parentBlockData, blockData.getMinterPublicKey(), minterLevel);
long timeDelta = timestamp - parentBlockData.getTimestamp();
BlockMintingInfo blockMintingInfo = new BlockMintingInfo();
blockMintingInfo.minterPublicKey = blockData.getMinterPublicKey();
blockMintingInfo.minterLevel = minterLevel;
blockMintingInfo.onlineAccountsCount = blockData.getOnlineAccountsCount();
blockMintingInfo.maxDistance = new BigDecimal(block.MAX_DISTANCE);
blockMintingInfo.keyDistance = distance;
blockMintingInfo.keyDistanceRatio = ratio;
blockMintingInfo.timestamp = timestamp;
blockMintingInfo.timeDelta = timeDelta;
return blockMintingInfo;
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@GET
@Path("/timestamp/{timestamp}")
@Operation(

3
src/main/java/org/qortal/api/resource/CrossChainResource.java
View File

@ -262,6 +262,7 @@ public class CrossChainResource {
// We want both a minimum of 5 trades and enough trades to span at least 4 hours
int minimumCount = 5;
int maximumCount = 10;
long minimumPeriod = 4 * 60 * 60 * 1000L; // ms
Boolean isFinished = Boolean.TRUE;
@ -276,7 +277,7 @@ public class CrossChainResource {
ACCT acct = acctInfo.getValue().get();
List<ATStateData> atStates = repository.getATRepository().getMatchingFinalATStatesQuorum(codeHash,
isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumCount, minimumPeriod);
isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumCount, maximumCount, minimumPeriod);
for (ATStateData atState : atStates) {
CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState);

2
src/main/java/org/qortal/block/Block.java
View File

@ -232,7 +232,7 @@ public class Block {
// Other useful constants
private static final BigInteger MAX_DISTANCE;
public static final BigInteger MAX_DISTANCE;
static {
byte[] maxValue = new byte[Transformer.PUBLIC_KEY_LENGTH];
Arrays.fill(maxValue, (byte) 0xFF);

9
src/main/java/org/qortal/controller/Controller.java
View File

@ -637,6 +637,11 @@ public class Controller extends Thread {
return peerChainTipData == null || peerChainTipData.getLastBlockSignature() == null || inferiorChainTips.contains(new ByteArray(peerChainTipData.getLastBlockSignature()));
};
public static final Predicate<Peer> hasOldVersion = peer -> {
final String minPeerVersion = Settings.getInstance().getMinPeerVersion();
return peer.isAtLeastVersion(minPeerVersion) == false;
};
private void potentiallySynchronize() throws InterruptedException {
// Already synchronizing via another thread?
if (this.isSynchronizing)
@ -653,11 +658,15 @@ public class Controller extends Thread {
// Disregard peers that don't have a recent block
peers.removeIf(hasNoRecentBlock);
// Disregard peers that are on an old version
peers.removeIf(hasOldVersion);
checkRecoveryModeForPeers(peers);
if (recoveryMode) {
peers = Network.getInstance().getHandshakedPeers();
peers.removeIf(hasOnlyGenesisBlock);
peers.removeIf(hasMisbehaved);
peers.removeIf(hasOldVersion);
}
// Check we have enough peers to potentially synchronize

14
src/main/java/org/qortal/network/Handshake.java
View File

@ -4,7 +4,6 @@ import java.util.Arrays;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@ -51,7 +50,7 @@ public enum Handshake {
String versionString = helloMessage.getVersionString();
Matcher matcher = VERSION_PATTERN.matcher(versionString);
Matcher matcher = peer.VERSION_PATTERN.matcher(versionString);
if (!matcher.lookingAt()) {
LOGGER.debug(() -> String.format("Peer %s sent invalid HELLO version string '%s'", peer, versionString));
return null;
@ -72,6 +71,15 @@ public enum Handshake {
peer.setPeersConnectionTimestamp(peersConnectionTimestamp);
peer.setPeersVersion(versionString, version);
if (Settings.getInstance().getAllowConnectionsWithOlderPeerVersions() == false) {
// Ensure the peer is running at least the minimum version allowed for connections
final String minPeerVersion = Settings.getInstance().getMinPeerVersion();
if (peer.isAtLeastVersion(minPeerVersion) == false) {
LOGGER.info(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString));
return null;
}
}
return CHALLENGE;
}
@ -244,8 +252,6 @@ public enum Handshake {
/** Maximum allowed difference between peer's reported timestamp and when they connected, in milliseconds. */
private static final long MAX_TIMESTAMP_DELTA = 30 * 1000L; // ms
private static final Pattern VERSION_PATTERN = Pattern.compile(Controller.VERSION_PREFIX + "(\\d{1,3})\\.(\\d{1,5})\\.(\\d{1,5})");
private static final long PEER_VERSION_131 = 0x0100030001L;
private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes

35
src/main/java/org/qortal/network/Peer.java
View File

@ -20,9 +20,12 @@ import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.data.block.CommonBlockData;
import org.qortal.data.network.PeerChainTipData;
import org.qortal.data.network.PeerData;
@ -87,6 +90,9 @@ public class Peer {
byte[] ourChallenge;
// Versioning
public static final Pattern VERSION_PATTERN = Pattern.compile(Controller.VERSION_PREFIX + "(\\d{1,3})\\.(\\d{1,5})\\.(\\d{1,5})");
// Peer info
private final Object peerInfoLock = new Object();
@ -651,6 +657,35 @@ public class Peer {
}
// Minimum version
public boolean isAtLeastVersion(String minVersionString) {
if (minVersionString == null)
return false;
// Add the version prefix
minVersionString = Controller.VERSION_PREFIX + minVersionString;
Matcher matcher = VERSION_PATTERN.matcher(minVersionString);
if (!matcher.lookingAt())
return false;
// We're expecting 3 positive shorts, so we can convert 1.2.3 into 0x0100020003
long minVersion = 0;
for (int g = 1; g <= 3; ++g) {
long value = Long.parseLong(matcher.group(g));
if (value < 0 || value > Short.MAX_VALUE)
return false;
minVersion <<= 16;
minVersion |= value;
}
return this.getPeersVersion() >= minVersion;
}
// Common block data
public boolean canUseCachedCommonBlockData() {

2
src/main/java/org/qortal/repository/ATRepository.java
View File

@ -98,7 +98,7 @@ public interface ATRepository {
*/
public List<ATStateData> getMatchingFinalATStatesQuorum(byte[] codeHash, Boolean isFinished,
Integer dataByteOffset, Long expectedValue,
int minimumCount, long minimumPeriod) throws DataException;
int minimumCount, int maximumCount, long minimumPeriod) throws DataException;
/**
* Returns all ATStateData for a given block height.

5
src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java
View File

@ -454,7 +454,7 @@ public class HSQLDBATRepository implements ATRepository {
@Override
public List<ATStateData> getMatchingFinalATStatesQuorum(byte[] codeHash, Boolean isFinished,
Integer dataByteOffset, Long expectedValue,
int minimumCount, long minimumPeriod) throws DataException {
int minimumCount, int maximumCount, long minimumPeriod) throws DataException {
// We need most recent entry first so we can use its timestamp to slice further results
List<ATStateData> mostRecentStates = this.getMatchingFinalATStates(codeHash, isFinished,
dataByteOffset, expectedValue, null,
@ -510,7 +510,8 @@ public class HSQLDBATRepository implements ATRepository {
bindParams.add(minimumHeight);
bindParams.add(minimumCount);
sql.append("ORDER BY FinalATStates.height DESC");
sql.append("ORDER BY FinalATStates.height DESC LIMIT ?");
bindParams.add(maximumCount);
List<ATStateData> atStates = new ArrayList<>();

11
src/main/java/org/qortal/settings/Settings.java
View File

@ -125,6 +125,13 @@ public class Settings {
/** Maximum number of retry attempts if a peer fails to respond with the requested data */
private int maxRetries = 2;
/** Minimum peer version number required in order to sync with them */
private String minPeerVersion = "1.5.0";
/** Whether to allow connections with peers below minPeerVersion
* If true, we won't sync with them but they can still sync with us, and will show in the peers list
* If false, sync will be blocked both ways, and they will not appear in the peers list */
private boolean allowConnectionsWithOlderPeerVersions = true;
/** Whether to sync multiple blocks at once in normal operation */
private boolean fastSyncEnabled = false;
/** Whether to sync multiple blocks at once when the peer has a different chain */
@ -421,6 +428,10 @@ public class Settings {
public int getMaxRetries() { return this.maxRetries; }
public String getMinPeerVersion() { return this.minPeerVersion; }
public boolean getAllowConnectionsWithOlderPeerVersions() { return this.allowConnectionsWithOlderPeerVersions; }
public String getBlockchainConfig() {
return this.blockchainConfig;
}

148
tools/block-timings.sh Executable file
View File

@ -0,0 +1,148 @@
#!/usr/bin/env bash
start_height=$1
count=$2
target=$3
deviation=$4
power=$5
if [ -z "${start_height}" ]; then
echo
echo "Error: missing start height."
echo
echo "Usage:"
echo "block-timings.sh <startheight> [count] [target] [deviation] [power]"
echo
echo "startheight: a block height, preferably within the untrimmed range, to avoid data gaps"
echo "count: the number of blocks to request and analyse after the start height. Default: 100"
echo "target: the target block time in milliseconds. Originates from blockchain.json. Default: 60000"
echo "deviation: the allowed block time deviation in milliseconds. Originates from blockchain.json. Default: 30000"
echo "power: used when transforming key distance to a time offset. Originates from blockchain.json. Default: 0.2"
echo
exit
fi
count=${count:=100}
target=${target:=60000}
deviation=${deviation:=30000}
power=${power:=0.2}
finish_height=$((start_height + count - 1))
height=$start_height
echo "Settings:"
echo "Target time offset: ${target}"
echo "Deviation: ${deviation}"
echo "Power transform: ${power}"
echo
function calculate_time_offset {
local key_distance_ratio=$1
local transformed=$( echo "" | awk "END {print ${key_distance_ratio} ^ ${power}}")
local time_offset=$(echo "${deviation}*2*${transformed}" | bc)
time_offset=${time_offset%.*}
echo $time_offset
}
function fetch_and_process_blocks {
echo "Fetching blocks from height ${start_height} to ${finish_height}..."
echo
total_time_offset=0
errors=0
while [ "${height}" -le "${finish_height}" ]; do
block_minting_info=$(curl -s "http://localhost:12391/blocks/byheight/${height}/mintinginfo")
error=$(echo "${block_minting_info}" | jq -r .error)
if [ "${error}" != "null" ]; then
echo "Error fetching minting info for block ${height}"
echo
errors=$((errors+1))
height=$((height+1))
continue;
fi
# Parse minting info
minter_level=$(echo "${block_minting_info}" | jq -r .minterLevel)
online_accounts_count=$(echo "${block_minting_info}" | jq -r .onlineAccountsCount)
key_distance_ratio=$(echo "${block_minting_info}" | jq -r .keyDistanceRatio)
time_delta=$(echo "${block_minting_info}" | jq -r .timeDelta)
time_offset=$(calculate_time_offset "${key_distance_ratio}")
block_time=$((target-deviation+time_offset))
echo "=== BLOCK ${height} ==="
echo "Minter level: ${minter_level}"
echo "Online accounts: ${online_accounts_count}"
echo "Key distance ratio: ${key_distance_ratio}"
echo "Time offset: ${time_offset}"
echo "Block time (real): ${time_delta}"
echo "Block time (calculated): ${block_time}"
if [ "${time_delta}" -ne "${block_time}" ]; then
echo "WARNING: Block time mismatch. This is to be expected when using custom settings."
fi
echo
total_time_offset=$((total_time_offset+block_time))
height=$((height+1))
done
adjusted_count=$((count-errors))
if [ "${adjusted_count}" -eq 0 ]; then
echo "No blocks were retrieved."
echo
exit;
fi
mean_time_offset=$((total_time_offset/adjusted_count))
time_offset_diff=$((mean_time_offset-target))
echo "==================="
echo "===== SUMMARY ====="
echo "==================="
echo "Total blocks retrieved: ${adjusted_count}"
echo "Total blocks failed: ${errors}"
echo "Mean time offset: ${mean_time_offset}ms"
echo "Target time offset: ${target}ms"
echo "Difference from target: ${time_offset_diff}ms"
echo
}
function estimate_key_distance_ratio_for_level {
local level=$1
local example_key_distance="0.5"
echo "(${example_key_distance}/${level})"
}
function estimate_block_timestamps {
min_block_time=9999999
max_block_time=0
echo "===== BLOCK TIME ESTIMATES ====="
for level in {1..10}; do
example_key_distance_ratio=$(estimate_key_distance_ratio_for_level "${level}")
time_offset=$(calculate_time_offset "${example_key_distance_ratio}")
block_time=$((target-deviation+time_offset))
if [ "${block_time}" -gt "${max_block_time}" ]; then
max_block_time=${block_time}
fi
if [ "${block_time}" -lt "${min_block_time}" ]; then
min_block_time=${block_time}
fi
echo "Level: ${level}, time offset: ${time_offset}, block time: ${block_time}"
done
block_time_range=$((max_block_time-min_block_time))
echo "Range: ${block_time_range}"
echo
}
fetch_and_process_blocks
estimate_block_timestamps