crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Merge remote-tracking branch 'catbref/schnorr-agg-BlockMinter-fix' into schnorr-agg-BlockMinter-fix

Browse Source 
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/java/org/qortal/controller/OnlineAccountsManager.java
#	src/main/java/org/qortal/network/message/BlockV2Message.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2.json
This commit is contained in:
CalDescent 2022-06-24 11:47:58 +01:00
commit 23408827b3
27 changed files with 3268 additions and 810 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/org/qortal/account/PrivateKeyAccount.java
View File

@ -11,15 +11,15 @@ public class PrivateKeyAccount extends PublicKeyAccount {
private final Ed25519PrivateKeyParameters edPrivateKeyParams;
/**
* Create PrivateKeyAccount using byte[32] seed.
* Create PrivateKeyAccount using byte[32] private key.
*
* @param seed
* @param privateKey
* byte[32] used to create private/public key pair
* @throws IllegalArgumentException
* if passed invalid seed
* if passed invalid privateKey
*/
public PrivateKeyAccount(Repository repository, byte[] seed) {
this(repository, new Ed25519PrivateKeyParameters(seed, 0));
public PrivateKeyAccount(Repository repository, byte[] privateKey) {
this(repository, new Ed25519PrivateKeyParameters(privateKey, 0));
}
private PrivateKeyAccount(Repository repository, Ed25519PrivateKeyParameters edPrivateKeyParams) {
@ -37,10 +37,6 @@ public class PrivateKeyAccount extends PublicKeyAccount {
return this.privateKey;
}
public static byte[] toPublicKey(byte[] seed) {
return new Ed25519PrivateKeyParameters(seed, 0).generatePublicKey().getEncoded();
}
public byte[] sign(byte[] message) {
return Crypto.sign(this.edPrivateKeyParams, message);
}

137
src/main/java/org/qortal/block/Block.java
View File

@ -10,6 +10,7 @@ import java.math.BigInteger;
import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.text.MessageFormat;
import java.text.NumberFormat;
import java.util.*;
import java.util.stream.Collectors;
@ -27,6 +28,7 @@ import org.qortal.block.BlockChain.BlockTimingByHeight;
import org.qortal.block.BlockChain.AccountLevelShareBin;
import org.qortal.controller.OnlineAccountsManager;
import org.qortal.crypto.Crypto;
import org.qortal.crypto.Qortal25519Extras;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.data.account.AccountData;
import org.qortal.data.account.EligibleQoraHolderData;
@ -221,11 +223,10 @@ public class Block {
return accountAmount;
}
}
/** Always use getExpandedAccounts() to access this, as it's lazy-instantiated. */
private List<ExpandedAccount> cachedExpandedAccounts = null;
/** Opportunistic cache of this block's valid online accounts. Only created by call to isValid(). */
private List<OnlineAccountData> cachedValidOnlineAccounts = null;
/** Opportunistic cache of this block's valid online reward-shares. Only created by call to isValid(). */
private List<RewardShareData> cachedOnlineRewardShares = null;
@ -347,18 +348,21 @@ public class Block {
int version = parentBlock.getNextBlockVersion();
byte[] reference = parentBlockData.getSignature();
// Fetch our list of online accounts
List<OnlineAccountData> onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts();
if (onlineAccounts.isEmpty()) {
LOGGER.error("No online accounts - not even our own?");
// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, minter.getPublicKey());
if (minterLevel == 0) {
LOGGER.error("Minter effective level returned zero?");
return null;
}
// Find newest online accounts timestamp
long onlineAccountsTimestamp = 0;
for (OnlineAccountData onlineAccountData : onlineAccounts) {
if (onlineAccountData.getTimestamp() > onlineAccountsTimestamp)
onlineAccountsTimestamp = onlineAccountData.getTimestamp();
long timestamp = calcTimestamp(parentBlockData, minter.getPublicKey(), minterLevel);
long onlineAccountsTimestamp = OnlineAccountsManager.getCurrentOnlineAccountTimestamp();
// Fetch our list of online accounts
List<OnlineAccountData> onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts(onlineAccountsTimestamp);
if (onlineAccounts.isEmpty()) {
LOGGER.error("No online accounts - not even our own?");
return null;
}
// Load sorted list of reward share public keys into memory, so that the indexes can be obtained.
@ -369,10 +373,6 @@ public class Block {
// Map using index into sorted list of reward-shares as key
Map<Integer, OnlineAccountData> indexedOnlineAccounts = new HashMap<>();
for (OnlineAccountData onlineAccountData : onlineAccounts) {
// Disregard online accounts with different timestamps
if (onlineAccountData.getTimestamp() != onlineAccountsTimestamp)
continue;
Integer accountIndex = getRewardShareIndex(onlineAccountData.getPublicKey(), allRewardSharePublicKeys);
if (accountIndex == null)
// Online account (reward-share) with current timestamp but reward-share cancelled
@ -389,26 +389,29 @@ public class Block {
byte[] encodedOnlineAccounts = BlockTransformer.encodeOnlineAccounts(onlineAccountsSet);
int onlineAccountsCount = onlineAccountsSet.size();
// Concatenate online account timestamp signatures (in correct order)
byte[] onlineAccountsSignatures = new byte[onlineAccountsCount * Transformer.SIGNATURE_LENGTH];
for (int i = 0; i < onlineAccountsCount; ++i) {
Integer accountIndex = accountIndexes.get(i);
OnlineAccountData onlineAccountData = indexedOnlineAccounts.get(accountIndex);
System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH);
byte[] onlineAccountsSignatures;
if (timestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp()) {
// Collate all signatures
Collection<byte[]> signaturesToAggregate = indexedOnlineAccounts.values()
.stream()
.map(OnlineAccountData::getSignature)
.collect(Collectors.toList());
// Aggregated, single signature
onlineAccountsSignatures = Qortal25519Extras.aggregateSignatures(signaturesToAggregate);
} else {
// Concatenate online account timestamp signatures (in correct order)
onlineAccountsSignatures = new byte[onlineAccountsCount * Transformer.SIGNATURE_LENGTH];
for (int i = 0; i < onlineAccountsCount; ++i) {
Integer accountIndex = accountIndexes.get(i);
OnlineAccountData onlineAccountData = indexedOnlineAccounts.get(accountIndex);
System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH);
}
}
byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData,
minter.getPublicKey(), encodedOnlineAccounts));
// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, minter.getPublicKey());
if (minterLevel == 0) {
LOGGER.error("Minter effective level returned zero?");
return null;
}
long timestamp = calcTimestamp(parentBlockData, minter.getPublicKey(), minterLevel);
int transactionCount = 0;
byte[] transactionsSignature = null;
int height = parentBlockData.getHeight() + 1;
@ -1013,49 +1016,59 @@ public class Block {
if (this.blockData.getOnlineAccountsSignatures() == null || this.blockData.getOnlineAccountsSignatures().length == 0)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MISSING;
if (this.blockData.getOnlineAccountsSignatures().length != onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED;
if (this.blockData.getTimestamp() >= BlockChain.getInstance().getAggregateSignatureTimestamp()) {
// We expect just the one, aggregated signature
if (this.blockData.getOnlineAccountsSignatures().length != Transformer.SIGNATURE_LENGTH)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED;
} else {
if (this.blockData.getOnlineAccountsSignatures().length != onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED;
}
// Check signatures
long onlineTimestamp = this.blockData.getOnlineAccountsTimestamp();
byte[] onlineTimestampBytes = Longs.toByteArray(onlineTimestamp);
// If this block is much older than current online timestamp, then there's no point checking current online accounts
List<OnlineAccountData> currentOnlineAccounts = onlineTimestamp < NTP.getTime() - OnlineAccountsManager.ONLINE_TIMESTAMP_MODULUS
? null
: OnlineAccountsManager.getInstance().getOnlineAccounts();
List<OnlineAccountData> latestBlocksOnlineAccounts = OnlineAccountsManager.getInstance().getLatestBlocksOnlineAccounts();
// Extract online accounts' timestamp signatures from block data
// Extract online accounts' timestamp signatures from block data. Only one signature if aggregated.
List<byte[]> onlineAccountsSignatures = BlockTransformer.decodeTimestampSignatures(this.blockData.getOnlineAccountsSignatures());
// We'll build up a list of online accounts to hand over to Controller if block is added to chain
// and this will become latestBlocksOnlineAccounts (above) to reduce CPU load when we process next block...
List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
if (this.blockData.getTimestamp() >= BlockChain.getInstance().getAggregateSignatureTimestamp()) {
// Aggregate all public keys
Collection<byte[]> publicKeys = onlineRewardShares.stream()
.map(RewardShareData::getRewardSharePublicKey)
.collect(Collectors.toList());
for (int i = 0; i < onlineAccountsSignatures.size(); ++i) {
byte[] signature = onlineAccountsSignatures.get(i);
byte[] publicKey = onlineRewardShares.get(i).getRewardSharePublicKey();
byte[] aggregatePublicKey = Qortal25519Extras.aggregatePublicKeys(publicKeys);
OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, signature, publicKey);
ourOnlineAccounts.add(onlineAccountData);
byte[] aggregateSignature = onlineAccountsSignatures.get(0);
// If signature is still current then no need to perform Ed25519 verify
if (currentOnlineAccounts != null && currentOnlineAccounts.remove(onlineAccountData))
// remove() returned true, so online account still current
// and one less entry in currentOnlineAccounts to check next time
continue;
// If signature was okay in latest block then no need to perform Ed25519 verify
if (latestBlocksOnlineAccounts != null && latestBlocksOnlineAccounts.contains(onlineAccountData))
continue;
if (!Crypto.verify(publicKey, signature, onlineTimestampBytes))
// One-step verification of aggregate signature using aggregate public key
if (!Qortal25519Extras.verifyAggregated(aggregatePublicKey, aggregateSignature, onlineTimestampBytes))
return ValidationResult.ONLINE_ACCOUNT_SIGNATURE_INCORRECT;
} else {
// Build block's view of online accounts
Set<OnlineAccountData> onlineAccounts = new HashSet<>();
for (int i = 0; i < onlineAccountsSignatures.size(); ++i) {
byte[] signature = onlineAccountsSignatures.get(i);
byte[] publicKey = onlineRewardShares.get(i).getRewardSharePublicKey();
OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, signature, publicKey);
onlineAccounts.add(onlineAccountData);
}
// Remove those already validated & cached by online accounts manager - no need to re-validate them
OnlineAccountsManager.getInstance().removeKnown(onlineAccounts, onlineTimestamp);
// Validate the rest
for (OnlineAccountData onlineAccount : onlineAccounts)
if (!Crypto.verify(onlineAccount.getPublicKey(), onlineAccount.getSignature(), onlineTimestampBytes))
return ValidationResult.ONLINE_ACCOUNT_SIGNATURE_INCORRECT;
// We've validated these, so allow online accounts manager to cache
OnlineAccountsManager.getInstance().addBlocksOnlineAccounts(onlineAccounts, onlineTimestamp);
}
// All online accounts valid, so save our list of online accounts for potential later use
this.cachedValidOnlineAccounts = ourOnlineAccounts;
this.cachedOnlineRewardShares = onlineRewardShares;
return ValidationResult.OK;
@ -1426,9 +1439,6 @@ public class Block {
postBlockTidy();
// Give Controller our cached, valid online accounts data (if any) to help reduce CPU load for next block
OnlineAccountsManager.getInstance().pushLatestBlocksOnlineAccounts(this.cachedValidOnlineAccounts);
// Log some debugging info relating to the block weight calculation
this.logDebugInfo();
}
@ -1644,9 +1654,6 @@ public class Block {
this.blockData.setHeight(null);
postBlockTidy();
// Remove any cached, valid online accounts data from Controller
OnlineAccountsManager.getInstance().popLatestBlocksOnlineAccounts();
}
protected void orphanTransactionsFromBlock() throws DataException {

7
src/main/java/org/qortal/block/BlockChain.java
View File

@ -71,7 +71,8 @@ public class BlockChain {
calcChainWeightTimestamp,
transactionV5Timestamp,
transactionV6Timestamp,
disableReferenceTimestamp
disableReferenceTimestamp,
aggregateSignatureTimestamp;
}
// Custom transaction fees
@ -415,6 +416,10 @@ public class BlockChain {
return this.featureTriggers.get(FeatureTrigger.disableReferenceTimestamp.name()).longValue();
}
public long getAggregateSignatureTimestamp() {
return this.featureTriggers.get(FeatureTrigger.aggregateSignatureTimestamp.name()).longValue();
}
// More complex getters for aspects that change by height or timestamp
public long getRewardAtHeight(int ourHeight) {

597
src/main/java/org/qortal/controller/BlockMinter.java
View File

@ -65,9 +65,8 @@ public class BlockMinter extends Thread {
// Lite nodes do not mint
return;
}
try (final Repository repository = RepositoryManager.getRepository()) {
if (Settings.getInstance().getWipeUnconfirmedOnStart()) {
if (Settings.getInstance().getWipeUnconfirmedOnStart()) {
try (final Repository repository = RepositoryManager.getRepository()) {
// Wipe existing unconfirmed transactions
List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions();
@ -77,30 +76,31 @@ public class BlockMinter extends Thread {
}
repository.saveChanges();
} catch (DataException e) {
LOGGER.warn("Repository issue trying to wipe unconfirmed transactions on start-up: {}", e.getMessage());
// Fall-through to normal behaviour in case we can recover
}
}
// Going to need this a lot...
BlockRepository blockRepository = repository.getBlockRepository();
BlockData previousBlockData = null;
BlockData previousBlockData = null;
// Vars to keep track of blocks that were skipped due to chain weight
byte[] parentSignatureForLastLowWeightBlock = null;
Long timeOfLastLowWeightBlock = null;
// Vars to keep track of blocks that were skipped due to chain weight
byte[] parentSignatureForLastLowWeightBlock = null;
Long timeOfLastLowWeightBlock = null;
List<Block> newBlocks = new ArrayList<>();
List<Block> newBlocks = new ArrayList<>();
// Flags for tracking change in whether minting is possible,
// so we can notify Controller, and further update SysTray, etc.
boolean isMintingPossible = false;
boolean wasMintingPossible = isMintingPossible;
while (running) {
repository.discardChanges(); // Free repository locks, if any
// Flags for tracking change in whether minting is possible,
// so we can notify Controller, and further update SysTray, etc.
boolean isMintingPossible = false;
boolean wasMintingPossible = isMintingPossible;
while (running) {
if (isMintingPossible != wasMintingPossible)
Controller.getInstance().onMintingPossibleChange(isMintingPossible);
if (isMintingPossible != wasMintingPossible)
Controller.getInstance().onMintingPossibleChange(isMintingPossible);
wasMintingPossible = isMintingPossible;
wasMintingPossible = isMintingPossible;
try {
// Sleep for a while
Thread.sleep(1000);
@ -114,319 +114,338 @@ public class BlockMinter extends Thread {
if (minLatestBlockTimestamp == null)
continue;
// No online accounts? (e.g. during startup)
if (OnlineAccountsManager.getInstance().getOnlineAccounts().isEmpty())
// No online accounts for current timestamp? (e.g. during startup)
if (!OnlineAccountsManager.getInstance().hasOnlineAccounts())
continue;
List<MintingAccountData> mintingAccountsData = repository.getAccountRepository().getMintingAccounts();
// No minting accounts?
if (mintingAccountsData.isEmpty())
continue;
try (final Repository repository = RepositoryManager.getRepository()) {
// Going to need this a lot...
BlockRepository blockRepository = repository.getBlockRepository();
// Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level
// Note that minting accounts are actually reward-shares in Qortal
Iterator<MintingAccountData> madi = mintingAccountsData.iterator();
while (madi.hasNext()) {
MintingAccountData mintingAccountData = madi.next();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
if (rewardShareData == null) {
// Reward-share doesn't exist - probably cancelled but not yet removed from node's list of minting accounts
madi.remove();
continue;
}
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
madi.remove();
continue;
}
// Optional (non-validated) prevention of block submissions below a defined level.
// This is an unvalidated version of Blockchain.minAccountLevelToMint
// and exists only to reduce block candidates by default.
int level = mintingAccount.getEffectiveMintingLevel();
if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
madi.remove();
continue;
}
}
// Needs a mutable copy of the unmodifiableList
List<Peer> peers = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers());
BlockData lastBlockData = blockRepository.getLastBlock();
// Disregard peers that have "misbehaved" recently
peers.removeIf(Controller.hasMisbehaved);
// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
if (Synchronizer.getInstance().getRecoveryMode() == false)
peers.removeIf(Controller.hasNoRecentBlock);
// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
continue;
// If we are stuck on an invalid block, we should allow an alternative to be minted
boolean recoverInvalidBlock = false;
if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) {
// We've had at least one invalid block
long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived;
long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived;
if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) {
if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) {
// Last valid block was more than 10 mins ago, but we've had an invalid block since then
// Assume that the chain has stalled because there is no alternative valid candidate
// Enter recovery mode to allow alternative, valid candidates to be minted
recoverInvalidBlock = true;
}
}
}
// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
if (Synchronizer.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false)
List<MintingAccountData> mintingAccountsData = repository.getAccountRepository().getMintingAccounts();
// No minting accounts?
if (mintingAccountsData.isEmpty())
continue;
// There are enough peers with a recent block and our latest block is recent
// so go ahead and mint a block if possible.
isMintingPossible = true;
// Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level
// Note that minting accounts are actually reward-shares in Qortal
Iterator<MintingAccountData> madi = mintingAccountsData.iterator();
while (madi.hasNext()) {
MintingAccountData mintingAccountData = madi.next();
// Check blockchain hasn't changed
if (previousBlockData == null || !Arrays.equals(previousBlockData.getSignature(), lastBlockData.getSignature())) {
previousBlockData = lastBlockData;
newBlocks.clear();
// Reduce log timeout
logTimeout = 10 * 1000L;
// Last low weight block is no longer valid
parentSignatureForLastLowWeightBlock = null;
}
// Discard accounts we have already built blocks with
mintingAccountsData.removeIf(mintingAccountData -> newBlocks.stream().anyMatch(newBlock -> Arrays.equals(newBlock.getBlockData().getMinterPublicKey(), mintingAccountData.getPublicKey())));
// Do we need to build any potential new blocks?
List<PrivateKeyAccount> newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
// We might need to sit the next block out, if one of our minting accounts signed the previous one
final byte[] previousBlockMinter = previousBlockData.getMinterPublicKey();
final boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter));
if (mintedLastBlock) {
LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one"));
continue;
}
if (parentSignatureForLastLowWeightBlock != null) {
// The last iteration found a higher weight block in the network, so sleep for a while
// to allow is to sync the higher weight chain. We are sleeping here rather than when
// detected as we don't want to hold the blockchain lock open.
LOGGER.debug("Sleeping for 10 seconds...");
Thread.sleep(10 * 1000L);
}
for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
// First block does the AT heavy-lifting
if (newBlocks.isEmpty()) {
Block newBlock = Block.mint(repository, previousBlockData, mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't build a to-be-minted block"));
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
if (rewardShareData == null) {
// Reward-share doesn't exist - probably cancelled but not yet removed from node's list of minting accounts
madi.remove();
continue;
}
newBlocks.add(newBlock);
} else {
// The blocks for other minters require less effort...
Block newBlock = newBlocks.get(0).remint(mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't rebuild a to-be-minted block"));
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
madi.remove();
continue;
}
newBlocks.add(newBlock);
// Optional (non-validated) prevention of block submissions below a defined level.
// This is an unvalidated version of Blockchain.minAccountLevelToMint
// and exists only to reduce block candidates by default.
int level = mintingAccount.getEffectiveMintingLevel();
if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
madi.remove();
continue;
}
}
}
// No potential block candidates?
if (newBlocks.isEmpty())
continue;
// Needs a mutable copy of the unmodifiableList
List<Peer> peers = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers());
BlockData lastBlockData = blockRepository.getLastBlock();
// Make sure we're the only thread modifying the blockchain
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (!blockchainLock.tryLock(30, TimeUnit.SECONDS)) {
LOGGER.debug("Couldn't acquire blockchain lock even after waiting 30 seconds");
continue;
}
// Disregard peers that have "misbehaved" recently
peers.removeIf(Controller.hasMisbehaved);
boolean newBlockMinted = false;
Block newBlock = null;
// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
if (Synchronizer.getInstance().getRecoveryMode() == false)
peers.removeIf(Controller.hasNoRecentBlock);
try {
// Clear repository session state so we have latest view of data
repository.discardChanges();
// Now that we have blockchain lock, do final check that chain hasn't changed
BlockData latestBlockData = blockRepository.getLastBlock();
if (!Arrays.equals(lastBlockData.getSignature(), latestBlockData.getSignature()))
// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
continue;
List<Block> goodBlocks = new ArrayList<>();
for (Block testBlock : newBlocks) {
// Is new block's timestamp valid yet?
// We do a separate check as some timestamp checks are skipped for testchains
if (testBlock.isTimestampValid() != ValidationResult.OK)
continue;
testBlock.preProcess();
// Is new block valid yet? (Before adding unconfirmed transactions)
ValidationResult result = testBlock.isValid();
if (result != ValidationResult.OK) {
moderatedLog(() -> LOGGER.error(String.format("To-be-minted block invalid '%s' before adding transactions?", result.name())));
continue;
}
goodBlocks.add(testBlock);
}
if (goodBlocks.isEmpty())
continue;
// Pick best block
final int parentHeight = previousBlockData.getHeight();
final byte[] parentBlockSignature = previousBlockData.getSignature();
BigInteger bestWeight = null;
for (int bi = 0; bi < goodBlocks.size(); ++bi) {
BlockData blockData = goodBlocks.get(bi).getBlockData();
BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
blockSummaryData.setMinterLevel(minterLevel);
BigInteger blockWeight = Block.calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData);
if (bestWeight == null || blockWeight.compareTo(bestWeight) < 0) {
newBlock = goodBlocks.get(bi);
bestWeight = blockWeight;
}
}
try {
if (this.higherWeightChainExists(repository, bestWeight)) {
// Check if the base block has updated since the last time we were here
if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null ||
!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) {
// We've switched to a different chain, so reset the timer
timeOfLastLowWeightBlock = NTP.getTime();
// If we are stuck on an invalid block, we should allow an alternative to be minted
boolean recoverInvalidBlock = false;
if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) {
// We've had at least one invalid block
long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived;
long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived;
if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) {
if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) {
// Last valid block was more than 10 mins ago, but we've had an invalid block since then
// Assume that the chain has stalled because there is no alternative valid candidate
// Enter recovery mode to allow alternative, valid candidates to be minted
recoverInvalidBlock = true;
}
parentSignatureForLastLowWeightBlock = previousBlockData.getSignature();
}
}
// If less than 30 seconds has passed since first detection the higher weight chain,
// we should skip our block submission to give us the opportunity to sync to the better chain
if (NTP.getTime() - timeOfLastLowWeightBlock < 30*1000L) {
LOGGER.debug("Higher weight chain found in peers, so not signing a block this round");
LOGGER.debug("Time since detected: {}ms", NTP.getTime() - timeOfLastLowWeightBlock);
// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
if (Synchronizer.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false)
continue;
// There are enough peers with a recent block and our latest block is recent
// so go ahead and mint a block if possible.
isMintingPossible = true;
// Reattach newBlocks to new repository handle
for (Block newBlock : newBlocks)
newBlock.setRepository(repository);
// Check blockchain hasn't changed
if (previousBlockData == null || !Arrays.equals(previousBlockData.getSignature(), lastBlockData.getSignature())) {
previousBlockData = lastBlockData;
newBlocks.clear();
// Reduce log timeout
logTimeout = 10 * 1000L;
// Last low weight block is no longer valid
parentSignatureForLastLowWeightBlock = null;
}
// Discard accounts we have already built blocks with
mintingAccountsData.removeIf(mintingAccountData -> newBlocks.stream().anyMatch(newBlock -> Arrays.equals(newBlock.getBlockData().getMinterPublicKey(), mintingAccountData.getPublicKey())));
// Do we need to build any potential new blocks?
List<PrivateKeyAccount> newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
// We might need to sit the next block out, if one of our minting accounts signed the previous one
byte[] previousBlockMinter = previousBlockData.getMinterPublicKey();
boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter));
if (mintedLastBlock) {
LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one"));
continue;
}
if (parentSignatureForLastLowWeightBlock != null) {
// The last iteration found a higher weight block in the network, so sleep for a while
// to allow is to sync the higher weight chain. We are sleeping here rather than when
// detected as we don't want to hold the blockchain lock open.
LOGGER.info("Sleeping for 10 seconds...");
Thread.sleep(10 * 1000L);
}
for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
// First block does the AT heavy-lifting
if (newBlocks.isEmpty()) {
Block newBlock = Block.mint(repository, previousBlockData, mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't build a to-be-minted block"));
continue;
}
else {
// More than 30 seconds have passed, so we should submit our block candidate anyway.
LOGGER.debug("More than 30 seconds passed, so proceeding to submit block candidate...");
newBlocks.add(newBlock);
} else {
// The blocks for other minters require less effort...
Block newBlock = newBlocks.get(0).remint(mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't rebuild a to-be-minted block"));
continue;
}
newBlocks.add(newBlock);
}
else {
LOGGER.debug("No higher weight chain found in peers");
}
} catch (DataException e) {
LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
}
// Discard any uncommitted changes as a result of the higher weight chain detection
repository.discardChanges();
// No potential block candidates?
if (newBlocks.isEmpty())
continue;
// Clear variables that track low weight blocks
parentSignatureForLastLowWeightBlock = null;
timeOfLastLowWeightBlock = null;
// Add unconfirmed transactions
addUnconfirmedTransactions(repository, newBlock);
// Sign to create block's signature
newBlock.sign();
// Is newBlock still valid?
ValidationResult validationResult = newBlock.isValid();
if (validationResult != ValidationResult.OK) {
// No longer valid? Report and discard
LOGGER.error(String.format("To-be-minted block now invalid '%s' after adding unconfirmed transactions?", validationResult.name()));
// Rebuild block candidates, just to be sure
newBlocks.clear();
// Make sure we're the only thread modifying the blockchain
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (!blockchainLock.tryLock(30, TimeUnit.SECONDS)) {
LOGGER.debug("Couldn't acquire blockchain lock even after waiting 30 seconds");
continue;
}
// Add to blockchain - something else will notice and broadcast new block to network
boolean newBlockMinted = false;
Block newBlock = null;
try {
newBlock.process();
// Clear repository session state so we have latest view of data
repository.discardChanges();
repository.saveChanges();
// Now that we have blockchain lock, do final check that chain hasn't changed
BlockData latestBlockData = blockRepository.getLastBlock();
if (!Arrays.equals(lastBlockData.getSignature(), latestBlockData.getSignature()))
continue;
LOGGER.info(String.format("Minted new block: %d", newBlock.getBlockData().getHeight()));
List<Block> goodBlocks = new ArrayList<>();
boolean wasInvalidBlockDiscarded = false;
Iterator<Block> newBlocksIterator = newBlocks.iterator();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey());
while (newBlocksIterator.hasNext()) {
Block testBlock = newBlocksIterator.next();
if (rewardShareData != null) {
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
Base58.encode(newBlock.getParent().getSignature()),
rewardShareData.getMinter(),
rewardShareData.getRecipient()));
} else {
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
Base58.encode(newBlock.getParent().getSignature()),
newBlock.getMinter().getAddress()));
// Is new block's timestamp valid yet?
// We do a separate check as some timestamp checks are skipped for testchains
if (testBlock.isTimestampValid() != ValidationResult.OK)
continue;
testBlock.preProcess();
// Is new block valid yet? (Before adding unconfirmed transactions)
ValidationResult result = testBlock.isValid();
if (result != ValidationResult.OK) {
moderatedLog(() -> LOGGER.error(String.format("To-be-minted block invalid '%s' before adding transactions?", result.name())));
newBlocksIterator.remove();
wasInvalidBlockDiscarded = true;
/*
* Bail out fast so that we loop around from the top again.
* This gives BlockMinter the possibility to remint this candidate block using another block from newBlocks,
* via the Blocks.remint() method, which avoids having to re-process Block ATs all over again.
* Particularly useful if some aspect of Blocks changes due a timestamp-based feature-trigger (see BlockChain class).
*/
break;
}
goodBlocks.add(testBlock);
}
// Notify network after we're released blockchain lock
newBlockMinted = true;
if (wasInvalidBlockDiscarded || goodBlocks.isEmpty())
continue;
// Notify Controller
repository.discardChanges(); // clear transaction status to prevent deadlocks
Controller.getInstance().onNewBlock(newBlock.getBlockData());
} catch (DataException e) {
// Unable to process block - report and discard
LOGGER.error("Unable to process newly minted block?", e);
newBlocks.clear();
// Pick best block
final int parentHeight = previousBlockData.getHeight();
final byte[] parentBlockSignature = previousBlockData.getSignature();
BigInteger bestWeight = null;
for (int bi = 0; bi < goodBlocks.size(); ++bi) {
BlockData blockData = goodBlocks.get(bi).getBlockData();
BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
blockSummaryData.setMinterLevel(minterLevel);
BigInteger blockWeight = Block.calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData);
if (bestWeight == null || blockWeight.compareTo(bestWeight) < 0) {
newBlock = goodBlocks.get(bi);
bestWeight = blockWeight;
}
}
try {
if (this.higherWeightChainExists(repository, bestWeight)) {
// Check if the base block has updated since the last time we were here
if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null ||
!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) {
// We've switched to a different chain, so reset the timer
timeOfLastLowWeightBlock = NTP.getTime();
}
parentSignatureForLastLowWeightBlock = previousBlockData.getSignature();
// If less than 30 seconds has passed since first detection the higher weight chain,
// we should skip our block submission to give us the opportunity to sync to the better chain
if (NTP.getTime() - timeOfLastLowWeightBlock < 30 * 1000L) {
LOGGER.info("Higher weight chain found in peers, so not signing a block this round");
LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock);
continue;
} else {
// More than 30 seconds have passed, so we should submit our block candidate anyway.
LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate...");
}
} else {
LOGGER.debug("No higher weight chain found in peers");
}
} catch (DataException e) {
LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
}
// Discard any uncommitted changes as a result of the higher weight chain detection
repository.discardChanges();
// Clear variables that track low weight blocks
parentSignatureForLastLowWeightBlock = null;
timeOfLastLowWeightBlock = null;
// Add unconfirmed transactions
addUnconfirmedTransactions(repository, newBlock);
// Sign to create block's signature
newBlock.sign();
// Is newBlock still valid?
ValidationResult validationResult = newBlock.isValid();
if (validationResult != ValidationResult.OK) {
// No longer valid? Report and discard
LOGGER.error(String.format("To-be-minted block now invalid '%s' after adding unconfirmed transactions?", validationResult.name()));
// Rebuild block candidates, just to be sure
newBlocks.clear();
continue;
}
// Add to blockchain - something else will notice and broadcast new block to network
try {
newBlock.process();
repository.saveChanges();
LOGGER.info(String.format("Minted new block: %d", newBlock.getBlockData().getHeight()));
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey());
if (rewardShareData != null) {
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
Base58.encode(newBlock.getParent().getSignature()),
rewardShareData.getMinter(),
rewardShareData.getRecipient()));
} else {
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
Base58.encode(newBlock.getParent().getSignature()),
newBlock.getMinter().getAddress()));
}
// Notify network after we're released blockchain lock
newBlockMinted = true;
// Notify Controller
repository.discardChanges(); // clear transaction status to prevent deadlocks
Controller.getInstance().onNewBlock(newBlock.getBlockData());
} catch (DataException e) {
// Unable to process block - report and discard
LOGGER.error("Unable to process newly minted block?", e);
newBlocks.clear();
}
} finally {
blockchainLock.unlock();
}
} finally {
blockchainLock.unlock();
}
if (newBlockMinted) {
// Broadcast our new chain to network
BlockData newBlockData = newBlock.getBlockData();
if (newBlockMinted) {
// Broadcast our new chain to network
BlockData newBlockData = newBlock.getBlockData();
Network network = Network.getInstance();
network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newBlockData));
Network network = Network.getInstance();
network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newBlockData));
}
} catch (DataException e) {
LOGGER.warn("Repository issue while running block minter", e);
}
} catch (InterruptedException e) {
// We've been interrupted - time to exit
return;
}
} catch (DataException e) {
LOGGER.warn("Repository issue while running block minter", e);
} catch (InterruptedException e) {
// We've been interrupted - time to exit
return;
}
}

4
src/main/java/org/qortal/controller/Controller.java
View File

@ -1241,6 +1241,10 @@ public class Controller extends Thread {
OnlineAccountsManager.getInstance().onNetworkOnlineAccountsV2Message(peer, message);
break;
case GET_ONLINE_ACCOUNTS_V3:
OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV3Message(peer, message);
break;
case GET_ARBITRARY_DATA:
// Not currently supported
break;

806
src/main/java/org/qortal/controller/OnlineAccountsManager.java
View File

@ -1,12 +1,15 @@
package org.qortal.controller;
import com.google.common.hash.HashCode;
import com.google.common.primitives.Longs;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.Account;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.account.PublicKeyAccount;
import org.qortal.block.Block;
import org.qortal.block.BlockChain;
import org.qortal.crypto.Crypto;
import org.qortal.crypto.Qortal25519Extras;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.account.RewardShareData;
import org.qortal.data.network.OnlineAccountData;
@ -18,270 +21,352 @@ import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import org.qortal.utils.NamedThreadFactory;
import java.util.*;
import java.util.concurrent.*;
import java.util.stream.Collectors;
public class OnlineAccountsManager extends Thread {
private class OurOnlineAccountsThread extends Thread {
public void run() {
try {
while (!isStopping) {
Thread.sleep(10000L);
// Refresh our online accounts signatures?
sendOurOnlineAccountsInfo();
}
} catch (InterruptedException e) {
// Fall through to exit thread
}
}
}
public class OnlineAccountsManager {
private static final Logger LOGGER = LogManager.getLogger(OnlineAccountsManager.class);
private static OnlineAccountsManager instance;
// 'Current' as in 'now'
/**
* How long online accounts signatures last before they expire.
*/
public static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000L;
/**
* How many 'current' timestamp-sets of online accounts we cache.
*/
private static final int MAX_CACHED_TIMESTAMP_SETS = 2;
/**
* How many timestamp-sets of online accounts we cache for 'latest blocks'.
*/
private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 3;
private static final long ONLINE_ACCOUNTS_QUEUE_INTERVAL = 100L; //ms
private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms
private static final long ONLINE_ACCOUNTS_LEGACY_BROADCAST_INTERVAL = 60 * 1000L; // ms
private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 15 * 1000L; // ms
private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L; // v3.2.0
private static final long ONLINE_ACCOUNTS_V3_PEER_VERSION = 0x03000300cbL; // v3.3.203
private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(4, new NamedThreadFactory("OnlineAccounts"));
private volatile boolean isStopping = false;
// To do with online accounts list
private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms
private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 1 * 60 * 1000L; // ms
public static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000L;
private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L);
/** How many (latest) blocks' worth of online accounts we cache */
private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2;
private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L;
private final Set<OnlineAccountData> onlineAccountsImportQueue = ConcurrentHashMap.newKeySet();
private long onlineAccountsTasksTimestamp = Controller.startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms
/**
* Cache of 'current' online accounts, keyed by timestamp
*/
private final Map<Long, Set<OnlineAccountData>> currentOnlineAccounts = new ConcurrentHashMap<>();
/**
* Cache of hash-summary of 'current' online accounts, keyed by timestamp, then leading byte of public key.
*/
private final Map<Long, Map<Byte, byte[]>> currentOnlineAccountsHashes = new ConcurrentHashMap<>();
private final List<OnlineAccountData> onlineAccountsImportQueue = Collections.synchronizedList(new ArrayList<>());
/**
* Cache of online accounts for latest blocks - not necessarily 'current' / now.
* <i>Probably</i> only accessed / modified by a single Synchronizer thread.
*/
private final SortedMap<Long, Set<OnlineAccountData>> latestBlocksOnlineAccounts = new ConcurrentSkipListMap<>();
private boolean hasOnlineAccounts = false;
private boolean hasOurOnlineAccounts = false;
public static Long getCurrentOnlineAccountTimestamp() {
Long now = NTP.getTime();
if (now == null)
return null;
/** Cache of current 'online accounts' */
List<OnlineAccountData> onlineAccounts = new ArrayList<>();
/** Cache of latest blocks' online accounts */
Deque<List<OnlineAccountData>> latestBlocksOnlineAccounts = new ArrayDeque<>(MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS);
public OnlineAccountsManager() {
return (now / ONLINE_TIMESTAMP_MODULUS) * ONLINE_TIMESTAMP_MODULUS;
}
public static synchronized OnlineAccountsManager getInstance() {
if (instance == null) {
instance = new OnlineAccountsManager();
}
return instance;
private OnlineAccountsManager() {
}
public void run() {
private static class SingletonContainer {
private static final OnlineAccountsManager INSTANCE = new OnlineAccountsManager();
}
// Start separate thread to prepare our online accounts
// This could be converted to a thread pool later if more concurrency is needed
OurOnlineAccountsThread ourOnlineAccountsThread = new OurOnlineAccountsThread();
ourOnlineAccountsThread.start();
public static OnlineAccountsManager getInstance() {
return SingletonContainer.INSTANCE;
}
try {
while (!Controller.isStopping()) {
Thread.sleep(100L);
public void start() {
// Expire old online accounts signatures
executor.scheduleAtFixedRate(this::expireOldOnlineAccounts, ONLINE_ACCOUNTS_TASKS_INTERVAL, ONLINE_ACCOUNTS_TASKS_INTERVAL, TimeUnit.MILLISECONDS);
final Long now = NTP.getTime();
if (now == null) {
continue;
}
// Send our online accounts
executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS);
// Perform tasks to do with managing online accounts list
if (now >= onlineAccountsTasksTimestamp) {
onlineAccountsTasksTimestamp = now + ONLINE_ACCOUNTS_TASKS_INTERVAL;
performOnlineAccountsTasks();
}
// Request online accounts from peers (legacy)
executor.scheduleAtFixedRate(this::requestLegacyRemoteOnlineAccounts, ONLINE_ACCOUNTS_LEGACY_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_LEGACY_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS);
// Request online accounts from peers (V3+)
executor.scheduleAtFixedRate(this::requestRemoteOnlineAccounts, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS);
// Process queued online account verifications
this.processOnlineAccountsImportQueue();
}
} catch (InterruptedException e) {
// Fall through to exit thread
}
ourOnlineAccountsThread.interrupt();
// Process import queue
executor.scheduleWithFixedDelay(this::processOnlineAccountsImportQueue, ONLINE_ACCOUNTS_QUEUE_INTERVAL, ONLINE_ACCOUNTS_QUEUE_INTERVAL, TimeUnit.MILLISECONDS);
}
public void shutdown() {
isStopping = true;
this.interrupt();
}
public boolean hasOnlineAccounts() {
return this.hasOnlineAccounts;
}
// Online accounts import queue
private void processOnlineAccountsImportQueue() {
if (this.onlineAccountsImportQueue.isEmpty()) {
// Nothing to do
return;
}
LOGGER.debug("Processing online accounts import queue (size: {})", this.onlineAccountsImportQueue.size());
try (final Repository repository = RepositoryManager.getRepository()) {
List<OnlineAccountData> onlineAccountDataCopy = new ArrayList<>(this.onlineAccountsImportQueue);
for (OnlineAccountData onlineAccountData : onlineAccountDataCopy) {
if (isStopping) {
return;
}
this.verifyAndAddAccount(repository, onlineAccountData);
// Remove from queue
onlineAccountsImportQueue.remove(onlineAccountData);
}
LOGGER.debug("Finished processing online accounts import queue");
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while verifying online accounts"), e);
}
}
// Utilities
private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException {
final Long now = NTP.getTime();
if (now == null)
return;
PublicKeyAccount otherAccount = new PublicKeyAccount(repository, onlineAccountData.getPublicKey());
// Check timestamp is 'recent' here
if (Math.abs(onlineAccountData.getTimestamp() - now) > ONLINE_TIMESTAMP_MODULUS * 2) {
LOGGER.trace(() -> String.format("Rejecting online account %s with out of range timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp()));
return;
}
// Verify
byte[] data = Longs.toByteArray(onlineAccountData.getTimestamp());
if (!otherAccount.verify(onlineAccountData.getSignature(), data)) {
LOGGER.trace(() -> String.format("Rejecting invalid online account %s", otherAccount.getAddress()));
return;
}
// Qortal: check online account is actually reward-share
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(onlineAccountData.getPublicKey());
if (rewardShareData == null) {
// Reward-share doesn't even exist - probably not a good sign
LOGGER.trace(() -> String.format("Rejecting unknown online reward-share public key %s", Base58.encode(onlineAccountData.getPublicKey())));
return;
}
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
LOGGER.trace(() -> String.format("Rejecting online reward-share with non-minting account %s", mintingAccount.getAddress()));
return;
}
synchronized (this.onlineAccounts) {
OnlineAccountData existingAccountData = this.onlineAccounts.stream().filter(account -> Arrays.equals(account.getPublicKey(), onlineAccountData.getPublicKey())).findFirst().orElse(null);
if (existingAccountData != null) {
if (existingAccountData.getTimestamp() < onlineAccountData.getTimestamp()) {
this.onlineAccounts.remove(existingAccountData);
LOGGER.trace(() -> String.format("Updated online account %s with timestamp %d (was %d)", otherAccount.getAddress(), onlineAccountData.getTimestamp(), existingAccountData.getTimestamp()));
} else {
LOGGER.trace(() -> String.format("Not updating existing online account %s", otherAccount.getAddress()));
return;
}
} else {
LOGGER.trace(() -> String.format("Added online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp()));
}
this.onlineAccounts.add(onlineAccountData);
}
executor.shutdownNow();
}
// Testing support
public void ensureTestingAccountsOnline(PrivateKeyAccount... onlineAccounts) {
if (!BlockChain.getInstance().isTestChain()) {
LOGGER.warn("Ignoring attempt to ensure test account is online for non-test chain!");
return;
}
final Long now = NTP.getTime();
if (now == null)
final Long onlineAccountsTimestamp = getCurrentOnlineAccountTimestamp();
if (onlineAccountsTimestamp == null)
return;
final long onlineAccountsTimestamp = toOnlineAccountTimestamp(now);
byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
final boolean useAggregateCompatibleSignature = onlineAccountsTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp();
synchronized (this.onlineAccounts) {
this.onlineAccounts.clear();
Set<OnlineAccountData> replacementAccounts = new HashSet<>();
for (PrivateKeyAccount onlineAccount : onlineAccounts) {
// Check mintingAccount is actually reward-share?
for (PrivateKeyAccount onlineAccount : onlineAccounts) {
// Check mintingAccount is actually reward-share?
byte[] signature = useAggregateCompatibleSignature
? Qortal25519Extras.signForAggregation(onlineAccount.getPrivateKey(), timestampBytes)
: onlineAccount.sign(timestampBytes);
byte[] publicKey = onlineAccount.getPublicKey();
byte[] signature = onlineAccount.sign(timestampBytes);
byte[] publicKey = onlineAccount.getPublicKey();
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey);
replacementAccounts.add(ourOnlineAccountData);
}
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey);
this.onlineAccounts.add(ourOnlineAccountData);
this.currentOnlineAccounts.clear();
addAccounts(replacementAccounts);
}
// Online accounts import queue
private void processOnlineAccountsImportQueue() {
if (this.onlineAccountsImportQueue.isEmpty())
// Nothing to do
return;
LOGGER.debug("Processing online accounts import queue (size: {})", this.onlineAccountsImportQueue.size());
Set<OnlineAccountData> onlineAccountsToAdd = new HashSet<>();
try (final Repository repository = RepositoryManager.getRepository()) {
for (OnlineAccountData onlineAccountData : this.onlineAccountsImportQueue) {
if (isStopping)
return;
boolean isValid = this.isValidCurrentAccount(repository, onlineAccountData);
if (isValid)
onlineAccountsToAdd.add(onlineAccountData);
// Remove from queue
onlineAccountsImportQueue.remove(onlineAccountData);
}
} catch (DataException e) {
LOGGER.error("Repository issue while verifying online accounts", e);
}
if (!onlineAccountsToAdd.isEmpty()) {
LOGGER.debug("Merging {} validated online accounts from import queue", onlineAccountsToAdd.size());
addAccounts(onlineAccountsToAdd);
}
}
private void performOnlineAccountsTasks() {
// Utilities
public static byte[] xorByteArrayInPlace(byte[] inplaceArray, byte[] otherArray) {
if (inplaceArray == null)
return Arrays.copyOf(otherArray, otherArray.length);
// Start from index 1 to enforce static leading byte
for (int i = 1; i < otherArray.length; i++)
inplaceArray[i] ^= otherArray[i];
return inplaceArray;
}
private static boolean isValidCurrentAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException {
final Long now = NTP.getTime();
if (now == null)
return false;
byte[] rewardSharePublicKey = onlineAccountData.getPublicKey();
long onlineAccountTimestamp = onlineAccountData.getTimestamp();
// Check timestamp is 'recent' here
if (Math.abs(onlineAccountTimestamp - now) > ONLINE_TIMESTAMP_MODULUS * 2) {
LOGGER.trace(() -> String.format("Rejecting online account %s with out of range timestamp %d", Base58.encode(rewardSharePublicKey), onlineAccountTimestamp));
return false;
}
// Verify signature
byte[] data = Longs.toByteArray(onlineAccountData.getTimestamp());
boolean isSignatureValid = onlineAccountTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp()
? Qortal25519Extras.verifyAggregated(rewardSharePublicKey, onlineAccountData.getSignature(), data)
: Crypto.verify(rewardSharePublicKey, onlineAccountData.getSignature(), data);
if (!isSignatureValid) {
LOGGER.trace(() -> String.format("Rejecting invalid online account %s", Base58.encode(rewardSharePublicKey)));
return false;
}
// Qortal: check online account is actually reward-share
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey);
if (rewardShareData == null) {
// Reward-share doesn't even exist - probably not a good sign
LOGGER.trace(() -> String.format("Rejecting unknown online reward-share public key %s", Base58.encode(rewardSharePublicKey)));
return false;
}
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
LOGGER.trace(() -> String.format("Rejecting online reward-share with non-minting account %s", mintingAccount.getAddress()));
return false;
}
return true;
}
/** Adds accounts, maybe rebuilds hashes, returns whether any new accounts were added / hashes rebuilt. */
private boolean addAccounts(Collection<OnlineAccountData> onlineAccountsToAdd) {
// For keeping track of which hashes to rebuild
Map<Long, Set<Byte>> hashesToRebuild = new HashMap<>();
for (OnlineAccountData onlineAccountData : onlineAccountsToAdd) {
boolean isNewEntry = this.addAccount(onlineAccountData);
if (isNewEntry)
hashesToRebuild.computeIfAbsent(onlineAccountData.getTimestamp(), k -> new HashSet<>()).add(onlineAccountData.getPublicKey()[0]);
}
if (hashesToRebuild.isEmpty())
return false;
for (var entry : hashesToRebuild.entrySet()) {
Long timestamp = entry.getKey();
LOGGER.debug(() -> String.format("Rehashing for timestamp %d and leading bytes %s",
timestamp,
entry.getValue().stream().sorted(Byte::compareUnsigned).map(leadingByte -> String.format("%02x", leadingByte)).collect(Collectors.joining(", "))
)
);
for (Byte leadingByte : entry.getValue()) {
byte[] pubkeyHash = currentOnlineAccounts.get(timestamp).stream()
.map(OnlineAccountData::getPublicKey)
.filter(publicKey -> leadingByte == publicKey[0])
.reduce(null, OnlineAccountsManager::xorByteArrayInPlace);
currentOnlineAccountsHashes.computeIfAbsent(timestamp, k -> new ConcurrentHashMap<>()).put(leadingByte, pubkeyHash);
LOGGER.trace(() -> String.format("Rebuilt hash %s for timestamp %d and leading byte %02x using %d public keys",
HashCode.fromBytes(pubkeyHash),
timestamp,
leadingByte,
currentOnlineAccounts.get(timestamp).stream()
.map(OnlineAccountData::getPublicKey)
.filter(publicKey -> leadingByte == publicKey[0])
.count()
));
}
}
LOGGER.info(String.format("we have online accounts for timestamps: %s", String.join(", ", this.currentOnlineAccounts.keySet().stream().map(l -> Long.toString(l)).collect(Collectors.joining(", ")))));
return true;
}
private boolean addAccount(OnlineAccountData onlineAccountData) {
byte[] rewardSharePublicKey = onlineAccountData.getPublicKey();
long onlineAccountTimestamp = onlineAccountData.getTimestamp();
Set<OnlineAccountData> onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountTimestamp, k -> ConcurrentHashMap.newKeySet());
boolean isNewEntry = onlineAccounts.add(onlineAccountData);
if (isNewEntry)
LOGGER.trace(() -> String.format("Added online account %s with timestamp %d", Base58.encode(rewardSharePublicKey), onlineAccountTimestamp));
else
LOGGER.trace(() -> String.format("Not updating existing online account %s with timestamp %d", Base58.encode(rewardSharePublicKey), onlineAccountTimestamp));
return isNewEntry;
}
/**
* Expire old entries.
*/
private void expireOldOnlineAccounts() {
final Long now = NTP.getTime();
if (now == null)
return;
// Expire old entries
final long cutoffThreshold = now - LAST_SEEN_EXPIRY_PERIOD;
synchronized (this.onlineAccounts) {
Iterator<OnlineAccountData> iterator = this.onlineAccounts.iterator();
while (iterator.hasNext()) {
OnlineAccountData onlineAccountData = iterator.next();
if (onlineAccountData.getTimestamp() < cutoffThreshold) {
iterator.remove();
LOGGER.trace(() -> {
PublicKeyAccount otherAccount = new PublicKeyAccount(null, onlineAccountData.getPublicKey());
return String.format("Removed expired online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp());
});
}
}
}
// Request data from other peers?
if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) {
List<OnlineAccountData> safeOnlineAccounts;
synchronized (this.onlineAccounts) {
safeOnlineAccounts = new ArrayList<>(this.onlineAccounts);
}
Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts);
Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts);
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
);
}
final long cutoffThreshold = now - MAX_CACHED_TIMESTAMP_SETS * ONLINE_TIMESTAMP_MODULUS;
this.currentOnlineAccounts.keySet().removeIf(timestamp -> timestamp < cutoffThreshold);
this.currentOnlineAccountsHashes.keySet().removeIf(timestamp -> timestamp < cutoffThreshold);
}
private void sendOurOnlineAccountsInfo() {
/**
* Request data from other peers. (Pre-V3)
*/
private void requestLegacyRemoteOnlineAccounts() {
final Long now = NTP.getTime();
if (now == null)
return;
// Don't bother if we're not up to date
if (!Controller.getInstance().isUpToDate())
return;
List<OnlineAccountData> mergedOnlineAccounts = Set.copyOf(this.currentOnlineAccounts.values()).stream().flatMap(Set::stream).collect(Collectors.toList());
Message messageV2 = new GetOnlineAccountsV2Message(mergedOnlineAccounts);
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() < ONLINE_ACCOUNTS_V3_PEER_VERSION
? messageV2
: null
);
}
/**
* Request data from other peers. V3+
*/
private void requestRemoteOnlineAccounts() {
final Long now = NTP.getTime();
if (now == null)
return;
// Don't bother if we're not up to date
if (!Controller.getInstance().isUpToDate())
return;
Message messageV3 = new GetOnlineAccountsV3Message(currentOnlineAccountsHashes);
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V3_PEER_VERSION
? messageV3
: null
);
}
/**
* Send online accounts that are minting on this node.
*/
private void sendOurOnlineAccountsInfo() {
// 'current' timestamp
final Long onlineAccountsTimestamp = getCurrentOnlineAccountTimestamp();
if (onlineAccountsTimestamp == null)
return;
Long now = NTP.getTime();
if (now == null) {
return;
}
@ -296,13 +381,12 @@ public class OnlineAccountsManager extends Thread {
try (final Repository repository = RepositoryManager.getRepository()) {
mintingAccounts = repository.getAccountRepository().getMintingAccounts();
// We have no accounts, but don't reset timestamp
// We have no accounts to send
if (mintingAccounts.isEmpty())
return;
// Only reward-share accounts allowed
// Only active reward-shares allowed
Iterator<MintingAccountData> iterator = mintingAccounts.iterator();
int i = 0;
while (iterator.hasNext()) {
MintingAccountData mintingAccountData = iterator.next();
@ -319,113 +403,138 @@ public class OnlineAccountsManager extends Thread {
iterator.remove();
continue;
}
if (++i > 1+1) {
iterator.remove();
continue;
}
}
} catch (DataException e) {
LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
return;
}
// 'current' timestamp
final long onlineAccountsTimestamp = toOnlineAccountTimestamp(now);
boolean hasInfoChanged = false;
boolean existingAccountsExist = false;
final boolean useAggregateCompatibleSignature = onlineAccountsTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp();
byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
MINTING_ACCOUNTS:
for (MintingAccountData mintingAccountData : mintingAccounts) {
PrivateKeyAccount mintingAccount = new PrivateKeyAccount(null, mintingAccountData.getPrivateKey());
byte[] privateKey = mintingAccountData.getPrivateKey();
byte[] publicKey = Crypto.toPublicKey(privateKey);
byte[] signature = mintingAccount.sign(timestampBytes);
byte[] publicKey = mintingAccount.getPublicKey();
byte[] signature = useAggregateCompatibleSignature
? Qortal25519Extras.signForAggregation(privateKey, timestampBytes)
: Crypto.sign(privateKey, timestampBytes);
// Our account is online
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey);
synchronized (this.onlineAccounts) {
Iterator<OnlineAccountData> iterator = this.onlineAccounts.iterator();
while (iterator.hasNext()) {
OnlineAccountData existingOnlineAccountData = iterator.next();
if (Arrays.equals(existingOnlineAccountData.getPublicKey(), ourOnlineAccountData.getPublicKey())) {
// If our online account is already present, with same timestamp, then move on to next mintingAccount
if (existingOnlineAccountData.getTimestamp() == onlineAccountsTimestamp) {
existingAccountsExist = true;
continue MINTING_ACCOUNTS;
}
// If our online account is already present, but with older timestamp, then remove it
iterator.remove();
break;
}
}
this.onlineAccounts.add(ourOnlineAccountData);
}
LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", mintingAccount.getAddress(), onlineAccountsTimestamp));
ourOnlineAccounts.add(ourOnlineAccountData);
hasInfoChanged = true;
}
// Keep track of whether we have online accounts circulating in the network, for systray status
this.hasOnlineAccounts = existingAccountsExist || !ourOnlineAccounts.isEmpty();
this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty();
boolean hasInfoChanged = addAccounts(ourOnlineAccounts);
if (!hasInfoChanged)
return;
Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts);
Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts);
Message messageV3 = new OnlineAccountsV2Message(ourOnlineAccounts); // TODO: V3 message
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V3_PEER_VERSION
? messageV3
: peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION
? messageV2
: messageV1
);
LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp));
LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp);
}
public static long toOnlineAccountTimestamp(long timestamp) {
return (timestamp / ONLINE_TIMESTAMP_MODULUS) * ONLINE_TIMESTAMP_MODULUS;
/**
* Returns whether online accounts manager has any online accounts with timestamp recent enough to be considered currently online.
*/
// BlockMinter: only calls this to check whether returned list is empty or not, to determine whether minting is even possible or not
public boolean hasOnlineAccounts() {
// 'current' timestamp
final Long onlineAccountsTimestamp = getCurrentOnlineAccountTimestamp();
if (onlineAccountsTimestamp == null)
return false;
return this.currentOnlineAccounts.containsKey(onlineAccountsTimestamp);
}
/** Returns list of online accounts with timestamp recent enough to be considered currently online. */
public boolean hasOurOnlineAccounts() {
return this.hasOurOnlineAccounts;
}
/**
* Returns list of online accounts matching given timestamp.
*/
// Block::mint() - only wants online accounts with (online) timestamp that matches block's (online) timestamp so they can be added to new block
public List<OnlineAccountData> getOnlineAccounts(long onlineTimestamp) {
LOGGER.info(String.format("caller's timestamp: %d, our timestamps: %s", onlineTimestamp, String.join(", ", this.currentOnlineAccounts.keySet().stream().map(l -> Long.toString(l)).collect(Collectors.joining(", ")))));
return new ArrayList<>(Set.copyOf(this.currentOnlineAccounts.getOrDefault(onlineTimestamp, Collections.emptySet())));
}
/**
* Returns list of online accounts with timestamp recent enough to be considered currently online.
*/
// API: calls this to return list of online accounts - probably expects ALL timestamps - but going to get 'current' from now on
public List<OnlineAccountData> getOnlineAccounts() {
final long onlineTimestamp = toOnlineAccountTimestamp(NTP.getTime());
// 'current' timestamp
final Long onlineAccountsTimestamp = getCurrentOnlineAccountTimestamp();
if (onlineAccountsTimestamp == null)
return Collections.emptyList();
synchronized (this.onlineAccounts) {
return this.onlineAccounts.stream().filter(account -> account.getTimestamp() == onlineTimestamp).collect(Collectors.toList());
}
return getOnlineAccounts(onlineAccountsTimestamp);
}
// Block processing
/** Returns cached, unmodifiable list of latest block's online accounts. */
public List<OnlineAccountData> getLatestBlocksOnlineAccounts() {
synchronized (this.latestBlocksOnlineAccounts) {
return this.latestBlocksOnlineAccounts.peekFirst();
}
/**
* Removes previously validated entries from block's online accounts.
* <p>
* Checks both 'current' and block caches.
* <p>
* Typically called by {@link Block#areOnlineAccountsValid()}
*/
public void removeKnown(Set<OnlineAccountData> blocksOnlineAccounts, Long timestamp) {
Set<OnlineAccountData> onlineAccounts = this.currentOnlineAccounts.get(timestamp);
// If not 'current' timestamp - try block cache instead
if (onlineAccounts == null)
onlineAccounts = this.latestBlocksOnlineAccounts.get(timestamp);
if (onlineAccounts != null)
blocksOnlineAccounts.removeAll(onlineAccounts);
}
/** Caches list of latest block's online accounts. Typically called by Block.process() */
public void pushLatestBlocksOnlineAccounts(List<OnlineAccountData> latestBlocksOnlineAccounts) {
synchronized (this.latestBlocksOnlineAccounts) {
if (this.latestBlocksOnlineAccounts.size() == MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS)
this.latestBlocksOnlineAccounts.pollLast();
this.latestBlocksOnlineAccounts.addFirst(latestBlocksOnlineAccounts == null
? Collections.emptyList()
: Collections.unmodifiableList(latestBlocksOnlineAccounts));
/**
* Adds block's online accounts to one of OnlineAccountManager's caches.
* <p>
* It is assumed that the online accounts have been verified.
* <p>
* Typically called by {@link Block#areOnlineAccountsValid()}
*/
public void addBlocksOnlineAccounts(Set<OnlineAccountData> blocksOnlineAccounts, Long timestamp) {
// We want to add to 'current' in preference if possible
if (this.currentOnlineAccounts.containsKey(timestamp)) {
addAccounts(blocksOnlineAccounts);
return;
}
}
/** Reverts list of latest block's online accounts. Typically called by Block.orphan() */
public void popLatestBlocksOnlineAccounts() {
synchronized (this.latestBlocksOnlineAccounts) {
this.latestBlocksOnlineAccounts.pollFirst();
// Add to block cache instead
this.latestBlocksOnlineAccounts.computeIfAbsent(timestamp, k -> ConcurrentHashMap.newKeySet())
.addAll(blocksOnlineAccounts);
// If block cache has grown too large then we need to trim.
if (this.latestBlocksOnlineAccounts.size() > MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS) {
// However, be careful to trim the opposite end to the entry we just added!
Long firstKey = this.latestBlocksOnlineAccounts.firstKey();
if (!firstKey.equals(timestamp))
this.latestBlocksOnlineAccounts.remove(firstKey);
else
this.latestBlocksOnlineAccounts.remove(this.latestBlocksOnlineAccounts.lastKey());
}
}
@ -438,45 +547,48 @@ public class OnlineAccountsManager extends Thread {
List<OnlineAccountData> excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts();
// Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts
List<OnlineAccountData> accountsToSend;
synchronized (this.onlineAccounts) {
accountsToSend = new ArrayList<>(this.onlineAccounts);
}
List<OnlineAccountData> accountsToSend = Set.copyOf(this.currentOnlineAccounts.values()).stream().flatMap(Set::stream).collect(Collectors.toList());
int prefilterSize = accountsToSend.size();
Iterator<OnlineAccountData> iterator = accountsToSend.iterator();
SEND_ITERATOR:
while (iterator.hasNext()) {
OnlineAccountData onlineAccountData = iterator.next();
for (int i = 0; i < excludeAccounts.size(); ++i) {
OnlineAccountData excludeAccountData = excludeAccounts.get(i);
for (OnlineAccountData excludeAccountData : excludeAccounts) {
if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) {
iterator.remove();
continue SEND_ITERATOR;
break;
}
}
}
if (accountsToSend.isEmpty())
return;
Message onlineAccountsMessage = new OnlineAccountsMessage(accountsToSend);
peer.sendMessage(onlineAccountsMessage);
LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer));
LOGGER.debug("Sent {} of our {} online accounts to {}", accountsToSend.size(), prefilterSize, peer);
}
public void onNetworkOnlineAccountsMessage(Peer peer, Message message) {
OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message;
List<OnlineAccountData> peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts();
LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer));
LOGGER.debug("Received {} online accounts from {}", peersOnlineAccounts.size(), peer);
try (final Repository repository = RepositoryManager.getRepository()) {
for (OnlineAccountData onlineAccountData : peersOnlineAccounts)
this.verifyAndAddAccount(repository, onlineAccountData);
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e);
int importCount = 0;
// Add any online accounts to the queue that aren't already present
for (OnlineAccountData onlineAccountData : peersOnlineAccounts) {
boolean isNewEntry = onlineAccountsImportQueue.add(onlineAccountData);
if (isNewEntry)
importCount++;
}
if (importCount > 0)
LOGGER.debug("Added {} online accounts to queue", importCount);
}
public void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) {
@ -485,58 +597,106 @@ public class OnlineAccountsManager extends Thread {
List<OnlineAccountData> excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts();
// Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts
List<OnlineAccountData> accountsToSend;
synchronized (this.onlineAccounts) {
accountsToSend = new ArrayList<>(this.onlineAccounts);
}
List<OnlineAccountData> accountsToSend = Set.copyOf(this.currentOnlineAccounts.values()).stream().flatMap(Set::stream).collect(Collectors.toList());
int prefilterSize = accountsToSend.size();
Iterator<OnlineAccountData> iterator = accountsToSend.iterator();
SEND_ITERATOR:
while (iterator.hasNext()) {
OnlineAccountData onlineAccountData = iterator.next();
for (int i = 0; i < excludeAccounts.size(); ++i) {
OnlineAccountData excludeAccountData = excludeAccounts.get(i);
for (OnlineAccountData excludeAccountData : excludeAccounts) {
if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) {
iterator.remove();
continue SEND_ITERATOR;
break;
}
}
}
if (accountsToSend.isEmpty())
return;
Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend);
peer.sendMessage(onlineAccountsMessage);
LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer));
LOGGER.debug("Sent {} of our {} online accounts to {}", accountsToSend.size(), prefilterSize, peer);
}
public void onNetworkOnlineAccountsV2Message(Peer peer, Message message) {
OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message;
List<OnlineAccountData> peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts();
LOGGER.debug(String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer));
LOGGER.debug("Received {} online accounts from {}", peersOnlineAccounts.size(), peer);
int importCount = 0;
// Add any online accounts to the queue that aren't already present
for (OnlineAccountData onlineAccountData : peersOnlineAccounts) {
boolean isNewEntry = onlineAccountsImportQueue.add(onlineAccountData);
// Do we already know about this online account data?
if (onlineAccounts.contains(onlineAccountData)) {
continue;
}
// Is it already in the import queue?
if (onlineAccountsImportQueue.contains(onlineAccountData)) {
continue;
}
onlineAccountsImportQueue.add(onlineAccountData);
importCount++;
if (isNewEntry)
importCount++;
}
LOGGER.debug(String.format("Added %d online accounts to queue", importCount));
if (importCount > 0)
LOGGER.debug("Added {} online accounts to queue", importCount);
}
public void onNetworkGetOnlineAccountsV3Message(Peer peer, Message message) {
GetOnlineAccountsV3Message getOnlineAccountsMessage = (GetOnlineAccountsV3Message) message;
Map<Long, Map<Byte, byte[]>> peersHashes = getOnlineAccountsMessage.getHashesByTimestampThenByte();
List<OnlineAccountData> outgoingOnlineAccounts = new ArrayList<>();
// Warning: no double-checking/fetching - we must be ConcurrentMap compatible!
// So no contains()-then-get() or multiple get()s on the same key/map.
// We also use getOrDefault() with emptySet() on currentOnlineAccounts in case corresponding timestamp entry isn't there.
for (var ourOuterMapEntry : currentOnlineAccountsHashes.entrySet()) {
Long timestamp = ourOuterMapEntry.getKey();
var ourInnerMap = ourOuterMapEntry.getValue();
var peersInnerMap = peersHashes.get(timestamp);
if (peersInnerMap == null) {
// Peer doesn't have this timestamp, so if it's valid (i.e. not too old) then we'd have to send all of ours
Set<OnlineAccountData> timestampsOnlineAccounts = this.currentOnlineAccounts.getOrDefault(timestamp, Collections.emptySet());
outgoingOnlineAccounts.addAll(timestampsOnlineAccounts);
LOGGER.debug(() -> String.format("Going to send all %d online accounts for timestamp %d", timestampsOnlineAccounts.size(), timestamp));
} else {
// Quick cache of which leading bytes to send so we only have to filter once
Set<Byte> outgoingLeadingBytes = new HashSet<>();
// We have entries for this timestamp so compare against peer's entries
for (var ourInnerMapEntry : ourInnerMap.entrySet()) {
Byte leadingByte = ourInnerMapEntry.getKey();
byte[] peersHash = peersInnerMap.get(leadingByte);
if (!Arrays.equals(ourInnerMapEntry.getValue(), peersHash)) {
// For this leading byte: hashes don't match or peer doesn't have entry
// Send all online accounts for this timestamp and leading byte
outgoingLeadingBytes.add(leadingByte);
}
}
int beforeAddSize = outgoingOnlineAccounts.size();
this.currentOnlineAccounts.getOrDefault(timestamp, Collections.emptySet()).stream()
.filter(account -> outgoingLeadingBytes.contains(account.getPublicKey()[0]))
.forEach(outgoingOnlineAccounts::add);
if (outgoingOnlineAccounts.size() > beforeAddSize)
LOGGER.debug(String.format("Going to send %d online accounts for timestamp %d and leading bytes %s",
outgoingOnlineAccounts.size() - beforeAddSize,
timestamp,
outgoingLeadingBytes.stream().sorted(Byte::compareUnsigned).map(leadingByte -> String.format("%02x", leadingByte)).collect(Collectors.joining(", "))
)
);
}
}
Message onlineAccountsMessage = new OnlineAccountsV2Message(outgoingOnlineAccounts); // TODO: V3 message
peer.sendMessage(onlineAccountsMessage);
LOGGER.debug("Sent {} online accounts to {}", outgoingOnlineAccounts.size(), peer);
}
}

2
src/main/java/org/qortal/controller/tradebot/TradeBot.java
View File

@ -292,7 +292,7 @@ public class TradeBot implements Listener {
}
public static byte[] deriveTradeNativePublicKey(byte[] privateKey) {
return PrivateKeyAccount.toPublicKey(privateKey);
return Crypto.toPublicKey(privateKey);
}
public static byte[] deriveTradeForeignPublicKey(byte[] privateKey) {

99
src/main/java/org/qortal/crypto/BouncyCastle25519.java
View File

@ -1,99 +0,0 @@
package org.qortal.crypto;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
import org.bouncycastle.crypto.Digest;
import org.bouncycastle.math.ec.rfc7748.X25519;
import org.bouncycastle.math.ec.rfc7748.X25519Field;
import org.bouncycastle.math.ec.rfc8032.Ed25519;
/** Additions to BouncyCastle providing Ed25519 to X25519 key conversion. */
public class BouncyCastle25519 {
private static final Class<?> pointAffineClass;
private static final Constructor<?> pointAffineCtor;
private static final Method decodePointVarMethod;
private static final Field yField;
static {
try {
Class<?> ed25519Class = Ed25519.class;
pointAffineClass = Arrays.stream(ed25519Class.getDeclaredClasses()).filter(clazz -> clazz.getSimpleName().equals("PointAffine")).findFirst().get();
if (pointAffineClass == null)
throw new ClassNotFoundException("Can't locate PointExt inner class inside Ed25519");
decodePointVarMethod = ed25519Class.getDeclaredMethod("decodePointVar", byte[].class, int.class, boolean.class, pointAffineClass);
decodePointVarMethod.setAccessible(true);
pointAffineCtor = pointAffineClass.getDeclaredConstructors()[0];
pointAffineCtor.setAccessible(true);
yField = pointAffineClass.getDeclaredField("y");
yField.setAccessible(true);
} catch (NoSuchMethodException | SecurityException | IllegalArgumentException | NoSuchFieldException | ClassNotFoundException e) {
throw new RuntimeException("Can't initialize BouncyCastle25519 shim", e);
}
}
private static int[] obtainYFromPublicKey(byte[] ed25519PublicKey) {
try {
Object pA = pointAffineCtor.newInstance();
Boolean result = (Boolean) decodePointVarMethod.invoke(null, ed25519PublicKey, 0, true, pA);
if (result == null || !result)
return null;
return (int[]) yField.get(pA);
} catch (SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
throw new RuntimeException("Can't reflect into BouncyCastle", e);
}
}
public static byte[] toX25519PublicKey(byte[] ed25519PublicKey) {
int[] one = new int[X25519Field.SIZE];
X25519Field.one(one);
int[] y = obtainYFromPublicKey(ed25519PublicKey);
int[] oneMinusY = new int[X25519Field.SIZE];
X25519Field.sub(one, y, oneMinusY);
int[] onePlusY = new int[X25519Field.SIZE];
X25519Field.add(one, y, onePlusY);
int[] oneMinusYInverted = new int[X25519Field.SIZE];
X25519Field.inv(oneMinusY, oneMinusYInverted);
int[] u = new int[X25519Field.SIZE];
X25519Field.mul(onePlusY, oneMinusYInverted, u);
X25519Field.normalize(u);
byte[] x25519PublicKey = new byte[X25519.SCALAR_SIZE];
X25519Field.encode(u, x25519PublicKey, 0);
return x25519PublicKey;
}
public static byte[] toX25519PrivateKey(byte[] ed25519PrivateKey) {
Digest d = Ed25519.createPrehash();
byte[] h = new byte[d.getDigestSize()];
d.update(ed25519PrivateKey, 0, ed25519PrivateKey.length);
d.doFinal(h, 0);
byte[] s = new byte[X25519.SCALAR_SIZE];
System.arraycopy(h, 0, s, 0, X25519.SCALAR_SIZE);
s[0] &= 0xF8;
s[X25519.SCALAR_SIZE - 1] &= 0x7F;
s[X25519.SCALAR_SIZE - 1] |= 0x40;
return s;
}
}

1427
src/main/java/org/qortal/crypto/BouncyCastleEd25519.java Normal file
View File

File diff suppressed because it is too large Load Diff

19
src/main/java/org/qortal/crypto/Crypto.java
View File

@ -253,6 +253,10 @@ public abstract class Crypto {
return false;
}
public static byte[] toPublicKey(byte[] privateKey) {
return new Ed25519PrivateKeyParameters(privateKey, 0).generatePublicKey().getEncoded();
}
public static boolean verify(byte[] publicKey, byte[] signature, byte[] message) {
try {
return Ed25519.verify(signature, 0, publicKey, 0, message, 0, message.length);
@ -264,16 +268,24 @@ public abstract class Crypto {
public static byte[] sign(Ed25519PrivateKeyParameters edPrivateKeyParams, byte[] message) {
byte[] signature = new byte[SIGNATURE_LENGTH];
edPrivateKeyParams.sign(Ed25519.Algorithm.Ed25519, edPrivateKeyParams.generatePublicKey(), null, message, 0, message.length, signature, 0);
edPrivateKeyParams.sign(Ed25519.Algorithm.Ed25519,null, message, 0, message.length, signature, 0);
return signature;
}
public static byte[] sign(byte[] privateKey, byte[] message) {
byte[] signature = new byte[SIGNATURE_LENGTH];
new Ed25519PrivateKeyParameters(privateKey, 0).sign(Ed25519.Algorithm.Ed25519,null, message, 0, message.length, signature, 0);
return signature;
}
public static byte[] getSharedSecret(byte[] privateKey, byte[] publicKey) {
byte[] x25519PrivateKey = BouncyCastle25519.toX25519PrivateKey(privateKey);
byte[] x25519PrivateKey = Qortal25519Extras.toX25519PrivateKey(privateKey);
X25519PrivateKeyParameters xPrivateKeyParams = new X25519PrivateKeyParameters(x25519PrivateKey, 0);
byte[] x25519PublicKey = BouncyCastle25519.toX25519PublicKey(publicKey);
byte[] x25519PublicKey = Qortal25519Extras.toX25519PublicKey(publicKey);
X25519PublicKeyParameters xPublicKeyParams = new X25519PublicKeyParameters(x25519PublicKey, 0);
byte[] sharedSecret = new byte[SHARED_SECRET_LENGTH];
@ -281,5 +293,4 @@ public abstract class Crypto {
return sharedSecret;
}
}

234
src/main/java/org/qortal/crypto/Qortal25519Extras.java Normal file
View File

@ -0,0 +1,234 @@
package org.qortal.crypto;
import org.bouncycastle.crypto.Digest;
import org.bouncycastle.crypto.digests.SHA512Digest;
import org.bouncycastle.math.ec.rfc7748.X25519;
import org.bouncycastle.math.ec.rfc7748.X25519Field;
import org.bouncycastle.math.ec.rfc8032.Ed25519;
import org.bouncycastle.math.raw.Nat256;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Collection;
/**
* Additions to BouncyCastle providing:
* <p></p>
* <ul>
* <li>Ed25519 to X25519 key conversion</li>
* <li>Aggregate public keys</li>
* <li>Aggregate signatures</li>
* </ul>
*/
public abstract class Qortal25519Extras extends BouncyCastleEd25519 {
private static final SecureRandom SECURE_RANDOM = new SecureRandom();
public static byte[] toX25519PublicKey(byte[] ed25519PublicKey) {
int[] one = new int[X25519Field.SIZE];
X25519Field.one(one);
PointAffine pA = new PointAffine();
if (!decodePointVar(ed25519PublicKey, 0, true, pA))
return null;
int[] y = pA.y;
int[] oneMinusY = new int[X25519Field.SIZE];
X25519Field.sub(one, y, oneMinusY);
int[] onePlusY = new int[X25519Field.SIZE];
X25519Field.add(one, y, onePlusY);
int[] oneMinusYInverted = new int[X25519Field.SIZE];
X25519Field.inv(oneMinusY, oneMinusYInverted);
int[] u = new int[X25519Field.SIZE];
X25519Field.mul(onePlusY, oneMinusYInverted, u);
X25519Field.normalize(u);
byte[] x25519PublicKey = new byte[X25519.SCALAR_SIZE];
X25519Field.encode(u, x25519PublicKey, 0);
return x25519PublicKey;
}
public static byte[] toX25519PrivateKey(byte[] ed25519PrivateKey) {
Digest d = Ed25519.createPrehash();
byte[] h = new byte[d.getDigestSize()];
d.update(ed25519PrivateKey, 0, ed25519PrivateKey.length);
d.doFinal(h, 0);
byte[] s = new byte[X25519.SCALAR_SIZE];
System.arraycopy(h, 0, s, 0, X25519.SCALAR_SIZE);
s[0] &= 0xF8;
s[X25519.SCALAR_SIZE - 1] &= 0x7F;
s[X25519.SCALAR_SIZE - 1] |= 0x40;
return s;
}
// Mostly for test support
public static PointAccum newPointAccum() {
return new PointAccum();
}
public static byte[] aggregatePublicKeys(Collection<byte[]> publicKeys) {
PointAccum rAccum = null;
for (byte[] publicKey : publicKeys) {
PointAffine pA = new PointAffine();
if (!decodePointVar(publicKey, 0, false, pA))
// Failed to decode
return null;
if (rAccum == null) {
rAccum = new PointAccum();
pointCopy(pA, rAccum);
} else {
pointAdd(pointCopy(pA), rAccum);
}
}
byte[] publicKey = new byte[SCALAR_BYTES];
if (0 == encodePoint(rAccum, publicKey, 0))
// Failed to encode
return null;
return publicKey;
}
public static byte[] aggregateSignatures(Collection<byte[]> signatures) {
// Signatures are (R, s)
// R is a point
// s is a scalar
PointAccum rAccum = null;
int[] sAccum = new int[SCALAR_INTS];
byte[] rEncoded = new byte[POINT_BYTES];
int[] sPart = new int[SCALAR_INTS];
for (byte[] signature : signatures) {
System.arraycopy(signature,0, rEncoded, 0, rEncoded.length);
PointAffine pA = new PointAffine();
if (!decodePointVar(rEncoded, 0, false, pA))
// Failed to decode
return null;
if (rAccum == null) {
rAccum = new PointAccum();
pointCopy(pA, rAccum);
decode32(signature, rEncoded.length, sAccum, 0, SCALAR_INTS);
} else {
pointAdd(pointCopy(pA), rAccum);
decode32(signature, rEncoded.length, sPart, 0, SCALAR_INTS);
Nat256.addTo(sPart, sAccum);
// "mod L" on sAccum
if (Nat256.gte(sAccum, L))
Nat256.subFrom(L, sAccum);
}
}
byte[] signature = new byte[SIGNATURE_SIZE];
if (0 == encodePoint(rAccum, signature, 0))
// Failed to encode
return null;
for (int i = 0; i < sAccum.length; ++i) {
encode32(sAccum[i], signature, POINT_BYTES + i * 4);
}
return signature;
}
public static byte[] signForAggregation(byte[] privateKey, byte[] message) {
// Very similar to BouncyCastle's implementation except we use secure random nonce and different hash
Digest d = new SHA512Digest();
byte[] h = new byte[d.getDigestSize()];
d.reset();
d.update(privateKey, 0, privateKey.length);
d.doFinal(h, 0);
byte[] sH = new byte[SCALAR_BYTES];
pruneScalar(h, 0, sH);
byte[] publicKey = new byte[SCALAR_BYTES];
scalarMultBaseEncoded(sH, publicKey, 0);
byte[] rSeed = new byte[d.getDigestSize()];
SECURE_RANDOM.nextBytes(rSeed);
byte[] r = new byte[SCALAR_BYTES];
pruneScalar(rSeed, 0, r);
byte[] R = new byte[POINT_BYTES];
scalarMultBaseEncoded(r, R, 0);
d.reset();
d.update(message, 0, message.length);
d.doFinal(h, 0);
byte[] k = reduceScalar(h);
byte[] s = calculateS(r, k, sH);
byte[] signature = new byte[SIGNATURE_SIZE];
System.arraycopy(R, 0, signature, 0, POINT_BYTES);
System.arraycopy(s, 0, signature, POINT_BYTES, SCALAR_BYTES);
return signature;
}
public static boolean verifyAggregated(byte[] publicKey, byte[] signature, byte[] message) {
byte[] R = Arrays.copyOfRange(signature, 0, POINT_BYTES);
byte[] s = Arrays.copyOfRange(signature, POINT_BYTES, POINT_BYTES + SCALAR_BYTES);
if (!checkPointVar(R))
// R out of bounds
return false;
if (!checkScalarVar(s))
// s out of bounds
return false;
byte[] S = new byte[POINT_BYTES];
scalarMultBaseEncoded(s, S, 0);
PointAffine pA = new PointAffine();
if (!decodePointVar(publicKey, 0, true, pA))
// Failed to decode
return false;
Digest d = new SHA512Digest();
byte[] h = new byte[d.getDigestSize()];
d.update(message, 0, message.length);
d.doFinal(h, 0);
byte[] k = reduceScalar(h);
int[] nS = new int[SCALAR_INTS];
decodeScalar(s, 0, nS);
int[] nA = new int[SCALAR_INTS];
decodeScalar(k, 0, nA);
/*PointAccum*/
PointAccum pR = new PointAccum();
scalarMultStrausVar(nS, nA, pA, pR);
byte[] check = new byte[POINT_BYTES];
if (0 == encodePoint(pR, check, 0))
// Failed to encode
return false;
return Arrays.equals(check, R);
}
}

19
src/main/java/org/qortal/data/network/OnlineAccountData.java
View File

@ -5,6 +5,7 @@ import java.util.Arrays;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlTransient;
import org.qortal.account.PublicKeyAccount;
@ -16,6 +17,9 @@ public class OnlineAccountData {
protected byte[] signature;
protected byte[] publicKey;
@XmlTransient
private int hash;
// Constructors
// necessary for JAXB serialization
@ -62,20 +66,23 @@ public class OnlineAccountData {
if (otherOnlineAccountData.timestamp != this.timestamp)
return false;
// Signature more likely to be unique than public key
if (!Arrays.equals(otherOnlineAccountData.signature, this.signature))
return false;
if (!Arrays.equals(otherOnlineAccountData.publicKey, this.publicKey))
return false;
// We don't compare signature because it's not our remit to verify and newer aggregate signatures use random nonces
return true;
}
@Override
public int hashCode() {
// Pretty lazy implementation
return (int) this.timestamp;
int h = this.hash;
if (h == 0) {
this.hash = h = Long.hashCode(this.timestamp)
^ Arrays.hashCode(this.publicKey);
// We don't use signature because newer aggregate signatures use random nonces
}
return h;
}
}

2
src/main/java/org/qortal/network/Network.java
View File

@ -469,6 +469,8 @@ public class Network {
class NetworkProcessor extends ExecuteProduceConsume {
private final Logger LOGGER = LogManager.getLogger(NetworkProcessor.class);
private final AtomicLong nextConnectTaskTimestamp = new AtomicLong(0L); // ms - try first connect once NTP syncs
private final AtomicLong nextBroadcastTimestamp = new AtomicLong(0L); // ms - try first broadcast once NTP syncs

112
src/main/java/org/qortal/network/message/GetOnlineAccountsV3Message.java Normal file
View File

@ -0,0 +1,112 @@
package org.qortal.network.message;
import com.google.common.primitives.Longs;
import org.qortal.transform.Transformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
/**
* For requesting online accounts info from remote peer, given our list of online accounts.
* <p></p>
* Different format to V1 and V2:<br>
* <ul>
* <li>V1 is: number of entries, then timestamp + pubkey for each entry</li>
* <li>V2 is: groups of: number of entries, timestamp, then pubkey for each entry</li>
* <li>V3 is: groups of: timestamp, number of entries (one per leading byte), then hash(pubkeys) for each entry</li>
* </ul>
* <p></p>
* End
*/
public class GetOnlineAccountsV3Message extends Message {
private static final Map<Long, Map<Byte, byte[]>> EMPTY_ONLINE_ACCOUNTS = Collections.emptyMap();
private Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte;
public GetOnlineAccountsV3Message(Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte) {
super(MessageType.GET_ONLINE_ACCOUNTS_V3);
// If we don't have ANY online accounts then it's an easier construction...
if (hashesByTimestampThenByte.isEmpty()) {
this.dataBytes = EMPTY_DATA_BYTES;
return;
}
// We should know exactly how many bytes to allocate now
int byteSize = hashesByTimestampThenByte.size() * (Transformer.TIMESTAMP_LENGTH + Transformer.BYTE_LENGTH);
byteSize += hashesByTimestampThenByte.values()
.stream()
.mapToInt(map -> map.size() * Transformer.PUBLIC_KEY_LENGTH)
.sum();
ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
// Warning: no double-checking/fetching! We must be ConcurrentMap compatible.
// So no contains() then get() or multiple get()s on the same key/map.
try {
for (var outerMapEntry : hashesByTimestampThenByte.entrySet()) {
bytes.write(Longs.toByteArray(outerMapEntry.getKey()));
var innerMap = outerMapEntry.getValue();
// Number of entries: 1 - 256, where 256 is represented by 0
bytes.write(innerMap.size() & 0xFF);
for (byte[] hashBytes : innerMap.values()) {
bytes.write(hashBytes);
}
}
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private GetOnlineAccountsV3Message(int id, Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte) {
super(id, MessageType.GET_ONLINE_ACCOUNTS_V3);
this.hashesByTimestampThenByte = hashesByTimestampThenByte;
}
public Map<Long, Map<Byte, byte[]>> getHashesByTimestampThenByte() {
return this.hashesByTimestampThenByte;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) {
// 'empty' case
if (!bytes.hasRemaining()) {
return new GetOnlineAccountsV3Message(id, EMPTY_ONLINE_ACCOUNTS);
}
Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte = new HashMap<>();
while (bytes.hasRemaining()) {
long timestamp = bytes.getLong();
int hashCount = bytes.get();
if (hashCount <= 0)
// 256 is represented by 0.
// Also converts negative signed value (e.g. -1) to proper positive unsigned value (255)
hashCount += 256;
Map<Byte, byte[]> hashesByByte = new HashMap<>();
for (int i = 0; i < hashCount; ++i) {
byte[] publicKeyHash = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKeyHash);
hashesByByte.put(publicKeyHash[0], publicKeyHash);
}
hashesByTimestampThenByte.put(timestamp, hashesByByte);
}
return new GetOnlineAccountsV3Message(id, hashesByTimestampThenByte);
}
}

3
src/main/java/org/qortal/network/message/Message.java
View File

@ -46,6 +46,7 @@ public abstract class Message {
private static final int MAX_DATA_SIZE = 10 * 1024 * 1024; // 10MB
protected static final byte[] EMPTY_DATA_BYTES = new byte[0];
private static final ByteBuffer EMPTY_READ_ONLY_BYTE_BUFFER = ByteBuffer.wrap(EMPTY_DATA_BYTES).asReadOnlyBuffer();
protected int id;
protected final MessageType type;
@ -126,7 +127,7 @@ public abstract class Message {
if (dataSize > 0 && dataSize + CHECKSUM_LENGTH > readOnlyBuffer.remaining())
return null;
ByteBuffer dataSlice = null;
ByteBuffer dataSlice = EMPTY_READ_ONLY_BYTE_BUFFER;
if (dataSize > 0) {
byte[] expectedChecksum = new byte[CHECKSUM_LENGTH];
readOnlyBuffer.get(expectedChecksum);

2
src/main/java/org/qortal/network/message/MessageType.java
View File

@ -46,6 +46,8 @@ public enum MessageType {
GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer),
ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer),
GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer),
// ONLINE_ACCOUNTS_V3(84, OnlineAccountsV3Message::fromByteBuffer),
GET_ONLINE_ACCOUNTS_V3(85, GetOnlineAccountsV3Message::fromByteBuffer),
ARBITRARY_DATA(90, ArbitraryDataMessage::fromByteBuffer),
GET_ARBITRARY_DATA(91, GetArbitraryDataMessage::fromByteBuffer),

3
src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java
View File

@ -23,7 +23,6 @@ import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.crypto.Crypto;
import org.qortal.globalization.Translator;
import org.qortal.gui.SysTray;
@ -1003,7 +1002,7 @@ public class HSQLDBRepository implements Repository {
if (privateKey == null)
return null;
return PrivateKeyAccount.toPublicKey(privateKey);
return Crypto.toPublicKey(privateKey);
}
public static String ed25519PublicKeyToAddress(byte[] publicKey) {

3
src/main/resources/blockchain.json
View File

@ -60,7 +60,8 @@
"calcChainWeightTimestamp": 1620579600000,
"transactionV5Timestamp": 1642176000000,
"transactionV6Timestamp": 9999999999999,
"disableReferenceTimestamp": 1655222400000
"disableReferenceTimestamp": 1655222400000,
"aggregateSignatureTimestamp": 9999999999999
},
"genesisInfo": {
"version": 4,

14
src/test/java/org/qortal/test/CryptoTests.java
View File

@ -4,7 +4,7 @@ import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.block.BlockChain;
import org.qortal.crypto.AES;
import org.qortal.crypto.BouncyCastle25519;
import org.qortal.crypto.Qortal25519Extras;
import org.qortal.crypto.Crypto;
import org.qortal.test.common.Common;
import org.qortal.utils.Base58;
@ -123,14 +123,14 @@ public class CryptoTests extends Common {
random.nextBytes(ed25519PrivateKey);
PrivateKeyAccount account = new PrivateKeyAccount(null, ed25519PrivateKey);
byte[] x25519PrivateKey = BouncyCastle25519.toX25519PrivateKey(account.getPrivateKey());
byte[] x25519PrivateKey = Qortal25519Extras.toX25519PrivateKey(account.getPrivateKey());
X25519PrivateKeyParameters x25519PrivateKeyParams = new X25519PrivateKeyParameters(x25519PrivateKey, 0);
// Derive X25519 public key from X25519 private key
byte[] x25519PublicKeyFromPrivate = x25519PrivateKeyParams.generatePublicKey().getEncoded();
// Derive X25519 public key from Ed25519 public key
byte[] x25519PublicKeyFromEd25519 = BouncyCastle25519.toX25519PublicKey(account.getPublicKey());
byte[] x25519PublicKeyFromEd25519 = Qortal25519Extras.toX25519PublicKey(account.getPublicKey());
assertEquals(String.format("Public keys do not match, from private key %s", Base58.encode(ed25519PrivateKey)), Base58.encode(x25519PublicKeyFromPrivate), Base58.encode(x25519PublicKeyFromEd25519));
}
@ -162,10 +162,10 @@ public class CryptoTests extends Common {
}
private static byte[] calcBCSharedSecret(byte[] ed25519PrivateKey, byte[] ed25519PublicKey) {
byte[] x25519PrivateKey = BouncyCastle25519.toX25519PrivateKey(ed25519PrivateKey);
byte[] x25519PrivateKey = Qortal25519Extras.toX25519PrivateKey(ed25519PrivateKey);
X25519PrivateKeyParameters privateKeyParams = new X25519PrivateKeyParameters(x25519PrivateKey, 0);
byte[] x25519PublicKey = BouncyCastle25519.toX25519PublicKey(ed25519PublicKey);
byte[] x25519PublicKey = Qortal25519Extras.toX25519PublicKey(ed25519PublicKey);
X25519PublicKeyParameters publicKeyParams = new X25519PublicKeyParameters(x25519PublicKey, 0);
byte[] sharedSecret = new byte[32];
@ -186,10 +186,10 @@ public class CryptoTests extends Common {
final String expectedTheirX25519PublicKey = "ANjnZLRSzW9B1aVamiYGKP3XtBooU9tGGDjUiibUfzp2";
final String expectedSharedSecret = "DTMZYG96x8XZuGzDvHFByVLsXedimqtjiXHhXPVe58Ap";
byte[] ourX25519PrivateKey = BouncyCastle25519.toX25519PrivateKey(ourPrivateKey);
byte[] ourX25519PrivateKey = Qortal25519Extras.toX25519PrivateKey(ourPrivateKey);
assertEquals("X25519 private key incorrect", expectedOurX25519PrivateKey, Base58.encode(ourX25519PrivateKey));
byte[] theirX25519PublicKey = BouncyCastle25519.toX25519PublicKey(theirPublicKey);
byte[] theirX25519PublicKey = Qortal25519Extras.toX25519PublicKey(theirPublicKey);
assertEquals("X25519 public key incorrect", expectedTheirX25519PublicKey, Base58.encode(theirX25519PublicKey));
byte[] sharedSecret = calcBCSharedSecret(ourPrivateKey, theirPublicKey);

190
src/test/java/org/qortal/test/SchnorrTests.java Normal file
View File

@ -0,0 +1,190 @@
package org.qortal.test;
import com.google.common.hash.HashCode;
import com.google.common.primitives.Bytes;
import com.google.common.primitives.Longs;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.junit.Test;
import org.qortal.crypto.Qortal25519Extras;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.transform.Transformer;
import java.math.BigInteger;
import java.security.SecureRandom;
import java.security.Security;
import java.util.*;
import java.util.stream.Collectors;
import static org.junit.Assert.*;
public class SchnorrTests extends Qortal25519Extras {
static {
// This must go before any calls to LogManager/Logger
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
Security.insertProviderAt(new BouncyCastleProvider(), 0);
Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
}
private static final SecureRandom SECURE_RANDOM = new SecureRandom();
@Test
public void testConversion() {
// Scalar form
byte[] scalarA = HashCode.fromString("0100000000000000000000000000000000000000000000000000000000000000".toLowerCase()).asBytes();
System.out.printf("a: %s%n", HashCode.fromBytes(scalarA));
byte[] pointA = HashCode.fromString("5866666666666666666666666666666666666666666666666666666666666666".toLowerCase()).asBytes();
BigInteger expectedY = new BigInteger("46316835694926478169428394003475163141307993866256225615783033603165251855960");
PointAccum pointAccum = Qortal25519Extras.newPointAccum();
scalarMultBase(scalarA, pointAccum);
byte[] encoded = new byte[POINT_BYTES];
if (0 == encodePoint(pointAccum, encoded, 0))
fail("Point encoding failed");
System.out.printf("aG: %s%n", HashCode.fromBytes(encoded));
assertArrayEquals(pointA, encoded);
byte[] yBytes = new byte[POINT_BYTES];
System.arraycopy(encoded,0, yBytes, 0, encoded.length);
Bytes.reverse(yBytes);
System.out.printf("yBytes: %s%n", HashCode.fromBytes(yBytes));
BigInteger yBI = new BigInteger(yBytes);
System.out.printf("aG y: %s%n", yBI);
assertEquals(expectedY, yBI);
}
@Test
public void testAddition() {
/*
* 1G: b'5866666666666666666666666666666666666666666666666666666666666666'
* 2G: b'c9a3f86aae465f0e56513864510f3997561fa2c9e85ea21dc2292309f3cd6022'
* 3G: b'd4b4f5784868c3020403246717ec169ff79e26608ea126a1ab69ee77d1b16712'
*/
// Scalar form
byte[] s1 = HashCode.fromString("0100000000000000000000000000000000000000000000000000000000000000".toLowerCase()).asBytes();
byte[] s2 = HashCode.fromString("0200000000000000000000000000000000000000000000000000000000000000".toLowerCase()).asBytes();
// Point form
byte[] g1 = HashCode.fromString("5866666666666666666666666666666666666666666666666666666666666666".toLowerCase()).asBytes();
byte[] g2 = HashCode.fromString("c9a3f86aae465f0e56513864510f3997561fa2c9e85ea21dc2292309f3cd6022".toLowerCase()).asBytes();
byte[] g3 = HashCode.fromString("d4b4f5784868c3020403246717ec169ff79e26608ea126a1ab69ee77d1b16712".toLowerCase()).asBytes();
PointAccum p1 = Qortal25519Extras.newPointAccum();
scalarMultBase(s1, p1);
PointAccum p2 = Qortal25519Extras.newPointAccum();
scalarMultBase(s2, p2);
pointAdd(pointCopy(p1), p2);
byte[] encoded = new byte[POINT_BYTES];
if (0 == encodePoint(p2, encoded, 0))
fail("Point encoding failed");
System.out.printf("sum: %s%n", HashCode.fromBytes(encoded));
assertArrayEquals(g3, encoded);
}
@Test
public void testSimpleSign() {
byte[] privateKey = HashCode.fromString("0100000000000000000000000000000000000000000000000000000000000000".toLowerCase()).asBytes();
byte[] message = HashCode.fromString("01234567".toLowerCase()).asBytes();
byte[] signature = signForAggregation(privateKey, message);
System.out.printf("signature: %s%n", HashCode.fromBytes(signature));
}
@Test
public void testSimpleVerify() {
byte[] privateKey = HashCode.fromString("0100000000000000000000000000000000000000000000000000000000000000".toLowerCase()).asBytes();
byte[] message = HashCode.fromString("01234567".toLowerCase()).asBytes();
byte[] signature = HashCode.fromString("13e58e88f3df9e06637d2d5bbb814c028e3ba135494530b9d3b120bdb31168d62c70a37ae9cfba816fe6038ee1ce2fb521b95c4a91c7ff0bb1dd2e67733f2b0d".toLowerCase()).asBytes();
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
Qortal25519Extras.generatePublicKey(privateKey, 0, publicKey, 0);
assertTrue(verifyAggregated(publicKey, signature, message));
}
@Test
public void testSimpleSignAndVerify() {
byte[] privateKey = HashCode.fromString("0100000000000000000000000000000000000000000000000000000000000000".toLowerCase()).asBytes();
byte[] message = HashCode.fromString("01234567".toLowerCase()).asBytes();
byte[] signature = signForAggregation(privateKey, message);
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
Qortal25519Extras.generatePublicKey(privateKey, 0, publicKey, 0);
assertTrue(verifyAggregated(publicKey, signature, message));
}
@Test
public void testSimpleAggregate() {
List<OnlineAccountData> onlineAccounts = generateOnlineAccounts(1);
byte[] aggregatePublicKey = aggregatePublicKeys(onlineAccounts.stream().map(OnlineAccountData::getPublicKey).collect(Collectors.toUnmodifiableList()));
System.out.printf("Aggregate public key: %s%n", HashCode.fromBytes(aggregatePublicKey));
byte[] aggregateSignature = aggregateSignatures(onlineAccounts.stream().map(OnlineAccountData::getSignature).collect(Collectors.toUnmodifiableList()));
System.out.printf("Aggregate signature: %s%n", HashCode.fromBytes(aggregateSignature));
OnlineAccountData onlineAccount = onlineAccounts.get(0);
assertArrayEquals(String.format("expected: %s, actual: %s", HashCode.fromBytes(onlineAccount.getPublicKey()), HashCode.fromBytes(aggregatePublicKey)), onlineAccount.getPublicKey(), aggregatePublicKey);
assertArrayEquals(String.format("expected: %s, actual: %s", HashCode.fromBytes(onlineAccount.getSignature()), HashCode.fromBytes(aggregateSignature)), onlineAccount.getSignature(), aggregateSignature);
// This is the crucial test:
long timestamp = onlineAccount.getTimestamp();
byte[] timestampBytes = Longs.toByteArray(timestamp);
assertTrue(verifyAggregated(aggregatePublicKey, aggregateSignature, timestampBytes));
}
@Test
public void testMultipleAggregate() {
List<OnlineAccountData> onlineAccounts = generateOnlineAccounts(5000);
byte[] aggregatePublicKey = aggregatePublicKeys(onlineAccounts.stream().map(OnlineAccountData::getPublicKey).collect(Collectors.toUnmodifiableList()));
System.out.printf("Aggregate public key: %s%n", HashCode.fromBytes(aggregatePublicKey));
byte[] aggregateSignature = aggregateSignatures(onlineAccounts.stream().map(OnlineAccountData::getSignature).collect(Collectors.toUnmodifiableList()));
System.out.printf("Aggregate signature: %s%n", HashCode.fromBytes(aggregateSignature));
OnlineAccountData onlineAccount = onlineAccounts.get(0);
// This is the crucial test:
long timestamp = onlineAccount.getTimestamp();
byte[] timestampBytes = Longs.toByteArray(timestamp);
assertTrue(verifyAggregated(aggregatePublicKey, aggregateSignature, timestampBytes));
}
private List<OnlineAccountData> generateOnlineAccounts(int numAccounts) {
List<OnlineAccountData> onlineAccounts = new ArrayList<>();
long timestamp = System.currentTimeMillis();
byte[] timestampBytes = Longs.toByteArray(timestamp);
for (int a = 0; a < numAccounts; ++a) {
byte[] privateKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
SECURE_RANDOM.nextBytes(privateKey);
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
Qortal25519Extras.generatePublicKey(privateKey, 0, publicKey, 0);
byte[] signature = signForAggregation(privateKey, timestampBytes);
onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey));
}
return onlineAccounts;
}
}

3
src/test/java/org/qortal/test/apps/RewardShareKeys.java
View File

@ -6,6 +6,7 @@ import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.account.PublicKeyAccount;
import org.qortal.crypto.Crypto;
import org.qortal.utils.Base58;
public class RewardShareKeys {
@ -28,7 +29,7 @@ public class RewardShareKeys {
PublicKeyAccount recipientAccount = new PublicKeyAccount(null, args.length > 1 ? Base58.decode(args[1]) : minterAccount.getPublicKey());
byte[] rewardSharePrivateKey = minterAccount.getRewardSharePrivateKey(recipientAccount.getPublicKey());
byte[] rewardSharePublicKey = PrivateKeyAccount.toPublicKey(rewardSharePrivateKey);
byte[] rewardSharePublicKey = Crypto.toPublicKey(rewardSharePrivateKey);
System.out.println(String.format("Minter account: %s", minterAccount.getAddress()));
System.out.println(String.format("Minter's public key: %s", Base58.encode(minterAccount.getPublicKey())));

3
src/test/java/org/qortal/test/common/AccountUtils.java
View File

@ -6,6 +6,7 @@ import java.util.HashMap;
import java.util.Map;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.crypto.Crypto;
import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.PaymentTransactionData;
import org.qortal.data.transaction.RewardShareTransactionData;
@ -45,7 +46,7 @@ public class AccountUtils {
long timestamp = repository.getTransactionRepository().fromSignature(reference).getTimestamp() + 1;
byte[] rewardSharePrivateKey = mintingAccount.getRewardSharePrivateKey(recipientAccount.getPublicKey());
byte[] rewardSharePublicKey = PrivateKeyAccount.toPublicKey(rewardSharePrivateKey);
byte[] rewardSharePublicKey = Crypto.toPublicKey(rewardSharePrivateKey);
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, mintingAccount.getPublicKey(), fee, null);
TransactionData transactionData = new RewardShareTransactionData(baseTransactionData, recipientAccount.getAddress(), rewardSharePublicKey, sharePercent);

63
src/test/java/org/qortal/test/minting/BlockTimestampTests.java Normal file
View File

@ -0,0 +1,63 @@
package org.qortal.test.minting;
import org.junit.Before;
import org.junit.Test;
import org.qortal.block.Block;
import org.qortal.data.block.BlockData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.transform.Transformer;
import org.qortal.utils.NTP;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
public class BlockTimestampTests extends Common {
private static class BlockTimestampDataPoint {
public byte[] minterPublicKey;
public int minterAccountLevel;
public long blockTimestamp;
}
private static final Random RANDOM = new Random();
@Before
public void beforeTest() throws DataException {
Common.useSettings("test-settings-v2-block-timestamps.json");
NTP.setFixedOffset(0L);
}
@Test
public void testTimestamps() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
Block parentBlock = BlockUtils.mintBlock(repository);
BlockData parentBlockData = parentBlock.getBlockData();
// Generate lots of test minters
List<BlockTimestampDataPoint> dataPoints = new ArrayList<>();
for (int i = 0; i < 20; i++) {
BlockTimestampDataPoint dataPoint = new BlockTimestampDataPoint();
dataPoint.minterPublicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
RANDOM.nextBytes(dataPoint.minterPublicKey);
dataPoint.minterAccountLevel = RANDOM.nextInt(5) + 5;
dataPoint.blockTimestamp = Block.calcTimestamp(parentBlockData, dataPoint.minterPublicKey, dataPoint.minterAccountLevel);
System.out.printf("[%d] level %d, blockTimestamp %d - parentTimestamp %d = %d%n",
i,
dataPoint.minterAccountLevel,
dataPoint.blockTimestamp,
parentBlockData.getTimestamp(),
dataPoint.blockTimestamp - parentBlockData.getTimestamp()
);
}
}
}
}

210
src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java Normal file
View File

@ -0,0 +1,210 @@
package org.qortal.test.network;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.junit.Ignore;
import org.junit.Test;
import org.qortal.controller.OnlineAccountsManager;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.network.message.*;
import org.qortal.transform.Transformer;
import java.nio.ByteBuffer;
import java.security.Security;
import java.util.*;
import static org.junit.Assert.*;
public class OnlineAccountsV3Tests {
private static final Random RANDOM = new Random();
static {
// This must go before any calls to LogManager/Logger
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
Security.insertProviderAt(new BouncyCastleProvider(), 0);
Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
}
@Ignore("For informational use")
@Test
public void compareV2ToV3() throws MessageException {
List<OnlineAccountData> onlineAccounts = generateOnlineAccounts(false);
// How many of each timestamp and leading byte (of public key)
Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte = convertToHashMaps(onlineAccounts);
byte[] v3DataBytes = new GetOnlineAccountsV3Message(hashesByTimestampThenByte).toBytes();
int v3ByteSize = v3DataBytes.length;
byte[] v2DataBytes = new GetOnlineAccountsV2Message(onlineAccounts).toBytes();
int v2ByteSize = v2DataBytes.length;
int numTimestamps = hashesByTimestampThenByte.size();
System.out.printf("For %d accounts split across %d timestamp%s: V2 size %d vs V3 size %d%n",
onlineAccounts.size(),
numTimestamps,
numTimestamps != 1 ? "s" : "",
v2ByteSize,
v3ByteSize
);
for (var outerMapEntry : hashesByTimestampThenByte.entrySet()) {
long timestamp = outerMapEntry.getKey();
var innerMap = outerMapEntry.getValue();
System.out.printf("For timestamp %d: %d / 256 slots used.%n",
timestamp,
innerMap.size()
);
}
}
private Map<Long, Map<Byte, byte[]>> convertToHashMaps(List<OnlineAccountData> onlineAccounts) {
// How many of each timestamp and leading byte (of public key)
Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte = new HashMap<>();
for (OnlineAccountData onlineAccountData : onlineAccounts) {
Long timestamp = onlineAccountData.getTimestamp();
Byte leadingByte = onlineAccountData.getPublicKey()[0];
hashesByTimestampThenByte
.computeIfAbsent(timestamp, k -> new HashMap<>())
.compute(leadingByte, (k, v) -> OnlineAccountsManager.xorByteArrayInPlace(v, onlineAccountData.getPublicKey()));
}
return hashesByTimestampThenByte;
}
@Test
public void testOnGetOnlineAccountsV3() {
List<OnlineAccountData> ourOnlineAccounts = generateOnlineAccounts(false);
List<OnlineAccountData> peersOnlineAccounts = generateOnlineAccounts(false);
Map<Long, Map<Byte, byte[]>> ourConvertedHashes = convertToHashMaps(ourOnlineAccounts);
Map<Long, Map<Byte, byte[]>> peersConvertedHashes = convertToHashMaps(peersOnlineAccounts);
List<String> mockReply = new ArrayList<>();
// Warning: no double-checking/fetching - we must be ConcurrentMap compatible!
// So no contains()-then-get() or multiple get()s on the same key/map.
for (var ourOuterMapEntry : ourConvertedHashes.entrySet()) {
Long timestamp = ourOuterMapEntry.getKey();
var ourInnerMap = ourOuterMapEntry.getValue();
var peersInnerMap = peersConvertedHashes.get(timestamp);
if (peersInnerMap == null) {
// Peer doesn't have this timestamp, so if it's valid (i.e. not too old) then we'd have to send all of ours
for (Byte leadingByte : ourInnerMap.keySet())
mockReply.add(timestamp + ":" + leadingByte);
} else {
// We have entries for this timestamp so compare against peer's entries
for (var ourInnerMapEntry : ourInnerMap.entrySet()) {
Byte leadingByte = ourInnerMapEntry.getKey();
byte[] peersHash = peersInnerMap.get(leadingByte);
if (!Arrays.equals(ourInnerMapEntry.getValue(), peersHash)) {
// We don't match peer, or peer doesn't have - send all online accounts for this timestamp and leading byte
mockReply.add(timestamp + ":" + leadingByte);
}
}
}
}
int numOurTimestamps = ourConvertedHashes.size();
System.out.printf("We have %d accounts split across %d timestamp%s%n",
ourOnlineAccounts.size(),
numOurTimestamps,
numOurTimestamps != 1 ? "s" : ""
);
int numPeerTimestamps = peersConvertedHashes.size();
System.out.printf("Peer sent %d accounts split across %d timestamp%s%n",
peersOnlineAccounts.size(),
numPeerTimestamps,
numPeerTimestamps != 1 ? "s" : ""
);
System.out.printf("We need to send: %d%n%s%n", mockReply.size(), String.join(", ", mockReply));
}
@Test
public void testSerialization() throws MessageException {
List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(true);
Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByteOut = convertToHashMaps(onlineAccountsOut);
validateSerialization(hashesByTimestampThenByteOut);
}
@Test
public void testEmptySerialization() throws MessageException {
Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByteOut = Collections.emptyMap();
validateSerialization(hashesByTimestampThenByteOut);
hashesByTimestampThenByteOut = new HashMap<>();
validateSerialization(hashesByTimestampThenByteOut);
}
private void validateSerialization(Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByteOut) throws MessageException {
Message messageOut = new GetOnlineAccountsV3Message(hashesByTimestampThenByteOut);
byte[] messageBytes = messageOut.toBytes();
ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes).asReadOnlyBuffer();
GetOnlineAccountsV3Message messageIn = (GetOnlineAccountsV3Message) Message.fromByteBuffer(byteBuffer);
Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByteIn = messageIn.getHashesByTimestampThenByte();
Set<Long> timestampsIn = hashesByTimestampThenByteIn.keySet();
Set<Long> timestampsOut = hashesByTimestampThenByteOut.keySet();
assertEquals("timestamp count mismatch", timestampsOut.size(), timestampsIn.size());
assertTrue("timestamps mismatch", timestampsIn.containsAll(timestampsOut));
for (Long timestamp : timestampsIn) {
Map<Byte, byte[]> hashesByByteIn = hashesByTimestampThenByteIn.get(timestamp);
Map<Byte, byte[]> hashesByByteOut = hashesByTimestampThenByteOut.get(timestamp);
assertNotNull("timestamp entry missing", hashesByByteOut);
Set<Byte> leadingBytesIn = hashesByByteIn.keySet();
Set<Byte> leadingBytesOut = hashesByByteOut.keySet();
assertEquals("leading byte entry count mismatch", leadingBytesOut.size(), leadingBytesIn.size());
assertTrue("leading byte entry mismatch", leadingBytesIn.containsAll(leadingBytesOut));
for (Byte leadingByte : leadingBytesOut) {
byte[] bytesIn = hashesByByteIn.get(leadingByte);
byte[] bytesOut = hashesByByteOut.get(leadingByte);
assertTrue("pubkey hash mismatch", Arrays.equals(bytesOut, bytesIn));
}
}
}
private List<OnlineAccountData> generateOnlineAccounts(boolean withSignatures) {
List<OnlineAccountData> onlineAccounts = new ArrayList<>();
int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2
for (int t = 0; t < numTimestamps; ++t) {
long timestamp = 1 << 31 + (t + 1) << 12;
int numAccounts = RANDOM.nextInt(3000);
for (int a = 0; a < numAccounts; ++a) {
byte[] sig = null;
if (withSignatures) {
sig = new byte[Transformer.SIGNATURE_LENGTH];
RANDOM.nextBytes(sig);
}
byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH];
RANDOM.nextBytes(pubkey);
onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey));
}
}
return onlineAccounts;
}
}

85
src/test/resources/test-chain-v2-block-timestamps.json Normal file
View File

@ -0,0 +1,85 @@
{
"isTestChain": true,
"blockTimestampMargin": 500,
"transactionExpiryPeriod": 86400000,
"maxBlockSize": 2097152,
"maxBytesPerUnitFee": 1024,
"unitFee": "0.1",
"nameRegistrationUnitFees": [
{ "timestamp": 1645372800000, "fee": "5" }
],
"requireGroupForApproval": false,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 20,
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 3600000,
"onlineAccountSignaturesMaxLifetime": 86400000,
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },
{ "height": 21, "reward": 1 }
],
"sharesByLevel": [
{ "levels": [ 1, 2 ], "share": 0.05 },
{ "levels": [ 3, 4 ], "share": 0.10 },
{ "levels": [ 5, 6 ], "share": 0.15 },
{ "levels": [ 7, 8 ], "share": 0.20 },
{ "levels": [ 9, 10 ], "share": 0.25 }
],
"qoraHoldersShare": 0.20,
"qoraPerQortReward": 250,
"blocksNeededByLevel": [ 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 ],
"blockTimingsByHeight": [
{ "height": 1, "target": 60000, "deviation": 30000, "power": 0.2 },
{ "height": 2, "target": 70000, "deviation": 10000, "power": 0.8 }
],
"ciyamAtSettings": {
"feePerStep": "0.0001",
"maxStepsPerRound": 500,
"stepsPerFunctionCall": 10,
"minutesPerBlock": 1
},
"featureTriggers": {
"messageHeight": 0,
"atHeight": 0,
"assetsTimestamp": 0,
"votingTimestamp": 0,
"arbitraryTimestamp": 0,
"powfixTimestamp": 0,
"qortalTimestamp": 0,
"newAssetPricingTimestamp": 0,
"groupApprovalTimestamp": 0,
"atFindNextTransactionFix": 0,
"newBlockSigHeight": 999999,
"shareBinFix": 999999,
"calcChainWeightTimestamp": 0,
"transactionV5Timestamp": 0,
"transactionV6Timestamp": 9999999999999,
"aggregateSignatureTimestamp": 9999999999999
},
"genesisInfo": {
"version": 4,
"timestamp": 0,
"transactions": [
{ "type": "ISSUE_ASSET", "assetName": "QORT", "description": "QORT native coin", "data": "", "quantity": 0, "isDivisible": true, "fee": 0 },
{ "type": "ISSUE_ASSET", "assetName": "Legacy-QORA", "description": "Representative legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true },
{ "type": "ISSUE_ASSET", "assetName": "QORT-from-QORA", "description": "QORT gained from holding legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true },
{ "type": "GENESIS", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "amount": "1000000000" },
{ "type": "GENESIS", "recipient": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "amount": "1000000" },
{ "type": "GENESIS", "recipient": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "amount": "1000000" },
{ "type": "GENESIS", "recipient": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "amount": "1000000" },
{ "type": "CREATE_GROUP", "creatorPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "groupName": "dev-group", "description": "developer group", "isOpen": false, "approvalThreshold": "PCT100", "minimumBlockDelay": 0, "maximumBlockDelay": 1440 },
{ "type": "ISSUE_ASSET", "issuerPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "assetName": "TEST", "description": "test asset", "data": "", "quantity": "1000000", "isDivisible": true, "fee": 0 },
{ "type": "ISSUE_ASSET", "issuerPublicKey": "C6wuddsBV3HzRrXUtezE7P5MoRXp5m3mEDokRDGZB6ry", "assetName": "OTHER", "description": "other test asset", "data": "", "quantity": "1000000", "isDivisible": true, "fee": 0 },
{ "type": "ISSUE_ASSET", "issuerPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "assetName": "GOLD", "description": "gold test asset", "data": "", "quantity": "1000000", "isDivisible": true, "fee": 0 },
{ "type": "ACCOUNT_FLAGS", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "andMask": -1, "orMask": 1, "xorMask": 0 },
{ "type": "REWARD_SHARE", "minterPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "rewardSharePublicKey": "7PpfnvLSG7y4HPh8hE7KoqAjLCkv7Ui6xw4mKAkbZtox", "sharePercent": "100" },
{ "type": "ACCOUNT_LEVEL", "target": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "level": 5 }
]
}
}

1
src/test/resources/test-chain-v2.json
View File

@ -55,6 +55,7 @@
"transactionV5Timestamp": 0,
"transactionV6Timestamp": 0,
"disableReferenceTimestamp": 9999999999999
"aggregateSignatureTimestamp": 2652560000000
},
"genesisInfo": {
"version": 4,

19
src/test/resources/test-settings-v2-block-timestamps.json Normal file
View File

@ -0,0 +1,19 @@
{
"repositoryPath": "testdb",
"bitcoinNet": "TEST3",
"litecoinNet": "TEST3",
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2-block-timestamps.json",
"exportPath": "qortal-backup-test",
"bootstrap": false,
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0,
"pruneBlockLimit": 100,
"bootstrapFilenamePrefix": "test-",
"dataPath": "data-test",
"tempDataPath": "data-test/_temp",
"listsPath": "lists-test",
"storagePolicy": "FOLLOWED_OR_VIEWED",
"maxStorageCapacity": 104857600
}