Qortal/qortal
1
0
Fork 1
mirror of https://github.com/Qortal/qortal.git synced 2025-07-23 04:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'catbref/schnorr-agg-BlockMinter-fix' into schnorr-agg-BlockMinter-fix

Browse Source 
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/java/org/qortal/controller/OnlineAccountsManager.java
#	src/main/java/org/qortal/network/message/BlockV2Message.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2.json
This commit is contained in:
CalDescent
2022-06-24 11:47:58 +01:00
parent ae6e2fab6f 431cbf01af
commit 23408827b3
27 changed files with 3268 additions and 810 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/org/qortal/account/PrivateKeyAccount.java
View File

@@ -11,15 +11,15 @@ public class PrivateKeyAccount extends PublicKeyAccount {
private final Ed25519PrivateKeyParameters edPrivateKeyParams;
/**
* Create PrivateKeyAccount using byte[32] seed.
* Create PrivateKeyAccount using byte[32] private key.
*
* @param seed
* @param privateKey
* byte[32] used to create private/public key pair
* @throws IllegalArgumentException
* if passed invalid seed
* if passed invalid privateKey
*/
public PrivateKeyAccount(Repository repository, byte[] seed) {
this(repository, new Ed25519PrivateKeyParameters(seed, 0));
public PrivateKeyAccount(Repository repository, byte[] privateKey) {
this(repository, new Ed25519PrivateKeyParameters(privateKey, 0));
}
private PrivateKeyAccount(Repository repository, Ed25519PrivateKeyParameters edPrivateKeyParams) {
@@ -37,10 +37,6 @@ public class PrivateKeyAccount extends PublicKeyAccount {
return this.privateKey;
}
public static byte[] toPublicKey(byte[] seed) {
return new Ed25519PrivateKeyParameters(seed, 0).generatePublicKey().getEncoded();
}
public byte[] sign(byte[] message) {
return Crypto.sign(this.edPrivateKeyParams, message);
}

137
src/main/java/org/qortal/block/Block.java
View File

@@ -10,6 +10,7 @@ import java.math.BigInteger;
import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.text.MessageFormat;
import java.text.NumberFormat;
import java.util.*;
import java.util.stream.Collectors;
@@ -27,6 +28,7 @@ import org.qortal.block.BlockChain.BlockTimingByHeight;
import org.qortal.block.BlockChain.AccountLevelShareBin;
import org.qortal.controller.OnlineAccountsManager;
import org.qortal.crypto.Crypto;
import org.qortal.crypto.Qortal25519Extras;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.data.account.AccountData;
import org.qortal.data.account.EligibleQoraHolderData;
@@ -221,11 +223,10 @@ public class Block {
return accountAmount;
}
}
/** Always use getExpandedAccounts() to access this, as it's lazy-instantiated. */
private List<ExpandedAccount> cachedExpandedAccounts = null;
/** Opportunistic cache of this block's valid online accounts. Only created by call to isValid(). */
private List<OnlineAccountData> cachedValidOnlineAccounts = null;
/** Opportunistic cache of this block's valid online reward-shares. Only created by call to isValid(). */
private List<RewardShareData> cachedOnlineRewardShares = null;
@@ -347,18 +348,21 @@ public class Block {
int version = parentBlock.getNextBlockVersion();
byte[] reference = parentBlockData.getSignature();
// Fetch our list of online accounts
List<OnlineAccountData> onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts();
if (onlineAccounts.isEmpty()) {
LOGGER.error("No online accounts - not even our own?");
// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, minter.getPublicKey());
if (minterLevel == 0) {
LOGGER.error("Minter effective level returned zero?");
return null;
}
// Find newest online accounts timestamp
long onlineAccountsTimestamp = 0;
for (OnlineAccountData onlineAccountData : onlineAccounts) {
if (onlineAccountData.getTimestamp() > onlineAccountsTimestamp)
onlineAccountsTimestamp = onlineAccountData.getTimestamp();
long timestamp = calcTimestamp(parentBlockData, minter.getPublicKey(), minterLevel);
long onlineAccountsTimestamp = OnlineAccountsManager.getCurrentOnlineAccountTimestamp();
// Fetch our list of online accounts
List<OnlineAccountData> onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts(onlineAccountsTimestamp);
if (onlineAccounts.isEmpty()) {
LOGGER.error("No online accounts - not even our own?");
return null;
}
// Load sorted list of reward share public keys into memory, so that the indexes can be obtained.
@@ -369,10 +373,6 @@ public class Block {
// Map using index into sorted list of reward-shares as key
Map<Integer, OnlineAccountData> indexedOnlineAccounts = new HashMap<>();
for (OnlineAccountData onlineAccountData : onlineAccounts) {
// Disregard online accounts with different timestamps
if (onlineAccountData.getTimestamp() != onlineAccountsTimestamp)
continue;
Integer accountIndex = getRewardShareIndex(onlineAccountData.getPublicKey(), allRewardSharePublicKeys);
if (accountIndex == null)
// Online account (reward-share) with current timestamp but reward-share cancelled
@@ -389,26 +389,29 @@ public class Block {
byte[] encodedOnlineAccounts = BlockTransformer.encodeOnlineAccounts(onlineAccountsSet);
int onlineAccountsCount = onlineAccountsSet.size();
// Concatenate online account timestamp signatures (in correct order)
byte[] onlineAccountsSignatures = new byte[onlineAccountsCount * Transformer.SIGNATURE_LENGTH];
for (int i = 0; i < onlineAccountsCount; ++i) {
Integer accountIndex = accountIndexes.get(i);
OnlineAccountData onlineAccountData = indexedOnlineAccounts.get(accountIndex);
System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH);
byte[] onlineAccountsSignatures;
if (timestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp()) {
// Collate all signatures
Collection<byte[]> signaturesToAggregate = indexedOnlineAccounts.values()
.stream()
.map(OnlineAccountData::getSignature)
.collect(Collectors.toList());
// Aggregated, single signature
onlineAccountsSignatures = Qortal25519Extras.aggregateSignatures(signaturesToAggregate);
} else {
// Concatenate online account timestamp signatures (in correct order)
onlineAccountsSignatures = new byte[onlineAccountsCount * Transformer.SIGNATURE_LENGTH];
for (int i = 0; i < onlineAccountsCount; ++i) {
Integer accountIndex = accountIndexes.get(i);
OnlineAccountData onlineAccountData = indexedOnlineAccounts.get(accountIndex);
System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH);
}
}
byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData,
minter.getPublicKey(), encodedOnlineAccounts));
// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, minter.getPublicKey());
if (minterLevel == 0) {
LOGGER.error("Minter effective level returned zero?");
return null;
}
long timestamp = calcTimestamp(parentBlockData, minter.getPublicKey(), minterLevel);
int transactionCount = 0;
byte[] transactionsSignature = null;
int height = parentBlockData.getHeight() + 1;
@@ -1013,49 +1016,59 @@ public class Block {
if (this.blockData.getOnlineAccountsSignatures() == null || this.blockData.getOnlineAccountsSignatures().length == 0)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MISSING;
if (this.blockData.getOnlineAccountsSignatures().length != onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED;
if (this.blockData.getTimestamp() >= BlockChain.getInstance().getAggregateSignatureTimestamp()) {
// We expect just the one, aggregated signature
if (this.blockData.getOnlineAccountsSignatures().length != Transformer.SIGNATURE_LENGTH)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED;
} else {
if (this.blockData.getOnlineAccountsSignatures().length != onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED;
}
// Check signatures
long onlineTimestamp = this.blockData.getOnlineAccountsTimestamp();
byte[] onlineTimestampBytes = Longs.toByteArray(onlineTimestamp);
// If this block is much older than current online timestamp, then there's no point checking current online accounts
List<OnlineAccountData> currentOnlineAccounts = onlineTimestamp < NTP.getTime() - OnlineAccountsManager.ONLINE_TIMESTAMP_MODULUS
? null
: OnlineAccountsManager.getInstance().getOnlineAccounts();
List<OnlineAccountData> latestBlocksOnlineAccounts = OnlineAccountsManager.getInstance().getLatestBlocksOnlineAccounts();
// Extract online accounts' timestamp signatures from block data
// Extract online accounts' timestamp signatures from block data. Only one signature if aggregated.
List<byte[]> onlineAccountsSignatures = BlockTransformer.decodeTimestampSignatures(this.blockData.getOnlineAccountsSignatures());
// We'll build up a list of online accounts to hand over to Controller if block is added to chain
// and this will become latestBlocksOnlineAccounts (above) to reduce CPU load when we process next block...
List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
if (this.blockData.getTimestamp() >= BlockChain.getInstance().getAggregateSignatureTimestamp()) {
// Aggregate all public keys
Collection<byte[]> publicKeys = onlineRewardShares.stream()
.map(RewardShareData::getRewardSharePublicKey)
.collect(Collectors.toList());
for (int i = 0; i < onlineAccountsSignatures.size(); ++i) {
byte[] signature = onlineAccountsSignatures.get(i);
byte[] publicKey = onlineRewardShares.get(i).getRewardSharePublicKey();
byte[] aggregatePublicKey = Qortal25519Extras.aggregatePublicKeys(publicKeys);
OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, signature, publicKey);
ourOnlineAccounts.add(onlineAccountData);
byte[] aggregateSignature = onlineAccountsSignatures.get(0);
// If signature is still current then no need to perform Ed25519 verify
if (currentOnlineAccounts != null && currentOnlineAccounts.remove(onlineAccountData))
// remove() returned true, so online account still current
// and one less entry in currentOnlineAccounts to check next time
continue;
// If signature was okay in latest block then no need to perform Ed25519 verify
if (latestBlocksOnlineAccounts != null && latestBlocksOnlineAccounts.contains(onlineAccountData))
continue;
if (!Crypto.verify(publicKey, signature, onlineTimestampBytes))
// One-step verification of aggregate signature using aggregate public key
if (!Qortal25519Extras.verifyAggregated(aggregatePublicKey, aggregateSignature, onlineTimestampBytes))
return ValidationResult.ONLINE_ACCOUNT_SIGNATURE_INCORRECT;
} else {
// Build block's view of online accounts
Set<OnlineAccountData> onlineAccounts = new HashSet<>();
for (int i = 0; i < onlineAccountsSignatures.size(); ++i) {
byte[] signature = onlineAccountsSignatures.get(i);
byte[] publicKey = onlineRewardShares.get(i).getRewardSharePublicKey();
OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, signature, publicKey);
onlineAccounts.add(onlineAccountData);
}
// Remove those already validated & cached by online accounts manager - no need to re-validate them
OnlineAccountsManager.getInstance().removeKnown(onlineAccounts, onlineTimestamp);
// Validate the rest
for (OnlineAccountData onlineAccount : onlineAccounts)
if (!Crypto.verify(onlineAccount.getPublicKey(), onlineAccount.getSignature(), onlineTimestampBytes))
return ValidationResult.ONLINE_ACCOUNT_SIGNATURE_INCORRECT;
// We've validated these, so allow online accounts manager to cache
OnlineAccountsManager.getInstance().addBlocksOnlineAccounts(onlineAccounts, onlineTimestamp);
}
// All online accounts valid, so save our list of online accounts for potential later use
this.cachedValidOnlineAccounts = ourOnlineAccounts;
this.cachedOnlineRewardShares = onlineRewardShares;
return ValidationResult.OK;
@@ -1426,9 +1439,6 @@ public class Block {
postBlockTidy();
// Give Controller our cached, valid online accounts data (if any) to help reduce CPU load for next block
OnlineAccountsManager.getInstance().pushLatestBlocksOnlineAccounts(this.cachedValidOnlineAccounts);
// Log some debugging info relating to the block weight calculation
this.logDebugInfo();
}
@@ -1644,9 +1654,6 @@ public class Block {
this.blockData.setHeight(null);
postBlockTidy();
// Remove any cached, valid online accounts data from Controller
OnlineAccountsManager.getInstance().popLatestBlocksOnlineAccounts();
}
protected void orphanTransactionsFromBlock() throws DataException {

7
src/main/java/org/qortal/block/BlockChain.java
View File

@@ -71,7 +71,8 @@ public class BlockChain {
calcChainWeightTimestamp,
transactionV5Timestamp,
transactionV6Timestamp,
disableReferenceTimestamp
disableReferenceTimestamp,
aggregateSignatureTimestamp;
}
// Custom transaction fees
@@ -415,6 +416,10 @@ public class BlockChain {
return this.featureTriggers.get(FeatureTrigger.disableReferenceTimestamp.name()).longValue();
}
public long getAggregateSignatureTimestamp() {
return this.featureTriggers.get(FeatureTrigger.aggregateSignatureTimestamp.name()).longValue();
}
// More complex getters for aspects that change by height or timestamp
public long getRewardAtHeight(int ourHeight) {

597
src/main/java/org/qortal/controller/BlockMinter.java
View File

@@ -65,9 +65,8 @@ public class BlockMinter extends Thread {
// Lite nodes do not mint
return;
}
try (final Repository repository = RepositoryManager.getRepository()) {
if (Settings.getInstance().getWipeUnconfirmedOnStart()) {
if (Settings.getInstance().getWipeUnconfirmedOnStart()) {
try (final Repository repository = RepositoryManager.getRepository()) {
// Wipe existing unconfirmed transactions
List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions();
@@ -77,30 +76,31 @@ public class BlockMinter extends Thread {
}
repository.saveChanges();
} catch (DataException e) {
LOGGER.warn("Repository issue trying to wipe unconfirmed transactions on start-up: {}", e.getMessage());
// Fall-through to normal behaviour in case we can recover
}
}
// Going to need this a lot...
BlockRepository blockRepository = repository.getBlockRepository();
BlockData previousBlockData = null;
BlockData previousBlockData = null;
// Vars to keep track of blocks that were skipped due to chain weight
byte[] parentSignatureForLastLowWeightBlock = null;
Long timeOfLastLowWeightBlock = null;
// Vars to keep track of blocks that were skipped due to chain weight
byte[] parentSignatureForLastLowWeightBlock = null;
Long timeOfLastLowWeightBlock = null;
List<Block> newBlocks = new ArrayList<>();
List<Block> newBlocks = new ArrayList<>();
// Flags for tracking change in whether minting is possible,
// so we can notify Controller, and further update SysTray, etc.
boolean isMintingPossible = false;
boolean wasMintingPossible = isMintingPossible;
while (running) {
repository.discardChanges(); // Free repository locks, if any
// Flags for tracking change in whether minting is possible,
// so we can notify Controller, and further update SysTray, etc.
boolean isMintingPossible = false;
boolean wasMintingPossible = isMintingPossible;
while (running) {
if (isMintingPossible != wasMintingPossible)
Controller.getInstance().onMintingPossibleChange(isMintingPossible);
if (isMintingPossible != wasMintingPossible)
Controller.getInstance().onMintingPossibleChange(isMintingPossible);
wasMintingPossible = isMintingPossible;
wasMintingPossible = isMintingPossible;
try {
// Sleep for a while
Thread.sleep(1000);
@@ -114,319 +114,338 @@ public class BlockMinter extends Thread {
if (minLatestBlockTimestamp == null)
continue;
// No online accounts? (e.g. during startup)
if (OnlineAccountsManager.getInstance().getOnlineAccounts().isEmpty())
// No online accounts for current timestamp? (e.g. during startup)
if (!OnlineAccountsManager.getInstance().hasOnlineAccounts())
continue;
List<MintingAccountData> mintingAccountsData = repository.getAccountRepository().getMintingAccounts();
// No minting accounts?
if (mintingAccountsData.isEmpty())
continue;
try (final Repository repository = RepositoryManager.getRepository()) {
// Going to need this a lot...
BlockRepository blockRepository = repository.getBlockRepository();
// Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level
// Note that minting accounts are actually reward-shares in Qortal
Iterator<MintingAccountData> madi = mintingAccountsData.iterator();
while (madi.hasNext()) {
MintingAccountData mintingAccountData = madi.next();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
if (rewardShareData == null) {
// Reward-share doesn't exist - probably cancelled but not yet removed from node's list of minting accounts
madi.remove();
continue;
}
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
madi.remove();
continue;
}
// Optional (non-validated) prevention of block submissions below a defined level.
// This is an unvalidated version of Blockchain.minAccountLevelToMint
// and exists only to reduce block candidates by default.
int level = mintingAccount.getEffectiveMintingLevel();
if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
madi.remove();
continue;
}
}
// Needs a mutable copy of the unmodifiableList
List<Peer> peers = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers());
BlockData lastBlockData = blockRepository.getLastBlock();
// Disregard peers that have "misbehaved" recently
peers.removeIf(Controller.hasMisbehaved);
// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
if (Synchronizer.getInstance().getRecoveryMode() == false)
peers.removeIf(Controller.hasNoRecentBlock);
// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
continue;
// If we are stuck on an invalid block, we should allow an alternative to be minted
boolean recoverInvalidBlock = false;
if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) {
// We've had at least one invalid block
long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived;
long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived;
if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) {
if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) {
// Last valid block was more than 10 mins ago, but we've had an invalid block since then
// Assume that the chain has stalled because there is no alternative valid candidate
// Enter recovery mode to allow alternative, valid candidates to be minted
recoverInvalidBlock = true;
}
}
}
// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
if (Synchronizer.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false)
List<MintingAccountData> mintingAccountsData = repository.getAccountRepository().getMintingAccounts();
// No minting accounts?
if (mintingAccountsData.isEmpty())
continue;
// There are enough peers with a recent block and our latest block is recent
// so go ahead and mint a block if possible.
isMintingPossible = true;
// Disregard minting accounts that are no longer valid, e.g. by transfer/loss of founder flag or account level
// Note that minting accounts are actually reward-shares in Qortal
Iterator<MintingAccountData> madi = mintingAccountsData.iterator();
while (madi.hasNext()) {
MintingAccountData mintingAccountData = madi.next();
// Check blockchain hasn't changed
if (previousBlockData == null || !Arrays.equals(previousBlockData.getSignature(), lastBlockData.getSignature())) {
previousBlockData = lastBlockData;
newBlocks.clear();
// Reduce log timeout
logTimeout = 10 * 1000L;
// Last low weight block is no longer valid
parentSignatureForLastLowWeightBlock = null;
}
// Discard accounts we have already built blocks with
mintingAccountsData.removeIf(mintingAccountData -> newBlocks.stream().anyMatch(newBlock -> Arrays.equals(newBlock.getBlockData().getMinterPublicKey(), mintingAccountData.getPublicKey())));
// Do we need to build any potential new blocks?
List<PrivateKeyAccount> newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
// We might need to sit the next block out, if one of our minting accounts signed the previous one
final byte[] previousBlockMinter = previousBlockData.getMinterPublicKey();
final boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter));
if (mintedLastBlock) {
LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one"));
continue;
}
if (parentSignatureForLastLowWeightBlock != null) {
// The last iteration found a higher weight block in the network, so sleep for a while
// to allow is to sync the higher weight chain. We are sleeping here rather than when
// detected as we don't want to hold the blockchain lock open.
LOGGER.debug("Sleeping for 10 seconds...");
Thread.sleep(10 * 1000L);
}
for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
// First block does the AT heavy-lifting
if (newBlocks.isEmpty()) {
Block newBlock = Block.mint(repository, previousBlockData, mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't build a to-be-minted block"));
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(mintingAccountData.getPublicKey());
if (rewardShareData == null) {
// Reward-share doesn't exist - probably cancelled but not yet removed from node's list of minting accounts
madi.remove();
continue;
}
newBlocks.add(newBlock);
} else {
// The blocks for other minters require less effort...
Block newBlock = newBlocks.get(0).remint(mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't rebuild a to-be-minted block"));
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
madi.remove();
continue;
}
newBlocks.add(newBlock);
// Optional (non-validated) prevention of block submissions below a defined level.
// This is an unvalidated version of Blockchain.minAccountLevelToMint
// and exists only to reduce block candidates by default.
int level = mintingAccount.getEffectiveMintingLevel();
if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
madi.remove();
continue;
}
}
}
// No potential block candidates?
if (newBlocks.isEmpty())
continue;
// Needs a mutable copy of the unmodifiableList
List<Peer> peers = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers());
BlockData lastBlockData = blockRepository.getLastBlock();
// Make sure we're the only thread modifying the blockchain
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (!blockchainLock.tryLock(30, TimeUnit.SECONDS)) {
LOGGER.debug("Couldn't acquire blockchain lock even after waiting 30 seconds");
continue;
}
// Disregard peers that have "misbehaved" recently
peers.removeIf(Controller.hasMisbehaved);
boolean newBlockMinted = false;
Block newBlock = null;
// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
if (Synchronizer.getInstance().getRecoveryMode() == false)
peers.removeIf(Controller.hasNoRecentBlock);
try {
// Clear repository session state so we have latest view of data
repository.discardChanges();
// Now that we have blockchain lock, do final check that chain hasn't changed
BlockData latestBlockData = blockRepository.getLastBlock();
if (!Arrays.equals(lastBlockData.getSignature(), latestBlockData.getSignature()))
// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
continue;
List<Block> goodBlocks = new ArrayList<>();
for (Block testBlock : newBlocks) {
// Is new block's timestamp valid yet?
// We do a separate check as some timestamp checks are skipped for testchains
if (testBlock.isTimestampValid() != ValidationResult.OK)
continue;
testBlock.preProcess();
// Is new block valid yet? (Before adding unconfirmed transactions)
ValidationResult result = testBlock.isValid();
if (result != ValidationResult.OK) {
moderatedLog(() -> LOGGER.error(String.format("To-be-minted block invalid '%s' before adding transactions?", result.name())));
continue;
}
goodBlocks.add(testBlock);
}
if (goodBlocks.isEmpty())
continue;
// Pick best block
final int parentHeight = previousBlockData.getHeight();
final byte[] parentBlockSignature = previousBlockData.getSignature();
BigInteger bestWeight = null;
for (int bi = 0; bi < goodBlocks.size(); ++bi) {
BlockData blockData = goodBlocks.get(bi).getBlockData();
BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
blockSummaryData.setMinterLevel(minterLevel);
BigInteger blockWeight = Block.calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData);
if (bestWeight == null || blockWeight.compareTo(bestWeight) < 0) {
newBlock = goodBlocks.get(bi);
bestWeight = blockWeight;
}
}
try {
if (this.higherWeightChainExists(repository, bestWeight)) {
// Check if the base block has updated since the last time we were here
if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null ||
!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) {
// We've switched to a different chain, so reset the timer
timeOfLastLowWeightBlock = NTP.getTime();
// If we are stuck on an invalid block, we should allow an alternative to be minted
boolean recoverInvalidBlock = false;
if (Synchronizer.getInstance().timeInvalidBlockLastReceived != null) {
// We've had at least one invalid block
long timeSinceLastValidBlock = NTP.getTime() - Synchronizer.getInstance().timeValidBlockLastReceived;
long timeSinceLastInvalidBlock = NTP.getTime() - Synchronizer.getInstance().timeInvalidBlockLastReceived;
if (timeSinceLastValidBlock > INVALID_BLOCK_RECOVERY_TIMEOUT) {
if (timeSinceLastInvalidBlock < INVALID_BLOCK_RECOVERY_TIMEOUT) {
// Last valid block was more than 10 mins ago, but we've had an invalid block since then
// Assume that the chain has stalled because there is no alternative valid candidate
// Enter recovery mode to allow alternative, valid candidates to be minted
recoverInvalidBlock = true;
}
parentSignatureForLastLowWeightBlock = previousBlockData.getSignature();
}
}
// If less than 30 seconds has passed since first detection the higher weight chain,
// we should skip our block submission to give us the opportunity to sync to the better chain
if (NTP.getTime() - timeOfLastLowWeightBlock < 30*1000L) {
LOGGER.debug("Higher weight chain found in peers, so not signing a block this round");
LOGGER.debug("Time since detected: {}ms", NTP.getTime() - timeOfLastLowWeightBlock);
// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
if (Synchronizer.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false)
continue;
// There are enough peers with a recent block and our latest block is recent
// so go ahead and mint a block if possible.
isMintingPossible = true;
// Reattach newBlocks to new repository handle
for (Block newBlock : newBlocks)
newBlock.setRepository(repository);
// Check blockchain hasn't changed
if (previousBlockData == null || !Arrays.equals(previousBlockData.getSignature(), lastBlockData.getSignature())) {
previousBlockData = lastBlockData;
newBlocks.clear();
// Reduce log timeout
logTimeout = 10 * 1000L;
// Last low weight block is no longer valid
parentSignatureForLastLowWeightBlock = null;
}
// Discard accounts we have already built blocks with
mintingAccountsData.removeIf(mintingAccountData -> newBlocks.stream().anyMatch(newBlock -> Arrays.equals(newBlock.getBlockData().getMinterPublicKey(), mintingAccountData.getPublicKey())));
// Do we need to build any potential new blocks?
List<PrivateKeyAccount> newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
// We might need to sit the next block out, if one of our minting accounts signed the previous one
byte[] previousBlockMinter = previousBlockData.getMinterPublicKey();
boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter));
if (mintedLastBlock) {
LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one"));
continue;
}
if (parentSignatureForLastLowWeightBlock != null) {
// The last iteration found a higher weight block in the network, so sleep for a while
// to allow is to sync the higher weight chain. We are sleeping here rather than when
// detected as we don't want to hold the blockchain lock open.
LOGGER.info("Sleeping for 10 seconds...");
Thread.sleep(10 * 1000L);
}
for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
// First block does the AT heavy-lifting
if (newBlocks.isEmpty()) {
Block newBlock = Block.mint(repository, previousBlockData, mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't build a to-be-minted block"));
continue;
}
else {
// More than 30 seconds have passed, so we should submit our block candidate anyway.
LOGGER.debug("More than 30 seconds passed, so proceeding to submit block candidate...");
newBlocks.add(newBlock);
} else {
// The blocks for other minters require less effort...
Block newBlock = newBlocks.get(0).remint(mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't rebuild a to-be-minted block"));
continue;
}
newBlocks.add(newBlock);
}
else {
LOGGER.debug("No higher weight chain found in peers");
}
} catch (DataException e) {
LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
}
// Discard any uncommitted changes as a result of the higher weight chain detection
repository.discardChanges();
// No potential block candidates?
if (newBlocks.isEmpty())
continue;
// Clear variables that track low weight blocks
parentSignatureForLastLowWeightBlock = null;
timeOfLastLowWeightBlock = null;
// Add unconfirmed transactions
addUnconfirmedTransactions(repository, newBlock);
// Sign to create block's signature
newBlock.sign();
// Is newBlock still valid?
ValidationResult validationResult = newBlock.isValid();
if (validationResult != ValidationResult.OK) {
// No longer valid? Report and discard
LOGGER.error(String.format("To-be-minted block now invalid '%s' after adding unconfirmed transactions?", validationResult.name()));
// Rebuild block candidates, just to be sure
newBlocks.clear();
// Make sure we're the only thread modifying the blockchain
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (!blockchainLock.tryLock(30, TimeUnit.SECONDS)) {
LOGGER.debug("Couldn't acquire blockchain lock even after waiting 30 seconds");
continue;
}
// Add to blockchain - something else will notice and broadcast new block to network
boolean newBlockMinted = false;
Block newBlock = null;
try {
newBlock.process();
// Clear repository session state so we have latest view of data
repository.discardChanges();
repository.saveChanges();
// Now that we have blockchain lock, do final check that chain hasn't changed
BlockData latestBlockData = blockRepository.getLastBlock();
if (!Arrays.equals(lastBlockData.getSignature(), latestBlockData.getSignature()))
continue;
LOGGER.info(String.format("Minted new block: %d", newBlock.getBlockData().getHeight()));
List<Block> goodBlocks = new ArrayList<>();
boolean wasInvalidBlockDiscarded = false;
Iterator<Block> newBlocksIterator = newBlocks.iterator();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey());
while (newBlocksIterator.hasNext()) {
Block testBlock = newBlocksIterator.next();
if (rewardShareData != null) {
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
Base58.encode(newBlock.getParent().getSignature()),
rewardShareData.getMinter(),
rewardShareData.getRecipient()));
} else {
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
Base58.encode(newBlock.getParent().getSignature()),
newBlock.getMinter().getAddress()));
// Is new block's timestamp valid yet?
// We do a separate check as some timestamp checks are skipped for testchains
if (testBlock.isTimestampValid() != ValidationResult.OK)
continue;
testBlock.preProcess();
// Is new block valid yet? (Before adding unconfirmed transactions)
ValidationResult result = testBlock.isValid();
if (result != ValidationResult.OK) {
moderatedLog(() -> LOGGER.error(String.format("To-be-minted block invalid '%s' before adding transactions?", result.name())));
newBlocksIterator.remove();
wasInvalidBlockDiscarded = true;
/*
* Bail out fast so that we loop around from the top again.
* This gives BlockMinter the possibility to remint this candidate block using another block from newBlocks,
* via the Blocks.remint() method, which avoids having to re-process Block ATs all over again.
* Particularly useful if some aspect of Blocks changes due a timestamp-based feature-trigger (see BlockChain class).
*/
break;
}
goodBlocks.add(testBlock);
}
// Notify network after we're released blockchain lock
newBlockMinted = true;
if (wasInvalidBlockDiscarded || goodBlocks.isEmpty())
continue;
// Notify Controller
repository.discardChanges(); // clear transaction status to prevent deadlocks
Controller.getInstance().onNewBlock(newBlock.getBlockData());
} catch (DataException e) {
// Unable to process block - report and discard
LOGGER.error("Unable to process newly minted block?", e);
newBlocks.clear();
// Pick best block
final int parentHeight = previousBlockData.getHeight();
final byte[] parentBlockSignature = previousBlockData.getSignature();
BigInteger bestWeight = null;
for (int bi = 0; bi < goodBlocks.size(); ++bi) {
BlockData blockData = goodBlocks.get(bi).getBlockData();
BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
blockSummaryData.setMinterLevel(minterLevel);
BigInteger blockWeight = Block.calcBlockWeight(parentHeight, parentBlockSignature, blockSummaryData);
if (bestWeight == null || blockWeight.compareTo(bestWeight) < 0) {
newBlock = goodBlocks.get(bi);
bestWeight = blockWeight;
}
}
try {
if (this.higherWeightChainExists(repository, bestWeight)) {
// Check if the base block has updated since the last time we were here
if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null ||
!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) {
// We've switched to a different chain, so reset the timer
timeOfLastLowWeightBlock = NTP.getTime();
}
parentSignatureForLastLowWeightBlock = previousBlockData.getSignature();
// If less than 30 seconds has passed since first detection the higher weight chain,
// we should skip our block submission to give us the opportunity to sync to the better chain
if (NTP.getTime() - timeOfLastLowWeightBlock < 30 * 1000L) {
LOGGER.info("Higher weight chain found in peers, so not signing a block this round");
LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock);
continue;
} else {
// More than 30 seconds have passed, so we should submit our block candidate anyway.
LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate...");
}
} else {
LOGGER.debug("No higher weight chain found in peers");
}
} catch (DataException e) {
LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
}
// Discard any uncommitted changes as a result of the higher weight chain detection
repository.discardChanges();
// Clear variables that track low weight blocks
parentSignatureForLastLowWeightBlock = null;
timeOfLastLowWeightBlock = null;
// Add unconfirmed transactions
addUnconfirmedTransactions(repository, newBlock);
// Sign to create block's signature
newBlock.sign();
// Is newBlock still valid?
ValidationResult validationResult = newBlock.isValid();
if (validationResult != ValidationResult.OK) {
// No longer valid? Report and discard
LOGGER.error(String.format("To-be-minted block now invalid '%s' after adding unconfirmed transactions?", validationResult.name()));
// Rebuild block candidates, just to be sure
newBlocks.clear();
continue;
}
// Add to blockchain - something else will notice and broadcast new block to network
try {
newBlock.process();
repository.saveChanges();
LOGGER.info(String.format("Minted new block: %d", newBlock.getBlockData().getHeight()));
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey());
if (rewardShareData != null) {
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
Base58.encode(newBlock.getParent().getSignature()),
rewardShareData.getMinter(),
rewardShareData.getRecipient()));
} else {
LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
Base58.encode(newBlock.getParent().getSignature()),
newBlock.getMinter().getAddress()));
}
// Notify network after we're released blockchain lock
newBlockMinted = true;
// Notify Controller
repository.discardChanges(); // clear transaction status to prevent deadlocks
Controller.getInstance().onNewBlock(newBlock.getBlockData());
} catch (DataException e) {
// Unable to process block - report and discard
LOGGER.error("Unable to process newly minted block?", e);
newBlocks.clear();
}
} finally {
blockchainLock.unlock();
}
} finally {
blockchainLock.unlock();
}
if (newBlockMinted) {
// Broadcast our new chain to network
BlockData newBlockData = newBlock.getBlockData();
if (newBlockMinted) {
// Broadcast our new chain to network
BlockData newBlockData = newBlock.getBlockData();
Network network = Network.getInstance();
network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newBlockData));
Network network = Network.getInstance();
network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newBlockData));
}
} catch (DataException e) {
LOGGER.warn("Repository issue while running block minter", e);
}
} catch (InterruptedException e) {
// We've been interrupted - time to exit
return;
}
} catch (DataException e) {
LOGGER.warn("Repository issue while running block minter", e);
} catch (InterruptedException e) {
// We've been interrupted - time to exit
return;
}
}

4
src/main/java/org/qortal/controller/Controller.java
View File

@@ -1241,6 +1241,10 @@ public class Controller extends Thread {
OnlineAccountsManager.getInstance().onNetworkOnlineAccountsV2Message(peer, message);
break;
case GET_ONLINE_ACCOUNTS_V3:
OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV3Message(peer, message);
break;
case GET_ARBITRARY_DATA:
// Not currently supported
break;

806
src/main/java/org/qortal/controller/OnlineAccountsManager.java
View File

@@ -1,12 +1,15 @@
package org.qortal.controller;
import com.google.common.hash.HashCode;
import com.google.common.primitives.Longs;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.Account;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.account.PublicKeyAccount;
import org.qortal.block.Block;
import org.qortal.block.BlockChain;
import org.qortal.crypto.Crypto;
import org.qortal.crypto.Qortal25519Extras;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.account.RewardShareData;
import org.qortal.data.network.OnlineAccountData;
@@ -18,270 +21,352 @@ import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import org.qortal.utils.NamedThreadFactory;
import java.util.*;
import java.util.concurrent.*;
import java.util.stream.Collectors;
public class OnlineAccountsManager extends Thread {
private class OurOnlineAccountsThread extends Thread {
public void run() {
try {
while (!isStopping) {
Thread.sleep(10000L);
// Refresh our online accounts signatures?
sendOurOnlineAccountsInfo();
}
} catch (InterruptedException e) {
// Fall through to exit thread
}
}
}
public class OnlineAccountsManager {
private static final Logger LOGGER = LogManager.getLogger(OnlineAccountsManager.class);
private static OnlineAccountsManager instance;
// 'Current' as in 'now'
/**
* How long online accounts signatures last before they expire.
*/
public static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000L;
/**
* How many 'current' timestamp-sets of online accounts we cache.
*/
private static final int MAX_CACHED_TIMESTAMP_SETS = 2;
/**
* How many timestamp-sets of online accounts we cache for 'latest blocks'.
*/
private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 3;
private static final long ONLINE_ACCOUNTS_QUEUE_INTERVAL = 100L; //ms
private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms
private static final long ONLINE_ACCOUNTS_LEGACY_BROADCAST_INTERVAL = 60 * 1000L; // ms
private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 15 * 1000L; // ms
private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L; // v3.2.0
private static final long ONLINE_ACCOUNTS_V3_PEER_VERSION = 0x03000300cbL; // v3.3.203
private final ScheduledExecutorService executor = Executors.newScheduledThreadPool(4, new NamedThreadFactory("OnlineAccounts"));
private volatile boolean isStopping = false;
// To do with online accounts list
private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms
private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 1 * 60 * 1000L; // ms
public static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000L;
private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L);
/** How many (latest) blocks' worth of online accounts we cache */
private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2;
private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L;
private final Set<OnlineAccountData> onlineAccountsImportQueue = ConcurrentHashMap.newKeySet();
private long onlineAccountsTasksTimestamp = Controller.startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms
/**
* Cache of 'current' online accounts, keyed by timestamp
*/
private final Map<Long, Set<OnlineAccountData>> currentOnlineAccounts = new ConcurrentHashMap<>();
/**
* Cache of hash-summary of 'current' online accounts, keyed by timestamp, then leading byte of public key.
*/
private final Map<Long, Map<Byte, byte[]>> currentOnlineAccountsHashes = new ConcurrentHashMap<>();
private final List<OnlineAccountData> onlineAccountsImportQueue = Collections.synchronizedList(new ArrayList<>());
/**
* Cache of online accounts for latest blocks - not necessarily 'current' / now.
* <i>Probably</i> only accessed / modified by a single Synchronizer thread.
*/
private final SortedMap<Long, Set<OnlineAccountData>> latestBlocksOnlineAccounts = new ConcurrentSkipListMap<>();
private boolean hasOnlineAccounts = false;
private boolean hasOurOnlineAccounts = false;
public static Long getCurrentOnlineAccountTimestamp() {
Long now = NTP.getTime();
if (now == null)
return null;
/** Cache of current 'online accounts' */
List<OnlineAccountData> onlineAccounts = new ArrayList<>();
/** Cache of latest blocks' online accounts */
Deque<List<OnlineAccountData>> latestBlocksOnlineAccounts = new ArrayDeque<>(MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS);
public OnlineAccountsManager() {
return (now / ONLINE_TIMESTAMP_MODULUS) * ONLINE_TIMESTAMP_MODULUS;
}
public static synchronized OnlineAccountsManager getInstance() {
if (instance == null) {
instance = new OnlineAccountsManager();
}
return instance;
private OnlineAccountsManager() {
}
public void run() {
private static class SingletonContainer {
private static final OnlineAccountsManager INSTANCE = new OnlineAccountsManager();
}
// Start separate thread to prepare our online accounts
// This could be converted to a thread pool later if more concurrency is needed
OurOnlineAccountsThread ourOnlineAccountsThread = new OurOnlineAccountsThread();
ourOnlineAccountsThread.start();
public static OnlineAccountsManager getInstance() {
return SingletonContainer.INSTANCE;
}
try {
while (!Controller.isStopping()) {
Thread.sleep(100L);
public void start() {
// Expire old online accounts signatures
executor.scheduleAtFixedRate(this::expireOldOnlineAccounts, ONLINE_ACCOUNTS_TASKS_INTERVAL, ONLINE_ACCOUNTS_TASKS_INTERVAL, TimeUnit.MILLISECONDS);
final Long now = NTP.getTime();
if (now == null) {
continue;
}
// Send our online accounts
executor.scheduleAtFixedRate(this::sendOurOnlineAccountsInfo, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS);
// Perform tasks to do with managing online accounts list
if (now >= onlineAccountsTasksTimestamp) {
onlineAccountsTasksTimestamp = now + ONLINE_ACCOUNTS_TASKS_INTERVAL;
performOnlineAccountsTasks();
}
// Request online accounts from peers (legacy)
executor.scheduleAtFixedRate(this::requestLegacyRemoteOnlineAccounts, ONLINE_ACCOUNTS_LEGACY_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_LEGACY_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS);
// Request online accounts from peers (V3+)
executor.scheduleAtFixedRate(this::requestRemoteOnlineAccounts, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, ONLINE_ACCOUNTS_BROADCAST_INTERVAL, TimeUnit.MILLISECONDS);
// Process queued online account verifications
this.processOnlineAccountsImportQueue();
}
} catch (InterruptedException e) {
// Fall through to exit thread
}
ourOnlineAccountsThread.interrupt();
// Process import queue
executor.scheduleWithFixedDelay(this::processOnlineAccountsImportQueue, ONLINE_ACCOUNTS_QUEUE_INTERVAL, ONLINE_ACCOUNTS_QUEUE_INTERVAL, TimeUnit.MILLISECONDS);
}
public void shutdown() {
isStopping = true;
this.interrupt();
}
public boolean hasOnlineAccounts() {
return this.hasOnlineAccounts;
}
// Online accounts import queue
private void processOnlineAccountsImportQueue() {
if (this.onlineAccountsImportQueue.isEmpty()) {
// Nothing to do
return;
}
LOGGER.debug("Processing online accounts import queue (size: {})", this.onlineAccountsImportQueue.size());
try (final Repository repository = RepositoryManager.getRepository()) {
List<OnlineAccountData> onlineAccountDataCopy = new ArrayList<>(this.onlineAccountsImportQueue);
for (OnlineAccountData onlineAccountData : onlineAccountDataCopy) {
if (isStopping) {
return;
}
this.verifyAndAddAccount(repository, onlineAccountData);
// Remove from queue
onlineAccountsImportQueue.remove(onlineAccountData);
}
LOGGER.debug("Finished processing online accounts import queue");
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while verifying online accounts"), e);
}
}
// Utilities
private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException {
final Long now = NTP.getTime();
if (now == null)
return;
PublicKeyAccount otherAccount = new PublicKeyAccount(repository, onlineAccountData.getPublicKey());
// Check timestamp is 'recent' here
if (Math.abs(onlineAccountData.getTimestamp() - now) > ONLINE_TIMESTAMP_MODULUS * 2) {
LOGGER.trace(() -> String.format("Rejecting online account %s with out of range timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp()));
return;
}
// Verify
byte[] data = Longs.toByteArray(onlineAccountData.getTimestamp());
if (!otherAccount.verify(onlineAccountData.getSignature(), data)) {
LOGGER.trace(() -> String.format("Rejecting invalid online account %s", otherAccount.getAddress()));
return;
}
// Qortal: check online account is actually reward-share
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(onlineAccountData.getPublicKey());
if (rewardShareData == null) {
// Reward-share doesn't even exist - probably not a good sign
LOGGER.trace(() -> String.format("Rejecting unknown online reward-share public key %s", Base58.encode(onlineAccountData.getPublicKey())));
return;
}
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
LOGGER.trace(() -> String.format("Rejecting online reward-share with non-minting account %s", mintingAccount.getAddress()));
return;
}
synchronized (this.onlineAccounts) {
OnlineAccountData existingAccountData = this.onlineAccounts.stream().filter(account -> Arrays.equals(account.getPublicKey(), onlineAccountData.getPublicKey())).findFirst().orElse(null);
if (existingAccountData != null) {
if (existingAccountData.getTimestamp() < onlineAccountData.getTimestamp()) {
this.onlineAccounts.remove(existingAccountData);
LOGGER.trace(() -> String.format("Updated online account %s with timestamp %d (was %d)", otherAccount.getAddress(), onlineAccountData.getTimestamp(), existingAccountData.getTimestamp()));
} else {
LOGGER.trace(() -> String.format("Not updating existing online account %s", otherAccount.getAddress()));
return;
}
} else {
LOGGER.trace(() -> String.format("Added online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp()));
}
this.onlineAccounts.add(onlineAccountData);
}
executor.shutdownNow();
}
// Testing support
public void ensureTestingAccountsOnline(PrivateKeyAccount... onlineAccounts) {
if (!BlockChain.getInstance().isTestChain()) {
LOGGER.warn("Ignoring attempt to ensure test account is online for non-test chain!");
return;
}
final Long now = NTP.getTime();
if (now == null)
final Long onlineAccountsTimestamp = getCurrentOnlineAccountTimestamp();
if (onlineAccountsTimestamp == null)
return;
final long onlineAccountsTimestamp = toOnlineAccountTimestamp(now);
byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
final boolean useAggregateCompatibleSignature = onlineAccountsTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp();
synchronized (this.onlineAccounts) {
this.onlineAccounts.clear();
Set<OnlineAccountData> replacementAccounts = new HashSet<>();
for (PrivateKeyAccount onlineAccount : onlineAccounts) {
// Check mintingAccount is actually reward-share?
for (PrivateKeyAccount onlineAccount : onlineAccounts) {
// Check mintingAccount is actually reward-share?
byte[] signature = useAggregateCompatibleSignature
? Qortal25519Extras.signForAggregation(onlineAccount.getPrivateKey(), timestampBytes)
: onlineAccount.sign(timestampBytes);
byte[] publicKey = onlineAccount.getPublicKey();
byte[] signature = onlineAccount.sign(timestampBytes);
byte[] publicKey = onlineAccount.getPublicKey();
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey);
replacementAccounts.add(ourOnlineAccountData);
}
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey);
this.onlineAccounts.add(ourOnlineAccountData);
this.currentOnlineAccounts.clear();
addAccounts(replacementAccounts);
}
// Online accounts import queue
private void processOnlineAccountsImportQueue() {
if (this.onlineAccountsImportQueue.isEmpty())
// Nothing to do
return;
LOGGER.debug("Processing online accounts import queue (size: {})", this.onlineAccountsImportQueue.size());
Set<OnlineAccountData> onlineAccountsToAdd = new HashSet<>();
try (final Repository repository = RepositoryManager.getRepository()) {
for (OnlineAccountData onlineAccountData : this.onlineAccountsImportQueue) {
if (isStopping)
return;
boolean isValid = this.isValidCurrentAccount(repository, onlineAccountData);
if (isValid)
onlineAccountsToAdd.add(onlineAccountData);
// Remove from queue
onlineAccountsImportQueue.remove(onlineAccountData);
}
} catch (DataException e) {
LOGGER.error("Repository issue while verifying online accounts", e);
}
if (!onlineAccountsToAdd.isEmpty()) {
LOGGER.debug("Merging {} validated online accounts from import queue", onlineAccountsToAdd.size());
addAccounts(onlineAccountsToAdd);
}
}
private void performOnlineAccountsTasks() {
// Utilities
public static byte[] xorByteArrayInPlace(byte[] inplaceArray, byte[] otherArray) {
if (inplaceArray == null)
return Arrays.copyOf(otherArray, otherArray.length);
// Start from index 1 to enforce static leading byte
for (int i = 1; i < otherArray.length; i++)
inplaceArray[i] ^= otherArray[i];
return inplaceArray;
}
private static boolean isValidCurrentAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException {
final Long now = NTP.getTime();
if (now == null)
return false;
byte[] rewardSharePublicKey = onlineAccountData.getPublicKey();
long onlineAccountTimestamp = onlineAccountData.getTimestamp();
// Check timestamp is 'recent' here
if (Math.abs(onlineAccountTimestamp - now) > ONLINE_TIMESTAMP_MODULUS * 2) {
LOGGER.trace(() -> String.format("Rejecting online account %s with out of range timestamp %d", Base58.encode(rewardSharePublicKey), onlineAccountTimestamp));
return false;
}
// Verify signature
byte[] data = Longs.toByteArray(onlineAccountData.getTimestamp());
boolean isSignatureValid = onlineAccountTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp()
? Qortal25519Extras.verifyAggregated(rewardSharePublicKey, onlineAccountData.getSignature(), data)
: Crypto.verify(rewardSharePublicKey, onlineAccountData.getSignature(), data);
if (!isSignatureValid) {
LOGGER.trace(() -> String.format("Rejecting invalid online account %s", Base58.encode(rewardSharePublicKey)));
return false;
}
// Qortal: check online account is actually reward-share
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey);
if (rewardShareData == null) {
// Reward-share doesn't even exist - probably not a good sign
LOGGER.trace(() -> String.format("Rejecting unknown online reward-share public key %s", Base58.encode(rewardSharePublicKey)));
return false;
}
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
LOGGER.trace(() -> String.format("Rejecting online reward-share with non-minting account %s", mintingAccount.getAddress()));
return false;
}
return true;
}
/** Adds accounts, maybe rebuilds hashes, returns whether any new accounts were added / hashes rebuilt. */
private boolean addAccounts(Collection<OnlineAccountData> onlineAccountsToAdd) {
// For keeping track of which hashes to rebuild
Map<Long, Set<Byte>> hashesToRebuild = new HashMap<>();
for (OnlineAccountData onlineAccountData : onlineAccountsToAdd) {
boolean isNewEntry = this.addAccount(onlineAccountData);
if (isNewEntry)
hashesToRebuild.computeIfAbsent(onlineAccountData.getTimestamp(), k -> new HashSet<>()).add(onlineAccountData.getPublicKey()[0]);
}
if (hashesToRebuild.isEmpty())
return false;
for (var entry : hashesToRebuild.entrySet()) {
Long timestamp = entry.getKey();
LOGGER.debug(() -> String.format("Rehashing for timestamp %d and leading bytes %s",
timestamp,
entry.getValue().stream().sorted(Byte::compareUnsigned).map(leadingByte -> String.format("%02x", leadingByte)).collect(Collectors.joining(", "))
)
);
for (Byte leadingByte : entry.getValue()) {
byte[] pubkeyHash = currentOnlineAccounts.get(timestamp).stream()
.map(OnlineAccountData::getPublicKey)
.filter(publicKey -> leadingByte == publicKey[0])
.reduce(null, OnlineAccountsManager::xorByteArrayInPlace);
currentOnlineAccountsHashes.computeIfAbsent(timestamp, k -> new ConcurrentHashMap<>()).put(leadingByte, pubkeyHash);
LOGGER.trace(() -> String.format("Rebuilt hash %s for timestamp %d and leading byte %02x using %d public keys",
HashCode.fromBytes(pubkeyHash),
timestamp,
leadingByte,
currentOnlineAccounts.get(timestamp).stream()
.map(OnlineAccountData::getPublicKey)
.filter(publicKey -> leadingByte == publicKey[0])
.count()
));
}
}
LOGGER.info(String.format("we have online accounts for timestamps: %s", String.join(", ", this.currentOnlineAccounts.keySet().stream().map(l -> Long.toString(l)).collect(Collectors.joining(", ")))));
return true;
}
private boolean addAccount(OnlineAccountData onlineAccountData) {
byte[] rewardSharePublicKey = onlineAccountData.getPublicKey();
long onlineAccountTimestamp = onlineAccountData.getTimestamp();
Set<OnlineAccountData> onlineAccounts = this.currentOnlineAccounts.computeIfAbsent(onlineAccountTimestamp, k -> ConcurrentHashMap.newKeySet());
boolean isNewEntry = onlineAccounts.add(onlineAccountData);
if (isNewEntry)
LOGGER.trace(() -> String.format("Added online account %s with timestamp %d", Base58.encode(rewardSharePublicKey), onlineAccountTimestamp));
else
LOGGER.trace(() -> String.format("Not updating existing online account %s with timestamp %d", Base58.encode(rewardSharePublicKey), onlineAccountTimestamp));
return isNewEntry;
}
/**
* Expire old entries.
*/
private void expireOldOnlineAccounts() {
final Long now = NTP.getTime();
if (now == null)
return;
// Expire old entries
final long cutoffThreshold = now - LAST_SEEN_EXPIRY_PERIOD;
synchronized (this.onlineAccounts) {
Iterator<OnlineAccountData> iterator = this.onlineAccounts.iterator();
while (iterator.hasNext()) {
OnlineAccountData onlineAccountData = iterator.next();
if (onlineAccountData.getTimestamp() < cutoffThreshold) {
iterator.remove();
LOGGER.trace(() -> {
PublicKeyAccount otherAccount = new PublicKeyAccount(null, onlineAccountData.getPublicKey());
return String.format("Removed expired online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp());
});
}
}
}
// Request data from other peers?
if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) {
List<OnlineAccountData> safeOnlineAccounts;
synchronized (this.onlineAccounts) {
safeOnlineAccounts = new ArrayList<>(this.onlineAccounts);
}
Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts);
Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts);
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
);
}
final long cutoffThreshold = now - MAX_CACHED_TIMESTAMP_SETS * ONLINE_TIMESTAMP_MODULUS;
this.currentOnlineAccounts.keySet().removeIf(timestamp -> timestamp < cutoffThreshold);
this.currentOnlineAccountsHashes.keySet().removeIf(timestamp -> timestamp < cutoffThreshold);
}
private void sendOurOnlineAccountsInfo() {
/**
* Request data from other peers. (Pre-V3)
*/
private void requestLegacyRemoteOnlineAccounts() {
final Long now = NTP.getTime();
if (now == null)
return;
// Don't bother if we're not up to date
if (!Controller.getInstance().isUpToDate())
return;
List<OnlineAccountData> mergedOnlineAccounts = Set.copyOf(this.currentOnlineAccounts.values()).stream().flatMap(Set::stream).collect(Collectors.toList());
Message messageV2 = new GetOnlineAccountsV2Message(mergedOnlineAccounts);
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() < ONLINE_ACCOUNTS_V3_PEER_VERSION
? messageV2
: null
);
}
/**
* Request data from other peers. V3+
*/
private void requestRemoteOnlineAccounts() {
final Long now = NTP.getTime();
if (now == null)
return;
// Don't bother if we're not up to date
if (!Controller.getInstance().isUpToDate())
return;
Message messageV3 = new GetOnlineAccountsV3Message(currentOnlineAccountsHashes);
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V3_PEER_VERSION
? messageV3
: null
);
}
/**
* Send online accounts that are minting on this node.
*/
private void sendOurOnlineAccountsInfo() {
// 'current' timestamp
final Long onlineAccountsTimestamp = getCurrentOnlineAccountTimestamp();
if (onlineAccountsTimestamp == null)
return;
Long now = NTP.getTime();
if (now == null) {
return;
}
@@ -296,13 +381,12 @@ public class OnlineAccountsManager extends Thread {
try (final Repository repository = RepositoryManager.getRepository()) {
mintingAccounts = repository.getAccountRepository().getMintingAccounts();
// We have no accounts, but don't reset timestamp
// We have no accounts to send
if (mintingAccounts.isEmpty())
return;
// Only reward-share accounts allowed
// Only active reward-shares allowed
Iterator<MintingAccountData> iterator = mintingAccounts.iterator();
int i = 0;
while (iterator.hasNext()) {
MintingAccountData mintingAccountData = iterator.next();
@@ -319,113 +403,138 @@ public class OnlineAccountsManager extends Thread {
iterator.remove();
continue;
}
if (++i > 1+1) {
iterator.remove();
continue;
}
}
} catch (DataException e) {
LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
return;
}
// 'current' timestamp
final long onlineAccountsTimestamp = toOnlineAccountTimestamp(now);
boolean hasInfoChanged = false;
boolean existingAccountsExist = false;
final boolean useAggregateCompatibleSignature = onlineAccountsTimestamp >= BlockChain.getInstance().getAggregateSignatureTimestamp();
byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
MINTING_ACCOUNTS:
for (MintingAccountData mintingAccountData : mintingAccounts) {
PrivateKeyAccount mintingAccount = new PrivateKeyAccount(null, mintingAccountData.getPrivateKey());
byte[] privateKey = mintingAccountData.getPrivateKey();
byte[] publicKey = Crypto.toPublicKey(privateKey);
byte[] signature = mintingAccount.sign(timestampBytes);
byte[] publicKey = mintingAccount.getPublicKey();
byte[] signature = useAggregateCompatibleSignature
? Qortal25519Extras.signForAggregation(privateKey, timestampBytes)
: Crypto.sign(privateKey, timestampBytes);
// Our account is online
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey);
synchronized (this.onlineAccounts) {
Iterator<OnlineAccountData> iterator = this.onlineAccounts.iterator();
while (iterator.hasNext()) {
OnlineAccountData existingOnlineAccountData = iterator.next();
if (Arrays.equals(existingOnlineAccountData.getPublicKey(), ourOnlineAccountData.getPublicKey())) {
// If our online account is already present, with same timestamp, then move on to next mintingAccount
if (existingOnlineAccountData.getTimestamp() == onlineAccountsTimestamp) {
existingAccountsExist = true;
continue MINTING_ACCOUNTS;
}
// If our online account is already present, but with older timestamp, then remove it
iterator.remove();
break;
}
}
this.onlineAccounts.add(ourOnlineAccountData);
}
LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", mintingAccount.getAddress(), onlineAccountsTimestamp));
ourOnlineAccounts.add(ourOnlineAccountData);
hasInfoChanged = true;
}
// Keep track of whether we have online accounts circulating in the network, for systray status
this.hasOnlineAccounts = existingAccountsExist || !ourOnlineAccounts.isEmpty();
this.hasOurOnlineAccounts = !ourOnlineAccounts.isEmpty();
boolean hasInfoChanged = addAccounts(ourOnlineAccounts);
if (!hasInfoChanged)
return;
Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts);
Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts);
Message messageV3 = new OnlineAccountsV2Message(ourOnlineAccounts); // TODO: V3 message
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V3_PEER_VERSION
? messageV3
: peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION
? messageV2
: messageV1
);
LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp));
LOGGER.debug("Broadcasted {} online account{} with timestamp {}", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp);
}
public static long toOnlineAccountTimestamp(long timestamp) {
return (timestamp / ONLINE_TIMESTAMP_MODULUS) * ONLINE_TIMESTAMP_MODULUS;
/**
* Returns whether online accounts manager has any online accounts with timestamp recent enough to be considered currently online.
*/
// BlockMinter: only calls this to check whether returned list is empty or not, to determine whether minting is even possible or not
public boolean hasOnlineAccounts() {
// 'current' timestamp
final Long onlineAccountsTimestamp = getCurrentOnlineAccountTimestamp();
if (onlineAccountsTimestamp == null)
return false;
return this.currentOnlineAccounts.containsKey(onlineAccountsTimestamp);
}
/** Returns list of online accounts with timestamp recent enough to be considered currently online. */
public boolean hasOurOnlineAccounts() {
return this.hasOurOnlineAccounts;
}
/**
* Returns list of online accounts matching given timestamp.
*/
// Block::mint() - only wants online accounts with (online) timestamp that matches block's (online) timestamp so they can be added to new block
public List<OnlineAccountData> getOnlineAccounts(long onlineTimestamp) {
LOGGER.info(String.format("caller's timestamp: %d, our timestamps: %s", onlineTimestamp, String.join(", ", this.currentOnlineAccounts.keySet().stream().map(l -> Long.toString(l)).collect(Collectors.joining(", ")))));
return new ArrayList<>(Set.copyOf(this.currentOnlineAccounts.getOrDefault(onlineTimestamp, Collections.emptySet())));
}
/**
* Returns list of online accounts with timestamp recent enough to be considered currently online.
*/
// API: calls this to return list of online accounts - probably expects ALL timestamps - but going to get 'current' from now on
public List<OnlineAccountData> getOnlineAccounts() {
final long onlineTimestamp = toOnlineAccountTimestamp(NTP.getTime());
// 'current' timestamp
final Long onlineAccountsTimestamp = getCurrentOnlineAccountTimestamp();
if (onlineAccountsTimestamp == null)
return Collections.emptyList();
synchronized (this.onlineAccounts) {
return this.onlineAccounts.stream().filter(account -> account.getTimestamp() == onlineTimestamp).collect(Collectors.toList());
}
return getOnlineAccounts(onlineAccountsTimestamp);
}
// Block processing
/** Returns cached, unmodifiable list of latest block's online accounts. */
public List<OnlineAccountData> getLatestBlocksOnlineAccounts() {
synchronized (this.latestBlocksOnlineAccounts) {
return this.latestBlocksOnlineAccounts.peekFirst();
}
/**
* Removes previously validated entries from block's online accounts.
* <p>
* Checks both 'current' and block caches.
* <p>
* Typically called by {@link Block#areOnlineAccountsValid()}
*/
public void removeKnown(Set<OnlineAccountData> blocksOnlineAccounts, Long timestamp) {
Set<OnlineAccountData> onlineAccounts = this.currentOnlineAccounts.get(timestamp);
// If not 'current' timestamp - try block cache instead
if (onlineAccounts == null)
onlineAccounts = this.latestBlocksOnlineAccounts.get(timestamp);
if (onlineAccounts != null)
blocksOnlineAccounts.removeAll(onlineAccounts);
}
/** Caches list of latest block's online accounts. Typically called by Block.process() */
public void pushLatestBlocksOnlineAccounts(List<OnlineAccountData> latestBlocksOnlineAccounts) {
synchronized (this.latestBlocksOnlineAccounts) {
if (this.latestBlocksOnlineAccounts.size() == MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS)
this.latestBlocksOnlineAccounts.pollLast();
this.latestBlocksOnlineAccounts.addFirst(latestBlocksOnlineAccounts == null
? Collections.emptyList()
: Collections.unmodifiableList(latestBlocksOnlineAccounts));
/**
* Adds block's online accounts to one of OnlineAccountManager's caches.
* <p>
* It is assumed that the online accounts have been verified.
* <p>
* Typically called by {@link Block#areOnlineAccountsValid()}
*/
public void addBlocksOnlineAccounts(Set<OnlineAccountData> blocksOnlineAccounts, Long timestamp) {
// We want to add to 'current' in preference if possible
if (this.currentOnlineAccounts.containsKey(timestamp)) {
addAccounts(blocksOnlineAccounts);
return;
}
}
/** Reverts list of latest block's online accounts. Typically called by Block.orphan() */
public void popLatestBlocksOnlineAccounts() {
synchronized (this.latestBlocksOnlineAccounts) {
this.latestBlocksOnlineAccounts.pollFirst();
// Add to block cache instead
this.latestBlocksOnlineAccounts.computeIfAbsent(timestamp, k -> ConcurrentHashMap.newKeySet())
.addAll(blocksOnlineAccounts);
// If block cache has grown too large then we need to trim.
if (this.latestBlocksOnlineAccounts.size() > MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS) {
// However, be careful to trim the opposite end to the entry we just added!
Long firstKey = this.latestBlocksOnlineAccounts.firstKey();
if (!firstKey.equals(timestamp))
this.latestBlocksOnlineAccounts.remove(firstKey);
else
this.latestBlocksOnlineAccounts.remove(this.latestBlocksOnlineAccounts.lastKey());
}
}
@@ -438,45 +547,48 @@ public class OnlineAccountsManager extends Thread {
List<OnlineAccountData> excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts();
// Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts
List<OnlineAccountData> accountsToSend;
synchronized (this.onlineAccounts) {
accountsToSend = new ArrayList<>(this.onlineAccounts);
}
List<OnlineAccountData> accountsToSend = Set.copyOf(this.currentOnlineAccounts.values()).stream().flatMap(Set::stream).collect(Collectors.toList());
int prefilterSize = accountsToSend.size();
Iterator<OnlineAccountData> iterator = accountsToSend.iterator();
SEND_ITERATOR:
while (iterator.hasNext()) {
OnlineAccountData onlineAccountData = iterator.next();
for (int i = 0; i < excludeAccounts.size(); ++i) {
OnlineAccountData excludeAccountData = excludeAccounts.get(i);
for (OnlineAccountData excludeAccountData : excludeAccounts) {
if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) {
iterator.remove();
continue SEND_ITERATOR;
break;
}
}
}
if (accountsToSend.isEmpty())
return;
Message onlineAccountsMessage = new OnlineAccountsMessage(accountsToSend);
peer.sendMessage(onlineAccountsMessage);
LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer));
LOGGER.debug("Sent {} of our {} online accounts to {}", accountsToSend.size(), prefilterSize, peer);
}
public void onNetworkOnlineAccountsMessage(Peer peer, Message message) {
OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message;
List<OnlineAccountData> peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts();
LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer));
LOGGER.debug("Received {} online accounts from {}", peersOnlineAccounts.size(), peer);
try (final Repository repository = RepositoryManager.getRepository()) {
for (OnlineAccountData onlineAccountData : peersOnlineAccounts)
this.verifyAndAddAccount(repository, onlineAccountData);
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e);
int importCount = 0;
// Add any online accounts to the queue that aren't already present
for (OnlineAccountData onlineAccountData : peersOnlineAccounts) {
boolean isNewEntry = onlineAccountsImportQueue.add(onlineAccountData);
if (isNewEntry)
importCount++;
}
if (importCount > 0)
LOGGER.debug("Added {} online accounts to queue", importCount);
}
public void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) {
@@ -485,58 +597,106 @@ public class OnlineAccountsManager extends Thread {
List<OnlineAccountData> excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts();
// Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts
List<OnlineAccountData> accountsToSend;
synchronized (this.onlineAccounts) {
accountsToSend = new ArrayList<>(this.onlineAccounts);
}
List<OnlineAccountData> accountsToSend = Set.copyOf(this.currentOnlineAccounts.values()).stream().flatMap(Set::stream).collect(Collectors.toList());
int prefilterSize = accountsToSend.size();
Iterator<OnlineAccountData> iterator = accountsToSend.iterator();
SEND_ITERATOR:
while (iterator.hasNext()) {
OnlineAccountData onlineAccountData = iterator.next();
for (int i = 0; i < excludeAccounts.size(); ++i) {
OnlineAccountData excludeAccountData = excludeAccounts.get(i);
for (OnlineAccountData excludeAccountData : excludeAccounts) {
if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) {
iterator.remove();
continue SEND_ITERATOR;
break;
}
}
}
if (accountsToSend.isEmpty())
return;
Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend);
peer.sendMessage(onlineAccountsMessage);
LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer));
LOGGER.debug("Sent {} of our {} online accounts to {}", accountsToSend.size(), prefilterSize, peer);
}
public void onNetworkOnlineAccountsV2Message(Peer peer, Message message) {
OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message;
List<OnlineAccountData> peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts();
LOGGER.debug(String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer));
LOGGER.debug("Received {} online accounts from {}", peersOnlineAccounts.size(), peer);
int importCount = 0;
// Add any online accounts to the queue that aren't already present
for (OnlineAccountData onlineAccountData : peersOnlineAccounts) {
boolean isNewEntry = onlineAccountsImportQueue.add(onlineAccountData);
// Do we already know about this online account data?
if (onlineAccounts.contains(onlineAccountData)) {
continue;
}
// Is it already in the import queue?
if (onlineAccountsImportQueue.contains(onlineAccountData)) {
continue;
}
onlineAccountsImportQueue.add(onlineAccountData);
importCount++;
if (isNewEntry)
importCount++;
}
LOGGER.debug(String.format("Added %d online accounts to queue", importCount));
if (importCount > 0)
LOGGER.debug("Added {} online accounts to queue", importCount);
}
public void onNetworkGetOnlineAccountsV3Message(Peer peer, Message message) {
GetOnlineAccountsV3Message getOnlineAccountsMessage = (GetOnlineAccountsV3Message) message;
Map<Long, Map<Byte, byte[]>> peersHashes = getOnlineAccountsMessage.getHashesByTimestampThenByte();
List<OnlineAccountData> outgoingOnlineAccounts = new ArrayList<>();
// Warning: no double-checking/fetching - we must be ConcurrentMap compatible!
// So no contains()-then-get() or multiple get()s on the same key/map.
// We also use getOrDefault() with emptySet() on currentOnlineAccounts in case corresponding timestamp entry isn't there.
for (var ourOuterMapEntry : currentOnlineAccountsHashes.entrySet()) {
Long timestamp = ourOuterMapEntry.getKey();
var ourInnerMap = ourOuterMapEntry.getValue();
var peersInnerMap = peersHashes.get(timestamp);
if (peersInnerMap == null) {
// Peer doesn't have this timestamp, so if it's valid (i.e. not too old) then we'd have to send all of ours
Set<OnlineAccountData> timestampsOnlineAccounts = this.currentOnlineAccounts.getOrDefault(timestamp, Collections.emptySet());
outgoingOnlineAccounts.addAll(timestampsOnlineAccounts);
LOGGER.debug(() -> String.format("Going to send all %d online accounts for timestamp %d", timestampsOnlineAccounts.size(), timestamp));
} else {
// Quick cache of which leading bytes to send so we only have to filter once
Set<Byte> outgoingLeadingBytes = new HashSet<>();
// We have entries for this timestamp so compare against peer's entries
for (var ourInnerMapEntry : ourInnerMap.entrySet()) {
Byte leadingByte = ourInnerMapEntry.getKey();
byte[] peersHash = peersInnerMap.get(leadingByte);
if (!Arrays.equals(ourInnerMapEntry.getValue(), peersHash)) {
// For this leading byte: hashes don't match or peer doesn't have entry
// Send all online accounts for this timestamp and leading byte
outgoingLeadingBytes.add(leadingByte);
}
}
int beforeAddSize = outgoingOnlineAccounts.size();
this.currentOnlineAccounts.getOrDefault(timestamp, Collections.emptySet()).stream()
.filter(account -> outgoingLeadingBytes.contains(account.getPublicKey()[0]))
.forEach(outgoingOnlineAccounts::add);
if (outgoingOnlineAccounts.size() > beforeAddSize)
LOGGER.debug(String.format("Going to send %d online accounts for timestamp %d and leading bytes %s",
outgoingOnlineAccounts.size() - beforeAddSize,
timestamp,
outgoingLeadingBytes.stream().sorted(Byte::compareUnsigned).map(leadingByte -> String.format("%02x", leadingByte)).collect(Collectors.joining(", "))
)
);
}
}
Message onlineAccountsMessage = new OnlineAccountsV2Message(outgoingOnlineAccounts); // TODO: V3 message
peer.sendMessage(onlineAccountsMessage);
LOGGER.debug("Sent {} online accounts to {}", outgoingOnlineAccounts.size(), peer);
}
}

2
src/main/java/org/qortal/controller/tradebot/TradeBot.java
View File

@@ -292,7 +292,7 @@ public class TradeBot implements Listener {
}
public static byte[] deriveTradeNativePublicKey(byte[] privateKey) {
return PrivateKeyAccount.toPublicKey(privateKey);
return Crypto.toPublicKey(privateKey);
}
public static byte[] deriveTradeForeignPublicKey(byte[] privateKey) {

99
src/main/java/org/qortal/crypto/BouncyCastle25519.java
View File

@@ -1,99 +0,0 @@
package org.qortal.crypto;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Arrays;
import org.bouncycastle.crypto.Digest;
import org.bouncycastle.math.ec.rfc7748.X25519;
import org.bouncycastle.math.ec.rfc7748.X25519Field;
import org.bouncycastle.math.ec.rfc8032.Ed25519;
/** Additions to BouncyCastle providing Ed25519 to X25519 key conversion. */
public class BouncyCastle25519 {
private static final Class<?> pointAffineClass;
private static final Constructor<?> pointAffineCtor;
private static final Method decodePointVarMethod;
private static final Field yField;
static {
try {
Class<?> ed25519Class = Ed25519.class;
pointAffineClass = Arrays.stream(ed25519Class.getDeclaredClasses()).filter(clazz -> clazz.getSimpleName().equals("PointAffine")).findFirst().get();
if (pointAffineClass == null)
throw new ClassNotFoundException("Can't locate PointExt inner class inside Ed25519");
decodePointVarMethod = ed25519Class.getDeclaredMethod("decodePointVar", byte[].class, int.class, boolean.class, pointAffineClass);
decodePointVarMethod.setAccessible(true);
pointAffineCtor = pointAffineClass.getDeclaredConstructors()[0];
pointAffineCtor.setAccessible(true);
yField = pointAffineClass.getDeclaredField("y");
yField.setAccessible(true);
} catch (NoSuchMethodException | SecurityException | IllegalArgumentException | NoSuchFieldException | ClassNotFoundException e) {
throw new RuntimeException("Can't initialize BouncyCastle25519 shim", e);
}
}
private static int[] obtainYFromPublicKey(byte[] ed25519PublicKey) {
try {
Object pA = pointAffineCtor.newInstance();
Boolean result = (Boolean) decodePointVarMethod.invoke(null, ed25519PublicKey, 0, true, pA);
if (result == null || !result)
return null;
return (int[]) yField.get(pA);
} catch (SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) {
throw new RuntimeException("Can't reflect into BouncyCastle", e);
}
}
public static byte[] toX25519PublicKey(byte[] ed25519PublicKey) {
int[] one = new int[X25519Field.SIZE];
X25519Field.one(one);
int[] y = obtainYFromPublicKey(ed25519PublicKey);
int[] oneMinusY = new int[X25519Field.SIZE];
X25519Field.sub(one, y, oneMinusY);
int[] onePlusY = new int[X25519Field.SIZE];
X25519Field.add(one, y, onePlusY);
int[] oneMinusYInverted = new int[X25519Field.SIZE];
X25519Field.inv(oneMinusY, oneMinusYInverted);
int[] u = new int[X25519Field.SIZE];
X25519Field.mul(onePlusY, oneMinusYInverted, u);
X25519Field.normalize(u);
byte[] x25519PublicKey = new byte[X25519.SCALAR_SIZE];
X25519Field.encode(u, x25519PublicKey, 0);
return x25519PublicKey;
}
public static byte[] toX25519PrivateKey(byte[] ed25519PrivateKey) {
Digest d = Ed25519.createPrehash();
byte[] h = new byte[d.getDigestSize()];
d.update(ed25519PrivateKey, 0, ed25519PrivateKey.length);
d.doFinal(h, 0);
byte[] s = new byte[X25519.SCALAR_SIZE];
System.arraycopy(h, 0, s, 0, X25519.SCALAR_SIZE);
s[0] &= 0xF8;
s[X25519.SCALAR_SIZE - 1] &= 0x7F;
s[X25519.SCALAR_SIZE - 1] |= 0x40;
return s;
}
}

1427
src/main/java/org/qortal/crypto/BouncyCastleEd25519.java Normal file
View File

File diff suppressed because it is too large Load Diff

19
src/main/java/org/qortal/crypto/Crypto.java
View File

@@ -253,6 +253,10 @@ public abstract class Crypto {
return false;
}
public static byte[] toPublicKey(byte[] privateKey) {
return new Ed25519PrivateKeyParameters(privateKey, 0).generatePublicKey().getEncoded();
}
public static boolean verify(byte[] publicKey, byte[] signature, byte[] message) {
try {
return Ed25519.verify(signature, 0, publicKey, 0, message, 0, message.length);
@@ -264,16 +268,24 @@ public abstract class Crypto {
public static byte[] sign(Ed25519PrivateKeyParameters edPrivateKeyParams, byte[] message) {
byte[] signature = new byte[SIGNATURE_LENGTH];
edPrivateKeyParams.sign(Ed25519.Algorithm.Ed25519, edPrivateKeyParams.generatePublicKey(), null, message, 0, message.length, signature, 0);
edPrivateKeyParams.sign(Ed25519.Algorithm.Ed25519,null, message, 0, message.length, signature, 0);
return signature;
}
public static byte[] sign(byte[] privateKey, byte[] message) {
byte[] signature = new byte[SIGNATURE_LENGTH];
new Ed25519PrivateKeyParameters(privateKey, 0).sign(Ed25519.Algorithm.Ed25519,null, message, 0, message.length, signature, 0);
return signature;
}
public static byte[] getSharedSecret(byte[] privateKey, byte[] publicKey) {
byte[] x25519PrivateKey = BouncyCastle25519.toX25519PrivateKey(privateKey);
byte[] x25519PrivateKey = Qortal25519Extras.toX25519PrivateKey(privateKey);
X25519PrivateKeyParameters xPrivateKeyParams = new X25519PrivateKeyParameters(x25519PrivateKey, 0);
byte[] x25519PublicKey = BouncyCastle25519.toX25519PublicKey(publicKey);
byte[] x25519PublicKey = Qortal25519Extras.toX25519PublicKey(publicKey);
X25519PublicKeyParameters xPublicKeyParams = new X25519PublicKeyParameters(x25519PublicKey, 0);
byte[] sharedSecret = new byte[SHARED_SECRET_LENGTH];
@@ -281,5 +293,4 @@ public abstract class Crypto {
return sharedSecret;
}
}

234
src/main/java/org/qortal/crypto/Qortal25519Extras.java Normal file
View File

@@ -0,0 +1,234 @@
package org.qortal.crypto;
import org.bouncycastle.crypto.Digest;
import org.bouncycastle.crypto.digests.SHA512Digest;
import org.bouncycastle.math.ec.rfc7748.X25519;
import org.bouncycastle.math.ec.rfc7748.X25519Field;
import org.bouncycastle.math.ec.rfc8032.Ed25519;
import org.bouncycastle.math.raw.Nat256;
import java.security.SecureRandom;
import java.util.Arrays;
import java.util.Collection;
/**
* Additions to BouncyCastle providing:
* <p></p>
* <ul>
* <li>Ed25519 to X25519 key conversion</li>
* <li>Aggregate public keys</li>
* <li>Aggregate signatures</li>
* </ul>
*/
public abstract class Qortal25519Extras extends BouncyCastleEd25519 {
private static final SecureRandom SECURE_RANDOM = new SecureRandom();
public static byte[] toX25519PublicKey(byte[] ed25519PublicKey) {
int[] one = new int[X25519Field.SIZE];
X25519Field.one(one);
PointAffine pA = new PointAffine();
if (!decodePointVar(ed25519PublicKey, 0, true, pA))
return null;
int[] y = pA.y;
int[] oneMinusY = new int[X25519Field.SIZE];
X25519Field.sub(one, y, oneMinusY);
int[] onePlusY = new int[X25519Field.SIZE];
X25519Field.add(one, y, onePlusY);
int[] oneMinusYInverted = new int[X25519Field.SIZE];
X25519Field.inv(oneMinusY, oneMinusYInverted);
int[] u = new int[X25519Field.SIZE];
X25519Field.mul(onePlusY, oneMinusYInverted, u);
X25519Field.normalize(u);
byte[] x25519PublicKey = new byte[X25519.SCALAR_SIZE];
X25519Field.encode(u, x25519PublicKey, 0);
return x25519PublicKey;
}
public static byte[] toX25519PrivateKey(byte[] ed25519PrivateKey) {
Digest d = Ed25519.createPrehash();
byte[] h = new byte[d.getDigestSize()];
d.update(ed25519PrivateKey, 0, ed25519PrivateKey.length);
d.doFinal(h, 0);
byte[] s = new byte[X25519.SCALAR_SIZE];
System.arraycopy(h, 0, s, 0, X25519.SCALAR_SIZE);
s[0] &= 0xF8;
s[X25519.SCALAR_SIZE - 1] &= 0x7F;
s[X25519.SCALAR_SIZE - 1] |= 0x40;
return s;
}
// Mostly for test support
public static PointAccum newPointAccum() {
return new PointAccum();
}
public static byte[] aggregatePublicKeys(Collection<byte[]> publicKeys) {
PointAccum rAccum = null;
for (byte[] publicKey : publicKeys) {
PointAffine pA = new PointAffine();
if (!decodePointVar(publicKey, 0, false, pA))
// Failed to decode
return null;
if (rAccum == null) {
rAccum = new PointAccum();
pointCopy(pA, rAccum);
} else {
pointAdd(pointCopy(pA), rAccum);
}
}
byte[] publicKey = new byte[SCALAR_BYTES];
if (0 == encodePoint(rAccum, publicKey, 0))
// Failed to encode
return null;
return publicKey;
}
public static byte[] aggregateSignatures(Collection<byte[]> signatures) {
// Signatures are (R, s)
// R is a point
// s is a scalar
PointAccum rAccum = null;
int[] sAccum = new int[SCALAR_INTS];
byte[] rEncoded = new byte[POINT_BYTES];
int[] sPart = new int[SCALAR_INTS];
for (byte[] signature : signatures) {
System.arraycopy(signature,0, rEncoded, 0, rEncoded.length);
PointAffine pA = new PointAffine();
if (!decodePointVar(rEncoded, 0, false, pA))
// Failed to decode
return null;
if (rAccum == null) {
rAccum = new PointAccum();
pointCopy(pA, rAccum);
decode32(signature, rEncoded.length, sAccum, 0, SCALAR_INTS);
} else {
pointAdd(pointCopy(pA), rAccum);
decode32(signature, rEncoded.length, sPart, 0, SCALAR_INTS);
Nat256.addTo(sPart, sAccum);
// "mod L" on sAccum
if (Nat256.gte(sAccum, L))
Nat256.subFrom(L, sAccum);
}
}
byte[] signature = new byte[SIGNATURE_SIZE];
if (0 == encodePoint(rAccum, signature, 0))
// Failed to encode
return null;
for (int i = 0; i < sAccum.length; ++i) {
encode32(sAccum[i], signature, POINT_BYTES + i * 4);
}
return signature;
}
public static byte[] signForAggregation(byte[] privateKey, byte[] message) {
// Very similar to BouncyCastle's implementation except we use secure random nonce and different hash
Digest d = new SHA512Digest();
byte[] h = new byte[d.getDigestSize()];
d.reset();
d.update(privateKey, 0, privateKey.length);
d.doFinal(h, 0);
byte[] sH = new byte[SCALAR_BYTES];
pruneScalar(h, 0, sH);
byte[] publicKey = new byte[SCALAR_BYTES];
scalarMultBaseEncoded(sH, publicKey, 0);
byte[] rSeed = new byte[d.getDigestSize()];
SECURE_RANDOM.nextBytes(rSeed);
byte[] r = new byte[SCALAR_BYTES];
pruneScalar(rSeed, 0, r);
byte[] R = new byte[POINT_BYTES];
scalarMultBaseEncoded(r, R, 0);
d.reset();
d.update(message, 0, message.length);
d.doFinal(h, 0);
byte[] k = reduceScalar(h);
byte[] s = calculateS(r, k, sH);
byte[] signature = new byte[SIGNATURE_SIZE];
System.arraycopy(R, 0, signature, 0, POINT_BYTES);
System.arraycopy(s, 0, signature, POINT_BYTES, SCALAR_BYTES);
return signature;
}
public static boolean verifyAggregated(byte[] publicKey, byte[] signature, byte[] message) {
byte[] R = Arrays.copyOfRange(signature, 0, POINT_BYTES);
byte[] s = Arrays.copyOfRange(signature, POINT_BYTES, POINT_BYTES + SCALAR_BYTES);
if (!checkPointVar(R))
// R out of bounds
return false;
if (!checkScalarVar(s))
// s out of bounds
return false;
byte[] S = new byte[POINT_BYTES];
scalarMultBaseEncoded(s, S, 0);
PointAffine pA = new PointAffine();
if (!decodePointVar(publicKey, 0, true, pA))
// Failed to decode
return false;
Digest d = new SHA512Digest();
byte[] h = new byte[d.getDigestSize()];
d.update(message, 0, message.length);
d.doFinal(h, 0);
byte[] k = reduceScalar(h);
int[] nS = new int[SCALAR_INTS];
decodeScalar(s, 0, nS);
int[] nA = new int[SCALAR_INTS];
decodeScalar(k, 0, nA);
/*PointAccum*/
PointAccum pR = new PointAccum();
scalarMultStrausVar(nS, nA, pA, pR);
byte[] check = new byte[POINT_BYTES];
if (0 == encodePoint(pR, check, 0))
// Failed to encode
return false;
return Arrays.equals(check, R);
}
}

19
src/main/java/org/qortal/data/network/OnlineAccountData.java
View File

@@ -5,6 +5,7 @@ import java.util.Arrays;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlTransient;
import org.qortal.account.PublicKeyAccount;
@@ -16,6 +17,9 @@ public class OnlineAccountData {
protected byte[] signature;
protected byte[] publicKey;
@XmlTransient
private int hash;
// Constructors
// necessary for JAXB serialization
@@ -62,20 +66,23 @@ public class OnlineAccountData {
if (otherOnlineAccountData.timestamp != this.timestamp)
return false;
// Signature more likely to be unique than public key
if (!Arrays.equals(otherOnlineAccountData.signature, this.signature))
return false;
if (!Arrays.equals(otherOnlineAccountData.publicKey, this.publicKey))
return false;
// We don't compare signature because it's not our remit to verify and newer aggregate signatures use random nonces
return true;
}
@Override
public int hashCode() {
// Pretty lazy implementation
return (int) this.timestamp;
int h = this.hash;
if (h == 0) {
this.hash = h = Long.hashCode(this.timestamp)
^ Arrays.hashCode(this.publicKey);
// We don't use signature because newer aggregate signatures use random nonces
}
return h;
}
}

2
src/main/java/org/qortal/network/Network.java
View File

@@ -469,6 +469,8 @@ public class Network {
class NetworkProcessor extends ExecuteProduceConsume {
private final Logger LOGGER = LogManager.getLogger(NetworkProcessor.class);
private final AtomicLong nextConnectTaskTimestamp = new AtomicLong(0L); // ms - try first connect once NTP syncs
private final AtomicLong nextBroadcastTimestamp = new AtomicLong(0L); // ms - try first broadcast once NTP syncs

112
src/main/java/org/qortal/network/message/GetOnlineAccountsV3Message.java Normal file
View File

@@ -0,0 +1,112 @@
package org.qortal.network.message;
import com.google.common.primitives.Longs;
import org.qortal.transform.Transformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
/**
* For requesting online accounts info from remote peer, given our list of online accounts.
* <p></p>
* Different format to V1 and V2:<br>
* <ul>
* <li>V1 is: number of entries, then timestamp + pubkey for each entry</li>
* <li>V2 is: groups of: number of entries, timestamp, then pubkey for each entry</li>
* <li>V3 is: groups of: timestamp, number of entries (one per leading byte), then hash(pubkeys) for each entry</li>
* </ul>
* <p></p>
* End
*/
public class GetOnlineAccountsV3Message extends Message {
private static final Map<Long, Map<Byte, byte[]>> EMPTY_ONLINE_ACCOUNTS = Collections.emptyMap();
private Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte;
public GetOnlineAccountsV3Message(Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte) {
super(MessageType.GET_ONLINE_ACCOUNTS_V3);
// If we don't have ANY online accounts then it's an easier construction...
if (hashesByTimestampThenByte.isEmpty()) {
this.dataBytes = EMPTY_DATA_BYTES;
return;
}
// We should know exactly how many bytes to allocate now
int byteSize = hashesByTimestampThenByte.size() * (Transformer.TIMESTAMP_LENGTH + Transformer.BYTE_LENGTH);
byteSize += hashesByTimestampThenByte.values()
.stream()
.mapToInt(map -> map.size() * Transformer.PUBLIC_KEY_LENGTH)
.sum();
ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
// Warning: no double-checking/fetching! We must be ConcurrentMap compatible.
// So no contains() then get() or multiple get()s on the same key/map.
try {
for (var outerMapEntry : hashesByTimestampThenByte.entrySet()) {
bytes.write(Longs.toByteArray(outerMapEntry.getKey()));
var innerMap = outerMapEntry.getValue();
// Number of entries: 1 - 256, where 256 is represented by 0
bytes.write(innerMap.size() & 0xFF);
for (byte[] hashBytes : innerMap.values()) {
bytes.write(hashBytes);
}
}
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private GetOnlineAccountsV3Message(int id, Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte) {
super(id, MessageType.GET_ONLINE_ACCOUNTS_V3);
this.hashesByTimestampThenByte = hashesByTimestampThenByte;
}
public Map<Long, Map<Byte, byte[]>> getHashesByTimestampThenByte() {
return this.hashesByTimestampThenByte;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) {
// 'empty' case
if (!bytes.hasRemaining()) {
return new GetOnlineAccountsV3Message(id, EMPTY_ONLINE_ACCOUNTS);
}
Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte = new HashMap<>();
while (bytes.hasRemaining()) {
long timestamp = bytes.getLong();
int hashCount = bytes.get();
if (hashCount <= 0)
// 256 is represented by 0.
// Also converts negative signed value (e.g. -1) to proper positive unsigned value (255)
hashCount += 256;
Map<Byte, byte[]> hashesByByte = new HashMap<>();
for (int i = 0; i < hashCount; ++i) {
byte[] publicKeyHash = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKeyHash);
hashesByByte.put(publicKeyHash[0], publicKeyHash);
}
hashesByTimestampThenByte.put(timestamp, hashesByByte);
}
return new GetOnlineAccountsV3Message(id, hashesByTimestampThenByte);
}
}

3
src/main/java/org/qortal/network/message/Message.java
View File

@@ -46,6 +46,7 @@ public abstract class Message {
private static final int MAX_DATA_SIZE = 10 * 1024 * 1024; // 10MB
protected static final byte[] EMPTY_DATA_BYTES = new byte[0];
private static final ByteBuffer EMPTY_READ_ONLY_BYTE_BUFFER = ByteBuffer.wrap(EMPTY_DATA_BYTES).asReadOnlyBuffer();
protected int id;
protected final MessageType type;
@@ -126,7 +127,7 @@ public abstract class Message {
if (dataSize > 0 && dataSize + CHECKSUM_LENGTH > readOnlyBuffer.remaining())
return null;
ByteBuffer dataSlice = null;
ByteBuffer dataSlice = EMPTY_READ_ONLY_BYTE_BUFFER;
if (dataSize > 0) {
byte[] expectedChecksum = new byte[CHECKSUM_LENGTH];
readOnlyBuffer.get(expectedChecksum);

2
src/main/java/org/qortal/network/message/MessageType.java
View File

@@ -46,6 +46,8 @@ public enum MessageType {
GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer),
ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer),
GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer),
// ONLINE_ACCOUNTS_V3(84, OnlineAccountsV3Message::fromByteBuffer),
GET_ONLINE_ACCOUNTS_V3(85, GetOnlineAccountsV3Message::fromByteBuffer),
ARBITRARY_DATA(90, ArbitraryDataMessage::fromByteBuffer),
GET_ARBITRARY_DATA(91, GetArbitraryDataMessage::fromByteBuffer),

3
src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java
View File

@@ -23,7 +23,6 @@ import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.crypto.Crypto;
import org.qortal.globalization.Translator;
import org.qortal.gui.SysTray;
@@ -1003,7 +1002,7 @@ public class HSQLDBRepository implements Repository {
if (privateKey == null)
return null;
return PrivateKeyAccount.toPublicKey(privateKey);
return Crypto.toPublicKey(privateKey);
}
public static String ed25519PublicKeyToAddress(byte[] publicKey) {

3
src/main/resources/blockchain.json
View File

@@ -60,7 +60,8 @@
"calcChainWeightTimestamp": 1620579600000,
"transactionV5Timestamp": 1642176000000,
"transactionV6Timestamp": 9999999999999,
"disableReferenceTimestamp": 1655222400000
"disableReferenceTimestamp": 1655222400000,
"aggregateSignatureTimestamp": 9999999999999
},
"genesisInfo": {
"version": 4,