More work on online accounts / blocks

Block data now includes number of online accounts, as encoded online account indexes can't be
validated by ConciseSet it seems.
Corresponding changes to repository, transformer, block validation, data object, block summaries...

Block timestamps are now calculated using parent block data and generator's public key,
instead of old qora1 generating balance code.

Generators are valid to forge if they have forging flag enabled. This will probably change
to an account-level check in the near future.

Added trimming of old online accounts signatures from blocks.

Tidied up SysTray/BlockGenerator generation enabled/possible flag.

Although we perform online accounts tasks (currently) every 10 seconds,
only broadcast our online accounts every 60 seconds.

In Controller.main(), if args are present then use first as a filename to
settings JSON file (overriding the default filename).

Still to do: change Block/BlockChain/Synchronizer to prefer blocks with more online accounts,
failing that use generator nearest 'ideal', etc.
This commit is contained in:
catbref 2019-09-18 14:49:47 +01:00
parent aa81c86cf1
commit 0dd5b1e65a
14 changed files with 277 additions and 188 deletions

View File

@ -16,11 +16,13 @@ import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qora.account.Account;
import org.qora.account.Forging;
import org.qora.account.PrivateKeyAccount;
import org.qora.account.PublicKeyAccount;
import org.qora.asset.Asset;
import org.qora.at.AT;
import org.qora.block.BlockChain;
import org.qora.block.BlockChain.BlockTimingByHeight;
import org.qora.controller.Controller;
import org.qora.crypto.Crypto;
import org.qora.data.account.ProxyForgerData;
@ -99,7 +101,7 @@ public class Block {
TRANSACTION_ALREADY_PROCESSED(54),
TRANSACTION_NEEDS_APPROVAL(55),
AT_STATES_MISMATCH(61),
ONLINE_ACCOUNT_LEVEL_ZERO(70),
ONLINE_ACCOUNTS_INVALID(70),
ONLINE_ACCOUNT_UNKNOWN(71),
ONLINE_ACCOUNT_SIGNATURES_MISSING(72),
ONLINE_ACCOUNT_SIGNATURES_MALFORMED(73),
@ -147,6 +149,13 @@ public class Block {
// TODO push this out to blockchain config file
public static final int MAX_BLOCK_BYTES = 1048576;
private static final BigInteger MAX_DISTANCE;
static {
byte[] maxValue = new byte[Transformer.PUBLIC_KEY_LENGTH];
Arrays.fill(maxValue, (byte) 0xFF);
MAX_DISTANCE = new BigInteger(1, maxValue);
}
public static final ConciseSet EMPTY_ONLINE_ACCOUNTS = new ConciseSet();
// Constructors
@ -251,10 +260,11 @@ public class Block {
ConciseSet onlineAccountsSet = new ConciseSet();
onlineAccountsSet = onlineAccountsSet.convert(accountIndexes);
byte[] encodedOnlineAccounts = BlockTransformer.encodeOnlineAccounts(onlineAccountsSet);
int onlineAccountsCount = onlineAccountsSet.size();
// Concatenate online account timestamp signatures (in correct order)
byte[] onlineAccountsSignatures = new byte[accountIndexes.size() * Transformer.SIGNATURE_LENGTH];
for (int i = 0; i < accountIndexes.size(); ++i) {
byte[] onlineAccountsSignatures = new byte[onlineAccountsCount * Transformer.SIGNATURE_LENGTH];
for (int i = 0; i < onlineAccountsCount; ++i) {
Integer accountIndex = accountIndexes.get(i);
OnlineAccount onlineAccount = indexedOnlineAccounts.get(accountIndex);
System.arraycopy(onlineAccount.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH);
@ -268,7 +278,7 @@ public class Block {
throw new DataException("Unable to calculate next block generator signature", e);
}
long timestamp = parentBlock.calcNextBlockTimestamp(version, generatorSignature, generator);
long timestamp = calcTimestamp(parentBlockData, generator.getPublicKey());
int transactionCount = 0;
byte[] transactionsSignature = null;
@ -282,7 +292,8 @@ public class Block {
// This instance used for AT processing
this.blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance,
generator.getPublicKey(), generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsTimestamp, onlineAccountsSignatures);
generator.getPublicKey(), generatorSignature, atCount, atFees,
encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures);
// Requires this.blockData and this.transactions, sets this.ourAtStates and this.ourAtFees
this.executeATs();
@ -294,7 +305,8 @@ public class Block {
// Rebuild blockData using post-AT-execute data
this.blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance,
generator.getPublicKey(), generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsTimestamp, onlineAccountsSignatures);
generator.getPublicKey(), generatorSignature, atCount, atFees,
encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures);
}
/**
@ -311,7 +323,6 @@ public class Block {
newBlock.generator = generator;
BlockData parentBlockData = this.getParent();
Block parentBlock = new Block(repository, parentBlockData);
// Copy AT state data
newBlock.ourAtStates = this.ourAtStates;
@ -331,7 +342,7 @@ public class Block {
throw new DataException("Unable to calculate next block generator signature", e);
}
long timestamp = parentBlock.calcNextBlockTimestamp(version, generatorSignature, generator);
long timestamp = calcTimestamp(parentBlockData, generator.getPublicKey());
newBlock.transactions = this.transactions;
int transactionCount = this.blockData.getTransactionCount();
@ -343,11 +354,12 @@ public class Block {
BigDecimal atFees = newBlock.ourAtFees;
byte[] encodedOnlineAccounts = this.blockData.getEncodedOnlineAccounts();
int onlineAccountsCount = this.blockData.getOnlineAccountsCount();
Long onlineAccountsTimestamp = this.blockData.getOnlineAccountsTimestamp();
byte[] onlineAccountsSignatures = this.blockData.getOnlineAccountsSignatures();
newBlock.blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance,
generator.getPublicKey(), generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsTimestamp, onlineAccountsSignatures);
generator.getPublicKey(), generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures);
// Resign to update transactions signature
newBlock.sign();
@ -474,86 +486,6 @@ public class Block {
return actualBlockTime;
}
private BigInteger calcGeneratorsTarget(PublicKeyAccount nextBlockGenerator) throws DataException {
// Start with 32-byte maximum integer representing all possible correct "guesses"
// Where a "correct guess" is an integer greater than the threshold represented by calcBlockHash()
byte[] targetBytes = new byte[32];
Arrays.fill(targetBytes, Byte.MAX_VALUE);
BigInteger target = new BigInteger(1, targetBytes);
// Divide by next block's base target
// So if next block requires a higher generating balance then there are fewer remaining "correct guesses"
BigInteger baseTarget = BigInteger.valueOf(calcBaseTarget(calcNextBlockGeneratingBalance()));
target = target.divide(baseTarget);
// If generator is actually proxy account then use forger's account to calculate target.
BigDecimal generatingBalance;
ProxyForgerData proxyForgerData = this.repository.getAccountRepository().getProxyForgeData(nextBlockGenerator.getPublicKey());
if (proxyForgerData != null)
generatingBalance = new PublicKeyAccount(this.repository, proxyForgerData.getForgerPublicKey()).getGeneratingBalance();
else
generatingBalance = nextBlockGenerator.getGeneratingBalance();
// Multiply by account's generating balance
// So the greater the account's generating balance then the greater the remaining "correct guesses"
target = target.multiply(generatingBalance.toBigInteger());
return target;
}
/** Returns pseudo-random, but deterministic, integer for this block (and block's generator for v3+ blocks) */
private BigInteger calcBlockHash() {
byte[] hashData;
if (this.blockData.getVersion() < 3)
hashData = this.blockData.getGeneratorSignature();
else
hashData = Bytes.concat(this.blockData.getReference(), generator.getPublicKey());
// Calculate 32-byte hash as pseudo-random, but deterministic, integer (unique to this generator for v3+ blocks)
byte[] hash = Crypto.digest(hashData);
// Convert hash to BigInteger form
return new BigInteger(1, hash);
}
/** Returns pseudo-random, but deterministic, integer for next block (and next block's generator for v3+ blocks) */
private BigInteger calcNextBlockHash(int nextBlockVersion, byte[] preVersion3GeneratorSignature, PublicKeyAccount nextBlockGenerator) {
byte[] hashData;
if (nextBlockVersion < 3)
hashData = preVersion3GeneratorSignature;
else
hashData = Bytes.concat(this.blockData.getSignature(), nextBlockGenerator.getPublicKey());
// Calculate 32-byte hash as pseudo-random, but deterministic, integer (unique to this generator for v3+ blocks)
byte[] hash = Crypto.digest(hashData);
// Convert hash to BigInteger form
return new BigInteger(1, hash);
}
/** Calculate next block's timestamp, given next block's version, generator signature and generator's public key */
private long calcNextBlockTimestamp(int nextBlockVersion, byte[] nextBlockGeneratorSignature, PublicKeyAccount nextBlockGenerator) throws DataException {
BigInteger hashValue = calcNextBlockHash(nextBlockVersion, nextBlockGeneratorSignature, nextBlockGenerator);
BigInteger target = calcGeneratorsTarget(nextBlockGenerator);
// If target is zero then generator has no balance so return longest value
if (target.compareTo(BigInteger.ZERO) == 0)
return Long.MAX_VALUE;
// Use ratio of "correct guesses" to calculate minimum delay until this generator can forge a block
BigInteger seconds = hashValue.divide(target).add(BigInteger.ONE);
// Calculate next block timestamp using delay
BigInteger timestamp = seconds.multiply(BigInteger.valueOf(1000)).add(BigInteger.valueOf(this.blockData.getTimestamp()));
// Limit timestamp to maximum long value
timestamp = timestamp.min(BigInteger.valueOf(Long.MAX_VALUE));
return timestamp.longValue();
}
/**
* Return block's transactions.
* <p>
@ -787,14 +719,55 @@ public class Block {
}
}
public static byte[] calcIdealGeneratorPublicKey(int parentBlockHeight, byte[] parentBlockSignature) {
return Crypto.digest(Bytes.concat(Longs.toByteArray(parentBlockHeight), parentBlockSignature));
}
public static byte[] calcHeightPerturbedPublicKey(int height, byte[] publicKey) {
return Crypto.digest(Bytes.concat(Longs.toByteArray(height), publicKey));
}
public static BigInteger calcGeneratorDistance(BlockData parentBlockData, byte[] generatorPublicKey) {
final int parentHeight = parentBlockData.getHeight();
final int thisHeight = parentHeight + 1;
// Convert all bits into unsigned BigInteger
BigInteger idealBI = new BigInteger(1, calcIdealGeneratorPublicKey(parentHeight, parentBlockData.getSignature()));
BigInteger generatorBI = new BigInteger(1, calcHeightPerturbedPublicKey(thisHeight, generatorPublicKey));
return idealBI.subtract(generatorBI).abs();
}
/**
* Returns timestamp based on previous block.
* Returns timestamp based on previous block and this block's generator.
* <p>
* For qora-core, we'll using the minimum from BlockChain config.
* Uses same proportion of this block's generator from 'ideal' generator
* with min to max target block periods, added to previous block's timestamp.
* <p>
* Example:<br>
* This block's generator is 20% of max distance from 'ideal' generator.<br>
* Min/Max block periods are 30s and 90s respectively.<br>
* 20% of (90s - 30s) is 12s<br>
* So this block's timestamp is previous block's timestamp + 30s + 12s.
*/
public static long calcTimestamp(BlockData parentBlockData, byte[] generatorPublicKey) {
BigInteger distance = calcGeneratorDistance(parentBlockData, generatorPublicKey);
final int thisHeight = parentBlockData.getHeight() + 1;
BlockTimingByHeight blockTiming = BlockChain.getInstance().getBlockTimingByHeight(thisHeight);
double ratio = new BigDecimal(distance).divide(new BigDecimal(MAX_DISTANCE), 40, RoundingMode.DOWN).doubleValue();
// Use power transform on ratio to spread out smaller values for bigger effect
double transformed = Math.pow(ratio, blockTiming.power);
long timeOffset = Double.valueOf(blockTiming.deviation * 2.0 * transformed).longValue();
return parentBlockData.getTimestamp() + blockTiming.target - blockTiming.deviation + timeOffset;
}
public static long calcMinimumTimestamp(BlockData parentBlockData) {
long minBlockTime = BlockChain.getInstance().getMinBlockTime(); // seconds
return parentBlockData.getTimestamp() + (minBlockTime * 1000L);
final int thisHeight = parentBlockData.getHeight() + 1;
BlockTimingByHeight blockTiming = BlockChain.getInstance().getBlockTimingByHeight(thisHeight);
return parentBlockData.getTimestamp() + blockTiming.target - blockTiming.deviation;
}
/**
@ -856,14 +829,13 @@ public class Block {
if (this.blockData.getTimestamp() - BlockChain.getInstance().getBlockTimestampMargin() > NTP.getTime())
return ValidationResult.TIMESTAMP_IN_FUTURE;
// Legacy gen1 test: check timestamp milliseconds is the same as parent timestamp milliseconds?
if (this.blockData.getTimestamp() % 1000 != parentBlockData.getTimestamp() % 1000)
return ValidationResult.TIMESTAMP_MS_INCORRECT;
// Check timestamp is at least minimum based on parent block
if (this.blockData.getTimestamp() < Block.calcMinimumTimestamp(parentBlockData))
return ValidationResult.TIMESTAMP_TOO_SOON;
// Too early to forge block?
// XXX DISABLED as it doesn't work - but why?
// if (this.blockData.getTimestamp() < Block.calcMinimumTimestamp(parentBlockData))
// return ValidationResult.TIMESTAMP_TOO_SOON;
long expectedTimestamp = calcTimestamp(parentBlockData, this.blockData.getGeneratorPublicKey());
if (this.blockData.getTimestamp() != expectedTimestamp)
return ValidationResult.TIMESTAMP_INCORRECT;
return ValidationResult.OK;
}
@ -875,6 +847,9 @@ public class Block {
// Expand block's online accounts indexes into actual accounts
ConciseSet accountIndexes = BlockTransformer.decodeOnlineAccounts(this.blockData.getEncodedOnlineAccounts());
// We use count of online accounts to validate decoded account indexes
if (accountIndexes.size() != this.blockData.getOnlineAccountsCount())
return ValidationResult.ONLINE_ACCOUNTS_INVALID;
List<ProxyForgerData> expandedAccounts = new ArrayList<>();
@ -1148,28 +1123,21 @@ public class Block {
/** Returns whether block's generator is actually allowed to forge this block. */
protected boolean isGeneratorValidToForge(Block parentBlock) throws DataException {
BlockData parentBlockData = parentBlock.getBlockData();
// Generator must have forging flag enabled
Account generator = new PublicKeyAccount(repository, this.blockData.getGeneratorPublicKey());
if (Forging.canForge(generator))
return true;
BigInteger hashValue = this.calcBlockHash();
// Check whether generator public key could be a proxy forge account
ProxyForgerData proxyForgerData = this.repository.getAccountRepository().getProxyForgeData(this.blockData.getGeneratorPublicKey());
if (proxyForgerData != null) {
Account forger = new PublicKeyAccount(this.repository, proxyForgerData.getForgerPublicKey());
// calcGeneratorsTarget handles proxy forging aspect
BigInteger target = parentBlock.calcGeneratorsTarget(this.generator);
if (Forging.canForge(forger))
return true;
}
// Multiply target by guesses
long guesses = (this.blockData.getTimestamp() - parentBlockData.getTimestamp()) / 1000;
BigInteger lowerTarget = target.multiply(BigInteger.valueOf(guesses - 1));
target = target.multiply(BigInteger.valueOf(guesses));
// Generator's target must exceed block's hashValue threshold
if (hashValue.compareTo(target) >= 0)
return false;
// Odd gen1 comment: "CHECK IF FIRST BLOCK OF USER"
// Each second elapsed allows generator to test a new "target" window against hashValue
if (hashValue.compareTo(lowerTarget) < 0)
return false;
return true;
return false;
}
/**

View File

@ -32,6 +32,7 @@ import org.qora.repository.DataException;
import org.qora.repository.Repository;
import org.qora.repository.RepositoryManager;
import org.qora.settings.Settings;
import org.qora.utils.NTP;
import org.qora.utils.StringLongMapXmlAdapter;
/**
@ -99,6 +100,15 @@ public class BlockChain {
}
List<RewardByHeight> rewardsByHeight;
/** Block times by block height */
public static class BlockTimingByHeight {
public int height;
public long target; // ms
public long deviation; // ms
public double power;
}
List<BlockTimingByHeight> blockTimingsByHeight;
/** Forging right tiers */
public static class ForgingTier {
/** Minimum number of blocks forged before account can enable minting on other accounts. */
@ -329,6 +339,14 @@ public class BlockChain {
return null;
}
public BlockTimingByHeight getBlockTimingByHeight(int ourHeight) {
for (int i = blockTimingsByHeight.size() - 1; i >= 0; --i)
if (blockTimingsByHeight.get(i).height <= ourHeight)
return blockTimingsByHeight.get(i);
throw new IllegalStateException(String.format("No block timing info available for height %d", ourHeight));
}
/** Validate blockchain config read from JSON */
private void validateConfig() {
if (this.genesisInfo == null) {
@ -440,4 +458,25 @@ public class BlockChain {
}
}
public static void trimOldOnlineAccountsSignatures() {
final Long now = NTP.getTime();
if (now == null)
return;
try (final Repository repository = RepositoryManager.tryRepository()) {
if (repository == null)
return;
int numBlocksTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(now - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime());
if (numBlocksTrimmed > 0)
LOGGER.debug(String.format("Trimmed old online accounts signatures from %d block%s", numBlocksTrimmed, (numBlocksTrimmed != 1 ? "s" : "")));
repository.saveChanges();
} catch (DataException e) {
LOGGER.warn("Repository issue trying to trim old online accounts signatures: " + e.getMessage());
return;
}
}
}

View File

@ -70,28 +70,36 @@ public class BlockGenerator extends Thread {
List<Block> newBlocks = new ArrayList<>();
// Flags that allow us to track whether generating is possible changes,
// so we can notify Controller, and further update SysTray, etc.
boolean isGenerationPossible = false;
boolean wasGenerationPossible = isGenerationPossible;
while (running) {
// Sleep for a while
try {
repository.discardChanges(); // Free repository locks, if any
if (isGenerationPossible != wasGenerationPossible)
Controller.getInstance().onGenerationPossibleChange(isGenerationPossible);
wasGenerationPossible = isGenerationPossible;
Thread.sleep(1000);
} catch (InterruptedException e) {
// We've been interrupted - time to exit
return;
}
// If Controller says we can't generate, then don't...
if (!Controller.getInstance().isGenerationAllowed())
isGenerationPossible = false;
final Long now = NTP.getTime();
if (now == null)
continue;
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
if (minLatestBlockTimestamp == null)
continue;
final Long now = NTP.getTime();
if (now == null)
continue;
// No online accounts? (e.g. during startup)
if (Controller.getInstance().getOnlineAccounts().isEmpty())
continue;
@ -121,6 +129,7 @@ public class BlockGenerator extends Thread {
// There are no peers with a recent block and/or our latest block is recent
// so go ahead and generate a block if possible.
isGenerationPossible = true;
// Check blockchain hasn't changed
if (previousBlock == null || !Arrays.equals(previousBlock.getSignature(), lastBlockData.getSignature())) {

View File

@ -106,6 +106,7 @@ public class Controller extends Thread {
// To do with online accounts list
private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000; // ms
private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 1 * 60 * 1000; // ms
private static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000;
private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000);
@ -125,8 +126,8 @@ public class Controller extends Thread {
private long deleteExpiredTimestamp = startTime + DELETE_EXPIRED_INTERVAL; // ms
private long onlineAccountsTasksTimestamp = startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms
/** Whether BlockGenerator is allowed to generate blocks. Mostly determined by system clock accuracy. */
private volatile boolean isGenerationAllowed = false;
/** Whether we can generate new blocks, as reported by BlockGenerator. */
private volatile boolean isGenerationPossible = false;
/** Signature of peer's latest block that will result in no sync action needed (e.g. INFERIOR_CHAIN, NOTHING_TO_DO, OK). */
private byte[] noSyncPeerBlockSignature = null;
@ -227,10 +228,6 @@ public class Controller extends Thread {
return this.blockchainLock;
}
public boolean isGenerationAllowed() {
return this.isGenerationAllowed;
}
// Entry point
public static void main(String args[]) {
@ -243,7 +240,10 @@ public class Controller extends Thread {
Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
// Load/check settings, which potentially sets up blockchain config, etc.
Settings.getInstance();
if (args.length > 0)
Settings.fileInstance(args[0]);
else
Settings.getInstance();
LOGGER.info("Starting NTP");
NTP.start();
@ -373,7 +373,6 @@ public class Controller extends Thread {
ntpCheckTimestamp = now + NTP_PRE_SYNC_CHECK_PERIOD;
}
isGenerationAllowed = ntpTime != null;
requestSysTrayUpdate = true;
}
@ -525,7 +524,7 @@ public class Controller extends Thread {
String connectionsText = Translator.INSTANCE.translate("SysTray", numberOfPeers != 1 ? "CONNECTIONS" : "CONNECTION");
String heightText = Translator.INSTANCE.translate("SysTray", "BLOCK_HEIGHT");
String generatingText = Translator.INSTANCE.translate("SysTray", isGenerationAllowed ? "GENERATING_ENABLED" : "GENERATING_DISABLED");
String generatingText = Translator.INSTANCE.translate("SysTray", isGenerationPossible ? "GENERATING_ENABLED" : "GENERATING_DISABLED");
String tooltip = String.format("%s - %d %s - %s %d", generatingText, numberOfPeers, connectionsText, heightText, height);
SysTray.getInstance().setToolTipText(tooltip);
@ -632,6 +631,11 @@ public class Controller extends Thread {
network.broadcast(peer -> network.buildGetUnconfirmedTransactionsMessage(peer));
}
public void onGenerationPossibleChange(boolean isGenerationPossible) {
this.isGenerationPossible = isGenerationPossible;
requestSysTrayUpdate = true;
}
public void onGeneratedBlock() {
// Broadcast our new height info
BlockData latestBlockData;
@ -646,6 +650,8 @@ public class Controller extends Thread {
Network network = Network.getInstance();
network.broadcast(peer -> network.buildHeightMessage(peer, latestBlockData));
requestSysTrayUpdate = true;
}
public void onNewTransaction(TransactionData transactionData) {
@ -1247,7 +1253,9 @@ public class Controller extends Thread {
}
private void performOnlineAccountsTasks() {
final long now = System.currentTimeMillis();
final Long now = NTP.getTime();
if (now == null)
return;
// Expire old entries
final long cutoffThreshold = now - LAST_SEEN_EXPIRY_PERIOD;
@ -1267,15 +1275,20 @@ public class Controller extends Thread {
}
}
// Request data from another peer
Message message;
synchronized (this.onlineAccounts) {
message = new GetOnlineAccountsMessage(this.onlineAccounts);
// Request data from other peers?
if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) {
Message message;
synchronized (this.onlineAccounts) {
message = new GetOnlineAccountsMessage(this.onlineAccounts);
}
Network.getInstance().broadcast((peer) -> message);
}
Network.getInstance().broadcast((peer) -> message);
// Refresh our onlineness?
// Refresh our online accounts signatures?
sendOurOnlineAccountsInfo();
// Trim blockchain by removing 'old' online accounts signatures
BlockChain.trimOldOnlineAccountsSignatures();
}
private void sendOurOnlineAccountsInfo() {
@ -1304,7 +1317,6 @@ public class Controller extends Thread {
return;
}
// 'current' timestamp
final long onlineAccountsTimestamp = Controller.toOnlineAccountTimestamp(now);
boolean hasInfoChanged = false;
@ -1351,7 +1363,7 @@ public class Controller extends Thread {
Message message = new OnlineAccountsMessage(ourOnlineAccounts);
Network.getInstance().broadcast((peer) -> message);
LOGGER.trace(( )-> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp));
LOGGER.trace(()-> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp));
}
public static long toOnlineAccountTimestamp(long timestamp) {

View File

@ -99,7 +99,6 @@ public class Synchronizer {
if (peerHeight == 1)
return SynchronizationResult.GENESIS_ONLY;
// XXX this may well be obsolete now
// If peer is too far behind us then don't them.
int minHeight = ourHeight - MAXIMUM_HEIGHT_DELTA;
if (!force && peerHeight < minHeight) {
@ -136,7 +135,6 @@ public class Synchronizer {
peerHeight = commonBlockHeight;
}
// XXX This may well be obsolete now
// If common block is peer's latest block then we simply have the same, or longer, chain to peer, so exit now
if (commonBlockHeight == peerHeight) {
if (peerHeight == ourHeight)
@ -154,7 +152,7 @@ public class Synchronizer {
return SynchronizationResult.TOO_DIVERGENT;
}
// If we have blocks after common block then decide whether we want to sync (lowest block signature wins)
// If we both have blocks after common block then decide whether we want to sync
int highestMutualHeight = Math.min(peerHeight, ourHeight);
// If our latest block is very old, we're very behind and should ditch our fork.

View File

@ -33,6 +33,7 @@ public class BlockData implements Serializable {
private int atCount;
private BigDecimal atFees;
private byte[] encodedOnlineAccounts;
private int onlineAccountsCount;
private Long onlineAccountsTimestamp;
private byte[] onlineAccountsSignatures;
@ -44,7 +45,7 @@ public class BlockData implements Serializable {
public BlockData(int version, byte[] reference, int transactionCount, BigDecimal totalFees, byte[] transactionsSignature, Integer height, long timestamp,
BigDecimal generatingBalance, byte[] generatorPublicKey, byte[] generatorSignature, int atCount, BigDecimal atFees,
byte[] encodedOnlineAccounts, Long onlineAccountsTimestamp, byte[] onlineAccountsSignatures) {
byte[] encodedOnlineAccounts, int onlineAccountsCount, Long onlineAccountsTimestamp, byte[] onlineAccountsSignatures) {
this.version = version;
this.reference = reference;
this.transactionCount = transactionCount;
@ -58,6 +59,7 @@ public class BlockData implements Serializable {
this.atCount = atCount;
this.atFees = atFees;
this.encodedOnlineAccounts = encodedOnlineAccounts;
this.onlineAccountsCount = onlineAccountsCount;
this.onlineAccountsTimestamp = onlineAccountsTimestamp;
this.onlineAccountsSignatures = onlineAccountsSignatures;
@ -70,7 +72,7 @@ public class BlockData implements Serializable {
public BlockData(int version, byte[] reference, int transactionCount, BigDecimal totalFees, byte[] transactionsSignature, Integer height, long timestamp,
BigDecimal generatingBalance, byte[] generatorPublicKey, byte[] generatorSignature, int atCount, BigDecimal atFees) {
this(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance, generatorPublicKey, generatorSignature, atCount, atFees,
null, null, null);
null, 0, null, null);
}
// Getters/setters
@ -167,8 +169,12 @@ public class BlockData implements Serializable {
return this.encodedOnlineAccounts;
}
public int getOnlineAccountsCount() {
return this.onlineAccountsCount;
}
public Long getOnlineAccountsTimestamp() {
return onlineAccountsTimestamp;
return this.onlineAccountsTimestamp;
}
public void setOnlineAccountsTimestamp(Long onlineAccountsTimestamp) {

View File

@ -1,21 +1,34 @@
package org.qora.data.block;
import org.qora.transform.block.BlockTransformer;
public class BlockSummaryData {
// Properties
private int height;
private byte[] signature;
private byte[] generatorPublicKey;
private int onlineAccountsCount;
// Constructors
public BlockSummaryData(int height, byte[] signature, byte[] generatorPublicKey) {
public BlockSummaryData(int height, byte[] signature, byte[] generatorPublicKey, int onlineAccountsCount) {
this.height = height;
this.signature = signature;
this.generatorPublicKey = generatorPublicKey;
this.onlineAccountsCount = onlineAccountsCount;
}
public BlockSummaryData(BlockData blockData) {
this(blockData.getHeight(), blockData.getSignature(), blockData.getGeneratorPublicKey());
this.height = blockData.getHeight();
this.signature = blockData.getSignature();
this.generatorPublicKey = blockData.getGeneratorPublicKey();
byte[] encodedOnlineAccounts = blockData.getEncodedOnlineAccounts();
if (encodedOnlineAccounts != null) {
this.onlineAccountsCount = BlockTransformer.decodeOnlineAccounts(encodedOnlineAccounts).size();
} else {
this.onlineAccountsCount = 0;
}
}
// Getters / setters
@ -32,4 +45,8 @@ public class BlockSummaryData {
return this.generatorPublicKey;
}
public int getOnlineAccountsCount() {
return this.onlineAccountsCount;
}
}

View File

@ -15,7 +15,7 @@ import com.google.common.primitives.Ints;
public class BlockSummariesMessage extends Message {
private static final int BLOCK_SUMMARY_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH + Transformer.INT_LENGTH + Transformer.PUBLIC_KEY_LENGTH;
private static final int BLOCK_SUMMARY_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH + Transformer.INT_LENGTH + Transformer.PUBLIC_KEY_LENGTH + Transformer.INT_LENGTH;
private List<BlockSummaryData> blockSummaries;
@ -49,7 +49,9 @@ public class BlockSummariesMessage extends Message {
byte[] generatorPublicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(generatorPublicKey);
BlockSummaryData blockSummary = new BlockSummaryData(height, signature, generatorPublicKey);
int onlineAccountsCount = bytes.getInt();
BlockSummaryData blockSummary = new BlockSummaryData(height, signature, generatorPublicKey, onlineAccountsCount);
blockSummaries.add(blockSummary);
}
@ -67,6 +69,7 @@ public class BlockSummariesMessage extends Message {
bytes.write(Ints.toByteArray(blockSummary.getHeight()));
bytes.write(blockSummary.getSignature());
bytes.write(blockSummary.getGeneratorPublicKey());
bytes.write(Ints.toByteArray(blockSummary.getOnlineAccountsCount()));
}
return bytes.toByteArray();

View File

@ -123,6 +123,14 @@ public interface BlockRepository {
*/
public List<BlockSummaryData> getBlockSummaries(int firstBlockHeight, int lastBlockHeight) throws DataException;
/**
* Trim online accounts signatures from blocks older than passed timestamp.
*
* @param timestamp
* @return number of blocks trimmed
*/
public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException;
/**
* Saves block into repository.
*

View File

@ -23,7 +23,7 @@ public class HSQLDBBlockRepository implements BlockRepository {
private static final String BLOCK_DB_COLUMNS = "version, reference, transaction_count, total_fees, "
+ "transactions_signature, height, generation, generating_balance, generator, generator_signature, "
+ "AT_count, AT_fees, online_accounts, online_accounts_timestamp, online_accounts_signatures";
+ "AT_count, AT_fees, online_accounts, online_accounts_count, online_accounts_timestamp, online_accounts_signatures";
protected HSQLDBRepository repository;
@ -49,11 +49,13 @@ public class HSQLDBBlockRepository implements BlockRepository {
int atCount = resultSet.getInt(11);
BigDecimal atFees = resultSet.getBigDecimal(12);
byte[] encodedOnlineAccounts = resultSet.getBytes(13);
Long onlineAccountsTimestamp = getZonedTimestampMilli(resultSet, 14);
byte[] onlineAccountsSignatures = resultSet.getBytes(15);
int onlineAccountsCount = resultSet.getInt(14);
Long onlineAccountsTimestamp = getZonedTimestampMilli(resultSet, 15);
byte[] onlineAccountsSignatures = resultSet.getBytes(16);
return new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance,
generatorPublicKey, generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsTimestamp, onlineAccountsSignatures);
generatorPublicKey, generatorSignature, atCount, atFees,
encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures);
} catch (SQLException e) {
throw new DataException("Error extracting data from result set", e);
}
@ -288,7 +290,7 @@ public class HSQLDBBlockRepository implements BlockRepository {
@Override
public List<BlockSummaryData> getBlockSummaries(int firstBlockHeight, int lastBlockHeight) throws DataException {
String sql = "SELECT signature, height, generator FROM Blocks WHERE height BETWEEN ? AND ?";
String sql = "SELECT signature, height, generator, online_accounts_count FROM Blocks WHERE height BETWEEN ? AND ?";
List<BlockSummaryData> blockSummaries = new ArrayList<>();
@ -300,8 +302,9 @@ public class HSQLDBBlockRepository implements BlockRepository {
byte[] signature = resultSet.getBytes(1);
int height = resultSet.getInt(2);
byte[] generatorPublicKey = resultSet.getBytes(3);
int onlineAccountsCount = resultSet.getInt(4);
BlockSummaryData blockSummary = new BlockSummaryData(height, signature, generatorPublicKey);
BlockSummaryData blockSummary = new BlockSummaryData(height, signature, generatorPublicKey, onlineAccountsCount);
blockSummaries.add(blockSummary);
} while (resultSet.next());
@ -311,6 +314,17 @@ public class HSQLDBBlockRepository implements BlockRepository {
}
}
@Override
public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException {
String sql = "UPDATE Blocks set online_accounts_signatures = NULL WHERE generation < ? AND online_accounts_signatures IS NOT NULL";
try {
return this.repository.checkedExecuteUpdateCount(sql, toOffsetDateTime(timestamp));
} catch (SQLException e) {
throw new DataException("Unable to trim old online accounts signatures in repository", e);
}
}
@Override
public void save(BlockData blockData) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("Blocks");
@ -321,7 +335,7 @@ public class HSQLDBBlockRepository implements BlockRepository {
.bind("generation", toOffsetDateTime(blockData.getTimestamp())).bind("generating_balance", blockData.getGeneratingBalance())
.bind("generator", blockData.getGeneratorPublicKey()).bind("generator_signature", blockData.getGeneratorSignature())
.bind("AT_count", blockData.getATCount()).bind("AT_fees", blockData.getATFees())
.bind("online_accounts", blockData.getEncodedOnlineAccounts())
.bind("online_accounts", blockData.getEncodedOnlineAccounts()).bind("online_accounts_count", blockData.getOnlineAccountsCount())
.bind("online_accounts_timestamp", toOffsetDateTime(blockData.getOnlineAccountsTimestamp()))
.bind("online_accounts_signatures", blockData.getOnlineAccountsSignatures());

View File

@ -787,6 +787,7 @@ public class HSQLDBDatabaseUpdates {
case 55:
// Storage of which level 1+ accounts were 'online' for a particular block. Used to distribute block rewards.
stmt.execute("ALTER TABLE Blocks ADD COLUMN online_accounts VARBINARY(1048576)");
stmt.execute("ALTER TABLE Blocks ADD COLUMN online_accounts_count INT NOT NULL DEFAULT 0");
stmt.execute("ALTER TABLE Blocks ADD COLUMN online_accounts_timestamp TIMESTAMP WITH TIME ZONE");
stmt.execute("ALTER TABLE Blocks ADD COLUMN online_accounts_signatures BLOB");
break;

View File

@ -460,17 +460,19 @@ public class HSQLDBRepository implements Repository {
* @return number of changed rows
* @throws SQLException
*/
private int checkedExecuteUpdateCount(PreparedStatement preparedStatement, Object... objects) throws SQLException {
prepareExecute(preparedStatement, objects);
/* package */ int checkedExecuteUpdateCount(String sql, Object... objects) throws SQLException {
try (PreparedStatement preparedStatement = this.prepareStatement(sql)) {
prepareExecute(preparedStatement, objects);
if (preparedStatement.execute())
throw new SQLException("Database produced results, not row count");
if (preparedStatement.execute())
throw new SQLException("Database produced results, not row count");
int rowCount = preparedStatement.getUpdateCount();
if (rowCount == -1)
throw new SQLException("Database returned invalid row count");
int rowCount = preparedStatement.getUpdateCount();
if (rowCount == -1)
throw new SQLException("Database returned invalid row count");
return rowCount;
return rowCount;
}
}
/**
@ -543,9 +545,7 @@ public class HSQLDBRepository implements Repository {
sql.append(" WHERE ");
sql.append(whereClause);
try (PreparedStatement preparedStatement = this.prepareStatement(sql.toString())) {
return this.checkedExecuteUpdateCount(preparedStatement, objects);
}
return this.checkedExecuteUpdateCount(sql.toString(), objects);
}
/**
@ -559,9 +559,7 @@ public class HSQLDBRepository implements Repository {
sql.append("DELETE FROM ");
sql.append(tableName);
try (PreparedStatement preparedStatement = this.prepareStatement(sql.toString())) {
return this.checkedExecuteUpdateCount(preparedStatement);
}
return this.checkedExecuteUpdateCount(sql.toString());
}
/**

View File

@ -49,6 +49,7 @@ public class BlockTransformer extends Transformer {
protected static final int AT_FEES_LENGTH = LONG_LENGTH;
protected static final int AT_LENGTH = AT_FEES_LENGTH + AT_BYTES_LENGTH;
protected static final int ONLINE_ACCOUNTS_COUNT_LENGTH = INT_LENGTH;
protected static final int ONLINE_ACCOUNTS_SIZE_LENGTH = INT_LENGTH;
protected static final int ONLINE_ACCOUNTS_TIMESTAMP_LENGTH = LONG_LENGTH;
protected static final int ONLINE_ACCOUNTS_SIGNATURES_COUNT_LENGTH = INT_LENGTH;
@ -194,11 +195,14 @@ public class BlockTransformer extends Transformer {
}
// Online accounts info?
byte[] onlineAccounts = null;
byte[] encodedOnlineAccounts = null;
int onlineAccountsCount = 0;
byte[] onlineAccountsSignatures = null;
Long onlineAccountsTimestamp = null;
if (version >= 4) {
onlineAccountsCount = byteBuffer.getInt();
int conciseSetLength = byteBuffer.getInt();
if (conciseSetLength > Block.MAX_BLOCK_BYTES)
@ -207,8 +211,13 @@ public class BlockTransformer extends Transformer {
if ((conciseSetLength & 3) != 0)
throw new TransformationException("Byte data length not multiple of 4 for online account info");
onlineAccounts = new byte[conciseSetLength];
byteBuffer.get(onlineAccounts);
encodedOnlineAccounts = new byte[conciseSetLength];
byteBuffer.get(encodedOnlineAccounts);
// Try to decode to ConciseSet
ConciseSet accountsIndexes = BlockTransformer.decodeOnlineAccounts(encodedOnlineAccounts);
if (accountsIndexes.size() != onlineAccountsCount)
throw new TransformationException("Block's online account data malformed");
// Note: number of signatures, not byte length
int onlineAccountsSignaturesCount = byteBuffer.getInt();
@ -228,7 +237,7 @@ public class BlockTransformer extends Transformer {
// We don't have a height!
Integer height = null;
BlockData blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance,
generatorPublicKey, generatorSignature, atCount, atFees, onlineAccounts, onlineAccountsTimestamp, onlineAccountsSignatures);
generatorPublicKey, generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures);
return new Triple<BlockData, List<TransactionData>, List<ATStateData>>(blockData, transactions, atStates);
}
@ -239,9 +248,11 @@ public class BlockTransformer extends Transformer {
if (blockData.getVersion() >= 4) {
blockLength += AT_BYTES_LENGTH + blockData.getATCount() * V4_AT_ENTRY_LENGTH;
blockLength += ONLINE_ACCOUNTS_SIZE_LENGTH + blockData.getEncodedOnlineAccounts().length;
blockLength += ONLINE_ACCOUNTS_COUNT_LENGTH + ONLINE_ACCOUNTS_SIZE_LENGTH + blockData.getEncodedOnlineAccounts().length;
blockLength += ONLINE_ACCOUNTS_SIGNATURES_COUNT_LENGTH;
if (blockData.getOnlineAccountsSignatures().length > 0)
byte[] onlineAccountsSignatures = blockData.getOnlineAccountsSignatures();
if (onlineAccountsSignatures != null && onlineAccountsSignatures.length > 0)
blockLength += ONLINE_ACCOUNTS_TIMESTAMP_LENGTH + blockData.getOnlineAccountsSignatures().length;
} else if (blockData.getVersion() >= 2)
blockLength += AT_FEES_LENGTH + AT_BYTES_LENGTH + blockData.getATCount() * V2_AT_ENTRY_LENGTH;
@ -315,25 +326,27 @@ public class BlockTransformer extends Transformer {
byte[] encodedOnlineAccounts = blockData.getEncodedOnlineAccounts();
if (encodedOnlineAccounts != null) {
bytes.write(Ints.toByteArray(blockData.getOnlineAccountsCount()));
bytes.write(Ints.toByteArray(encodedOnlineAccounts.length));
bytes.write(encodedOnlineAccounts);
} else {
bytes.write(Ints.toByteArray(0));
bytes.write(Ints.toByteArray(0)); // onlineAccountsCount
bytes.write(Ints.toByteArray(0)); // encodedOnlineAccounts length
}
byte[] onlineAccountsSignatures = blockData.getOnlineAccountsSignatures();
if (onlineAccountsSignatures != null) {
if (onlineAccountsSignatures != null && onlineAccountsSignatures.length > 0) {
// Note: we write the number of signatures, not the number of bytes
bytes.write(Ints.toByteArray(onlineAccountsSignatures.length / Transformer.SIGNATURE_LENGTH));
if (onlineAccountsSignatures.length > 0) {
// Only write online accounts timestamp if we have signatures
bytes.write(Longs.toByteArray(blockData.getOnlineAccountsTimestamp()));
// We only write online accounts timestamp if we have signatures
bytes.write(Longs.toByteArray(blockData.getOnlineAccountsTimestamp()));
bytes.write(onlineAccountsSignatures);
}
bytes.write(onlineAccountsSignatures);
} else {
// Zero online accounts signatures (timestamp omitted also)
bytes.write(Ints.toByteArray(0));
}
}

View File

@ -15,7 +15,7 @@
"onlineAccountSignaturesMaxLifetime": 3196800000,
"genesisInfo": {
"version": 4,
"timestamp": "1568360000000",
"timestamp": "1568720000000",
"generatingBalance": "100000",
"transactions": [
{ "type": "ISSUE_ASSET", "owner": "QUwGVHPPxJNJ2dq95abQNe79EyBN2K26zM", "assetName": "QORT", "description": "QORTAL coin", "quantity": 10000000, "isDivisible": true, "fee": 0, "reference": "28u54WRcMfGujtQMZ9dNKFXVqucY7XfPihXAqPFsnx853NPUwfDJy1sMH5boCkahFgjUNYqc5fkduxdBhQTKgUsC", "data": "{}" },
@ -48,6 +48,9 @@
{ "minBlocks": 50, "maxSubAccounts": 1 },
{ "minBlocks": 0, "maxSubAccounts": 0 }
],
"blockTimingsByHeight": [
{ "height": 1, "target": 60000, "deviation": 30000, "power": 0.2 }
],
"featureTriggers": {
"messageHeight": 0,
"atHeight": 0,