diff --git a/src/main/java/org/qora/block/Block.java b/src/main/java/org/qora/block/Block.java index aed0577a..d050c969 100644 --- a/src/main/java/org/qora/block/Block.java +++ b/src/main/java/org/qora/block/Block.java @@ -16,11 +16,13 @@ import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qora.account.Account; +import org.qora.account.Forging; import org.qora.account.PrivateKeyAccount; import org.qora.account.PublicKeyAccount; import org.qora.asset.Asset; import org.qora.at.AT; import org.qora.block.BlockChain; +import org.qora.block.BlockChain.BlockTimingByHeight; import org.qora.controller.Controller; import org.qora.crypto.Crypto; import org.qora.data.account.ProxyForgerData; @@ -99,7 +101,7 @@ public class Block { TRANSACTION_ALREADY_PROCESSED(54), TRANSACTION_NEEDS_APPROVAL(55), AT_STATES_MISMATCH(61), - ONLINE_ACCOUNT_LEVEL_ZERO(70), + ONLINE_ACCOUNTS_INVALID(70), ONLINE_ACCOUNT_UNKNOWN(71), ONLINE_ACCOUNT_SIGNATURES_MISSING(72), ONLINE_ACCOUNT_SIGNATURES_MALFORMED(73), @@ -147,6 +149,13 @@ public class Block { // TODO push this out to blockchain config file public static final int MAX_BLOCK_BYTES = 1048576; + private static final BigInteger MAX_DISTANCE; + static { + byte[] maxValue = new byte[Transformer.PUBLIC_KEY_LENGTH]; + Arrays.fill(maxValue, (byte) 0xFF); + MAX_DISTANCE = new BigInteger(1, maxValue); + } + public static final ConciseSet EMPTY_ONLINE_ACCOUNTS = new ConciseSet(); // Constructors @@ -251,10 +260,11 @@ public class Block { ConciseSet onlineAccountsSet = new ConciseSet(); onlineAccountsSet = onlineAccountsSet.convert(accountIndexes); byte[] encodedOnlineAccounts = BlockTransformer.encodeOnlineAccounts(onlineAccountsSet); + int onlineAccountsCount = onlineAccountsSet.size(); // Concatenate online account timestamp signatures (in correct order) - byte[] onlineAccountsSignatures = new byte[accountIndexes.size() * Transformer.SIGNATURE_LENGTH]; - for (int i = 0; i < accountIndexes.size(); ++i) { + byte[] onlineAccountsSignatures = new byte[onlineAccountsCount * Transformer.SIGNATURE_LENGTH]; + for (int i = 0; i < onlineAccountsCount; ++i) { Integer accountIndex = accountIndexes.get(i); OnlineAccount onlineAccount = indexedOnlineAccounts.get(accountIndex); System.arraycopy(onlineAccount.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH); @@ -268,7 +278,7 @@ public class Block { throw new DataException("Unable to calculate next block generator signature", e); } - long timestamp = parentBlock.calcNextBlockTimestamp(version, generatorSignature, generator); + long timestamp = calcTimestamp(parentBlockData, generator.getPublicKey()); int transactionCount = 0; byte[] transactionsSignature = null; @@ -282,7 +292,8 @@ public class Block { // This instance used for AT processing this.blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance, - generator.getPublicKey(), generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsTimestamp, onlineAccountsSignatures); + generator.getPublicKey(), generatorSignature, atCount, atFees, + encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures); // Requires this.blockData and this.transactions, sets this.ourAtStates and this.ourAtFees this.executeATs(); @@ -294,7 +305,8 @@ public class Block { // Rebuild blockData using post-AT-execute data this.blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance, - generator.getPublicKey(), generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsTimestamp, onlineAccountsSignatures); + generator.getPublicKey(), generatorSignature, atCount, atFees, + encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures); } /** @@ -311,7 +323,6 @@ public class Block { newBlock.generator = generator; BlockData parentBlockData = this.getParent(); - Block parentBlock = new Block(repository, parentBlockData); // Copy AT state data newBlock.ourAtStates = this.ourAtStates; @@ -331,7 +342,7 @@ public class Block { throw new DataException("Unable to calculate next block generator signature", e); } - long timestamp = parentBlock.calcNextBlockTimestamp(version, generatorSignature, generator); + long timestamp = calcTimestamp(parentBlockData, generator.getPublicKey()); newBlock.transactions = this.transactions; int transactionCount = this.blockData.getTransactionCount(); @@ -343,11 +354,12 @@ public class Block { BigDecimal atFees = newBlock.ourAtFees; byte[] encodedOnlineAccounts = this.blockData.getEncodedOnlineAccounts(); + int onlineAccountsCount = this.blockData.getOnlineAccountsCount(); Long onlineAccountsTimestamp = this.blockData.getOnlineAccountsTimestamp(); byte[] onlineAccountsSignatures = this.blockData.getOnlineAccountsSignatures(); newBlock.blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance, - generator.getPublicKey(), generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsTimestamp, onlineAccountsSignatures); + generator.getPublicKey(), generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures); // Resign to update transactions signature newBlock.sign(); @@ -474,86 +486,6 @@ public class Block { return actualBlockTime; } - private BigInteger calcGeneratorsTarget(PublicKeyAccount nextBlockGenerator) throws DataException { - // Start with 32-byte maximum integer representing all possible correct "guesses" - // Where a "correct guess" is an integer greater than the threshold represented by calcBlockHash() - byte[] targetBytes = new byte[32]; - Arrays.fill(targetBytes, Byte.MAX_VALUE); - BigInteger target = new BigInteger(1, targetBytes); - - // Divide by next block's base target - // So if next block requires a higher generating balance then there are fewer remaining "correct guesses" - BigInteger baseTarget = BigInteger.valueOf(calcBaseTarget(calcNextBlockGeneratingBalance())); - target = target.divide(baseTarget); - - // If generator is actually proxy account then use forger's account to calculate target. - BigDecimal generatingBalance; - ProxyForgerData proxyForgerData = this.repository.getAccountRepository().getProxyForgeData(nextBlockGenerator.getPublicKey()); - if (proxyForgerData != null) - generatingBalance = new PublicKeyAccount(this.repository, proxyForgerData.getForgerPublicKey()).getGeneratingBalance(); - else - generatingBalance = nextBlockGenerator.getGeneratingBalance(); - - // Multiply by account's generating balance - // So the greater the account's generating balance then the greater the remaining "correct guesses" - target = target.multiply(generatingBalance.toBigInteger()); - - return target; - } - - /** Returns pseudo-random, but deterministic, integer for this block (and block's generator for v3+ blocks) */ - private BigInteger calcBlockHash() { - byte[] hashData; - - if (this.blockData.getVersion() < 3) - hashData = this.blockData.getGeneratorSignature(); - else - hashData = Bytes.concat(this.blockData.getReference(), generator.getPublicKey()); - - // Calculate 32-byte hash as pseudo-random, but deterministic, integer (unique to this generator for v3+ blocks) - byte[] hash = Crypto.digest(hashData); - - // Convert hash to BigInteger form - return new BigInteger(1, hash); - } - - /** Returns pseudo-random, but deterministic, integer for next block (and next block's generator for v3+ blocks) */ - private BigInteger calcNextBlockHash(int nextBlockVersion, byte[] preVersion3GeneratorSignature, PublicKeyAccount nextBlockGenerator) { - byte[] hashData; - - if (nextBlockVersion < 3) - hashData = preVersion3GeneratorSignature; - else - hashData = Bytes.concat(this.blockData.getSignature(), nextBlockGenerator.getPublicKey()); - - // Calculate 32-byte hash as pseudo-random, but deterministic, integer (unique to this generator for v3+ blocks) - byte[] hash = Crypto.digest(hashData); - - // Convert hash to BigInteger form - return new BigInteger(1, hash); - } - - /** Calculate next block's timestamp, given next block's version, generator signature and generator's public key */ - private long calcNextBlockTimestamp(int nextBlockVersion, byte[] nextBlockGeneratorSignature, PublicKeyAccount nextBlockGenerator) throws DataException { - BigInteger hashValue = calcNextBlockHash(nextBlockVersion, nextBlockGeneratorSignature, nextBlockGenerator); - BigInteger target = calcGeneratorsTarget(nextBlockGenerator); - - // If target is zero then generator has no balance so return longest value - if (target.compareTo(BigInteger.ZERO) == 0) - return Long.MAX_VALUE; - - // Use ratio of "correct guesses" to calculate minimum delay until this generator can forge a block - BigInteger seconds = hashValue.divide(target).add(BigInteger.ONE); - - // Calculate next block timestamp using delay - BigInteger timestamp = seconds.multiply(BigInteger.valueOf(1000)).add(BigInteger.valueOf(this.blockData.getTimestamp())); - - // Limit timestamp to maximum long value - timestamp = timestamp.min(BigInteger.valueOf(Long.MAX_VALUE)); - - return timestamp.longValue(); - } - /** * Return block's transactions. *

@@ -787,14 +719,55 @@ public class Block { } } + public static byte[] calcIdealGeneratorPublicKey(int parentBlockHeight, byte[] parentBlockSignature) { + return Crypto.digest(Bytes.concat(Longs.toByteArray(parentBlockHeight), parentBlockSignature)); + } + + public static byte[] calcHeightPerturbedPublicKey(int height, byte[] publicKey) { + return Crypto.digest(Bytes.concat(Longs.toByteArray(height), publicKey)); + } + + public static BigInteger calcGeneratorDistance(BlockData parentBlockData, byte[] generatorPublicKey) { + final int parentHeight = parentBlockData.getHeight(); + final int thisHeight = parentHeight + 1; + + // Convert all bits into unsigned BigInteger + BigInteger idealBI = new BigInteger(1, calcIdealGeneratorPublicKey(parentHeight, parentBlockData.getSignature())); + BigInteger generatorBI = new BigInteger(1, calcHeightPerturbedPublicKey(thisHeight, generatorPublicKey)); + return idealBI.subtract(generatorBI).abs(); + } + /** - * Returns timestamp based on previous block. + * Returns timestamp based on previous block and this block's generator. *

- * For qora-core, we'll using the minimum from BlockChain config. + * Uses same proportion of this block's generator from 'ideal' generator + * with min to max target block periods, added to previous block's timestamp. + *

+ * Example:
+ * This block's generator is 20% of max distance from 'ideal' generator.
+ * Min/Max block periods are 30s and 90s respectively.
+ * 20% of (90s - 30s) is 12s
+ * So this block's timestamp is previous block's timestamp + 30s + 12s. */ + public static long calcTimestamp(BlockData parentBlockData, byte[] generatorPublicKey) { + BigInteger distance = calcGeneratorDistance(parentBlockData, generatorPublicKey); + final int thisHeight = parentBlockData.getHeight() + 1; + BlockTimingByHeight blockTiming = BlockChain.getInstance().getBlockTimingByHeight(thisHeight); + + double ratio = new BigDecimal(distance).divide(new BigDecimal(MAX_DISTANCE), 40, RoundingMode.DOWN).doubleValue(); + + // Use power transform on ratio to spread out smaller values for bigger effect + double transformed = Math.pow(ratio, blockTiming.power); + + long timeOffset = Double.valueOf(blockTiming.deviation * 2.0 * transformed).longValue(); + + return parentBlockData.getTimestamp() + blockTiming.target - blockTiming.deviation + timeOffset; + } + public static long calcMinimumTimestamp(BlockData parentBlockData) { - long minBlockTime = BlockChain.getInstance().getMinBlockTime(); // seconds - return parentBlockData.getTimestamp() + (minBlockTime * 1000L); + final int thisHeight = parentBlockData.getHeight() + 1; + BlockTimingByHeight blockTiming = BlockChain.getInstance().getBlockTimingByHeight(thisHeight); + return parentBlockData.getTimestamp() + blockTiming.target - blockTiming.deviation; } /** @@ -856,14 +829,13 @@ public class Block { if (this.blockData.getTimestamp() - BlockChain.getInstance().getBlockTimestampMargin() > NTP.getTime()) return ValidationResult.TIMESTAMP_IN_FUTURE; - // Legacy gen1 test: check timestamp milliseconds is the same as parent timestamp milliseconds? - if (this.blockData.getTimestamp() % 1000 != parentBlockData.getTimestamp() % 1000) - return ValidationResult.TIMESTAMP_MS_INCORRECT; + // Check timestamp is at least minimum based on parent block + if (this.blockData.getTimestamp() < Block.calcMinimumTimestamp(parentBlockData)) + return ValidationResult.TIMESTAMP_TOO_SOON; - // Too early to forge block? - // XXX DISABLED as it doesn't work - but why? - // if (this.blockData.getTimestamp() < Block.calcMinimumTimestamp(parentBlockData)) - // return ValidationResult.TIMESTAMP_TOO_SOON; + long expectedTimestamp = calcTimestamp(parentBlockData, this.blockData.getGeneratorPublicKey()); + if (this.blockData.getTimestamp() != expectedTimestamp) + return ValidationResult.TIMESTAMP_INCORRECT; return ValidationResult.OK; } @@ -875,6 +847,9 @@ public class Block { // Expand block's online accounts indexes into actual accounts ConciseSet accountIndexes = BlockTransformer.decodeOnlineAccounts(this.blockData.getEncodedOnlineAccounts()); + // We use count of online accounts to validate decoded account indexes + if (accountIndexes.size() != this.blockData.getOnlineAccountsCount()) + return ValidationResult.ONLINE_ACCOUNTS_INVALID; List expandedAccounts = new ArrayList<>(); @@ -1148,28 +1123,21 @@ public class Block { /** Returns whether block's generator is actually allowed to forge this block. */ protected boolean isGeneratorValidToForge(Block parentBlock) throws DataException { - BlockData parentBlockData = parentBlock.getBlockData(); + // Generator must have forging flag enabled + Account generator = new PublicKeyAccount(repository, this.blockData.getGeneratorPublicKey()); + if (Forging.canForge(generator)) + return true; - BigInteger hashValue = this.calcBlockHash(); + // Check whether generator public key could be a proxy forge account + ProxyForgerData proxyForgerData = this.repository.getAccountRepository().getProxyForgeData(this.blockData.getGeneratorPublicKey()); + if (proxyForgerData != null) { + Account forger = new PublicKeyAccount(this.repository, proxyForgerData.getForgerPublicKey()); - // calcGeneratorsTarget handles proxy forging aspect - BigInteger target = parentBlock.calcGeneratorsTarget(this.generator); + if (Forging.canForge(forger)) + return true; + } - // Multiply target by guesses - long guesses = (this.blockData.getTimestamp() - parentBlockData.getTimestamp()) / 1000; - BigInteger lowerTarget = target.multiply(BigInteger.valueOf(guesses - 1)); - target = target.multiply(BigInteger.valueOf(guesses)); - - // Generator's target must exceed block's hashValue threshold - if (hashValue.compareTo(target) >= 0) - return false; - - // Odd gen1 comment: "CHECK IF FIRST BLOCK OF USER" - // Each second elapsed allows generator to test a new "target" window against hashValue - if (hashValue.compareTo(lowerTarget) < 0) - return false; - - return true; + return false; } /** diff --git a/src/main/java/org/qora/block/BlockChain.java b/src/main/java/org/qora/block/BlockChain.java index 584cbdd4..2e2daff7 100644 --- a/src/main/java/org/qora/block/BlockChain.java +++ b/src/main/java/org/qora/block/BlockChain.java @@ -32,6 +32,7 @@ import org.qora.repository.DataException; import org.qora.repository.Repository; import org.qora.repository.RepositoryManager; import org.qora.settings.Settings; +import org.qora.utils.NTP; import org.qora.utils.StringLongMapXmlAdapter; /** @@ -99,6 +100,15 @@ public class BlockChain { } List rewardsByHeight; + /** Block times by block height */ + public static class BlockTimingByHeight { + public int height; + public long target; // ms + public long deviation; // ms + public double power; + } + List blockTimingsByHeight; + /** Forging right tiers */ public static class ForgingTier { /** Minimum number of blocks forged before account can enable minting on other accounts. */ @@ -329,6 +339,14 @@ public class BlockChain { return null; } + public BlockTimingByHeight getBlockTimingByHeight(int ourHeight) { + for (int i = blockTimingsByHeight.size() - 1; i >= 0; --i) + if (blockTimingsByHeight.get(i).height <= ourHeight) + return blockTimingsByHeight.get(i); + + throw new IllegalStateException(String.format("No block timing info available for height %d", ourHeight)); + } + /** Validate blockchain config read from JSON */ private void validateConfig() { if (this.genesisInfo == null) { @@ -440,4 +458,25 @@ public class BlockChain { } } + public static void trimOldOnlineAccountsSignatures() { + final Long now = NTP.getTime(); + if (now == null) + return; + + try (final Repository repository = RepositoryManager.tryRepository()) { + if (repository == null) + return; + + int numBlocksTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(now - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime()); + + if (numBlocksTrimmed > 0) + LOGGER.debug(String.format("Trimmed old online accounts signatures from %d block%s", numBlocksTrimmed, (numBlocksTrimmed != 1 ? "s" : ""))); + + repository.saveChanges(); + } catch (DataException e) { + LOGGER.warn("Repository issue trying to trim old online accounts signatures: " + e.getMessage()); + return; + } + } + } diff --git a/src/main/java/org/qora/block/BlockGenerator.java b/src/main/java/org/qora/block/BlockGenerator.java index 7e7540fb..7727358c 100644 --- a/src/main/java/org/qora/block/BlockGenerator.java +++ b/src/main/java/org/qora/block/BlockGenerator.java @@ -70,28 +70,36 @@ public class BlockGenerator extends Thread { List newBlocks = new ArrayList<>(); + // Flags that allow us to track whether generating is possible changes, + // so we can notify Controller, and further update SysTray, etc. + boolean isGenerationPossible = false; + boolean wasGenerationPossible = isGenerationPossible; while (running) { // Sleep for a while try { repository.discardChanges(); // Free repository locks, if any + + if (isGenerationPossible != wasGenerationPossible) + Controller.getInstance().onGenerationPossibleChange(isGenerationPossible); + + wasGenerationPossible = isGenerationPossible; + Thread.sleep(1000); } catch (InterruptedException e) { // We've been interrupted - time to exit return; } - // If Controller says we can't generate, then don't... - if (!Controller.getInstance().isGenerationAllowed()) + isGenerationPossible = false; + + final Long now = NTP.getTime(); + if (now == null) continue; final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); if (minLatestBlockTimestamp == null) continue; - final Long now = NTP.getTime(); - if (now == null) - continue; - // No online accounts? (e.g. during startup) if (Controller.getInstance().getOnlineAccounts().isEmpty()) continue; @@ -121,6 +129,7 @@ public class BlockGenerator extends Thread { // There are no peers with a recent block and/or our latest block is recent // so go ahead and generate a block if possible. + isGenerationPossible = true; // Check blockchain hasn't changed if (previousBlock == null || !Arrays.equals(previousBlock.getSignature(), lastBlockData.getSignature())) { diff --git a/src/main/java/org/qora/controller/Controller.java b/src/main/java/org/qora/controller/Controller.java index 6c220005..1fe1f731 100644 --- a/src/main/java/org/qora/controller/Controller.java +++ b/src/main/java/org/qora/controller/Controller.java @@ -106,6 +106,7 @@ public class Controller extends Thread { // To do with online accounts list private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000; // ms + private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 1 * 60 * 1000; // ms private static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000; private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000); @@ -125,8 +126,8 @@ public class Controller extends Thread { private long deleteExpiredTimestamp = startTime + DELETE_EXPIRED_INTERVAL; // ms private long onlineAccountsTasksTimestamp = startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms - /** Whether BlockGenerator is allowed to generate blocks. Mostly determined by system clock accuracy. */ - private volatile boolean isGenerationAllowed = false; + /** Whether we can generate new blocks, as reported by BlockGenerator. */ + private volatile boolean isGenerationPossible = false; /** Signature of peer's latest block that will result in no sync action needed (e.g. INFERIOR_CHAIN, NOTHING_TO_DO, OK). */ private byte[] noSyncPeerBlockSignature = null; @@ -227,10 +228,6 @@ public class Controller extends Thread { return this.blockchainLock; } - public boolean isGenerationAllowed() { - return this.isGenerationAllowed; - } - // Entry point public static void main(String args[]) { @@ -243,7 +240,10 @@ public class Controller extends Thread { Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); // Load/check settings, which potentially sets up blockchain config, etc. - Settings.getInstance(); + if (args.length > 0) + Settings.fileInstance(args[0]); + else + Settings.getInstance(); LOGGER.info("Starting NTP"); NTP.start(); @@ -373,7 +373,6 @@ public class Controller extends Thread { ntpCheckTimestamp = now + NTP_PRE_SYNC_CHECK_PERIOD; } - isGenerationAllowed = ntpTime != null; requestSysTrayUpdate = true; } @@ -525,7 +524,7 @@ public class Controller extends Thread { String connectionsText = Translator.INSTANCE.translate("SysTray", numberOfPeers != 1 ? "CONNECTIONS" : "CONNECTION"); String heightText = Translator.INSTANCE.translate("SysTray", "BLOCK_HEIGHT"); - String generatingText = Translator.INSTANCE.translate("SysTray", isGenerationAllowed ? "GENERATING_ENABLED" : "GENERATING_DISABLED"); + String generatingText = Translator.INSTANCE.translate("SysTray", isGenerationPossible ? "GENERATING_ENABLED" : "GENERATING_DISABLED"); String tooltip = String.format("%s - %d %s - %s %d", generatingText, numberOfPeers, connectionsText, heightText, height); SysTray.getInstance().setToolTipText(tooltip); @@ -632,6 +631,11 @@ public class Controller extends Thread { network.broadcast(peer -> network.buildGetUnconfirmedTransactionsMessage(peer)); } + public void onGenerationPossibleChange(boolean isGenerationPossible) { + this.isGenerationPossible = isGenerationPossible; + requestSysTrayUpdate = true; + } + public void onGeneratedBlock() { // Broadcast our new height info BlockData latestBlockData; @@ -646,6 +650,8 @@ public class Controller extends Thread { Network network = Network.getInstance(); network.broadcast(peer -> network.buildHeightMessage(peer, latestBlockData)); + + requestSysTrayUpdate = true; } public void onNewTransaction(TransactionData transactionData) { @@ -1247,7 +1253,9 @@ public class Controller extends Thread { } private void performOnlineAccountsTasks() { - final long now = System.currentTimeMillis(); + final Long now = NTP.getTime(); + if (now == null) + return; // Expire old entries final long cutoffThreshold = now - LAST_SEEN_EXPIRY_PERIOD; @@ -1267,15 +1275,20 @@ public class Controller extends Thread { } } - // Request data from another peer - Message message; - synchronized (this.onlineAccounts) { - message = new GetOnlineAccountsMessage(this.onlineAccounts); + // Request data from other peers? + if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) { + Message message; + synchronized (this.onlineAccounts) { + message = new GetOnlineAccountsMessage(this.onlineAccounts); + } + Network.getInstance().broadcast((peer) -> message); } - Network.getInstance().broadcast((peer) -> message); - // Refresh our onlineness? + // Refresh our online accounts signatures? sendOurOnlineAccountsInfo(); + + // Trim blockchain by removing 'old' online accounts signatures + BlockChain.trimOldOnlineAccountsSignatures(); } private void sendOurOnlineAccountsInfo() { @@ -1304,7 +1317,6 @@ public class Controller extends Thread { return; } - // 'current' timestamp final long onlineAccountsTimestamp = Controller.toOnlineAccountTimestamp(now); boolean hasInfoChanged = false; @@ -1351,7 +1363,7 @@ public class Controller extends Thread { Message message = new OnlineAccountsMessage(ourOnlineAccounts); Network.getInstance().broadcast((peer) -> message); - LOGGER.trace(( )-> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp)); + LOGGER.trace(()-> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp)); } public static long toOnlineAccountTimestamp(long timestamp) { diff --git a/src/main/java/org/qora/controller/Synchronizer.java b/src/main/java/org/qora/controller/Synchronizer.java index 75f0718b..c7720f7d 100644 --- a/src/main/java/org/qora/controller/Synchronizer.java +++ b/src/main/java/org/qora/controller/Synchronizer.java @@ -99,7 +99,6 @@ public class Synchronizer { if (peerHeight == 1) return SynchronizationResult.GENESIS_ONLY; - // XXX this may well be obsolete now // If peer is too far behind us then don't them. int minHeight = ourHeight - MAXIMUM_HEIGHT_DELTA; if (!force && peerHeight < minHeight) { @@ -136,7 +135,6 @@ public class Synchronizer { peerHeight = commonBlockHeight; } - // XXX This may well be obsolete now // If common block is peer's latest block then we simply have the same, or longer, chain to peer, so exit now if (commonBlockHeight == peerHeight) { if (peerHeight == ourHeight) @@ -154,7 +152,7 @@ public class Synchronizer { return SynchronizationResult.TOO_DIVERGENT; } - // If we have blocks after common block then decide whether we want to sync (lowest block signature wins) + // If we both have blocks after common block then decide whether we want to sync int highestMutualHeight = Math.min(peerHeight, ourHeight); // If our latest block is very old, we're very behind and should ditch our fork. diff --git a/src/main/java/org/qora/data/block/BlockData.java b/src/main/java/org/qora/data/block/BlockData.java index 071532d1..455a0d32 100644 --- a/src/main/java/org/qora/data/block/BlockData.java +++ b/src/main/java/org/qora/data/block/BlockData.java @@ -33,6 +33,7 @@ public class BlockData implements Serializable { private int atCount; private BigDecimal atFees; private byte[] encodedOnlineAccounts; + private int onlineAccountsCount; private Long onlineAccountsTimestamp; private byte[] onlineAccountsSignatures; @@ -44,7 +45,7 @@ public class BlockData implements Serializable { public BlockData(int version, byte[] reference, int transactionCount, BigDecimal totalFees, byte[] transactionsSignature, Integer height, long timestamp, BigDecimal generatingBalance, byte[] generatorPublicKey, byte[] generatorSignature, int atCount, BigDecimal atFees, - byte[] encodedOnlineAccounts, Long onlineAccountsTimestamp, byte[] onlineAccountsSignatures) { + byte[] encodedOnlineAccounts, int onlineAccountsCount, Long onlineAccountsTimestamp, byte[] onlineAccountsSignatures) { this.version = version; this.reference = reference; this.transactionCount = transactionCount; @@ -58,6 +59,7 @@ public class BlockData implements Serializable { this.atCount = atCount; this.atFees = atFees; this.encodedOnlineAccounts = encodedOnlineAccounts; + this.onlineAccountsCount = onlineAccountsCount; this.onlineAccountsTimestamp = onlineAccountsTimestamp; this.onlineAccountsSignatures = onlineAccountsSignatures; @@ -70,7 +72,7 @@ public class BlockData implements Serializable { public BlockData(int version, byte[] reference, int transactionCount, BigDecimal totalFees, byte[] transactionsSignature, Integer height, long timestamp, BigDecimal generatingBalance, byte[] generatorPublicKey, byte[] generatorSignature, int atCount, BigDecimal atFees) { this(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance, generatorPublicKey, generatorSignature, atCount, atFees, - null, null, null); + null, 0, null, null); } // Getters/setters @@ -167,8 +169,12 @@ public class BlockData implements Serializable { return this.encodedOnlineAccounts; } + public int getOnlineAccountsCount() { + return this.onlineAccountsCount; + } + public Long getOnlineAccountsTimestamp() { - return onlineAccountsTimestamp; + return this.onlineAccountsTimestamp; } public void setOnlineAccountsTimestamp(Long onlineAccountsTimestamp) { diff --git a/src/main/java/org/qora/data/block/BlockSummaryData.java b/src/main/java/org/qora/data/block/BlockSummaryData.java index c840e676..7396f12d 100644 --- a/src/main/java/org/qora/data/block/BlockSummaryData.java +++ b/src/main/java/org/qora/data/block/BlockSummaryData.java @@ -1,21 +1,34 @@ package org.qora.data.block; +import org.qora.transform.block.BlockTransformer; + public class BlockSummaryData { // Properties private int height; private byte[] signature; private byte[] generatorPublicKey; + private int onlineAccountsCount; // Constructors - public BlockSummaryData(int height, byte[] signature, byte[] generatorPublicKey) { + public BlockSummaryData(int height, byte[] signature, byte[] generatorPublicKey, int onlineAccountsCount) { this.height = height; this.signature = signature; this.generatorPublicKey = generatorPublicKey; + this.onlineAccountsCount = onlineAccountsCount; } public BlockSummaryData(BlockData blockData) { - this(blockData.getHeight(), blockData.getSignature(), blockData.getGeneratorPublicKey()); + this.height = blockData.getHeight(); + this.signature = blockData.getSignature(); + this.generatorPublicKey = blockData.getGeneratorPublicKey(); + + byte[] encodedOnlineAccounts = blockData.getEncodedOnlineAccounts(); + if (encodedOnlineAccounts != null) { + this.onlineAccountsCount = BlockTransformer.decodeOnlineAccounts(encodedOnlineAccounts).size(); + } else { + this.onlineAccountsCount = 0; + } } // Getters / setters @@ -32,4 +45,8 @@ public class BlockSummaryData { return this.generatorPublicKey; } + public int getOnlineAccountsCount() { + return this.onlineAccountsCount; + } + } diff --git a/src/main/java/org/qora/network/message/BlockSummariesMessage.java b/src/main/java/org/qora/network/message/BlockSummariesMessage.java index 3a330292..5d0ef3a6 100644 --- a/src/main/java/org/qora/network/message/BlockSummariesMessage.java +++ b/src/main/java/org/qora/network/message/BlockSummariesMessage.java @@ -15,7 +15,7 @@ import com.google.common.primitives.Ints; public class BlockSummariesMessage extends Message { - private static final int BLOCK_SUMMARY_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH + Transformer.INT_LENGTH + Transformer.PUBLIC_KEY_LENGTH; + private static final int BLOCK_SUMMARY_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH + Transformer.INT_LENGTH + Transformer.PUBLIC_KEY_LENGTH + Transformer.INT_LENGTH; private List blockSummaries; @@ -49,7 +49,9 @@ public class BlockSummariesMessage extends Message { byte[] generatorPublicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; bytes.get(generatorPublicKey); - BlockSummaryData blockSummary = new BlockSummaryData(height, signature, generatorPublicKey); + int onlineAccountsCount = bytes.getInt(); + + BlockSummaryData blockSummary = new BlockSummaryData(height, signature, generatorPublicKey, onlineAccountsCount); blockSummaries.add(blockSummary); } @@ -67,6 +69,7 @@ public class BlockSummariesMessage extends Message { bytes.write(Ints.toByteArray(blockSummary.getHeight())); bytes.write(blockSummary.getSignature()); bytes.write(blockSummary.getGeneratorPublicKey()); + bytes.write(Ints.toByteArray(blockSummary.getOnlineAccountsCount())); } return bytes.toByteArray(); diff --git a/src/main/java/org/qora/repository/BlockRepository.java b/src/main/java/org/qora/repository/BlockRepository.java index 65b9ffce..d12e490f 100644 --- a/src/main/java/org/qora/repository/BlockRepository.java +++ b/src/main/java/org/qora/repository/BlockRepository.java @@ -123,6 +123,14 @@ public interface BlockRepository { */ public List getBlockSummaries(int firstBlockHeight, int lastBlockHeight) throws DataException; + /** + * Trim online accounts signatures from blocks older than passed timestamp. + * + * @param timestamp + * @return number of blocks trimmed + */ + public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException; + /** * Saves block into repository. * diff --git a/src/main/java/org/qora/repository/hsqldb/HSQLDBBlockRepository.java b/src/main/java/org/qora/repository/hsqldb/HSQLDBBlockRepository.java index 4649c73b..bff9132f 100644 --- a/src/main/java/org/qora/repository/hsqldb/HSQLDBBlockRepository.java +++ b/src/main/java/org/qora/repository/hsqldb/HSQLDBBlockRepository.java @@ -23,7 +23,7 @@ public class HSQLDBBlockRepository implements BlockRepository { private static final String BLOCK_DB_COLUMNS = "version, reference, transaction_count, total_fees, " + "transactions_signature, height, generation, generating_balance, generator, generator_signature, " - + "AT_count, AT_fees, online_accounts, online_accounts_timestamp, online_accounts_signatures"; + + "AT_count, AT_fees, online_accounts, online_accounts_count, online_accounts_timestamp, online_accounts_signatures"; protected HSQLDBRepository repository; @@ -49,11 +49,13 @@ public class HSQLDBBlockRepository implements BlockRepository { int atCount = resultSet.getInt(11); BigDecimal atFees = resultSet.getBigDecimal(12); byte[] encodedOnlineAccounts = resultSet.getBytes(13); - Long onlineAccountsTimestamp = getZonedTimestampMilli(resultSet, 14); - byte[] onlineAccountsSignatures = resultSet.getBytes(15); + int onlineAccountsCount = resultSet.getInt(14); + Long onlineAccountsTimestamp = getZonedTimestampMilli(resultSet, 15); + byte[] onlineAccountsSignatures = resultSet.getBytes(16); return new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance, - generatorPublicKey, generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsTimestamp, onlineAccountsSignatures); + generatorPublicKey, generatorSignature, atCount, atFees, + encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures); } catch (SQLException e) { throw new DataException("Error extracting data from result set", e); } @@ -288,7 +290,7 @@ public class HSQLDBBlockRepository implements BlockRepository { @Override public List getBlockSummaries(int firstBlockHeight, int lastBlockHeight) throws DataException { - String sql = "SELECT signature, height, generator FROM Blocks WHERE height BETWEEN ? AND ?"; + String sql = "SELECT signature, height, generator, online_accounts_count FROM Blocks WHERE height BETWEEN ? AND ?"; List blockSummaries = new ArrayList<>(); @@ -300,8 +302,9 @@ public class HSQLDBBlockRepository implements BlockRepository { byte[] signature = resultSet.getBytes(1); int height = resultSet.getInt(2); byte[] generatorPublicKey = resultSet.getBytes(3); + int onlineAccountsCount = resultSet.getInt(4); - BlockSummaryData blockSummary = new BlockSummaryData(height, signature, generatorPublicKey); + BlockSummaryData blockSummary = new BlockSummaryData(height, signature, generatorPublicKey, onlineAccountsCount); blockSummaries.add(blockSummary); } while (resultSet.next()); @@ -311,6 +314,17 @@ public class HSQLDBBlockRepository implements BlockRepository { } } + @Override + public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException { + String sql = "UPDATE Blocks set online_accounts_signatures = NULL WHERE generation < ? AND online_accounts_signatures IS NOT NULL"; + + try { + return this.repository.checkedExecuteUpdateCount(sql, toOffsetDateTime(timestamp)); + } catch (SQLException e) { + throw new DataException("Unable to trim old online accounts signatures in repository", e); + } + } + @Override public void save(BlockData blockData) throws DataException { HSQLDBSaver saveHelper = new HSQLDBSaver("Blocks"); @@ -321,7 +335,7 @@ public class HSQLDBBlockRepository implements BlockRepository { .bind("generation", toOffsetDateTime(blockData.getTimestamp())).bind("generating_balance", blockData.getGeneratingBalance()) .bind("generator", blockData.getGeneratorPublicKey()).bind("generator_signature", blockData.getGeneratorSignature()) .bind("AT_count", blockData.getATCount()).bind("AT_fees", blockData.getATFees()) - .bind("online_accounts", blockData.getEncodedOnlineAccounts()) + .bind("online_accounts", blockData.getEncodedOnlineAccounts()).bind("online_accounts_count", blockData.getOnlineAccountsCount()) .bind("online_accounts_timestamp", toOffsetDateTime(blockData.getOnlineAccountsTimestamp())) .bind("online_accounts_signatures", blockData.getOnlineAccountsSignatures()); diff --git a/src/main/java/org/qora/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qora/repository/hsqldb/HSQLDBDatabaseUpdates.java index a5ddd4b5..a1a091b1 100644 --- a/src/main/java/org/qora/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qora/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -787,6 +787,7 @@ public class HSQLDBDatabaseUpdates { case 55: // Storage of which level 1+ accounts were 'online' for a particular block. Used to distribute block rewards. stmt.execute("ALTER TABLE Blocks ADD COLUMN online_accounts VARBINARY(1048576)"); + stmt.execute("ALTER TABLE Blocks ADD COLUMN online_accounts_count INT NOT NULL DEFAULT 0"); stmt.execute("ALTER TABLE Blocks ADD COLUMN online_accounts_timestamp TIMESTAMP WITH TIME ZONE"); stmt.execute("ALTER TABLE Blocks ADD COLUMN online_accounts_signatures BLOB"); break; diff --git a/src/main/java/org/qora/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qora/repository/hsqldb/HSQLDBRepository.java index b5b80d68..5c7897ac 100644 --- a/src/main/java/org/qora/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qora/repository/hsqldb/HSQLDBRepository.java @@ -460,17 +460,19 @@ public class HSQLDBRepository implements Repository { * @return number of changed rows * @throws SQLException */ - private int checkedExecuteUpdateCount(PreparedStatement preparedStatement, Object... objects) throws SQLException { - prepareExecute(preparedStatement, objects); + /* package */ int checkedExecuteUpdateCount(String sql, Object... objects) throws SQLException { + try (PreparedStatement preparedStatement = this.prepareStatement(sql)) { + prepareExecute(preparedStatement, objects); - if (preparedStatement.execute()) - throw new SQLException("Database produced results, not row count"); + if (preparedStatement.execute()) + throw new SQLException("Database produced results, not row count"); - int rowCount = preparedStatement.getUpdateCount(); - if (rowCount == -1) - throw new SQLException("Database returned invalid row count"); + int rowCount = preparedStatement.getUpdateCount(); + if (rowCount == -1) + throw new SQLException("Database returned invalid row count"); - return rowCount; + return rowCount; + } } /** @@ -543,9 +545,7 @@ public class HSQLDBRepository implements Repository { sql.append(" WHERE "); sql.append(whereClause); - try (PreparedStatement preparedStatement = this.prepareStatement(sql.toString())) { - return this.checkedExecuteUpdateCount(preparedStatement, objects); - } + return this.checkedExecuteUpdateCount(sql.toString(), objects); } /** @@ -559,9 +559,7 @@ public class HSQLDBRepository implements Repository { sql.append("DELETE FROM "); sql.append(tableName); - try (PreparedStatement preparedStatement = this.prepareStatement(sql.toString())) { - return this.checkedExecuteUpdateCount(preparedStatement); - } + return this.checkedExecuteUpdateCount(sql.toString()); } /** diff --git a/src/main/java/org/qora/transform/block/BlockTransformer.java b/src/main/java/org/qora/transform/block/BlockTransformer.java index 1ccaea17..f6938885 100644 --- a/src/main/java/org/qora/transform/block/BlockTransformer.java +++ b/src/main/java/org/qora/transform/block/BlockTransformer.java @@ -49,6 +49,7 @@ public class BlockTransformer extends Transformer { protected static final int AT_FEES_LENGTH = LONG_LENGTH; protected static final int AT_LENGTH = AT_FEES_LENGTH + AT_BYTES_LENGTH; + protected static final int ONLINE_ACCOUNTS_COUNT_LENGTH = INT_LENGTH; protected static final int ONLINE_ACCOUNTS_SIZE_LENGTH = INT_LENGTH; protected static final int ONLINE_ACCOUNTS_TIMESTAMP_LENGTH = LONG_LENGTH; protected static final int ONLINE_ACCOUNTS_SIGNATURES_COUNT_LENGTH = INT_LENGTH; @@ -194,11 +195,14 @@ public class BlockTransformer extends Transformer { } // Online accounts info? - byte[] onlineAccounts = null; + byte[] encodedOnlineAccounts = null; + int onlineAccountsCount = 0; byte[] onlineAccountsSignatures = null; Long onlineAccountsTimestamp = null; if (version >= 4) { + onlineAccountsCount = byteBuffer.getInt(); + int conciseSetLength = byteBuffer.getInt(); if (conciseSetLength > Block.MAX_BLOCK_BYTES) @@ -207,8 +211,13 @@ public class BlockTransformer extends Transformer { if ((conciseSetLength & 3) != 0) throw new TransformationException("Byte data length not multiple of 4 for online account info"); - onlineAccounts = new byte[conciseSetLength]; - byteBuffer.get(onlineAccounts); + encodedOnlineAccounts = new byte[conciseSetLength]; + byteBuffer.get(encodedOnlineAccounts); + + // Try to decode to ConciseSet + ConciseSet accountsIndexes = BlockTransformer.decodeOnlineAccounts(encodedOnlineAccounts); + if (accountsIndexes.size() != onlineAccountsCount) + throw new TransformationException("Block's online account data malformed"); // Note: number of signatures, not byte length int onlineAccountsSignaturesCount = byteBuffer.getInt(); @@ -228,7 +237,7 @@ public class BlockTransformer extends Transformer { // We don't have a height! Integer height = null; BlockData blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, generatingBalance, - generatorPublicKey, generatorSignature, atCount, atFees, onlineAccounts, onlineAccountsTimestamp, onlineAccountsSignatures); + generatorPublicKey, generatorSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures); return new Triple, List>(blockData, transactions, atStates); } @@ -239,9 +248,11 @@ public class BlockTransformer extends Transformer { if (blockData.getVersion() >= 4) { blockLength += AT_BYTES_LENGTH + blockData.getATCount() * V4_AT_ENTRY_LENGTH; - blockLength += ONLINE_ACCOUNTS_SIZE_LENGTH + blockData.getEncodedOnlineAccounts().length; + blockLength += ONLINE_ACCOUNTS_COUNT_LENGTH + ONLINE_ACCOUNTS_SIZE_LENGTH + blockData.getEncodedOnlineAccounts().length; blockLength += ONLINE_ACCOUNTS_SIGNATURES_COUNT_LENGTH; - if (blockData.getOnlineAccountsSignatures().length > 0) + + byte[] onlineAccountsSignatures = blockData.getOnlineAccountsSignatures(); + if (onlineAccountsSignatures != null && onlineAccountsSignatures.length > 0) blockLength += ONLINE_ACCOUNTS_TIMESTAMP_LENGTH + blockData.getOnlineAccountsSignatures().length; } else if (blockData.getVersion() >= 2) blockLength += AT_FEES_LENGTH + AT_BYTES_LENGTH + blockData.getATCount() * V2_AT_ENTRY_LENGTH; @@ -315,25 +326,27 @@ public class BlockTransformer extends Transformer { byte[] encodedOnlineAccounts = blockData.getEncodedOnlineAccounts(); if (encodedOnlineAccounts != null) { + bytes.write(Ints.toByteArray(blockData.getOnlineAccountsCount())); + bytes.write(Ints.toByteArray(encodedOnlineAccounts.length)); bytes.write(encodedOnlineAccounts); } else { - bytes.write(Ints.toByteArray(0)); + bytes.write(Ints.toByteArray(0)); // onlineAccountsCount + bytes.write(Ints.toByteArray(0)); // encodedOnlineAccounts length } byte[] onlineAccountsSignatures = blockData.getOnlineAccountsSignatures(); - if (onlineAccountsSignatures != null) { + if (onlineAccountsSignatures != null && onlineAccountsSignatures.length > 0) { // Note: we write the number of signatures, not the number of bytes bytes.write(Ints.toByteArray(onlineAccountsSignatures.length / Transformer.SIGNATURE_LENGTH)); - if (onlineAccountsSignatures.length > 0) { - // Only write online accounts timestamp if we have signatures - bytes.write(Longs.toByteArray(blockData.getOnlineAccountsTimestamp())); + // We only write online accounts timestamp if we have signatures + bytes.write(Longs.toByteArray(blockData.getOnlineAccountsTimestamp())); - bytes.write(onlineAccountsSignatures); - } + bytes.write(onlineAccountsSignatures); } else { + // Zero online accounts signatures (timestamp omitted also) bytes.write(Ints.toByteArray(0)); } } diff --git a/src/main/resources/blockchain.json b/src/main/resources/blockchain.json index d9be50b8..f828afea 100644 --- a/src/main/resources/blockchain.json +++ b/src/main/resources/blockchain.json @@ -15,7 +15,7 @@ "onlineAccountSignaturesMaxLifetime": 3196800000, "genesisInfo": { "version": 4, - "timestamp": "1568360000000", + "timestamp": "1568720000000", "generatingBalance": "100000", "transactions": [ { "type": "ISSUE_ASSET", "owner": "QUwGVHPPxJNJ2dq95abQNe79EyBN2K26zM", "assetName": "QORT", "description": "QORTAL coin", "quantity": 10000000, "isDivisible": true, "fee": 0, "reference": "28u54WRcMfGujtQMZ9dNKFXVqucY7XfPihXAqPFsnx853NPUwfDJy1sMH5boCkahFgjUNYqc5fkduxdBhQTKgUsC", "data": "{}" }, @@ -48,6 +48,9 @@ { "minBlocks": 50, "maxSubAccounts": 1 }, { "minBlocks": 0, "maxSubAccounts": 0 } ], + "blockTimingsByHeight": [ + { "height": 1, "target": 60000, "deviation": 30000, "power": 0.2 } + ], "featureTriggers": { "messageHeight": 0, "atHeight": 0,