diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index 6552fc25..e1273072 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -29,6 +29,7 @@ import org.qortal.controller.Controller; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; +import org.qortal.data.account.EligibleQoraHolderData; import org.qortal.data.account.QortFromQoraData; import org.qortal.data.account.RewardShareData; import org.qortal.data.at.ATData; @@ -53,7 +54,6 @@ import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.utils.Amounts; import org.qortal.utils.Base58; import org.qortal.utils.NTP; -import org.roaringbitmap.IntIterator; import com.google.common.primitives.Bytes; import com.google.common.primitives.Longs; @@ -128,7 +128,7 @@ public class Block { @FunctionalInterface private interface BlockRewardDistributor { - long distribute(long amount, Map balanceChanges) throws DataException; + long distribute(long amount, Map balanceChanges) throws DataException; } /** Lazy-instantiated expanded info on block's online accounts. */ @@ -144,8 +144,8 @@ public class Block { private final Account recipientAccount; private final AccountData recipientAccountData; - ExpandedAccount(Repository repository, int accountIndex) throws DataException { - this.rewardShareData = repository.getAccountRepository().getRewardShareByIndex(accountIndex); + ExpandedAccount(Repository repository, RewardShareData rewardShareData) throws DataException { + this.rewardShareData = rewardShareData; this.sharePercent = this.rewardShareData.getSharePercent(); this.mintingAccount = new Account(repository, this.rewardShareData.getMinter()); @@ -188,12 +188,12 @@ public class Block { return shareBinsByLevel[accountLevel]; } - public long distribute(long accountAmount, Map balanceChanges) { + public long distribute(long accountAmount, Map balanceChanges) { if (this.isRecipientAlsoMinter) { // minter & recipient the same - simpler case LOGGER.trace(() -> String.format("Minter/recipient account %s share: %s", this.mintingAccount.getAddress(), Amounts.prettyAmount(accountAmount))); if (accountAmount != 0) - balanceChanges.merge(this.mintingAccount, accountAmount, Long::sum); + balanceChanges.merge(this.mintingAccount.getAddress(), accountAmount, Long::sum); } else { // minter & recipient different - extra work needed long recipientAmount = (accountAmount * this.sharePercent) / 100L / 100L; // because scaled by 2dp and 'percent' means "per 100" @@ -201,11 +201,11 @@ public class Block { LOGGER.trace(() -> String.format("Minter account %s share: %s", this.mintingAccount.getAddress(), Amounts.prettyAmount(minterAmount))); if (minterAmount != 0) - balanceChanges.merge(this.mintingAccount, minterAmount, Long::sum); + balanceChanges.merge(this.mintingAccount.getAddress(), minterAmount, Long::sum); LOGGER.trace(() -> String.format("Recipient account %s share: %s", this.recipientAccount.getAddress(), Amounts.prettyAmount(recipientAmount))); if (recipientAmount != 0) - balanceChanges.merge(this.recipientAccount, recipientAmount, Long::sum); + balanceChanges.merge(this.recipientAccount.getAddress(), recipientAmount, Long::sum); } // We always distribute all of the amount @@ -217,6 +217,8 @@ public class Block { /** Opportunistic cache of this block's valid online accounts. Only created by call to isValid(). */ private List cachedValidOnlineAccounts = null; + /** Opportunistic cache of this block's valid online reward-shares. Only created by call to isValid(). */ + private List cachedOnlineRewardShares = null; // Other useful constants @@ -567,22 +569,28 @@ public class Block { /** * Return expanded info on block's online accounts. *

+ * Typically called as part of Block.process() or Block.orphan() + * so ideally after any calls to Block.isValid(). + * * @throws DataException */ public List getExpandedAccounts() throws DataException { if (this.cachedExpandedAccounts != null) return this.cachedExpandedAccounts; - ConciseSet accountIndexes = BlockTransformer.decodeOnlineAccounts(this.blockData.getEncodedOnlineAccounts()); + // We might already have a cache of online, reward-shares thanks to isValid() + if (this.cachedOnlineRewardShares == null) { + ConciseSet accountIndexes = BlockTransformer.decodeOnlineAccounts(this.blockData.getEncodedOnlineAccounts()); + this.cachedOnlineRewardShares = repository.getAccountRepository().getRewardSharesByIndexes(accountIndexes.toArray()); + + if (this.cachedOnlineRewardShares == null) + throw new DataException("Online accounts invalid?"); + } + List expandedAccounts = new ArrayList<>(); - IntIterator iterator = accountIndexes.iterator(); - while (iterator.hasNext()) { - int accountIndex = iterator.next(); - - ExpandedAccount accountInfo = new ExpandedAccount(repository, accountIndex); - expandedAccounts.add(accountInfo); - } + for (RewardShareData rewardShare : this.cachedOnlineRewardShares) + expandedAccounts.add(new ExpandedAccount(repository, rewardShare)); this.cachedExpandedAccounts = expandedAccounts; @@ -917,19 +925,9 @@ public class Block { if (accountIndexes.size() != this.blockData.getOnlineAccountsCount()) return ValidationResult.ONLINE_ACCOUNTS_INVALID; - List expandedAccounts = new ArrayList<>(); - - IntIterator iterator = accountIndexes.iterator(); - while (iterator.hasNext()) { - int accountIndex = iterator.next(); - RewardShareData rewardShareData = repository.getAccountRepository().getRewardShareByIndex(accountIndex); - - // Check that claimed online account actually exists - if (rewardShareData == null) - return ValidationResult.ONLINE_ACCOUNT_UNKNOWN; - - expandedAccounts.add(rewardShareData); - } + List onlineRewardShares = repository.getAccountRepository().getRewardSharesByIndexes(accountIndexes.toArray()); + if (onlineRewardShares == null) + return ValidationResult.ONLINE_ACCOUNT_UNKNOWN; // If block is past a certain age then we simply assume the signatures were correct long signatureRequirementThreshold = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMinLifetime(); @@ -939,7 +937,7 @@ public class Block { if (this.blockData.getOnlineAccountsSignatures() == null || this.blockData.getOnlineAccountsSignatures().length == 0) return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MISSING; - if (this.blockData.getOnlineAccountsSignatures().length != expandedAccounts.size() * Transformer.SIGNATURE_LENGTH) + if (this.blockData.getOnlineAccountsSignatures().length != onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH) return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED; // Check signatures @@ -961,7 +959,7 @@ public class Block { for (int i = 0; i < onlineAccountsSignatures.size(); ++i) { byte[] signature = onlineAccountsSignatures.get(i); - byte[] publicKey = expandedAccounts.get(i).getRewardSharePublicKey(); + byte[] publicKey = onlineRewardShares.get(i).getRewardSharePublicKey(); OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, signature, publicKey); ourOnlineAccounts.add(onlineAccountData); @@ -982,6 +980,7 @@ public class Block { // All online accounts valid, so save our list of online accounts for potential later use this.cachedValidOnlineAccounts = ourOnlineAccounts; + this.cachedOnlineRewardShares = onlineRewardShares; return ValidationResult.OK; } @@ -1316,13 +1315,16 @@ public class Block { allUniqueExpandedAccounts.add(expandedAccount.recipientAccountData); } - // Decrease blocks minted count for all accounts + // Increase blocks minted count for all accounts + + // Batch update in repository + repository.getAccountRepository().modifyMintedBlockCounts(allUniqueExpandedAccounts.stream().map(AccountData::getAddress).collect(Collectors.toList()), +1); + + // Local changes and also checks for level bump for (AccountData accountData : allUniqueExpandedAccounts) { // Adjust count locally (in Java) accountData.setBlocksMinted(accountData.getBlocksMinted() + 1); - - int rowCount = repository.getAccountRepository().modifyMintedBlockCount(accountData.getAddress(), +1); - LOGGER.trace(() -> String.format("Block minter %s up to %d minted block%s (rowCount: %d)", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""), rowCount)); + LOGGER.trace(() -> String.format("Block minter %s up to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""))); final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment(); @@ -1612,12 +1614,14 @@ public class Block { } // Decrease blocks minted count for all accounts + + // Batch update in repository + repository.getAccountRepository().modifyMintedBlockCounts(allUniqueExpandedAccounts.stream().map(AccountData::getAddress).collect(Collectors.toList()), -1); + for (AccountData accountData : allUniqueExpandedAccounts) { // Adjust count locally (in Java) accountData.setBlocksMinted(accountData.getBlocksMinted() - 1); - - int rowCount = repository.getAccountRepository().modifyMintedBlockCount(accountData.getAddress(), -1); - LOGGER.trace(() -> String.format("Block minter %s down to %d minted block%s (rowCount: %d)", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""), rowCount)); + LOGGER.trace(() -> String.format("Block minter %s down to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""))); final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment(); @@ -1646,7 +1650,7 @@ public class Block { this.distributionMethod = distributionMethod; } - public long distribute(long distibutionAmount, Map balanceChanges) throws DataException { + public long distribute(long distibutionAmount, Map balanceChanges) throws DataException { return this.distributionMethod.distribute(distibutionAmount, balanceChanges); } } @@ -1663,7 +1667,7 @@ public class Block { // Now distribute to candidates // Collate all balance changes and then apply in one final step - Map balanceChanges = new HashMap<>(); + Map balanceChanges = new HashMap<>(); long remainingAmount = totalAmount; for (int r = 0; r < rewardCandidates.size(); ++r) { @@ -1688,8 +1692,10 @@ public class Block { } // Apply balance changes - for (Map.Entry balanceChange : balanceChanges.entrySet()) - balanceChange.getKey().modifyAssetBalance(Asset.QORT, balanceChange.getValue()); + List accountBalanceDeltas = balanceChanges.entrySet().stream() + .map(entry -> new AccountBalanceData(entry.getKey(), Asset.QORT, entry.getValue())) + .collect(Collectors.toList()); + this.repository.getAccountRepository().modifyAssetBalances(accountBalanceDeltas); } protected List determineBlockRewardCandidates(boolean isProcessingNotOrphaning) throws DataException { @@ -1759,7 +1765,7 @@ public class Block { } // Fetch list of legacy QORA holders who haven't reached their cap of QORT reward. - List qoraHolders = this.repository.getAccountRepository().getEligibleLegacyQoraHolders(isProcessingNotOrphaning ? null : this.blockData.getHeight()); + List qoraHolders = this.repository.getAccountRepository().getEligibleLegacyQoraHolders(isProcessingNotOrphaning ? null : this.blockData.getHeight()); final boolean haveQoraHolders = !qoraHolders.isEmpty(); final long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShare(); @@ -1812,7 +1818,7 @@ public class Block { return rewardCandidates; } - private static long distributeBlockRewardShare(long distributionAmount, List accounts, Map balanceChanges) { + private static long distributeBlockRewardShare(long distributionAmount, List accounts, Map balanceChanges) { // Collate all expanded accounts by minting account Map> accountsByMinter = new HashMap<>(); @@ -1841,7 +1847,7 @@ public class Block { return sharedAmount; } - private static long distributeBlockRewardToQoraHolders(long qoraHoldersAmount, List qoraHolders, Map balanceChanges, Block block) throws DataException { + private static long distributeBlockRewardToQoraHolders(long qoraHoldersAmount, List qoraHolders, Map balanceChanges, Block block) throws DataException { final boolean isProcessingNotOrphaning = qoraHoldersAmount >= 0; long qoraPerQortReward = BlockChain.getInstance().getQoraPerQortReward(); @@ -1849,7 +1855,7 @@ public class Block { long totalQoraHeld = 0; for (int i = 0; i < qoraHolders.size(); ++i) - totalQoraHeld += qoraHolders.get(i).getBalance(); + totalQoraHeld += qoraHolders.get(i).getQoraBalance(); long finalTotalQoraHeld = totalQoraHeld; LOGGER.trace(() -> String.format("Total legacy QORA held: %s", Amounts.prettyAmount(finalTotalQoraHeld))); @@ -1862,9 +1868,13 @@ public class Block { BigInteger totalQoraHeldBI = BigInteger.valueOf(totalQoraHeld); long sharedAmount = 0; + // For batched update of QORT_FROM_QORA balances + List newQortFromQoraBalances = new ArrayList<>(); + for (int h = 0; h < qoraHolders.size(); ++h) { - AccountBalanceData qoraHolder = qoraHolders.get(h); - BigInteger qoraHolderBalanceBI = BigInteger.valueOf(qoraHolder.getBalance()); + EligibleQoraHolderData qoraHolder = qoraHolders.get(h); + BigInteger qoraHolderBalanceBI = BigInteger.valueOf(qoraHolder.getQoraBalance()); + String qoraHolderAddress = qoraHolder.getAddress(); // This is where a 128bit integer library could help: // long holderReward = (qoraHoldersAmount * qoraHolder.getBalance()) / totalQoraHeld; @@ -1872,15 +1882,13 @@ public class Block { final long holderRewardForLogging = holderReward; LOGGER.trace(() -> String.format("QORA holder %s has %s / %s QORA so share: %s", - qoraHolder.getAddress(), Amounts.prettyAmount(qoraHolder.getBalance()), finalTotalQoraHeld, Amounts.prettyAmount(holderRewardForLogging))); + qoraHolderAddress, Amounts.prettyAmount(qoraHolder.getQoraBalance()), finalTotalQoraHeld, Amounts.prettyAmount(holderRewardForLogging))); // Too small to register this time? if (holderReward == 0) continue; - Account qoraHolderAccount = new Account(block.repository, qoraHolder.getAddress()); - - long newQortFromQoraBalance = qoraHolderAccount.getConfirmedBalance(Asset.QORT_FROM_QORA) + holderReward; + long newQortFromQoraBalance = qoraHolder.getQortFromQoraBalance() + holderReward; // If processing, make sure we don't overpay if (isProcessingNotOrphaning) { @@ -1894,44 +1902,43 @@ public class Block { newQortFromQoraBalance -= adjustment; // This is also the QORA holder's final QORT-from-QORA block - QortFromQoraData qortFromQoraData = new QortFromQoraData(qoraHolder.getAddress(), holderReward, block.blockData.getHeight()); + QortFromQoraData qortFromQoraData = new QortFromQoraData(qoraHolderAddress, holderReward, block.blockData.getHeight()); block.repository.getAccountRepository().save(qortFromQoraData); long finalAdjustedHolderReward = holderReward; LOGGER.trace(() -> String.format("QORA holder %s final share %s at height %d", - qoraHolder.getAddress(), Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight())); + qoraHolderAddress, Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight())); } } else { // Orphaning - QortFromQoraData qortFromQoraData = block.repository.getAccountRepository().getQortFromQoraInfo(qoraHolder.getAddress()); - if (qortFromQoraData != null) { + if (qoraHolder.getFinalBlockHeight() != null) { // Final QORT-from-QORA amount from repository was stored during processing, and hence positive. // So we use + here as qortFromQora is negative during orphaning. // More efficient than "holderReward - (0 - final-qort-from-qora)" - long adjustment = holderReward + qortFromQoraData.getFinalQortFromQora(); + long adjustment = holderReward + qoraHolder.getFinalQortFromQora(); holderReward -= adjustment; newQortFromQoraBalance -= adjustment; - block.repository.getAccountRepository().deleteQortFromQoraInfo(qoraHolder.getAddress()); + block.repository.getAccountRepository().deleteQortFromQoraInfo(qoraHolderAddress); long finalAdjustedHolderReward = holderReward; LOGGER.trace(() -> String.format("QORA holder %s final share %s was at height %d", - qoraHolder.getAddress(), Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight())); + qoraHolderAddress, Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight())); } } - balanceChanges.merge(qoraHolderAccount, holderReward, Long::sum); + balanceChanges.merge(qoraHolderAddress, holderReward, Long::sum); - if (newQortFromQoraBalance > 0) - qoraHolderAccount.setConfirmedBalance(Asset.QORT_FROM_QORA, newQortFromQoraBalance); - else - // Remove QORT_FROM_QORA balance as it's zero - qoraHolderAccount.deleteBalance(Asset.QORT_FROM_QORA); + // Add to batched QORT_FROM_QORA balance update list + newQortFromQoraBalances.add(new AccountBalanceData(qoraHolderAddress, Asset.QORT_FROM_QORA, newQortFromQoraBalance)); sharedAmount += holderReward; } + // Perform batched update of QORT_FROM_QORA balances + block.repository.getAccountRepository().setAssetBalances(newQortFromQoraBalances); + return sharedAmount; } diff --git a/src/main/java/org/qortal/data/account/EligibleQoraHolderData.java b/src/main/java/org/qortal/data/account/EligibleQoraHolderData.java new file mode 100644 index 00000000..f3f02862 --- /dev/null +++ b/src/main/java/org/qortal/data/account/EligibleQoraHolderData.java @@ -0,0 +1,48 @@ +package org.qortal.data.account; + +public class EligibleQoraHolderData { + + // Properties + + private String address; + + private long qoraBalance; + private long qortFromQoraBalance; + + private Long finalQortFromQora; + private Integer finalBlockHeight; + + // Constructors + + public EligibleQoraHolderData(String address, long qoraBalance, long qortFromQoraBalance, Long finalQortFromQora, + Integer finalBlockHeight) { + this.address = address; + this.qoraBalance = qoraBalance; + this.qortFromQoraBalance = qortFromQoraBalance; + this.finalQortFromQora = finalQortFromQora; + this.finalBlockHeight = finalBlockHeight; + } + + // Getters/Setters + + public String getAddress() { + return this.address; + } + + public long getQoraBalance() { + return this.qoraBalance; + } + + public long getQortFromQoraBalance() { + return this.qortFromQoraBalance; + } + + public Long getFinalQortFromQora() { + return this.finalQortFromQora; + } + + public Integer getFinalBlockHeight() { + return this.finalBlockHeight; + } + +} diff --git a/src/main/java/org/qortal/repository/AccountRepository.java b/src/main/java/org/qortal/repository/AccountRepository.java index 1c96f2fa..a23771f9 100644 --- a/src/main/java/org/qortal/repository/AccountRepository.java +++ b/src/main/java/org/qortal/repository/AccountRepository.java @@ -4,6 +4,7 @@ import java.util.List; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; +import org.qortal.data.account.EligibleQoraHolderData; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.QortFromQoraData; import org.qortal.data.account.RewardShareData; @@ -89,6 +90,13 @@ public interface AccountRepository { */ public int modifyMintedBlockCount(String address, int delta) throws DataException; + /** + * Modifies batch of accounts' minted block count only. + *

+ * This is a one-shot, batch version of modifyMintedBlockCount(String, int) above. + */ + public void modifyMintedBlockCounts(List addresses, int delta) throws DataException; + /** Delete account from repository. */ public void delete(String address) throws DataException; @@ -106,6 +114,9 @@ public interface AccountRepository { */ public AccountBalanceData getBalance(String address, long assetId) throws DataException; + /** Returns all account balances for given assetID, optionally excluding zero balances. */ + public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException; + /** How to order results when fetching asset balances. */ public enum BalanceOrdering { /** assetID first, then balance, then account address */ @@ -116,15 +127,18 @@ public interface AccountRepository { ASSET_ACCOUNT } - /** Returns all account balances for given assetID, optionally excluding zero balances. */ - public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException; - /** Returns account balances for matching addresses / assetIDs, optionally excluding zero balances, with pagination, used by API. */ public List getAssetBalances(List addresses, List assetIds, BalanceOrdering balanceOrdering, Boolean excludeZero, Integer limit, Integer offset, Boolean reverse) throws DataException; /** Modifies account's asset balance by deltaBalance. */ public void modifyAssetBalance(String address, long assetId, long deltaBalance) throws DataException; + /** Modifies a batch of account asset balances, treating AccountBalanceData.balance as deltaBalance. */ + public void modifyAssetBalances(List accountBalanceDeltas) throws DataException; + + /** Batch update of account asset balances. */ + public void setAssetBalances(List accountBalances) throws DataException; + public void save(AccountBalanceData accountBalanceData) throws DataException; public void delete(String address, long assetId) throws DataException; @@ -156,6 +170,16 @@ public interface AccountRepository { */ public RewardShareData getRewardShareByIndex(int index) throws DataException; + /** + * Returns list of reward-share data using array of indexes into list of reward-shares (sorted by reward-share public key). + *

+ * This is a one-shot, batch form of the above getRewardShareByIndex(int) call. + * + * @return list of reward-share data, or null if one (or more) index is invalid + * @throws DataException + */ + public List getRewardSharesByIndexes(int[] indexes) throws DataException; + public boolean rewardShareExists(byte[] rewardSharePublicKey) throws DataException; public void save(RewardShareData rewardShareData) throws DataException; @@ -175,7 +199,7 @@ public interface AccountRepository { // Managing QORT from legacy QORA /** - * Returns balance data for accounts with legacy QORA asset that are eligible + * Returns full info for accounts with legacy QORA asset that are eligible * for more block reward (block processing) or for block reward removal (block orphaning). *

* For block processing, accounts that have already received their final QORT reward for owning @@ -187,7 +211,7 @@ public interface AccountRepository { * @param blockHeight QORT reward must have be present at this height (for orphaning only) * @throws DataException */ - public List getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException; + public List getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException; public QortFromQoraData getQortFromQoraInfo(String address) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java index 38862ef6..418a8493 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java @@ -1,13 +1,17 @@ package org.qortal.repository.hsqldb; +import static org.qortal.utils.Amounts.prettyAmount; + import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import org.qortal.asset.Asset; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; +import org.qortal.data.account.EligibleQoraHolderData; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.QortFromQoraData; import org.qortal.data.account.RewardShareData; @@ -145,7 +149,7 @@ public class HSQLDBAccountRepository implements AccountRepository { public void ensureAccount(AccountData accountData) throws DataException { String sql = "INSERT IGNORE INTO Accounts (account, public_key) VALUES (?, ?)"; // MySQL syntax try { - this.repository.checkedExecuteUpdateCount(sql, accountData.getAddress(), accountData.getPublicKey()); + this.repository.executeCheckedUpdate(sql, accountData.getAddress(), accountData.getPublicKey()); } catch (SQLException e) { throw new DataException("Unable to ensure minimal account in repository", e); } @@ -260,12 +264,26 @@ public class HSQLDBAccountRepository implements AccountRepository { "ON DUPLICATE KEY UPDATE blocks_minted = blocks_minted + ?"; try { - return this.repository.checkedExecuteUpdateCount(sql, address, delta, delta); + return this.repository.executeCheckedUpdate(sql, address, delta, delta); } catch (SQLException e) { throw new DataException("Unable to modify account's minted block count in repository", e); } } + @Override + public void modifyMintedBlockCounts(List addresses, int delta) throws DataException { + String sql = "INSERT INTO Accounts (account, blocks_minted) VALUES (?, ?) " + + "ON DUPLICATE KEY UPDATE blocks_minted = blocks_minted + ?"; + + List bindParamRows = addresses.stream().map(address -> new Object[] { address, delta, delta }).collect(Collectors.toList()); + + try { + this.repository.executeCheckedBatchUpdate(sql, bindParamRows); + } catch (SQLException e) { + throw new DataException("Unable to modify many account minted block counts in repository", e); + } + } + @Override public void delete(String address) throws DataException { // NOTE: Account balances are deleted automatically by the database thanks to "ON DELETE CASCADE" in AccountBalances' FOREIGN KEY @@ -447,7 +465,7 @@ public class HSQLDBAccountRepository implements AccountRepository { // Perform actual balance change String sql = "UPDATE AccountBalances set balance = balance + ? WHERE account = ? AND asset_id = ?"; try { - this.repository.checkedExecuteUpdateCount(sql, deltaBalance, address, assetId); + this.repository.executeCheckedUpdate(sql, deltaBalance, address, assetId); } catch (SQLException e) { throw new DataException("Unable to reduce account balance in repository", e); } @@ -455,7 +473,7 @@ public class HSQLDBAccountRepository implements AccountRepository { // We have to ensure parent row exists to satisfy foreign key constraint try { String sql = "INSERT IGNORE INTO Accounts (account) VALUES (?)"; // MySQL syntax - this.repository.checkedExecuteUpdateCount(sql, address); + this.repository.executeCheckedUpdate(sql, address); } catch (SQLException e) { throw new DataException("Unable to ensure minimal account in repository", e); } @@ -464,13 +482,95 @@ public class HSQLDBAccountRepository implements AccountRepository { String sql = "INSERT INTO AccountBalances (account, asset_id, balance) VALUES (?, ?, ?) " + "ON DUPLICATE KEY UPDATE balance = balance + ?"; try { - this.repository.checkedExecuteUpdateCount(sql, address, assetId, deltaBalance, deltaBalance); + this.repository.executeCheckedUpdate(sql, address, assetId, deltaBalance, deltaBalance); } catch (SQLException e) { throw new DataException("Unable to increase account balance in repository", e); } } } + public void modifyAssetBalances(List accountBalanceDeltas) throws DataException { + // Nothing to do? + if (accountBalanceDeltas == null || accountBalanceDeltas.isEmpty()) + return; + + // Map balance changes into SQL bind params, filtering out no-op changes + List modifyBalanceParams = accountBalanceDeltas.stream() + .filter(accountBalance -> accountBalance.getBalance() != 0L) + .map(accountBalance -> new Object[] { accountBalance.getAddress(), accountBalance.getAssetId(), accountBalance.getBalance(), accountBalance.getBalance() }) + .collect(Collectors.toList()); + + // Before we modify balances, ensure parent accounts exist + String ensureSql = "INSERT IGNORE INTO Accounts (account) VALUES (?)"; // MySQL syntax + try { + this.repository.executeCheckedBatchUpdate(ensureSql, modifyBalanceParams.stream().map(objects -> new Object[] { objects[0] }).collect(Collectors.toList())); + } catch (SQLException e) { + throw new DataException("Unable to ensure minimal accounts in repository", e); + } + + // Perform actual balance changes + String sql = "INSERT INTO AccountBalances (account, asset_id, balance) VALUES (?, ?, ?) " + + "ON DUPLICATE KEY UPDATE balance = balance + ?"; + try { + this.repository.executeCheckedBatchUpdate(sql, modifyBalanceParams); + } catch (SQLException e) { + throw new DataException("Unable to modify account balances in repository", e); + } + } + + + @Override + public void setAssetBalances(List accountBalances) throws DataException { + // Nothing to do? + if (accountBalances == null || accountBalances.isEmpty()) + return; + + /* + * Split workload into zero and non-zero balances, + * checking for negative balances as we progress. + */ + + List zeroAccountBalanceParams = new ArrayList<>(); + List nonZeroAccountBalanceParams = new ArrayList<>(); + + for (AccountBalanceData accountBalanceData : accountBalances) { + final long balance = accountBalanceData.getBalance(); + + if (balance < 0) + throw new DataException(String.format("Refusing to set negative balance %s [assetId %d] for %s", + prettyAmount(balance), accountBalanceData.getAssetId(), accountBalanceData.getAddress())); + + if (balance == 0) + zeroAccountBalanceParams.add(new Object[] { accountBalanceData.getAddress(), accountBalanceData.getAssetId() }); + else + nonZeroAccountBalanceParams.add(new Object[] { accountBalanceData.getAddress(), accountBalanceData.getAssetId(), balance, balance }); + } + + // Batch update (actually delete) of zero balances + try { + this.repository.deleteBatch("AccountBalances", "account = ? AND asset_id = ?", zeroAccountBalanceParams); + } catch (SQLException e) { + throw new DataException("Unable to delete account balances from repository", e); + } + + // Before we set new balances, ensure parent accounts exist + String ensureSql = "INSERT IGNORE INTO Accounts (account) VALUES (?)"; // MySQL syntax + try { + this.repository.executeCheckedBatchUpdate(ensureSql, nonZeroAccountBalanceParams.stream().map(objects -> new Object[] { objects[0] }).collect(Collectors.toList())); + } catch (SQLException e) { + throw new DataException("Unable to ensure minimal accounts in repository", e); + } + + // Now set all balances in one go + String setSql = "INSERT INTO AccountBalances (account, asset_id, balance) VALUES (?, ?, ?) " + + "ON DUPLICATE KEY UPDATE balance = ?"; + try { + this.repository.executeCheckedBatchUpdate(setSql, nonZeroAccountBalanceParams); + } catch (SQLException e) { + throw new DataException("Unable to set account balances in repository", e); + } + } + @Override public void save(AccountBalanceData accountBalanceData) throws DataException { HSQLDBSaver saveHelper = new HSQLDBSaver("AccountBalances"); @@ -699,7 +799,52 @@ public class HSQLDBAccountRepository implements AccountRepository { return new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent); } catch (SQLException e) { - throw new DataException("Unable to fetch reward-share info from repository", e); + throw new DataException("Unable to fetch indexed reward-share from repository", e); + } + } + + @Override + public List getRewardSharesByIndexes(int[] indexes) throws DataException { + String sql = "SELECT minter_public_key, minter, recipient, share_percent, reward_share_public_key FROM RewardShares " + + "ORDER BY reward_share_public_key ASC"; + + if (indexes == null) + return null; + + List rewardShares = new ArrayList<>(); + if (indexes.length == 0) + return rewardShares; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return null; + + int rowNum = 1; + for (int i = 0; i < indexes.length; ++i) { + final int index = indexes[i]; + + while (rowNum < index + 1) { // +1 because in JDBC, first row is row 1 + if (!resultSet.next()) + // Index is out of bounds + return null; + + ++rowNum; + } + + byte[] minterPublicKey = resultSet.getBytes(1); + String minter = resultSet.getString(2); + String recipient = resultSet.getString(3); + int sharePercent = resultSet.getInt(4); + byte[] rewardSharePublicKey = resultSet.getBytes(5); + + RewardShareData rewardShareData = new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent); + + rewardShares.add(rewardShareData); + } + + return rewardShares; + } catch (SQLException e) { + throw new DataException("Unable to fetch indexed reward-shares from repository", e); } } @@ -785,11 +930,14 @@ public class HSQLDBAccountRepository implements AccountRepository { // Managing QORT from legacy QORA @Override - public List getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException { + public List getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException { StringBuilder sql = new StringBuilder(1024); - sql.append("SELECT account, balance from AccountBalances "); + sql.append("SELECT account, Qora.balance, QortFromQora.balance, final_qort_from_qora, final_block_height "); + sql.append("FROM AccountBalances AS Qora "); sql.append("LEFT OUTER JOIN AccountQortFromQoraInfo USING (account) "); - sql.append("WHERE asset_id = "); + sql.append("LEFT OUTER JOIN AccountBalances AS QortFromQora ON QortFromQora.account = Qora.account AND QortFromQora.asset_id = "); + sql.append(Asset.QORT_FROM_QORA); // int is safe to use literally + sql.append(" WHERE Qora.asset_id = "); sql.append(Asset.LEGACY_QORA); // int is safe to use literally sql.append(" AND (final_block_height IS NULL"); @@ -800,20 +948,29 @@ public class HSQLDBAccountRepository implements AccountRepository { sql.append(")"); - List accountBalances = new ArrayList<>(); + List eligibleLegacyQoraHolders = new ArrayList<>(); try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) { if (resultSet == null) - return accountBalances; + return eligibleLegacyQoraHolders; do { String address = resultSet.getString(1); - long balance = resultSet.getLong(2); + long qoraBalance = resultSet.getLong(2); + long qortFromQoraBalance = resultSet.getLong(3); - accountBalances.add(new AccountBalanceData(address, Asset.LEGACY_QORA, balance)); + Long finalQortFromQora = resultSet.getLong(4); + if (finalQortFromQora == 0 && resultSet.wasNull()) + finalQortFromQora = null; + + Integer finalBlockHeight = resultSet.getInt(5); + if (finalBlockHeight == 0 && resultSet.wasNull()) + finalBlockHeight = null; + + eligibleLegacyQoraHolders.add(new EligibleQoraHolderData(address, qoraBalance, qortFromQoraBalance, finalQortFromQora, finalBlockHeight)); } while (resultSet.next()); - return accountBalances; + return eligibleLegacyQoraHolders; } catch (SQLException e) { throw new DataException("Unable to fetch eligible legacy QORA holders from repository", e); } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java index 7f3f2f7c..1fd50b83 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java @@ -464,7 +464,7 @@ public class HSQLDBBlockRepository implements BlockRepository { String sql = "UPDATE Blocks set online_accounts_signatures = NULL WHERE minted_when < ? AND online_accounts_signatures IS NOT NULL"; try { - return this.repository.checkedExecuteUpdateCount(sql, timestamp); + return this.repository.executeCheckedUpdate(sql, timestamp); } catch (SQLException e) { throw new DataException("Unable to trim old online accounts signatures in repository", e); } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index dbee6cc0..ad0b6ec9 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -16,9 +16,12 @@ import java.sql.Savepoint; import java.sql.Statement; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.Deque; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -49,18 +52,18 @@ public class HSQLDBRepository implements Repository { private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepository.class); protected Connection connection; - protected Deque savepoints; + protected Deque savepoints = new ArrayDeque<>(3); protected boolean debugState = false; protected Long slowQueryThreshold = null; protected List sqlStatements; protected long sessionId; + protected Map preparedStatementCache = new HashMap<>(); // Constructors // NB: no visibility modifier so only callable from within same package /* package */ HSQLDBRepository(Connection connection) throws DataException { this.connection = connection; - this.savepoints = new ArrayDeque<>(3); this.slowQueryThreshold = Settings.getInstance().getSlowQueryThreshold(); if (this.slowQueryThreshold != null) @@ -240,7 +243,12 @@ public class HSQLDBRepository implements Repository { try (Statement stmt = this.connection.createStatement()) { assertEmptyTransaction("connection close"); - // give connection back to the pool + // Assume we are not going to be GC'd for a while + this.preparedStatementCache.clear(); + this.sqlStatements = null; + this.savepoints.clear(); + + // Give connection back to the pool this.connection.close(); this.connection = null; } catch (SQLException e) { @@ -414,6 +422,17 @@ public class HSQLDBRepository implements Repository { if (this.sqlStatements != null) this.sqlStatements.add(sql); + /* + * We cache a duplicate PreparedStatement for this SQL string, + * which we never close, which means HSQLDB also caches a parsed, + * prepared statement that can be reused for subsequent + * calls to HSQLDB.prepareStatement(sql). + * + * See org.hsqldb.StatementManager for more details. + */ + if (!this.preparedStatementCache.containsKey(sql)) + this.preparedStatementCache.put(sql, this.connection.prepareStatement(sql)); + return this.connection.prepareStatement(sql); } @@ -460,7 +479,7 @@ public class HSQLDBRepository implements Repository { * @param objects * @throws SQLException */ - private void prepareExecute(PreparedStatement preparedStatement, Object... objects) throws SQLException { + private void bindStatementParams(PreparedStatement preparedStatement, Object... objects) throws SQLException { for (int i = 0; i < objects.length; ++i) // Special treatment for BigDecimals so that they retain their "scale", // which would otherwise be assumed as 0. @@ -481,7 +500,7 @@ public class HSQLDBRepository implements Repository { * @throws SQLException */ private ResultSet checkedExecuteResultSet(PreparedStatement preparedStatement, Object... objects) throws SQLException { - prepareExecute(preparedStatement, objects); + bindStatementParams(preparedStatement, objects); if (!preparedStatement.execute()) throw new SQLException("Fetching from database produced no results"); @@ -504,14 +523,32 @@ public class HSQLDBRepository implements Repository { * @return number of changed rows * @throws SQLException */ - /* package */ int checkedExecuteUpdateCount(String sql, Object... objects) throws SQLException { + /* package */ int executeCheckedUpdate(String sql, Object... objects) throws SQLException { + return this.executeCheckedBatchUpdate(sql, Collections.singletonList(objects)); + } + + /** + * Execute batched PreparedStatement + * + * @param preparedStatement + * @param objects + * @return number of changed rows + * @throws SQLException + */ + /* package */ int executeCheckedBatchUpdate(String sql, List batchedObjects) throws SQLException { + // Nothing to do? + if (batchedObjects == null || batchedObjects.isEmpty()) + return 0; + try (PreparedStatement preparedStatement = this.prepareStatement(sql)) { - prepareExecute(preparedStatement, objects); + for (Object[] objects : batchedObjects) { + this.bindStatementParams(preparedStatement, objects); + preparedStatement.addBatch(); + } long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis(); - if (preparedStatement.execute()) - throw new SQLException("Database produced results, not row count"); + int[] updateCounts = preparedStatement.executeBatch(); if (this.slowQueryThreshold != null) { long queryTime = System.currentTimeMillis() - beforeQuery; @@ -523,11 +560,15 @@ public class HSQLDBRepository implements Repository { } } - int rowCount = preparedStatement.getUpdateCount(); - if (rowCount == -1) - throw new SQLException("Database returned invalid row count"); + int totalCount = 0; + for (int i = 0; i < updateCounts.length; ++i) { + if (updateCounts[i] < 0) + throw new SQLException("Database returned invalid row count"); - return rowCount; + totalCount += updateCounts[i]; + } + + return totalCount; } } @@ -598,7 +639,25 @@ public class HSQLDBRepository implements Repository { sql.append(" WHERE "); sql.append(whereClause); - return this.checkedExecuteUpdateCount(sql.toString(), objects); + return this.executeCheckedUpdate(sql.toString(), objects); + } + + /** + * Delete rows from database table. + * + * @param tableName + * @param whereClause + * @param objects + * @throws SQLException + */ + public int deleteBatch(String tableName, String whereClause, List batchedObjects) throws SQLException { + StringBuilder sql = new StringBuilder(256); + sql.append("DELETE FROM "); + sql.append(tableName); + sql.append(" WHERE "); + sql.append(whereClause); + + return this.executeCheckedBatchUpdate(sql.toString(), batchedObjects); } /** @@ -612,7 +671,7 @@ public class HSQLDBRepository implements Repository { sql.append("DELETE FROM "); sql.append(tableName); - return this.checkedExecuteUpdateCount(sql.toString()); + return this.executeCheckedUpdate(sql.toString()); } /** diff --git a/src/test/java/org/qortal/test/AccountBalanceTests.java b/src/test/java/org/qortal/test/AccountBalanceTests.java index 35a9fb17..cd2822ac 100644 --- a/src/test/java/org/qortal/test/AccountBalanceTests.java +++ b/src/test/java/org/qortal/test/AccountBalanceTests.java @@ -171,4 +171,84 @@ public class AccountBalanceTests extends Common { Common.useDefaultSettings(); } + /** Test batch set/delete of account balances */ + @Test + public void testBatchedBalanceChanges() throws DataException, SQLException { + Random random = new Random(); + int ai; + + try (final Repository repository = RepositoryManager.getRepository()) { + System.out.println("Creating random accounts..."); + + // Generate some random accounts + List accounts = new ArrayList<>(); + for (ai = 0; ai < 2000; ++ai) { + byte[] publicKey = new byte[32]; + random.nextBytes(publicKey); + + PublicKeyAccount account = new PublicKeyAccount(repository, publicKey); + accounts.add(account); + } + + List accountBalances = new ArrayList<>(); + + System.out.println("Setting random balances..."); + + // Fill with lots of random balances + for (ai = 0; ai < accounts.size(); ++ai) { + Account account = accounts.get(ai); + int assetId = random.nextInt(2); + // random zero, or non-zero, balance + long balance = random.nextBoolean() ? 0L : random.nextInt(100000); + + accountBalances.add(new AccountBalanceData(account.getAddress(), assetId, balance)); + } + + repository.getAccountRepository().setAssetBalances(accountBalances); + repository.saveChanges(); + + System.out.println("Setting new random balances..."); + + // Now flip zero-ness for first half of balances + for (ai = 0; ai < accountBalances.size() / 2; ++ai) { + AccountBalanceData accountBalanceData = accountBalances.get(ai); + + accountBalanceData.setBalance(accountBalanceData.getBalance() != 0 ? 0L : random.nextInt(100000)); + } + // ...and randomize the rest + for (/*use ai from before*/; ai < accountBalances.size(); ++ai) { + AccountBalanceData accountBalanceData = accountBalances.get(ai); + + accountBalanceData.setBalance(random.nextBoolean() ? 0L : random.nextInt(100000)); + } + + repository.getAccountRepository().setAssetBalances(accountBalances); + repository.saveChanges(); + + System.out.println("Modifying random balances..."); + + // Fill with lots of random balance changes + for (ai = 0; ai < accounts.size(); ++ai) { + Account account = accounts.get(ai); + int assetId = random.nextInt(2); + // random zero, or non-zero, balance + long balance = random.nextBoolean() ? 0L : random.nextInt(100000); + + accountBalances.add(new AccountBalanceData(account.getAddress(), assetId, balance)); + } + + repository.getAccountRepository().modifyAssetBalances(accountBalances); + repository.saveChanges(); + + System.out.println("Deleting all balances..."); + + // Now simply delete all balances + for (ai = 0; ai < accountBalances.size(); ++ai) + accountBalances.get(ai).setBalance(0L); + + repository.getAccountRepository().setAssetBalances(accountBalances); + repository.saveChanges(); + } + } + } \ No newline at end of file diff --git a/src/test/java/org/qortal/test/RepositoryTests.java b/src/test/java/org/qortal/test/RepositoryTests.java index 557736a2..b453ce7b 100644 --- a/src/test/java/org/qortal/test/RepositoryTests.java +++ b/src/test/java/org/qortal/test/RepositoryTests.java @@ -17,6 +17,7 @@ import static org.junit.Assert.*; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.Executors; @@ -129,12 +130,18 @@ public class RepositoryTests extends Common { /** Check that the sub-query used to fetch highest block height is optimized by HSQLDB. */ @Test public void testBlockHeightSpeed() throws DataException, SQLException { + final int mintBlockCount = 30000; + try (final Repository repository = RepositoryManager.getRepository()) { // Mint some blocks - System.out.println("Minting test blocks - should take approx. 30 seconds..."); - for (int i = 0; i < 30000; ++i) + System.out.println(String.format("Minting %d test blocks - should take approx. 30 seconds...", mintBlockCount)); + + long beforeBigMint = System.currentTimeMillis(); + for (int i = 0; i < mintBlockCount; ++i) BlockUtils.mintBlock(repository); + System.out.println(String.format("Minting %d blocks actually took %d seconds", mintBlockCount, (System.currentTimeMillis() - beforeBigMint) / 1000L)); + final HSQLDBRepository hsqldb = (HSQLDBRepository) repository; // Too slow: @@ -287,6 +294,21 @@ public class RepositoryTests extends Common { } } + /** Test batched DELETE */ + @Test + public void testBatchedDelete() { + // Generate test data + List batchedObjects = new ArrayList<>(); + for (int i = 0; i < 100; ++i) + batchedObjects.add(new Object[] { String.valueOf(i), 1L }); + + try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) { + hsqldb.deleteBatch("AccountBalances", "account = ? AND asset_id = ?", batchedObjects); + } catch (DataException | SQLException e) { + fail("Batched delete failed: " + e.getMessage()); + } + } + public static void hsqldbSleep(int millis) throws SQLException { System.out.println(String.format("HSQLDB sleep() thread ID: %s", Thread.currentThread().getId())); diff --git a/src/test/java/org/qortal/test/minting/RewardShareTests.java b/src/test/java/org/qortal/test/minting/RewardShareTests.java index 25a4a38c..cde3c2ff 100644 --- a/src/test/java/org/qortal/test/minting/RewardShareTests.java +++ b/src/test/java/org/qortal/test/minting/RewardShareTests.java @@ -121,7 +121,7 @@ public class RewardShareTests extends Common { Transaction transaction = Transaction.fromData(repository, transactionData); ValidationResult validationResult = transaction.isValidUnconfirmed(); - assertEquals("Initial 0% share should be invalid", ValidationResult.INVALID_REWARD_SHARE_PERCENT, validationResult); + assertNotSame("Creating reward-share with 'cancel' share-percent should be invalid", ValidationResult.OK, validationResult); } }