AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Repository optimizations!

Browse Source 
Added Qortal-side HSQLDB PreparedStatement cache, hashed
by SQL query string, to reduce re-preparing statements.
(HSQLDB actually does the work in avoiding re-preparing
by comparing its own query-to-statement cache map, but we
need to keep an 'open' statement on our side for this to
happen).

Support added for batched INSERT/UPDATE SQL statements to
update many rows in one call.

Several specific repository calls, e.g. modifyMintedBlockCount
or modifyAssetBalance, now have batch versions that allow
many rows to be updated in one call.

In Block, when distributing block rewards, although we still
build a map of balance changes to apply after all calculations,
this map is now handed off wholesale to the repository to
apply in one (or two) queries, instead of a repository call
per account. The balanceChanges map is now keyed by account
address, as opposed to actual Account.

Also in Block, we try to cache the fetched online reward-shares
(typically in Block.isValid et al) to avoid re-fetching them
later when calculating block rewards.

In addition, actually fetching online reward-shares is no longer
done index-by-index, but the whole array of indexes is passed
wholesale to the repository which then returns the corresponding
reward-shares as a list.

In Block.increaseAccountLevels, blocks minted counts are also
updated in one single repository call, rather than one
repository call per account.

When distributing Block rewards to legacy QORA holders,
all necessary info is fetched from the repository in one hit
instead of two-phases of: 1. fetching eligible QORA holders,
and 2. fetching extra data for that QORA holder as needed.

In addition, updated QORT_FROM_QORA asset balances are done
via one batch repository call, rather than per update.
This commit is contained in:
catbref 2020-08-26 17:16:45 +01:00
parent 96eb60dca3
commit 1ca5b864a9
9 changed files with 500 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
src/main/java/org/qortal/block/Block.java
View File

@ -29,6 +29,7 @@ import org.qortal.controller.Controller;
import org.qortal.crypto.Crypto;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.data.account.AccountData;
import org.qortal.data.account.EligibleQoraHolderData;
import org.qortal.data.account.QortFromQoraData;
import org.qortal.data.account.RewardShareData;
import org.qortal.data.at.ATData;
@ -53,7 +54,6 @@ import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Amounts;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import org.roaringbitmap.IntIterator;
import com.google.common.primitives.Bytes;
import com.google.common.primitives.Longs;
@ -128,7 +128,7 @@ public class Block {
@FunctionalInterface
private interface BlockRewardDistributor {
long distribute(long amount, Map<Account, Long> balanceChanges) throws DataException;
long distribute(long amount, Map<String, Long> balanceChanges) throws DataException;
}
/** Lazy-instantiated expanded info on block's online accounts. */
@ -144,8 +144,8 @@ public class Block {
private final Account recipientAccount;
private final AccountData recipientAccountData;
ExpandedAccount(Repository repository, int accountIndex) throws DataException {
this.rewardShareData = repository.getAccountRepository().getRewardShareByIndex(accountIndex);
ExpandedAccount(Repository repository, RewardShareData rewardShareData) throws DataException {
this.rewardShareData = rewardShareData;
this.sharePercent = this.rewardShareData.getSharePercent();
this.mintingAccount = new Account(repository, this.rewardShareData.getMinter());
@ -188,12 +188,12 @@ public class Block {
return shareBinsByLevel[accountLevel];
}
public long distribute(long accountAmount, Map<Account, Long> balanceChanges) {
public long distribute(long accountAmount, Map<String, Long> balanceChanges) {
if (this.isRecipientAlsoMinter) {
// minter & recipient the same - simpler case
LOGGER.trace(() -> String.format("Minter/recipient account %s share: %s", this.mintingAccount.getAddress(), Amounts.prettyAmount(accountAmount)));
if (accountAmount != 0)
balanceChanges.merge(this.mintingAccount, accountAmount, Long::sum);
balanceChanges.merge(this.mintingAccount.getAddress(), accountAmount, Long::sum);
} else {
// minter & recipient different - extra work needed
long recipientAmount = (accountAmount * this.sharePercent) / 100L / 100L; // because scaled by 2dp and 'percent' means "per 100"
@ -201,11 +201,11 @@ public class Block {
LOGGER.trace(() -> String.format("Minter account %s share: %s", this.mintingAccount.getAddress(), Amounts.prettyAmount(minterAmount)));
if (minterAmount != 0)
balanceChanges.merge(this.mintingAccount, minterAmount, Long::sum);
balanceChanges.merge(this.mintingAccount.getAddress(), minterAmount, Long::sum);
LOGGER.trace(() -> String.format("Recipient account %s share: %s", this.recipientAccount.getAddress(), Amounts.prettyAmount(recipientAmount)));
if (recipientAmount != 0)
balanceChanges.merge(this.recipientAccount, recipientAmount, Long::sum);
balanceChanges.merge(this.recipientAccount.getAddress(), recipientAmount, Long::sum);
}
// We always distribute all of the amount
@ -217,6 +217,8 @@ public class Block {
/** Opportunistic cache of this block's valid online accounts. Only created by call to isValid(). */
private List<OnlineAccountData> cachedValidOnlineAccounts = null;
/** Opportunistic cache of this block's valid online reward-shares. Only created by call to isValid(). */
private List<RewardShareData> cachedOnlineRewardShares = null;
// Other useful constants
@ -567,22 +569,28 @@ public class Block {
/**
* Return expanded info on block's online accounts.
* <p>
* Typically called as part of Block.process() or Block.orphan()
* so ideally after any calls to Block.isValid().
*
* @throws DataException
*/
public List<ExpandedAccount> getExpandedAccounts() throws DataException {
if (this.cachedExpandedAccounts != null)
return this.cachedExpandedAccounts;
ConciseSet accountIndexes = BlockTransformer.decodeOnlineAccounts(this.blockData.getEncodedOnlineAccounts());
// We might already have a cache of online, reward-shares thanks to isValid()
if (this.cachedOnlineRewardShares == null) {
ConciseSet accountIndexes = BlockTransformer.decodeOnlineAccounts(this.blockData.getEncodedOnlineAccounts());
this.cachedOnlineRewardShares = repository.getAccountRepository().getRewardSharesByIndexes(accountIndexes.toArray());
if (this.cachedOnlineRewardShares == null)
throw new DataException("Online accounts invalid?");
}
List<ExpandedAccount> expandedAccounts = new ArrayList<>();
IntIterator iterator = accountIndexes.iterator();
while (iterator.hasNext()) {
int accountIndex = iterator.next();
ExpandedAccount accountInfo = new ExpandedAccount(repository, accountIndex);
expandedAccounts.add(accountInfo);
}
for (RewardShareData rewardShare : this.cachedOnlineRewardShares)
expandedAccounts.add(new ExpandedAccount(repository, rewardShare));
this.cachedExpandedAccounts = expandedAccounts;
@ -917,19 +925,9 @@ public class Block {
if (accountIndexes.size() != this.blockData.getOnlineAccountsCount())
return ValidationResult.ONLINE_ACCOUNTS_INVALID;
List<RewardShareData> expandedAccounts = new ArrayList<>();
IntIterator iterator = accountIndexes.iterator();
while (iterator.hasNext()) {
int accountIndex = iterator.next();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShareByIndex(accountIndex);
// Check that claimed online account actually exists
if (rewardShareData == null)
return ValidationResult.ONLINE_ACCOUNT_UNKNOWN;
expandedAccounts.add(rewardShareData);
}
List<RewardShareData> onlineRewardShares = repository.getAccountRepository().getRewardSharesByIndexes(accountIndexes.toArray());
if (onlineRewardShares == null)
return ValidationResult.ONLINE_ACCOUNT_UNKNOWN;
// If block is past a certain age then we simply assume the signatures were correct
long signatureRequirementThreshold = NTP.getTime() - BlockChain.getInstance().getOnlineAccountSignaturesMinLifetime();
@ -939,7 +937,7 @@ public class Block {
if (this.blockData.getOnlineAccountsSignatures() == null || this.blockData.getOnlineAccountsSignatures().length == 0)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MISSING;
if (this.blockData.getOnlineAccountsSignatures().length != expandedAccounts.size() * Transformer.SIGNATURE_LENGTH)
if (this.blockData.getOnlineAccountsSignatures().length != onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED;
// Check signatures
@ -961,7 +959,7 @@ public class Block {
for (int i = 0; i < onlineAccountsSignatures.size(); ++i) {
byte[] signature = onlineAccountsSignatures.get(i);
byte[] publicKey = expandedAccounts.get(i).getRewardSharePublicKey();
byte[] publicKey = onlineRewardShares.get(i).getRewardSharePublicKey();
OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, signature, publicKey);
ourOnlineAccounts.add(onlineAccountData);
@ -982,6 +980,7 @@ public class Block {
// All online accounts valid, so save our list of online accounts for potential later use
this.cachedValidOnlineAccounts = ourOnlineAccounts;
this.cachedOnlineRewardShares = onlineRewardShares;
return ValidationResult.OK;
}
@ -1316,13 +1315,16 @@ public class Block {
allUniqueExpandedAccounts.add(expandedAccount.recipientAccountData);
}
// Decrease blocks minted count for all accounts
// Increase blocks minted count for all accounts
// Batch update in repository
repository.getAccountRepository().modifyMintedBlockCounts(allUniqueExpandedAccounts.stream().map(AccountData::getAddress).collect(Collectors.toList()), +1);
// Local changes and also checks for level bump
for (AccountData accountData : allUniqueExpandedAccounts) {
// Adjust count locally (in Java)
accountData.setBlocksMinted(accountData.getBlocksMinted() + 1);
int rowCount = repository.getAccountRepository().modifyMintedBlockCount(accountData.getAddress(), +1);
LOGGER.trace(() -> String.format("Block minter %s up to %d minted block%s (rowCount: %d)", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""), rowCount));
LOGGER.trace(() -> String.format("Block minter %s up to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : "")));
final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment();
@ -1612,12 +1614,14 @@ public class Block {
}
// Decrease blocks minted count for all accounts
// Batch update in repository
repository.getAccountRepository().modifyMintedBlockCounts(allUniqueExpandedAccounts.stream().map(AccountData::getAddress).collect(Collectors.toList()), -1);
for (AccountData accountData : allUniqueExpandedAccounts) {
// Adjust count locally (in Java)
accountData.setBlocksMinted(accountData.getBlocksMinted() - 1);
int rowCount = repository.getAccountRepository().modifyMintedBlockCount(accountData.getAddress(), -1);
LOGGER.trace(() -> String.format("Block minter %s down to %d minted block%s (rowCount: %d)", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : ""), rowCount));
LOGGER.trace(() -> String.format("Block minter %s down to %d minted block%s", accountData.getAddress(), accountData.getBlocksMinted(), (accountData.getBlocksMinted() != 1 ? "s" : "")));
final int effectiveBlocksMinted = accountData.getBlocksMinted() + accountData.getBlocksMintedAdjustment();
@ -1646,7 +1650,7 @@ public class Block {
this.distributionMethod = distributionMethod;
}
public long distribute(long distibutionAmount, Map<Account, Long> balanceChanges) throws DataException {
public long distribute(long distibutionAmount, Map<String, Long> balanceChanges) throws DataException {
return this.distributionMethod.distribute(distibutionAmount, balanceChanges);
}
}
@ -1663,7 +1667,7 @@ public class Block {
// Now distribute to candidates
// Collate all balance changes and then apply in one final step
Map<Account, Long> balanceChanges = new HashMap<>();
Map<String, Long> balanceChanges = new HashMap<>();
long remainingAmount = totalAmount;
for (int r = 0; r < rewardCandidates.size(); ++r) {
@ -1688,8 +1692,10 @@ public class Block {
}
// Apply balance changes
for (Map.Entry<Account, Long> balanceChange : balanceChanges.entrySet())
balanceChange.getKey().modifyAssetBalance(Asset.QORT, balanceChange.getValue());
List<AccountBalanceData> accountBalanceDeltas = balanceChanges.entrySet().stream()
.map(entry -> new AccountBalanceData(entry.getKey(), Asset.QORT, entry.getValue()))
.collect(Collectors.toList());
this.repository.getAccountRepository().modifyAssetBalances(accountBalanceDeltas);
}
protected List<BlockRewardCandidate> determineBlockRewardCandidates(boolean isProcessingNotOrphaning) throws DataException {
@ -1759,7 +1765,7 @@ public class Block {
}
// Fetch list of legacy QORA holders who haven't reached their cap of QORT reward.
List<AccountBalanceData> qoraHolders = this.repository.getAccountRepository().getEligibleLegacyQoraHolders(isProcessingNotOrphaning ? null : this.blockData.getHeight());
List<EligibleQoraHolderData> qoraHolders = this.repository.getAccountRepository().getEligibleLegacyQoraHolders(isProcessingNotOrphaning ? null : this.blockData.getHeight());
final boolean haveQoraHolders = !qoraHolders.isEmpty();
final long qoraHoldersShare = BlockChain.getInstance().getQoraHoldersShare();
@ -1812,7 +1818,7 @@ public class Block {
return rewardCandidates;
}
private static long distributeBlockRewardShare(long distributionAmount, List<ExpandedAccount> accounts, Map<Account, Long> balanceChanges) {
private static long distributeBlockRewardShare(long distributionAmount, List<ExpandedAccount> accounts, Map<String, Long> balanceChanges) {
// Collate all expanded accounts by minting account
Map<String, List<ExpandedAccount>> accountsByMinter = new HashMap<>();
@ -1841,7 +1847,7 @@ public class Block {
return sharedAmount;
}
private static long distributeBlockRewardToQoraHolders(long qoraHoldersAmount, List<AccountBalanceData> qoraHolders, Map<Account, Long> balanceChanges, Block block) throws DataException {
private static long distributeBlockRewardToQoraHolders(long qoraHoldersAmount, List<EligibleQoraHolderData> qoraHolders, Map<String, Long> balanceChanges, Block block) throws DataException {
final boolean isProcessingNotOrphaning = qoraHoldersAmount >= 0;
long qoraPerQortReward = BlockChain.getInstance().getQoraPerQortReward();
@ -1849,7 +1855,7 @@ public class Block {
long totalQoraHeld = 0;
for (int i = 0; i < qoraHolders.size(); ++i)
totalQoraHeld += qoraHolders.get(i).getBalance();
totalQoraHeld += qoraHolders.get(i).getQoraBalance();
long finalTotalQoraHeld = totalQoraHeld;
LOGGER.trace(() -> String.format("Total legacy QORA held: %s", Amounts.prettyAmount(finalTotalQoraHeld)));
@ -1862,9 +1868,13 @@ public class Block {
BigInteger totalQoraHeldBI = BigInteger.valueOf(totalQoraHeld);
long sharedAmount = 0;
// For batched update of QORT_FROM_QORA balances
List<AccountBalanceData> newQortFromQoraBalances = new ArrayList<>();
for (int h = 0; h < qoraHolders.size(); ++h) {
AccountBalanceData qoraHolder = qoraHolders.get(h);
BigInteger qoraHolderBalanceBI = BigInteger.valueOf(qoraHolder.getBalance());
EligibleQoraHolderData qoraHolder = qoraHolders.get(h);
BigInteger qoraHolderBalanceBI = BigInteger.valueOf(qoraHolder.getQoraBalance());
String qoraHolderAddress = qoraHolder.getAddress();
// This is where a 128bit integer library could help:
// long holderReward = (qoraHoldersAmount * qoraHolder.getBalance()) / totalQoraHeld;
@ -1872,15 +1882,13 @@ public class Block {
final long holderRewardForLogging = holderReward;
LOGGER.trace(() -> String.format("QORA holder %s has %s / %s QORA so share: %s",
qoraHolder.getAddress(), Amounts.prettyAmount(qoraHolder.getBalance()), finalTotalQoraHeld, Amounts.prettyAmount(holderRewardForLogging)));
qoraHolderAddress, Amounts.prettyAmount(qoraHolder.getQoraBalance()), finalTotalQoraHeld, Amounts.prettyAmount(holderRewardForLogging)));
// Too small to register this time?
if (holderReward == 0)
continue;
Account qoraHolderAccount = new Account(block.repository, qoraHolder.getAddress());
long newQortFromQoraBalance = qoraHolderAccount.getConfirmedBalance(Asset.QORT_FROM_QORA) + holderReward;
long newQortFromQoraBalance = qoraHolder.getQortFromQoraBalance() + holderReward;
// If processing, make sure we don't overpay
if (isProcessingNotOrphaning) {
@ -1894,44 +1902,43 @@ public class Block {
newQortFromQoraBalance -= adjustment;
// This is also the QORA holder's final QORT-from-QORA block
QortFromQoraData qortFromQoraData = new QortFromQoraData(qoraHolder.getAddress(), holderReward, block.blockData.getHeight());
QortFromQoraData qortFromQoraData = new QortFromQoraData(qoraHolderAddress, holderReward, block.blockData.getHeight());
block.repository.getAccountRepository().save(qortFromQoraData);
long finalAdjustedHolderReward = holderReward;
LOGGER.trace(() -> String.format("QORA holder %s final share %s at height %d",
qoraHolder.getAddress(), Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight()));
qoraHolderAddress, Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight()));
}
} else {
// Orphaning
QortFromQoraData qortFromQoraData = block.repository.getAccountRepository().getQortFromQoraInfo(qoraHolder.getAddress());
if (qortFromQoraData != null) {
if (qoraHolder.getFinalBlockHeight() != null) {
// Final QORT-from-QORA amount from repository was stored during processing, and hence positive.
// So we use + here as qortFromQora is negative during orphaning.
// More efficient than "holderReward - (0 - final-qort-from-qora)"
long adjustment = holderReward + qortFromQoraData.getFinalQortFromQora();
long adjustment = holderReward + qoraHolder.getFinalQortFromQora();
holderReward -= adjustment;
newQortFromQoraBalance -= adjustment;
block.repository.getAccountRepository().deleteQortFromQoraInfo(qoraHolder.getAddress());
block.repository.getAccountRepository().deleteQortFromQoraInfo(qoraHolderAddress);
long finalAdjustedHolderReward = holderReward;
LOGGER.trace(() -> String.format("QORA holder %s final share %s was at height %d",
qoraHolder.getAddress(), Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight()));
qoraHolderAddress, Amounts.prettyAmount(finalAdjustedHolderReward), block.blockData.getHeight()));
}
}
balanceChanges.merge(qoraHolderAccount, holderReward, Long::sum);
balanceChanges.merge(qoraHolderAddress, holderReward, Long::sum);
if (newQortFromQoraBalance > 0)
qoraHolderAccount.setConfirmedBalance(Asset.QORT_FROM_QORA, newQortFromQoraBalance);
else
// Remove QORT_FROM_QORA balance as it's zero
qoraHolderAccount.deleteBalance(Asset.QORT_FROM_QORA);
// Add to batched QORT_FROM_QORA balance update list
newQortFromQoraBalances.add(new AccountBalanceData(qoraHolderAddress, Asset.QORT_FROM_QORA, newQortFromQoraBalance));
sharedAmount += holderReward;
}
// Perform batched update of QORT_FROM_QORA balances
block.repository.getAccountRepository().setAssetBalances(newQortFromQoraBalances);
return sharedAmount;
}

48
src/main/java/org/qortal/data/account/EligibleQoraHolderData.java Normal file
View File

@ -0,0 +1,48 @@
package org.qortal.data.account;
public class EligibleQoraHolderData {
// Properties
private String address;
private long qoraBalance;
private long qortFromQoraBalance;
private Long finalQortFromQora;
private Integer finalBlockHeight;
// Constructors
public EligibleQoraHolderData(String address, long qoraBalance, long qortFromQoraBalance, Long finalQortFromQora,
Integer finalBlockHeight) {
this.address = address;
this.qoraBalance = qoraBalance;
this.qortFromQoraBalance = qortFromQoraBalance;
this.finalQortFromQora = finalQortFromQora;
this.finalBlockHeight = finalBlockHeight;
}
// Getters/Setters
public String getAddress() {
return this.address;
}
public long getQoraBalance() {
return this.qoraBalance;
}
public long getQortFromQoraBalance() {
return this.qortFromQoraBalance;
}
public Long getFinalQortFromQora() {
return this.finalQortFromQora;
}
public Integer getFinalBlockHeight() {
return this.finalBlockHeight;
}
}

34
src/main/java/org/qortal/repository/AccountRepository.java
View File

@ -4,6 +4,7 @@ import java.util.List;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.data.account.AccountData;
import org.qortal.data.account.EligibleQoraHolderData;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.account.QortFromQoraData;
import org.qortal.data.account.RewardShareData;
@ -89,6 +90,13 @@ public interface AccountRepository {
*/
public int modifyMintedBlockCount(String address, int delta) throws DataException;
/**
* Modifies batch of accounts' minted block count only.
* <p>
* This is a one-shot, batch version of modifyMintedBlockCount(String, int) above.
*/
public void modifyMintedBlockCounts(List<String> addresses, int delta) throws DataException;
/** Delete account from repository. */
public void delete(String address) throws DataException;
@ -106,6 +114,9 @@ public interface AccountRepository {
*/
public AccountBalanceData getBalance(String address, long assetId) throws DataException;
/** Returns all account balances for given assetID, optionally excluding zero balances. */
public List<AccountBalanceData> getAssetBalances(long assetId, Boolean excludeZero) throws DataException;
/** How to order results when fetching asset balances. */
public enum BalanceOrdering {
/** assetID first, then balance, then account address */
@ -116,15 +127,18 @@ public interface AccountRepository {
ASSET_ACCOUNT
}
/** Returns all account balances for given assetID, optionally excluding zero balances. */
public List<AccountBalanceData> getAssetBalances(long assetId, Boolean excludeZero) throws DataException;
/** Returns account balances for matching addresses / assetIDs, optionally excluding zero balances, with pagination, used by API. */
public List<AccountBalanceData> getAssetBalances(List<String> addresses, List<Long> assetIds, BalanceOrdering balanceOrdering, Boolean excludeZero, Integer limit, Integer offset, Boolean reverse) throws DataException;
/** Modifies account's asset balance by <tt>deltaBalance</tt>. */
public void modifyAssetBalance(String address, long assetId, long deltaBalance) throws DataException;
/** Modifies a batch of account asset balances, treating AccountBalanceData.balance as <tt>deltaBalance</tt>. */
public void modifyAssetBalances(List<AccountBalanceData> accountBalanceDeltas) throws DataException;
/** Batch update of account asset balances. */
public void setAssetBalances(List<AccountBalanceData> accountBalances) throws DataException;
public void save(AccountBalanceData accountBalanceData) throws DataException;
public void delete(String address, long assetId) throws DataException;
@ -156,6 +170,16 @@ public interface AccountRepository {
*/
public RewardShareData getRewardShareByIndex(int index) throws DataException;
/**
* Returns list of reward-share data using array of indexes into list of reward-shares (sorted by reward-share public key).
* <p>
* This is a one-shot, batch form of the above <tt>getRewardShareByIndex(int)</tt> call.
*
* @return list of reward-share data, or null if one (or more) index is invalid
* @throws DataException
*/
public List<RewardShareData> getRewardSharesByIndexes(int[] indexes) throws DataException;
public boolean rewardShareExists(byte[] rewardSharePublicKey) throws DataException;
public void save(RewardShareData rewardShareData) throws DataException;
@ -175,7 +199,7 @@ public interface AccountRepository {
// Managing QORT from legacy QORA
/**
* Returns balance data for accounts with legacy QORA asset that are eligible
* Returns full info for accounts with legacy QORA asset that are eligible
* for more block reward (block processing) or for block reward removal (block orphaning).
* <p>
* For block processing, accounts that have already received their final QORT reward for owning
@ -187,7 +211,7 @@ public interface AccountRepository {
* @param blockHeight QORT reward must have be present at this height (for orphaning only)
* @throws DataException
*/
public List<AccountBalanceData> getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException;
public List<EligibleQoraHolderData> getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException;
public QortFromQoraData getQortFromQoraInfo(String address) throws DataException;

185
src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java
View File

@ -1,13 +1,17 @@
package org.qortal.repository.hsqldb;
import static org.qortal.utils.Amounts.prettyAmount;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.qortal.asset.Asset;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.data.account.AccountData;
import org.qortal.data.account.EligibleQoraHolderData;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.account.QortFromQoraData;
import org.qortal.data.account.RewardShareData;
@ -145,7 +149,7 @@ public class HSQLDBAccountRepository implements AccountRepository {
public void ensureAccount(AccountData accountData) throws DataException {
String sql = "INSERT IGNORE INTO Accounts (account, public_key) VALUES (?, ?)"; // MySQL syntax
try {
this.repository.checkedExecuteUpdateCount(sql, accountData.getAddress(), accountData.getPublicKey());
this.repository.executeCheckedUpdate(sql, accountData.getAddress(), accountData.getPublicKey());
} catch (SQLException e) {
throw new DataException("Unable to ensure minimal account in repository", e);
}
@ -260,12 +264,26 @@ public class HSQLDBAccountRepository implements AccountRepository {
"ON DUPLICATE KEY UPDATE blocks_minted = blocks_minted + ?";
try {
return this.repository.checkedExecuteUpdateCount(sql, address, delta, delta);
return this.repository.executeCheckedUpdate(sql, address, delta, delta);
} catch (SQLException e) {
throw new DataException("Unable to modify account's minted block count in repository", e);
}
}
@Override
public void modifyMintedBlockCounts(List<String> addresses, int delta) throws DataException {
String sql = "INSERT INTO Accounts (account, blocks_minted) VALUES (?, ?) " +
"ON DUPLICATE KEY UPDATE blocks_minted = blocks_minted + ?";
List<Object[]> bindParamRows = addresses.stream().map(address -> new Object[] { address, delta, delta }).collect(Collectors.toList());
try {
this.repository.executeCheckedBatchUpdate(sql, bindParamRows);
} catch (SQLException e) {
throw new DataException("Unable to modify many account minted block counts in repository", e);
}
}
@Override
public void delete(String address) throws DataException {
// NOTE: Account balances are deleted automatically by the database thanks to "ON DELETE CASCADE" in AccountBalances' FOREIGN KEY
@ -447,7 +465,7 @@ public class HSQLDBAccountRepository implements AccountRepository {
// Perform actual balance change
String sql = "UPDATE AccountBalances set balance = balance + ? WHERE account = ? AND asset_id = ?";
try {
this.repository.checkedExecuteUpdateCount(sql, deltaBalance, address, assetId);
this.repository.executeCheckedUpdate(sql, deltaBalance, address, assetId);
} catch (SQLException e) {
throw new DataException("Unable to reduce account balance in repository", e);
}
@ -455,7 +473,7 @@ public class HSQLDBAccountRepository implements AccountRepository {
// We have to ensure parent row exists to satisfy foreign key constraint
try {
String sql = "INSERT IGNORE INTO Accounts (account) VALUES (?)"; // MySQL syntax
this.repository.checkedExecuteUpdateCount(sql, address);
this.repository.executeCheckedUpdate(sql, address);
} catch (SQLException e) {
throw new DataException("Unable to ensure minimal account in repository", e);
}
@ -464,13 +482,95 @@ public class HSQLDBAccountRepository implements AccountRepository {
String sql = "INSERT INTO AccountBalances (account, asset_id, balance) VALUES (?, ?, ?) " +
"ON DUPLICATE KEY UPDATE balance = balance + ?";
try {
this.repository.checkedExecuteUpdateCount(sql, address, assetId, deltaBalance, deltaBalance);
this.repository.executeCheckedUpdate(sql, address, assetId, deltaBalance, deltaBalance);
} catch (SQLException e) {
throw new DataException("Unable to increase account balance in repository", e);
}
}
}
public void modifyAssetBalances(List<AccountBalanceData> accountBalanceDeltas) throws DataException {
// Nothing to do?
if (accountBalanceDeltas == null || accountBalanceDeltas.isEmpty())
return;
// Map balance changes into SQL bind params, filtering out no-op changes
List<Object[]> modifyBalanceParams = accountBalanceDeltas.stream()
.filter(accountBalance -> accountBalance.getBalance() != 0L)
.map(accountBalance -> new Object[] { accountBalance.getAddress(), accountBalance.getAssetId(), accountBalance.getBalance(), accountBalance.getBalance() })
.collect(Collectors.toList());
// Before we modify balances, ensure parent accounts exist
String ensureSql = "INSERT IGNORE INTO Accounts (account) VALUES (?)"; // MySQL syntax
try {
this.repository.executeCheckedBatchUpdate(ensureSql, modifyBalanceParams.stream().map(objects -> new Object[] { objects[0] }).collect(Collectors.toList()));
} catch (SQLException e) {
throw new DataException("Unable to ensure minimal accounts in repository", e);
}
// Perform actual balance changes
String sql = "INSERT INTO AccountBalances (account, asset_id, balance) VALUES (?, ?, ?) " +
"ON DUPLICATE KEY UPDATE balance = balance + ?";
try {
this.repository.executeCheckedBatchUpdate(sql, modifyBalanceParams);
} catch (SQLException e) {
throw new DataException("Unable to modify account balances in repository", e);
}
}
@Override
public void setAssetBalances(List<AccountBalanceData> accountBalances) throws DataException {
// Nothing to do?
if (accountBalances == null || accountBalances.isEmpty())
return;
/*
* Split workload into zero and non-zero balances,
* checking for negative balances as we progress.
*/
List<Object[]> zeroAccountBalanceParams = new ArrayList<>();
List<Object[]> nonZeroAccountBalanceParams = new ArrayList<>();
for (AccountBalanceData accountBalanceData : accountBalances) {
final long balance = accountBalanceData.getBalance();
if (balance < 0)
throw new DataException(String.format("Refusing to set negative balance %s [assetId %d] for %s",
prettyAmount(balance), accountBalanceData.getAssetId(), accountBalanceData.getAddress()));
if (balance == 0)
zeroAccountBalanceParams.add(new Object[] { accountBalanceData.getAddress(), accountBalanceData.getAssetId() });
else
nonZeroAccountBalanceParams.add(new Object[] { accountBalanceData.getAddress(), accountBalanceData.getAssetId(), balance, balance });
}
// Batch update (actually delete) of zero balances
try {
this.repository.deleteBatch("AccountBalances", "account = ? AND asset_id = ?", zeroAccountBalanceParams);
} catch (SQLException e) {
throw new DataException("Unable to delete account balances from repository", e);
}
// Before we set new balances, ensure parent accounts exist
String ensureSql = "INSERT IGNORE INTO Accounts (account) VALUES (?)"; // MySQL syntax
try {
this.repository.executeCheckedBatchUpdate(ensureSql, nonZeroAccountBalanceParams.stream().map(objects -> new Object[] { objects[0] }).collect(Collectors.toList()));
} catch (SQLException e) {
throw new DataException("Unable to ensure minimal accounts in repository", e);
}
// Now set all balances in one go
String setSql = "INSERT INTO AccountBalances (account, asset_id, balance) VALUES (?, ?, ?) " +
"ON DUPLICATE KEY UPDATE balance = ?";
try {
this.repository.executeCheckedBatchUpdate(setSql, nonZeroAccountBalanceParams);
} catch (SQLException e) {
throw new DataException("Unable to set account balances in repository", e);
}
}
@Override
public void save(AccountBalanceData accountBalanceData) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("AccountBalances");
@ -699,7 +799,52 @@ public class HSQLDBAccountRepository implements AccountRepository {
return new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent);
} catch (SQLException e) {
throw new DataException("Unable to fetch reward-share info from repository", e);
throw new DataException("Unable to fetch indexed reward-share from repository", e);
}
}
@Override
public List<RewardShareData> getRewardSharesByIndexes(int[] indexes) throws DataException {
String sql = "SELECT minter_public_key, minter, recipient, share_percent, reward_share_public_key FROM RewardShares "
+ "ORDER BY reward_share_public_key ASC";
if (indexes == null)
return null;
List<RewardShareData> rewardShares = new ArrayList<>();
if (indexes.length == 0)
return rewardShares;
try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
if (resultSet == null)
return null;
int rowNum = 1;
for (int i = 0; i < indexes.length; ++i) {
final int index = indexes[i];
while (rowNum < index + 1) { // +1 because in JDBC, first row is row 1
if (!resultSet.next())
// Index is out of bounds
return null;
++rowNum;
}
byte[] minterPublicKey = resultSet.getBytes(1);
String minter = resultSet.getString(2);
String recipient = resultSet.getString(3);
int sharePercent = resultSet.getInt(4);
byte[] rewardSharePublicKey = resultSet.getBytes(5);
RewardShareData rewardShareData = new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent);
rewardShares.add(rewardShareData);
}
return rewardShares;
} catch (SQLException e) {
throw new DataException("Unable to fetch indexed reward-shares from repository", e);
}
}
@ -785,11 +930,14 @@ public class HSQLDBAccountRepository implements AccountRepository {
// Managing QORT from legacy QORA
@Override
public List<AccountBalanceData> getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException {
public List<EligibleQoraHolderData> getEligibleLegacyQoraHolders(Integer blockHeight) throws DataException {
StringBuilder sql = new StringBuilder(1024);
sql.append("SELECT account, balance from AccountBalances ");
sql.append("SELECT account, Qora.balance, QortFromQora.balance, final_qort_from_qora, final_block_height ");
sql.append("FROM AccountBalances AS Qora ");
sql.append("LEFT OUTER JOIN AccountQortFromQoraInfo USING (account) ");
sql.append("WHERE asset_id = ");
sql.append("LEFT OUTER JOIN AccountBalances AS QortFromQora ON QortFromQora.account = Qora.account AND QortFromQora.asset_id = ");
sql.append(Asset.QORT_FROM_QORA); // int is safe to use literally
sql.append(" WHERE Qora.asset_id = ");
sql.append(Asset.LEGACY_QORA); // int is safe to use literally
sql.append(" AND (final_block_height IS NULL");
@ -800,20 +948,29 @@ public class HSQLDBAccountRepository implements AccountRepository {
sql.append(")");
List<AccountBalanceData> accountBalances = new ArrayList<>();
List<EligibleQoraHolderData> eligibleLegacyQoraHolders = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) {
if (resultSet == null)
return accountBalances;
return eligibleLegacyQoraHolders;
do {
String address = resultSet.getString(1);
long balance = resultSet.getLong(2);
long qoraBalance = resultSet.getLong(2);
long qortFromQoraBalance = resultSet.getLong(3);
accountBalances.add(new AccountBalanceData(address, Asset.LEGACY_QORA, balance));
Long finalQortFromQora = resultSet.getLong(4);
if (finalQortFromQora == 0 && resultSet.wasNull())
finalQortFromQora = null;
Integer finalBlockHeight = resultSet.getInt(5);
if (finalBlockHeight == 0 && resultSet.wasNull())
finalBlockHeight = null;
eligibleLegacyQoraHolders.add(new EligibleQoraHolderData(address, qoraBalance, qortFromQoraBalance, finalQortFromQora, finalBlockHeight));
} while (resultSet.next());
return accountBalances;
return eligibleLegacyQoraHolders;
} catch (SQLException e) {
throw new DataException("Unable to fetch eligible legacy QORA holders from repository", e);
}

2
src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java
View File

@ -464,7 +464,7 @@ public class HSQLDBBlockRepository implements BlockRepository {
String sql = "UPDATE Blocks set online_accounts_signatures = NULL WHERE minted_when < ? AND online_accounts_signatures IS NOT NULL";
try {
return this.repository.checkedExecuteUpdateCount(sql, timestamp);
return this.repository.executeCheckedUpdate(sql, timestamp);
} catch (SQLException e) {
throw new DataException("Unable to trim old online accounts signatures in repository", e);
}

89
src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java
View File

@ -16,9 +16,12 @@ import java.sql.Savepoint;
import java.sql.Statement;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Deque;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -49,18 +52,18 @@ public class HSQLDBRepository implements Repository {
private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepository.class);
protected Connection connection;
protected Deque<Savepoint> savepoints;
protected Deque<Savepoint> savepoints = new ArrayDeque<>(3);
protected boolean debugState = false;
protected Long slowQueryThreshold = null;
protected List<String> sqlStatements;
protected long sessionId;
protected Map<String, PreparedStatement> preparedStatementCache = new HashMap<>();
// Constructors
// NB: no visibility modifier so only callable from within same package
/* package */ HSQLDBRepository(Connection connection) throws DataException {
this.connection = connection;
this.savepoints = new ArrayDeque<>(3);
this.slowQueryThreshold = Settings.getInstance().getSlowQueryThreshold();
if (this.slowQueryThreshold != null)
@ -240,7 +243,12 @@ public class HSQLDBRepository implements Repository {
try (Statement stmt = this.connection.createStatement()) {
assertEmptyTransaction("connection close");
// give connection back to the pool
// Assume we are not going to be GC'd for a while
this.preparedStatementCache.clear();
this.sqlStatements = null;
this.savepoints.clear();
// Give connection back to the pool
this.connection.close();
this.connection = null;
} catch (SQLException e) {
@ -414,6 +422,17 @@ public class HSQLDBRepository implements Repository {
if (this.sqlStatements != null)
this.sqlStatements.add(sql);
/*
* We cache a duplicate PreparedStatement for this SQL string,
* which we never close, which means HSQLDB also caches a parsed,
* prepared statement that can be reused for subsequent
* calls to HSQLDB.prepareStatement(sql).
*
* See org.hsqldb.StatementManager for more details.
*/
if (!this.preparedStatementCache.containsKey(sql))
this.preparedStatementCache.put(sql, this.connection.prepareStatement(sql));
return this.connection.prepareStatement(sql);
}
@ -460,7 +479,7 @@ public class HSQLDBRepository implements Repository {
* @param objects
* @throws SQLException
*/
private void prepareExecute(PreparedStatement preparedStatement, Object... objects) throws SQLException {
private void bindStatementParams(PreparedStatement preparedStatement, Object... objects) throws SQLException {
for (int i = 0; i < objects.length; ++i)
// Special treatment for BigDecimals so that they retain their "scale",
// which would otherwise be assumed as 0.
@ -481,7 +500,7 @@ public class HSQLDBRepository implements Repository {
* @throws SQLException
*/
private ResultSet checkedExecuteResultSet(PreparedStatement preparedStatement, Object... objects) throws SQLException {
prepareExecute(preparedStatement, objects);
bindStatementParams(preparedStatement, objects);
if (!preparedStatement.execute())
throw new SQLException("Fetching from database produced no results");
@ -504,14 +523,32 @@ public class HSQLDBRepository implements Repository {
* @return number of changed rows
* @throws SQLException
*/
/* package */ int checkedExecuteUpdateCount(String sql, Object... objects) throws SQLException {
/* package */ int executeCheckedUpdate(String sql, Object... objects) throws SQLException {
return this.executeCheckedBatchUpdate(sql, Collections.singletonList(objects));
}
/**
* Execute batched PreparedStatement
*
* @param preparedStatement
* @param objects
* @return number of changed rows
* @throws SQLException
*/
/* package */ int executeCheckedBatchUpdate(String sql, List<Object[]> batchedObjects) throws SQLException {
// Nothing to do?
if (batchedObjects == null || batchedObjects.isEmpty())
return 0;
try (PreparedStatement preparedStatement = this.prepareStatement(sql)) {
prepareExecute(preparedStatement, objects);
for (Object[] objects : batchedObjects) {
this.bindStatementParams(preparedStatement, objects);
preparedStatement.addBatch();
}
long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis();
if (preparedStatement.execute())
throw new SQLException("Database produced results, not row count");
int[] updateCounts = preparedStatement.executeBatch();
if (this.slowQueryThreshold != null) {
long queryTime = System.currentTimeMillis() - beforeQuery;
@ -523,11 +560,15 @@ public class HSQLDBRepository implements Repository {
}
}
int rowCount = preparedStatement.getUpdateCount();
if (rowCount == -1)
throw new SQLException("Database returned invalid row count");
int totalCount = 0;
for (int i = 0; i < updateCounts.length; ++i) {
if (updateCounts[i] < 0)
throw new SQLException("Database returned invalid row count");
return rowCount;
totalCount += updateCounts[i];
}
return totalCount;
}
}
@ -598,7 +639,25 @@ public class HSQLDBRepository implements Repository {
sql.append(" WHERE ");
sql.append(whereClause);
return this.checkedExecuteUpdateCount(sql.toString(), objects);
return this.executeCheckedUpdate(sql.toString(), objects);
}
/**
* Delete rows from database table.
*
* @param tableName
* @param whereClause
* @param objects
* @throws SQLException
*/
public int deleteBatch(String tableName, String whereClause, List<Object[]> batchedObjects) throws SQLException {
StringBuilder sql = new StringBuilder(256);
sql.append("DELETE FROM ");
sql.append(tableName);
sql.append(" WHERE ");
sql.append(whereClause);
return this.executeCheckedBatchUpdate(sql.toString(), batchedObjects);
}
/**
@ -612,7 +671,7 @@ public class HSQLDBRepository implements Repository {
sql.append("DELETE FROM ");
sql.append(tableName);
return this.checkedExecuteUpdateCount(sql.toString());
return this.executeCheckedUpdate(sql.toString());
}
/**

80
src/test/java/org/qortal/test/AccountBalanceTests.java
View File

@ -171,4 +171,84 @@ public class AccountBalanceTests extends Common {
Common.useDefaultSettings();
}
/** Test batch set/delete of account balances */
@Test
public void testBatchedBalanceChanges() throws DataException, SQLException {
Random random = new Random();
int ai;
try (final Repository repository = RepositoryManager.getRepository()) {
System.out.println("Creating random accounts...");
// Generate some random accounts
List<Account> accounts = new ArrayList<>();
for (ai = 0; ai < 2000; ++ai) {
byte[] publicKey = new byte[32];
random.nextBytes(publicKey);
PublicKeyAccount account = new PublicKeyAccount(repository, publicKey);
accounts.add(account);
}
List<AccountBalanceData> accountBalances = new ArrayList<>();
System.out.println("Setting random balances...");
// Fill with lots of random balances
for (ai = 0; ai < accounts.size(); ++ai) {
Account account = accounts.get(ai);
int assetId = random.nextInt(2);
// random zero, or non-zero, balance
long balance = random.nextBoolean() ? 0L : random.nextInt(100000);
accountBalances.add(new AccountBalanceData(account.getAddress(), assetId, balance));
}
repository.getAccountRepository().setAssetBalances(accountBalances);
repository.saveChanges();
System.out.println("Setting new random balances...");
// Now flip zero-ness for first half of balances
for (ai = 0; ai < accountBalances.size() / 2; ++ai) {
AccountBalanceData accountBalanceData = accountBalances.get(ai);
accountBalanceData.setBalance(accountBalanceData.getBalance() != 0 ? 0L : random.nextInt(100000));
}
// ...and randomize the rest
for (/*use ai from before*/; ai < accountBalances.size(); ++ai) {
AccountBalanceData accountBalanceData = accountBalances.get(ai);
accountBalanceData.setBalance(random.nextBoolean() ? 0L : random.nextInt(100000));
}
repository.getAccountRepository().setAssetBalances(accountBalances);
repository.saveChanges();
System.out.println("Modifying random balances...");
// Fill with lots of random balance changes
for (ai = 0; ai < accounts.size(); ++ai) {
Account account = accounts.get(ai);
int assetId = random.nextInt(2);
// random zero, or non-zero, balance
long balance = random.nextBoolean() ? 0L : random.nextInt(100000);
accountBalances.add(new AccountBalanceData(account.getAddress(), assetId, balance));
}
repository.getAccountRepository().modifyAssetBalances(accountBalances);
repository.saveChanges();
System.out.println("Deleting all balances...");
// Now simply delete all balances
for (ai = 0; ai < accountBalances.size(); ++ai)
accountBalances.get(ai).setBalance(0L);
repository.getAccountRepository().setAssetBalances(accountBalances);
repository.saveChanges();
}
}
}

26
src/test/java/org/qortal/test/RepositoryTests.java
View File

@ -17,6 +17,7 @@ import static org.junit.Assert.*;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Executors;
@ -129,12 +130,18 @@ public class RepositoryTests extends Common {
/** Check that the <i>sub-query</i> used to fetch highest block height is optimized by HSQLDB. */
@Test
public void testBlockHeightSpeed() throws DataException, SQLException {
final int mintBlockCount = 30000;
try (final Repository repository = RepositoryManager.getRepository()) {
// Mint some blocks
System.out.println("Minting test blocks - should take approx. 30 seconds...");
for (int i = 0; i < 30000; ++i)
System.out.println(String.format("Minting %d test blocks - should take approx. 30 seconds...", mintBlockCount));
long beforeBigMint = System.currentTimeMillis();
for (int i = 0; i < mintBlockCount; ++i)
BlockUtils.mintBlock(repository);
System.out.println(String.format("Minting %d blocks actually took %d seconds", mintBlockCount, (System.currentTimeMillis() - beforeBigMint) / 1000L));
final HSQLDBRepository hsqldb = (HSQLDBRepository) repository;
// Too slow:
@ -287,6 +294,21 @@ public class RepositoryTests extends Common {
}
}
/** Test batched DELETE */
@Test
public void testBatchedDelete() {
// Generate test data
List<Object[]> batchedObjects = new ArrayList<>();
for (int i = 0; i < 100; ++i)
batchedObjects.add(new Object[] { String.valueOf(i), 1L });
try (final HSQLDBRepository hsqldb = (HSQLDBRepository) RepositoryManager.getRepository()) {
hsqldb.deleteBatch("AccountBalances", "account = ? AND asset_id = ?", batchedObjects);
} catch (DataException | SQLException e) {
fail("Batched delete failed: " + e.getMessage());
}
}
public static void hsqldbSleep(int millis) throws SQLException {
System.out.println(String.format("HSQLDB sleep() thread ID: %s", Thread.currentThread().getId()));

2
src/test/java/org/qortal/test/minting/RewardShareTests.java
View File

@ -121,7 +121,7 @@ public class RewardShareTests extends Common {
Transaction transaction = Transaction.fromData(repository, transactionData);
ValidationResult validationResult = transaction.isValidUnconfirmed();
assertEquals("Initial 0% share should be invalid", ValidationResult.INVALID_REWARD_SHARE_PERCENT, validationResult);
assertNotSame("Creating reward-share with 'cancel' share-percent should be invalid", ValidationResult.OK, validationResult);
}
}