Redoing account balances with block height.

*** WARNING ***
Possible block reward bug in this commit. Further investigation needed.

Reverted AccountBalances back to height-less form.
Added HistoricAccountBalances table that is populated via trigger on AccountBalances.
This saves work when performing common requests for latest/confirmed balances,
shunting the extra work to when requesting height-related account balances.

Unified API call GET /addresses/balance/{address} by having address/assetId/height as
query params.

Simpler call for fetching legacy QORA holders during block rewarding.

Improved SQL for fetching asset balances, in all conditions,
e.g. with/without filtering addresses, with/without filtering assetIds,
etc.

Unit test for above to make sure query execution is fast enough.
(At one point, some SQL query was taking 6 seconds!)

Added optional 'height' Integer to AccountBalanceData, but this
is not populated/used very often.

HSQLDBAccountRepository.save(AccountBalanceData) now checks zero balance saves
to see if the row can be deleted instead. This fixes a lot of unhappy tests
that complain that there are residual account balance rows left after
post-test orphaning back to genesis block.

Yet more tests.
Removed very old 'TransactionTests' which are mostly covered in more specific tests elsewhere.
Added cancel-sell-name test from above.

Fixed AssetsApiTests to check for QORT not QORA!

Changed hard-coded assetIDs in test.common.AssetUtils in light of new LEGACY_QORA & QORT_FROM_QORA genesis assets.

Some test blockchain config changes.
This commit is contained in:
catbref
2019-11-08 17:30:09 +00:00
parent 31cbc1f15b
commit 30df320e7f
17 changed files with 390 additions and 1323 deletions

View File

@@ -5,14 +5,11 @@ import java.util.List;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qora.block.Block;
import org.qora.block.BlockChain;
import org.qora.data.account.AccountBalanceData;
import org.qora.data.account.AccountData;
import org.qora.data.account.RewardShareData;
import org.qora.data.block.BlockData;
import org.qora.data.transaction.TransactionData;
import org.qora.repository.BlockRepository;
import org.qora.repository.DataException;
import org.qora.repository.Repository;
import org.qora.transaction.Transaction;
@@ -54,35 +51,14 @@ public class Account {
return new AccountData(this.address);
}
// Balance manipulations - assetId is 0 for QORA
// Balance manipulations - assetId is 0 for QORT
public BigDecimal getBalance(long assetId, int confirmations) throws DataException {
// Simple case: we only need balance with 1 confirmation
if (confirmations == 1)
return this.getConfirmedBalance(assetId);
public BigDecimal getBalance(long assetId, int height) throws DataException {
AccountBalanceData accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId, height);
if (accountBalanceData == null)
return BigDecimal.ZERO.setScale(8);
/*
* For a balance with more confirmations work back from last block, undoing transactions involving this account, until we have processed required number
* of blocks.
*/
BlockRepository blockRepository = this.repository.getBlockRepository();
BigDecimal balance = this.getConfirmedBalance(assetId);
BlockData blockData = blockRepository.getLastBlock();
// Note: "blockData.getHeight() > 1" to make sure we don't examine genesis block
for (int i = 1; i < confirmations && blockData != null && blockData.getHeight() > 1; ++i) {
Block block = new Block(this.repository, blockData);
// CIYAM AT transactions should be fetched from repository so no special handling needed here
for (Transaction transaction : block.getTransactions())
if (transaction.isInvolved(this))
balance = balance.subtract(transaction.getAmount(this));
blockData = block.getParent();
}
// Return balance
return balance;
return accountBalanceData.getBalance();
}
public BigDecimal getConfirmedBalance(long assetId) throws DataException {

View File

@@ -191,7 +191,8 @@ public class AddressesResource {
@GET
@Path("/balance/{address}")
@Operation(
summary = "Returns the confirmed balance of the given address",
summary = "Returns account balance",
description = "Returns account's balance, optionally of given asset and at given height",
responses = {
@ApiResponse(
description = "the balance",
@@ -199,14 +200,27 @@ public class AddressesResource {
)
}
)
@ApiErrors({ApiError.INVALID_ADDRESS, ApiError.REPOSITORY_ISSUE})
public BigDecimal getConfirmedBalance(@PathParam("address") String address) {
@ApiErrors({ApiError.INVALID_ADDRESS, ApiError.INVALID_ASSET_ID, ApiError.INVALID_HEIGHT, ApiError.REPOSITORY_ISSUE})
public BigDecimal getBalance(@PathParam("address") String address,
@QueryParam("assetId") Long assetId,
@QueryParam("height") Integer height) {
if (!Crypto.isValidAddress(address))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
try (final Repository repository = RepositoryManager.getRepository()) {
Account account = new Account(repository, address);
return account.getConfirmedBalance(Asset.QORT);
if (assetId == null)
assetId = Asset.QORT;
else if (!repository.getAssetRepository().assetExists(assetId))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ASSET_ID);
if (height == null)
height = repository.getBlockRepository().getBlockchainHeight();
else if (height <= 0)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT);
return account.getBalance(assetId, height);
} catch (ApiException e) {
throw e;
} catch (DataException e) {
@@ -214,21 +228,6 @@ public class AddressesResource {
}
}
@GET
@Path("/balance/{address}/{confirmations}")
@Operation(
summary = "Calculates the balance of the given address for the given confirmations",
responses = {
@ApiResponse(
description = "the balance",
content = @Content(schema = @Schema(type = "string", format = "number"))
)
}
)
public String getConfirmedBalance(@PathParam("address") String address, @PathParam("confirmations") int confirmations) {
throw new UnsupportedOperationException();
}
@GET
@Path("/publickey/{address}")
@Operation(

View File

@@ -8,7 +8,6 @@ import java.math.BigInteger;
import java.math.RoundingMode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -40,7 +39,6 @@ import org.qora.data.block.BlockTransactionData;
import org.qora.data.network.OnlineAccountData;
import org.qora.data.transaction.TransactionData;
import org.qora.repository.ATRepository;
import org.qora.repository.AccountRepository.BalanceOrdering;
import org.qora.repository.DataException;
import org.qora.repository.Repository;
import org.qora.repository.TransactionRepository;
@@ -1621,9 +1619,7 @@ public class Block {
BigDecimal qoraHoldersAmount = BlockChain.getInstance().getQoraHoldersShare().multiply(totalAmount).setScale(8, RoundingMode.DOWN);
LOGGER.trace(() -> String.format("Legacy QORA holders share of %s: %s", totalAmount.toPlainString(), qoraHoldersAmount.toPlainString()));
List<String> assetAddresses = Collections.emptyList();
List<Long> assetIds = Collections.singletonList(Asset.LEGACY_QORA);
List<AccountBalanceData> qoraHolders = this.repository.getAccountRepository().getAssetBalances(assetAddresses, assetIds, BalanceOrdering.ASSET_ACCOUNT, true, null, null, null);
List<AccountBalanceData> qoraHolders = this.repository.getAccountRepository().getAssetBalances(Asset.LEGACY_QORA, true);
// Filter out qoraHolders who have received max QORT due to holding legacy QORA, (ratio from blockchain config)
BigDecimal qoraPerQortReward = BlockChain.getInstance().getQoraPerQortReward();
@@ -1708,7 +1704,12 @@ public class Block {
}
qoraHolderAccount.setConfirmedBalance(Asset.QORT, qoraHolderAccount.getConfirmedBalance(Asset.QORT).add(holderReward));
qoraHolderAccount.setConfirmedBalance(Asset.QORT_FROM_QORA, newQortFromQoraBalance);
if (newQortFromQoraBalance.signum() > 0)
qoraHolderAccount.setConfirmedBalance(Asset.QORT_FROM_QORA, newQortFromQoraBalance);
else
// Remove QORT_FROM_QORA balance as it's zero
qoraHolderAccount.deleteBalance(Asset.QORT_FROM_QORA);
sharedAmount = sharedAmount.add(holderReward);
}

View File

@@ -13,7 +13,9 @@ public class AccountBalanceData {
private String address;
private long assetId;
private BigDecimal balance;
// Not always present:
private Integer height;
private String assetName;
// Constructors
@@ -22,15 +24,22 @@ public class AccountBalanceData {
protected AccountBalanceData() {
}
public AccountBalanceData(String address, long assetId, BigDecimal balance, String assetName) {
public AccountBalanceData(String address, long assetId, BigDecimal balance) {
this.address = address;
this.assetId = assetId;
this.balance = balance;
this.assetName = assetName;
}
public AccountBalanceData(String address, long assetId, BigDecimal balance) {
this(address, assetId, balance, null);
public AccountBalanceData(String address, long assetId, BigDecimal balance, int height) {
this(address, assetId, balance);
this.height = height;
}
public AccountBalanceData(String address, long assetId, BigDecimal balance, String assetName) {
this(address, assetId, balance);
this.assetName = assetName;
}
// Getters/Setters
@@ -51,6 +60,10 @@ public class AccountBalanceData {
this.balance = balance;
}
public Integer getHeight() {
return this.height;
}
public String getAssetName() {
return this.assetName;
}

View File

@@ -89,6 +89,9 @@ public interface AccountRepository {
/** Returns account balance data for address & assetId at (or before) passed block height. */
public AccountBalanceData getBalance(String address, long assetId, int height) throws DataException;
/** Returns per-height historic balance for address & assetId. */
public List<AccountBalanceData> getHistoricBalances(String address, long assetId) throws DataException;
public enum BalanceOrdering {
ASSET_BALANCE_ACCOUNT,
ACCOUNT_ASSET,

View File

@@ -259,7 +259,7 @@ public class HSQLDBAccountRepository implements AccountRepository {
@Override
public AccountBalanceData getBalance(String address, long assetId) throws DataException {
String sql = "SELECT balance FROM AccountBalances WHERE account = ? AND asset_id = ? ORDER BY height DESC LIMIT 1";
String sql = "SELECT IFNULL(balance, 0) FROM AccountBalances WHERE account = ? AND asset_id = ? LIMIT 1";
try (ResultSet resultSet = this.repository.checkedExecute(sql, address, assetId)) {
if (resultSet == null)
@@ -275,7 +275,7 @@ public class HSQLDBAccountRepository implements AccountRepository {
@Override
public AccountBalanceData getBalance(String address, long assetId, int height) throws DataException {
String sql = "SELECT balance FROM AccountBalances WHERE account = ? AND asset_id = ? AND height <= ? ORDER BY height DESC LIMIT 1";
String sql = "SELECT IFNULL(balance, 0) FROM HistoricAccountBalances WHERE account = ? AND asset_id = ? AND height <= ? ORDER BY height DESC LIMIT 1";
try (ResultSet resultSet = this.repository.checkedExecute(sql, address, assetId, height)) {
if (resultSet == null)
@@ -289,10 +289,32 @@ public class HSQLDBAccountRepository implements AccountRepository {
}
}
@Override
public List<AccountBalanceData> getHistoricBalances(String address, long assetId) throws DataException {
String sql = "SELECT height, balance FROM HistoricAccountBalances WHERE account = ? AND asset_id = ? ORDER BY height DESC";
List<AccountBalanceData> historicBalances = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql, address, assetId)) {
if (resultSet == null)
return historicBalances;
do {
int height = resultSet.getInt(1);
BigDecimal balance = resultSet.getBigDecimal(2);
historicBalances.add(new AccountBalanceData(address, assetId, balance, height));
} while (resultSet.next());
return historicBalances;
} catch (SQLException e) {
throw new DataException("Unable to fetch historic account balances from repository", e);
}
}
@Override
public List<AccountBalanceData> getAssetBalances(long assetId, Boolean excludeZero) throws DataException {
StringBuilder sql = new StringBuilder(1024);
sql.append("SELECT account, IFNULL(balance, 0) FROM NewestAccountBalances WHERE asset_id = ?");
sql.append("SELECT account, IFNULL(balance, 0) FROM AccountBalances WHERE asset_id = ?");
if (excludeZero != null && excludeZero)
sql.append(" AND balance != 0");
@@ -321,76 +343,83 @@ public class HSQLDBAccountRepository implements AccountRepository {
StringBuilder sql = new StringBuilder(1024);
sql.append("SELECT account, asset_id, IFNULL(balance, 0), asset_name FROM ");
if (!addresses.isEmpty()) {
sql.append("(VALUES ");
final boolean haveAddresses = addresses != null && !addresses.isEmpty();
final boolean haveAssetIds = assetIds != null && !assetIds.isEmpty();
final int addressesSize = addresses.size();
for (int ai = 0; ai < addressesSize; ++ai) {
if (ai != 0)
sql.append(", ");
// Fill temporary table with filtering addresses/assetIDs
if (haveAddresses)
HSQLDBRepository.temporaryValuesTableSql(sql, addresses.size(), "TmpAccounts", "account");
sql.append("(?)");
if (haveAssetIds) {
if (haveAddresses)
sql.append("CROSS JOIN ");
HSQLDBRepository.temporaryValuesTableSql(sql, assetIds, "TmpAssetIds", "asset_id");
}
if (haveAddresses || haveAssetIds) {
// Now use temporary table to filter AccountBalances (using index) and optional zero balance exclusion
sql.append("JOIN AccountBalances ON ");
if (haveAddresses)
sql.append("AccountBalances.account = TmpAccounts.account ");
if (haveAssetIds) {
if (haveAddresses)
sql.append("AND ");
sql.append("AccountBalances.asset_id = TmpAssetIds.asset_id ");
}
sql.append(") AS Accounts (account) ");
sql.append("CROSS JOIN Assets LEFT OUTER JOIN NewestAccountBalances USING (asset_id, account) ");
if (!haveAddresses || (excludeZero != null && excludeZero))
sql.append("AND AccountBalances.balance != 0 ");
} else {
// Simplier, no-address query
sql.append("NewestAccountBalances NATURAL JOIN Assets ");
// Simpler form if no filtering
sql.append("AccountBalances ");
// Zero balance exclusion comes later
}
if (!assetIds.isEmpty()) {
// longs are safe enough to use literally
sql.append("WHERE asset_id IN (");
// Join for asset name
sql.append("JOIN Assets ON Assets.asset_id = AccountBalances.asset_id ");
final int assetIdsSize = assetIds.size();
for (int ai = 0; ai < assetIdsSize; ++ai) {
if (ai != 0)
sql.append(", ");
// Zero balance exclusion if no filtering
if (!haveAddresses && !haveAssetIds && excludeZero != null && excludeZero)
sql.append("WHERE AccountBalances.balance != 0 ");
sql.append(assetIds.get(ai));
if (balanceOrdering != null) {
String[] orderingColumns;
switch (balanceOrdering) {
case ACCOUNT_ASSET:
orderingColumns = new String[] { "account", "asset_id" };
break;
case ASSET_ACCOUNT:
orderingColumns = new String[] { "asset_id", "account" };
break;
case ASSET_BALANCE_ACCOUNT:
orderingColumns = new String[] { "asset_id", "balance", "account" };
break;
default:
throw new DataException(String.format("Unsupported asset balance result ordering: %s", balanceOrdering.name()));
}
sql.append(") ");
}
sql.append("ORDER BY ");
for (int oi = 0; oi < orderingColumns.length; ++oi) {
if (oi != 0)
sql.append(", ");
// For no-address queries, or unless specifically requested, only return accounts with non-zero balance
if (addresses.isEmpty() || (excludeZero != null && excludeZero)) {
sql.append(assetIds.isEmpty() ? " WHERE " : " AND ");
sql.append("balance != 0 ");
}
String[] orderingColumns;
switch (balanceOrdering) {
case ACCOUNT_ASSET:
orderingColumns = new String[] { "account", "asset_id" };
break;
case ASSET_ACCOUNT:
orderingColumns = new String[] { "asset_id", "account" };
break;
case ASSET_BALANCE_ACCOUNT:
orderingColumns = new String[] { "asset_id", "balance", "account" };
break;
default:
throw new DataException(String.format("Unsupported asset balance result ordering: %s", balanceOrdering.name()));
}
sql.append("ORDER BY ");
for (int oi = 0; oi < orderingColumns.length; ++oi) {
if (oi != 0)
sql.append(", ");
sql.append(orderingColumns[oi]);
if (reverse != null && reverse)
sql.append(" DESC");
sql.append(orderingColumns[oi]);
if (reverse != null && reverse)
sql.append(" DESC");
}
}
HSQLDBRepository.limitOffsetSql(sql, limit, offset);
String[] addressesArray = addresses.toArray(new String[addresses.size()]);
String[] addressesArray = addresses == null ? new String[0] : addresses.toArray(new String[addresses.size()]);
List<AccountBalanceData> accountBalances = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), (Object[]) addressesArray)) {
@@ -414,15 +443,41 @@ public class HSQLDBAccountRepository implements AccountRepository {
@Override
public void save(AccountBalanceData accountBalanceData) throws DataException {
// If balance is zero and there are no prior historic balance, then simply delete balances for this assetId (typically during orphaning)
if (accountBalanceData.getBalance().signum() == 0) {
boolean hasPriorBalances;
try {
hasPriorBalances = this.repository.exists("HistoricAccountBalances", "account = ? AND asset_id = ? AND height < (SELECT IFNULL(MAX(height), 1) FROM Blocks)",
accountBalanceData.getAddress(), accountBalanceData.getAssetId());
} catch (SQLException e) {
throw new DataException("Unable to check for historic account balances in repository", e);
}
if (!hasPriorBalances) {
try {
this.repository.delete("AccountBalances", "account = ? AND asset_id = ?", accountBalanceData.getAddress(), accountBalanceData.getAssetId());
} catch (SQLException e) {
throw new DataException("Unable to delete account balance from repository", e);
}
// I don't think we need to do this as Block.orphan() would do this for us?
try {
this.repository.delete("HistoricAccountBalances", "account = ? AND asset_id = ?", accountBalanceData.getAddress(), accountBalanceData.getAssetId());
} catch (SQLException e) {
throw new DataException("Unable to delete historic account balances from repository", e);
}
return;
}
}
HSQLDBSaver saveHelper = new HSQLDBSaver("AccountBalances");
saveHelper.bind("account", accountBalanceData.getAddress()).bind("asset_id", accountBalanceData.getAssetId()).bind("balance",
accountBalanceData.getBalance());
try {
// Fill in 'height'
int height = this.repository.checkedExecute("SELECT COUNT(*) + 1 FROM Blocks").getInt(1);
saveHelper.bind("height", height);
// HistoricAccountBalances auto-updated via trigger
saveHelper.execute(this.repository);
} catch (SQLException e) {
@@ -437,14 +492,20 @@ public class HSQLDBAccountRepository implements AccountRepository {
} catch (SQLException e) {
throw new DataException("Unable to delete account balance from repository", e);
}
try {
this.repository.delete("HistoricAccountBalances", "account = ? AND asset_id = ?", address, assetId);
} catch (SQLException e) {
throw new DataException("Unable to delete historic account balances from repository", e);
}
}
@Override
public int deleteBalancesFromHeight(int height) throws DataException {
try {
return this.repository.delete("AccountBalances", "height >= ?", height);
return this.repository.delete("HistoricAccountBalances", "height >= ?", height);
} catch (SQLException e) {
throw new DataException("Unable to delete old account balances from repository", e);
throw new DataException("Unable to delete historic account balances from repository", e);
}
}

View File

@@ -847,21 +847,19 @@ public class HSQLDBDatabaseUpdates {
+ "PRIMARY KEY (account), FOREIGN KEY (account) REFERENCES Accounts (account) ON DELETE CASCADE)");
break;
case 60: // Adding height to account balances
// We need to drop primary key first
stmt.execute("ALTER TABLE AccountBalances DROP PRIMARY KEY");
// Add height to account balances
stmt.execute("ALTER TABLE AccountBalances ADD COLUMN height INT NOT NULL DEFAULT 0 BEFORE BALANCE");
// Add new primary key
stmt.execute("ALTER TABLE AccountBalances ADD PRIMARY KEY (asset_id, account, height)");
/// Create a view for account balances at greatest height
stmt.execute("CREATE VIEW NewestAccountBalances (account, asset_id, balance) AS "
+ "SELECT AccountBalances.account, AccountBalances.asset_id, AccountBalances.balance FROM AccountBalances "
+ "LEFT OUTER JOIN AccountBalances AS NewerAccountBalances "
+ "ON NewerAccountBalances.account = AccountBalances.account "
+ "AND NewerAccountBalances.asset_id = AccountBalances.asset_id "
+ "AND NewerAccountBalances.height > AccountBalances.height "
+ "WHERE NewerAccountBalances.height IS NULL");
case 60:
// Index for speeding up fetch legacy QORA holders for Block processing
stmt.execute("CREATE INDEX AccountBalances_Asset_Balance_Index ON AccountBalances (asset_id, balance)");
// Tracking height-history to account balances
stmt.execute("CREATE TABLE HistoricAccountBalances (account QoraAddress, asset_id AssetID, height INT DEFAULT 1, balance QoraAmount NOT NULL, "
+ "PRIMARY KEY (account, asset_id, height), FOREIGN KEY (account) REFERENCES Accounts (account) ON DELETE CASCADE)");
// Create triggers on changes to AccountBalances rows to update historic
stmt.execute("CREATE TRIGGER Historic_account_balance_insert_trigger AFTER INSERT ON AccountBalances REFERENCING NEW ROW AS new_row FOR EACH ROW "
+ "INSERT INTO HistoricAccountBalances VALUES (new_row.account, new_row.asset_id, (SELECT IFNULL(MAX(height), 0) + 1 FROM Blocks), new_row.balance) "
+ "ON DUPLICATE KEY UPDATE balance = new_row.balance");
stmt.execute("CREATE TRIGGER Historic_account_balance_update_trigger AFTER UPDATE ON AccountBalances REFERENCING NEW ROW AS new_row FOR EACH ROW "
+ "INSERT INTO HistoricAccountBalances VALUES (new_row.account, new_row.asset_id, (SELECT IFNULL(MAX(height), 0) + 1 FROM Blocks), new_row.balance) "
+ "ON DUPLICATE KEY UPDATE balance = new_row.balance");
break;
default:

View File

@@ -577,6 +577,52 @@ public class HSQLDBRepository implements Repository {
}
}
/**
* Appends SQL for filling a temporary VALUES table, values NOT supplied.
* <p>
* (Convenience method for HSQLDB repository subclasses).
*/
/* package */ static void temporaryValuesTableSql(StringBuilder stringBuilder, int valuesCount, String tableName, String columnName) {
stringBuilder.append("(VALUES ");
for (int i = 0; i < valuesCount; ++i) {
if (i != 0)
stringBuilder.append(", ");
stringBuilder.append("(?)");
}
stringBuilder.append(") AS ");
stringBuilder.append(tableName);
stringBuilder.append(" (");
stringBuilder.append(columnName);
stringBuilder.append(") ");
}
/**
* Appends SQL for filling a temporary VALUES table, literal values ARE supplied.
* <p>
* (Convenience method for HSQLDB repository subclasses).
*/
/* package */ static void temporaryValuesTableSql(StringBuilder stringBuilder, List<? extends Object> values, String tableName, String columnName) {
stringBuilder.append("(VALUES ");
for (int i = 0; i < values.size(); ++i) {
if (i != 0)
stringBuilder.append(", ");
stringBuilder.append("(");
stringBuilder.append(values.get(i));
stringBuilder.append(")");
}
stringBuilder.append(") AS ");
stringBuilder.append(tableName);
stringBuilder.append(" (");
stringBuilder.append(columnName);
stringBuilder.append(") ");
}
/** Logs other HSQLDB sessions then re-throws passed exception */
public SQLException examineException(SQLException e) throws SQLException {
LOGGER.error(String.format("HSQLDB error (session %d): %s", this.sessionId, e.getMessage()), e);