AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Rip out historic account balances as they take up too much DB space.

Browse Source
This commit is contained in:
catbref 2020-03-26 11:48:04 +00:00
parent 558263521c
commit 7bb2f841ad
7 changed files with 15 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/qortal/account/Account.java
View File

@ -53,8 +53,8 @@ public class Account {
// Balance manipulations - assetId is 0 for QORT
public BigDecimal getBalance(long assetId, int height) throws DataException {
AccountBalanceData accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId, height);
public BigDecimal getBalance(long assetId) throws DataException {
AccountBalanceData accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId);
if (accountBalanceData == null)
return BigDecimal.ZERO.setScale(8);

12
src/main/java/org/qortal/api/resource/AddressesResource.java
View File

@ -192,7 +192,7 @@ public class AddressesResource {
@Path("/balance/{address}")
@Operation(
summary = "Returns account balance",
description = "Returns account's balance, optionally of given asset and at given height",
description = "Returns account's QORT balance, or of other specified asset",
responses = {
@ApiResponse(
description = "the balance",
@ -202,8 +202,7 @@ public class AddressesResource {
)
@ApiErrors({ApiError.INVALID_ADDRESS, ApiError.INVALID_ASSET_ID, ApiError.INVALID_HEIGHT, ApiError.REPOSITORY_ISSUE})
public BigDecimal getBalance(@PathParam("address") String address,
@QueryParam("assetId") Long assetId,
@QueryParam("height") Integer height) {
@QueryParam("assetId") Long assetId) {
if (!Crypto.isValidAddress(address))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
@ -215,12 +214,7 @@ public class AddressesResource {
else if (!repository.getAssetRepository().assetExists(assetId))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ASSET_ID);
if (height == null)
height = repository.getBlockRepository().getBlockchainHeight();
else if (height <= 0)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT);
return account.getBalance(assetId, height);
return account.getBalance(assetId);
} catch (ApiException e) {
throw e;
} catch (DataException e) {

3
src/main/java/org/qortal/block/Block.java
View File

@ -1466,9 +1466,6 @@ public class Block {
decreaseAccountLevels();
}
// Delete orphaned balances
this.repository.getAccountRepository().deleteBalancesFromHeight(this.blockData.getHeight());
// Delete block from blockchain
this.repository.getBlockRepository().delete(this.blockData);
this.blockData.setHeight(null);

9
src/main/java/org/qortal/repository/AccountRepository.java
View File

@ -96,12 +96,6 @@ public interface AccountRepository {
public AccountBalanceData getBalance(String address, long assetId) throws DataException;
/** Returns account balance data for address & assetId at (or before) passed block height. */
public AccountBalanceData getBalance(String address, long assetId, int height) throws DataException;
/** Returns per-height historic balance for address & assetId. */
public List<AccountBalanceData> getHistoricBalances(String address, long assetId) throws DataException;
public enum BalanceOrdering {
ASSET_BALANCE_ACCOUNT,
ACCOUNT_ASSET,
@ -118,9 +112,6 @@ public interface AccountRepository {
public void delete(String address, long assetId) throws DataException;
/** Deletes orphaned balances at block height >= <tt>height</tt>. */
public int deleteBalancesFromHeight(int height) throws DataException;
// Reward-shares
public RewardShareData getRewardShare(byte[] mintingAccountPublicKey, String recipientAccount) throws DataException;

101
src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java
View File

@ -327,44 +327,6 @@ public class HSQLDBAccountRepository implements AccountRepository {
}
}
@Override
public AccountBalanceData getBalance(String address, long assetId, int height) throws DataException {
String sql = "SELECT IFNULL(balance, 0) FROM HistoricAccountBalances WHERE account = ? AND asset_id = ? AND height <= ? ORDER BY height DESC LIMIT 1";
try (ResultSet resultSet = this.repository.checkedExecute(sql, address, assetId, height)) {
if (resultSet == null)
return null;
BigDecimal balance = resultSet.getBigDecimal(1).setScale(8);
return new AccountBalanceData(address, assetId, balance);
} catch (SQLException e) {
throw new DataException("Unable to fetch account balance from repository", e);
}
}
@Override
public List<AccountBalanceData> getHistoricBalances(String address, long assetId) throws DataException {
String sql = "SELECT height, balance FROM HistoricAccountBalances WHERE account = ? AND asset_id = ? ORDER BY height DESC";
List<AccountBalanceData> historicBalances = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql, address, assetId)) {
if (resultSet == null)
return historicBalances;
do {
int height = resultSet.getInt(1);
BigDecimal balance = resultSet.getBigDecimal(2);
historicBalances.add(new AccountBalanceData(address, assetId, balance, height));
} while (resultSet.next());
return historicBalances;
} catch (SQLException e) {
throw new DataException("Unable to fetch historic account balances from repository", e);
}
}
@Override
public List<AccountBalanceData> getAssetBalances(long assetId, Boolean excludeZero) throws DataException {
StringBuilder sql = new StringBuilder(1024);
@ -510,19 +472,6 @@ public class HSQLDBAccountRepository implements AccountRepository {
} catch (SQLException e) {
throw new DataException("Unable to reduce account balance in repository", e);
}
// If balance is now zero, and there are no prior historic balances, then simply delete row for this address-assetId (typically during orphaning)
String deleteWhereSql = "account = ? AND asset_id = ? AND balance = 0 " + // covers "if balance now zero"
"AND (" +
"SELECT TRUE FROM HistoricAccountBalances " +
"WHERE account = ? AND asset_id = ? AND height < (SELECT height - 1 FROM NextBlockHeight) " +
"LIMIT 1" +
")";
try {
this.repository.delete("AccountBalances", deleteWhereSql, address, assetId, address, assetId);
} catch (SQLException e) {
throw new DataException("Unable to prune account balance in repository", e);
}
} else {
// We have to ensure parent row exists to satisfy foreign key constraint
try {
@ -545,47 +494,12 @@ public class HSQLDBAccountRepository implements AccountRepository {
@Override
public void save(AccountBalanceData accountBalanceData) throws DataException {
// If balance is zero and there are no prior historic balance, then simply delete balances for this assetId (typically during orphaning)
if (accountBalanceData.getBalance().signum() == 0) {
String existsSql = "account = ? AND asset_id = ? AND height < (SELECT height - 1 FROM NextBlockHeight)"; // height prior to current block. no matches (obviously) prior to genesis block
boolean hasPriorBalances;
try {
hasPriorBalances = this.repository.exists("HistoricAccountBalances", existsSql, accountBalanceData.getAddress(), accountBalanceData.getAssetId());
} catch (SQLException e) {
throw new DataException("Unable to check for historic account balances in repository", e);
}
if (!hasPriorBalances) {
try {
this.repository.delete("AccountBalances", "account = ? AND asset_id = ?", accountBalanceData.getAddress(), accountBalanceData.getAssetId());
} catch (SQLException e) {
throw new DataException("Unable to delete account balance from repository", e);
}
/*
* I don't think we need to do this as Block.orphan() would do this for us?
try {
this.repository.delete("HistoricAccountBalances", "account = ? AND asset_id = ?", accountBalanceData.getAddress(), accountBalanceData.getAssetId());
} catch (SQLException e) {
throw new DataException("Unable to delete historic account balances from repository", e);
}
*/
return;
}
}
HSQLDBSaver saveHelper = new HSQLDBSaver("AccountBalances");
saveHelper.bind("account", accountBalanceData.getAddress()).bind("asset_id", accountBalanceData.getAssetId()).bind("balance",
accountBalanceData.getBalance());
try {
// HistoricAccountBalances auto-updated via trigger
saveHelper.execute(this.repository);
} catch (SQLException e) {
throw new DataException("Unable to save account balance into repository", e);
@ -599,21 +513,6 @@ public class HSQLDBAccountRepository implements AccountRepository {
} catch (SQLException e) {
throw new DataException("Unable to delete account balance from repository", e);
}
try {
this.repository.delete("HistoricAccountBalances", "account = ? AND asset_id = ?", address, assetId);
} catch (SQLException e) {
throw new DataException("Unable to delete historic account balances from repository", e);
}
}
@Override
public int deleteBalancesFromHeight(int height) throws DataException {
try {
return this.repository.delete("HistoricAccountBalances", "height >= ?", height);
} catch (SQLException e) {
throw new DataException("Unable to delete historic account balances from repository", e);
}
}
// Reward-Share

10
src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
View File

@ -956,6 +956,16 @@ public class HSQLDBDatabaseUpdates {
stmt.execute("SET FILES WRITE DELAY 5"); // only fsync() every 5 seconds
break;
case 69:
// Get rid of historic account balances as they simply use up way too much space
stmt.execute("DROP TRIGGER Historic_Account_Balance_Insert_Trigger");
stmt.execute("DROP TRIGGER Historic_Account_Balance_Update_Trigger");
stmt.execute("DROP TABLE HistoricAccountBalances");
// Reclaim space
stmt.execute("CHECKPOINT");
stmt.execute("CHECKPOINT DEFRAG");
break;
default:
// nothing to do
return false;

123
src/test/java/org/qortal/test/AccountBalanceTests.java
View File

@ -14,15 +14,10 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.Account;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.account.PublicKeyAccount;
import org.qortal.asset.Asset;
import org.qortal.block.BlockChain;
import org.qortal.data.account.AccountBalanceData;
import org.qortal.data.account.AccountData;
import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.PaymentTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
@ -30,7 +25,6 @@ import org.qortal.repository.AccountRepository.BalanceOrdering;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TestAccount;
import org.qortal.test.common.TransactionUtils;
public class AccountBalanceTests extends Common {
@ -88,123 +82,6 @@ public class AccountBalanceTests extends Common {
}
}
/** Tests we can fetch initial balance when newer balance exists. */
@Test
public void testGetBalanceAtHeight() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
TestAccount alice = Common.getTestAccount(repository, "alice");
BigDecimal initialBalance = testNewerBalance(repository, alice);
// Fetch all historic balances
List<AccountBalanceData> historicBalances = repository.getAccountRepository().getHistoricBalances(alice.getAddress(), Asset.QORT);
for (AccountBalanceData historicBalance : historicBalances)
System.out.println(String.format("Balance at height %d: %s", historicBalance.getHeight(), historicBalance.getBalance().toPlainString()));
// Fetch balance at height 1, even though newer balance exists
AccountBalanceData accountBalanceData = repository.getAccountRepository().getBalance(alice.getAddress(), Asset.QORT, 1);
BigDecimal genesisBalance = accountBalanceData.getBalance();
// Confirm genesis balance is same as initial
assertEqualBigDecimals("Genesis balance should match initial", initialBalance, genesisBalance);
}
}
/** Tests we can fetch balance with a height where no balance change occurred. */
@Test
public void testGetBalanceAtNearestHeight() throws DataException {
Random random = new Random();
byte[] publicKey = new byte[32];
random.nextBytes(publicKey);
try (final Repository repository = RepositoryManager.getRepository()) {
PublicKeyAccount recipientAccount = new PublicKeyAccount(repository, publicKey);
System.out.println(String.format("Test recipient: %s", recipientAccount.getAddress()));
// Mint a few blocks
for (int i = 0; i < 10; ++i)
BlockUtils.mintBlock(repository);
// Confirm recipient balance is zero
BigDecimal balance = recipientAccount.getConfirmedBalance(Asset.QORT);
assertEqualBigDecimals("recipient's balance should be zero", BigDecimal.ZERO, balance);
// Confirm recipient has no historic balances
List<AccountBalanceData> historicBalances = repository.getAccountRepository().getHistoricBalances(recipientAccount.getAddress(), Asset.QORT);
for (AccountBalanceData historicBalance : historicBalances)
System.err.println(String.format("Block %d: %s", historicBalance.getHeight(), historicBalance.getBalance().toPlainString()));
assertTrue("recipient should not have historic balances yet", historicBalances.isEmpty());
// Send 1 QORT to recipient
TestAccount sendingAccount = Common.getTestAccount(repository, "alice");
pay(repository, sendingAccount, recipientAccount, BigDecimal.ONE);
// Mint some more blocks
for (int i = 0; i < 10; ++i)
BlockUtils.mintBlock(repository);
// Send more QORT to recipient
BigDecimal amount = BigDecimal.valueOf(random.nextInt(123456));
pay(repository, sendingAccount, recipientAccount, amount);
BigDecimal totalAmount = BigDecimal.ONE.add(amount);
// Mint some more blocks
for (int i = 0; i < 10; ++i)
BlockUtils.mintBlock(repository);
// Confirm recipient balance is as expected
balance = recipientAccount.getConfirmedBalance(Asset.QORT);
assertEqualBigDecimals("recipient's balance incorrect", totalAmount, balance);
historicBalances = repository.getAccountRepository().getHistoricBalances(recipientAccount.getAddress(), Asset.QORT);
for (AccountBalanceData historicBalance : historicBalances)
System.out.println(String.format("Block %d: %s", historicBalance.getHeight(), historicBalance.getBalance().toPlainString()));
// Confirm balance as of 2 blocks ago
int height = repository.getBlockRepository().getBlockchainHeight();
balance = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 2).getBalance();
assertEqualBigDecimals("recipient's historic balance incorrect", totalAmount, balance);
// Confirm balance prior to last payment
balance = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 15).getBalance();
assertEqualBigDecimals("recipient's historic balance incorrect", BigDecimal.ONE, balance);
// Orphan blocks to before last payment
BlockUtils.orphanBlocks(repository, 10 + 5);
// Re-check balance from (now) invalid height
AccountBalanceData accountBalanceData = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 2);
balance = accountBalanceData.getBalance();
assertEqualBigDecimals("recipient's invalid-height balance should be one", BigDecimal.ONE, balance);
// Orphan blocks to before initial 1 QORT payment
BlockUtils.orphanBlocks(repository, 10 + 5);
// Re-check balance from (now) invalid height
accountBalanceData = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 2);
assertNull("recipient's invalid-height balance data should be null", accountBalanceData);
// Confirm recipient has no historic balances
historicBalances = repository.getAccountRepository().getHistoricBalances(recipientAccount.getAddress(), Asset.QORT);
for (AccountBalanceData historicBalance : historicBalances)
System.err.println(String.format("Block %d: %s", historicBalance.getHeight(), historicBalance.getBalance().toPlainString()));
assertTrue("recipient should have no remaining historic balances", historicBalances.isEmpty());
}
}
private void pay(Repository repository, PrivateKeyAccount sendingAccount, Account recipientAccount, BigDecimal amount) throws DataException {
byte[] reference = sendingAccount.getLastReference();
long timestamp = repository.getTransactionRepository().fromSignature(reference).getTimestamp() + 1;
int txGroupId = 0;
BigDecimal fee = BlockChain.getInstance().getUnitFee();
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, sendingAccount.getPublicKey(), fee, null);
TransactionData transactionData = new PaymentTransactionData(baseTransactionData, recipientAccount.getAddress(), amount);
TransactionUtils.signAndMint(repository, transactionData, sendingAccount);
}
/** Tests SQL query speed for account balance fetches. */
@Test
public void testRepositorySpeed() throws DataException, SQLException {