From 7bb2f841adfa38149ff951d41fafbd31931ff48f Mon Sep 17 00:00:00 2001 From: catbref Date: Thu, 26 Mar 2020 11:48:04 +0000 Subject: [PATCH] Rip out historic account balances as they take up too much DB space. --- src/main/java/org/qortal/account/Account.java | 4 +- .../api/resource/AddressesResource.java | 12 +- src/main/java/org/qortal/block/Block.java | 3 - .../qortal/repository/AccountRepository.java | 9 -- .../hsqldb/HSQLDBAccountRepository.java | 101 -------------- .../hsqldb/HSQLDBDatabaseUpdates.java | 10 ++ .../org/qortal/test/AccountBalanceTests.java | 123 ------------------ 7 files changed, 15 insertions(+), 247 deletions(-) diff --git a/src/main/java/org/qortal/account/Account.java b/src/main/java/org/qortal/account/Account.java index df1c2db9..829243c3 100644 --- a/src/main/java/org/qortal/account/Account.java +++ b/src/main/java/org/qortal/account/Account.java @@ -53,8 +53,8 @@ public class Account { // Balance manipulations - assetId is 0 for QORT - public BigDecimal getBalance(long assetId, int height) throws DataException { - AccountBalanceData accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId, height); + public BigDecimal getBalance(long assetId) throws DataException { + AccountBalanceData accountBalanceData = this.repository.getAccountRepository().getBalance(this.address, assetId); if (accountBalanceData == null) return BigDecimal.ZERO.setScale(8); diff --git a/src/main/java/org/qortal/api/resource/AddressesResource.java b/src/main/java/org/qortal/api/resource/AddressesResource.java index 26108d92..f13a622c 100644 --- a/src/main/java/org/qortal/api/resource/AddressesResource.java +++ b/src/main/java/org/qortal/api/resource/AddressesResource.java @@ -192,7 +192,7 @@ public class AddressesResource { @Path("/balance/{address}") @Operation( summary = "Returns account balance", - description = "Returns account's balance, optionally of given asset and at given height", + description = "Returns account's QORT balance, or of other specified asset", responses = { @ApiResponse( description = "the balance", @@ -202,8 +202,7 @@ public class AddressesResource { ) @ApiErrors({ApiError.INVALID_ADDRESS, ApiError.INVALID_ASSET_ID, ApiError.INVALID_HEIGHT, ApiError.REPOSITORY_ISSUE}) public BigDecimal getBalance(@PathParam("address") String address, - @QueryParam("assetId") Long assetId, - @QueryParam("height") Integer height) { + @QueryParam("assetId") Long assetId) { if (!Crypto.isValidAddress(address)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); @@ -215,12 +214,7 @@ public class AddressesResource { else if (!repository.getAssetRepository().assetExists(assetId)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ASSET_ID); - if (height == null) - height = repository.getBlockRepository().getBlockchainHeight(); - else if (height <= 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT); - - return account.getBalance(assetId, height); + return account.getBalance(assetId); } catch (ApiException e) { throw e; } catch (DataException e) { diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index d0eaeab7..483090bc 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -1466,9 +1466,6 @@ public class Block { decreaseAccountLevels(); } - // Delete orphaned balances - this.repository.getAccountRepository().deleteBalancesFromHeight(this.blockData.getHeight()); - // Delete block from blockchain this.repository.getBlockRepository().delete(this.blockData); this.blockData.setHeight(null); diff --git a/src/main/java/org/qortal/repository/AccountRepository.java b/src/main/java/org/qortal/repository/AccountRepository.java index f983e8a4..15aba42a 100644 --- a/src/main/java/org/qortal/repository/AccountRepository.java +++ b/src/main/java/org/qortal/repository/AccountRepository.java @@ -96,12 +96,6 @@ public interface AccountRepository { public AccountBalanceData getBalance(String address, long assetId) throws DataException; - /** Returns account balance data for address & assetId at (or before) passed block height. */ - public AccountBalanceData getBalance(String address, long assetId, int height) throws DataException; - - /** Returns per-height historic balance for address & assetId. */ - public List getHistoricBalances(String address, long assetId) throws DataException; - public enum BalanceOrdering { ASSET_BALANCE_ACCOUNT, ACCOUNT_ASSET, @@ -118,9 +112,6 @@ public interface AccountRepository { public void delete(String address, long assetId) throws DataException; - /** Deletes orphaned balances at block height >= height. */ - public int deleteBalancesFromHeight(int height) throws DataException; - // Reward-shares public RewardShareData getRewardShare(byte[] mintingAccountPublicKey, String recipientAccount) throws DataException; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java index 0d406854..c38fa438 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBAccountRepository.java @@ -327,44 +327,6 @@ public class HSQLDBAccountRepository implements AccountRepository { } } - @Override - public AccountBalanceData getBalance(String address, long assetId, int height) throws DataException { - String sql = "SELECT IFNULL(balance, 0) FROM HistoricAccountBalances WHERE account = ? AND asset_id = ? AND height <= ? ORDER BY height DESC LIMIT 1"; - - try (ResultSet resultSet = this.repository.checkedExecute(sql, address, assetId, height)) { - if (resultSet == null) - return null; - - BigDecimal balance = resultSet.getBigDecimal(1).setScale(8); - - return new AccountBalanceData(address, assetId, balance); - } catch (SQLException e) { - throw new DataException("Unable to fetch account balance from repository", e); - } - } - - @Override - public List getHistoricBalances(String address, long assetId) throws DataException { - String sql = "SELECT height, balance FROM HistoricAccountBalances WHERE account = ? AND asset_id = ? ORDER BY height DESC"; - - List historicBalances = new ArrayList<>(); - try (ResultSet resultSet = this.repository.checkedExecute(sql, address, assetId)) { - if (resultSet == null) - return historicBalances; - - do { - int height = resultSet.getInt(1); - BigDecimal balance = resultSet.getBigDecimal(2); - - historicBalances.add(new AccountBalanceData(address, assetId, balance, height)); - } while (resultSet.next()); - - return historicBalances; - } catch (SQLException e) { - throw new DataException("Unable to fetch historic account balances from repository", e); - } - } - @Override public List getAssetBalances(long assetId, Boolean excludeZero) throws DataException { StringBuilder sql = new StringBuilder(1024); @@ -510,19 +472,6 @@ public class HSQLDBAccountRepository implements AccountRepository { } catch (SQLException e) { throw new DataException("Unable to reduce account balance in repository", e); } - - // If balance is now zero, and there are no prior historic balances, then simply delete row for this address-assetId (typically during orphaning) - String deleteWhereSql = "account = ? AND asset_id = ? AND balance = 0 " + // covers "if balance now zero" - "AND (" + - "SELECT TRUE FROM HistoricAccountBalances " + - "WHERE account = ? AND asset_id = ? AND height < (SELECT height - 1 FROM NextBlockHeight) " + - "LIMIT 1" + - ")"; - try { - this.repository.delete("AccountBalances", deleteWhereSql, address, assetId, address, assetId); - } catch (SQLException e) { - throw new DataException("Unable to prune account balance in repository", e); - } } else { // We have to ensure parent row exists to satisfy foreign key constraint try { @@ -545,47 +494,12 @@ public class HSQLDBAccountRepository implements AccountRepository { @Override public void save(AccountBalanceData accountBalanceData) throws DataException { - // If balance is zero and there are no prior historic balance, then simply delete balances for this assetId (typically during orphaning) - if (accountBalanceData.getBalance().signum() == 0) { - String existsSql = "account = ? AND asset_id = ? AND height < (SELECT height - 1 FROM NextBlockHeight)"; // height prior to current block. no matches (obviously) prior to genesis block - - boolean hasPriorBalances; - try { - hasPriorBalances = this.repository.exists("HistoricAccountBalances", existsSql, accountBalanceData.getAddress(), accountBalanceData.getAssetId()); - } catch (SQLException e) { - throw new DataException("Unable to check for historic account balances in repository", e); - } - - if (!hasPriorBalances) { - try { - this.repository.delete("AccountBalances", "account = ? AND asset_id = ?", accountBalanceData.getAddress(), accountBalanceData.getAssetId()); - } catch (SQLException e) { - throw new DataException("Unable to delete account balance from repository", e); - } - - /* - * I don't think we need to do this as Block.orphan() would do this for us? - - try { - this.repository.delete("HistoricAccountBalances", "account = ? AND asset_id = ?", accountBalanceData.getAddress(), accountBalanceData.getAssetId()); - } catch (SQLException e) { - throw new DataException("Unable to delete historic account balances from repository", e); - } - - */ - - return; - } - } - HSQLDBSaver saveHelper = new HSQLDBSaver("AccountBalances"); saveHelper.bind("account", accountBalanceData.getAddress()).bind("asset_id", accountBalanceData.getAssetId()).bind("balance", accountBalanceData.getBalance()); try { - // HistoricAccountBalances auto-updated via trigger - saveHelper.execute(this.repository); } catch (SQLException e) { throw new DataException("Unable to save account balance into repository", e); @@ -599,21 +513,6 @@ public class HSQLDBAccountRepository implements AccountRepository { } catch (SQLException e) { throw new DataException("Unable to delete account balance from repository", e); } - - try { - this.repository.delete("HistoricAccountBalances", "account = ? AND asset_id = ?", address, assetId); - } catch (SQLException e) { - throw new DataException("Unable to delete historic account balances from repository", e); - } - } - - @Override - public int deleteBalancesFromHeight(int height) throws DataException { - try { - return this.repository.delete("HistoricAccountBalances", "height >= ?", height); - } catch (SQLException e) { - throw new DataException("Unable to delete historic account balances from repository", e); - } } // Reward-Share diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 10489ab4..d49b7e7a 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -956,6 +956,16 @@ public class HSQLDBDatabaseUpdates { stmt.execute("SET FILES WRITE DELAY 5"); // only fsync() every 5 seconds break; + case 69: + // Get rid of historic account balances as they simply use up way too much space + stmt.execute("DROP TRIGGER Historic_Account_Balance_Insert_Trigger"); + stmt.execute("DROP TRIGGER Historic_Account_Balance_Update_Trigger"); + stmt.execute("DROP TABLE HistoricAccountBalances"); + // Reclaim space + stmt.execute("CHECKPOINT"); + stmt.execute("CHECKPOINT DEFRAG"); + break; + default: // nothing to do return false; diff --git a/src/test/java/org/qortal/test/AccountBalanceTests.java b/src/test/java/org/qortal/test/AccountBalanceTests.java index a47f0e16..b46e87fb 100644 --- a/src/test/java/org/qortal/test/AccountBalanceTests.java +++ b/src/test/java/org/qortal/test/AccountBalanceTests.java @@ -14,15 +14,10 @@ import org.junit.After; import org.junit.Before; import org.junit.Test; import org.qortal.account.Account; -import org.qortal.account.PrivateKeyAccount; import org.qortal.account.PublicKeyAccount; import org.qortal.asset.Asset; -import org.qortal.block.BlockChain; import org.qortal.data.account.AccountBalanceData; import org.qortal.data.account.AccountData; -import org.qortal.data.transaction.BaseTransactionData; -import org.qortal.data.transaction.PaymentTransactionData; -import org.qortal.data.transaction.TransactionData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -30,7 +25,6 @@ import org.qortal.repository.AccountRepository.BalanceOrdering; import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TestAccount; -import org.qortal.test.common.TransactionUtils; public class AccountBalanceTests extends Common { @@ -88,123 +82,6 @@ public class AccountBalanceTests extends Common { } } - /** Tests we can fetch initial balance when newer balance exists. */ - @Test - public void testGetBalanceAtHeight() throws DataException { - try (final Repository repository = RepositoryManager.getRepository()) { - TestAccount alice = Common.getTestAccount(repository, "alice"); - - BigDecimal initialBalance = testNewerBalance(repository, alice); - - // Fetch all historic balances - List historicBalances = repository.getAccountRepository().getHistoricBalances(alice.getAddress(), Asset.QORT); - for (AccountBalanceData historicBalance : historicBalances) - System.out.println(String.format("Balance at height %d: %s", historicBalance.getHeight(), historicBalance.getBalance().toPlainString())); - - // Fetch balance at height 1, even though newer balance exists - AccountBalanceData accountBalanceData = repository.getAccountRepository().getBalance(alice.getAddress(), Asset.QORT, 1); - BigDecimal genesisBalance = accountBalanceData.getBalance(); - - // Confirm genesis balance is same as initial - assertEqualBigDecimals("Genesis balance should match initial", initialBalance, genesisBalance); - } - } - - /** Tests we can fetch balance with a height where no balance change occurred. */ - @Test - public void testGetBalanceAtNearestHeight() throws DataException { - Random random = new Random(); - - byte[] publicKey = new byte[32]; - random.nextBytes(publicKey); - - try (final Repository repository = RepositoryManager.getRepository()) { - PublicKeyAccount recipientAccount = new PublicKeyAccount(repository, publicKey); - System.out.println(String.format("Test recipient: %s", recipientAccount.getAddress())); - - // Mint a few blocks - for (int i = 0; i < 10; ++i) - BlockUtils.mintBlock(repository); - - // Confirm recipient balance is zero - BigDecimal balance = recipientAccount.getConfirmedBalance(Asset.QORT); - assertEqualBigDecimals("recipient's balance should be zero", BigDecimal.ZERO, balance); - - // Confirm recipient has no historic balances - List historicBalances = repository.getAccountRepository().getHistoricBalances(recipientAccount.getAddress(), Asset.QORT); - for (AccountBalanceData historicBalance : historicBalances) - System.err.println(String.format("Block %d: %s", historicBalance.getHeight(), historicBalance.getBalance().toPlainString())); - assertTrue("recipient should not have historic balances yet", historicBalances.isEmpty()); - - // Send 1 QORT to recipient - TestAccount sendingAccount = Common.getTestAccount(repository, "alice"); - pay(repository, sendingAccount, recipientAccount, BigDecimal.ONE); - - // Mint some more blocks - for (int i = 0; i < 10; ++i) - BlockUtils.mintBlock(repository); - - // Send more QORT to recipient - BigDecimal amount = BigDecimal.valueOf(random.nextInt(123456)); - pay(repository, sendingAccount, recipientAccount, amount); - BigDecimal totalAmount = BigDecimal.ONE.add(amount); - - // Mint some more blocks - for (int i = 0; i < 10; ++i) - BlockUtils.mintBlock(repository); - - // Confirm recipient balance is as expected - balance = recipientAccount.getConfirmedBalance(Asset.QORT); - assertEqualBigDecimals("recipient's balance incorrect", totalAmount, balance); - - historicBalances = repository.getAccountRepository().getHistoricBalances(recipientAccount.getAddress(), Asset.QORT); - for (AccountBalanceData historicBalance : historicBalances) - System.out.println(String.format("Block %d: %s", historicBalance.getHeight(), historicBalance.getBalance().toPlainString())); - - // Confirm balance as of 2 blocks ago - int height = repository.getBlockRepository().getBlockchainHeight(); - balance = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 2).getBalance(); - assertEqualBigDecimals("recipient's historic balance incorrect", totalAmount, balance); - - // Confirm balance prior to last payment - balance = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 15).getBalance(); - assertEqualBigDecimals("recipient's historic balance incorrect", BigDecimal.ONE, balance); - - // Orphan blocks to before last payment - BlockUtils.orphanBlocks(repository, 10 + 5); - - // Re-check balance from (now) invalid height - AccountBalanceData accountBalanceData = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 2); - balance = accountBalanceData.getBalance(); - assertEqualBigDecimals("recipient's invalid-height balance should be one", BigDecimal.ONE, balance); - - // Orphan blocks to before initial 1 QORT payment - BlockUtils.orphanBlocks(repository, 10 + 5); - - // Re-check balance from (now) invalid height - accountBalanceData = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 2); - assertNull("recipient's invalid-height balance data should be null", accountBalanceData); - - // Confirm recipient has no historic balances - historicBalances = repository.getAccountRepository().getHistoricBalances(recipientAccount.getAddress(), Asset.QORT); - for (AccountBalanceData historicBalance : historicBalances) - System.err.println(String.format("Block %d: %s", historicBalance.getHeight(), historicBalance.getBalance().toPlainString())); - assertTrue("recipient should have no remaining historic balances", historicBalances.isEmpty()); - } - } - - private void pay(Repository repository, PrivateKeyAccount sendingAccount, Account recipientAccount, BigDecimal amount) throws DataException { - byte[] reference = sendingAccount.getLastReference(); - long timestamp = repository.getTransactionRepository().fromSignature(reference).getTimestamp() + 1; - - int txGroupId = 0; - BigDecimal fee = BlockChain.getInstance().getUnitFee(); - BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, sendingAccount.getPublicKey(), fee, null); - TransactionData transactionData = new PaymentTransactionData(baseTransactionData, recipientAccount.getAddress(), amount); - - TransactionUtils.signAndMint(repository, transactionData, sendingAccount); - } - /** Tests SQL query speed for account balance fetches. */ @Test public void testRepositorySpeed() throws DataException, SQLException {