From 0d17f02191c0e4832281c66c69bf9def210fd79b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 28 Sep 2021 20:29:53 +0100 Subject: [PATCH] Pass a repository instance into the bulk archiving and pruning methods. This is a better approach than opening a new session for each, and it makes it easier to write unit tests. --- .../org/qortal/controller/Controller.java | 10 +- .../qortal/repository/RepositoryManager.java | 13 +- .../hsqldb/HSQLDBDatabaseArchiving.java | 77 ++- .../hsqldb/HSQLDBDatabasePruning.java | 457 +++++++++--------- 4 files changed, 278 insertions(+), 279 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index a00ffdac..feb4b309 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -2,8 +2,11 @@ package org.qortal.controller; import java.awt.TrayIcon.MessageType; import java.io.File; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.SecureRandom; import java.security.Security; import java.time.LocalDateTime; @@ -409,8 +412,11 @@ public class Controller extends Thread { try { RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl()); RepositoryManager.setRepositoryFactory(repositoryFactory); - RepositoryManager.archive(); - RepositoryManager.prune(); + + try (final Repository repository = RepositoryManager.getRepository()) { + RepositoryManager.archive(repository); + RepositoryManager.prune(repository); + } } catch (DataException e) { // If exception has no cause then repository is in use by some other process. if (e.getCause() == null) { diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 480edc59..7b96e08c 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving; import org.qortal.repository.hsqldb.HSQLDBDatabasePruning; +import org.qortal.repository.hsqldb.HSQLDBRepository; import org.qortal.settings.Settings; import java.sql.SQLException; @@ -58,12 +59,12 @@ public abstract class RepositoryManager { } } - public static boolean archive() { + public static boolean archive(Repository repository) { // Bulk archive the database the first time we use archive mode if (Settings.getInstance().isArchiveEnabled()) { if (RepositoryManager.canArchiveOrPrune()) { try { - return HSQLDBDatabaseArchiving.buildBlockArchive(); + return HSQLDBDatabaseArchiving.buildBlockArchive(repository); } catch (DataException e) { LOGGER.info("Unable to build block archive. The database may have been left in an inconsistent state."); @@ -76,18 +77,18 @@ public abstract class RepositoryManager { return false; } - public static boolean prune() { + public static boolean prune(Repository repository) { // Bulk prune the database the first time we use pruning mode if (Settings.getInstance().isPruningEnabled() || Settings.getInstance().isArchiveEnabled()) { if (RepositoryManager.canArchiveOrPrune()) { try { - boolean prunedATStates = HSQLDBDatabasePruning.pruneATStates(); - boolean prunedBlocks = HSQLDBDatabasePruning.pruneBlocks(); + boolean prunedATStates = HSQLDBDatabasePruning.pruneATStates((HSQLDBRepository) repository); + boolean prunedBlocks = HSQLDBDatabasePruning.pruneBlocks((HSQLDBRepository) repository); // Perform repository maintenance to shrink the db size down if (prunedATStates && prunedBlocks) { - HSQLDBDatabasePruning.performMaintenance(); + HSQLDBDatabasePruning.performMaintenance(repository); return true; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java index 618d5115..e9892a0b 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; import org.qortal.repository.BlockArchiveWriter; import org.qortal.repository.DataException; +import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.transform.TransformationException; @@ -29,53 +30,51 @@ public class HSQLDBDatabaseArchiving { private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabaseArchiving.class); - public static boolean buildBlockArchive() throws DataException { - try (final HSQLDBRepository repository = (HSQLDBRepository)RepositoryManager.getRepository()) { + public static boolean buildBlockArchive(Repository repository) throws DataException { - // Only build the archive if we have never done so before - int archiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); - if (archiveHeight > 0) { - // Already archived - return false; - } + // Only build the archive if we have never done so before + int archiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + if (archiveHeight > 0) { + // Already archived + return false; + } - LOGGER.info("Building block archive - this process could take a while... (approx. 15 mins on high spec)"); + LOGGER.info("Building block archive - this process could take a while... (approx. 15 mins on high spec)"); - final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); - int startHeight = 0; + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + int startHeight = 0; - while (!Controller.isStopping()) { - try { - BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository); - BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); - switch (result) { - case OK: - // Increment block archive height - startHeight += writer.getWrittenCount(); - repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight); - repository.saveChanges(); - break; + while (!Controller.isStopping()) { + try { + BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository); + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + switch (result) { + case OK: + // Increment block archive height + startHeight += writer.getWrittenCount(); + repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight); + repository.saveChanges(); + break; - case STOPPING: - return false; + case STOPPING: + return false; - case NOT_ENOUGH_BLOCKS: - // We've reached the limit of the blocks we can archive - // Return from the whole method - return true; + case NOT_ENOUGH_BLOCKS: + // We've reached the limit of the blocks we can archive + // Return from the whole method + return true; - case BLOCK_NOT_FOUND: - // We tried to archive a block that didn't exist. This is a major failure and likely means - // that a bootstrap or re-sync is needed. Return rom the method - LOGGER.info("Error: block not found when building archive. If this error persists, " + - "a bootstrap or re-sync may be needed."); - return false; - } - - } catch (IOException | TransformationException | InterruptedException e) { - LOGGER.info("Caught exception when creating block cache", e); - return false; + case BLOCK_NOT_FOUND: + // We tried to archive a block that didn't exist. This is a major failure and likely means + // that a bootstrap or re-sync is needed. Return rom the method + LOGGER.info("Error: block not found when building archive. If this error persists, " + + "a bootstrap or re-sync may be needed."); + return false; } + + } catch (IOException | TransformationException | InterruptedException e) { + LOGGER.info("Caught exception when creating block cache", e); + return false; } } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java index f0673c3f..3a9c4f02 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java @@ -6,6 +6,7 @@ import org.qortal.controller.Controller; import org.qortal.data.block.BlockData; import org.qortal.repository.BlockArchiveWriter; import org.qortal.repository.DataException; +import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; @@ -38,286 +39,278 @@ public class HSQLDBDatabasePruning { private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabasePruning.class); - public static boolean pruneATStates() throws SQLException, DataException { - try (final HSQLDBRepository repository = (HSQLDBRepository)RepositoryManager.getRepository()) { + public static boolean pruneATStates(HSQLDBRepository repository) throws SQLException, DataException { - // Only bulk prune AT states if we have never done so before - int pruneHeight = repository.getATRepository().getAtPruneHeight(); - if (pruneHeight > 0) { - // Already pruned AT states + // Only bulk prune AT states if we have never done so before + int pruneHeight = repository.getATRepository().getAtPruneHeight(); + if (pruneHeight > 0) { + // Already pruned AT states + return false; + } + + if (Settings.getInstance().isArchiveEnabled()) { + // Only proceed if we can see that the archiver has already finished + // This way, if the archiver failed for any reason, we can prune once it has had + // some opportunities to try again + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository); + if (!upToDate) { return false; } + } - if (Settings.getInstance().isArchiveEnabled()) { - // Only proceed if we can see that the archiver has already finished - // This way, if the archiver failed for any reason, we can prune once it has had - // some opportunities to try again - boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository); - if (!upToDate) { - return false; - } - } + LOGGER.info("Starting bulk prune of AT states - this process could take a while... " + + "(approx. 2 mins on high spec, or upwards of 30 mins in some cases)"); - LOGGER.info("Starting bulk prune of AT states - this process could take a while... " + - "(approx. 2 mins on high spec, or upwards of 30 mins in some cases)"); + // Create new AT-states table to hold smaller dataset + repository.executeCheckedUpdate("DROP TABLE IF EXISTS ATStatesNew"); + repository.executeCheckedUpdate("CREATE TABLE ATStatesNew (" + + "AT_address QortalAddress, height INTEGER NOT NULL, state_hash ATStateHash NOT NULL, " + + "fees QortalAmount NOT NULL, is_initial BOOLEAN NOT NULL, sleep_until_message_timestamp BIGINT, " + + "PRIMARY KEY (AT_address, height), " + + "FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)"); + repository.executeCheckedUpdate("SET TABLE ATStatesNew NEW SPACE"); + repository.executeCheckedUpdate("CHECKPOINT"); - // Create new AT-states table to hold smaller dataset - repository.executeCheckedUpdate("DROP TABLE IF EXISTS ATStatesNew"); - repository.executeCheckedUpdate("CREATE TABLE ATStatesNew (" - + "AT_address QortalAddress, height INTEGER NOT NULL, state_hash ATStateHash NOT NULL, " - + "fees QortalAmount NOT NULL, is_initial BOOLEAN NOT NULL, sleep_until_message_timestamp BIGINT, " - + "PRIMARY KEY (AT_address, height), " - + "FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)"); - repository.executeCheckedUpdate("SET TABLE ATStatesNew NEW SPACE"); - repository.executeCheckedUpdate("CHECKPOINT"); - - // Add a height index - LOGGER.info("Adding index to AT states table..."); - repository.executeCheckedUpdate("CREATE INDEX IF NOT EXISTS ATStatesNewHeightIndex ON ATStatesNew (height)"); - repository.executeCheckedUpdate("CHECKPOINT"); + // Add a height index + LOGGER.info("Adding index to AT states table..."); + repository.executeCheckedUpdate("CREATE INDEX IF NOT EXISTS ATStatesNewHeightIndex ON ATStatesNew (height)"); + repository.executeCheckedUpdate("CHECKPOINT"); - // Find our latest block - BlockData latestBlock = repository.getBlockRepository().getLastBlock(); - if (latestBlock == null) { - LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning"); - return false; - } + // Find our latest block + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + if (latestBlock == null) { + LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning"); + return false; + } - // Calculate some constants for later use - final int blockchainHeight = latestBlock.getHeight(); - int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); - if (Settings.getInstance().isArchiveEnabled()) { - // Archive mode - don't prune anything that hasn't been archived yet - maximumBlockToTrim = Math.min(maximumBlockToTrim, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); - } - final int startHeight = maximumBlockToTrim; - final int endHeight = blockchainHeight; - final int blockStep = 10000; + // Calculate some constants for later use + final int blockchainHeight = latestBlock.getHeight(); + int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + if (Settings.getInstance().isArchiveEnabled()) { + // Archive mode - don't prune anything that hasn't been archived yet + maximumBlockToTrim = Math.min(maximumBlockToTrim, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); + } + final int startHeight = maximumBlockToTrim; + final int endHeight = blockchainHeight; + final int blockStep = 10000; - // It's essential that we rebuild the latest AT states here, as we are using this data in the next query. - // Failing to do this will result in important AT states being deleted, rendering the database unusable. - repository.getATRepository().rebuildLatestAtStates(); + // It's essential that we rebuild the latest AT states here, as we are using this data in the next query. + // Failing to do this will result in important AT states being deleted, rendering the database unusable. + repository.getATRepository().rebuildLatestAtStates(); - // Loop through all the LatestATStates and copy them to the new table - LOGGER.info("Copying AT states..."); - for (int height = 0; height < endHeight; height += blockStep) { - //LOGGER.info(String.format("Copying AT states between %d and %d...", height, height + blockStep - 1)); + // Loop through all the LatestATStates and copy them to the new table + LOGGER.info("Copying AT states..."); + for (int height = 0; height < endHeight; height += blockStep) { + //LOGGER.info(String.format("Copying AT states between %d and %d...", height, height + blockStep - 1)); - String sql = "SELECT height, AT_address FROM LatestATStates WHERE height BETWEEN ? AND ?"; - try (ResultSet latestAtStatesResultSet = repository.checkedExecute(sql, height, height + blockStep - 1)) { - if (latestAtStatesResultSet != null) { - do { - int latestAtHeight = latestAtStatesResultSet.getInt(1); - String latestAtAddress = latestAtStatesResultSet.getString(2); + String sql = "SELECT height, AT_address FROM LatestATStates WHERE height BETWEEN ? AND ?"; + try (ResultSet latestAtStatesResultSet = repository.checkedExecute(sql, height, height + blockStep - 1)) { + if (latestAtStatesResultSet != null) { + do { + int latestAtHeight = latestAtStatesResultSet.getInt(1); + String latestAtAddress = latestAtStatesResultSet.getString(2); - // Copy this latest ATState to the new table - //LOGGER.info(String.format("Copying AT %s at height %d...", latestAtAddress, latestAtHeight)); - try { - String updateSql = "INSERT INTO ATStatesNew (" - + "SELECT AT_address, height, state_hash, fees, is_initial, sleep_until_message_timestamp " - + "FROM ATStates " - + "WHERE height = ? AND AT_address = ?)"; - repository.executeCheckedUpdate(updateSql, latestAtHeight, latestAtAddress); - } catch (SQLException e) { - repository.examineException(e); - throw new DataException("Unable to copy ATStates", e); - } + // Copy this latest ATState to the new table + //LOGGER.info(String.format("Copying AT %s at height %d...", latestAtAddress, latestAtHeight)); + try { + String updateSql = "INSERT INTO ATStatesNew (" + + "SELECT AT_address, height, state_hash, fees, is_initial, sleep_until_message_timestamp " + + "FROM ATStates " + + "WHERE height = ? AND AT_address = ?)"; + repository.executeCheckedUpdate(updateSql, latestAtHeight, latestAtAddress); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to copy ATStates", e); + } - if (height >= startHeight) { - // Now copy this AT's states for each recent block they is present in - for (int i = startHeight; i < endHeight; i++) { - if (latestAtHeight < i) { - // This AT finished before this block so there is nothing to copy - continue; - } + if (height >= startHeight) { + // Now copy this AT's states for each recent block they is present in + for (int i = startHeight; i < endHeight; i++) { + if (latestAtHeight < i) { + // This AT finished before this block so there is nothing to copy + continue; + } - //LOGGER.info(String.format("Copying recent AT %s at height %d...", latestAtAddress, i)); - try { - // Copy each LatestATState to the new table - String updateSql = "INSERT IGNORE INTO ATStatesNew (" - + "SELECT AT_address, height, state_hash, fees, is_initial, sleep_until_message_timestamp " - + "FROM ATStates " - + "WHERE height = ? AND AT_address = ?)"; - repository.executeCheckedUpdate(updateSql, i, latestAtAddress); - } catch (SQLException e) { - repository.examineException(e); - throw new DataException("Unable to copy ATStates", e); - } + //LOGGER.info(String.format("Copying recent AT %s at height %d...", latestAtAddress, i)); + try { + // Copy each LatestATState to the new table + String updateSql = "INSERT IGNORE INTO ATStatesNew (" + + "SELECT AT_address, height, state_hash, fees, is_initial, sleep_until_message_timestamp " + + "FROM ATStates " + + "WHERE height = ? AND AT_address = ?)"; + repository.executeCheckedUpdate(updateSql, i, latestAtAddress); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to copy ATStates", e); } } + } - } while (latestAtStatesResultSet.next()); - } - } catch (SQLException e) { - throw new DataException("Unable to copy AT states", e); + } while (latestAtStatesResultSet.next()); } + } catch (SQLException e) { + throw new DataException("Unable to copy AT states", e); } - - repository.saveChanges(); - - - // Finally, drop the original table and rename - LOGGER.info("Deleting old AT states..."); - repository.executeCheckedUpdate("DROP TABLE ATStates"); - repository.executeCheckedUpdate("ALTER TABLE ATStatesNew RENAME TO ATStates"); - repository.executeCheckedUpdate("ALTER INDEX ATStatesNewHeightIndex RENAME TO ATStatesHeightIndex"); - repository.executeCheckedUpdate("CHECKPOINT"); - - // Update the prune height - repository.getATRepository().setAtPruneHeight(maximumBlockToTrim); - repository.saveChanges(); - - repository.executeCheckedUpdate("CHECKPOINT"); - - // Now prune/trim the ATStatesData, as this currently goes back over a month - return HSQLDBDatabasePruning.pruneATStateData(); } + + repository.saveChanges(); + + + // Finally, drop the original table and rename + LOGGER.info("Deleting old AT states..."); + repository.executeCheckedUpdate("DROP TABLE ATStates"); + repository.executeCheckedUpdate("ALTER TABLE ATStatesNew RENAME TO ATStates"); + repository.executeCheckedUpdate("ALTER INDEX ATStatesNewHeightIndex RENAME TO ATStatesHeightIndex"); + repository.executeCheckedUpdate("CHECKPOINT"); + + // Update the prune height + repository.getATRepository().setAtPruneHeight(maximumBlockToTrim); + repository.saveChanges(); + + repository.executeCheckedUpdate("CHECKPOINT"); + + // Now prune/trim the ATStatesData, as this currently goes back over a month + return HSQLDBDatabasePruning.pruneATStateData(repository); } /* * Bulk prune ATStatesData to catch up with the now pruned ATStates table * This uses the existing AT States trimming code but with a much higher end block */ - private static boolean pruneATStateData() throws SQLException, DataException { - try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) { - - if (Settings.getInstance().isArchiveEnabled()) { - // Don't prune ATStatesData in archive mode - return true; - } - - BlockData latestBlock = repository.getBlockRepository().getLastBlock(); - if (latestBlock == null) { - LOGGER.info("Unable to determine blockchain height, necessary for bulk ATStatesData pruning"); - return false; - } - final int blockchainHeight = latestBlock.getHeight(); - int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); - // ATStateData is already trimmed - so carry on from where we left off in the past - int pruneStartHeight = repository.getATRepository().getAtTrimHeight(); - - LOGGER.info("Starting bulk prune of AT states data - this process could take a while... (approx. 3 mins on high spec)"); - - while (pruneStartHeight < upperPrunableHeight) { - // Prune all AT state data up until our latest minus pruneBlockLimit (or our archive height) - - if (Controller.isStopping()) { - return false; - } - - // Override batch size in the settings because this is a one-off process - final int batchSize = 1000; - final int rowLimitPerBatch = 50000; - int upperBatchHeight = pruneStartHeight + batchSize; - int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); - - LOGGER.trace(String.format("Pruning AT states data between %d and %d...", pruneStartHeight, upperPruneHeight)); - - int numATStatesPruned = repository.getATRepository().trimAtStates(pruneStartHeight, upperPruneHeight, rowLimitPerBatch); - repository.saveChanges(); - - if (numATStatesPruned > 0) { - LOGGER.trace(String.format("Pruned %d AT states data rows between blocks %d and %d", - numATStatesPruned, pruneStartHeight, upperPruneHeight)); - } else { - repository.getATRepository().setAtTrimHeight(upperBatchHeight); - // No need to rebuild the latest AT states as we aren't currently synchronizing - repository.saveChanges(); - LOGGER.debug(String.format("Bumping AT states trim height to %d", upperBatchHeight)); - - // Can we move onto next batch? - if (upperPrunableHeight > upperBatchHeight) { - pruneStartHeight = upperBatchHeight; - } - else { - // We've finished pruning - break; - } - } - } + private static boolean pruneATStateData(Repository repository) throws DataException { + if (Settings.getInstance().isArchiveEnabled()) { + // Don't prune ATStatesData in archive mode return true; } - } - public static boolean pruneBlocks() throws SQLException, DataException { - try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) { + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + if (latestBlock == null) { + LOGGER.info("Unable to determine blockchain height, necessary for bulk ATStatesData pruning"); + return false; + } + final int blockchainHeight = latestBlock.getHeight(); + int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + // ATStateData is already trimmed - so carry on from where we left off in the past + int pruneStartHeight = repository.getATRepository().getAtTrimHeight(); - // Only bulk prune AT states if we have never done so before - int pruneHeight = repository.getBlockRepository().getBlockPruneHeight(); - if (pruneHeight > 0) { - // Already pruned blocks + LOGGER.info("Starting bulk prune of AT states data - this process could take a while... (approx. 3 mins on high spec)"); + + while (pruneStartHeight < upperPrunableHeight) { + // Prune all AT state data up until our latest minus pruneBlockLimit (or our archive height) + + if (Controller.isStopping()) { return false; } - if (Settings.getInstance().isArchiveEnabled()) { - // Only proceed if we can see that the archiver has already finished - // This way, if the archiver failed for any reason, we can prune once it has had - // some opportunities to try again - boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository); - if (!upToDate) { - return false; - } - } + // Override batch size in the settings because this is a one-off process + final int batchSize = 1000; + final int rowLimitPerBatch = 50000; + int upperBatchHeight = pruneStartHeight + batchSize; + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); - BlockData latestBlock = repository.getBlockRepository().getLastBlock(); - if (latestBlock == null) { - LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning"); - return false; - } - final int blockchainHeight = latestBlock.getHeight(); - int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); - int pruneStartHeight = 0; + LOGGER.trace(String.format("Pruning AT states data between %d and %d...", pruneStartHeight, upperPruneHeight)); - if (Settings.getInstance().isArchiveEnabled()) { - // Archive mode - don't prune anything that hasn't been archived yet - upperPrunableHeight = Math.min(upperPrunableHeight, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); - } + int numATStatesPruned = repository.getATRepository().trimAtStates(pruneStartHeight, upperPruneHeight, rowLimitPerBatch); + repository.saveChanges(); - LOGGER.info("Starting bulk prune of blocks - this process could take a while... (approx. 5 mins on high spec)"); - - while (pruneStartHeight < upperPrunableHeight) { - // Prune all blocks up until our latest minus pruneBlockLimit - - int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize(); - int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); - - LOGGER.info(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight)); - - int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight); + if (numATStatesPruned > 0) { + LOGGER.trace(String.format("Pruned %d AT states data rows between blocks %d and %d", + numATStatesPruned, pruneStartHeight, upperPruneHeight)); + } else { + repository.getATRepository().setAtTrimHeight(upperBatchHeight); + // No need to rebuild the latest AT states as we aren't currently synchronizing repository.saveChanges(); + LOGGER.debug(String.format("Bumping AT states trim height to %d", upperBatchHeight)); - if (numBlocksPruned > 0) { - LOGGER.info(String.format("Pruned %d block%s between %d and %d", - numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""), - pruneStartHeight, upperPruneHeight)); - } else { - repository.getBlockRepository().setBlockPruneHeight(upperBatchHeight); - repository.saveChanges(); - LOGGER.debug(String.format("Bumping block base prune height to %d", upperBatchHeight)); - - // Can we move onto next batch? - if (upperPrunableHeight > upperBatchHeight) { - pruneStartHeight = upperBatchHeight; - } - else { - // We've finished pruning - break; - } + // Can we move onto next batch? + if (upperPrunableHeight > upperBatchHeight) { + pruneStartHeight = upperBatchHeight; + } + else { + // We've finished pruning + break; } } - - return true; } + + return true; } - public static void performMaintenance() throws SQLException, DataException { - try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) { - repository.performPeriodicMaintenance(); + public static boolean pruneBlocks(Repository repository) throws SQLException, DataException { + + // Only bulk prune AT states if we have never done so before + int pruneHeight = repository.getBlockRepository().getBlockPruneHeight(); + if (pruneHeight > 0) { + // Already pruned blocks + return false; } + + if (Settings.getInstance().isArchiveEnabled()) { + // Only proceed if we can see that the archiver has already finished + // This way, if the archiver failed for any reason, we can prune once it has had + // some opportunities to try again + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository); + if (!upToDate) { + return false; + } + } + + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); + if (latestBlock == null) { + LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning"); + return false; + } + final int blockchainHeight = latestBlock.getHeight(); + int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + int pruneStartHeight = 0; + + if (Settings.getInstance().isArchiveEnabled()) { + // Archive mode - don't prune anything that hasn't been archived yet + upperPrunableHeight = Math.min(upperPrunableHeight, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); + } + + LOGGER.info("Starting bulk prune of blocks - this process could take a while... (approx. 5 mins on high spec)"); + + while (pruneStartHeight < upperPrunableHeight) { + // Prune all blocks up until our latest minus pruneBlockLimit + + int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize(); + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + LOGGER.info(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight); + repository.saveChanges(); + + if (numBlocksPruned > 0) { + LOGGER.info(String.format("Pruned %d block%s between %d and %d", + numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""), + pruneStartHeight, upperPruneHeight)); + } else { + repository.getBlockRepository().setBlockPruneHeight(upperBatchHeight); + repository.saveChanges(); + LOGGER.debug(String.format("Bumping block base prune height to %d", upperBatchHeight)); + + // Can we move onto next batch? + if (upperPrunableHeight > upperBatchHeight) { + pruneStartHeight = upperBatchHeight; + } + else { + // We've finished pruning + break; + } + } + } + + return true; + } + + public static void performMaintenance(Repository repository) throws SQLException, DataException { + repository.performPeriodicMaintenance(); } }