diff --git a/src/main/java/org/qortal/controller/pruning/BlockPruner.java b/src/main/java/org/qortal/controller/pruning/BlockPruner.java new file mode 100644 index 00000000..8ae25224 --- /dev/null +++ b/src/main/java/org/qortal/controller/pruning/BlockPruner.java @@ -0,0 +1,86 @@ +package org.qortal.controller.pruning; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; +import org.qortal.utils.NTP; + +public class BlockPruner implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(BlockPruner.class); + + @Override + public void run() { + Thread.currentThread().setName("Block pruner"); + + if (!Settings.getInstance().isPruningEnabled()) { + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + int pruneStartHeight = repository.getBlockRepository().getBlockPruneHeight(); + + while (!Controller.isStopping()) { + repository.discardChanges(); + + Thread.sleep(Settings.getInstance().getBlockPruneInterval()); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) + continue; + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Controller.getInstance().isSynchronizing()) + continue; + + // Prune all blocks up until our latest minus pruneBlockLimit + final int ourLatestHeight = chainTip.getHeight(); + final int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + + int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize(); + int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); + + if (pruneStartHeight >= upperPruneHeight) { + continue; + } + + LOGGER.debug(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight)); + + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight); + repository.saveChanges(); + + if (numBlocksPruned > 0) { + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Pruned %d block%s between %d and %d", + numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""), + finalPruneStartHeight, upperPruneHeight)); + } else { + // Can we move onto next batch? + if (upperPrunableHeight > upperBatchHeight) { + pruneStartHeight = upperBatchHeight; + repository.getBlockRepository().setBlockPruneHeight(pruneStartHeight); + repository.saveChanges(); + + final int finalPruneStartHeight = pruneStartHeight; + LOGGER.debug(() -> String.format("Bumping block base prune height to %d", finalPruneStartHeight)); + } + else { + // We've pruned up to the upper prunable height + // Back off for a while to save CPU for syncing + Thread.sleep(10*60*1000L); + } + } + } + } catch (DataException e) { + LOGGER.warn(String.format("Repository issue trying to prune blocks: %s", e.getMessage())); + } catch (InterruptedException e) { + // Time to exit + } + } + +} diff --git a/src/main/java/org/qortal/controller/pruning/PruneManager.java b/src/main/java/org/qortal/controller/pruning/PruneManager.java index dcd7391d..66019d01 100644 --- a/src/main/java/org/qortal/controller/pruning/PruneManager.java +++ b/src/main/java/org/qortal/controller/pruning/PruneManager.java @@ -23,6 +23,7 @@ public class PruneManager { // Start individual pruning processes ExecutorService pruneExecutor = Executors.newCachedThreadPool(new DaemonThreadFactory()); pruneExecutor.execute(new AtStatesPruner()); + pruneExecutor.execute(new BlockPruner()); } public static synchronized PruneManager getInstance() { diff --git a/src/main/java/org/qortal/repository/BlockRepository.java b/src/main/java/org/qortal/repository/BlockRepository.java index 78eba399..5ca61e66 100644 --- a/src/main/java/org/qortal/repository/BlockRepository.java +++ b/src/main/java/org/qortal/repository/BlockRepository.java @@ -166,6 +166,20 @@ public interface BlockRepository { */ public BlockData getDetachedBlockSignature(int startHeight) throws DataException; + + /** Returns height of first prunable block. */ + public int getBlockPruneHeight() throws DataException; + + /** Sets new base height for block pruning. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setBlockPruneHeight(int pruneHeight) throws DataException; + + /** Prunes full block data between passed heights. Returns number of pruned rows. */ + public int pruneBlocks(int minHeight, int maxHeight) throws DataException; + + /** * Saves block into repository. * diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java index b486e6a0..2f7e4ad2 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java @@ -509,6 +509,53 @@ public class HSQLDBBlockRepository implements BlockRepository { } } + + @Override + public int getBlockPruneHeight() throws DataException { + String sql = "SELECT block_prune_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch block prune height from repository", e); + } + } + + @Override + public void setBlockPruneHeight(int pruneHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET block_prune_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, pruneHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to set block prune height in repository", e); + } + } + } + + @Override + public int pruneBlocks(int minHeight, int maxHeight) throws DataException { + // Don't prune the genesis block + if (minHeight <= 1) { + minHeight = 2; + } + + try { + return this.repository.delete("Blocks", "height BETWEEN ? AND ?", minHeight, maxHeight); + } catch (SQLException e) { + throw new DataException("Unable to prune blocks from repository", e); + } + } + + @Override public BlockData getDetachedBlockSignature(int startHeight) throws DataException { String sql = "SELECT " + BLOCK_DB_COLUMNS + " FROM Blocks " diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index 94e753e8..d696351f 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -870,6 +870,7 @@ public class HSQLDBDatabaseUpdates { case 35: // Support for pruning stmt.execute("ALTER TABLE DatabaseInfo ADD AT_prune_height INT NOT NULL DEFAULT 0"); + stmt.execute("ALTER TABLE DatabaseInfo ADD block_prune_height INT NOT NULL DEFAULT 0"); break; default: diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index f63bdbb9..89a63ad1 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -103,24 +103,32 @@ public class Settings { /** Max number of AT states to trim in one go. */ private int atStatesTrimLimit = 4000; // records - /** Whether we should prune old data to reduce database size - * This prevents the node from being able to serve older blocks */ - private boolean pruningEnabled = false; - /** The amount of recent blocks we should keep when pruning */ - private int pruneBlockLimit = 1440; - - /** How often to attempt AT state pruning (ms). */ - private long atStatesPruneInterval = 3219L; // milliseconds - /** Block height range to scan for trimmable AT states.
- * This has a significant effect on execution time. */ - private int atStatesPruneBatchSize = 10; // blocks - /** How often to attempt online accounts signatures trimming (ms). */ private long onlineSignaturesTrimInterval = 9876L; // milliseconds /** Block height range to scan for trimmable online accounts signatures.
* This has a significant effect on execution time. */ private int onlineSignaturesTrimBatchSize = 100; // blocks + + /** Whether we should prune old data to reduce database size + * This prevents the node from being able to serve older blocks */ + private boolean pruningEnabled = false; + /** The amount of recent blocks we should keep when pruning */ + private int pruneBlockLimit = 1450; + + /** How often to attempt AT state pruning (ms). */ + private long atStatesPruneInterval = 3219L; // milliseconds + /** Block height range to scan for prunable AT states.
+ * This has a significant effect on execution time. */ + private int atStatesPruneBatchSize = 10; // blocks + + /** How often to attempt block pruning (ms). */ + private long blockPruneInterval = 3219L; // milliseconds + /** Block height range to scan for prunable blocks.
+ * This has a significant effect on execution time. */ + private int blockPruneBatchSize = 10000; // blocks + + // Peer-to-peer related private boolean isTestNet = false; /** Port number for inbound peer-to-peer connections. */ @@ -533,6 +541,15 @@ public class Settings { return this.atStatesTrimLimit; } + public long getOnlineSignaturesTrimInterval() { + return this.onlineSignaturesTrimInterval; + } + + public int getOnlineSignaturesTrimBatchSize() { + return this.onlineSignaturesTrimBatchSize; + } + + public boolean isPruningEnabled() { return this.pruningEnabled; } @@ -549,12 +566,12 @@ public class Settings { return this.atStatesPruneBatchSize; } - public long getOnlineSignaturesTrimInterval() { - return this.onlineSignaturesTrimInterval; + public long getBlockPruneInterval() { + return this.blockPruneInterval; } - public int getOnlineSignaturesTrimBatchSize() { - return this.onlineSignaturesTrimBatchSize; + public int getBlockPruneBatchSize() { + return this.blockPruneBatchSize; } }