From 446f9243803e31d4d7f765a7fb27548a83378597 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 1 Oct 2021 09:32:35 +0100 Subject: [PATCH] Added bulk pruning test, which highlighted some bugs in both bulk and regular pruning. --- .../controller/repository/BlockPruner.java | 19 +++-- .../qortal/repository/BlockArchiveWriter.java | 10 ++- .../qortal/repository/RepositoryManager.java | 2 +- .../hsqldb/HSQLDBDatabaseArchiving.java | 5 +- .../hsqldb/HSQLDBDatabasePruning.java | 12 ++-- .../org/qortal/test/BlockArchiveTests.java | 71 ++++++++++++++++++- 6 files changed, 99 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/qortal/controller/repository/BlockPruner.java b/src/main/java/org/qortal/controller/repository/BlockPruner.java index f5be6ee8..46cf919b 100644 --- a/src/main/java/org/qortal/controller/repository/BlockPruner.java +++ b/src/main/java/org/qortal/controller/repository/BlockPruner.java @@ -83,19 +83,18 @@ public class BlockPruner implements Runnable { repository.saveChanges(); if (numBlocksPruned > 0) { - final int finalPruneStartHeight = pruneStartHeight; - LOGGER.debug(() -> String.format("Pruned %d block%s between %d and %d", + LOGGER.debug(String.format("Pruned %d block%s between %d and %d", numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""), - finalPruneStartHeight, upperPruneHeight)); + pruneStartHeight, upperPruneHeight)); } else { - // Can we move onto next batch? - if (upperPrunableHeight > upperBatchHeight) { - pruneStartHeight = upperBatchHeight; - repository.getBlockRepository().setBlockPruneHeight(pruneStartHeight); - repository.saveChanges(); + final int nextPruneHeight = upperPruneHeight + 1; + repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight); + repository.saveChanges(); + LOGGER.debug(String.format("Bumping block base prune height to %d", pruneStartHeight)); - final int finalPruneStartHeight = pruneStartHeight; - LOGGER.debug(() -> String.format("Bumping block base prune height to %d", finalPruneStartHeight)); + // Can we move onto next batch? + if (upperPrunableHeight > nextPruneHeight) { + pruneStartHeight = nextPruneHeight; } else { // We've pruned up to the upper prunable height diff --git a/src/main/java/org/qortal/repository/BlockArchiveWriter.java b/src/main/java/org/qortal/repository/BlockArchiveWriter.java index 611cecea..39c28fd6 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveWriter.java +++ b/src/main/java/org/qortal/repository/BlockArchiveWriter.java @@ -29,14 +29,17 @@ public class BlockArchiveWriter { private static final Logger LOGGER = LogManager.getLogger(BlockArchiveWriter.class); + public static final long DEFAULT_FILE_SIZE_TARGET = 100 * 1024 * 1024; // 100MiB + private int startHeight; private final int endHeight; private final Repository repository; - private long fileSizeTarget = 100 * 1024 * 1024; // 100MiB + private long fileSizeTarget = DEFAULT_FILE_SIZE_TARGET; private boolean shouldEnforceFileSizeTarget = true; private int writtenCount; + private int lastWrittenHeight; private Path outputPath; public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) { @@ -169,6 +172,7 @@ public class BlockArchiveWriter { BlockArchiveReader.getInstance().invalidateFileListCache(); this.writtenCount = i; + this.lastWrittenHeight = endHeight; this.outputPath = Paths.get(filePath); return BlockArchiveWriteResult.OK; } @@ -177,6 +181,10 @@ public class BlockArchiveWriter { return this.writtenCount; } + public int getLastWrittenHeight() { + return this.lastWrittenHeight; + } + public Path getOutputPath() { return this.outputPath; } diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 7b96e08c..1b398057 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -64,7 +64,7 @@ public abstract class RepositoryManager { if (Settings.getInstance().isArchiveEnabled()) { if (RepositoryManager.canArchiveOrPrune()) { try { - return HSQLDBDatabaseArchiving.buildBlockArchive(repository); + return HSQLDBDatabaseArchiving.buildBlockArchive(repository, BlockArchiveWriter.DEFAULT_FILE_SIZE_TARGET); } catch (DataException e) { LOGGER.info("Unable to build block archive. The database may have been left in an inconsistent state."); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java index e9892a0b..0ad315e3 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java @@ -30,7 +30,7 @@ public class HSQLDBDatabaseArchiving { private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabaseArchiving.class); - public static boolean buildBlockArchive(Repository repository) throws DataException { + public static boolean buildBlockArchive(Repository repository, long fileSizeTarget) throws DataException { // Only build the archive if we have never done so before int archiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); @@ -47,11 +47,12 @@ public class HSQLDBDatabaseArchiving { while (!Controller.isStopping()) { try { BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository); + writer.setFileSizeTarget(fileSizeTarget); BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); switch (result) { case OK: // Increment block archive height - startHeight += writer.getWrittenCount(); + startHeight = writer.getLastWrittenHeight() + 1; repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight); repository.saveChanges(); break; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java index 3a9c4f02..49f54150 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java @@ -167,7 +167,8 @@ public class HSQLDBDatabasePruning { repository.executeCheckedUpdate("CHECKPOINT"); // Update the prune height - repository.getATRepository().setAtPruneHeight(maximumBlockToTrim); + int nextPruneHeight = maximumBlockToTrim + 1; + repository.getATRepository().setAtPruneHeight(nextPruneHeight); repository.saveChanges(); repository.executeCheckedUpdate("CHECKPOINT"); @@ -291,13 +292,14 @@ public class HSQLDBDatabasePruning { numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""), pruneStartHeight, upperPruneHeight)); } else { - repository.getBlockRepository().setBlockPruneHeight(upperBatchHeight); + final int nextPruneHeight = upperPruneHeight + 1; + repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight); repository.saveChanges(); - LOGGER.debug(String.format("Bumping block base prune height to %d", upperBatchHeight)); + LOGGER.debug(String.format("Bumping block base prune height to %d", nextPruneHeight)); // Can we move onto next batch? - if (upperPrunableHeight > upperBatchHeight) { - pruneStartHeight = upperBatchHeight; + if (upperPrunableHeight > nextPruneHeight) { + pruneStartHeight = nextPruneHeight; } else { // We've finished pruning diff --git a/src/test/java/org/qortal/test/BlockArchiveTests.java b/src/test/java/org/qortal/test/BlockArchiveTests.java index af7804f9..08d760b8 100644 --- a/src/test/java/org/qortal/test/BlockArchiveTests.java +++ b/src/test/java/org/qortal/test/BlockArchiveTests.java @@ -10,6 +10,8 @@ import org.qortal.data.at.ATStateData; import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; import org.qortal.repository.*; +import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving; +import org.qortal.repository.hsqldb.HSQLDBDatabasePruning; import org.qortal.repository.hsqldb.HSQLDBRepository; import org.qortal.settings.Settings; import org.qortal.test.common.AtUtils; @@ -24,7 +26,6 @@ import org.qortal.utils.Triple; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.SQLException; @@ -347,6 +348,74 @@ public class BlockArchiveTests extends Common { } } + @Test + public void testBulkArchiveAndPrune() throws DataException, InterruptedException, TransformationException, IOException, SQLException { + try (final Repository repository = RepositoryManager.getRepository()) { + HSQLDBRepository hsqldb = (HSQLDBRepository) repository; + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Assume 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Check the current archive height + assertEquals(0, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Write blocks 2-900 to the archive (using bulk method) + int fileSizeTarget = 425000; // Pre-calculated size of 900 blocks + assertTrue(HSQLDBDatabaseArchiving.buildBlockArchive(repository, 425000)); + + // Ensure the block archive height has increased + assertEquals(901, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the SQL repository contains blocks 2 and 900... + assertNotNull(repository.getBlockRepository().fromHeight(2)); + assertNotNull(repository.getBlockRepository().fromHeight(900)); + + // Check the current prune heights + assertEquals(0, repository.getBlockRepository().getBlockPruneHeight()); + assertEquals(0, repository.getATRepository().getAtPruneHeight()); + + // Prune all the archived blocks and AT states (using bulk method) + assertTrue(HSQLDBDatabasePruning.pruneBlocks(hsqldb)); + assertTrue(HSQLDBDatabasePruning.pruneATStates(hsqldb)); + + // Ensure the current prune heights have increased + assertEquals(901, repository.getBlockRepository().getBlockPruneHeight()); + assertEquals(901, repository.getATRepository().getAtPruneHeight()); + + // Now ensure the SQL repository is missing blocks 2 and 900... + assertNull(repository.getBlockRepository().fromHeight(2)); + assertNull(repository.getBlockRepository().fromHeight(900)); + + // ... but it's not missing blocks 1 and 901 (we don't prune the genesis block) + assertNotNull(repository.getBlockRepository().fromHeight(1)); + assertNotNull(repository.getBlockRepository().fromHeight(901)); + + // Validate the latest block height in the repository + assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + } + } + @Test public void testTrimArchivePruneAndOrphan() throws DataException, InterruptedException, TransformationException, IOException { try (final Repository repository = RepositoryManager.getRepository()) {