From 49749a0bc78eac926ac8e5a69e549659d01ea14a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 1 Oct 2021 11:03:56 +0100 Subject: [PATCH] Added more precise checking of database states to the bulk pruning test. This highlighted a major bug in the bulk prune process whereby the recent AT states weren't being retained. --- .../hsqldb/HSQLDBDatabasePruning.java | 15 +++++---- .../org/qortal/test/BlockArchiveTests.java | 32 ++++++++++++++++++- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java index 49f54150..4eea59be 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java @@ -91,7 +91,6 @@ public class HSQLDBDatabasePruning { // Archive mode - don't prune anything that hasn't been archived yet maximumBlockToTrim = Math.min(maximumBlockToTrim, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); } - final int startHeight = maximumBlockToTrim; final int endHeight = blockchainHeight; final int blockStep = 10000; @@ -104,10 +103,11 @@ public class HSQLDBDatabasePruning { // Loop through all the LatestATStates and copy them to the new table LOGGER.info("Copying AT states..."); for (int height = 0; height < endHeight; height += blockStep) { - //LOGGER.info(String.format("Copying AT states between %d and %d...", height, height + blockStep - 1)); + final int batchEndHeight = height + blockStep - 1; + //LOGGER.info(String.format("Copying AT states between %d and %d...", height, batchEndHeight)); String sql = "SELECT height, AT_address FROM LatestATStates WHERE height BETWEEN ? AND ?"; - try (ResultSet latestAtStatesResultSet = repository.checkedExecute(sql, height, height + blockStep - 1)) { + try (ResultSet latestAtStatesResultSet = repository.checkedExecute(sql, height, batchEndHeight)) { if (latestAtStatesResultSet != null) { do { int latestAtHeight = latestAtStatesResultSet.getInt(1); @@ -126,9 +126,12 @@ public class HSQLDBDatabasePruning { throw new DataException("Unable to copy ATStates", e); } - if (height >= startHeight) { - // Now copy this AT's states for each recent block they is present in - for (int i = startHeight; i < endHeight; i++) { + // If this batch includes blocks after the maximum block to trim, we will need to copy + // each of its AT states above maximumBlockToTrim as they are considered "recent". We + // need to do this for _all_ AT states in these blocks, regardless of their latest state. + if (batchEndHeight >= maximumBlockToTrim) { + // Now copy this AT's states for each recent block they are present in + for (int i = maximumBlockToTrim; i < endHeight; i++) { if (latestAtHeight < i) { // This AT finished before this block so there is nothing to copy continue; diff --git a/src/test/java/org/qortal/test/BlockArchiveTests.java b/src/test/java/org/qortal/test/BlockArchiveTests.java index 08d760b8..4f12bad8 100644 --- a/src/test/java/org/qortal/test/BlockArchiveTests.java +++ b/src/test/java/org/qortal/test/BlockArchiveTests.java @@ -381,7 +381,7 @@ public class BlockArchiveTests extends Common { // Write blocks 2-900 to the archive (using bulk method) int fileSizeTarget = 425000; // Pre-calculated size of 900 blocks - assertTrue(HSQLDBDatabaseArchiving.buildBlockArchive(repository, 425000)); + assertTrue(HSQLDBDatabaseArchiving.buildBlockArchive(repository, fileSizeTarget)); // Ensure the block archive height has increased assertEquals(901, repository.getBlockArchiveRepository().getBlockArchiveHeight()); @@ -394,6 +394,14 @@ public class BlockArchiveTests extends Common { assertEquals(0, repository.getBlockRepository().getBlockPruneHeight()); assertEquals(0, repository.getATRepository().getAtPruneHeight()); + // Prior to archiving or pruning, ensure blocks 2 to 1002 and their AT states are available in the db + for (int i=2; i<=1002; i++) { + assertNotNull(repository.getBlockRepository().fromHeight(i)); + List atStates = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStates); + assertEquals(1, atStates.size()); + } + // Prune all the archived blocks and AT states (using bulk method) assertTrue(HSQLDBDatabasePruning.pruneBlocks(hsqldb)); assertTrue(HSQLDBDatabasePruning.pruneATStates(hsqldb)); @@ -413,6 +421,28 @@ public class BlockArchiveTests extends Common { // Validate the latest block height in the repository assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight()); + // Ensure blocks 2-900 are all available in the archive + for (int i=2; i<=900; i++) { + assertNotNull(repository.getBlockArchiveRepository().fromHeight(i)); + } + + // Ensure blocks 2-900 are NOT available in the db + for (int i=2; i<=900; i++) { + assertNull(repository.getBlockRepository().fromHeight(i)); + } + + // Ensure blocks 901 to 1002 and their AT states are available in the db + for (int i=901; i<=1002; i++) { + assertNotNull(repository.getBlockRepository().fromHeight(i)); + List atStates = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStates); + assertEquals(1, atStates.size()); + } + + // Ensure blocks 901 to 1002 are not available in the archive + for (int i=901; i<=1002; i++) { + assertNull(repository.getBlockArchiveRepository().fromHeight(i)); + } } }