Added bulk pruning test, which highlighted some bugs in both bulk and regular pruning.

src/main/java/org/qortal/controller/repository/BlockPruner.java

@@ -83,19 +83,18 @@ public class BlockPruner implements Runnable {
 				repository.saveChanges();
 
 				if (numBlocksPruned > 0) {
-					final int finalPruneStartHeight = pruneStartHeight;
+					LOGGER.debug(String.format("Pruned %d block%s between %d and %d",
-					LOGGER.debug(() -> String.format("Pruned %d block%s between %d and %d",
 							numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""),
-							finalPruneStartHeight, upperPruneHeight));
+							pruneStartHeight, upperPruneHeight));
 				} else {
-					// Can we move onto next batch?
+					final int nextPruneHeight = upperPruneHeight + 1;
-					if (upperPrunableHeight > upperBatchHeight) {
+					repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight);
-						pruneStartHeight = upperBatchHeight;
+					repository.saveChanges();
-						repository.getBlockRepository().setBlockPruneHeight(pruneStartHeight);
+					LOGGER.debug(String.format("Bumping block base prune height to %d", pruneStartHeight));
-						repository.saveChanges();
 
-						final int finalPruneStartHeight = pruneStartHeight;
+					// Can we move onto next batch?
-						LOGGER.debug(() -> String.format("Bumping block base prune height to %d", finalPruneStartHeight));
+					if (upperPrunableHeight > nextPruneHeight) {
+						pruneStartHeight = nextPruneHeight;
 					}
 					else {
 						// We've pruned up to the upper prunable height

src/main/java/org/qortal/repository/BlockArchiveWriter.java

@@ -29,14 +29,17 @@ public class BlockArchiveWriter {
 
     private static final Logger LOGGER = LogManager.getLogger(BlockArchiveWriter.class);
 
+    public static final long DEFAULT_FILE_SIZE_TARGET = 100 * 1024 * 1024; // 100MiB
+
     private int startHeight;
     private final int endHeight;
     private final Repository repository;
 
-    private long fileSizeTarget = 100 * 1024 * 1024; // 100MiB
+    private long fileSizeTarget = DEFAULT_FILE_SIZE_TARGET;
     private boolean shouldEnforceFileSizeTarget = true;
 
     private int writtenCount;
+    private int lastWrittenHeight;
     private Path outputPath;
 
     public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) {
@@ -169,6 +172,7 @@ public class BlockArchiveWriter {
         BlockArchiveReader.getInstance().invalidateFileListCache();
 
         this.writtenCount = i;
+        this.lastWrittenHeight = endHeight;
         this.outputPath = Paths.get(filePath);
         return BlockArchiveWriteResult.OK;
     }
@@ -177,6 +181,10 @@ public class BlockArchiveWriter {
         return this.writtenCount;
     }
 
+    public int getLastWrittenHeight() {
+        return this.lastWrittenHeight;
+    }
+
     public Path getOutputPath() {
         return this.outputPath;
     }

src/main/java/org/qortal/repository/RepositoryManager.java

@@ -64,7 +64,7 @@ public abstract class RepositoryManager {
 		if (Settings.getInstance().isArchiveEnabled()) {
 			if (RepositoryManager.canArchiveOrPrune()) {
 				try {
-					return HSQLDBDatabaseArchiving.buildBlockArchive(repository);
+					return HSQLDBDatabaseArchiving.buildBlockArchive(repository, BlockArchiveWriter.DEFAULT_FILE_SIZE_TARGET);
 
 				} catch (DataException e) {
 					LOGGER.info("Unable to build block archive. The database may have been left in an inconsistent state.");

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java

@@ -30,7 +30,7 @@ public class HSQLDBDatabaseArchiving {
     private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabaseArchiving.class);
 
 
-    public static boolean buildBlockArchive(Repository repository) throws DataException {
+    public static boolean buildBlockArchive(Repository repository, long fileSizeTarget) throws DataException {
 
         // Only build the archive if we have never done so before
         int archiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight();
@@ -47,11 +47,12 @@ public class HSQLDBDatabaseArchiving {
         while (!Controller.isStopping()) {
             try {
                 BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository);
+                writer.setFileSizeTarget(fileSizeTarget);
                 BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
                 switch (result) {
                     case OK:
                         // Increment block archive height
-                        startHeight += writer.getWrittenCount();
+                        startHeight = writer.getLastWrittenHeight() + 1;
                         repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight);
                         repository.saveChanges();
                         break;

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java

@@ -167,7 +167,8 @@ public class HSQLDBDatabasePruning {
         repository.executeCheckedUpdate("CHECKPOINT");
 
         // Update the prune height
-        repository.getATRepository().setAtPruneHeight(maximumBlockToTrim);
+        int nextPruneHeight = maximumBlockToTrim + 1;
+        repository.getATRepository().setAtPruneHeight(nextPruneHeight);
         repository.saveChanges();
 
         repository.executeCheckedUpdate("CHECKPOINT");
@@ -291,13 +292,14 @@ public class HSQLDBDatabasePruning {
                         numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""),
                         pruneStartHeight, upperPruneHeight));
             } else {
-                repository.getBlockRepository().setBlockPruneHeight(upperBatchHeight);
+                final int nextPruneHeight = upperPruneHeight + 1;
+                repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight);
                 repository.saveChanges();
-                LOGGER.debug(String.format("Bumping block base prune height to %d", upperBatchHeight));
+                LOGGER.debug(String.format("Bumping block base prune height to %d", nextPruneHeight));
 
                 // Can we move onto next batch?
-                if (upperPrunableHeight > upperBatchHeight) {
+                if (upperPrunableHeight > nextPruneHeight) {
-                    pruneStartHeight = upperBatchHeight;
+                    pruneStartHeight = nextPruneHeight;
                 }
                 else {
                     // We've finished pruning

src/test/java/org/qortal/test/BlockArchiveTests.java

@@ -10,6 +10,8 @@ import org.qortal.data.at.ATStateData;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.repository.*;
+import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving;
+import org.qortal.repository.hsqldb.HSQLDBDatabasePruning;
 import org.qortal.repository.hsqldb.HSQLDBRepository;
 import org.qortal.settings.Settings;
 import org.qortal.test.common.AtUtils;
@@ -24,7 +26,6 @@ import org.qortal.utils.Triple;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.ByteBuffer;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.sql.SQLException;
@@ -347,6 +348,74 @@ public class BlockArchiveTests extends Common {
 		}
 	}
 
+	@Test
+	public void testBulkArchiveAndPrune() throws DataException, InterruptedException, TransformationException, IOException, SQLException {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			HSQLDBRepository hsqldb = (HSQLDBRepository) repository;
+
+			// Alice self share online
+			List<PrivateKeyAccount> mintingAndOnlineAccounts = new ArrayList<>();
+			PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share");
+			mintingAndOnlineAccounts.add(aliceSelfShare);
+
+			// Deploy an AT so that we have AT state data
+			PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
+			byte[] creationBytes = AtUtils.buildSimpleAT();
+			long fundingAmount = 1_00000000L;
+			AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
+
+			// Mint some blocks so that we are able to archive them later
+			for (int i = 0; i < 1000; i++)
+				BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0]));
+
+			// Assume 900 blocks are trimmed (this specifies the first untrimmed height)
+			repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901);
+			repository.getATRepository().setAtTrimHeight(901);
+
+			// Check the max archive height - this should be one less than the first untrimmed height
+			final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
+			assertEquals(900, maximumArchiveHeight);
+
+			// Check the current archive height
+			assertEquals(0, repository.getBlockArchiveRepository().getBlockArchiveHeight());
+
+			// Write blocks 2-900 to the archive (using bulk method)
+			int fileSizeTarget = 425000; // Pre-calculated size of 900 blocks
+			assertTrue(HSQLDBDatabaseArchiving.buildBlockArchive(repository, 425000));
+
+			// Ensure the block archive height has increased
+			assertEquals(901, repository.getBlockArchiveRepository().getBlockArchiveHeight());
+
+			// Ensure the SQL repository contains blocks 2 and 900...
+			assertNotNull(repository.getBlockRepository().fromHeight(2));
+			assertNotNull(repository.getBlockRepository().fromHeight(900));
+
+			// Check the current prune heights
+			assertEquals(0, repository.getBlockRepository().getBlockPruneHeight());
+			assertEquals(0, repository.getATRepository().getAtPruneHeight());
+
+			// Prune all the archived blocks and AT states (using bulk method)
+			assertTrue(HSQLDBDatabasePruning.pruneBlocks(hsqldb));
+			assertTrue(HSQLDBDatabasePruning.pruneATStates(hsqldb));
+
+			// Ensure the current prune heights have increased
+			assertEquals(901, repository.getBlockRepository().getBlockPruneHeight());
+			assertEquals(901, repository.getATRepository().getAtPruneHeight());
+
+			// Now ensure the SQL repository is missing blocks 2 and 900...
+			assertNull(repository.getBlockRepository().fromHeight(2));
+			assertNull(repository.getBlockRepository().fromHeight(900));
+
+			// ... but it's not missing blocks 1 and 901 (we don't prune the genesis block)
+			assertNotNull(repository.getBlockRepository().fromHeight(1));
+			assertNotNull(repository.getBlockRepository().fromHeight(901));
+
+			// Validate the latest block height in the repository
+			assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight());
+
+		}
+	}
+
 	@Test
 	public void testTrimArchivePruneAndOrphan() throws DataException, InterruptedException, TransformationException, IOException {
 		try (final Repository repository = RepositoryManager.getRepository()) {