Added bootstrap tests

This involved adding a feature to the test suite in include the option of using a repository located on disk rather than in memory. Also moved the bootstrap compression/extraction working directories to temporary folders.

src/test/java/org/qortal/test/BootstrapTests.java

@@ -0,0 +1,204 @@
+package org.qortal.test;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.qortal.account.PrivateKeyAccount;
+import org.qortal.controller.BlockMinter;
+import org.qortal.controller.Controller;
+import org.qortal.data.block.BlockData;
+import org.qortal.repository.*;
+import org.qortal.settings.Settings;
+import org.qortal.test.common.AtUtils;
+import org.qortal.test.common.Common;
+import org.qortal.transform.TransformationException;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.nio.file.NoSuchFileException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class BootstrapTests extends Common {
+
+    @Before
+    public void beforeTest() throws DataException, IOException {
+        Common.useSettingsAndDb(Common.testSettingsFilename, false);
+        NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset());
+        this.deleteBootstraps();
+    }
+
+    @After
+    public void afterTest() throws DataException, IOException {
+        this.deleteBootstraps();
+        this.deleteExportDirectory();
+    }
+
+    @Test
+    public void testCanBootstrap() throws DataException, InterruptedException, TransformationException, IOException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            this.buildDummyBlockchain(repository);
+
+            Bootstrap bootstrap = new Bootstrap(repository);
+            assertTrue(bootstrap.canBootstrap());
+
+        }
+    }
+
+    @Test
+    public void testValidateBlockchain() throws DataException, InterruptedException, TransformationException, IOException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            this.buildDummyBlockchain(repository);
+
+            Bootstrap bootstrap = new Bootstrap(repository);
+            assertTrue(bootstrap.validateBlockchain());
+
+        }
+    }
+
+
+    @Test
+    public void testCreateAndImportBootstrap() throws DataException, InterruptedException, TransformationException, IOException {
+
+        Path bootstrapPath = Paths.get(String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap.7z"));
+        Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive", "2-900.dat");
+        BlockData block1000;
+        byte[] originalArchiveContents;
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            this.buildDummyBlockchain(repository);
+
+            // Ensure the compressed bootstrap doesn't exist
+            assertFalse(Files.exists(bootstrapPath));
+
+            Bootstrap bootstrap = new Bootstrap(repository);
+            bootstrap.create();
+
+            // Ensure the compressed bootstrap exists
+            assertTrue(Files.exists(bootstrapPath));
+
+            // Ensure the original block archive file exists
+            assertTrue(Files.exists(archivePath));
+            originalArchiveContents = Files.readAllBytes(archivePath);
+
+            // Ensure block 1000 exists in the repository
+            block1000 = repository.getBlockRepository().fromHeight(1000);
+            assertNotNull(block1000);
+
+            // Ensure we can retrieve block 10 from the archive
+            assertNotNull(repository.getBlockArchiveRepository().fromHeight(10));
+
+            // Now delete block 1000
+            repository.getBlockRepository().delete(block1000);
+            assertNull(repository.getBlockRepository().fromHeight(1000));
+
+            // Overwrite the archive with dummy data, and verify it
+            try (PrintWriter out = new PrintWriter(archivePath.toFile())) {
+                out.println("testdata");
+            }
+            String newline = System.getProperty("line.separator");
+            assertEquals("testdata", Files.readString(archivePath).replace(newline, ""));
+
+            // Ensure we can no longer retrieve block 10 from the archive
+            assertNull(repository.getBlockArchiveRepository().fromHeight(10));
+
+            // Import the bootstrap back in
+            bootstrap.importFromPath(bootstrapPath);
+        }
+
+        // We need a new connection because we have switched to a new repository
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Ensure the block archive file exists
+            assertTrue(Files.exists(archivePath));
+
+            // and that its contents match the original
+            assertArrayEquals(originalArchiveContents, Files.readAllBytes(archivePath));
+
+            // Make sure that block 1000 exists again
+            BlockData newBlock1000 = repository.getBlockRepository().fromHeight(1000);
+            assertNotNull(newBlock1000);
+
+            // and ensure that the signatures match
+            assertArrayEquals(block1000.getSignature(), newBlock1000.getSignature());
+
+            // Ensure we can retrieve block 10 from the archive
+            assertNotNull(repository.getBlockArchiveRepository().fromHeight(10));
+        }
+    }
+
+
+    private void buildDummyBlockchain(Repository repository) throws DataException, InterruptedException, TransformationException, IOException {
+        // Alice self share online
+        List<PrivateKeyAccount> mintingAndOnlineAccounts = new ArrayList<>();
+        PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share");
+        mintingAndOnlineAccounts.add(aliceSelfShare);
+
+        // Deploy an AT so that we have AT state data
+        PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
+        byte[] creationBytes = AtUtils.buildSimpleAT();
+        long fundingAmount = 1_00000000L;
+        AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
+
+        // Mint some blocks so that we are able to archive them later
+        for (int i = 0; i < 1000; i++)
+            BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0]));
+
+        // Assume 900 blocks are trimmed (this specifies the first untrimmed height)
+        repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901);
+        repository.getATRepository().setAtTrimHeight(901);
+
+        // Check the max archive height - this should be one less than the first untrimmed height
+        final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
+
+        // Write blocks 2-900 to the archive
+        BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
+        writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
+        BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
+
+        // Increment block archive height
+        repository.getBlockArchiveRepository().setBlockArchiveHeight(901);
+
+        // Prune all the archived blocks
+        repository.getBlockRepository().pruneBlocks(0, 900);
+        repository.getBlockRepository().setBlockPruneHeight(901);
+
+        // Prune the AT states for the archived blocks
+        repository.getATRepository().rebuildLatestAtStates();
+        repository.getATRepository().pruneAtStates(0, 900);
+        repository.getATRepository().setAtPruneHeight(901);
+
+        // Refill cache, used by Controller.getMinimumLatestBlockTimestamp() and other methods
+        Controller.getInstance().refillLatestBlocksCache();
+
+        repository.saveChanges();
+    }
+
+    private void deleteBootstraps() throws IOException {
+        try {
+            Path path = Paths.get(String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap.7z"));
+            Files.delete(path);
+
+        } catch (NoSuchFileException e) {
+            // Nothing to delete
+        }
+    }
+
+    private void deleteExportDirectory() {
+        // Delete archive directory if exists
+        Path archivePath = Paths.get(Settings.getInstance().getExportPath());
+        try {
+            FileUtils.deleteDirectory(archivePath.toFile());
+        } catch (IOException e) {
+
+        }
+    }
+
+}

src/test/java/org/qortal/test/common/Common.java

@@ -2,8 +2,11 @@ package org.qortal.test.common;
 
 import static org.junit.Assert.*;
 
+import java.io.IOException;
 import java.math.BigDecimal;
 import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.security.Security;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -15,6 +18,7 @@ import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
@@ -46,9 +50,15 @@ public class Common {
 
 	private static final Logger LOGGER = LogManager.getLogger(Common.class);
 
-	public static final String testConnectionUrl = "jdbc:hsqldb:mem:testdb";
-	// For debugging, use this instead to write DB to disk for examination:
-	// public static final String testConnectionUrl = "jdbc:hsqldb:file:testdb/blockchain;create=true";
+	public static final String testConnectionUrlMemory = "jdbc:hsqldb:mem:testdb";
+	public static final String testConnectionUrlDisk = "jdbc:hsqldb:file:%s/blockchain;create=true";
+
+	// For debugging, use testConnectionUrlDisk instead of memory, to write DB to disk for examination.
+	// This can be achieved using `Common.useSettingsAndDb(Common.testSettingsFilename, false);`
+	// where `false` specifies to use a repository on disk rather than one in memory.
+	// Make sure to also comment out `Common.deleteTestRepository();` in closeRepository() below, so that
+	// the files remain after the test finishes.
+
 
 	public static final String testSettingsFilename = "test-settings-v2.json";
 
@@ -100,7 +110,7 @@ public class Common {
 		return testAccountsByName.values().stream().map(account -> new TestAccount(repository, account)).collect(Collectors.toList());
 	}
 
-	public static void useSettings(String settingsFilename) throws DataException {
+	public static void useSettingsAndDb(String settingsFilename, boolean dbInMemory) throws DataException {
 		closeRepository();
 
 		// Load/check settings, which potentially sets up blockchain config, etc.
@@ -109,11 +119,15 @@ public class Common {
 		assertNotNull("Test settings JSON file not found", testSettingsUrl);
 		Settings.fileInstance(testSettingsUrl.getPath());
 
-		setRepository();
+		setRepository(dbInMemory);
 
 		resetBlockchain();
 	}
 
+	public static void useSettings(String settingsFilename) throws DataException {
+		Common.useSettingsAndDb(settingsFilename, true);
+	}
+
 	public static void useDefaultSettings() throws DataException {
 		useSettings(testSettingsFilename);
 		NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset());
@@ -186,15 +200,33 @@ public class Common {
 		assertTrue(String.format("Non-genesis %s remains", typeName), remainingClone.isEmpty());
 	}
 
-	@BeforeClass
-	public static void setRepository() throws DataException {
-		RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(testConnectionUrl);
+	public static void setRepository(boolean inMemory) throws DataException {
+		String connectionUrlDisk = String.format(testConnectionUrlDisk, Settings.getInstance().getRepositoryPath());
+		String connectionUrl = inMemory ? testConnectionUrlMemory : connectionUrlDisk;
+		RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(connectionUrl);
 		RepositoryManager.setRepositoryFactory(repositoryFactory);
 	}
 
+	public static void deleteTestRepository() throws DataException {
+		// Delete repository directory if exists
+		Path repositoryPath = Paths.get(Settings.getInstance().getRepositoryPath());
+		try {
+			FileUtils.deleteDirectory(repositoryPath.toFile());
+		} catch (IOException e) {
+			throw new DataException(String.format("Unable to delete test repository: %s", e.getMessage()));
+		}
+	}
+
+	@BeforeClass
+	public static void setRepositoryInMemory() throws DataException {
+		Common.deleteTestRepository();
+		Common.setRepository(true);
+	}
+
 	@AfterClass
 	public static void closeRepository() throws DataException {
 		RepositoryManager.closeRepositoryFactory();
+		Common.deleteTestRepository(); // Comment out this line in you need to inspect the database after running a test
 	}
 
 	// Test assertions

src/test/resources/test-settings-v2-bitcoin-regtest.json

@@ -1,4 +1,5 @@
 {
+  "repositoryPath": "testdb",
   "bitcoinNet": "REGTEST",
   "litecoinNet": "REGTEST",
   "restrictedApi": false,
@@ -7,5 +8,6 @@
   "bootstrap": false,
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
-  "minPeers": 0
+  "minPeers": 0,
+  "pruneBlockLimit": 100
 }

src/test/resources/test-settings-v2-block-archive.json

@@ -8,6 +8,6 @@
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0,
-  "pruneBlockLimit": 1450,
+  "pruneBlockLimit": 100,
   "repositoryPath": "dbtest"
 }

src/test/resources/test-settings-v2-founder-rewards.json

@@ -1,9 +1,11 @@
 {
+  "repositoryPath": "testdb",
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-founder-rewards.json",
   "exportPath": "qortal-backup-test",
   "bootstrap": false,
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
-  "minPeers": 0
+  "minPeers": 0,
+  "pruneBlockLimit": 100
 }

src/test/resources/test-settings-v2-leftover-reward.json

@@ -1,9 +1,11 @@
 {
+  "repositoryPath": "testdb",
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-leftover-reward.json",
   "exportPath": "qortal-backup-test",
   "bootstrap": false,
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
-  "minPeers": 0
+  "minPeers": 0,
+  "pruneBlockLimit": 100
 }

src/test/resources/test-settings-v2-minting.json

@@ -1,9 +1,11 @@
 {
+  "repositoryPath": "testdb",
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-minting.json",
   "exportPath": "qortal-backup-test",
   "bootstrap": false,
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
-  "minPeers": 0
+  "minPeers": 0,
+  "pruneBlockLimit": 100
 }

src/test/resources/test-settings-v2-qora-holder-extremes.json

@@ -1,9 +1,11 @@
 {
+  "repositoryPath": "testdb",
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder-extremes.json",
   "exportPath": "qortal-backup-test",
   "bootstrap": false,
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
-  "minPeers": 0
+  "minPeers": 0,
+  "pruneBlockLimit": 100
 }

src/test/resources/test-settings-v2-qora-holder.json

@@ -1,9 +1,11 @@
 {
+  "repositoryPath": "testdb",
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder.json",
   "exportPath": "qortal-backup-test",
   "bootstrap": false,
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
-  "minPeers": 0
+  "minPeers": 0,
+  "pruneBlockLimit": 100
 }

src/test/resources/test-settings-v2-reward-levels.json

@@ -1,9 +1,11 @@
 {
+  "repositoryPath": "testdb",
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-reward-levels.json",
   "exportPath": "qortal-backup-test",
   "bootstrap": false,
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
-  "minPeers": 0
+  "minPeers": 0,
+  "pruneBlockLimit": 100
 }

src/test/resources/test-settings-v2-reward-scaling.json

@@ -1,9 +1,11 @@
 {
+  "repositoryPath": "testdb",
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-reward-scaling.json",
   "exportPath": "qortal-backup-test",
   "bootstrap": false,
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
-  "minPeers": 0
+  "minPeers": 0,
+  "pruneBlockLimit": 100
 }

src/test/resources/test-settings-v2.json

@@ -1,4 +1,5 @@
 {
+  "repositoryPath": "testdb",
   "bitcoinNet": "TEST3",
   "litecoinNet": "TEST3",
   "restrictedApi": false,
@@ -7,5 +8,7 @@
   "bootstrap": false,
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
-  "minPeers": 0
+  "minPeers": 0,
+  "pruneBlockLimit": 100,
+  "bootstrapFilenamePrefix": "test-"
 }