Added bootstrap tests

This involved adding a feature to the test suite in include the option of using a repository located on disk rather than in memory. Also moved the bootstrap compression/extraction working directories to temporary folders.

src/main/java/org/qortal/api/resource/BootstrapResource.java

@@ -12,6 +12,8 @@ import org.qortal.api.ApiExceptionFactory;
 import org.qortal.api.Security;
 import org.qortal.repository.Bootstrap;
 import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.*;
@@ -44,17 +46,18 @@ public class BootstrapResource {
 	public String createBootstrap() {
 		Security.checkApiCallAllowed(request);
 
-		Bootstrap bootstrap = new Bootstrap();
-		if (!bootstrap.canBootstrap()) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
-		}
+		try (final Repository repository = RepositoryManager.getRepository()) {
 
-		boolean isBlockchainValid = bootstrap.validateBlockchain();
-		if (!isBlockchainValid) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
-		}
+			Bootstrap bootstrap = new Bootstrap(repository);
+			if (!bootstrap.canBootstrap()) {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+			}
+
+			boolean isBlockchainValid = bootstrap.validateBlockchain();
+			if (!isBlockchainValid) {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
+			}
 
-		try {
 			return bootstrap.create();
 
 		} catch (DataException | InterruptedException | IOException e) {
@@ -78,7 +81,13 @@ public class BootstrapResource {
 	public boolean validateBootstrap() {
 		Security.checkApiCallAllowed(request);
 
-		Bootstrap bootstrap = new Bootstrap();
-		return bootstrap.validateCompleteBlockchain();
+		try (final Repository repository = RepositoryManager.getRepository()) {
+
+			Bootstrap bootstrap = new Bootstrap(repository);
+			return bootstrap.validateCompleteBlockchain();
+
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
+		}
 	}
 }

src/main/java/org/qortal/block/BlockChain.java

@@ -508,6 +508,7 @@ public class BlockChain {
 		}
 
 		boolean pruningEnabled = Settings.getInstance().isPruningEnabled();
+		boolean archiveEnabled = Settings.getInstance().isArchiveEnabled();
 		boolean hasBlocks = (chainTip != null && chainTip.getHeight() > 1);
 
 		if (pruningEnabled && hasBlocks) {
@@ -527,31 +528,16 @@ public class BlockChain {
 
 			// Set the number of blocks to validate based on the pruned state of the chain
 			// If pruned, subtract an extra 10 to allow room for error
-			int blocksToValidate = pruningEnabled ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440;
+			int blocksToValidate = (pruningEnabled || archiveEnabled) ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440;
 
 			int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - blocksToValidate, 1);
 			BlockData detachedBlockData = repository.getBlockRepository().getDetachedBlockSignature(startHeight);
 
 			if (detachedBlockData != null) {
-				LOGGER.error(String.format("Block %d's reference does not match any block's signature", detachedBlockData.getHeight()));
-
-				// Orphan if we aren't a pruning node
-				if (!Settings.getInstance().isPruningEnabled()) {
-
-					// Wait for blockchain lock (whereas orphan() only tries to get lock)
-					ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
-					blockchainLock.lock();
-					try {
-						LOGGER.info(String.format("Orphaning back to block %d", detachedBlockData.getHeight() - 1));
-						orphan(detachedBlockData.getHeight() - 1);
-					} finally {
-						blockchainLock.unlock();
-					}
-				}
-				else {
-					LOGGER.error(String.format("Not orphaning because we are in pruning mode. You may be on an " +
-							"invalid chain and should consider bootstrapping or re-syncing from genesis."));
-				}
+				LOGGER.error(String.format("Block %d's reference does not match any block's signature",
+						detachedBlockData.getHeight()));
+				LOGGER.error(String.format("Your chain may be invalid and you should consider bootstrapping" +
+						" or re-syncing from genesis."));
 			}
 		}
 	}
@@ -618,8 +604,10 @@ public class BlockChain {
 		boolean shouldBootstrap = Settings.getInstance().getBootstrap();
 		if (shouldBootstrap) {
 			// Settings indicate that we should apply a bootstrap rather than rebuilding and syncing from genesis
-			Bootstrap bootstrap = new Bootstrap();
-			bootstrap.startImport();
+			try (final Repository repository = RepositoryManager.getRepository()) {
+				Bootstrap bootstrap = new Bootstrap(repository);
+				bootstrap.startImport();
+			}
 			return;
 		}
 

src/main/java/org/qortal/repository/Bootstrap.java

@@ -9,27 +9,25 @@ import org.qortal.data.account.MintingAccountData;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.crosschain.TradeBotData;
 import org.qortal.repository.hsqldb.HSQLDBImportExport;
-import org.qortal.repository.hsqldb.HSQLDBRepository;
 import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
 import org.qortal.settings.Settings;
 import org.qortal.utils.NTP;
 import org.qortal.utils.SevenZ;
 
-import java.io.BufferedInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
+import java.nio.file.*;
 import java.util.List;
 import java.util.concurrent.locks.ReentrantLock;
 
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+
 
 public class Bootstrap {
 
+    private Repository repository;
+
     private static final Logger LOGGER = LogManager.getLogger(Bootstrap.class);
 
     /** The maximum number of untrimmed blocks allowed to be included in a bootstrap, beyond the trim threshold */
@@ -39,8 +37,8 @@ public class Bootstrap {
     private static final int MAXIMUM_UNPRUNED_BLOCKS = 100;
 
 
-    public Bootstrap() {
-
+    public Bootstrap(Repository repository) {
+        this.repository = repository;
     }
 
     /**
@@ -50,9 +48,8 @@ public class Bootstrap {
      * All failure reasons are logged
      */
     public boolean canBootstrap() {
-        LOGGER.info("Checking repository state...");
-
-        try (final Repository repository = RepositoryManager.getRepository()) {
+        try {
+            LOGGER.info("Checking repository state...");
 
             final boolean pruningEnabled = Settings.getInstance().isPruningEnabled();
             final boolean archiveEnabled = Settings.getInstance().isArchiveEnabled();
@@ -203,73 +200,80 @@ public class Bootstrap {
     }
 
     public String create() throws DataException, InterruptedException, IOException {
-        try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) {
 
-            LOGGER.info("Acquiring blockchain lock...");
-            ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
-            blockchainLock.lockInterruptibly();
+        LOGGER.info("Acquiring blockchain lock...");
+        ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+        blockchainLock.lockInterruptibly();
 
-            Path inputPath = null;
+        Path inputPath = null;
+        Path outputPath = null;
 
+        try {
+
+            LOGGER.info("Exporting local data...");
+            repository.exportNodeLocalData();
+
+            LOGGER.info("Deleting trade bot states...");
+            List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
+            for (TradeBotData tradeBotData : allTradeBotData) {
+                repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
+            }
+
+            LOGGER.info("Deleting minting accounts...");
+            List<MintingAccountData> mintingAccounts = repository.getAccountRepository().getMintingAccounts();
+            for (MintingAccountData mintingAccount : mintingAccounts) {
+                repository.getAccountRepository().delete(mintingAccount.getPrivateKey());
+            }
+
+            repository.saveChanges();
+
+            LOGGER.info("Performing repository maintenance...");
+            repository.performPeriodicMaintenance();
+
+            LOGGER.info("Creating bootstrap...");
+            repository.backup(true, "bootstrap");
+
+            LOGGER.info("Moving files to output directory...");
+            inputPath = Paths.get(Settings.getInstance().getRepositoryPath(), "bootstrap");
+            outputPath = Paths.get(Files.createTempDirectory("qortal-bootstrap").toString(), "bootstrap");
+
+
+            // Move the db backup to a "bootstrap" folder in the root directory
+            Files.move(inputPath, outputPath, REPLACE_EXISTING);
+
+            // Copy the archive folder to inside the bootstrap folder
+            FileUtils.copyDirectory(
+                    Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toFile(),
+                    Paths.get(outputPath.toString(), "archive").toFile()
+            );
+
+            LOGGER.info("Compressing...");
+            String compressedOutputPath = String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap.7z");
             try {
+                Files.delete(Paths.get(compressedOutputPath));
+            } catch (NoSuchFileException e) {
+                // Doesn't exist, so no need to delete
+            }
+            SevenZ.compress(compressedOutputPath, outputPath.toFile());
 
-                LOGGER.info("Exporting local data...");
-                repository.exportNodeLocalData();
+            // Return the path to the compressed bootstrap file
+            Path finalPath = Paths.get(outputPath.toString(), compressedOutputPath);
+            return finalPath.toAbsolutePath().toString();
 
-                LOGGER.info("Deleting trade bot states...");
-                List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
-                for (TradeBotData tradeBotData : allTradeBotData) {
-                    repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
-                }
+        } finally {
+            LOGGER.info("Re-importing local data...");
+            Path exportPath = HSQLDBImportExport.getExportDirectory(false);
+            repository.importDataFromFile(Paths.get(exportPath.toString(), "TradeBotStates.json").toString());
+            repository.importDataFromFile(Paths.get(exportPath.toString(), "MintingAccounts.json").toString());
 
-                LOGGER.info("Deleting minting accounts...");
-                List<MintingAccountData> mintingAccounts = repository.getAccountRepository().getMintingAccounts();
-                for (MintingAccountData mintingAccount : mintingAccounts) {
-                    repository.getAccountRepository().delete(mintingAccount.getPrivateKey());
-                }
+            blockchainLock.unlock();
 
-                repository.saveChanges();
-
-                LOGGER.info("Performing repository maintenance...");
-                repository.performPeriodicMaintenance();
-
-                LOGGER.info("Creating bootstrap...");
-                repository.backup(true, "bootstrap");
-
-                LOGGER.info("Moving files to output directory...");
-                inputPath = Paths.get(Settings.getInstance().getRepositoryPath(), "bootstrap");
-                Path outputPath = Paths.get("bootstrap");
+            // Cleanup
+            if (inputPath != null) {
+                FileUtils.deleteDirectory(inputPath.toFile());
+            }
+            if (outputPath != null) {
                 FileUtils.deleteDirectory(outputPath.toFile());
-
-                // Move the db backup to a "bootstrap" folder in the root directory
-                Files.move(inputPath, outputPath);
-
-                // Copy the archive folder to inside the bootstrap folder
-                FileUtils.copyDirectory(
-                        Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toFile(),
-                        Paths.get(outputPath.toString(), "archive").toFile()
-                );
-
-                LOGGER.info("Compressing...");
-                String fileName = "bootstrap.7z";
-                SevenZ.compress(fileName, outputPath.toFile());
-
-                // Return the path to the compressed bootstrap file
-                Path finalPath = Paths.get(outputPath.toString(), fileName);
-                return finalPath.toAbsolutePath().toString();
-
-            } finally {
-                LOGGER.info("Re-importing local data...");
-                Path exportPath = HSQLDBImportExport.getExportDirectory(false);
-                repository.importDataFromFile(Paths.get(exportPath.toString(), "TradeBotStates.json").toString());
-                repository.importDataFromFile(Paths.get(exportPath.toString(), "MintingAccounts.json").toString());
-
-                blockchainLock.unlock();
-
-                // Cleanup
-                if (inputPath != null) {
-                    FileUtils.deleteDirectory(inputPath.toFile());
-                }
             }
         }
     }
@@ -305,7 +309,7 @@ public class Bootstrap {
             try {
                 LOGGER.info("Downloading bootstrap...");
                 InputStream in = new URL(bootstrapUrl).openStream();
-                Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING);
+                Files.copy(in, path, REPLACE_EXISTING);
                 break;
 
             } catch (IOException e) {
@@ -324,7 +328,7 @@ public class Bootstrap {
         throw new DataException("Unable to download bootstrap");
     }
 
-    private void importFromPath(Path path) throws InterruptedException, DataException, IOException {
+    public void importFromPath(Path path) throws InterruptedException, DataException, IOException {
 
         ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
         blockchainLock.lockInterruptibly();
@@ -332,13 +336,18 @@ public class Bootstrap {
         try {
             LOGGER.info("Extracting bootstrap...");
             Path input = path.toAbsolutePath();
-            Path output = path.getParent().toAbsolutePath();
+            Path output = path.toAbsolutePath().getParent().toAbsolutePath();
             SevenZ.decompress(input.toString(), output.toFile());
 
             LOGGER.info("Stopping repository...");
+            // Close the repository while we are still able to
+            // Otherwise, the caller will run into difficulties when it tries to close it
+            repository.discardChanges();
+            repository.close();
+            // Now close the repository factory so that we can swap out the database files
             RepositoryManager.closeRepositoryFactory();
 
-            Path inputPath = Paths.get("bootstrap");
+            Path inputPath = Paths.get(output.toString(), "bootstrap");
             Path outputPath = Paths.get(Settings.getInstance().getRepositoryPath());
             if (!inputPath.toFile().exists()) {
                 throw new DataException("Extracted bootstrap doesn't exist");

src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java

@@ -267,7 +267,7 @@ public class HSQLDBRepository implements Repository {
 	public void close() throws DataException {
 		// Already closed? No need to do anything but maybe report double-call
 		if (this.connection == null) {
-			LOGGER.warn("HSQLDBRepository.close() called when repository already closed", new Exception("Repository already closed"));
+			LOGGER.warn("HSQLDBRepository.close() called when repository already closed. This is expected when bootstrapping.");
 			return;
 		}
 

src/main/java/org/qortal/settings/Settings.java

@@ -191,6 +191,9 @@ public class Settings {
 	// Export/import
 	private String exportPath = "qortal-backup";
 
+	// Bootstrap
+	private String bootstrapFilenamePrefix = "";
+
 	// Auto-update sources
 	private String[] autoUpdateRepos = new String[] {
 		"https://github.com/Qortal/qortal/raw/%s/qortal.update",
@@ -513,6 +516,10 @@ public class Settings {
 		return this.exportPath;
 	}
 
+	public String getBootstrapFilenamePrefix() {
+		return this.bootstrapFilenamePrefix;
+	}
+
 	public boolean isAutoUpdateEnabled() {
 		return this.autoUpdateEnabled;
 	}

src/main/java/org/qortal/utils/SevenZ.java

@@ -18,8 +18,8 @@ public class SevenZ {
 
     }
 
-    public static void compress(String name, File... files) throws IOException {
-        try (SevenZOutputFile out = new SevenZOutputFile(new File(name))){
+    public static void compress(String outputPath, File... files) throws IOException {
+        try (SevenZOutputFile out = new SevenZOutputFile(new File(outputPath))){
             for (File file : files){
                 addToArchiveCompression(out, file, ".");
             }