From 7375357b1115c59d7dd6a6a338877bc565bdb8ab Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 1 Oct 2021 07:44:33 +0100 Subject: [PATCH] Added bootstrap tests This involved adding a feature to the test suite in include the option of using a repository located on disk rather than in memory. Also moved the bootstrap compression/extraction working directories to temporary folders. --- .../api/resource/BootstrapResource.java | 31 ++- .../java/org/qortal/block/BlockChain.java | 32 +-- .../java/org/qortal/repository/Bootstrap.java | 157 +++++++------- .../repository/hsqldb/HSQLDBRepository.java | 2 +- .../java/org/qortal/settings/Settings.java | 7 + src/main/java/org/qortal/utils/SevenZ.java | 4 +- .../java/org/qortal/test/BootstrapTests.java | 204 ++++++++++++++++++ .../java/org/qortal/test/common/Common.java | 48 ++++- .../test-settings-v2-bitcoin-regtest.json | 4 +- .../test-settings-v2-block-archive.json | 2 +- .../test-settings-v2-founder-rewards.json | 4 +- .../test-settings-v2-leftover-reward.json | 4 +- .../resources/test-settings-v2-minting.json | 4 +- ...test-settings-v2-qora-holder-extremes.json | 4 +- .../test-settings-v2-qora-holder.json | 4 +- .../test-settings-v2-reward-levels.json | 4 +- .../test-settings-v2-reward-scaling.json | 4 +- src/test/resources/test-settings-v2.json | 5 +- 18 files changed, 396 insertions(+), 128 deletions(-) create mode 100644 src/test/java/org/qortal/test/BootstrapTests.java diff --git a/src/main/java/org/qortal/api/resource/BootstrapResource.java b/src/main/java/org/qortal/api/resource/BootstrapResource.java index fe2ed378..6cb5e996 100644 --- a/src/main/java/org/qortal/api/resource/BootstrapResource.java +++ b/src/main/java/org/qortal/api/resource/BootstrapResource.java @@ -12,6 +12,8 @@ import org.qortal.api.ApiExceptionFactory; import org.qortal.api.Security; import org.qortal.repository.Bootstrap; import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.*; @@ -44,17 +46,18 @@ public class BootstrapResource { public String createBootstrap() { Security.checkApiCallAllowed(request); - Bootstrap bootstrap = new Bootstrap(); - if (!bootstrap.canBootstrap()) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); - } + try (final Repository repository = RepositoryManager.getRepository()) { - boolean isBlockchainValid = bootstrap.validateBlockchain(); - if (!isBlockchainValid) { - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); - } + Bootstrap bootstrap = new Bootstrap(repository); + if (!bootstrap.canBootstrap()) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); + } + + boolean isBlockchainValid = bootstrap.validateBlockchain(); + if (!isBlockchainValid) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } - try { return bootstrap.create(); } catch (DataException | InterruptedException | IOException e) { @@ -78,7 +81,13 @@ public class BootstrapResource { public boolean validateBootstrap() { Security.checkApiCallAllowed(request); - Bootstrap bootstrap = new Bootstrap(); - return bootstrap.validateCompleteBlockchain(); + try (final Repository repository = RepositoryManager.getRepository()) { + + Bootstrap bootstrap = new Bootstrap(repository); + return bootstrap.validateCompleteBlockchain(); + + } catch (DataException e) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE); + } } } diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java index 98b8d4fd..a0aca44d 100644 --- a/src/main/java/org/qortal/block/BlockChain.java +++ b/src/main/java/org/qortal/block/BlockChain.java @@ -508,6 +508,7 @@ public class BlockChain { } boolean pruningEnabled = Settings.getInstance().isPruningEnabled(); + boolean archiveEnabled = Settings.getInstance().isArchiveEnabled(); boolean hasBlocks = (chainTip != null && chainTip.getHeight() > 1); if (pruningEnabled && hasBlocks) { @@ -527,31 +528,16 @@ public class BlockChain { // Set the number of blocks to validate based on the pruned state of the chain // If pruned, subtract an extra 10 to allow room for error - int blocksToValidate = pruningEnabled ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440; + int blocksToValidate = (pruningEnabled || archiveEnabled) ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440; int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - blocksToValidate, 1); BlockData detachedBlockData = repository.getBlockRepository().getDetachedBlockSignature(startHeight); if (detachedBlockData != null) { - LOGGER.error(String.format("Block %d's reference does not match any block's signature", detachedBlockData.getHeight())); - - // Orphan if we aren't a pruning node - if (!Settings.getInstance().isPruningEnabled()) { - - // Wait for blockchain lock (whereas orphan() only tries to get lock) - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - blockchainLock.lock(); - try { - LOGGER.info(String.format("Orphaning back to block %d", detachedBlockData.getHeight() - 1)); - orphan(detachedBlockData.getHeight() - 1); - } finally { - blockchainLock.unlock(); - } - } - else { - LOGGER.error(String.format("Not orphaning because we are in pruning mode. You may be on an " + - "invalid chain and should consider bootstrapping or re-syncing from genesis.")); - } + LOGGER.error(String.format("Block %d's reference does not match any block's signature", + detachedBlockData.getHeight())); + LOGGER.error(String.format("Your chain may be invalid and you should consider bootstrapping" + + " or re-syncing from genesis.")); } } } @@ -618,8 +604,10 @@ public class BlockChain { boolean shouldBootstrap = Settings.getInstance().getBootstrap(); if (shouldBootstrap) { // Settings indicate that we should apply a bootstrap rather than rebuilding and syncing from genesis - Bootstrap bootstrap = new Bootstrap(); - bootstrap.startImport(); + try (final Repository repository = RepositoryManager.getRepository()) { + Bootstrap bootstrap = new Bootstrap(repository); + bootstrap.startImport(); + } return; } diff --git a/src/main/java/org/qortal/repository/Bootstrap.java b/src/main/java/org/qortal/repository/Bootstrap.java index 2289db5e..a7d9df37 100644 --- a/src/main/java/org/qortal/repository/Bootstrap.java +++ b/src/main/java/org/qortal/repository/Bootstrap.java @@ -9,27 +9,25 @@ import org.qortal.data.account.MintingAccountData; import org.qortal.data.block.BlockData; import org.qortal.data.crosschain.TradeBotData; import org.qortal.repository.hsqldb.HSQLDBImportExport; -import org.qortal.repository.hsqldb.HSQLDBRepository; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; import org.qortal.utils.NTP; import org.qortal.utils.SevenZ; -import java.io.BufferedInputStream; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; +import java.nio.file.*; import java.util.List; import java.util.concurrent.locks.ReentrantLock; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + public class Bootstrap { + private Repository repository; + private static final Logger LOGGER = LogManager.getLogger(Bootstrap.class); /** The maximum number of untrimmed blocks allowed to be included in a bootstrap, beyond the trim threshold */ @@ -39,8 +37,8 @@ public class Bootstrap { private static final int MAXIMUM_UNPRUNED_BLOCKS = 100; - public Bootstrap() { - + public Bootstrap(Repository repository) { + this.repository = repository; } /** @@ -50,9 +48,8 @@ public class Bootstrap { * All failure reasons are logged */ public boolean canBootstrap() { - LOGGER.info("Checking repository state..."); - - try (final Repository repository = RepositoryManager.getRepository()) { + try { + LOGGER.info("Checking repository state..."); final boolean pruningEnabled = Settings.getInstance().isPruningEnabled(); final boolean archiveEnabled = Settings.getInstance().isArchiveEnabled(); @@ -203,73 +200,80 @@ public class Bootstrap { } public String create() throws DataException, InterruptedException, IOException { - try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) { - LOGGER.info("Acquiring blockchain lock..."); - ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); - blockchainLock.lockInterruptibly(); + LOGGER.info("Acquiring blockchain lock..."); + ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); + blockchainLock.lockInterruptibly(); - Path inputPath = null; + Path inputPath = null; + Path outputPath = null; + try { + + LOGGER.info("Exporting local data..."); + repository.exportNodeLocalData(); + + LOGGER.info("Deleting trade bot states..."); + List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); + for (TradeBotData tradeBotData : allTradeBotData) { + repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); + } + + LOGGER.info("Deleting minting accounts..."); + List mintingAccounts = repository.getAccountRepository().getMintingAccounts(); + for (MintingAccountData mintingAccount : mintingAccounts) { + repository.getAccountRepository().delete(mintingAccount.getPrivateKey()); + } + + repository.saveChanges(); + + LOGGER.info("Performing repository maintenance..."); + repository.performPeriodicMaintenance(); + + LOGGER.info("Creating bootstrap..."); + repository.backup(true, "bootstrap"); + + LOGGER.info("Moving files to output directory..."); + inputPath = Paths.get(Settings.getInstance().getRepositoryPath(), "bootstrap"); + outputPath = Paths.get(Files.createTempDirectory("qortal-bootstrap").toString(), "bootstrap"); + + + // Move the db backup to a "bootstrap" folder in the root directory + Files.move(inputPath, outputPath, REPLACE_EXISTING); + + // Copy the archive folder to inside the bootstrap folder + FileUtils.copyDirectory( + Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toFile(), + Paths.get(outputPath.toString(), "archive").toFile() + ); + + LOGGER.info("Compressing..."); + String compressedOutputPath = String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap.7z"); try { + Files.delete(Paths.get(compressedOutputPath)); + } catch (NoSuchFileException e) { + // Doesn't exist, so no need to delete + } + SevenZ.compress(compressedOutputPath, outputPath.toFile()); - LOGGER.info("Exporting local data..."); - repository.exportNodeLocalData(); + // Return the path to the compressed bootstrap file + Path finalPath = Paths.get(outputPath.toString(), compressedOutputPath); + return finalPath.toAbsolutePath().toString(); - LOGGER.info("Deleting trade bot states..."); - List allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData(); - for (TradeBotData tradeBotData : allTradeBotData) { - repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey()); - } + } finally { + LOGGER.info("Re-importing local data..."); + Path exportPath = HSQLDBImportExport.getExportDirectory(false); + repository.importDataFromFile(Paths.get(exportPath.toString(), "TradeBotStates.json").toString()); + repository.importDataFromFile(Paths.get(exportPath.toString(), "MintingAccounts.json").toString()); - LOGGER.info("Deleting minting accounts..."); - List mintingAccounts = repository.getAccountRepository().getMintingAccounts(); - for (MintingAccountData mintingAccount : mintingAccounts) { - repository.getAccountRepository().delete(mintingAccount.getPrivateKey()); - } + blockchainLock.unlock(); - repository.saveChanges(); - - LOGGER.info("Performing repository maintenance..."); - repository.performPeriodicMaintenance(); - - LOGGER.info("Creating bootstrap..."); - repository.backup(true, "bootstrap"); - - LOGGER.info("Moving files to output directory..."); - inputPath = Paths.get(Settings.getInstance().getRepositoryPath(), "bootstrap"); - Path outputPath = Paths.get("bootstrap"); + // Cleanup + if (inputPath != null) { + FileUtils.deleteDirectory(inputPath.toFile()); + } + if (outputPath != null) { FileUtils.deleteDirectory(outputPath.toFile()); - - // Move the db backup to a "bootstrap" folder in the root directory - Files.move(inputPath, outputPath); - - // Copy the archive folder to inside the bootstrap folder - FileUtils.copyDirectory( - Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toFile(), - Paths.get(outputPath.toString(), "archive").toFile() - ); - - LOGGER.info("Compressing..."); - String fileName = "bootstrap.7z"; - SevenZ.compress(fileName, outputPath.toFile()); - - // Return the path to the compressed bootstrap file - Path finalPath = Paths.get(outputPath.toString(), fileName); - return finalPath.toAbsolutePath().toString(); - - } finally { - LOGGER.info("Re-importing local data..."); - Path exportPath = HSQLDBImportExport.getExportDirectory(false); - repository.importDataFromFile(Paths.get(exportPath.toString(), "TradeBotStates.json").toString()); - repository.importDataFromFile(Paths.get(exportPath.toString(), "MintingAccounts.json").toString()); - - blockchainLock.unlock(); - - // Cleanup - if (inputPath != null) { - FileUtils.deleteDirectory(inputPath.toFile()); - } } } } @@ -305,7 +309,7 @@ public class Bootstrap { try { LOGGER.info("Downloading bootstrap..."); InputStream in = new URL(bootstrapUrl).openStream(); - Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); + Files.copy(in, path, REPLACE_EXISTING); break; } catch (IOException e) { @@ -324,7 +328,7 @@ public class Bootstrap { throw new DataException("Unable to download bootstrap"); } - private void importFromPath(Path path) throws InterruptedException, DataException, IOException { + public void importFromPath(Path path) throws InterruptedException, DataException, IOException { ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock(); blockchainLock.lockInterruptibly(); @@ -332,13 +336,18 @@ public class Bootstrap { try { LOGGER.info("Extracting bootstrap..."); Path input = path.toAbsolutePath(); - Path output = path.getParent().toAbsolutePath(); + Path output = path.toAbsolutePath().getParent().toAbsolutePath(); SevenZ.decompress(input.toString(), output.toFile()); LOGGER.info("Stopping repository..."); + // Close the repository while we are still able to + // Otherwise, the caller will run into difficulties when it tries to close it + repository.discardChanges(); + repository.close(); + // Now close the repository factory so that we can swap out the database files RepositoryManager.closeRepositoryFactory(); - Path inputPath = Paths.get("bootstrap"); + Path inputPath = Paths.get(output.toString(), "bootstrap"); Path outputPath = Paths.get(Settings.getInstance().getRepositoryPath()); if (!inputPath.toFile().exists()) { throw new DataException("Extracted bootstrap doesn't exist"); diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 8c69e0f2..6be8b9ea 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -267,7 +267,7 @@ public class HSQLDBRepository implements Repository { public void close() throws DataException { // Already closed? No need to do anything but maybe report double-call if (this.connection == null) { - LOGGER.warn("HSQLDBRepository.close() called when repository already closed", new Exception("Repository already closed")); + LOGGER.warn("HSQLDBRepository.close() called when repository already closed. This is expected when bootstrapping."); return; } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 8e1ed51b..7175c60a 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -191,6 +191,9 @@ public class Settings { // Export/import private String exportPath = "qortal-backup"; + // Bootstrap + private String bootstrapFilenamePrefix = ""; + // Auto-update sources private String[] autoUpdateRepos = new String[] { "https://github.com/Qortal/qortal/raw/%s/qortal.update", @@ -513,6 +516,10 @@ public class Settings { return this.exportPath; } + public String getBootstrapFilenamePrefix() { + return this.bootstrapFilenamePrefix; + } + public boolean isAutoUpdateEnabled() { return this.autoUpdateEnabled; } diff --git a/src/main/java/org/qortal/utils/SevenZ.java b/src/main/java/org/qortal/utils/SevenZ.java index 7af7ffc0..2c812e99 100644 --- a/src/main/java/org/qortal/utils/SevenZ.java +++ b/src/main/java/org/qortal/utils/SevenZ.java @@ -18,8 +18,8 @@ public class SevenZ { } - public static void compress(String name, File... files) throws IOException { - try (SevenZOutputFile out = new SevenZOutputFile(new File(name))){ + public static void compress(String outputPath, File... files) throws IOException { + try (SevenZOutputFile out = new SevenZOutputFile(new File(outputPath))){ for (File file : files){ addToArchiveCompression(out, file, "."); } diff --git a/src/test/java/org/qortal/test/BootstrapTests.java b/src/test/java/org/qortal/test/BootstrapTests.java new file mode 100644 index 00000000..6e76a386 --- /dev/null +++ b/src/test/java/org/qortal/test/BootstrapTests.java @@ -0,0 +1,204 @@ +package org.qortal.test; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.BlockMinter; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockData; +import org.qortal.repository.*; +import org.qortal.settings.Settings; +import org.qortal.test.common.AtUtils; +import org.qortal.test.common.Common; +import org.qortal.transform.TransformationException; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class BootstrapTests extends Common { + + @Before + public void beforeTest() throws DataException, IOException { + Common.useSettingsAndDb(Common.testSettingsFilename, false); + NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset()); + this.deleteBootstraps(); + } + + @After + public void afterTest() throws DataException, IOException { + this.deleteBootstraps(); + this.deleteExportDirectory(); + } + + @Test + public void testCanBootstrap() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + this.buildDummyBlockchain(repository); + + Bootstrap bootstrap = new Bootstrap(repository); + assertTrue(bootstrap.canBootstrap()); + + } + } + + @Test + public void testValidateBlockchain() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + this.buildDummyBlockchain(repository); + + Bootstrap bootstrap = new Bootstrap(repository); + assertTrue(bootstrap.validateBlockchain()); + + } + } + + + @Test + public void testCreateAndImportBootstrap() throws DataException, InterruptedException, TransformationException, IOException { + + Path bootstrapPath = Paths.get(String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap.7z")); + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive", "2-900.dat"); + BlockData block1000; + byte[] originalArchiveContents; + + try (final Repository repository = RepositoryManager.getRepository()) { + this.buildDummyBlockchain(repository); + + // Ensure the compressed bootstrap doesn't exist + assertFalse(Files.exists(bootstrapPath)); + + Bootstrap bootstrap = new Bootstrap(repository); + bootstrap.create(); + + // Ensure the compressed bootstrap exists + assertTrue(Files.exists(bootstrapPath)); + + // Ensure the original block archive file exists + assertTrue(Files.exists(archivePath)); + originalArchiveContents = Files.readAllBytes(archivePath); + + // Ensure block 1000 exists in the repository + block1000 = repository.getBlockRepository().fromHeight(1000); + assertNotNull(block1000); + + // Ensure we can retrieve block 10 from the archive + assertNotNull(repository.getBlockArchiveRepository().fromHeight(10)); + + // Now delete block 1000 + repository.getBlockRepository().delete(block1000); + assertNull(repository.getBlockRepository().fromHeight(1000)); + + // Overwrite the archive with dummy data, and verify it + try (PrintWriter out = new PrintWriter(archivePath.toFile())) { + out.println("testdata"); + } + String newline = System.getProperty("line.separator"); + assertEquals("testdata", Files.readString(archivePath).replace(newline, "")); + + // Ensure we can no longer retrieve block 10 from the archive + assertNull(repository.getBlockArchiveRepository().fromHeight(10)); + + // Import the bootstrap back in + bootstrap.importFromPath(bootstrapPath); + } + + // We need a new connection because we have switched to a new repository + try (final Repository repository = RepositoryManager.getRepository()) { + + // Ensure the block archive file exists + assertTrue(Files.exists(archivePath)); + + // and that its contents match the original + assertArrayEquals(originalArchiveContents, Files.readAllBytes(archivePath)); + + // Make sure that block 1000 exists again + BlockData newBlock1000 = repository.getBlockRepository().fromHeight(1000); + assertNotNull(newBlock1000); + + // and ensure that the signatures match + assertArrayEquals(block1000.getSignature(), newBlock1000.getSignature()); + + // Ensure we can retrieve block 10 from the archive + assertNotNull(repository.getBlockArchiveRepository().fromHeight(10)); + } + } + + + private void buildDummyBlockchain(Repository repository) throws DataException, InterruptedException, TransformationException, IOException { + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Assume 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(901); + + // Prune all the archived blocks + repository.getBlockRepository().pruneBlocks(0, 900); + repository.getBlockRepository().setBlockPruneHeight(901); + + // Prune the AT states for the archived blocks + repository.getATRepository().rebuildLatestAtStates(); + repository.getATRepository().pruneAtStates(0, 900); + repository.getATRepository().setAtPruneHeight(901); + + // Refill cache, used by Controller.getMinimumLatestBlockTimestamp() and other methods + Controller.getInstance().refillLatestBlocksCache(); + + repository.saveChanges(); + } + + private void deleteBootstraps() throws IOException { + try { + Path path = Paths.get(String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap.7z")); + Files.delete(path); + + } catch (NoSuchFileException e) { + // Nothing to delete + } + } + + private void deleteExportDirectory() { + // Delete archive directory if exists + Path archivePath = Paths.get(Settings.getInstance().getExportPath()); + try { + FileUtils.deleteDirectory(archivePath.toFile()); + } catch (IOException e) { + + } + } + +} diff --git a/src/test/java/org/qortal/test/common/Common.java b/src/test/java/org/qortal/test/common/Common.java index 24c86690..c45fcfd7 100644 --- a/src/test/java/org/qortal/test/common/Common.java +++ b/src/test/java/org/qortal/test/common/Common.java @@ -2,8 +2,11 @@ package org.qortal.test.common; import static org.junit.Assert.*; +import java.io.IOException; import java.math.BigDecimal; import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.security.Security; import java.util.ArrayList; import java.util.Collections; @@ -15,6 +18,7 @@ import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; +import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bouncycastle.jce.provider.BouncyCastleProvider; @@ -46,9 +50,15 @@ public class Common { private static final Logger LOGGER = LogManager.getLogger(Common.class); - public static final String testConnectionUrl = "jdbc:hsqldb:mem:testdb"; - // For debugging, use this instead to write DB to disk for examination: - // public static final String testConnectionUrl = "jdbc:hsqldb:file:testdb/blockchain;create=true"; + public static final String testConnectionUrlMemory = "jdbc:hsqldb:mem:testdb"; + public static final String testConnectionUrlDisk = "jdbc:hsqldb:file:%s/blockchain;create=true"; + + // For debugging, use testConnectionUrlDisk instead of memory, to write DB to disk for examination. + // This can be achieved using `Common.useSettingsAndDb(Common.testSettingsFilename, false);` + // where `false` specifies to use a repository on disk rather than one in memory. + // Make sure to also comment out `Common.deleteTestRepository();` in closeRepository() below, so that + // the files remain after the test finishes. + public static final String testSettingsFilename = "test-settings-v2.json"; @@ -100,7 +110,7 @@ public class Common { return testAccountsByName.values().stream().map(account -> new TestAccount(repository, account)).collect(Collectors.toList()); } - public static void useSettings(String settingsFilename) throws DataException { + public static void useSettingsAndDb(String settingsFilename, boolean dbInMemory) throws DataException { closeRepository(); // Load/check settings, which potentially sets up blockchain config, etc. @@ -109,11 +119,15 @@ public class Common { assertNotNull("Test settings JSON file not found", testSettingsUrl); Settings.fileInstance(testSettingsUrl.getPath()); - setRepository(); + setRepository(dbInMemory); resetBlockchain(); } + public static void useSettings(String settingsFilename) throws DataException { + Common.useSettingsAndDb(settingsFilename, true); + } + public static void useDefaultSettings() throws DataException { useSettings(testSettingsFilename); NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset()); @@ -186,15 +200,33 @@ public class Common { assertTrue(String.format("Non-genesis %s remains", typeName), remainingClone.isEmpty()); } - @BeforeClass - public static void setRepository() throws DataException { - RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(testConnectionUrl); + public static void setRepository(boolean inMemory) throws DataException { + String connectionUrlDisk = String.format(testConnectionUrlDisk, Settings.getInstance().getRepositoryPath()); + String connectionUrl = inMemory ? testConnectionUrlMemory : connectionUrlDisk; + RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(connectionUrl); RepositoryManager.setRepositoryFactory(repositoryFactory); } + public static void deleteTestRepository() throws DataException { + // Delete repository directory if exists + Path repositoryPath = Paths.get(Settings.getInstance().getRepositoryPath()); + try { + FileUtils.deleteDirectory(repositoryPath.toFile()); + } catch (IOException e) { + throw new DataException(String.format("Unable to delete test repository: %s", e.getMessage())); + } + } + + @BeforeClass + public static void setRepositoryInMemory() throws DataException { + Common.deleteTestRepository(); + Common.setRepository(true); + } + @AfterClass public static void closeRepository() throws DataException { RepositoryManager.closeRepositoryFactory(); + Common.deleteTestRepository(); // Comment out this line in you need to inspect the database after running a test } // Test assertions diff --git a/src/test/resources/test-settings-v2-bitcoin-regtest.json b/src/test/resources/test-settings-v2-bitcoin-regtest.json index 7f03b447..687c240d 100644 --- a/src/test/resources/test-settings-v2-bitcoin-regtest.json +++ b/src/test/resources/test-settings-v2-bitcoin-regtest.json @@ -1,4 +1,5 @@ { + "repositoryPath": "testdb", "bitcoinNet": "REGTEST", "litecoinNet": "REGTEST", "restrictedApi": false, @@ -7,5 +8,6 @@ "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-block-archive.json b/src/test/resources/test-settings-v2-block-archive.json index 7cac32b6..c5ed1aa8 100644 --- a/src/test/resources/test-settings-v2-block-archive.json +++ b/src/test/resources/test-settings-v2-block-archive.json @@ -8,6 +8,6 @@ "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, "minPeers": 0, - "pruneBlockLimit": 1450, + "pruneBlockLimit": 100, "repositoryPath": "dbtest" } diff --git a/src/test/resources/test-settings-v2-founder-rewards.json b/src/test/resources/test-settings-v2-founder-rewards.json index fedd5de4..02d71d76 100644 --- a/src/test/resources/test-settings-v2-founder-rewards.json +++ b/src/test/resources/test-settings-v2-founder-rewards.json @@ -1,9 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-founder-rewards.json", "exportPath": "qortal-backup-test", "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-leftover-reward.json b/src/test/resources/test-settings-v2-leftover-reward.json index 45f86ff3..185bbeba 100644 --- a/src/test/resources/test-settings-v2-leftover-reward.json +++ b/src/test/resources/test-settings-v2-leftover-reward.json @@ -1,9 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-leftover-reward.json", "exportPath": "qortal-backup-test", "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-minting.json b/src/test/resources/test-settings-v2-minting.json index c2522774..b5645812 100644 --- a/src/test/resources/test-settings-v2-minting.json +++ b/src/test/resources/test-settings-v2-minting.json @@ -1,9 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-minting.json", "exportPath": "qortal-backup-test", "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-qora-holder-extremes.json b/src/test/resources/test-settings-v2-qora-holder-extremes.json index a4422562..e20fddf0 100644 --- a/src/test/resources/test-settings-v2-qora-holder-extremes.json +++ b/src/test/resources/test-settings-v2-qora-holder-extremes.json @@ -1,9 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder-extremes.json", "exportPath": "qortal-backup-test", "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-qora-holder.json b/src/test/resources/test-settings-v2-qora-holder.json index f8777ca1..9d7d2567 100644 --- a/src/test/resources/test-settings-v2-qora-holder.json +++ b/src/test/resources/test-settings-v2-qora-holder.json @@ -1,9 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder.json", "exportPath": "qortal-backup-test", "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-reward-levels.json b/src/test/resources/test-settings-v2-reward-levels.json index 02a91d28..3ee0179d 100644 --- a/src/test/resources/test-settings-v2-reward-levels.json +++ b/src/test/resources/test-settings-v2-reward-levels.json @@ -1,9 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-reward-levels.json", "exportPath": "qortal-backup-test", "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2-reward-scaling.json b/src/test/resources/test-settings-v2-reward-scaling.json index 87f77d44..fa02ebe7 100644 --- a/src/test/resources/test-settings-v2-reward-scaling.json +++ b/src/test/resources/test-settings-v2-reward-scaling.json @@ -1,9 +1,11 @@ { + "repositoryPath": "testdb", "restrictedApi": false, "blockchainConfig": "src/test/resources/test-chain-v2-reward-scaling.json", "exportPath": "qortal-backup-test", "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100 } diff --git a/src/test/resources/test-settings-v2.json b/src/test/resources/test-settings-v2.json index 4dfaeac1..83bdf197 100644 --- a/src/test/resources/test-settings-v2.json +++ b/src/test/resources/test-settings-v2.json @@ -1,4 +1,5 @@ { + "repositoryPath": "testdb", "bitcoinNet": "TEST3", "litecoinNet": "TEST3", "restrictedApi": false, @@ -7,5 +8,7 @@ "bootstrap": false, "wipeUnconfirmedOnStart": false, "testNtpOffset": 0, - "minPeers": 0 + "minPeers": 0, + "pruneBlockLimit": 100, + "bootstrapFilenamePrefix": "test-" }