From 74ea2a847dd2ce543d9d28329e2ca7b2d7f0a1dc Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 12 Sep 2021 10:13:52 +0100 Subject: [PATCH] Added unit tests for trimming, pruning, and archiving. --- .../org/qortal/controller/BlockMinter.java | 3 +- .../qortal/repository/BlockArchiveWriter.java | 25 +- .../org/qortal/test/BlockArchiveTests.java | 500 ++++++++++++++++++ src/test/java/org/qortal/test/PruneTests.java | 91 ++++ .../org/qortal/test/at/AtRepositoryTests.java | 158 +++--- .../java/org/qortal/test/common/AtUtils.java | 81 +++ .../test-settings-v2-block-archive.json | 11 + 7 files changed, 781 insertions(+), 88 deletions(-) create mode 100644 src/test/java/org/qortal/test/BlockArchiveTests.java create mode 100644 src/test/java/org/qortal/test/PruneTests.java create mode 100644 src/test/java/org/qortal/test/common/AtUtils.java create mode 100644 src/test/resources/test-settings-v2-block-archive.json diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index 0cf33f43..33431258 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -442,7 +442,8 @@ public class BlockMinter extends Thread { // Add to blockchain newBlock.process(); - LOGGER.info(String.format("Minted new test block: %d", newBlock.getBlockData().getHeight())); + LOGGER.info(String.format("Minted new test block: %d sig: %.8s", + newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature()))); repository.saveChanges(); diff --git a/src/main/java/org/qortal/repository/BlockArchiveWriter.java b/src/main/java/org/qortal/repository/BlockArchiveWriter.java index 11151e17..77c98d96 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveWriter.java +++ b/src/main/java/org/qortal/repository/BlockArchiveWriter.java @@ -33,7 +33,11 @@ public class BlockArchiveWriter { private final int endHeight; private final Repository repository; + private long fileSizeTarget = 100 * 1024 * 1024; // 100MiB + private boolean shouldEnforceFileSizeTarget = true; + private int writtenCount; + private Path outputPath; public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) { this.startHeight = startHeight; @@ -87,8 +91,9 @@ public class BlockArchiveWriter { LOGGER.info(String.format("Fetching blocks from height %d...", startHeight)); int i = 0; - long fileSizeTarget = 100 * 1024 * 1024; // 100MiB - while (headerBytes.size() + bytes.size() < fileSizeTarget) { + while (headerBytes.size() + bytes.size() < this.fileSizeTarget + || this.shouldEnforceFileSizeTarget == false) { + if (Controller.isStopping()) { return BlockArchiveWriteResult.STOPPING; } @@ -132,7 +137,7 @@ public class BlockArchiveWriter { LOGGER.info(String.format("Total length of %d blocks is %d bytes", i, totalLength)); // Validate file size, in case something went wrong - if (totalLength < fileSizeTarget) { + if (totalLength < fileSizeTarget && this.shouldEnforceFileSizeTarget) { return BlockArchiveWriteResult.NOT_ENOUGH_BLOCKS; } @@ -164,6 +169,7 @@ public class BlockArchiveWriter { BlockArchiveReader.getInstance().invalidateFileListCache(); this.writtenCount = i; + this.outputPath = Paths.get(filePath); return BlockArchiveWriteResult.OK; } @@ -171,4 +177,17 @@ public class BlockArchiveWriter { return this.writtenCount; } + public Path getOutputPath() { + return this.outputPath; + } + + public void setFileSizeTarget(long fileSizeTarget) { + this.fileSizeTarget = fileSizeTarget; + } + + // For testing, to avoid having to pre-calculate file sizes + public void setShouldEnforceFileSizeTarget(boolean shouldEnforceFileSizeTarget) { + this.shouldEnforceFileSizeTarget = shouldEnforceFileSizeTarget; + } + } diff --git a/src/test/java/org/qortal/test/BlockArchiveTests.java b/src/test/java/org/qortal/test/BlockArchiveTests.java new file mode 100644 index 00000000..c05915cd --- /dev/null +++ b/src/test/java/org/qortal/test/BlockArchiveTests.java @@ -0,0 +1,500 @@ +package org.qortal.test; + +import org.apache.commons.io.FileUtils; +import org.ciyam.at.CompilationException; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.controller.BlockMinter; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.*; +import org.qortal.settings.Settings; +import org.qortal.test.common.AtUtils; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.transform.TransformationException; +import org.qortal.utils.BlockArchiveUtils; +import org.qortal.utils.Triple; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class BlockArchiveTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); // Necessary to set NTP offset + Common.useSettings("test-settings-v2-block-archive.json"); + this.deleteArchiveDirectory(); + } + + @After + public void afterTest() throws DataException { + this.deleteArchiveDirectory(); + } + + + @Test + public void testWriter() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(900 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + } + } + + @Test + public void testWriterAndReader() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(900 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Read block 2 from the archive + BlockArchiveReader reader = BlockArchiveReader.getInstance(); + Triple, List> block2Info = reader.fetchBlockAtHeight(2); + BlockData block2ArchiveData = block2Info.getA(); + + // Read block 2 from the repository + BlockData block2RepositoryData = repository.getBlockRepository().fromHeight(2); + + // Ensure the values match + assertEquals(block2ArchiveData.getHeight(), block2RepositoryData.getHeight()); + assertArrayEquals(block2ArchiveData.getSignature(), block2RepositoryData.getSignature()); + + // Test some values in the archive + assertEquals(1, block2ArchiveData.getOnlineAccountsCount()); + + // Read block 900 from the archive + Triple, List> block900Info = reader.fetchBlockAtHeight(900); + BlockData block900ArchiveData = block900Info.getA(); + + // Read block 900 from the repository + BlockData block900RepositoryData = repository.getBlockRepository().fromHeight(900); + + // Ensure the values match + assertEquals(block900ArchiveData.getHeight(), block900RepositoryData.getHeight()); + assertArrayEquals(block900ArchiveData.getSignature(), block900RepositoryData.getSignature()); + + // Test some values in the archive + assertEquals(1, block900ArchiveData.getOnlineAccountsCount()); + + } + } + + @Test + public void testArchivedAtStates() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 10; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // 9 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(10); + repository.getATRepository().setAtTrimHeight(10); + + // Check the max archive height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(9, maximumArchiveHeight); + + // Write blocks 2-9 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(9 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(9 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Check blocks 3-9 + for (Integer testHeight = 2; testHeight <= 9; testHeight++) { + + // Read a block from the archive + BlockArchiveReader reader = BlockArchiveReader.getInstance(); + Triple, List> blockInfo = reader.fetchBlockAtHeight(testHeight); + BlockData archivedBlockData = blockInfo.getA(); + ATStateData archivedAtStateData = blockInfo.getC().isEmpty() ? null : blockInfo.getC().get(0); + List archivedTransactions = blockInfo.getB(); + + // Read the same block from the repository + BlockData repositoryBlockData = repository.getBlockRepository().fromHeight(testHeight); + ATStateData repositoryAtStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight); + + // Ensure the repository has full AT state data + assertNotNull(repositoryAtStateData.getStateHash()); + assertNotNull(repositoryAtStateData.getStateData()); + + // Check the archived AT state + if (testHeight == 2) { + // Block 2 won't have an AT state hash because it's initial (and has the DEPLOY_AT in the same block) + assertNull(archivedAtStateData); + + assertEquals(1, archivedTransactions.size()); + assertEquals(Transaction.TransactionType.DEPLOY_AT, archivedTransactions.get(0).getType()); + } + else { + // For blocks 3+, ensure the archive has the AT state data, but not the hashes + assertNotNull(archivedAtStateData.getStateHash()); + assertNull(archivedAtStateData.getStateData()); + + // They also shouldn't have any transactions + assertTrue(archivedTransactions.isEmpty()); + } + + // Also check the online accounts count and height + assertEquals(1, archivedBlockData.getOnlineAccountsCount()); + assertEquals(testHeight, archivedBlockData.getHeight()); + + // Ensure the values match + assertEquals(archivedBlockData.getHeight(), repositoryBlockData.getHeight()); + assertArrayEquals(archivedBlockData.getSignature(), repositoryBlockData.getSignature()); + assertEquals(archivedBlockData.getOnlineAccountsCount(), repositoryBlockData.getOnlineAccountsCount()); + assertArrayEquals(archivedBlockData.getMinterSignature(), repositoryBlockData.getMinterSignature()); + assertEquals(archivedBlockData.getATCount(), repositoryBlockData.getATCount()); + assertEquals(archivedBlockData.getOnlineAccountsCount(), repositoryBlockData.getOnlineAccountsCount()); + assertArrayEquals(archivedBlockData.getReference(), repositoryBlockData.getReference()); + assertEquals(archivedBlockData.getTimestamp(), repositoryBlockData.getTimestamp()); + assertEquals(archivedBlockData.getATFees(), repositoryBlockData.getATFees()); + assertEquals(archivedBlockData.getTotalFees(), repositoryBlockData.getTotalFees()); + assertEquals(archivedBlockData.getTransactionCount(), repositoryBlockData.getTransactionCount()); + assertArrayEquals(archivedBlockData.getTransactionsSignature(), repositoryBlockData.getTransactionsSignature()); + + if (testHeight != 2) { + assertArrayEquals(archivedAtStateData.getStateHash(), repositoryAtStateData.getStateHash()); + } + } + + // Check block 10 (unarchived) + BlockArchiveReader reader = BlockArchiveReader.getInstance(); + Triple, List> blockInfo = reader.fetchBlockAtHeight(10); + assertNull(blockInfo); + + } + + } + + @Test + public void testArchiveAndPrune() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Assume 900 blocks are trimmed (this specifies the first untrimmed height) + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901); + repository.getATRepository().setAtTrimHeight(901); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(900, maximumArchiveHeight); + + // Write blocks 2-900 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(900 - 1, writer.getWrittenCount()); + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Ensure the SQL repository contains blocks 2 and 900... + assertNotNull(repository.getBlockRepository().fromHeight(2)); + assertNotNull(repository.getBlockRepository().fromHeight(900)); + + // Prune all the archived blocks + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 900); + assertEquals(900-1, numBlocksPruned); + repository.getBlockRepository().setBlockPruneHeight(901); + + // Prune the AT states for the archived blocks + repository.getATRepository().rebuildLatestAtStates(); + int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 900); + assertEquals(900-1, numATStatesPruned); + repository.getATRepository().setAtPruneHeight(901); + + // Now ensure the SQL repository is missing blocks 2 and 900... + assertNull(repository.getBlockRepository().fromHeight(2)); + assertNull(repository.getBlockRepository().fromHeight(900)); + + // ... but it's not missing blocks 1 and 901 (we don't prune the genesis block) + assertNotNull(repository.getBlockRepository().fromHeight(1)); + assertNotNull(repository.getBlockRepository().fromHeight(901)); + + // Validate the latest block height in the repository + assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + } + } + + @Test + public void testTrimArchivePruneAndOrphan() throws DataException, InterruptedException, TransformationException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks so that we are able to archive them later + for (int i = 0; i < 1000; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Make sure that block 500 has full AT state data and data hash + List block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500); + ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + + // Trim the first 500 blocks + repository.getBlockRepository().trimOldOnlineAccountsSignatures(0, 500); + repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(501); + repository.getATRepository().trimAtStates(0, 500, 1000); + repository.getATRepository().setAtTrimHeight(501); + + // Now block 500 should only have the AT state data hash + block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500); + atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500); + assertNotNull(atStatesData.getStateHash()); + assertNull(atStatesData.getStateData()); + + // ... but block 501 should have the full data + List block501AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(501); + atStatesData = repository.getATRepository().getATStateAtHeight(block501AtStatesData.get(0).getATAddress(), 501); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + + // Check the max archive height - this should be one less than the first untrimmed height + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository); + assertEquals(500, maximumArchiveHeight); + + BlockData block3DataPreArchive = repository.getBlockRepository().fromHeight(3); + + // Write blocks 2-500 to the archive + BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository); + writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result); + + // Make sure that the archive contains the correct number of blocks + assertEquals(500 - 1, writer.getWrittenCount()); // -1 for the genesis block + + // Increment block archive height + repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount()); + repository.saveChanges(); + assertEquals(500 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight()); + + // Ensure the file exists + File outputFile = writer.getOutputPath().toFile(); + assertTrue(outputFile.exists()); + + // Ensure the SQL repository contains blocks 2 and 500... + assertNotNull(repository.getBlockRepository().fromHeight(2)); + assertNotNull(repository.getBlockRepository().fromHeight(500)); + + // Prune all the archived blocks + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 500); + assertEquals(500-1, numBlocksPruned); + repository.getBlockRepository().setBlockPruneHeight(501); + + // Prune the AT states for the archived blocks + repository.getATRepository().rebuildLatestAtStates(); + int numATStatesPruned = repository.getATRepository().pruneAtStates(2, 500); + assertEquals(499, numATStatesPruned); + repository.getATRepository().setAtPruneHeight(501); + + // Now ensure the SQL repository is missing blocks 2 and 500... + assertNull(repository.getBlockRepository().fromHeight(2)); + assertNull(repository.getBlockRepository().fromHeight(500)); + + // ... but it's not missing blocks 1 and 501 (we don't prune the genesis block) + assertNotNull(repository.getBlockRepository().fromHeight(1)); + assertNotNull(repository.getBlockRepository().fromHeight(501)); + + // Validate the latest block height in the repository + assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // Now orphan some unarchived blocks. + BlockUtils.orphanBlocks(repository, 500); + assertEquals(502, (int) repository.getBlockRepository().getLastBlock().getHeight()); + + // We're close to the lower limit of the SQL database now, so + // we need to import some blocks from the archive + BlockArchiveUtils.importFromArchive(401, 500, repository); + + // Ensure the SQL repository now contains block 401 but not 400... + assertNotNull(repository.getBlockRepository().fromHeight(401)); + assertNull(repository.getBlockRepository().fromHeight(400)); + + // Import the remaining 399 blocks + BlockArchiveUtils.importFromArchive(2, 400, repository); + + // Verify that block 3 matches the original + BlockData block3DataPostArchive = repository.getBlockRepository().fromHeight(3); + assertArrayEquals(block3DataPreArchive.getSignature(), block3DataPostArchive.getSignature()); + assertEquals(block3DataPreArchive.getHeight(), block3DataPostArchive.getHeight()); + + // Orphan 1 more block, which should be the last one that is possible to be orphaned + BlockUtils.orphanBlocks(repository, 1); + + // Orphan another block, which should fail + Exception exception = null; + try { + BlockUtils.orphanBlocks(repository, 1); + } catch (DataException e) { + exception = e; + } + + // Ensure that a DataException is thrown because there is no more AT states data available + assertNotNull(exception); + assertEquals(DataException.class, exception.getClass()); + + // FUTURE: we may be able to retain unique AT states when trimming, to avoid this exception + // and allow orphaning back through blocks with trimmed AT states. + + } + } + + + private void deleteArchiveDirectory() { + // Delete archive directory if exists + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath(); + try { + FileUtils.deleteDirectory(archivePath.toFile()); + } catch (IOException e) { + + } + } + +} diff --git a/src/test/java/org/qortal/test/PruneTests.java b/src/test/java/org/qortal/test/PruneTests.java new file mode 100644 index 00000000..0914d794 --- /dev/null +++ b/src/test/java/org/qortal/test/PruneTests.java @@ -0,0 +1,91 @@ +package org.qortal.test; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.controller.BlockMinter; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.AtUtils; +import org.qortal.test.common.Common; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.*; + +public class PruneTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testPruning() throws DataException { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Alice self share online + List mintingAndOnlineAccounts = new ArrayList<>(); + PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share"); + mintingAndOnlineAccounts.add(aliceSelfShare); + + // Deploy an AT so that we have AT state data + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + byte[] creationBytes = AtUtils.buildSimpleAT(); + long fundingAmount = 1_00000000L; + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + + // Mint some blocks + for (int i = 2; i <= 10; i++) + BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0])); + + // Make sure that all blocks have full AT state data and data hash + for (Integer i=2; i <= 10; i++) { + BlockData blockData = repository.getBlockRepository().fromHeight(i); + assertNotNull(blockData.getSignature()); + assertEquals(i, blockData.getHeight()); + List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStatesDataList); + assertFalse(atStatesDataList.isEmpty()); + ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + } + + // Prune blocks 2-5 + int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 5); + assertEquals(4, numBlocksPruned); + repository.getBlockRepository().setBlockPruneHeight(6); + + // Prune AT states for blocks 2-5 + int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 5); + assertEquals(4, numATStatesPruned); + repository.getATRepository().setAtPruneHeight(6); + + // Make sure that blocks 2-5 are now missing block data and AT states data + for (Integer i=2; i <= 5; i++) { + BlockData blockData = repository.getBlockRepository().fromHeight(i); + assertNull(blockData); + List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); + assertTrue(atStatesDataList.isEmpty()); + } + + // ... but blocks 6-10 have block data and full AT states data + for (Integer i=6; i <= 10; i++) { + BlockData blockData = repository.getBlockRepository().fromHeight(i); + assertNotNull(blockData.getSignature()); + List atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i); + assertNotNull(atStatesDataList); + assertFalse(atStatesDataList.isEmpty()); + ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i); + assertNotNull(atStatesData.getStateHash()); + assertNotNull(atStatesData.getStateData()); + } + } + } + +} diff --git a/src/test/java/org/qortal/test/at/AtRepositoryTests.java b/src/test/java/org/qortal/test/at/AtRepositoryTests.java index 0b302435..8ef4c774 100644 --- a/src/test/java/org/qortal/test/at/AtRepositoryTests.java +++ b/src/test/java/org/qortal/test/at/AtRepositoryTests.java @@ -21,6 +21,7 @@ import org.qortal.group.Group; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.AtUtils; import org.qortal.test.common.BlockUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; @@ -35,13 +36,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetATStateAtHeightWithData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -58,13 +59,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetATStateAtHeightWithoutData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -87,13 +88,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetLatestATStateWithData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -111,13 +112,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetLatestATStatePostTrimming() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -144,14 +145,66 @@ public class AtRepositoryTests extends Common { } @Test - public void testGetMatchingFinalATStatesWithoutDataValue() throws DataException { - byte[] creationBytes = buildSimpleAT(); + public void testOrphanTrimmedATStates() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + // Mint a few blocks + for (int i = 0; i < 10; ++i) + BlockUtils.mintBlock(repository); + + int blockchainHeight = repository.getBlockRepository().getBlockchainHeight(); + int maxTrimHeight = blockchainHeight - 4; + Integer testHeight = maxTrimHeight + 1; + + // Trim AT state data + repository.getATRepository().rebuildLatestAtStates(); + repository.saveChanges(); + repository.getATRepository().trimAtStates(2, maxTrimHeight, 1000); + + // Orphan 3 blocks + // This leaves one more untrimmed block, so the latest AT state should be available + BlockUtils.orphanBlocks(repository, 3); + + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + assertEquals(testHeight, atStateData.getHeight()); + + // We should always have the latest AT state data available + assertNotNull(atStateData.getStateData()); + + // Orphan 1 more block + Exception exception = null; + try { + BlockUtils.orphanBlocks(repository, 1); + } catch (DataException e) { + exception = e; + } + + // Ensure that a DataException is thrown because there is no more AT states data available + assertNotNull(exception); + assertEquals(DataException.class, exception.getClass()); + assertEquals(String.format("Can't find previous AT state data for %s", atAddress), exception.getMessage()); + + // FUTURE: we may be able to retain unique AT states when trimming, to avoid this exception + // and allow orphaning back through blocks with trimmed AT states. + } + } + + @Test + public void testGetMatchingFinalATStatesWithoutDataValue() throws DataException { + byte[] creationBytes = AtUtils.buildSimpleAT(); + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -191,13 +244,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetMatchingFinalATStatesWithDataValue() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -237,13 +290,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetBlockATStatesAtHeightWithData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - doDeploy(repository, deployer, creationBytes, fundingAmount); + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); // Mint a few blocks for (int i = 0; i < 10; ++i) @@ -264,13 +317,13 @@ public class AtRepositoryTests extends Common { @Test public void testGetBlockATStatesAtHeightWithoutData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - doDeploy(repository, deployer, creationBytes, fundingAmount); + AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); // Mint a few blocks for (int i = 0; i < 10; ++i) @@ -297,13 +350,13 @@ public class AtRepositoryTests extends Common { @Test public void testSaveATStateWithData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -328,13 +381,13 @@ public class AtRepositoryTests extends Common { @Test public void testSaveATStateWithoutData() throws DataException { - byte[] creationBytes = buildSimpleAT(); + byte[] creationBytes = AtUtils.buildSimpleAT(); try (final Repository repository = RepositoryManager.getRepository()) { PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); long fundingAmount = 1_00000000L; - DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount); String atAddress = deployAtTransaction.getATAccount().getAddress(); // Mint a few blocks @@ -364,67 +417,4 @@ public class AtRepositoryTests extends Common { assertNull(atStateData.getStateData()); } } - - private byte[] buildSimpleAT() { - // Pretend we use 4 values in data segment - int addrCounter = 4; - - // Data segment - ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); - - ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); - - // Two-pass version - for (int pass = 0; pass < 2; ++pass) { - codeByteBuffer.clear(); - - try { - // Stop and wait for next block - codeByteBuffer.put(OpCode.STP_IMD.compile()); - } catch (CompilationException e) { - throw new IllegalStateException("Unable to compile AT?", e); - } - } - - codeByteBuffer.flip(); - - byte[] codeBytes = new byte[codeByteBuffer.limit()]; - codeByteBuffer.get(codeBytes); - - final short ciyamAtVersion = 2; - final short numCallStackPages = 0; - final short numUserStackPages = 0; - final long minActivationAmount = 0L; - - return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); - } - - private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { - long txTimestamp = System.currentTimeMillis(); - byte[] lastReference = deployer.getLastReference(); - - if (lastReference == null) { - System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); - System.exit(2); - } - - Long fee = null; - String name = "Test AT"; - String description = "Test AT"; - String atType = "Test"; - String tags = "TEST"; - - BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); - TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); - - DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); - - fee = deployAtTransaction.calcRecommendedFee(); - deployAtTransactionData.setFee(fee); - - TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); - - return deployAtTransaction; - } - } diff --git a/src/test/java/org/qortal/test/common/AtUtils.java b/src/test/java/org/qortal/test/common/AtUtils.java new file mode 100644 index 00000000..3bc2b235 --- /dev/null +++ b/src/test/java/org/qortal/test/common/AtUtils.java @@ -0,0 +1,81 @@ +package org.qortal.test.common; + +import org.ciyam.at.CompilationException; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.transaction.DeployAtTransaction; + +import java.nio.ByteBuffer; + +public class AtUtils { + + public static byte[] buildSimpleAT() { + // Pretend we use 4 values in data segment + int addrCounter = 4; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile AT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + public static DeployAtTransaction doDeployAT(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "Test AT"; + String description = "Test AT"; + String atType = "Test"; + String tags = "TEST"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } +} diff --git a/src/test/resources/test-settings-v2-block-archive.json b/src/test/resources/test-settings-v2-block-archive.json new file mode 100644 index 00000000..612c8658 --- /dev/null +++ b/src/test/resources/test-settings-v2-block-archive.json @@ -0,0 +1,11 @@ +{ + "bitcoinNet": "TEST3", + "litecoinNet": "TEST3", + "restrictedApi": false, + "blockchainConfig": "src/test/resources/test-chain-v2.json", + "wipeUnconfirmedOnStart": false, + "testNtpOffset": 0, + "minPeers": 0, + "pruneBlockLimit": 1450, + "repositoryPath": "dbtest" +}