Added unit tests for trimming, pruning, and archiving.

This commit is contained in:
CalDescent 2021-09-12 10:13:52 +01:00
parent 1d8351f921
commit 14f6fd19ef
7 changed files with 781 additions and 88 deletions

View File

@ -421,7 +421,8 @@ public class BlockMinter extends Thread {
// Add to blockchain
newBlock.process();
LOGGER.info(String.format("Minted new test block: %d", newBlock.getBlockData().getHeight()));
LOGGER.info(String.format("Minted new test block: %d sig: %.8s",
newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getBlockData().getSignature())));
repository.saveChanges();

View File

@ -33,7 +33,11 @@ public class BlockArchiveWriter {
private final int endHeight;
private final Repository repository;
private long fileSizeTarget = 100 * 1024 * 1024; // 100MiB
private boolean shouldEnforceFileSizeTarget = true;
private int writtenCount;
private Path outputPath;
public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) {
this.startHeight = startHeight;
@ -87,8 +91,9 @@ public class BlockArchiveWriter {
LOGGER.info(String.format("Fetching blocks from height %d...", startHeight));
int i = 0;
long fileSizeTarget = 100 * 1024 * 1024; // 100MiB
while (headerBytes.size() + bytes.size() < fileSizeTarget) {
while (headerBytes.size() + bytes.size() < this.fileSizeTarget
|| this.shouldEnforceFileSizeTarget == false) {
if (Controller.isStopping()) {
return BlockArchiveWriteResult.STOPPING;
}
@ -132,7 +137,7 @@ public class BlockArchiveWriter {
LOGGER.info(String.format("Total length of %d blocks is %d bytes", i, totalLength));
// Validate file size, in case something went wrong
if (totalLength < fileSizeTarget) {
if (totalLength < fileSizeTarget && this.shouldEnforceFileSizeTarget) {
return BlockArchiveWriteResult.NOT_ENOUGH_BLOCKS;
}
@ -164,6 +169,7 @@ public class BlockArchiveWriter {
BlockArchiveReader.getInstance().invalidateFileListCache();
this.writtenCount = i;
this.outputPath = Paths.get(filePath);
return BlockArchiveWriteResult.OK;
}
@ -171,4 +177,17 @@ public class BlockArchiveWriter {
return this.writtenCount;
}
public Path getOutputPath() {
return this.outputPath;
}
public void setFileSizeTarget(long fileSizeTarget) {
this.fileSizeTarget = fileSizeTarget;
}
// For testing, to avoid having to pre-calculate file sizes
public void setShouldEnforceFileSizeTarget(boolean shouldEnforceFileSizeTarget) {
this.shouldEnforceFileSizeTarget = shouldEnforceFileSizeTarget;
}
}

View File

@ -0,0 +1,500 @@
package org.qortal.test;
import org.apache.commons.io.FileUtils;
import org.ciyam.at.CompilationException;
import org.ciyam.at.MachineState;
import org.ciyam.at.OpCode;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.asset.Asset;
import org.qortal.controller.BlockMinter;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.DeployAtTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.group.Group;
import org.qortal.repository.*;
import org.qortal.settings.Settings;
import org.qortal.test.common.AtUtils;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.transaction.DeployAtTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.transform.TransformationException;
import org.qortal.utils.BlockArchiveUtils;
import org.qortal.utils.Triple;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.*;
public class BlockArchiveTests extends Common {
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings(); // Necessary to set NTP offset
Common.useSettings("test-settings-v2-block-archive.json");
this.deleteArchiveDirectory();
}
@After
public void afterTest() throws DataException {
this.deleteArchiveDirectory();
}
@Test
public void testWriter() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Alice self share online
List<PrivateKeyAccount> mintingAndOnlineAccounts = new ArrayList<>();
PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share");
mintingAndOnlineAccounts.add(aliceSelfShare);
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++)
BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0]));
// 900 blocks are trimmed (this specifies the first untrimmed height)
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901);
repository.getATRepository().setAtTrimHeight(901);
// Check the max archive height - this should be one less than the first untrimmed height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(900, maximumArchiveHeight);
// Write blocks 2-900 to the archive
BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result);
// Make sure that the archive contains the correct number of blocks
assertEquals(900 - 1, writer.getWrittenCount());
// Increment block archive height
repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount());
repository.saveChanges();
assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the file exists
File outputFile = writer.getOutputPath().toFile();
assertTrue(outputFile.exists());
}
}
@Test
public void testWriterAndReader() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Alice self share online
List<PrivateKeyAccount> mintingAndOnlineAccounts = new ArrayList<>();
PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share");
mintingAndOnlineAccounts.add(aliceSelfShare);
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++)
BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0]));
// 900 blocks are trimmed (this specifies the first untrimmed height)
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901);
repository.getATRepository().setAtTrimHeight(901);
// Check the max archive height - this should be one less than the first untrimmed height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(900, maximumArchiveHeight);
// Write blocks 2-900 to the archive
BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result);
// Make sure that the archive contains the correct number of blocks
assertEquals(900 - 1, writer.getWrittenCount());
// Increment block archive height
repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount());
repository.saveChanges();
assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the file exists
File outputFile = writer.getOutputPath().toFile();
assertTrue(outputFile.exists());
// Read block 2 from the archive
BlockArchiveReader reader = BlockArchiveReader.getInstance();
Triple<BlockData, List<TransactionData>, List<ATStateData>> block2Info = reader.fetchBlockAtHeight(2);
BlockData block2ArchiveData = block2Info.getA();
// Read block 2 from the repository
BlockData block2RepositoryData = repository.getBlockRepository().fromHeight(2);
// Ensure the values match
assertEquals(block2ArchiveData.getHeight(), block2RepositoryData.getHeight());
assertArrayEquals(block2ArchiveData.getSignature(), block2RepositoryData.getSignature());
// Test some values in the archive
assertEquals(1, block2ArchiveData.getOnlineAccountsCount());
// Read block 900 from the archive
Triple<BlockData, List<TransactionData>, List<ATStateData>> block900Info = reader.fetchBlockAtHeight(900);
BlockData block900ArchiveData = block900Info.getA();
// Read block 900 from the repository
BlockData block900RepositoryData = repository.getBlockRepository().fromHeight(900);
// Ensure the values match
assertEquals(block900ArchiveData.getHeight(), block900RepositoryData.getHeight());
assertArrayEquals(block900ArchiveData.getSignature(), block900RepositoryData.getSignature());
// Test some values in the archive
assertEquals(1, block900ArchiveData.getOnlineAccountsCount());
}
}
@Test
public void testArchivedAtStates() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Alice self share online
List<PrivateKeyAccount> mintingAndOnlineAccounts = new ArrayList<>();
PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share");
mintingAndOnlineAccounts.add(aliceSelfShare);
// Deploy an AT so that we have AT state data
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
byte[] creationBytes = AtUtils.buildSimpleAT();
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 10; i++)
BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0]));
// 9 blocks are trimmed (this specifies the first untrimmed height)
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(10);
repository.getATRepository().setAtTrimHeight(10);
// Check the max archive height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(9, maximumArchiveHeight);
// Write blocks 2-9 to the archive
BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result);
// Make sure that the archive contains the correct number of blocks
assertEquals(9 - 1, writer.getWrittenCount());
// Increment block archive height
repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount());
repository.saveChanges();
assertEquals(9 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the file exists
File outputFile = writer.getOutputPath().toFile();
assertTrue(outputFile.exists());
// Check blocks 3-9
for (Integer testHeight = 2; testHeight <= 9; testHeight++) {
// Read a block from the archive
BlockArchiveReader reader = BlockArchiveReader.getInstance();
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = reader.fetchBlockAtHeight(testHeight);
BlockData archivedBlockData = blockInfo.getA();
ATStateData archivedAtStateData = blockInfo.getC().isEmpty() ? null : blockInfo.getC().get(0);
List<TransactionData> archivedTransactions = blockInfo.getB();
// Read the same block from the repository
BlockData repositoryBlockData = repository.getBlockRepository().fromHeight(testHeight);
ATStateData repositoryAtStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight);
// Ensure the repository has full AT state data
assertNotNull(repositoryAtStateData.getStateHash());
assertNotNull(repositoryAtStateData.getStateData());
// Check the archived AT state
if (testHeight == 2) {
// Block 2 won't have an AT state hash because it's initial (and has the DEPLOY_AT in the same block)
assertNull(archivedAtStateData);
assertEquals(1, archivedTransactions.size());
assertEquals(Transaction.TransactionType.DEPLOY_AT, archivedTransactions.get(0).getType());
}
else {
// For blocks 3+, ensure the archive has the AT state data, but not the hashes
assertNotNull(archivedAtStateData.getStateHash());
assertNull(archivedAtStateData.getStateData());
// They also shouldn't have any transactions
assertTrue(archivedTransactions.isEmpty());
}
// Also check the online accounts count and height
assertEquals(1, archivedBlockData.getOnlineAccountsCount());
assertEquals(testHeight, archivedBlockData.getHeight());
// Ensure the values match
assertEquals(archivedBlockData.getHeight(), repositoryBlockData.getHeight());
assertArrayEquals(archivedBlockData.getSignature(), repositoryBlockData.getSignature());
assertEquals(archivedBlockData.getOnlineAccountsCount(), repositoryBlockData.getOnlineAccountsCount());
assertArrayEquals(archivedBlockData.getMinterSignature(), repositoryBlockData.getMinterSignature());
assertEquals(archivedBlockData.getATCount(), repositoryBlockData.getATCount());
assertEquals(archivedBlockData.getOnlineAccountsCount(), repositoryBlockData.getOnlineAccountsCount());
assertArrayEquals(archivedBlockData.getReference(), repositoryBlockData.getReference());
assertEquals(archivedBlockData.getTimestamp(), repositoryBlockData.getTimestamp());
assertEquals(archivedBlockData.getATFees(), repositoryBlockData.getATFees());
assertEquals(archivedBlockData.getTotalFees(), repositoryBlockData.getTotalFees());
assertEquals(archivedBlockData.getTransactionCount(), repositoryBlockData.getTransactionCount());
assertArrayEquals(archivedBlockData.getTransactionsSignature(), repositoryBlockData.getTransactionsSignature());
if (testHeight != 2) {
assertArrayEquals(archivedAtStateData.getStateHash(), repositoryAtStateData.getStateHash());
}
}
// Check block 10 (unarchived)
BlockArchiveReader reader = BlockArchiveReader.getInstance();
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = reader.fetchBlockAtHeight(10);
assertNull(blockInfo);
}
}
@Test
public void testArchiveAndPrune() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Alice self share online
List<PrivateKeyAccount> mintingAndOnlineAccounts = new ArrayList<>();
PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share");
mintingAndOnlineAccounts.add(aliceSelfShare);
// Deploy an AT so that we have AT state data
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
byte[] creationBytes = AtUtils.buildSimpleAT();
long fundingAmount = 1_00000000L;
AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++)
BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0]));
// Assume 900 blocks are trimmed (this specifies the first untrimmed height)
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901);
repository.getATRepository().setAtTrimHeight(901);
// Check the max archive height - this should be one less than the first untrimmed height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(900, maximumArchiveHeight);
// Write blocks 2-900 to the archive
BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result);
// Make sure that the archive contains the correct number of blocks
assertEquals(900 - 1, writer.getWrittenCount());
// Increment block archive height
repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount());
repository.saveChanges();
assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the file exists
File outputFile = writer.getOutputPath().toFile();
assertTrue(outputFile.exists());
// Ensure the SQL repository contains blocks 2 and 900...
assertNotNull(repository.getBlockRepository().fromHeight(2));
assertNotNull(repository.getBlockRepository().fromHeight(900));
// Prune all the archived blocks
int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 900);
assertEquals(900-1, numBlocksPruned);
repository.getBlockRepository().setBlockPruneHeight(901);
// Prune the AT states for the archived blocks
repository.getATRepository().rebuildLatestAtStates();
int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 900);
assertEquals(900-1, numATStatesPruned);
repository.getATRepository().setAtPruneHeight(901);
// Now ensure the SQL repository is missing blocks 2 and 900...
assertNull(repository.getBlockRepository().fromHeight(2));
assertNull(repository.getBlockRepository().fromHeight(900));
// ... but it's not missing blocks 1 and 901 (we don't prune the genesis block)
assertNotNull(repository.getBlockRepository().fromHeight(1));
assertNotNull(repository.getBlockRepository().fromHeight(901));
// Validate the latest block height in the repository
assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight());
}
}
@Test
public void testTrimArchivePruneAndOrphan() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Alice self share online
List<PrivateKeyAccount> mintingAndOnlineAccounts = new ArrayList<>();
PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share");
mintingAndOnlineAccounts.add(aliceSelfShare);
// Deploy an AT so that we have AT state data
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
byte[] creationBytes = AtUtils.buildSimpleAT();
long fundingAmount = 1_00000000L;
AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++)
BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0]));
// Make sure that block 500 has full AT state data and data hash
List<ATStateData> block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500);
ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500);
assertNotNull(atStatesData.getStateHash());
assertNotNull(atStatesData.getStateData());
// Trim the first 500 blocks
repository.getBlockRepository().trimOldOnlineAccountsSignatures(0, 500);
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(501);
repository.getATRepository().trimAtStates(0, 500, 1000);
repository.getATRepository().setAtTrimHeight(501);
// Now block 500 should only have the AT state data hash
block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500);
atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500);
assertNotNull(atStatesData.getStateHash());
assertNull(atStatesData.getStateData());
// ... but block 501 should have the full data
List<ATStateData> block501AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(501);
atStatesData = repository.getATRepository().getATStateAtHeight(block501AtStatesData.get(0).getATAddress(), 501);
assertNotNull(atStatesData.getStateHash());
assertNotNull(atStatesData.getStateData());
// Check the max archive height - this should be one less than the first untrimmed height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(500, maximumArchiveHeight);
BlockData block3DataPreArchive = repository.getBlockRepository().fromHeight(3);
// Write blocks 2-500 to the archive
BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result);
// Make sure that the archive contains the correct number of blocks
assertEquals(500 - 1, writer.getWrittenCount()); // -1 for the genesis block
// Increment block archive height
repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount());
repository.saveChanges();
assertEquals(500 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the file exists
File outputFile = writer.getOutputPath().toFile();
assertTrue(outputFile.exists());
// Ensure the SQL repository contains blocks 2 and 500...
assertNotNull(repository.getBlockRepository().fromHeight(2));
assertNotNull(repository.getBlockRepository().fromHeight(500));
// Prune all the archived blocks
int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 500);
assertEquals(500-1, numBlocksPruned);
repository.getBlockRepository().setBlockPruneHeight(501);
// Prune the AT states for the archived blocks
repository.getATRepository().rebuildLatestAtStates();
int numATStatesPruned = repository.getATRepository().pruneAtStates(2, 500);
assertEquals(499, numATStatesPruned);
repository.getATRepository().setAtPruneHeight(501);
// Now ensure the SQL repository is missing blocks 2 and 500...
assertNull(repository.getBlockRepository().fromHeight(2));
assertNull(repository.getBlockRepository().fromHeight(500));
// ... but it's not missing blocks 1 and 501 (we don't prune the genesis block)
assertNotNull(repository.getBlockRepository().fromHeight(1));
assertNotNull(repository.getBlockRepository().fromHeight(501));
// Validate the latest block height in the repository
assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight());
// Now orphan some unarchived blocks.
BlockUtils.orphanBlocks(repository, 500);
assertEquals(502, (int) repository.getBlockRepository().getLastBlock().getHeight());
// We're close to the lower limit of the SQL database now, so
// we need to import some blocks from the archive
BlockArchiveUtils.importFromArchive(401, 500, repository);
// Ensure the SQL repository now contains block 401 but not 400...
assertNotNull(repository.getBlockRepository().fromHeight(401));
assertNull(repository.getBlockRepository().fromHeight(400));
// Import the remaining 399 blocks
BlockArchiveUtils.importFromArchive(2, 400, repository);
// Verify that block 3 matches the original
BlockData block3DataPostArchive = repository.getBlockRepository().fromHeight(3);
assertArrayEquals(block3DataPreArchive.getSignature(), block3DataPostArchive.getSignature());
assertEquals(block3DataPreArchive.getHeight(), block3DataPostArchive.getHeight());
// Orphan 1 more block, which should be the last one that is possible to be orphaned
BlockUtils.orphanBlocks(repository, 1);
// Orphan another block, which should fail
Exception exception = null;
try {
BlockUtils.orphanBlocks(repository, 1);
} catch (DataException e) {
exception = e;
}
// Ensure that a DataException is thrown because there is no more AT states data available
assertNotNull(exception);
assertEquals(DataException.class, exception.getClass());
// FUTURE: we may be able to retain unique AT states when trimming, to avoid this exception
// and allow orphaning back through blocks with trimmed AT states.
}
}
private void deleteArchiveDirectory() {
// Delete archive directory if exists
Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath();
try {
FileUtils.deleteDirectory(archivePath.toFile());
} catch (IOException e) {
}
}
}

View File

@ -0,0 +1,91 @@
package org.qortal.test;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.controller.BlockMinter;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.AtUtils;
import org.qortal.test.common.Common;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.*;
public class PruneTests extends Common {
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
}
@Test
public void testPruning() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Alice self share online
List<PrivateKeyAccount> mintingAndOnlineAccounts = new ArrayList<>();
PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share");
mintingAndOnlineAccounts.add(aliceSelfShare);
// Deploy an AT so that we have AT state data
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
byte[] creationBytes = AtUtils.buildSimpleAT();
long fundingAmount = 1_00000000L;
AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
// Mint some blocks
for (int i = 2; i <= 10; i++)
BlockMinter.mintTestingBlock(repository, mintingAndOnlineAccounts.toArray(new PrivateKeyAccount[0]));
// Make sure that all blocks have full AT state data and data hash
for (Integer i=2; i <= 10; i++) {
BlockData blockData = repository.getBlockRepository().fromHeight(i);
assertNotNull(blockData.getSignature());
assertEquals(i, blockData.getHeight());
List<ATStateData> atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i);
assertNotNull(atStatesDataList);
assertFalse(atStatesDataList.isEmpty());
ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i);
assertNotNull(atStatesData.getStateHash());
assertNotNull(atStatesData.getStateData());
}
// Prune blocks 2-5
int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 5);
assertEquals(4, numBlocksPruned);
repository.getBlockRepository().setBlockPruneHeight(6);
// Prune AT states for blocks 2-5
int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 5);
assertEquals(4, numATStatesPruned);
repository.getATRepository().setAtPruneHeight(6);
// Make sure that blocks 2-5 are now missing block data and AT states data
for (Integer i=2; i <= 5; i++) {
BlockData blockData = repository.getBlockRepository().fromHeight(i);
assertNull(blockData);
List<ATStateData> atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i);
assertTrue(atStatesDataList.isEmpty());
}
// ... but blocks 6-10 have block data and full AT states data
for (Integer i=6; i <= 10; i++) {
BlockData blockData = repository.getBlockRepository().fromHeight(i);
assertNotNull(blockData.getSignature());
List<ATStateData> atStatesDataList = repository.getATRepository().getBlockATStatesAtHeight(i);
assertNotNull(atStatesDataList);
assertFalse(atStatesDataList.isEmpty());
ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(atStatesDataList.get(0).getATAddress(), i);
assertNotNull(atStatesData.getStateHash());
assertNotNull(atStatesData.getStateData());
}
}
}
}

View File

@ -21,6 +21,7 @@ import org.qortal.group.Group;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.AtUtils;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
@ -35,13 +36,13 @@ public class AtRepositoryTests extends Common {
@Test
public void testGetATStateAtHeightWithData() throws DataException {
byte[] creationBytes = buildSimpleAT();
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount);
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint a few blocks
@ -58,13 +59,13 @@ public class AtRepositoryTests extends Common {
@Test
public void testGetATStateAtHeightWithoutData() throws DataException {
byte[] creationBytes = buildSimpleAT();
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount);
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint a few blocks
@ -87,13 +88,13 @@ public class AtRepositoryTests extends Common {
@Test
public void testGetLatestATStateWithData() throws DataException {
byte[] creationBytes = buildSimpleAT();
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount);
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint a few blocks
@ -111,13 +112,13 @@ public class AtRepositoryTests extends Common {
@Test
public void testGetLatestATStatePostTrimming() throws DataException {
byte[] creationBytes = buildSimpleAT();
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount);
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint a few blocks
@ -144,14 +145,66 @@ public class AtRepositoryTests extends Common {
}
@Test
public void testGetMatchingFinalATStatesWithoutDataValue() throws DataException {
byte[] creationBytes = buildSimpleAT();
public void testOrphanTrimmedATStates() throws DataException {
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount);
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint a few blocks
for (int i = 0; i < 10; ++i)
BlockUtils.mintBlock(repository);
int blockchainHeight = repository.getBlockRepository().getBlockchainHeight();
int maxTrimHeight = blockchainHeight - 4;
Integer testHeight = maxTrimHeight + 1;
// Trim AT state data
repository.getATRepository().rebuildLatestAtStates();
repository.saveChanges();
repository.getATRepository().trimAtStates(2, maxTrimHeight, 1000);
// Orphan 3 blocks
// This leaves one more untrimmed block, so the latest AT state should be available
BlockUtils.orphanBlocks(repository, 3);
ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress);
assertEquals(testHeight, atStateData.getHeight());
// We should always have the latest AT state data available
assertNotNull(atStateData.getStateData());
// Orphan 1 more block
Exception exception = null;
try {
BlockUtils.orphanBlocks(repository, 1);
} catch (DataException e) {
exception = e;
}
// Ensure that a DataException is thrown because there is no more AT states data available
assertNotNull(exception);
assertEquals(DataException.class, exception.getClass());
assertEquals(String.format("Can't find previous AT state data for %s", atAddress), exception.getMessage());
// FUTURE: we may be able to retain unique AT states when trimming, to avoid this exception
// and allow orphaning back through blocks with trimmed AT states.
}
}
@Test
public void testGetMatchingFinalATStatesWithoutDataValue() throws DataException {
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint a few blocks
@ -191,13 +244,13 @@ public class AtRepositoryTests extends Common {
@Test
public void testGetMatchingFinalATStatesWithDataValue() throws DataException {
byte[] creationBytes = buildSimpleAT();
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount);
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint a few blocks
@ -237,13 +290,13 @@ public class AtRepositoryTests extends Common {
@Test
public void testGetBlockATStatesAtHeightWithData() throws DataException {
byte[] creationBytes = buildSimpleAT();
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
doDeploy(repository, deployer, creationBytes, fundingAmount);
AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
// Mint a few blocks
for (int i = 0; i < 10; ++i)
@ -264,13 +317,13 @@ public class AtRepositoryTests extends Common {
@Test
public void testGetBlockATStatesAtHeightWithoutData() throws DataException {
byte[] creationBytes = buildSimpleAT();
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
doDeploy(repository, deployer, creationBytes, fundingAmount);
AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
// Mint a few blocks
for (int i = 0; i < 10; ++i)
@ -297,13 +350,13 @@ public class AtRepositoryTests extends Common {
@Test
public void testSaveATStateWithData() throws DataException {
byte[] creationBytes = buildSimpleAT();
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount);
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint a few blocks
@ -328,13 +381,13 @@ public class AtRepositoryTests extends Common {
@Test
public void testSaveATStateWithoutData() throws DataException {
byte[] creationBytes = buildSimpleAT();
byte[] creationBytes = AtUtils.buildSimpleAT();
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount);
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint a few blocks
@ -364,67 +417,4 @@ public class AtRepositoryTests extends Common {
assertNull(atStateData.getStateData());
}
}
private byte[] buildSimpleAT() {
// Pretend we use 4 values in data segment
int addrCounter = 4;
// Data segment
ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE);
ByteBuffer codeByteBuffer = ByteBuffer.allocate(512);
// Two-pass version
for (int pass = 0; pass < 2; ++pass) {
codeByteBuffer.clear();
try {
// Stop and wait for next block
codeByteBuffer.put(OpCode.STP_IMD.compile());
} catch (CompilationException e) {
throw new IllegalStateException("Unable to compile AT?", e);
}
}
codeByteBuffer.flip();
byte[] codeBytes = new byte[codeByteBuffer.limit()];
codeByteBuffer.get(codeBytes);
final short ciyamAtVersion = 2;
final short numCallStackPages = 0;
final short numUserStackPages = 0;
final long minActivationAmount = 0L;
return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount);
}
private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException {
long txTimestamp = System.currentTimeMillis();
byte[] lastReference = deployer.getLastReference();
if (lastReference == null) {
System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress()));
System.exit(2);
}
Long fee = null;
String name = "Test AT";
String description = "Test AT";
String atType = "Test";
String tags = "TEST";
BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null);
TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT);
DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData);
fee = deployAtTransaction.calcRecommendedFee();
deployAtTransactionData.setFee(fee);
TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer);
return deployAtTransaction;
}
}

View File

@ -0,0 +1,81 @@
package org.qortal.test.common;
import org.ciyam.at.CompilationException;
import org.ciyam.at.MachineState;
import org.ciyam.at.OpCode;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.asset.Asset;
import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.DeployAtTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.group.Group;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.transaction.DeployAtTransaction;
import java.nio.ByteBuffer;
public class AtUtils {
public static byte[] buildSimpleAT() {
// Pretend we use 4 values in data segment
int addrCounter = 4;
// Data segment
ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE);
ByteBuffer codeByteBuffer = ByteBuffer.allocate(512);
// Two-pass version
for (int pass = 0; pass < 2; ++pass) {
codeByteBuffer.clear();
try {
// Stop and wait for next block
codeByteBuffer.put(OpCode.STP_IMD.compile());
} catch (CompilationException e) {
throw new IllegalStateException("Unable to compile AT?", e);
}
}
codeByteBuffer.flip();
byte[] codeBytes = new byte[codeByteBuffer.limit()];
codeByteBuffer.get(codeBytes);
final short ciyamAtVersion = 2;
final short numCallStackPages = 0;
final short numUserStackPages = 0;
final long minActivationAmount = 0L;
return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount);
}
public static DeployAtTransaction doDeployAT(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException {
long txTimestamp = System.currentTimeMillis();
byte[] lastReference = deployer.getLastReference();
if (lastReference == null) {
System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress()));
System.exit(2);
}
Long fee = null;
String name = "Test AT";
String description = "Test AT";
String atType = "Test";
String tags = "TEST";
BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null);
TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT);
DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData);
fee = deployAtTransaction.calcRecommendedFee();
deployAtTransactionData.setFee(fee);
TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer);
return deployAtTransaction;
}
}

View File

@ -0,0 +1,11 @@
{
"bitcoinNet": "TEST3",
"litecoinNet": "TEST3",
"restrictedApi": false,
"blockchainConfig": "src/test/resources/test-chain-v2.json",
"wipeUnconfirmedOnStart": false,
"testNtpOffset": 0,
"minPeers": 0,
"pruneBlockLimit": 1450,
"repositoryPath": "dbtest"
}