AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'rebuild-archive'

Browse Source
This commit is contained in:
CalDescent 2023-03-10 11:50:09 +00:00
commit 4afbca7ed2
17 changed files with 1001 additions and 741 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
src/main/java/org/qortal/api/resource/AdminResource.java
View File

@ -45,6 +45,7 @@ import org.qortal.block.BlockChain;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.controller.Synchronizer.SynchronizationResult;
import org.qortal.controller.repository.BlockArchiveRebuilder;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.account.RewardShareData;
import org.qortal.network.Network;
@ -734,6 +735,64 @@ public class AdminResource {
}
}
@POST
@Path("/repository/archive/rebuild")
@Operation(
summary = "Rebuild archive",
description = "Rebuilds archive files, using the specified serialization version",
requestBody = @RequestBody(
required = true,
content = @Content(
mediaType = MediaType.TEXT_PLAIN,
schema = @Schema(
type = "number", example = "2"
)
)
),
responses = {
@ApiResponse(
description = "\"true\"",
content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string"))
)
}
)
@ApiErrors({ApiError.REPOSITORY_ISSUE})
@SecurityRequirement(name = "apiKey")
public String rebuildArchive(@HeaderParam(Security.API_KEY_HEADER) String apiKey, Integer serializationVersion) {
Security.checkApiCallAllowed(request);
// Default serialization version to value specified in settings
if (serializationVersion == null) {
serializationVersion = Settings.getInstance().getDefaultArchiveVersion();
}
try {
// We don't actually need to lock the blockchain here, but we'll do it anyway so that
// the node can focus on rebuilding rather than synchronizing / minting.
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
blockchainLock.lockInterruptibly();
try {
BlockArchiveRebuilder blockArchiveRebuilder = new BlockArchiveRebuilder(serializationVersion);
blockArchiveRebuilder.start();
return "true";
} catch (IOException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e);
} finally {
blockchainLock.unlock();
}
} catch (InterruptedException e) {
// We couldn't lock blockchain to perform rebuild
return "false";
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@DELETE
@Path("/repository")
@Operation(

12
src/main/java/org/qortal/api/resource/BlocksResource.java
View File

@ -48,6 +48,7 @@ import org.qortal.repository.RepositoryManager;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformer;
import org.qortal.utils.Base58;
import org.qortal.utils.Triple;
@Path("/blocks")
@Tag(name = "Blocks")
@ -165,10 +166,13 @@ public class BlocksResource {
}
// Not found, so try the block archive
byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, false, repository);
if (bytes != null) {
if (version != 1) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Archived blocks require version 1");
Triple<byte[], Integer, Integer> serializedBlock = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, false, repository);
if (serializedBlock != null) {
byte[] bytes = serializedBlock.getA();
Integer serializationVersion = serializedBlock.getB();
if (version != serializationVersion) {
// TODO: we could quite easily reserialize the block with the requested version
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Block is not stored using requested serialization version.");
}
return Base58.encode(bytes);
}

4
src/main/java/org/qortal/block/Block.java
View File

@ -657,6 +657,10 @@ public class Block {
return this.atStates;
}
public byte[] getAtStatesHash() {
return this.atStatesHash;
}
/**
* Return expanded info on block's online accounts.
* <p>

29
src/main/java/org/qortal/controller/Controller.java
View File

@ -400,12 +400,8 @@ public class Controller extends Thread {
RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl());
RepositoryManager.setRepositoryFactory(repositoryFactory);
RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
try (final Repository repository = RepositoryManager.getRepository()) {
RepositoryManager.archive(repository);
RepositoryManager.prune(repository);
}
} catch (DataException e) {
}
catch (DataException e) {
// If exception has no cause then repository is in use by some other process.
if (e.getCause() == null) {
LOGGER.info("Repository in use by another process?");
@ -1379,9 +1375,24 @@ public class Controller extends Thread {
// If we have no block data, we should check the archive in case it's there
if (blockData == null) {
if (Settings.getInstance().isArchiveEnabled()) {
byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, true, repository);
if (bytes != null) {
CachedBlockMessage blockMessage = new CachedBlockMessage(bytes);
Triple<byte[], Integer, Integer> serializedBlock = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, true, repository);
if (serializedBlock != null) {
byte[] bytes = serializedBlock.getA();
Integer serializationVersion = serializedBlock.getB();
Message blockMessage;
switch (serializationVersion) {
case 1:
blockMessage = new CachedBlockMessage(bytes);
break;
case 2:
blockMessage = new CachedBlockV2Message(bytes);
break;
default:
return;
}
blockMessage.setId(message.getId());
// This call also causes the other needed data to be pulled in from repository

121
src/main/java/org/qortal/controller/repository/BlockArchiveRebuilder.java Normal file
View File

@ -0,0 +1,121 @@
package org.qortal.controller.repository;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.repository.*;
import org.qortal.settings.Settings;
import org.qortal.transform.TransformationException;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
public class BlockArchiveRebuilder {
private static final Logger LOGGER = LogManager.getLogger(BlockArchiveRebuilder.class);
private final int serializationVersion;
public BlockArchiveRebuilder(int serializationVersion) {
this.serializationVersion = serializationVersion;
}
public void start() throws DataException, IOException {
if (!Settings.getInstance().isArchiveEnabled() || Settings.getInstance().isLite()) {
return;
}
// New archive path is in a different location from original archive path, to avoid conflicts.
// It will be moved later, once the process is complete.
final Path newArchivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive-rebuild");
final Path originalArchivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive");
// Delete archive-rebuild if it exists from a previous attempt
FileUtils.deleteDirectory(newArchivePath.toFile());
try (final Repository repository = RepositoryManager.getRepository()) {
int startHeight = 1; // We need to rebuild the entire archive
LOGGER.info("Rebuilding block archive from height {}...", startHeight);
while (!Controller.isStopping()) {
repository.discardChanges();
Thread.sleep(1000L);
// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
if (Synchronizer.getInstance().isSynchronizing()) {
continue;
}
// Rebuild archive
try {
final int maximumArchiveHeight = BlockArchiveReader.getInstance().getHeightOfLastArchivedBlock();
if (startHeight >= maximumArchiveHeight) {
// We've finished.
// Delete existing archive and move the newly built one into its place
FileUtils.deleteDirectory(originalArchivePath.toFile());
FileUtils.moveDirectory(newArchivePath.toFile(), originalArchivePath.toFile());
BlockArchiveReader.getInstance().invalidateFileListCache();
LOGGER.info("Block archive successfully rebuilt");
return;
}
BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, serializationVersion, newArchivePath, repository);
// Set data source to BLOCK_ARCHIVE as we are rebuilding
writer.setDataSource(BlockArchiveWriter.BlockArchiveDataSource.BLOCK_ARCHIVE);
// We can't enforce the 100MB file size target, as the final file needs to contain all blocks
// that exist in the current archive. Otherwise, the final blocks in the archive will be lost.
writer.setShouldEnforceFileSizeTarget(false);
// We want to log the rebuild progress
writer.setShouldLogProgress(true);
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
switch (result) {
case OK:
// Increment block archive height
startHeight += writer.getWrittenCount();
repository.saveChanges();
break;
case STOPPING:
return;
// We've reached the limit of the blocks we can archive
// Sleep for a while to allow more to become available
case NOT_ENOUGH_BLOCKS:
// This shouldn't happen, as we're not enforcing minimum file sizes
repository.discardChanges();
throw new DataException("Unable to rebuild archive due to unexpected NOT_ENOUGH_BLOCKS response.");
case BLOCK_NOT_FOUND:
// We tried to archive a block that didn't exist. This is a major failure and likely means
// that a bootstrap or re-sync is needed. Try again every minute until then.
LOGGER.info("Error: block not found when rebuilding archive. If this error persists, " +
"a bootstrap or re-sync may be needed.");
repository.discardChanges();
throw new DataException("Unable to rebuild archive because a block is missing.");
}
} catch (IOException | TransformationException e) {
LOGGER.info("Caught exception when rebuilding block archive", e);
throw new DataException("Unable to rebuild block archive");
}
}
} catch (InterruptedException e) {
// Do nothing
} finally {
// Delete archive-rebuild if it still exists, as that means something went wrong
FileUtils.deleteDirectory(newArchivePath.toFile());
}
}
}

43
src/main/java/org/qortal/network/message/CachedBlockV2Message.java Normal file
View File

@ -0,0 +1,43 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import org.qortal.block.Block;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
// This is an OUTGOING-only Message which more readily lends itself to being cached
public class CachedBlockV2Message extends Message implements Cloneable {
public CachedBlockV2Message(Block block) throws TransformationException {
super(MessageType.BLOCK_V2);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try {
bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
bytes.write(BlockTransformer.toBytes(block));
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
public CachedBlockV2Message(byte[] cachedBytes) {
super(MessageType.BLOCK_V2);
this.dataBytes = cachedBytes;
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) {
throw new UnsupportedOperationException("CachedBlockMessageV2 is for outgoing messages only");
}
}

85
src/main/java/org/qortal/repository/BlockArchiveReader.java
View File

@ -3,10 +3,7 @@ package org.qortal.repository;
import com.google.common.primitives.Ints;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockArchiveData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.settings.Settings;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
@ -67,20 +64,51 @@ public class BlockArchiveReader {
this.fileListCache = Map.copyOf(map);
}
public Integer fetchSerializationVersionForHeight(int height) {
if (this.fileListCache == null) {
this.fetchFileList();
}
Triple<byte[], Integer, Integer> serializedBlock = this.fetchSerializedBlockBytesForHeight(height);
if (serializedBlock == null) {
return null;
}
Integer serializationVersion = serializedBlock.getB();
return serializationVersion;
}
public BlockTransformation fetchBlockAtHeight(int height) {
if (this.fileListCache == null) {
this.fetchFileList();
}
byte[] serializedBytes = this.fetchSerializedBlockBytesForHeight(height);
if (serializedBytes == null) {
Triple<byte[], Integer, Integer> serializedBlock = this.fetchSerializedBlockBytesForHeight(height);
if (serializedBlock == null) {
return null;
}
byte[] serializedBytes = serializedBlock.getA();
Integer serializationVersion = serializedBlock.getB();
if (serializedBytes == null || serializationVersion == null) {
return null;
}
ByteBuffer byteBuffer = ByteBuffer.wrap(serializedBytes);
BlockTransformation blockInfo = null;
try {
blockInfo = BlockTransformer.fromByteBuffer(byteBuffer);
switch (serializationVersion) {
case 1:
blockInfo = BlockTransformer.fromByteBuffer(byteBuffer);
break;
case 2:
blockInfo = BlockTransformer.fromByteBufferV2(byteBuffer);
break;
default:
// Invalid serialization version
return null;
}
if (blockInfo != null && blockInfo.getBlockData() != null) {
// Block height is stored outside of the main serialized bytes, so it
// won't be set automatically.
@ -168,15 +196,20 @@ public class BlockArchiveReader {
return null;
}
public byte[] fetchSerializedBlockBytesForSignature(byte[] signature, boolean includeHeightPrefix, Repository repository) {
public Triple<byte[], Integer, Integer> fetchSerializedBlockBytesForSignature(byte[] signature, boolean includeHeightPrefix, Repository repository) {
if (this.fileListCache == null) {
this.fetchFileList();
}
Integer height = this.fetchHeightForSignature(signature, repository);
if (height != null) {
byte[] blockBytes = this.fetchSerializedBlockBytesForHeight(height);
if (blockBytes == null) {
Triple<byte[], Integer, Integer> serializedBlock = this.fetchSerializedBlockBytesForHeight(height);
if (serializedBlock == null) {
return null;
}
byte[] blockBytes = serializedBlock.getA();
Integer version = serializedBlock.getB();
if (blockBytes == null || version == null) {
return null;
}
@ -187,18 +220,18 @@ public class BlockArchiveReader {
try {
bytes.write(Ints.toByteArray(height));
bytes.write(blockBytes);
return bytes.toByteArray();
return new Triple<>(bytes.toByteArray(), version, height);
} catch (IOException e) {
return null;
}
}
return blockBytes;
return new Triple<>(blockBytes, version, height);
}
return null;
}
public byte[] fetchSerializedBlockBytesForHeight(int height) {
public Triple<byte[], Integer, Integer> fetchSerializedBlockBytesForHeight(int height) {
String filename = this.getFilenameForHeight(height);
if (filename == null) {
// We don't have this block in the archive
@ -221,7 +254,7 @@ public class BlockArchiveReader {
// End of fixed length header
// Make sure the version is one we recognize
if (version != 1) {
if (version != 1 && version != 2) {
LOGGER.info("Error: unknown version in file {}: {}", filename, version);
return null;
}
@ -258,7 +291,7 @@ public class BlockArchiveReader {
byte[] blockBytes = new byte[blockLength];
file.read(blockBytes);
return blockBytes;
return new Triple<>(blockBytes, version, height);
} catch (FileNotFoundException e) {
LOGGER.info("File {} not found: {}", filename, e.getMessage());
@ -279,6 +312,30 @@ public class BlockArchiveReader {
}
}
public int getHeightOfLastArchivedBlock() {
if (this.fileListCache == null) {
this.fetchFileList();
}
int maxEndHeight = 0;
Iterator it = this.fileListCache.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pair = (Map.Entry) it.next();
if (pair == null && pair.getKey() == null && pair.getValue() == null) {
continue;
}
Triple<Integer, Integer, Integer> heightInfo = (Triple<Integer, Integer, Integer>) pair.getValue();
Integer endHeight = heightInfo.getB();
if (endHeight != null && endHeight > maxEndHeight) {
maxEndHeight = endHeight;
}
}
return maxEndHeight;
}
public void invalidateFileListCache() {
this.fileListCache = null;
}

129
src/main/java/org/qortal/repository/BlockArchiveWriter.java
View File

@ -6,10 +6,13 @@ import org.apache.logging.log4j.Logger;
import org.qortal.block.Block;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockArchiveData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.settings.Settings;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.transform.block.BlockTransformer;
import java.io.ByteArrayOutputStream;
@ -18,6 +21,7 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
public class BlockArchiveWriter {
@ -28,25 +32,71 @@ public class BlockArchiveWriter {
BLOCK_NOT_FOUND
}
public enum BlockArchiveDataSource {
BLOCK_REPOSITORY, // To build an archive from the Blocks table
BLOCK_ARCHIVE // To build a new archive from an existing archive
}
private static final Logger LOGGER = LogManager.getLogger(BlockArchiveWriter.class);
public static final long DEFAULT_FILE_SIZE_TARGET = 100 * 1024 * 1024; // 100MiB
private int startHeight;
private final int endHeight;
private final Integer serializationVersion;
private final Path archivePath;
private final Repository repository;
private long fileSizeTarget = DEFAULT_FILE_SIZE_TARGET;
private boolean shouldEnforceFileSizeTarget = true;
// Default data source to BLOCK_REPOSITORY; can optionally be overridden
private BlockArchiveDataSource dataSource = BlockArchiveDataSource.BLOCK_REPOSITORY;
private boolean shouldLogProgress = false;
private int writtenCount;
private int lastWrittenHeight;
private Path outputPath;
public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) {
/**
* Instantiate a BlockArchiveWriter using a custom archive path
* @param startHeight
* @param endHeight
* @param repository
*/
public BlockArchiveWriter(int startHeight, int endHeight, Integer serializationVersion, Path archivePath, Repository repository) {
this.startHeight = startHeight;
this.endHeight = endHeight;
this.archivePath = archivePath.toAbsolutePath();
this.repository = repository;
if (serializationVersion == null) {
// When serialization version isn't specified, fetch it from the existing archive
serializationVersion = this.findSerializationVersion();
}
this.serializationVersion = serializationVersion;
}
/**
* Instantiate a BlockArchiveWriter using the default archive path and version
* @param startHeight
* @param endHeight
* @param repository
*/
public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) {
this(startHeight, endHeight, null, Paths.get(Settings.getInstance().getRepositoryPath(), "archive"), repository);
}
private int findSerializationVersion() {
// Attempt to fetch the serialization version from the existing archive
Integer block2SerializationVersion = BlockArchiveReader.getInstance().fetchSerializationVersionForHeight(2);
if (block2SerializationVersion != null) {
return block2SerializationVersion;
}
// Default to version specified in settings
return Settings.getInstance().getDefaultArchiveVersion();
}
public static int getMaxArchiveHeight(Repository repository) throws DataException {
@ -72,8 +122,7 @@ public class BlockArchiveWriter {
public BlockArchiveWriteResult write() throws DataException, IOException, TransformationException, InterruptedException {
// Create the archive folder if it doesn't exist
// This is a subfolder of the db directory, to make bootstrapping easier
Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath();
// This is generally a subfolder of the db directory, to make bootstrapping easier
try {
Files.createDirectories(archivePath);
} catch (IOException e) {
@ -95,13 +144,13 @@ public class BlockArchiveWriter {
LOGGER.info(String.format("Fetching blocks from height %d...", startHeight));
int i = 0;
while (headerBytes.size() + bytes.size() < this.fileSizeTarget
|| this.shouldEnforceFileSizeTarget == false) {
while (headerBytes.size() + bytes.size() < this.fileSizeTarget) {
if (Controller.isStopping()) {
return BlockArchiveWriteResult.STOPPING;
}
if (Synchronizer.getInstance().isSynchronizing()) {
Thread.sleep(1000L);
continue;
}
@ -112,7 +161,28 @@ public class BlockArchiveWriter {
//LOGGER.info("Fetching block {}...", currentHeight);
BlockData blockData = repository.getBlockRepository().fromHeight(currentHeight);
BlockData blockData = null;
List<TransactionData> transactions = null;
List<ATStateData> atStates = null;
byte[] atStatesHash = null;
switch (this.dataSource) {
case BLOCK_ARCHIVE:
BlockTransformation archivedBlock = BlockArchiveReader.getInstance().fetchBlockAtHeight(currentHeight);
if (archivedBlock != null) {
blockData = archivedBlock.getBlockData();
transactions = archivedBlock.getTransactions();
atStates = archivedBlock.getAtStates();
atStatesHash = archivedBlock.getAtStatesHash();
}
break;
case BLOCK_REPOSITORY:
default:
blockData = repository.getBlockRepository().fromHeight(currentHeight);
break;
}
if (blockData == null) {
return BlockArchiveWriteResult.BLOCK_NOT_FOUND;
}
@ -122,18 +192,50 @@ public class BlockArchiveWriter {
repository.getBlockArchiveRepository().save(blockArchiveData);
repository.saveChanges();
// Build the block
Block block;
if (atStatesHash != null) {
block = new Block(repository, blockData, transactions, atStatesHash);
}
else if (atStates != null) {
block = new Block(repository, blockData, transactions, atStates);
}
else {
block = new Block(repository, blockData);
}
// Write the block data to some byte buffers
Block block = new Block(repository, blockData);
int blockIndex = bytes.size();
// Write block index to header
headerBytes.write(Ints.toByteArray(blockIndex));
// Write block height
bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
byte[] blockBytes = BlockTransformer.toBytes(block);
// Get serialized block bytes
byte[] blockBytes;
switch (serializationVersion) {
case 1:
blockBytes = BlockTransformer.toBytes(block);
break;
case 2:
blockBytes = BlockTransformer.toBytesV2(block);
break;
default:
throw new DataException("Invalid serialization version");
}
// Write block length
bytes.write(Ints.toByteArray(blockBytes.length));
// Write block bytes
bytes.write(blockBytes);
// Log every 1000 blocks
if (this.shouldLogProgress && i % 1000 == 0) {
LOGGER.info("Archived up to block height {}. Size of current file: {} bytes", currentHeight, (headerBytes.size() + bytes.size()));
}
i++;
}
@ -147,11 +249,10 @@ public class BlockArchiveWriter {
// We have enough blocks to create a new file
int endHeight = startHeight + i - 1;
int version = 1;
String filePath = String.format("%s/%d-%d.dat", archivePath.toString(), startHeight, endHeight);
FileOutputStream fileOutputStream = new FileOutputStream(filePath);
// Write version number
fileOutputStream.write(Ints.toByteArray(version));
fileOutputStream.write(Ints.toByteArray(serializationVersion));
// Write start height
fileOutputStream.write(Ints.toByteArray(startHeight));
// Write end height
@ -199,4 +300,12 @@ public class BlockArchiveWriter {
this.shouldEnforceFileSizeTarget = shouldEnforceFileSizeTarget;
}
public void setDataSource(BlockArchiveDataSource dataSource) {
this.dataSource = dataSource;
}
public void setShouldLogProgress(boolean shouldLogProgress) {
this.shouldLogProgress = shouldLogProgress;
}
}

61
src/main/java/org/qortal/repository/RepositoryManager.java
View File

@ -2,11 +2,6 @@ package org.qortal.repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.gui.SplashFrame;
import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving;
import org.qortal.repository.hsqldb.HSQLDBDatabasePruning;
import org.qortal.repository.hsqldb.HSQLDBRepository;
import org.qortal.settings.Settings;
import java.sql.SQLException;
import java.util.concurrent.TimeoutException;
@ -61,62 +56,6 @@ public abstract class RepositoryManager {
}
}
public static boolean archive(Repository repository) {
if (Settings.getInstance().isLite()) {
// Lite nodes have no blockchain
return false;
}
// Bulk archive the database the first time we use archive mode
if (Settings.getInstance().isArchiveEnabled()) {
if (RepositoryManager.canArchiveOrPrune()) {
try {
return HSQLDBDatabaseArchiving.buildBlockArchive(repository, BlockArchiveWriter.DEFAULT_FILE_SIZE_TARGET);
} catch (DataException e) {
LOGGER.info("Unable to build block archive. The database may have been left in an inconsistent state.");
}
}
else {
LOGGER.info("Unable to build block archive due to missing ATStatesHeightIndex. Bootstrapping is recommended.");
LOGGER.info("To bootstrap, stop the core and delete the db folder, then start the core again.");
SplashFrame.getInstance().updateStatus("Missing index. Bootstrapping is recommended.");
}
}
return false;
}
public static boolean prune(Repository repository) {
if (Settings.getInstance().isLite()) {
// Lite nodes have no blockchain
return false;
}
// Bulk prune the database the first time we use top-only or block archive mode
if (Settings.getInstance().isTopOnly() ||
Settings.getInstance().isArchiveEnabled()) {
if (RepositoryManager.canArchiveOrPrune()) {
try {
boolean prunedATStates = HSQLDBDatabasePruning.pruneATStates((HSQLDBRepository) repository);
boolean prunedBlocks = HSQLDBDatabasePruning.pruneBlocks((HSQLDBRepository) repository);
// Perform repository maintenance to shrink the db size down
if (prunedATStates && prunedBlocks) {
HSQLDBDatabasePruning.performMaintenance(repository);
return true;
}
} catch (SQLException | DataException e) {
LOGGER.info("Unable to bulk prune AT states. The database may have been left in an inconsistent state.");
}
}
else {
LOGGER.info("Unable to prune blocks due to missing ATStatesHeightIndex. Bootstrapping is recommended.");
}
}
return false;
}
public static void setRequestedCheckpoint(Boolean quick) {
quickCheckpointRequested = quick;
}

88
src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java
View File

@ -1,88 +0,0 @@
package org.qortal.repository.hsqldb;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.gui.SplashFrame;
import org.qortal.repository.BlockArchiveWriter;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.transform.TransformationException;
import java.io.IOException;
/**
*
* When switching to an archiving node, we need to archive most of the database contents.
* This involves copying its data into flat files.
* If we do this entirely as a background process, it is very slow and can interfere with syncing.
* However, if we take the approach of doing this in bulk, before starting up the rest of the
* processes, this makes it much faster and less invasive.
*
* From that point, the original background archiving process will run, but can be dialled right down
* so not to interfere with syncing.
*
*/
public class HSQLDBDatabaseArchiving {
private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabaseArchiving.class);
public static boolean buildBlockArchive(Repository repository, long fileSizeTarget) throws DataException {
// Only build the archive if we haven't already got one that is up to date
boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository);
if (upToDate) {
// Already archived
return false;
}
LOGGER.info("Building block archive - this process could take a while...");
SplashFrame.getInstance().updateStatus("Building block archive...");
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
int startHeight = 0;
while (!Controller.isStopping()) {
try {
BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository);
writer.setFileSizeTarget(fileSizeTarget);
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
switch (result) {
case OK:
// Increment block archive height
startHeight = writer.getLastWrittenHeight() + 1;
repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight);
repository.saveChanges();
break;
case STOPPING:
return false;
case NOT_ENOUGH_BLOCKS:
// We've reached the limit of the blocks we can archive
// Return from the whole method
return true;
case BLOCK_NOT_FOUND:
// We tried to archive a block that didn't exist. This is a major failure and likely means
// that a bootstrap or re-sync is needed. Return rom the method
LOGGER.info("Error: block not found when building archive. If this error persists, " +
"a bootstrap or re-sync may be needed.");
return false;
}
} catch (IOException | TransformationException | InterruptedException e) {
LOGGER.info("Caught exception when creating block cache", e);
return false;
}
}
// If we got this far then something went wrong (most likely the app is stopping)
return false;
}
}

332
src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java
View File

@ -1,332 +0,0 @@
package org.qortal.repository.hsqldb;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.data.block.BlockData;
import org.qortal.gui.SplashFrame;
import org.qortal.repository.BlockArchiveWriter;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.settings.Settings;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.concurrent.TimeoutException;
/**
*
* When switching from a full node to a pruning node, we need to delete most of the database contents.
* If we do this entirely as a background process, it is very slow and can interfere with syncing.
* However, if we take the approach of transferring only the necessary rows to a new table and then
* deleting the original table, this makes the process much faster. It was taking several days to
* delete the AT states in the background, but only a couple of minutes to copy them to a new table.
*
* The trade off is that we have to go through a form of "reshape" when starting the app for the first
* time after enabling pruning mode. But given that this is an opt-in mode, I don't think it will be
* a problem.
*
* Once the pruning is complete, it automatically performs a CHECKPOINT DEFRAG in order to
* shrink the database file size down to a fraction of what it was before.
*
* From this point, the original background process will run, but can be dialled right down so not
* to interfere with syncing.
*
*/
public class HSQLDBDatabasePruning {
private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabasePruning.class);
public static boolean pruneATStates(HSQLDBRepository repository) throws SQLException, DataException {
// Only bulk prune AT states if we have never done so before
int pruneHeight = repository.getATRepository().getAtPruneHeight();
if (pruneHeight > 0) {
// Already pruned AT states
return false;
}
if (Settings.getInstance().isArchiveEnabled()) {
// Only proceed if we can see that the archiver has already finished
// This way, if the archiver failed for any reason, we can prune once it has had
// some opportunities to try again
boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository);
if (!upToDate) {
return false;
}
}
LOGGER.info("Starting bulk prune of AT states - this process could take a while... " +
"(approx. 2 mins on high spec, or upwards of 30 mins in some cases)");
SplashFrame.getInstance().updateStatus("Pruning database (takes up to 30 mins)...");
// Create new AT-states table to hold smaller dataset
repository.executeCheckedUpdate("DROP TABLE IF EXISTS ATStatesNew");
repository.executeCheckedUpdate("CREATE TABLE ATStatesNew ("
+ "AT_address QortalAddress, height INTEGER NOT NULL, state_hash ATStateHash NOT NULL, "
+ "fees QortalAmount NOT NULL, is_initial BOOLEAN NOT NULL, sleep_until_message_timestamp BIGINT, "
+ "PRIMARY KEY (AT_address, height), "
+ "FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)");
repository.executeCheckedUpdate("SET TABLE ATStatesNew NEW SPACE");
repository.executeCheckedUpdate("CHECKPOINT");
// Add a height index
LOGGER.info("Adding index to AT states table...");
repository.executeCheckedUpdate("CREATE INDEX IF NOT EXISTS ATStatesNewHeightIndex ON ATStatesNew (height)");
repository.executeCheckedUpdate("CHECKPOINT");
// Find our latest block
BlockData latestBlock = repository.getBlockRepository().getLastBlock();
if (latestBlock == null) {
LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning");
return false;
}
// Calculate some constants for later use
final int blockchainHeight = latestBlock.getHeight();
int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit();
if (Settings.getInstance().isArchiveEnabled()) {
// Archive mode - don't prune anything that hasn't been archived yet
maximumBlockToTrim = Math.min(maximumBlockToTrim, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1);
}
final int endHeight = blockchainHeight;
final int blockStep = 10000;
// It's essential that we rebuild the latest AT states here, as we are using this data in the next query.
// Failing to do this will result in important AT states being deleted, rendering the database unusable.
repository.getATRepository().rebuildLatestAtStates(endHeight);
// Loop through all the LatestATStates and copy them to the new table
LOGGER.info("Copying AT states...");
for (int height = 0; height < endHeight; height += blockStep) {
final int batchEndHeight = height + blockStep - 1;
//LOGGER.info(String.format("Copying AT states between %d and %d...", height, batchEndHeight));
String sql = "SELECT height, AT_address FROM LatestATStates WHERE height BETWEEN ? AND ?";
try (ResultSet latestAtStatesResultSet = repository.checkedExecute(sql, height, batchEndHeight)) {
if (latestAtStatesResultSet != null) {
do {
int latestAtHeight = latestAtStatesResultSet.getInt(1);
String latestAtAddress = latestAtStatesResultSet.getString(2);
// Copy this latest ATState to the new table
//LOGGER.info(String.format("Copying AT %s at height %d...", latestAtAddress, latestAtHeight));
try {
String updateSql = "INSERT INTO ATStatesNew ("
+ "SELECT AT_address, height, state_hash, fees, is_initial, sleep_until_message_timestamp "
+ "FROM ATStates "
+ "WHERE height = ? AND AT_address = ?)";
repository.executeCheckedUpdate(updateSql, latestAtHeight, latestAtAddress);
} catch (SQLException e) {
repository.examineException(e);
throw new DataException("Unable to copy ATStates", e);
}
// If this batch includes blocks after the maximum block to trim, we will need to copy
// each of its AT states above maximumBlockToTrim as they are considered "recent". We
// need to do this for _all_ AT states in these blocks, regardless of their latest state.
if (batchEndHeight >= maximumBlockToTrim) {
// Now copy this AT's states for each recent block they are present in
for (int i = maximumBlockToTrim; i < endHeight; i++) {
if (latestAtHeight < i) {
// This AT finished before this block so there is nothing to copy
continue;
}
//LOGGER.info(String.format("Copying recent AT %s at height %d...", latestAtAddress, i));
try {
// Copy each LatestATState to the new table
String updateSql = "INSERT IGNORE INTO ATStatesNew ("
+ "SELECT AT_address, height, state_hash, fees, is_initial, sleep_until_message_timestamp "
+ "FROM ATStates "
+ "WHERE height = ? AND AT_address = ?)";
repository.executeCheckedUpdate(updateSql, i, latestAtAddress);
} catch (SQLException e) {
repository.examineException(e);
throw new DataException("Unable to copy ATStates", e);
}
}
}
repository.saveChanges();
} while (latestAtStatesResultSet.next());
}
} catch (SQLException e) {
throw new DataException("Unable to copy AT states", e);
}
}
// Finally, drop the original table and rename
LOGGER.info("Deleting old AT states...");
repository.executeCheckedUpdate("DROP TABLE ATStates");
repository.executeCheckedUpdate("ALTER TABLE ATStatesNew RENAME TO ATStates");
repository.executeCheckedUpdate("ALTER INDEX ATStatesNewHeightIndex RENAME TO ATStatesHeightIndex");
repository.executeCheckedUpdate("CHECKPOINT");
// Update the prune height
int nextPruneHeight = maximumBlockToTrim + 1;
repository.getATRepository().setAtPruneHeight(nextPruneHeight);
repository.saveChanges();
repository.executeCheckedUpdate("CHECKPOINT");
// Now prune/trim the ATStatesData, as this currently goes back over a month
return HSQLDBDatabasePruning.pruneATStateData(repository);
}
/*
* Bulk prune ATStatesData to catch up with the now pruned ATStates table
* This uses the existing AT States trimming code but with a much higher end block
*/
private static boolean pruneATStateData(Repository repository) throws DataException {
if (Settings.getInstance().isArchiveEnabled()) {
// Don't prune ATStatesData in archive mode
return true;
}
BlockData latestBlock = repository.getBlockRepository().getLastBlock();
if (latestBlock == null) {
LOGGER.info("Unable to determine blockchain height, necessary for bulk ATStatesData pruning");
return false;
}
final int blockchainHeight = latestBlock.getHeight();
int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit();
// ATStateData is already trimmed - so carry on from where we left off in the past
int pruneStartHeight = repository.getATRepository().getAtTrimHeight();
LOGGER.info("Starting bulk prune of AT states data - this process could take a while... (approx. 3 mins on high spec)");
while (pruneStartHeight < upperPrunableHeight) {
// Prune all AT state data up until our latest minus pruneBlockLimit (or our archive height)
if (Controller.isStopping()) {
return false;
}
// Override batch size in the settings because this is a one-off process
final int batchSize = 1000;
final int rowLimitPerBatch = 50000;
int upperBatchHeight = pruneStartHeight + batchSize;
int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight);
LOGGER.trace(String.format("Pruning AT states data between %d and %d...", pruneStartHeight, upperPruneHeight));
int numATStatesPruned = repository.getATRepository().trimAtStates(pruneStartHeight, upperPruneHeight, rowLimitPerBatch);
repository.saveChanges();
if (numATStatesPruned > 0) {
LOGGER.trace(String.format("Pruned %d AT states data rows between blocks %d and %d",
numATStatesPruned, pruneStartHeight, upperPruneHeight));
} else {
repository.getATRepository().setAtTrimHeight(upperBatchHeight);
// No need to rebuild the latest AT states as we aren't currently synchronizing
repository.saveChanges();
LOGGER.debug(String.format("Bumping AT states trim height to %d", upperBatchHeight));
// Can we move onto next batch?
if (upperPrunableHeight > upperBatchHeight) {
pruneStartHeight = upperBatchHeight;
}
else {
// We've finished pruning
break;
}
}
}
return true;
}
public static boolean pruneBlocks(Repository repository) throws SQLException, DataException {
// Only bulk prune AT states if we have never done so before
int pruneHeight = repository.getBlockRepository().getBlockPruneHeight();
if (pruneHeight > 0) {
// Already pruned blocks
return false;
}
if (Settings.getInstance().isArchiveEnabled()) {
// Only proceed if we can see that the archiver has already finished
// This way, if the archiver failed for any reason, we can prune once it has had
// some opportunities to try again
boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository);
if (!upToDate) {
return false;
}
}
BlockData latestBlock = repository.getBlockRepository().getLastBlock();
if (latestBlock == null) {
LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning");
return false;
}
final int blockchainHeight = latestBlock.getHeight();
int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit();
int pruneStartHeight = 0;
if (Settings.getInstance().isArchiveEnabled()) {
// Archive mode - don't prune anything that hasn't been archived yet
upperPrunableHeight = Math.min(upperPrunableHeight, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1);
}
LOGGER.info("Starting bulk prune of blocks - this process could take a while... (approx. 5 mins on high spec)");
while (pruneStartHeight < upperPrunableHeight) {
// Prune all blocks up until our latest minus pruneBlockLimit
int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize();
int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight);
LOGGER.info(String.format("Pruning blocks between %d and %d...", pruneStartHeight, upperPruneHeight));
int numBlocksPruned = repository.getBlockRepository().pruneBlocks(pruneStartHeight, upperPruneHeight);
repository.saveChanges();
if (numBlocksPruned > 0) {
LOGGER.info(String.format("Pruned %d block%s between %d and %d",
numBlocksPruned, (numBlocksPruned != 1 ? "s" : ""),
pruneStartHeight, upperPruneHeight));
} else {
final int nextPruneHeight = upperPruneHeight + 1;
repository.getBlockRepository().setBlockPruneHeight(nextPruneHeight);
repository.saveChanges();
LOGGER.debug(String.format("Bumping block base prune height to %d", nextPruneHeight));
// Can we move onto next batch?
if (upperPrunableHeight > nextPruneHeight) {
pruneStartHeight = nextPruneHeight;
}
else {
// We've finished pruning
break;
}
}
}
return true;
}
public static void performMaintenance(Repository repository) throws SQLException, DataException {
try {
SplashFrame.getInstance().updateStatus("Performing maintenance...");
// Timeout if the database isn't ready for backing up after 5 minutes
// Nothing else should be using the db at this point, so a timeout shouldn't happen
long timeout = 5 * 60 * 1000L;
repository.performPeriodicMaintenance(timeout);
} catch (TimeoutException e) {
LOGGER.info("Attempt to perform maintenance failed due to timeout: {}", e.getMessage());
}
}
}

6
src/main/java/org/qortal/settings/Settings.java
View File

@ -178,6 +178,8 @@ public class Settings {
private boolean archiveEnabled = true;
/** How often to attempt archiving (ms). */
private long archiveInterval = 7171L; // milliseconds
/** Serialization version to use when building an archive */
private int defaultArchiveVersion = 1;
/** Whether to automatically bootstrap instead of syncing from genesis */
@ -927,6 +929,10 @@ public class Settings {
return this.archiveInterval;
}
public int getDefaultArchiveVersion() {
return this.defaultArchiveVersion;
}
public boolean getBootstrap() {
return this.bootstrap;

24
src/main/java/org/qortal/transform/block/BlockTransformer.java
View File

@ -312,16 +312,24 @@ public class BlockTransformer extends Transformer {
ByteArrayOutputStream atHashBytes = new ByteArrayOutputStream(atBytesLength);
long atFees = 0;
for (ATStateData atStateData : block.getATStates()) {
// Skip initial states generated by DEPLOY_AT transactions in the same block
if (atStateData.isInitial())
continue;
if (block.getAtStatesHash() != null) {
// We already have the AT states hash
atFees = blockData.getATFees();
atHashBytes.write(block.getAtStatesHash());
}
else {
// We need to build the AT states hash
for (ATStateData atStateData : block.getATStates()) {
// Skip initial states generated by DEPLOY_AT transactions in the same block
if (atStateData.isInitial())
continue;
atHashBytes.write(atStateData.getATAddress().getBytes(StandardCharsets.UTF_8));
atHashBytes.write(atStateData.getStateHash());
atHashBytes.write(Longs.toByteArray(atStateData.getFees()));
atHashBytes.write(atStateData.getATAddress().getBytes(StandardCharsets.UTF_8));
atHashBytes.write(atStateData.getStateHash());
atHashBytes.write(Longs.toByteArray(atStateData.getFees()));
atFees += atStateData.getFees();
atFees += atStateData.getFees();
}
}
bytes.write(Ints.toByteArray(blockData.getATCount()));

25
src/main/java/org/qortal/utils/BlockArchiveUtils.java
View File

@ -21,6 +21,16 @@ public class BlockArchiveUtils {
* into the HSQLDB, in order to make it SQL-compatible
* again.
* <p>
* This is only fully compatible with archives that use
* serialization version 1. For version 2 (or above),
* we are unable to import individual AT states as we
* only have a single combined hash, so the use cases
* for this are greatly limited.
* <p>
* A version 1 archive should ultimately be rebuildable
* via a resync or reindex from genesis, allowing
* access to this feature once again.
* <p>
* Note: calls discardChanges() and saveChanges(), so
* make sure that you commit any existing repository
* changes before calling this method.
@ -61,9 +71,18 @@ public class BlockArchiveUtils {
repository.getBlockRepository().save(blockInfo.getBlockData());
// Save AT state data hashes
for (ATStateData atStateData : blockInfo.getAtStates()) {
atStateData.setHeight(blockInfo.getBlockData().getHeight());
repository.getATRepository().save(atStateData);
if (blockInfo.getAtStates() != null) {
for (ATStateData atStateData : blockInfo.getAtStates()) {
atStateData.setHeight(blockInfo.getBlockData().getHeight());
repository.getATRepository().save(atStateData);
}
}
else {
// We don't have AT state hashes, so we are only importing a partial state.
// This can still be useful to allow orphaning to very old blocks, when we
// need to access other chainstate info (such as balances) at an earlier block.
// In order to do this, the orphan process must be temporarily adjusted to avoid
// orphaning AT states, as it will otherwise fail due to having no previous state.
}
} catch (DataException e) {

217
src/test/java/org/qortal/test/BlockArchiveTests.java → src/test/java/org/qortal/test/BlockArchiveV1Tests.java
View File

@ -1,6 +1,7 @@
package org.qortal.test;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -10,8 +11,6 @@ import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.*;
import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving;
import org.qortal.repository.hsqldb.HSQLDBDatabasePruning;
import org.qortal.repository.hsqldb.HSQLDBRepository;
import org.qortal.settings.Settings;
import org.qortal.test.common.AtUtils;
@ -26,7 +25,6 @@ import org.qortal.utils.NTP;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.SQLException;
@ -34,13 +32,16 @@ import java.util.List;
import static org.junit.Assert.*;
public class BlockArchiveTests extends Common {
public class BlockArchiveV1Tests extends Common {
@Before
public void beforeTest() throws DataException {
public void beforeTest() throws DataException, IllegalAccessException {
Common.useSettings("test-settings-v2-block-archive.json");
NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset());
this.deleteArchiveDirectory();
// Set default archive version to 1, so that archive builds in these tests use V2
FieldUtils.writeField(Settings.getInstance(), "defaultArchiveVersion", 1, true);
}
@After
@ -333,212 +334,6 @@ public class BlockArchiveTests extends Common {
}
}
@Test
public void testBulkArchiveAndPrune() throws DataException, SQLException {
try (final Repository repository = RepositoryManager.getRepository()) {
HSQLDBRepository hsqldb = (HSQLDBRepository) repository;
// Deploy an AT so that we have AT state data
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
byte[] creationBytes = AtUtils.buildSimpleAT();
long fundingAmount = 1_00000000L;
AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++) {
BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
}
// Assume 900 blocks are trimmed (this specifies the first untrimmed height)
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901);
repository.getATRepository().setAtTrimHeight(901);
// Check the max archive height - this should be one less than the first untrimmed height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(900, maximumArchiveHeight);
// Check the current archive height
assertEquals(0, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Write blocks 2-900 to the archive (using bulk method)
int fileSizeTarget = 428600; // Pre-calculated size of 900 blocks
assertTrue(HSQLDBDatabaseArchiving.buildBlockArchive(repository, fileSizeTarget));
// Ensure the block archive height has increased
assertEquals(901, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the SQL repository contains blocks 2 and 900...
assertNotNull(repository.getBlockRepository().fromHeight(2));
assertNotNull(repository.getBlockRepository().fromHeight(900));
// Check the current prune heights
assertEquals(0, repository.getBlockRepository().getBlockPruneHeight());
assertEquals(0, repository.getATRepository().getAtPruneHeight());
// Prior to archiving or pruning, ensure blocks 2 to 1002 and their AT states are available in the db
for (int i=2; i<=1002; i++) {
assertNotNull(repository.getBlockRepository().fromHeight(i));
List<ATStateData> atStates = repository.getATRepository().getBlockATStatesAtHeight(i);
assertNotNull(atStates);
assertEquals(1, atStates.size());
}
// Prune all the archived blocks and AT states (using bulk method)
assertTrue(HSQLDBDatabasePruning.pruneBlocks(hsqldb));
assertTrue(HSQLDBDatabasePruning.pruneATStates(hsqldb));
// Ensure the current prune heights have increased
assertEquals(901, repository.getBlockRepository().getBlockPruneHeight());
assertEquals(901, repository.getATRepository().getAtPruneHeight());
// Now ensure the SQL repository is missing blocks 2 and 900...
assertNull(repository.getBlockRepository().fromHeight(2));
assertNull(repository.getBlockRepository().fromHeight(900));
// ... but it's not missing blocks 1 and 901 (we don't prune the genesis block)
assertNotNull(repository.getBlockRepository().fromHeight(1));
assertNotNull(repository.getBlockRepository().fromHeight(901));
// Validate the latest block height in the repository
assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight());
// Ensure blocks 2-900 are all available in the archive
for (int i=2; i<=900; i++) {
assertNotNull(repository.getBlockArchiveRepository().fromHeight(i));
}
// Ensure blocks 2-900 are NOT available in the db
for (int i=2; i<=900; i++) {
assertNull(repository.getBlockRepository().fromHeight(i));
}
// Ensure blocks 901 to 1002 and their AT states are available in the db
for (int i=901; i<=1002; i++) {
assertNotNull(repository.getBlockRepository().fromHeight(i));
List<ATStateData> atStates = repository.getATRepository().getBlockATStatesAtHeight(i);
assertNotNull(atStates);
assertEquals(1, atStates.size());
}
// Ensure blocks 901 to 1002 are not available in the archive
for (int i=901; i<=1002; i++) {
assertNull(repository.getBlockArchiveRepository().fromHeight(i));
}
}
}
@Test
public void testBulkArchiveAndPruneMultipleFiles() throws DataException, SQLException {
try (final Repository repository = RepositoryManager.getRepository()) {
HSQLDBRepository hsqldb = (HSQLDBRepository) repository;
// Deploy an AT so that we have AT state data
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
byte[] creationBytes = AtUtils.buildSimpleAT();
long fundingAmount = 1_00000000L;
AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++) {
BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
}
// Assume 900 blocks are trimmed (this specifies the first untrimmed height)
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901);
repository.getATRepository().setAtTrimHeight(901);
// Check the max archive height - this should be one less than the first untrimmed height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(900, maximumArchiveHeight);
// Check the current archive height
assertEquals(0, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Write blocks 2-900 to the archive (using bulk method)
int fileSizeTarget = 42360; // Pre-calculated size of approx 90 blocks
assertTrue(HSQLDBDatabaseArchiving.buildBlockArchive(repository, fileSizeTarget));
// Ensure 10 archive files have been created
Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive");
assertEquals(10, new File(archivePath.toString()).list().length);
// Check the files exist
assertTrue(Files.exists(Paths.get(archivePath.toString(), "2-90.dat")));
assertTrue(Files.exists(Paths.get(archivePath.toString(), "91-179.dat")));
assertTrue(Files.exists(Paths.get(archivePath.toString(), "180-268.dat")));
assertTrue(Files.exists(Paths.get(archivePath.toString(), "269-357.dat")));
assertTrue(Files.exists(Paths.get(archivePath.toString(), "358-446.dat")));
assertTrue(Files.exists(Paths.get(archivePath.toString(), "447-535.dat")));
assertTrue(Files.exists(Paths.get(archivePath.toString(), "536-624.dat")));
assertTrue(Files.exists(Paths.get(archivePath.toString(), "625-713.dat")));
assertTrue(Files.exists(Paths.get(archivePath.toString(), "714-802.dat")));
assertTrue(Files.exists(Paths.get(archivePath.toString(), "803-891.dat")));
// Ensure the block archive height has increased
// It won't be as high as 901, because blocks 892-901 were too small to reach the file size
// target of the 11th file
assertEquals(892, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the SQL repository contains blocks 2 and 891...
assertNotNull(repository.getBlockRepository().fromHeight(2));
assertNotNull(repository.getBlockRepository().fromHeight(891));
// Check the current prune heights
assertEquals(0, repository.getBlockRepository().getBlockPruneHeight());
assertEquals(0, repository.getATRepository().getAtPruneHeight());
// Prior to archiving or pruning, ensure blocks 2 to 1002 and their AT states are available in the db
for (int i=2; i<=1002; i++) {
assertNotNull(repository.getBlockRepository().fromHeight(i));
List<ATStateData> atStates = repository.getATRepository().getBlockATStatesAtHeight(i);
assertNotNull(atStates);
assertEquals(1, atStates.size());
}
// Prune all the archived blocks and AT states (using bulk method)
assertTrue(HSQLDBDatabasePruning.pruneBlocks(hsqldb));
assertTrue(HSQLDBDatabasePruning.pruneATStates(hsqldb));
// Ensure the current prune heights have increased
assertEquals(892, repository.getBlockRepository().getBlockPruneHeight());
assertEquals(892, repository.getATRepository().getAtPruneHeight());
// Now ensure the SQL repository is missing blocks 2 and 891...
assertNull(repository.getBlockRepository().fromHeight(2));
assertNull(repository.getBlockRepository().fromHeight(891));
// ... but it's not missing blocks 1 and 901 (we don't prune the genesis block)
assertNotNull(repository.getBlockRepository().fromHeight(1));
assertNotNull(repository.getBlockRepository().fromHeight(892));
// Validate the latest block height in the repository
assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight());
// Ensure blocks 2-891 are all available in the archive
for (int i=2; i<=891; i++) {
assertNotNull(repository.getBlockArchiveRepository().fromHeight(i));
}
// Ensure blocks 2-891 are NOT available in the db
for (int i=2; i<=891; i++) {
assertNull(repository.getBlockRepository().fromHeight(i));
}
// Ensure blocks 892 to 1002 and their AT states are available in the db
for (int i=892; i<=1002; i++) {
assertNotNull(repository.getBlockRepository().fromHeight(i));
List<ATStateData> atStates = repository.getATRepository().getBlockATStatesAtHeight(i);
assertNotNull(atStates);
assertEquals(1, atStates.size());
}
// Ensure blocks 892 to 1002 are not available in the archive
for (int i=892; i<=1002; i++) {
assertNull(repository.getBlockArchiveRepository().fromHeight(i));
}
}
}
@Test
public void testTrimArchivePruneAndOrphan() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {

504
src/test/java/org/qortal/test/BlockArchiveV2Tests.java Normal file
View File

@ -0,0 +1,504 @@
package org.qortal.test;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.controller.BlockMinter;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.*;
import org.qortal.repository.hsqldb.HSQLDBRepository;
import org.qortal.settings.Settings;
import org.qortal.test.common.AtUtils;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.transaction.DeployAtTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.utils.BlockArchiveUtils;
import org.qortal.utils.NTP;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.sql.SQLException;
import java.util.List;
import static org.junit.Assert.*;
public class BlockArchiveV2Tests extends Common {
@Before
public void beforeTest() throws DataException, IllegalAccessException {
Common.useSettings("test-settings-v2-block-archive.json");
NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset());
this.deleteArchiveDirectory();
// Set default archive version to 2, so that archive builds in these tests use V2
FieldUtils.writeField(Settings.getInstance(), "defaultArchiveVersion", 2, true);
}
@After
public void afterTest() throws DataException {
this.deleteArchiveDirectory();
}
@Test
public void testWriter() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++) {
BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
}
// 900 blocks are trimmed (this specifies the first untrimmed height)
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901);
repository.getATRepository().setAtTrimHeight(901);
// Check the max archive height - this should be one less than the first untrimmed height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(900, maximumArchiveHeight);
// Write blocks 2-900 to the archive
BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result);
// Make sure that the archive contains the correct number of blocks
assertEquals(900 - 1, writer.getWrittenCount());
// Increment block archive height
repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount());
repository.saveChanges();
assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the file exists
File outputFile = writer.getOutputPath().toFile();
assertTrue(outputFile.exists());
}
}
@Test
public void testWriterAndReader() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++) {
BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
}
// 900 blocks are trimmed (this specifies the first untrimmed height)
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901);
repository.getATRepository().setAtTrimHeight(901);
// Check the max archive height - this should be one less than the first untrimmed height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(900, maximumArchiveHeight);
// Write blocks 2-900 to the archive
BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result);
// Make sure that the archive contains the correct number of blocks
assertEquals(900 - 1, writer.getWrittenCount());
// Increment block archive height
repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount());
repository.saveChanges();
assertEquals(900 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the file exists
File outputFile = writer.getOutputPath().toFile();
assertTrue(outputFile.exists());
// Read block 2 from the archive
BlockArchiveReader reader = BlockArchiveReader.getInstance();
BlockTransformation block2Info = reader.fetchBlockAtHeight(2);
BlockData block2ArchiveData = block2Info.getBlockData();
// Read block 2 from the repository
BlockData block2RepositoryData = repository.getBlockRepository().fromHeight(2);
// Ensure the values match
assertEquals(block2ArchiveData.getHeight(), block2RepositoryData.getHeight());
assertArrayEquals(block2ArchiveData.getSignature(), block2RepositoryData.getSignature());
// Test some values in the archive
assertEquals(1, block2ArchiveData.getOnlineAccountsCount());
// Read block 900 from the archive
BlockTransformation block900Info = reader.fetchBlockAtHeight(900);
BlockData block900ArchiveData = block900Info.getBlockData();
// Read block 900 from the repository
BlockData block900RepositoryData = repository.getBlockRepository().fromHeight(900);
// Ensure the values match
assertEquals(block900ArchiveData.getHeight(), block900RepositoryData.getHeight());
assertArrayEquals(block900ArchiveData.getSignature(), block900RepositoryData.getSignature());
// Test some values in the archive
assertEquals(1, block900ArchiveData.getOnlineAccountsCount());
}
}
@Test
public void testArchivedAtStates() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Deploy an AT so that we have AT state data
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
byte[] creationBytes = AtUtils.buildSimpleAT();
long fundingAmount = 1_00000000L;
DeployAtTransaction deployAtTransaction = AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
String atAddress = deployAtTransaction.getATAccount().getAddress();
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++) {
BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
}
// 9 blocks are trimmed (this specifies the first untrimmed height)
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(10);
repository.getATRepository().setAtTrimHeight(10);
// Check the max archive height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(9, maximumArchiveHeight);
// Write blocks 2-9 to the archive
BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result);
// Make sure that the archive contains the correct number of blocks
assertEquals(9 - 1, writer.getWrittenCount());
// Increment block archive height
repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount());
repository.saveChanges();
assertEquals(9 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the file exists
File outputFile = writer.getOutputPath().toFile();
assertTrue(outputFile.exists());
// Check blocks 3-9
for (Integer testHeight = 2; testHeight <= 9; testHeight++) {
// Read a block from the archive
BlockArchiveReader reader = BlockArchiveReader.getInstance();
BlockTransformation blockInfo = reader.fetchBlockAtHeight(testHeight);
BlockData archivedBlockData = blockInfo.getBlockData();
byte[] archivedAtStateHash = blockInfo.getAtStatesHash();
List<TransactionData> archivedTransactions = blockInfo.getTransactions();
// Read the same block from the repository
BlockData repositoryBlockData = repository.getBlockRepository().fromHeight(testHeight);
ATStateData repositoryAtStateData = repository.getATRepository().getATStateAtHeight(atAddress, testHeight);
// Ensure the repository has full AT state data
assertNotNull(repositoryAtStateData.getStateHash());
assertNotNull(repositoryAtStateData.getStateData());
// Check the archived AT state
if (testHeight == 2) {
assertEquals(1, archivedTransactions.size());
assertEquals(Transaction.TransactionType.DEPLOY_AT, archivedTransactions.get(0).getType());
}
else {
// Blocks 3+ shouldn't have any transactions
assertTrue(archivedTransactions.isEmpty());
}
// Ensure the archive has the AT states hash
assertNotNull(archivedAtStateHash);
// Also check the online accounts count and height
assertEquals(1, archivedBlockData.getOnlineAccountsCount());
assertEquals(testHeight, archivedBlockData.getHeight());
// Ensure the values match
assertEquals(archivedBlockData.getHeight(), repositoryBlockData.getHeight());
assertArrayEquals(archivedBlockData.getSignature(), repositoryBlockData.getSignature());
assertEquals(archivedBlockData.getOnlineAccountsCount(), repositoryBlockData.getOnlineAccountsCount());
assertArrayEquals(archivedBlockData.getMinterSignature(), repositoryBlockData.getMinterSignature());
assertEquals(archivedBlockData.getATCount(), repositoryBlockData.getATCount());
assertEquals(archivedBlockData.getOnlineAccountsCount(), repositoryBlockData.getOnlineAccountsCount());
assertArrayEquals(archivedBlockData.getReference(), repositoryBlockData.getReference());
assertEquals(archivedBlockData.getTimestamp(), repositoryBlockData.getTimestamp());
assertEquals(archivedBlockData.getATFees(), repositoryBlockData.getATFees());
assertEquals(archivedBlockData.getTotalFees(), repositoryBlockData.getTotalFees());
assertEquals(archivedBlockData.getTransactionCount(), repositoryBlockData.getTransactionCount());
assertArrayEquals(archivedBlockData.getTransactionsSignature(), repositoryBlockData.getTransactionsSignature());
// TODO: build atStatesHash and compare against value in archive
}
// Check block 10 (unarchived)
BlockArchiveReader reader = BlockArchiveReader.getInstance();
BlockTransformation blockInfo = reader.fetchBlockAtHeight(10);
assertNull(blockInfo);
}
}
@Test
public void testArchiveAndPrune() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Deploy an AT so that we have AT state data
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
byte[] creationBytes = AtUtils.buildSimpleAT();
long fundingAmount = 1_00000000L;
AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++) {
BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
}
// Assume 900 blocks are trimmed (this specifies the first untrimmed height)
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(901);
repository.getATRepository().setAtTrimHeight(901);
// Check the max archive height - this should be one less than the first untrimmed height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(900, maximumArchiveHeight);
// Write blocks 2-900 to the archive
BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result);
// Make sure that the archive contains the correct number of blocks
assertEquals(900 - 1, writer.getWrittenCount());
// Increment block archive height
repository.getBlockArchiveRepository().setBlockArchiveHeight(901);
repository.saveChanges();
assertEquals(901, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the file exists
File outputFile = writer.getOutputPath().toFile();
assertTrue(outputFile.exists());
// Ensure the SQL repository contains blocks 2 and 900...
assertNotNull(repository.getBlockRepository().fromHeight(2));
assertNotNull(repository.getBlockRepository().fromHeight(900));
// Prune all the archived blocks
int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 900);
assertEquals(900-1, numBlocksPruned);
repository.getBlockRepository().setBlockPruneHeight(901);
// Prune the AT states for the archived blocks
repository.getATRepository().rebuildLatestAtStates(900);
repository.saveChanges();
int numATStatesPruned = repository.getATRepository().pruneAtStates(0, 900);
assertEquals(900-2, numATStatesPruned); // Minus 1 for genesis block, and another for the latest AT state
repository.getATRepository().setAtPruneHeight(901);
// Now ensure the SQL repository is missing blocks 2 and 900...
assertNull(repository.getBlockRepository().fromHeight(2));
assertNull(repository.getBlockRepository().fromHeight(900));
// ... but it's not missing blocks 1 and 901 (we don't prune the genesis block)
assertNotNull(repository.getBlockRepository().fromHeight(1));
assertNotNull(repository.getBlockRepository().fromHeight(901));
// Validate the latest block height in the repository
assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight());
}
}
@Test
public void testTrimArchivePruneAndOrphan() throws DataException, InterruptedException, TransformationException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Deploy an AT so that we have AT state data
PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice");
byte[] creationBytes = AtUtils.buildSimpleAT();
long fundingAmount = 1_00000000L;
AtUtils.doDeployAT(repository, deployer, creationBytes, fundingAmount);
// Mint some blocks so that we are able to archive them later
for (int i = 0; i < 1000; i++) {
BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
}
// Make sure that block 500 has full AT state data and data hash
List<ATStateData> block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500);
ATStateData atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500);
assertNotNull(atStatesData.getStateHash());
assertNotNull(atStatesData.getStateData());
// Trim the first 500 blocks
repository.getBlockRepository().trimOldOnlineAccountsSignatures(0, 500);
repository.getBlockRepository().setOnlineAccountsSignaturesTrimHeight(501);
repository.getATRepository().rebuildLatestAtStates(500);
repository.getATRepository().trimAtStates(0, 500, 1000);
repository.getATRepository().setAtTrimHeight(501);
// Now block 499 should only have the AT state data hash
List<ATStateData> block499AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(499);
atStatesData = repository.getATRepository().getATStateAtHeight(block499AtStatesData.get(0).getATAddress(), 499);
assertNotNull(atStatesData.getStateHash());
assertNull(atStatesData.getStateData());
// ... but block 500 should have the full data (due to being retained as the "latest" AT state in the trimmed range
block500AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(500);
atStatesData = repository.getATRepository().getATStateAtHeight(block500AtStatesData.get(0).getATAddress(), 500);
assertNotNull(atStatesData.getStateHash());
assertNotNull(atStatesData.getStateData());
// ... and block 501 should also have the full data
List<ATStateData> block501AtStatesData = repository.getATRepository().getBlockATStatesAtHeight(501);
atStatesData = repository.getATRepository().getATStateAtHeight(block501AtStatesData.get(0).getATAddress(), 501);
assertNotNull(atStatesData.getStateHash());
assertNotNull(atStatesData.getStateData());
// Check the max archive height - this should be one less than the first untrimmed height
final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository);
assertEquals(500, maximumArchiveHeight);
BlockData block3DataPreArchive = repository.getBlockRepository().fromHeight(3);
// Write blocks 2-500 to the archive
BlockArchiveWriter writer = new BlockArchiveWriter(0, maximumArchiveHeight, repository);
writer.setShouldEnforceFileSizeTarget(false); // To avoid the need to pre-calculate file sizes
BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
assertEquals(BlockArchiveWriter.BlockArchiveWriteResult.OK, result);
// Make sure that the archive contains the correct number of blocks
assertEquals(500 - 1, writer.getWrittenCount()); // -1 for the genesis block
// Increment block archive height
repository.getBlockArchiveRepository().setBlockArchiveHeight(writer.getWrittenCount());
repository.saveChanges();
assertEquals(500 - 1, repository.getBlockArchiveRepository().getBlockArchiveHeight());
// Ensure the file exists
File outputFile = writer.getOutputPath().toFile();
assertTrue(outputFile.exists());
// Ensure the SQL repository contains blocks 2 and 500...
assertNotNull(repository.getBlockRepository().fromHeight(2));
assertNotNull(repository.getBlockRepository().fromHeight(500));
// Prune all the archived blocks
int numBlocksPruned = repository.getBlockRepository().pruneBlocks(0, 500);
assertEquals(500-1, numBlocksPruned);
repository.getBlockRepository().setBlockPruneHeight(501);
// Prune the AT states for the archived blocks
repository.getATRepository().rebuildLatestAtStates(500);
repository.saveChanges();
int numATStatesPruned = repository.getATRepository().pruneAtStates(2, 500);
assertEquals(498, numATStatesPruned); // Minus 1 for genesis block, and another for the latest AT state
repository.getATRepository().setAtPruneHeight(501);
// Now ensure the SQL repository is missing blocks 2 and 500...
assertNull(repository.getBlockRepository().fromHeight(2));
assertNull(repository.getBlockRepository().fromHeight(500));
// ... but it's not missing blocks 1 and 501 (we don't prune the genesis block)
assertNotNull(repository.getBlockRepository().fromHeight(1));
assertNotNull(repository.getBlockRepository().fromHeight(501));
// Validate the latest block height in the repository
assertEquals(1002, (int) repository.getBlockRepository().getLastBlock().getHeight());
// Now orphan some unarchived blocks.
BlockUtils.orphanBlocks(repository, 500);
assertEquals(502, (int) repository.getBlockRepository().getLastBlock().getHeight());
// We're close to the lower limit of the SQL database now, so
// we need to import some blocks from the archive
BlockArchiveUtils.importFromArchive(401, 500, repository);
// Ensure the SQL repository now contains block 401 but not 400...
assertNotNull(repository.getBlockRepository().fromHeight(401));
assertNull(repository.getBlockRepository().fromHeight(400));
// Import the remaining 399 blocks
BlockArchiveUtils.importFromArchive(2, 400, repository);
// Verify that block 3 matches the original
BlockData block3DataPostArchive = repository.getBlockRepository().fromHeight(3);
assertArrayEquals(block3DataPreArchive.getSignature(), block3DataPostArchive.getSignature());
assertEquals(block3DataPreArchive.getHeight(), block3DataPostArchive.getHeight());
// Orphan 2 more block, which should be the last one that is possible to be orphaned
// TODO: figure out why this is 1 block more than in the equivalent block archive V1 test
BlockUtils.orphanBlocks(repository, 2);
// Orphan another block, which should fail
Exception exception = null;
try {
BlockUtils.orphanBlocks(repository, 1);
} catch (DataException e) {
exception = e;
}
// Ensure that a DataException is thrown because there is no more AT states data available
assertNotNull(exception);
assertEquals(DataException.class, exception.getClass());
// FUTURE: we may be able to retain unique AT states when trimming, to avoid this exception
// and allow orphaning back through blocks with trimmed AT states.
}
}
/**
* Many nodes are missing an ATStatesHeightIndex due to an earlier bug
* In these cases we disable archiving and pruning as this index is a
* very essential component in these processes.
*/
@Test
public void testMissingAtStatesHeightIndex() throws DataException, SQLException {
try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) {
// Firstly check that we're able to prune or archive when the index exists
assertTrue(repository.getATRepository().hasAtStatesHeightIndex());
assertTrue(RepositoryManager.canArchiveOrPrune());
// Delete the index
repository.prepareStatement("DROP INDEX ATSTATESHEIGHTINDEX").execute();
// Ensure check that we're unable to prune or archive when the index doesn't exist
assertFalse(repository.getATRepository().hasAtStatesHeightIndex());
assertFalse(RepositoryManager.canArchiveOrPrune());
}
}
private void deleteArchiveDirectory() {
// Delete archive directory if exists
Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath();
try {
FileUtils.deleteDirectory(archivePath.toFile());
} catch (IOException e) {
}
}
}

3
src/test/resources/test-settings-v2-block-archive.json
View File

@ -9,5 +9,6 @@
"testNtpOffset": 0,
"minPeers": 0,
"pruneBlockLimit": 100,
"repositoryPath": "dbtest"
"repositoryPath": "dbtest",
"defaultArchiveVersion": 1
}