diff --git a/src/main/java/org/qortal/api/resource/AdminResource.java b/src/main/java/org/qortal/api/resource/AdminResource.java index 88dd0065..3e666fe4 100644 --- a/src/main/java/org/qortal/api/resource/AdminResource.java +++ b/src/main/java/org/qortal/api/resource/AdminResource.java @@ -35,6 +35,7 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.RollingFileAppender; import org.qortal.account.Account; @@ -67,6 +68,8 @@ import com.google.common.collect.Lists; @Tag(name = "Admin") public class AdminResource { + private static final Logger LOGGER = LogManager.getLogger(AdminResource.class); + private static final int MAX_LOG_LINES = 500; @Context @@ -459,6 +462,23 @@ public class AdminResource { if (targetHeight <= 0 || targetHeight > Controller.getInstance().getChainHeight()) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT); + // Make sure we're not orphaning as far back as the archived blocks + // FUTURE: we could support this by first importing earlier blocks from the archive + if (Settings.getInstance().isPruningEnabled() || + Settings.getInstance().isArchiveEnabled()) { + + try (final Repository repository = RepositoryManager.getRepository()) { + // Find the first unarchived block + int oldestBlock = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + // Add some extra blocks just in case we're currently archiving/pruning + oldestBlock += 100; + if (targetHeight <= oldestBlock) { + LOGGER.info("Unable to orphan beyond block {} because it is archived", oldestBlock); + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT); + } + } + } + if (BlockChain.orphan(targetHeight)) return "true"; else diff --git a/src/main/java/org/qortal/api/resource/BlocksResource.java b/src/main/java/org/qortal/api/resource/BlocksResource.java index 8920ecc1..6dc13c8a 100644 --- a/src/main/java/org/qortal/api/resource/BlocksResource.java +++ b/src/main/java/org/qortal/api/resource/BlocksResource.java @@ -15,6 +15,8 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; import java.util.List; import javax.servlet.http.HttpServletRequest; @@ -33,11 +35,13 @@ import org.qortal.api.ApiExceptionFactory; import org.qortal.api.model.BlockMintingInfo; import org.qortal.api.model.BlockSignerSummary; import org.qortal.block.Block; +import org.qortal.controller.Controller; import org.qortal.crypto.Crypto; import org.qortal.data.account.AccountData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.transaction.TransactionData; +import org.qortal.repository.BlockArchiveReader; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -81,11 +85,19 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { + // Check the database first BlockData blockData = repository.getBlockRepository().fromSignature(signature); - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (blockData != null) { + return blockData; + } - return blockData; + // Not found, so try the block archive + blockData = repository.getBlockArchiveRepository().fromSignature(signature); + if (blockData != null) { + return blockData; + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -116,16 +128,24 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { + + // Check the database first BlockData blockData = repository.getBlockRepository().fromSignature(signature); - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (blockData != null) { + Block block = new Block(repository, blockData); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); + bytes.write(BlockTransformer.toBytes(block)); + return Base58.encode(bytes.toByteArray()); + } - Block block = new Block(repository, blockData); - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); - bytes.write(BlockTransformer.toBytes(block)); - return Base58.encode(bytes.toByteArray()); + // Not found, so try the block archive + byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, repository); + if (bytes != null) { + return Base58.encode(bytes); + } + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); } catch (TransformationException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); } catch (DataException | IOException e) { @@ -170,8 +190,12 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { - if (repository.getBlockRepository().getHeightFromSignature(signature) == 0) + // Check if the block exists in either the database or archive + if (repository.getBlockRepository().getHeightFromSignature(signature) == 0 && + repository.getBlockArchiveRepository().getHeightFromSignature(signature) == 0) { + // Not found in either the database or archive throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } return repository.getBlockRepository().getTransactionsFromSignature(signature, limit, offset, reverse); } catch (DataException e) { @@ -200,7 +224,19 @@ public class BlocksResource { }) public BlockData getFirstBlock() { try (final Repository repository = RepositoryManager.getRepository()) { - return repository.getBlockRepository().fromHeight(1); + // Check the database first + BlockData blockData = repository.getBlockRepository().fromHeight(1); + if (blockData != null) { + return blockData; + } + + // Try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(1); + if (blockData != null) { + return blockData; + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -262,17 +298,28 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { + BlockData childBlockData = null; + + // Check if block exists in database BlockData blockData = repository.getBlockRepository().fromSignature(signature); + if (blockData != null) { + return repository.getBlockRepository().fromReference(signature); + } - // Check block exists - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); - - BlockData childBlockData = repository.getBlockRepository().fromReference(signature); + // Not found, so try the archive + // This also checks that the parent block exists + // It will return null if either the parent or child don't exit + childBlockData = repository.getBlockArchiveRepository().fromReference(signature); // Check child block exists - if (childBlockData == null) + if (childBlockData == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + + // Check child block's reference matches the supplied signature + if (!Arrays.equals(childBlockData.getReference(), signature)) { + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } return childBlockData; } catch (DataException e) { @@ -338,13 +385,20 @@ public class BlocksResource { } try (final Repository repository = RepositoryManager.getRepository()) { + // Firstly check the database BlockData blockData = repository.getBlockRepository().fromSignature(signature); + if (blockData != null) { + return blockData.getHeight(); + } - // Check block exists - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + // Not found, so try the archive + blockData = repository.getBlockArchiveRepository().fromSignature(signature); + if (blockData != null) { + return blockData.getHeight(); + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); - return blockData.getHeight(); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -371,11 +425,20 @@ public class BlocksResource { }) public BlockData getByHeight(@PathParam("height") int height) { try (final Repository repository = RepositoryManager.getRepository()) { + // Firstly check the database BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (blockData != null) { + return blockData; + } + + // Not found, so try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + if (blockData != null) { + return blockData; + } + + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); - return blockData; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -402,12 +465,31 @@ public class BlocksResource { }) public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) { try (final Repository repository = RepositoryManager.getRepository()) { + // Try the database BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + if (blockData == null) { + + // Not found, so try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + if (blockData == null) { + + // Still not found + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + } Block block = new Block(repository, blockData); BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference()); + if (parentBlockData == null) { + // Parent block not found - try the archive + parentBlockData = repository.getBlockArchiveRepository().fromSignature(blockData.getReference()); + if (parentBlockData == null) { + + // Still not found + throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } + } + int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey()); if (minterLevel == 0) // This may be unavailable when requesting a trimmed block @@ -454,13 +536,26 @@ public class BlocksResource { }) public BlockData getByTimestamp(@PathParam("timestamp") long timestamp) { try (final Repository repository = RepositoryManager.getRepository()) { - int height = repository.getBlockRepository().getHeightFromTimestamp(timestamp); - if (height == 0) - throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + BlockData blockData = null; - BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) + // Try the Blocks table + int height = repository.getBlockRepository().getHeightFromTimestamp(timestamp); + if (height > 0) { + // Found match in Blocks table + return repository.getBlockRepository().fromHeight(height); + } + + // Not found in Blocks table, so try the archive + height = repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp); + if (height > 0) { + // Found match in archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + } + + // Ensure block exists + if (blockData == null) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN); + } return blockData; } catch (DataException e) { @@ -497,9 +592,14 @@ public class BlocksResource { for (/* count already set */; count > 0; --count, ++height) { BlockData blockData = repository.getBlockRepository().fromHeight(height); - if (blockData == null) - // Run out of blocks! - break; + if (blockData == null) { + // Not found - try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(height); + if (blockData == null) { + // Run out of blocks! + break; + } + } blocks.add(blockData); } @@ -544,7 +644,29 @@ public class BlocksResource { if (accountData == null || accountData.getPublicKey() == null) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.PUBLIC_KEY_NOT_FOUND); - return repository.getBlockRepository().getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse); + + List summaries = repository.getBlockRepository() + .getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse); + + // Add any from the archive + List archivedSummaries = repository.getBlockArchiveRepository() + .getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse); + if (archivedSummaries != null && !archivedSummaries.isEmpty()) { + summaries.addAll(archivedSummaries); + } + else { + summaries = archivedSummaries; + } + + // Sort the results (because they may have been obtained from two places) + if (reverse != null && reverse) { + summaries.sort((s1, s2) -> Integer.valueOf(s2.getHeight()).compareTo(Integer.valueOf(s1.getHeight()))); + } + else { + summaries.sort(Comparator.comparing(s -> Integer.valueOf(s.getHeight()))); + } + + return summaries; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -580,7 +702,8 @@ public class BlocksResource { if (!Crypto.isValidAddress(address)) throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS); - return repository.getBlockRepository().getBlockSigners(addresses, limit, offset, reverse); + // This method pulls data from both Blocks and BlockArchive, so no need to query serparately + return repository.getBlockArchiveRepository().getBlockSigners(addresses, limit, offset, reverse); } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } @@ -620,7 +743,76 @@ public class BlocksResource { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); try (final Repository repository = RepositoryManager.getRepository()) { - return repository.getBlockRepository().getBlockSummaries(startHeight, endHeight, count); + + /* + * start end count result + * 10 40 null blocks 10 to 39 (excludes end block, ignore count) + * + * null null null blocks 1 to 50 (assume count=50, maybe start=1) + * 30 null null blocks 30 to 79 (assume count=50) + * 30 null 10 blocks 30 to 39 + * + * null null 50 last 50 blocks? so if max(blocks.height) is 200, then blocks 151 to 200 + * null 200 null blocks 150 to 199 (excludes end block, assume count=50) + * null 200 10 blocks 190 to 199 (excludes end block) + */ + + List blockSummaries = new ArrayList<>(); + + // Use the latest X blocks if only a count is specified + if (startHeight == null && endHeight == null && count != null) { + BlockData chainTip = Controller.getInstance().getChainTip(); + startHeight = chainTip.getHeight() - count; + endHeight = chainTip.getHeight(); + } + + // ... otherwise default the start height to 1 + if (startHeight == null && endHeight == null) { + startHeight = 1; + } + + // Default the count to 50 + if (count == null) { + count = 50; + } + + // If both a start and end height exist, ignore the count + if (startHeight != null && endHeight != null) { + if (startHeight > 0 && endHeight > 0) { + count = Integer.MAX_VALUE; + } + } + + // Derive start height from end height if missing + if (startHeight == null || startHeight == 0) { + if (endHeight != null && endHeight > 0) { + if (count != null) { + startHeight = endHeight - count; + } + } + } + + for (/* count already set */; count > 0; --count, ++startHeight) { + if (endHeight != null && startHeight >= endHeight) { + break; + } + BlockData blockData = repository.getBlockRepository().fromHeight(startHeight); + if (blockData == null) { + // Not found - try the archive + blockData = repository.getBlockArchiveRepository().fromHeight(startHeight); + if (blockData == null) { + // Run out of blocks! + break; + } + } + + if (blockData != null) { + BlockSummaryData blockSummaryData = new BlockSummaryData(blockData); + blockSummaries.add(blockSummaryData); + } + } + + return blockSummaries; } catch (DataException e) { throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index f9d48c70..f03dd504 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -83,20 +83,14 @@ import org.qortal.network.message.OnlineAccountsMessage; import org.qortal.network.message.SignaturesMessage; import org.qortal.network.message.TransactionMessage; import org.qortal.network.message.TransactionSignaturesMessage; -import org.qortal.repository.DataException; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryFactory; -import org.qortal.repository.RepositoryManager; +import org.qortal.repository.*; import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.ValidationResult; -import org.qortal.utils.Base58; -import org.qortal.utils.ByteArray; -import org.qortal.utils.NTP; -import org.qortal.utils.Triple; +import org.qortal.utils.*; import com.google.common.primitives.Longs; @@ -414,6 +408,7 @@ public class Controller extends Thread { try { RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl()); RepositoryManager.setRepositoryFactory(repositoryFactory); + RepositoryManager.archive(); RepositoryManager.prune(); } catch (DataException e) { // If exception has no cause then repository is in use by some other process. @@ -1286,6 +1281,34 @@ public class Controller extends Thread { } } + // If we have no block data, we should check the archive in case it's there + if (blockData == null) { + if (Settings.getInstance().isArchiveEnabled()) { + byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, repository); + if (bytes != null) { + CachedBlockMessage blockMessage = new CachedBlockMessage(bytes); + blockMessage.setId(message.getId()); + + // This call also causes the other needed data to be pulled in from repository + if (!peer.sendMessage(blockMessage)) { + peer.disconnect("failed to send block"); + // Don't fall-through to caching because failure to send might be from failure to build message + return; + } + + // If request is for a recent block, cache it + if (getChainHeight() - blockData.getHeight() <= blockCacheSize) { + this.stats.getBlockMessageStats.cacheFills.incrementAndGet(); + + this.blockMessageCache.put(new ByteArray(blockData.getSignature()), blockMessage); + } + + // Sent successfully from archive, so nothing more to do + return; + } + } + } + if (blockData == null) { // We don't have this block this.stats.getBlockMessageStats.unknownBlocks.getAndIncrement(); diff --git a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java index 30d7f136..1493f478 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesPruner.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesPruner.java @@ -18,15 +18,24 @@ public class AtStatesPruner implements Runnable { public void run() { Thread.currentThread().setName("AT States pruner"); + boolean archiveMode = false; if (!Settings.getInstance().isPruningEnabled()) { - return; + // Pruning isn't enabled, but we might want to prune for the purposes of archiving + if (!Settings.getInstance().isArchiveEnabled()) { + // No pruning or archiving, so we must not prune anything + return; + } + else { + // We're allowed to prune blocks that have already been archived + archiveMode = true; + } } try (final Repository repository = RepositoryManager.getRepository()) { int pruneStartHeight = repository.getATRepository().getAtPruneHeight(); + repository.discardChanges(); repository.getATRepository().rebuildLatestAtStates(); - repository.saveChanges(); while (!Controller.isStopping()) { repository.discardChanges(); @@ -43,7 +52,14 @@ public class AtStatesPruner implements Runnable { // Prune AT states for all blocks up until our latest minus pruneBlockLimit final int ourLatestHeight = chainTip.getHeight(); - final int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + + // In archive mode we are only allowed to trim blocks that have already been archived + if (archiveMode) { + upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1; + + // TODO: validate that the actual archived data exists before pruning it? + } int upperBatchHeight = pruneStartHeight + Settings.getInstance().getAtStatesPruneBatchSize(); int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); diff --git a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java index ed02ee47..98a1a889 100644 --- a/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java +++ b/src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java @@ -21,8 +21,8 @@ public class AtStatesTrimmer implements Runnable { try (final Repository repository = RepositoryManager.getRepository()) { int trimStartHeight = repository.getATRepository().getAtTrimHeight(); + repository.discardChanges(); repository.getATRepository().rebuildLatestAtStates(); - repository.saveChanges(); while (!Controller.isStopping()) { repository.discardChanges(); diff --git a/src/main/java/org/qortal/controller/repository/BlockArchiver.java b/src/main/java/org/qortal/controller/repository/BlockArchiver.java new file mode 100644 index 00000000..f7bafe7d --- /dev/null +++ b/src/main/java/org/qortal/controller/repository/BlockArchiver.java @@ -0,0 +1,105 @@ +package org.qortal.controller.repository; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockData; +import org.qortal.repository.*; +import org.qortal.settings.Settings; +import org.qortal.transform.TransformationException; +import org.qortal.utils.NTP; + +import java.io.IOException; + +public class BlockArchiver implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(BlockArchiver.class); + + private static final long INITIAL_SLEEP_PERIOD = 0L; // TODO: 5 * 60 * 1000L + 1234L; // ms + + public void run() { + Thread.currentThread().setName("Block archiver"); + + if (!Settings.getInstance().isArchiveEnabled()) { + return; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + int startHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + + // Don't even start building until initial rush has ended + Thread.sleep(INITIAL_SLEEP_PERIOD); + + LOGGER.info("Starting block archiver..."); + + while (!Controller.isStopping()) { + repository.discardChanges(); + + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, true); + + Thread.sleep(Settings.getInstance().getArchiveInterval()); + + BlockData chainTip = Controller.getInstance().getChainTip(); + if (chainTip == null || NTP.getTime() == null) { + continue; + } + + // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages + if (Controller.getInstance().isSynchronizing()) { + continue; + } + + // Don't attempt to archive if we're not synced yet + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) { + continue; + } + + + // Build cache of blocks + try { + BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository); + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + switch (result) { + case OK: + // Increment block archive height + startHeight += writer.getWrittenCount(); + repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight); + repository.saveChanges(); + break; + + case STOPPING: + return; + + case BLOCK_LIMIT_REACHED: + // We've reached the limit of the blocks we can archive + // Sleep for a while to allow more to become available + case NOT_ENOUGH_BLOCKS: + // We didn't reach our file size target, so that must mean that we don't have enough blocks + // yet or something went wrong. Sleep for a while and then try again. + Thread.sleep(60 * 60 * 1000L); // 1 hour + break; + + case BLOCK_NOT_FOUND: + // We tried to archive a block that didn't exist. This is a major failure and likely means + // that a bootstrap or re-sync is needed. Try again every minute until then. + LOGGER.info("Error: block not found when building archive. If this error persists, " + + "a bootstrap or re-sync may be needed."); + Thread.sleep( 60 * 1000L); // 1 minute + break; + } + + } catch (IOException | TransformationException e) { + LOGGER.info("Caught exception when creating block cache", e); + } + + } + } catch (DataException e) { + LOGGER.info("Caught exception when creating block cache", e); + } catch (InterruptedException e) { + // Do nothing + } + + } + +} diff --git a/src/main/java/org/qortal/controller/repository/BlockPruner.java b/src/main/java/org/qortal/controller/repository/BlockPruner.java index 6d3180a8..f8fd2195 100644 --- a/src/main/java/org/qortal/controller/repository/BlockPruner.java +++ b/src/main/java/org/qortal/controller/repository/BlockPruner.java @@ -18,8 +18,17 @@ public class BlockPruner implements Runnable { public void run() { Thread.currentThread().setName("Block pruner"); + boolean archiveMode = false; if (!Settings.getInstance().isPruningEnabled()) { - return; + // Pruning isn't enabled, but we might want to prune for the purposes of archiving + if (!Settings.getInstance().isArchiveEnabled()) { + // No pruning or archiving, so we must not prune anything + return; + } + else { + // We're allowed to prune blocks that have already been archived + archiveMode = true; + } } try (final Repository repository = RepositoryManager.getRepository()) { @@ -35,12 +44,24 @@ public class BlockPruner implements Runnable { continue; // Don't even attempt if we're mid-sync as our repository requests will be delayed for ages - if (Controller.getInstance().isSynchronizing()) + if (Controller.getInstance().isSynchronizing()) { continue; + } + + // Don't attempt to prune if we're not synced yet + final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp(); + if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) { + continue; + } // Prune all blocks up until our latest minus pruneBlockLimit final int ourLatestHeight = chainTip.getHeight(); - final int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit(); + + // In archive mode we are only allowed to trim blocks that have already been archived + if (archiveMode) { + upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1; + } int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize(); int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight); diff --git a/src/main/java/org/qortal/controller/repository/PruneManager.java b/src/main/java/org/qortal/controller/repository/PruneManager.java index 5f92c75d..dcb21181 100644 --- a/src/main/java/org/qortal/controller/repository/PruneManager.java +++ b/src/main/java/org/qortal/controller/repository/PruneManager.java @@ -35,29 +35,70 @@ public class PruneManager { public void start() { this.executorService = Executors.newCachedThreadPool(new DaemonThreadFactory()); - // Don't allow both the pruner and the trimmer to run at the same time. - // In pruning mode, we are already deleting far more than we would when trimming. - // In non-pruning mode, we still need to trim to keep the non-essential data - // out of the database. There isn't a case where both are needed at once. - // If we ever do need to enable both at once, be very careful with the AT state - // trimming, since both currently rely on having exclusive access to the - // prepareForAtStateTrimming() method. For both trimming and pruning to take place - // at once, we would need to synchronize this method in a way that both can't - // call it at the same time, as otherwise active ATs would be pruned/trimmed when - // they should have been kept. - - if (Settings.getInstance().isPruningEnabled()) { - // Pruning enabled - start the pruning processes - this.executorService.execute(new AtStatesPruner()); - this.executorService.execute(new BlockPruner()); + if (Settings.getInstance().isPruningEnabled() && + !Settings.getInstance().isArchiveEnabled()) { + // Top-only-sync + this.startTopOnlySyncMode(); + } + else if (Settings.getInstance().isArchiveEnabled()) { + // Full node with block archive + this.startFullNodeWithBlockArchive(); } else { - // Pruning disabled - use trimming instead - this.executorService.execute(new AtStatesTrimmer()); - this.executorService.execute(new OnlineAccountsSignaturesTrimmer()); + // Full node with full SQL support + this.startFullSQLNode(); } } + /** + * Top-only-sync + * In this mode, we delete (prune) all blocks except + * a small number of recent ones. There is no need for + * trimming or archiving, because all relevant blocks + * are deleted. + */ + private void startTopOnlySyncMode() { + this.startPruning(); + } + + /** + * Full node with block archive + * In this mode we archive trimmed blocks, and then + * prune archived blocks to keep the database small + */ + private void startFullNodeWithBlockArchive() { + this.startTrimming(); + this.startArchiving(); + this.startPruning(); + } + + /** + * Full node with full SQL support + * In this mode we trim the database but don't prune + * or archive any data, because we want to maintain + * full SQL support of old blocks. This mode will not + * be actively maintained but can be used by those who + * need to perform SQL analysis on older blocks. + */ + private void startFullSQLNode() { + this.startTrimming(); + } + + + private void startPruning() { + this.executorService.execute(new AtStatesPruner()); + this.executorService.execute(new BlockPruner()); + } + + private void startTrimming() { + this.executorService.execute(new AtStatesTrimmer()); + this.executorService.execute(new OnlineAccountsSignaturesTrimmer()); + } + + private void startArchiving() { + this.executorService.execute(new BlockArchiver()); + } + public void stop() { this.executorService.shutdownNow(); diff --git a/src/main/java/org/qortal/data/block/BlockArchiveData.java b/src/main/java/org/qortal/data/block/BlockArchiveData.java new file mode 100644 index 00000000..c9db4032 --- /dev/null +++ b/src/main/java/org/qortal/data/block/BlockArchiveData.java @@ -0,0 +1,47 @@ +package org.qortal.data.block; + +import org.qortal.block.Block; + +public class BlockArchiveData { + + // Properties + private byte[] signature; + private Integer height; + private Long timestamp; + private byte[] minterPublicKey; + + // Constructors + + public BlockArchiveData(byte[] signature, Integer height, long timestamp, byte[] minterPublicKey) { + this.signature = signature; + this.height = height; + this.timestamp = timestamp; + this.minterPublicKey = minterPublicKey; + } + + public BlockArchiveData(BlockData blockData) { + this.signature = blockData.getSignature(); + this.height = blockData.getHeight(); + this.timestamp = blockData.getTimestamp(); + this.minterPublicKey = blockData.getMinterPublicKey(); + } + + // Getters/setters + + public byte[] getSignature() { + return this.signature; + } + + public Integer getHeight() { + return this.height; + } + + public Long getTimestamp() { + return this.timestamp; + } + + public byte[] getMinterPublicKey() { + return this.minterPublicKey; + } + +} diff --git a/src/main/java/org/qortal/network/message/CachedBlockMessage.java b/src/main/java/org/qortal/network/message/CachedBlockMessage.java index 7a175810..e5029ab0 100644 --- a/src/main/java/org/qortal/network/message/CachedBlockMessage.java +++ b/src/main/java/org/qortal/network/message/CachedBlockMessage.java @@ -23,7 +23,7 @@ public class CachedBlockMessage extends Message { this.block = block; } - private CachedBlockMessage(byte[] cachedBytes) { + public CachedBlockMessage(byte[] cachedBytes) { super(MessageType.BLOCK); this.block = null; diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index 74fb19ab..9316875d 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -113,7 +113,10 @@ public interface ATRepository { public List getBlockATStatesAtHeight(int height) throws DataException; - /** Rebuild the latest AT states cache, necessary for AT state trimming/pruning. */ + /** Rebuild the latest AT states cache, necessary for AT state trimming/pruning. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ public void rebuildLatestAtStates() throws DataException; diff --git a/src/main/java/org/qortal/repository/BlockArchiveReader.java b/src/main/java/org/qortal/repository/BlockArchiveReader.java new file mode 100644 index 00000000..1b68a7c5 --- /dev/null +++ b/src/main/java/org/qortal/repository/BlockArchiveReader.java @@ -0,0 +1,251 @@ +package org.qortal.repository; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.settings.Settings; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformer; +import org.qortal.utils.Triple; + +import static org.qortal.transform.Transformer.INT_LENGTH; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + +public class BlockArchiveReader { + + private static BlockArchiveReader instance; + private Map> fileListCache = Collections.synchronizedMap(new HashMap<>()); + + private static final Logger LOGGER = LogManager.getLogger(BlockArchiveReader.class); + + public BlockArchiveReader() { + + } + + public static synchronized BlockArchiveReader getInstance() { + if (instance == null) { + instance = new BlockArchiveReader(); + } + + return instance; + } + + private void fetchFileList() { + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath(); + File archiveDirFile = archivePath.toFile(); + String[] files = archiveDirFile.list(); + Map> map = new HashMap<>(); + + for (String file : files) { + Path filePath = Paths.get(file); + String filename = filePath.getFileName().toString(); + + // Parse the filename + if (filename == null || !filename.contains("-") || !filename.contains(".")) { + // Not a usable file + continue; + } + // Remove the extension and split into two parts + String[] parts = filename.substring(0, filename.lastIndexOf('.')).split("-"); + Integer startHeight = Integer.parseInt(parts[0]); + Integer endHeight = Integer.parseInt(parts[1]); + Integer range = endHeight - startHeight; + map.put(filename, new Triple(startHeight, endHeight, range)); + } + this.fileListCache = map; + } + + public Triple, List> fetchBlockAtHeight(int height) { + if (this.fileListCache.isEmpty()) { + this.fetchFileList(); + } + + byte[] serializedBytes = this.fetchSerializedBlockBytesForHeight(height); + if (serializedBytes == null) { + return null; + } + + ByteBuffer byteBuffer = ByteBuffer.wrap(serializedBytes); + Triple, List> blockInfo = null; + try { + blockInfo = BlockTransformer.fromByteBuffer(byteBuffer); + if (blockInfo != null && blockInfo.getA() != null) { + // Block height is stored outside of the main serialized bytes, so it + // won't be set automatically. + blockInfo.getA().setHeight(height); + } + } catch (TransformationException e) { + return null; + } + return blockInfo; + } + + public Triple, List> fetchBlockWithSignature( + byte[] signature, Repository repository) { + + if (this.fileListCache.isEmpty()) { + this.fetchFileList(); + } + + Integer height = this.fetchHeightForSignature(signature, repository); + if (height != null) { + return this.fetchBlockAtHeight(height); + } + return null; + } + + public Integer fetchHeightForSignature(byte[] signature, Repository repository) { + // Lookup the height for the requested signature + try { + BlockArchiveData archivedBlock = repository.getBlockArchiveRepository().getBlockArchiveDataForSignature(signature); + if (archivedBlock.getHeight() == null) { + return null; + } + return archivedBlock.getHeight(); + + } catch (DataException e) { + return null; + } + } + + public int fetchHeightForTimestamp(long timestamp, Repository repository) { + // Lookup the height for the requested signature + try { + return repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp); + + } catch (DataException e) { + return 0; + } + } + + private String getFilenameForHeight(int height) { + Iterator it = this.fileListCache.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry)it.next(); + if (pair == null && pair.getKey() == null && pair.getValue() == null) { + continue; + } + Triple heightInfo = (Triple) pair.getValue(); + Integer startHeight = heightInfo.getA(); + Integer endHeight = heightInfo.getB(); + + if (height >= startHeight && height <= endHeight) { + // Found the correct file + String filename = (String) pair.getKey(); + return filename; + } + } + + return null; + } + + public byte[] fetchSerializedBlockBytesForSignature(byte[] signature, Repository repository) { + + if (this.fileListCache.isEmpty()) { + this.fetchFileList(); + } + + Integer height = this.fetchHeightForSignature(signature, repository); + if (height != null) { + return this.fetchSerializedBlockBytesForHeight(height); + } + return null; + } + + public byte[] fetchSerializedBlockBytesForHeight(int height) { + String filename = this.getFilenameForHeight(height); + if (filename == null) { + // We don't have this block in the archive + // Invalidate the file list cache in case it is out of date + this.invalidateFileListCache(); + return null; + } + + Path filePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive", filename).toAbsolutePath(); + RandomAccessFile file = null; + try { + file = new RandomAccessFile(filePath.toString(), "r"); + // Get info about this file (the "fixed length header") + final int version = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + final int startHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + final int endHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + file.readInt(); // Block count (unused) // Do not remove or comment out, as it is moving the file pointer + final int variableHeaderLength = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + final int fixedHeaderLength = (int)file.getFilePointer(); + // End of fixed length header + + // Make sure the version is one we recognize + if (version != 1) { + LOGGER.info("Error: unknown version in file {}: {}", filename, version); + return null; + } + + // Verify that the block is within the reported range + if (height < startHeight || height > endHeight) { + LOGGER.info("Error: requested height {} but the range of file {} is {}-{}", + height, filename, startHeight, endHeight); + return null; + } + + // Seek to the location of the block index in the variable length header + final int locationOfBlockIndexInVariableHeaderSegment = (height - startHeight) * INT_LENGTH; + file.seek(fixedHeaderLength + locationOfBlockIndexInVariableHeaderSegment); + + // Read the value to obtain the index of this block in the data segment + int locationOfBlockInDataSegment = file.readInt(); + + // Now seek to the block data itself + int dataSegmentStartIndex = fixedHeaderLength + variableHeaderLength + INT_LENGTH; // Confirmed correct + file.seek(dataSegmentStartIndex + locationOfBlockInDataSegment); + + // Read the block metadata + int blockHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + int blockLength = file.readInt(); // Do not remove or comment out, as it is moving the file pointer + + // Ensure the block height matches the one requested + if (blockHeight != height) { + LOGGER.info("Error: height {} does not match requested: {}", blockHeight, height); + return null; + } + + // Now retrieve the block's serialized bytes + byte[] blockBytes = new byte[blockLength]; + file.read(blockBytes); + + return blockBytes; + + } catch (FileNotFoundException e) { + LOGGER.info("File {} not found: {}", filename, e.getMessage()); + return null; + } catch (IOException e) { + LOGGER.info("Unable to read block {} from archive: {}", height, e.getMessage()); + return null; + } + finally { + // Close the file + if (file != null) { + try { + file.close(); + } catch (IOException e) { + // Failed to close, but no need to handle this + } + } + } + } + + public void invalidateFileListCache() { + this.fileListCache.clear(); + } + +} diff --git a/src/main/java/org/qortal/repository/BlockArchiveRepository.java b/src/main/java/org/qortal/repository/BlockArchiveRepository.java new file mode 100644 index 00000000..c702a7ef --- /dev/null +++ b/src/main/java/org/qortal/repository/BlockArchiveRepository.java @@ -0,0 +1,118 @@ +package org.qortal.repository; + +import org.qortal.api.model.BlockSignerSummary; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.data.block.BlockSummaryData; + +import java.util.List; + +public interface BlockArchiveRepository { + + /** + * Returns BlockData from archive using block signature. + * + * @param signature + * @return block data, or null if not found in archive. + * @throws DataException + */ + public BlockData fromSignature(byte[] signature) throws DataException; + + /** + * Return height of block in archive using block's signature. + * + * @param signature + * @return height, or 0 if not found in blockchain. + * @throws DataException + */ + public int getHeightFromSignature(byte[] signature) throws DataException; + + /** + * Returns BlockData from archive using block height. + * + * @param height + * @return block data, or null if not found in blockchain. + * @throws DataException + */ + public BlockData fromHeight(int height) throws DataException; + + /** + * Returns BlockData from archive using block reference. + * Currently relies on a child block being the one block + * higher than its parent. This limitation can be removed + * by storing the reference in the BlockArchive table, but + * this has been avoided to reduce space. + * + * @param reference + * @return block data, or null if either parent or child + * not found in the archive. + * @throws DataException + */ + public BlockData fromReference(byte[] reference) throws DataException; + + /** + * Return height of block with timestamp just before passed timestamp. + * + * @param timestamp + * @return height, or 0 if not found in blockchain. + * @throws DataException + */ + public int getHeightFromTimestamp(long timestamp) throws DataException; + + /** + * Returns block summaries for blocks signed by passed public key, or reward-share with minter with passed public key. + */ + public List getBlockSummariesBySigner(byte[] signerPublicKey, Integer limit, Integer offset, Boolean reverse) throws DataException; + + /** + * Returns summaries of block signers, optionally limited to passed addresses. + * This combines both the BlockArchive and the Blocks data into a single result set. + */ + public List getBlockSigners(List addresses, Integer limit, Integer offset, Boolean reverse) throws DataException; + + + /** Returns height of first unarchived block. */ + public int getBlockArchiveHeight() throws DataException; + + /** Sets new height for block archiving. + *

+ * NOTE: performs implicit repository.saveChanges(). + */ + public void setBlockArchiveHeight(int archiveHeight) throws DataException; + + + /** + * Returns the block archive data for a given signature, from the block archive. + *

+ * This method will return null if no block archive has been built for the + * requested signature. In those cases, the height (and other data) can be + * looked up using the Blocks table. This allows a block to be located in + * the archive when we only know its signature. + *

+ * + * @param signature + * @throws DataException + */ + public BlockArchiveData getBlockArchiveDataForSignature(byte[] signature) throws DataException; + + /** + * Saves a block archive entry into the repository. + *

+ * This can be used to find the height of a block by its signature, without + * having access to the block data itself. + *

+ * + * @param blockArchiveData + * @throws DataException + */ + public void save(BlockArchiveData blockArchiveData) throws DataException; + + /** + * Deletes a block archive entry from the repository. + * + * @param blockArchiveData + * @throws DataException + */ + public void delete(BlockArchiveData blockArchiveData) throws DataException; + +} diff --git a/src/main/java/org/qortal/repository/BlockArchiveWriter.java b/src/main/java/org/qortal/repository/BlockArchiveWriter.java new file mode 100644 index 00000000..4aeb1a32 --- /dev/null +++ b/src/main/java/org/qortal/repository/BlockArchiveWriter.java @@ -0,0 +1,193 @@ +package org.qortal.repository; + +import com.google.common.primitives.Ints; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.Block; +import org.qortal.controller.Controller; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.settings.Settings; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformer; + +import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class BlockArchiveWriter { + + public enum BlockArchiveWriteResult { + OK, + STOPPING, + NOT_ENOUGH_BLOCKS, + BLOCK_LIMIT_REACHED, + BLOCK_NOT_FOUND + } + + private static final Logger LOGGER = LogManager.getLogger(BlockArchiveWriter.class); + + private int startHeight; + private final int endHeight; + private final Repository repository; + + private int writtenCount; + + public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) { + this.startHeight = startHeight; + this.endHeight = endHeight; + this.repository = repository; + } + + public static int getMaxArchiveHeight(Repository repository, boolean useMaximumDuplicatedLimit) throws DataException { + // We must only archive trimmed blocks, or the archive will grow far too large + final int accountSignaturesTrimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight(); + final int atTrimStartHeight = repository.getATRepository().getAtTrimHeight(); + final int trimStartHeight = Math.min(accountSignaturesTrimStartHeight, atTrimStartHeight); + + // In some cases we want to restrict the upper height of the archiver to save space + if (useMaximumDuplicatedLimit) { + // To save on disk space, it's best to not allow the archiver to get too far ahead of the pruner + // This reduces the amount of data that is held twice during the transition + final int blockPruneStartHeight = repository.getBlockRepository().getBlockPruneHeight(); + final int atPruneStartHeight = repository.getATRepository().getAtPruneHeight(); + final int pruneStartHeight = Math.min(blockPruneStartHeight, atPruneStartHeight); + final int maximumDuplicatedBlocks = Settings.getInstance().getMaxDuplicatedBlocksWhenArchiving(); + + // To summarize the above: + // - We must never archive anything greater than or equal to trimStartHeight + // - We should avoid archiving anything maximumDuplicatedBlocks higher than pruneStartHeight + return Math.min(trimStartHeight, pruneStartHeight + maximumDuplicatedBlocks); + } + else { + // We don't want to apply the maximum duplicated limit + return trimStartHeight; + } + } + + public static boolean isArchiverUpToDate(Repository repository, boolean useMaximumDuplicatedLimit) throws DataException { + final int maxArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, useMaximumDuplicatedLimit); + final int actualArchiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + final float progress = (float)actualArchiveHeight / (float) maxArchiveHeight; + LOGGER.info(String.format("maxArchiveHeight: %d, actualArchiveHeight: %d, progress: %f", + maxArchiveHeight, actualArchiveHeight, progress)); + + // If archiver is within 90% of the maximum, treat it as up to date + // We need several percent as an allowance because the archiver will only + // save files when they reach the target size + return (progress >= 0.90); + } + + public BlockArchiveWriteResult write() throws DataException, IOException, TransformationException, InterruptedException { + // Create the archive folder if it doesn't exist + // This is a subfolder of the db directory, to make bootstrapping easier + Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath(); + try { + Files.createDirectories(archivePath); + } catch (IOException e) { + LOGGER.info("Unable to create archive folder"); + throw new DataException("Unable to create archive folder"); + } + + // Determine start height of blocks to fetch + if (startHeight <= 2) { + // Skip genesis block, as it's not designed to be transmitted, and we can build that from blockchain.json + // TODO: include genesis block if we can + startHeight = 2; + } + + // Header bytes will store the block indexes + ByteArrayOutputStream headerBytes = new ByteArrayOutputStream(); + // Bytes will store the actual block data + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + LOGGER.info(String.format("Fetching blocks from height %d...", startHeight)); + int i = 0; + long fileSizeTarget = 100 * 1024 * 1024; // 100MiB + while (headerBytes.size() + bytes.size() < fileSizeTarget) { + if (Controller.isStopping()) { + return BlockArchiveWriteResult.STOPPING; + } + if (Controller.getInstance().isSynchronizing()) { + continue; + } + + int currentHeight = startHeight + i; + if (currentHeight >= endHeight) { + return BlockArchiveWriteResult.BLOCK_LIMIT_REACHED; + } + + //LOGGER.info("Fetching block {}...", currentHeight); + + BlockData blockData = repository.getBlockRepository().fromHeight(currentHeight); + if (blockData == null) { + return BlockArchiveWriteResult.BLOCK_NOT_FOUND; + } + + // Write the signature and height into the BlockArchive table + BlockArchiveData blockArchiveData = new BlockArchiveData(blockData); + repository.getBlockArchiveRepository().save(blockArchiveData); + repository.saveChanges(); + + // Write the block data to some byte buffers + Block block = new Block(repository, blockData); + int blockIndex = bytes.size(); + // Write block index to header + headerBytes.write(Ints.toByteArray(blockIndex)); + // Write block height + bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); + byte[] blockBytes = BlockTransformer.toBytes(block); + // Write block length + bytes.write(Ints.toByteArray(blockBytes.length)); + // Write block bytes + bytes.write(blockBytes); + i++; + + } + int totalLength = headerBytes.size() + bytes.size(); + LOGGER.info(String.format("Total length of %d blocks is %d bytes", i, totalLength)); + + // Validate file size, in case something went wrong + if (totalLength < fileSizeTarget) { + return BlockArchiveWriteResult.NOT_ENOUGH_BLOCKS; + } + + // We have enough blocks to create a new file + int endHeight = startHeight + i - 1; + int version = 1; + String filePath = String.format("%s/%d-%d.dat", archivePath.toString(), startHeight, endHeight); + FileOutputStream fileOutputStream = new FileOutputStream(filePath); + // Write version number + fileOutputStream.write(Ints.toByteArray(version)); + // Write start height + fileOutputStream.write(Ints.toByteArray(startHeight)); + // Write end height + fileOutputStream.write(Ints.toByteArray(endHeight)); + // Write total count + fileOutputStream.write(Ints.toByteArray(i)); + // Write dynamic header (block indexes) segment length + fileOutputStream.write(Ints.toByteArray(headerBytes.size())); + // Write dynamic header (block indexes) data + headerBytes.writeTo(fileOutputStream); + // Write data segment (block data) length + fileOutputStream.write(Ints.toByteArray(bytes.size())); + // Write data + bytes.writeTo(fileOutputStream); + // Close the file + fileOutputStream.close(); + + // Invalidate cache so that the rest of the app picks up the new file + BlockArchiveReader.getInstance().invalidateFileListCache(); + + this.writtenCount = i; + return BlockArchiveWriteResult.OK; + } + + public int getWrittenCount() { + return this.writtenCount; + } + +} diff --git a/src/main/java/org/qortal/repository/BlockRepository.java b/src/main/java/org/qortal/repository/BlockRepository.java index 5ca61e66..76891c36 100644 --- a/src/main/java/org/qortal/repository/BlockRepository.java +++ b/src/main/java/org/qortal/repository/BlockRepository.java @@ -137,11 +137,6 @@ public interface BlockRepository { */ public List getBlockSummaries(int firstBlockHeight, int lastBlockHeight) throws DataException; - /** - * Returns block summaries for the passed height range, for API use. - */ - public List getBlockSummaries(Integer startHeight, Integer endHeight, Integer count) throws DataException; - /** Returns height of first trimmable online accounts signatures. */ public int getOnlineAccountsSignaturesTrimHeight() throws DataException; diff --git a/src/main/java/org/qortal/repository/Repository.java b/src/main/java/org/qortal/repository/Repository.java index 656e6e1e..fab48a14 100644 --- a/src/main/java/org/qortal/repository/Repository.java +++ b/src/main/java/org/qortal/repository/Repository.java @@ -12,6 +12,8 @@ public interface Repository extends AutoCloseable { public BlockRepository getBlockRepository(); + public BlockArchiveRepository getBlockArchiveRepository(); + public ChatRepository getChatRepository(); public CrossChainRepository getCrossChainRepository(); diff --git a/src/main/java/org/qortal/repository/RepositoryManager.java b/src/main/java/org/qortal/repository/RepositoryManager.java index 5e9c71c2..f7557750 100644 --- a/src/main/java/org/qortal/repository/RepositoryManager.java +++ b/src/main/java/org/qortal/repository/RepositoryManager.java @@ -2,6 +2,7 @@ package org.qortal.repository; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving; import org.qortal.repository.hsqldb.HSQLDBDatabasePruning; import org.qortal.settings.Settings; @@ -57,9 +58,23 @@ public abstract class RepositoryManager { } } - public static void prune() { + public static boolean archive() { + // Bulk archive the database the first time we use archive mode + if (Settings.getInstance().isArchiveEnabled()) { + try { + return HSQLDBDatabaseArchiving.buildBlockArchive(); + + } catch (DataException e) { + LOGGER.info("Unable to bulk prune AT states. The database may have been left in an inconsistent state."); + } + } + return false; + } + + public static boolean prune() { // Bulk prune the database the first time we use pruning mode - if (Settings.getInstance().isPruningEnabled()) { + if (Settings.getInstance().isPruningEnabled() || + Settings.getInstance().isArchiveEnabled()) { try { boolean prunedATStates = HSQLDBDatabasePruning.pruneATStates(); boolean prunedBlocks = HSQLDBDatabasePruning.pruneBlocks(); @@ -67,12 +82,14 @@ public abstract class RepositoryManager { // Perform repository maintenance to shrink the db size down if (prunedATStates && prunedBlocks) { HSQLDBDatabasePruning.performMaintenance(); + return true; } } catch (SQLException | DataException e) { LOGGER.info("Unable to bulk prune AT states. The database may have been left in an inconsistent state."); } } + return false; } public static void setRequestedCheckpoint(Boolean quick) { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index 522fafb7..e0baa136 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -608,6 +608,7 @@ public class HSQLDBATRepository implements ATRepository { // that could result in one process using a partial or empty dataset // because it was in the process of being rebuilt by another thread synchronized (this.repository.latestATStatesLock) { + LOGGER.trace("Rebuilding latest AT states..."); // Rebuild cache of latest AT states that we can't trim String deleteSql = "DELETE FROM LatestATStates"; @@ -632,6 +633,8 @@ public class HSQLDBATRepository implements ATRepository { repository.examineException(e); throw new DataException("Unable to populate temporary latest AT states cache in repository", e); } + this.repository.saveChanges(); + LOGGER.trace("Rebuilt latest AT states"); } } @@ -661,7 +664,7 @@ public class HSQLDBATRepository implements ATRepository { this.repository.executeCheckedUpdate(updateSql, trimHeight); this.repository.saveChanges(); } catch (SQLException e) { - repository.examineException(e); + this.repository.examineException(e); throw new DataException("Unable to set AT state trim height in repository", e); } } @@ -689,7 +692,10 @@ public class HSQLDBATRepository implements ATRepository { + "LIMIT ?"; try { - return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit); + int modifiedRows = this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit); + this.repository.saveChanges(); + return modifiedRows; + } catch (SQLException e) { repository.examineException(e); throw new DataException("Unable to trim AT states in repository", e); @@ -757,7 +763,7 @@ public class HSQLDBATRepository implements ATRepository { } while (resultSet.next()); } } catch (SQLException e) { - throw new DataException("Unable to fetch flagged accounts from repository", e); + throw new DataException("Unable to fetch latest AT states from repository", e); } List atStates = this.getBlockATStatesAtHeight(height); @@ -785,6 +791,7 @@ public class HSQLDBATRepository implements ATRepository { } } } + this.repository.saveChanges(); return deletedCount; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java new file mode 100644 index 00000000..c491f862 --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java @@ -0,0 +1,277 @@ +package org.qortal.repository.hsqldb; + +import org.qortal.api.ApiError; +import org.qortal.api.ApiExceptionFactory; +import org.qortal.api.model.BlockSignerSummary; +import org.qortal.data.block.BlockArchiveData; +import org.qortal.data.block.BlockData; +import org.qortal.data.block.BlockSummaryData; +import org.qortal.repository.BlockArchiveReader; +import org.qortal.repository.BlockArchiveRepository; +import org.qortal.repository.DataException; +import org.qortal.utils.Triple; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository { + + protected HSQLDBRepository repository; + + public HSQLDBBlockArchiveRepository(HSQLDBRepository repository) { + this.repository = repository; + } + + + @Override + public BlockData fromSignature(byte[] signature) throws DataException { + Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockWithSignature(signature, this.repository); + if (blockInfo != null) { + return (BlockData) blockInfo.getA(); + } + return null; + } + + @Override + public int getHeightFromSignature(byte[] signature) throws DataException { + Integer height = BlockArchiveReader.getInstance().fetchHeightForSignature(signature, this.repository); + if (height == null || height == 0) { + return 0; + } + return height; + } + + @Override + public BlockData fromHeight(int height) throws DataException { + Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height); + if (blockInfo != null) { + return (BlockData) blockInfo.getA(); + } + return null; + } + + @Override + public BlockData fromReference(byte[] reference) throws DataException { + BlockData referenceBlock = this.repository.getBlockArchiveRepository().fromSignature(reference); + if (referenceBlock != null) { + int height = referenceBlock.getHeight(); + if (height > 0) { + // Request the block at height + 1 + Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height + 1); + if (blockInfo != null) { + return (BlockData) blockInfo.getA(); + } + } + } + return null; + } + + @Override + public int getHeightFromTimestamp(long timestamp) throws DataException { + String sql = "SELECT height FROM BlockArchive WHERE minted_when <= ? ORDER BY minted_when DESC, height DESC LIMIT 1"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, timestamp)) { + if (resultSet == null) { + return 0; + } + return resultSet.getInt(1); + + } catch (SQLException e) { + throw new DataException("Error fetching height from BlockArchive repository", e); + } + } + + @Override + public List getBlockSummariesBySigner(byte[] signerPublicKey, Integer limit, Integer offset, Boolean reverse) throws DataException { + StringBuilder sql = new StringBuilder(512); + sql.append("SELECT signature, height, BlockArchive.minter FROM "); + + // List of minter account's public key and reward-share public keys with minter's public key + sql.append("(SELECT * FROM (VALUES (CAST(? AS QortalPublicKey))) UNION (SELECT reward_share_public_key FROM RewardShares WHERE minter_public_key = ?)) AS PublicKeys (public_key) "); + + // Match BlockArchive blocks signed with public key from above list + sql.append("JOIN BlockArchive ON BlockArchive.minter = public_key "); + + sql.append("ORDER BY BlockArchive.height "); + if (reverse != null && reverse) + sql.append("DESC "); + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List blockSummaries = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), signerPublicKey, signerPublicKey)) { + if (resultSet == null) + return blockSummaries; + + do { + byte[] signature = resultSet.getBytes(1); + int height = resultSet.getInt(2); + byte[] blockMinterPublicKey = resultSet.getBytes(3); + + // Fetch additional info from the archive itself + int onlineAccountsCount = 0; + BlockData blockData = this.fromSignature(signature); + if (blockData != null) { + onlineAccountsCount = blockData.getOnlineAccountsCount(); + } + + BlockSummaryData blockSummary = new BlockSummaryData(height, signature, blockMinterPublicKey, onlineAccountsCount); + blockSummaries.add(blockSummary); + } while (resultSet.next()); + + return blockSummaries; + } catch (SQLException e) { + throw new DataException("Unable to fetch minter's block summaries from repository", e); + } + } + + @Override + public List getBlockSigners(List addresses, Integer limit, Integer offset, Boolean reverse) throws DataException { + String subquerySql = "SELECT minter, COUNT(signature) FROM (" + + "(SELECT minter, signature FROM Blocks) UNION ALL (SELECT minter, signature FROM BlockArchive)" + + ") GROUP BY minter"; + + StringBuilder sql = new StringBuilder(1024); + sql.append("SELECT DISTINCT block_minter, n_blocks, minter_public_key, minter, recipient FROM ("); + sql.append(subquerySql); + sql.append(") AS Minters (block_minter, n_blocks) LEFT OUTER JOIN RewardShares ON reward_share_public_key = block_minter "); + + if (addresses != null && !addresses.isEmpty()) { + sql.append(" LEFT OUTER JOIN Accounts AS BlockMinterAccounts ON BlockMinterAccounts.public_key = block_minter "); + sql.append(" LEFT OUTER JOIN Accounts AS RewardShareMinterAccounts ON RewardShareMinterAccounts.public_key = minter_public_key "); + sql.append(" JOIN (VALUES "); + + final int addressesSize = addresses.size(); + for (int ai = 0; ai < addressesSize; ++ai) { + if (ai != 0) + sql.append(", "); + + sql.append("(?)"); + } + + sql.append(") AS FilterAccounts (account) "); + sql.append(" ON FilterAccounts.account IN (recipient, BlockMinterAccounts.account, RewardShareMinterAccounts.account) "); + } else { + addresses = Collections.emptyList(); + } + + sql.append("ORDER BY n_blocks "); + if (reverse != null && reverse) + sql.append("DESC "); + + HSQLDBRepository.limitOffsetSql(sql, limit, offset); + + List summaries = new ArrayList<>(); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), addresses.toArray())) { + if (resultSet == null) + return summaries; + + do { + byte[] blockMinterPublicKey = resultSet.getBytes(1); + int nBlocks = resultSet.getInt(2); + + // May not be present if no reward-share: + byte[] mintingAccountPublicKey = resultSet.getBytes(3); + String minterAccount = resultSet.getString(4); + String recipientAccount = resultSet.getString(5); + + BlockSignerSummary blockSignerSummary; + if (recipientAccount == null) + blockSignerSummary = new BlockSignerSummary(blockMinterPublicKey, nBlocks); + else + blockSignerSummary = new BlockSignerSummary(blockMinterPublicKey, nBlocks, mintingAccountPublicKey, minterAccount, recipientAccount); + + summaries.add(blockSignerSummary); + } while (resultSet.next()); + + return summaries; + } catch (SQLException e) { + throw new DataException("Unable to fetch block minters from repository", e); + } + } + + + @Override + public int getBlockArchiveHeight() throws DataException { + String sql = "SELECT block_archive_height FROM DatabaseInfo"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql)) { + if (resultSet == null) + return 0; + + return resultSet.getInt(1); + } catch (SQLException e) { + throw new DataException("Unable to fetch block archive height from repository", e); + } + } + + @Override + public void setBlockArchiveHeight(int archiveHeight) throws DataException { + // trimHeightsLock is to prevent concurrent update on DatabaseInfo + // that could result in "transaction rollback: serialization failure" + synchronized (this.repository.trimHeightsLock) { + String updateSql = "UPDATE DatabaseInfo SET block_archive_height = ?"; + + try { + this.repository.executeCheckedUpdate(updateSql, archiveHeight); + this.repository.saveChanges(); + } catch (SQLException e) { + repository.examineException(e); + throw new DataException("Unable to set block archive height in repository", e); + } + } + } + + + @Override + public BlockArchiveData getBlockArchiveDataForSignature(byte[] signature) throws DataException { + String sql = "SELECT height, signature, minted_when, minter FROM BlockArchive WHERE signature = ? LIMIT 1"; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, signature)) { + if (resultSet == null) { + return null; + } + int height = resultSet.getInt(1); + byte[] sig = resultSet.getBytes(2); + long timestamp = resultSet.getLong(3); + byte[] minterPublicKey = resultSet.getBytes(4); + return new BlockArchiveData(sig, height, timestamp, minterPublicKey); + + } catch (SQLException e) { + throw new DataException("Error fetching height from BlockArchive repository", e); + } + } + + + @Override + public void save(BlockArchiveData blockArchiveData) throws DataException { + HSQLDBSaver saveHelper = new HSQLDBSaver("BlockArchive"); + + saveHelper.bind("signature", blockArchiveData.getSignature()) + .bind("height", blockArchiveData.getHeight()) + .bind("minted_when", blockArchiveData.getTimestamp()) + .bind("minter", blockArchiveData.getMinterPublicKey()); + + try { + saveHelper.execute(this.repository); + } catch (SQLException e) { + throw new DataException("Unable to save SimpleBlockData into BlockArchive repository", e); + } + } + + @Override + public void delete(BlockArchiveData blockArchiveData) throws DataException { + try { + this.repository.delete("BlockArchive", + "block_signature = ?", blockArchiveData.getSignature()); + } catch (SQLException e) { + throw new DataException("Unable to delete SimpleBlockData from BlockArchive repository", e); + } + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java index 2f7e4ad2..b8238085 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java @@ -10,6 +10,7 @@ import org.qortal.api.model.BlockSignerSummary; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.data.block.BlockTransactionData; +import org.qortal.data.block.BlockArchiveData; import org.qortal.data.transaction.TransactionData; import org.qortal.repository.BlockRepository; import org.qortal.repository.DataException; @@ -382,86 +383,6 @@ public class HSQLDBBlockRepository implements BlockRepository { } } - @Override - public List getBlockSummaries(Integer startHeight, Integer endHeight, Integer count) throws DataException { - StringBuilder sql = new StringBuilder(512); - List bindParams = new ArrayList<>(); - - sql.append("SELECT signature, height, minter, online_accounts_count, minted_when, transaction_count "); - - /* - * start end count result - * 10 40 null blocks 10 to 39 (excludes end block, ignore count) - * - * null null null blocks 1 to 50 (assume count=50, maybe start=1) - * 30 null null blocks 30 to 79 (assume count=50) - * 30 null 10 blocks 30 to 39 - * - * null null 50 last 50 blocks? so if max(blocks.height) is 200, then blocks 151 to 200 - * null 200 null blocks 150 to 199 (excludes end block, assume count=50) - * null 200 10 blocks 190 to 199 (excludes end block) - */ - - if (startHeight != null && endHeight != null) { - sql.append("FROM Blocks "); - sql.append("WHERE height BETWEEN ? AND ?"); - bindParams.add(startHeight); - bindParams.add(Integer.valueOf(endHeight - 1)); - } else if (endHeight != null || (startHeight == null && count != null)) { - // we are going to return blocks from the end of the chain - if (count == null) - count = 50; - - if (endHeight == null) { - sql.append("FROM (SELECT height FROM Blocks ORDER BY height DESC LIMIT 1) AS MaxHeights (max_height) "); - sql.append("JOIN Blocks ON height BETWEEN (max_height - ? + 1) AND max_height "); - bindParams.add(count); - } else { - sql.append("FROM Blocks "); - sql.append("WHERE height BETWEEN ? AND ?"); - bindParams.add(Integer.valueOf(endHeight - count)); - bindParams.add(Integer.valueOf(endHeight - 1)); - } - - } else { - // we are going to return blocks from the start of the chain - if (startHeight == null) - startHeight = 1; - - if (count == null) - count = 50; - - sql.append("FROM Blocks "); - sql.append("WHERE height BETWEEN ? AND ?"); - bindParams.add(startHeight); - bindParams.add(Integer.valueOf(startHeight + count - 1)); - } - - List blockSummaries = new ArrayList<>(); - - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { - if (resultSet == null) - return blockSummaries; - - do { - byte[] signature = resultSet.getBytes(1); - int height = resultSet.getInt(2); - byte[] minterPublicKey = resultSet.getBytes(3); - int onlineAccountsCount = resultSet.getInt(4); - long timestamp = resultSet.getLong(5); - int transactionCount = resultSet.getInt(6); - - BlockSummaryData blockSummary = new BlockSummaryData(height, signature, minterPublicKey, onlineAccountsCount, - timestamp, transactionCount); - blockSummaries.add(blockSummary); - } while (resultSet.next()); - - return blockSummaries; - } catch (SQLException e) { - throw new DataException("Unable to fetch height-ranged block summaries from repository", e); - } - } - @Override public int getOnlineAccountsSignaturesTrimHeight() throws DataException { String sql = "SELECT online_signatures_trim_height FROM DatabaseInfo"; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java new file mode 100644 index 00000000..930da828 --- /dev/null +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java @@ -0,0 +1,87 @@ +package org.qortal.repository.hsqldb; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.controller.Controller; +import org.qortal.repository.BlockArchiveWriter; +import org.qortal.repository.DataException; +import org.qortal.repository.RepositoryManager; +import org.qortal.transform.TransformationException; + +import java.io.IOException; + +/** + * + * When switching to an archiving node, we need to archive most of the database contents. + * This involves copying its data into flat files. + * If we do this entirely as a background process, it is very slow and can interfere with syncing. + * However, if we take the approach of doing this in bulk, before starting up the rest of the + * processes, this makes it much faster and less invasive. + * + * From that point, the original background archiving process will run, but can be dialled right down + * so not to interfere with syncing. + * + */ + + +public class HSQLDBDatabaseArchiving { + + private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabaseArchiving.class); + + + public static boolean buildBlockArchive() throws DataException { + try (final HSQLDBRepository repository = (HSQLDBRepository)RepositoryManager.getRepository()) { + + // Only build the archive if we have never done so before + int archiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight(); + if (archiveHeight > 0) { + // Already archived + return false; + } + + LOGGER.info("Building block archive - this process could take a while... (approx. 15 mins on high spec)"); + + final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, false); + int startHeight = 0; + + while (!Controller.isStopping()) { + try { + BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository); + BlockArchiveWriter.BlockArchiveWriteResult result = writer.write(); + switch (result) { + case OK: + // Increment block archive height + startHeight += writer.getWrittenCount(); + repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight); + repository.saveChanges(); + break; + + case STOPPING: + return false; + + case BLOCK_LIMIT_REACHED: + case NOT_ENOUGH_BLOCKS: + // We've reached the limit of the blocks we can archive + // Return from the whole method + return true; + + case BLOCK_NOT_FOUND: + // We tried to archive a block that didn't exist. This is a major failure and likely means + // that a bootstrap or re-sync is needed. Return rom the method + LOGGER.info("Error: block not found when building archive. If this error persists, " + + "a bootstrap or re-sync may be needed."); + return false; + } + + } catch (IOException | TransformationException | InterruptedException e) { + LOGGER.info("Caught exception when creating block cache", e); + return false; + } + } + } + + // If we got this far then something went wrong (most likely the app is stopping) + return false; + } + +} diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java index ba170bf6..969c954c 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java @@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.controller.Controller; import org.qortal.data.block.BlockData; +import org.qortal.repository.BlockArchiveWriter; import org.qortal.repository.DataException; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; @@ -36,6 +37,7 @@ public class HSQLDBDatabasePruning { private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabasePruning.class); + public static boolean pruneATStates() throws SQLException, DataException { try (final HSQLDBRepository repository = (HSQLDBRepository)RepositoryManager.getRepository()) { @@ -46,7 +48,18 @@ public class HSQLDBDatabasePruning { return false; } - LOGGER.info("Starting bulk prune of AT states - this process could take a while... (approx. 2 mins on high spec)"); + if (Settings.getInstance().isArchiveEnabled()) { + // Only proceed if we can see that the archiver has already finished + // This way, if the archiver failed for any reason, we can prune once it has had + // some opportunities to try again + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository, false); + if (!upToDate) { + return false; + } + } + + LOGGER.info("Starting bulk prune of AT states - this process could take a while... " + + "(approx. 2 mins on high spec, or upwards of 30 mins in some cases)"); // Create new AT-states table to hold smaller dataset repository.executeCheckedUpdate("DROP TABLE IF EXISTS ATStatesNew"); @@ -68,11 +81,17 @@ public class HSQLDBDatabasePruning { // Calculate some constants for later use final int blockchainHeight = latestBlock.getHeight(); - final int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + if (Settings.getInstance().isArchiveEnabled()) { + // Archive mode - don't prune anything that hasn't been archived yet + maximumBlockToTrim = Math.min(maximumBlockToTrim, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); + } final int startHeight = maximumBlockToTrim; final int endHeight = blockchainHeight; final int blockStep = 10000; + + // Loop through all the LatestATStates and copy them to the new table LOGGER.info("Copying AT states..."); for (int height = 0; height < endHeight; height += blockStep) { @@ -99,7 +118,7 @@ public class HSQLDBDatabasePruning { } if (height >= startHeight) { - // Now copy this AT states for each recent block it is present in + // Now copy this AT's states for each recent block they is present in for (int i = startHeight; i < endHeight; i++) { if (latestAtHeight < i) { // This AT finished before this block so there is nothing to copy @@ -159,20 +178,25 @@ public class HSQLDBDatabasePruning { private static boolean pruneATStateData() throws SQLException, DataException { try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) { + if (Settings.getInstance().isArchiveEnabled()) { + // Don't prune ATStatesData in archive mode + return true; + } + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); if (latestBlock == null) { LOGGER.info("Unable to determine blockchain height, necessary for bulk ATStatesData pruning"); return false; } final int blockchainHeight = latestBlock.getHeight(); - final int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); // ATStateData is already trimmed - so carry on from where we left off in the past int pruneStartHeight = repository.getATRepository().getAtTrimHeight(); LOGGER.info("Starting bulk prune of AT states data - this process could take a while... (approx. 3 mins on high spec)"); while (pruneStartHeight < upperPrunableHeight) { - // Prune all AT state data up until our latest minus pruneBlockLimit + // Prune all AT state data up until our latest minus pruneBlockLimit (or our archive height) if (Controller.isStopping()) { return false; @@ -225,15 +249,30 @@ public class HSQLDBDatabasePruning { return false; } + if (Settings.getInstance().isArchiveEnabled()) { + // Only proceed if we can see that the archiver has already finished + // This way, if the archiver failed for any reason, we can prune once it has had + // some opportunities to try again + boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository, false); + if (!upToDate) { + return false; + } + } + BlockData latestBlock = repository.getBlockRepository().getLastBlock(); if (latestBlock == null) { LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning"); return false; } final int blockchainHeight = latestBlock.getHeight(); - final int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); + int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit(); int pruneStartHeight = 0; + if (Settings.getInstance().isArchiveEnabled()) { + // Archive mode - don't prune anything that hasn't been archived yet + upperPrunableHeight = Math.min(upperPrunableHeight, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1); + } + LOGGER.info("Starting bulk prune of blocks - this process could take a while... (approx. 5 mins on high spec)"); while (pruneStartHeight < upperPrunableHeight) { diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java index d696351f..66fe9029 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -873,6 +873,25 @@ public class HSQLDBDatabaseUpdates { stmt.execute("ALTER TABLE DatabaseInfo ADD block_prune_height INT NOT NULL DEFAULT 0"); break; + case 36: + // Block archive support + stmt.execute("ALTER TABLE DatabaseInfo ADD block_archive_height INT NOT NULL DEFAULT 0"); + + // Block archive (lookup table to map signature to height) + // Actual data is stored in archive files outside of the database + stmt.execute("CREATE TABLE BlockArchive (signature BlockSignature, height INTEGER NOT NULL, " + + "minted_when EpochMillis NOT NULL, minter QortalPublicKey NOT NULL, " + + "PRIMARY KEY (signature))"); + // For finding blocks by height. + stmt.execute("CREATE INDEX BlockArchiveHeightIndex ON BlockArchive (height)"); + // For finding blocks by the account that minted them. + stmt.execute("CREATE INDEX BlockArchiveMinterIndex ON BlockArchive (minter)"); + // For finding blocks by timestamp or finding height of latest block immediately before timestamp, etc. + stmt.execute("CREATE INDEX BlockArchiveTimestampHeightIndex ON BlockArchive (minted_when, height)"); + // Use a separate table space as this table will be very large. + stmt.execute("SET TABLE BlockArchive NEW SPACE"); + break; + default: // nothing to do return false; diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java index 3a947cd6..6807c100 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java @@ -31,22 +31,7 @@ import org.qortal.crypto.Crypto; import org.qortal.data.crosschain.TradeBotData; import org.qortal.globalization.Translator; import org.qortal.gui.SysTray; -import org.qortal.repository.ATRepository; -import org.qortal.repository.AccountRepository; -import org.qortal.repository.ArbitraryRepository; -import org.qortal.repository.AssetRepository; -import org.qortal.repository.BlockRepository; -import org.qortal.repository.ChatRepository; -import org.qortal.repository.CrossChainRepository; -import org.qortal.repository.DataException; -import org.qortal.repository.GroupRepository; -import org.qortal.repository.MessageRepository; -import org.qortal.repository.NameRepository; -import org.qortal.repository.NetworkRepository; -import org.qortal.repository.Repository; -import org.qortal.repository.RepositoryManager; -import org.qortal.repository.TransactionRepository; -import org.qortal.repository.VotingRepository; +import org.qortal.repository.*; import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository; import org.qortal.settings.Settings; import org.qortal.utils.Base58; @@ -76,6 +61,7 @@ public class HSQLDBRepository implements Repository { private final ArbitraryRepository arbitraryRepository = new HSQLDBArbitraryRepository(this); private final AssetRepository assetRepository = new HSQLDBAssetRepository(this); private final BlockRepository blockRepository = new HSQLDBBlockRepository(this); + private final BlockArchiveRepository blockArchiveRepository = new HSQLDBBlockArchiveRepository(this); private final ChatRepository chatRepository = new HSQLDBChatRepository(this); private final CrossChainRepository crossChainRepository = new HSQLDBCrossChainRepository(this); private final GroupRepository groupRepository = new HSQLDBGroupRepository(this); @@ -143,6 +129,11 @@ public class HSQLDBRepository implements Repository { return this.blockRepository; } + @Override + public BlockArchiveRepository getBlockArchiveRepository() { + return this.blockArchiveRepository; + } + @Override public ChatRepository getChatRepository() { return this.chatRepository; diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 6ac7342c..6527d7e0 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -129,6 +129,15 @@ public class Settings { private int blockPruneBatchSize = 10000; // blocks + /** Whether we should archive old data to reduce the database size */ + private boolean archiveEnabled = true; + /** How often to attempt archiving (ms). */ + private long archiveInterval = 7171L; // milliseconds + /** The maximum number of blocks that can exist in both the + * database and the archive at the same time */ + private int maxDuplicatedBlocksWhenArchiving = 100000; + + // Peer-to-peer related private boolean isTestNet = false; /** Port number for inbound peer-to-peer connections. */ @@ -574,4 +583,17 @@ public class Settings { return this.blockPruneBatchSize; } + + public boolean isArchiveEnabled() { + return this.archiveEnabled; + } + + public long getArchiveInterval() { + return this.archiveInterval; + } + + public int getMaxDuplicatedBlocksWhenArchiving() { + return this.maxDuplicatedBlocksWhenArchiving; + } + }