Added block archive mode

This takes all trimmed blocks (which should now be all but the last 1450 or so) and moves them into flat files. Each file contains the serialized bytes of as many blocks that can fit within the file size target of 100MiB.

As a result, the HSQLDB size drops to less than 1GB, making it much faster and easier to maintain. It also significantly reduces the total size of each full node, because the data is stored in a highly optimized way.

HSQLDB then works similarly to the way it does in pruning mode - it holds all transactions, the latest state of every AT, as well as the full AT states data and hashes for the past 1450 blocks.

Each archive file contains headers and indexes in order to quickly locate blocks. When a peer requests a block that is within the archive, the serialized bytes are sent directly without the need to go via a BlockData object. Now that there are no slow queries or data serialization processes needed, it should greatly speed up the block serving.

The /block API endpoints have been modified in such a way that they will also check and retrieve blocks from the archive when needed.

A lightweight "BlockArchive" table is needed in HSQLDB to map block heights to signatures minters and timestamps. It made more sense to keep SQL support for these basic attributes of each block. These are located in a separate table from the full blocks, in order to create a clear distinction between HSQLDB blocks and archived blocks, and also to speed up query times in the Blocks table, which is the one we are using 99% of the time.

There is currently a restriction on the /admin/orphan API endpoint to prevent orphaning beyond the threshold of the block archive.

src/main/java/org/qortal/api/resource/AdminResource.java

@@ -35,6 +35,7 @@ import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 
 import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.core.LoggerContext;
 import org.apache.logging.log4j.core.appender.RollingFileAppender;
 import org.qortal.account.Account;
@@ -67,6 +68,8 @@ import com.google.common.collect.Lists;
 @Tag(name = "Admin")
 public class AdminResource {
 
+	private static final Logger LOGGER = LogManager.getLogger(AdminResource.class);
+
 	private static final int MAX_LOG_LINES = 500;
 
 	@Context
@@ -459,6 +462,23 @@ public class AdminResource {
 			if (targetHeight <= 0 || targetHeight > Controller.getInstance().getChainHeight())
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT);
 
+			// Make sure we're not orphaning as far back as the archived blocks
+			// FUTURE: we could support this by first importing earlier blocks from the archive
+			if (Settings.getInstance().isPruningEnabled() ||
+				Settings.getInstance().isArchiveEnabled()) {
+
+				try (final Repository repository = RepositoryManager.getRepository()) {
+					// Find the first unarchived block
+					int oldestBlock = repository.getBlockArchiveRepository().getBlockArchiveHeight();
+					// Add some extra blocks just in case we're currently archiving/pruning
+					oldestBlock += 100;
+					if (targetHeight <= oldestBlock) {
+						LOGGER.info("Unable to orphan beyond block {} because it is archived", oldestBlock);
+						throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_HEIGHT);
+					}
+				}
+			}
+
 			if (BlockChain.orphan(targetHeight))
 				return "true";
 			else

src/main/java/org/qortal/api/resource/BlocksResource.java

@@ -15,6 +15,8 @@ import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.math.RoundingMode;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
@@ -33,11 +35,13 @@ import org.qortal.api.ApiExceptionFactory;
 import org.qortal.api.model.BlockMintingInfo;
 import org.qortal.api.model.BlockSignerSummary;
 import org.qortal.block.Block;
+import org.qortal.controller.Controller;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.account.AccountData;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.block.BlockSummaryData;
 import org.qortal.data.transaction.TransactionData;
+import org.qortal.repository.BlockArchiveReader;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
@@ -81,11 +85,19 @@ public class BlocksResource {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
+		    // Check the database first
 			BlockData blockData = repository.getBlockRepository().fromSignature(signature);
-			if (blockData == null)
+			if (blockData != null) {
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+				return blockData;
+			}
 
-			return blockData;
+            // Not found, so try the block archive
+			blockData = repository.getBlockArchiveRepository().fromSignature(signature);
+			if (blockData != null) {
+				return blockData;
+			}
+
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -116,16 +128,24 @@ public class BlocksResource {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Check the database first
 			BlockData blockData = repository.getBlockRepository().fromSignature(signature);
-			if (blockData == null)
+			if (blockData != null) {
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+                Block block = new Block(repository, blockData);
+                ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+                bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
+                bytes.write(BlockTransformer.toBytes(block));
+                return Base58.encode(bytes.toByteArray());
+            }
 
-			Block block = new Block(repository, blockData);
+            // Not found, so try the block archive
-			ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+            byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, repository);
-			bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
+            if (bytes != null) {
-			bytes.write(BlockTransformer.toBytes(block));
+                return Base58.encode(bytes);
-			return Base58.encode(bytes.toByteArray());
+            }
 
+            throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 		} catch (TransformationException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e);
 		} catch (DataException | IOException e) {
@@ -170,8 +190,12 @@ public class BlocksResource {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			if (repository.getBlockRepository().getHeightFromSignature(signature) == 0)
+		    // Check if the block exists in either the database or archive
+			if (repository.getBlockRepository().getHeightFromSignature(signature) == 0 &&
+					repository.getBlockArchiveRepository().getHeightFromSignature(signature) == 0) {
+				// Not found in either the database or archive
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+            }
 
 			return repository.getBlockRepository().getTransactionsFromSignature(signature, limit, offset, reverse);
 		} catch (DataException e) {
@@ -200,7 +224,19 @@ public class BlocksResource {
 	})
 	public BlockData getFirstBlock() {
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			return repository.getBlockRepository().fromHeight(1);
+			// Check the database first
+			BlockData blockData = repository.getBlockRepository().fromHeight(1);
+			if (blockData != null) {
+				return blockData;
+			}
+
+			// Try the archive
+			blockData = repository.getBlockArchiveRepository().fromHeight(1);
+			if (blockData != null) {
+				return blockData;
+			}
+
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -262,17 +298,28 @@ public class BlocksResource {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
+			BlockData childBlockData = null;
+
+			// Check if block exists in database
 			BlockData blockData = repository.getBlockRepository().fromSignature(signature);
+			if (blockData != null) {
+				return repository.getBlockRepository().fromReference(signature);
+			}
 
-			// Check block exists
+			// Not found, so try the archive
-			if (blockData == null)
+			// This also checks that the parent block exists
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+			// It will return null if either the parent or child don't exit
-
+			childBlockData = repository.getBlockArchiveRepository().fromReference(signature);
-			BlockData childBlockData = repository.getBlockRepository().fromReference(signature);
 
 			// Check child block exists
-			if (childBlockData == null)
+			if (childBlockData == null) {
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+			}
+
+			// Check child block's reference matches the supplied signature
+			if (!Arrays.equals(childBlockData.getReference(), signature)) {
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+			}
 
 			return childBlockData;
 		} catch (DataException e) {
@@ -338,13 +385,20 @@ public class BlocksResource {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
+			// Firstly check the database
 			BlockData blockData = repository.getBlockRepository().fromSignature(signature);
+			if (blockData != null) {
+				return blockData.getHeight();
+			}
 
-			// Check block exists
+			// Not found, so try the archive
-			if (blockData == null)
+			blockData = repository.getBlockArchiveRepository().fromSignature(signature);
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+			if (blockData != null) {
+				return blockData.getHeight();
+			}
+
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 
-			return blockData.getHeight();
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -371,11 +425,20 @@ public class BlocksResource {
 	})
 	public BlockData getByHeight(@PathParam("height") int height) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
+			// Firstly check the database
 			BlockData blockData = repository.getBlockRepository().fromHeight(height);
-			if (blockData == null)
+			if (blockData != null) {
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+				return blockData;
+			}
+
+			// Not found, so try the archive
+			blockData = repository.getBlockArchiveRepository().fromHeight(height);
+			if (blockData != null) {
+				return blockData;
+			}
+
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 
-			return blockData;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -402,12 +465,31 @@ public class BlocksResource {
 	})
 	public BlockMintingInfo getBlockMintingInfoByHeight(@PathParam("height") int height) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
+			// Try the database
 			BlockData blockData = repository.getBlockRepository().fromHeight(height);
-			if (blockData == null)
+			if (blockData == null) {
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+
+				// Not found, so try the archive
+				blockData = repository.getBlockArchiveRepository().fromHeight(height);
+				if (blockData == null) {
+
+					// Still not found
+					throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+				}
+			}
 
 			Block block = new Block(repository, blockData);
 			BlockData parentBlockData = repository.getBlockRepository().fromSignature(blockData.getReference());
+			if (parentBlockData == null) {
+				// Parent block not found - try the archive
+				parentBlockData = repository.getBlockArchiveRepository().fromSignature(blockData.getReference());
+				if (parentBlockData == null) {
+
+					// Still not found
+					throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+				}
+			}
+
 			int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, blockData.getMinterPublicKey());
 			if (minterLevel == 0)
 				// This may be unavailable when requesting a trimmed block
@@ -454,13 +536,26 @@ public class BlocksResource {
 	})
 	public BlockData getByTimestamp(@PathParam("timestamp") long timestamp) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			int height = repository.getBlockRepository().getHeightFromTimestamp(timestamp);
+			BlockData blockData = null;
-			if (height == 0)
-				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
 
-			BlockData blockData = repository.getBlockRepository().fromHeight(height);
+			// Try the Blocks table
-			if (blockData == null)
+			int height = repository.getBlockRepository().getHeightFromTimestamp(timestamp);
+			if (height > 0) {
+				// Found match in Blocks table
+				return repository.getBlockRepository().fromHeight(height);
+			}
+
+			// Not found in Blocks table, so try the archive
+			height = repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp);
+			if (height > 0) {
+				// Found match in archive
+				blockData = repository.getBlockArchiveRepository().fromHeight(height);
+			}
+
+			// Ensure block exists
+			if (blockData == null) {
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
+			}
 
 			return blockData;
 		} catch (DataException e) {
@@ -497,9 +592,14 @@ public class BlocksResource {
 
 			for (/* count already set */; count > 0; --count, ++height) {
 				BlockData blockData = repository.getBlockRepository().fromHeight(height);
-				if (blockData == null)
+				if (blockData == null) {
-					// Run out of blocks!
+					// Not found - try the archive
-					break;
+					blockData = repository.getBlockArchiveRepository().fromHeight(height);
+					if (blockData == null) {
+						// Run out of blocks!
+						break;
+					}
+				}
 
 				blocks.add(blockData);
 			}
@@ -544,7 +644,29 @@ public class BlocksResource {
 			if (accountData == null || accountData.getPublicKey() == null)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.PUBLIC_KEY_NOT_FOUND);
 
-			return repository.getBlockRepository().getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse);
+
+			List<BlockSummaryData> summaries = repository.getBlockRepository()
+					.getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse);
+
+			// Add any from the archive
+			List<BlockSummaryData> archivedSummaries = repository.getBlockArchiveRepository()
+					.getBlockSummariesBySigner(accountData.getPublicKey(), limit, offset, reverse);
+			if (archivedSummaries != null && !archivedSummaries.isEmpty()) {
+				summaries.addAll(archivedSummaries);
+			}
+			else {
+				summaries = archivedSummaries;
+			}
+
+			// Sort the results (because they may have been obtained from two places)
+			if (reverse != null && reverse) {
+				summaries.sort((s1, s2) -> Integer.valueOf(s2.getHeight()).compareTo(Integer.valueOf(s1.getHeight())));
+			}
+			else {
+				summaries.sort(Comparator.comparing(s -> Integer.valueOf(s.getHeight())));
+			}
+
+			return summaries;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -580,7 +702,8 @@ public class BlocksResource {
 				if (!Crypto.isValidAddress(address))
 					throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_ADDRESS);
 
-			return repository.getBlockRepository().getBlockSigners(addresses, limit, offset, reverse);
+			// This method pulls data from both Blocks and BlockArchive, so no need to query serparately
+			return repository.getBlockArchiveRepository().getBlockSigners(addresses, limit, offset, reverse);
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -620,7 +743,76 @@ public class BlocksResource {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			return repository.getBlockRepository().getBlockSummaries(startHeight, endHeight, count);
+
+			/*
+			 * start	end		count		result
+			 * 10		40		null		blocks 10 to 39 (excludes end block, ignore count)
+			 *
+			 * null		null	null		blocks 1 to 50 (assume count=50, maybe start=1)
+			 * 30		null	null		blocks 30 to 79 (assume count=50)
+			 * 30		null	10			blocks 30 to 39
+			 *
+			 * null		null	50			last 50 blocks? so if max(blocks.height) is 200, then blocks 151 to 200
+			 * null		200		null		blocks 150 to 199 (excludes end block, assume count=50)
+			 * null		200		10			blocks 190 to 199 (excludes end block)
+			 */
+
+			List<BlockSummaryData> blockSummaries = new ArrayList<>();
+
+			// Use the latest X blocks if only a count is specified
+			if (startHeight == null && endHeight == null && count != null) {
+				BlockData chainTip = Controller.getInstance().getChainTip();
+				startHeight = chainTip.getHeight() - count;
+				endHeight = chainTip.getHeight();
+			}
+
+			// ... otherwise default the start height to 1
+			if (startHeight == null && endHeight == null) {
+				startHeight = 1;
+			}
+
+			// Default the count to 50
+			if (count == null) {
+				count = 50;
+			}
+
+			// If both a start and end height exist, ignore the count
+			if (startHeight != null && endHeight != null) {
+				if (startHeight > 0 && endHeight > 0) {
+					count = Integer.MAX_VALUE;
+				}
+			}
+
+			// Derive start height from end height if missing
+			if (startHeight == null || startHeight == 0) {
+				if (endHeight != null && endHeight > 0) {
+					if (count != null) {
+						startHeight = endHeight - count;
+					}
+				}
+			}
+
+			for (/* count already set */; count > 0; --count, ++startHeight) {
+				if (endHeight != null && startHeight >= endHeight) {
+					break;
+				}
+				BlockData blockData = repository.getBlockRepository().fromHeight(startHeight);
+				if (blockData == null) {
+					// Not found - try the archive
+					blockData = repository.getBlockArchiveRepository().fromHeight(startHeight);
+					if (blockData == null) {
+						// Run out of blocks!
+						break;
+					}
+				}
+
+				if (blockData != null) {
+					BlockSummaryData blockSummaryData = new BlockSummaryData(blockData);
+					blockSummaries.add(blockSummaryData);
+				}
+			}
+
+			return blockSummaries;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}

src/main/java/org/qortal/controller/Controller.java

@@ -83,20 +83,14 @@ import org.qortal.network.message.OnlineAccountsMessage;
 import org.qortal.network.message.SignaturesMessage;
 import org.qortal.network.message.TransactionMessage;
 import org.qortal.network.message.TransactionSignaturesMessage;
-import org.qortal.repository.DataException;
+import org.qortal.repository.*;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryFactory;
-import org.qortal.repository.RepositoryManager;
 import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
 import org.qortal.settings.Settings;
 import org.qortal.transaction.ArbitraryTransaction;
 import org.qortal.transaction.Transaction;
 import org.qortal.transaction.Transaction.TransactionType;
 import org.qortal.transaction.Transaction.ValidationResult;
-import org.qortal.utils.Base58;
+import org.qortal.utils.*;
-import org.qortal.utils.ByteArray;
-import org.qortal.utils.NTP;
-import org.qortal.utils.Triple;
 
 import com.google.common.primitives.Longs;
 
@@ -414,6 +408,7 @@ public class Controller extends Thread {
 		try {
 			RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl());
 			RepositoryManager.setRepositoryFactory(repositoryFactory);
+			RepositoryManager.archive();
 			RepositoryManager.prune();
 		} catch (DataException e) {
 			// If exception has no cause then repository is in use by some other process.
@@ -1286,6 +1281,34 @@ public class Controller extends Thread {
 				}
 			}
 
+			// If we have no block data, we should check the archive in case it's there
+			if (blockData == null) {
+				if (Settings.getInstance().isArchiveEnabled()) {
+					byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, repository);
+					if (bytes != null) {
+						CachedBlockMessage blockMessage = new CachedBlockMessage(bytes);
+						blockMessage.setId(message.getId());
+
+						// This call also causes the other needed data to be pulled in from repository
+						if (!peer.sendMessage(blockMessage)) {
+							peer.disconnect("failed to send block");
+							// Don't fall-through to caching because failure to send might be from failure to build message
+							return;
+						}
+
+						// If request is for a recent block, cache it
+						if (getChainHeight() - blockData.getHeight() <= blockCacheSize) {
+							this.stats.getBlockMessageStats.cacheFills.incrementAndGet();
+
+							this.blockMessageCache.put(new ByteArray(blockData.getSignature()), blockMessage);
+						}
+
+						// Sent successfully from archive, so nothing more to do
+						return;
+					}
+				}
+			}
+
 			if (blockData == null) {
 				// We don't have this block
 				this.stats.getBlockMessageStats.unknownBlocks.getAndIncrement();

src/main/java/org/qortal/controller/repository/AtStatesPruner.java

@@ -18,15 +18,24 @@ public class AtStatesPruner implements Runnable {
 	public void run() {
 		Thread.currentThread().setName("AT States pruner");
 
+		boolean archiveMode = false;
 		if (!Settings.getInstance().isPruningEnabled()) {
-			return;
+			// Pruning isn't enabled, but we might want to prune for the purposes of archiving
+			if (!Settings.getInstance().isArchiveEnabled()) {
+				// No pruning or archiving, so we must not prune anything
+				return;
+			}
+			else {
+				// We're allowed to prune blocks that have already been archived
+				archiveMode = true;
+			}
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			int pruneStartHeight = repository.getATRepository().getAtPruneHeight();
 
+			repository.discardChanges();
 			repository.getATRepository().rebuildLatestAtStates();
-			repository.saveChanges();
 
 			while (!Controller.isStopping()) {
 				repository.discardChanges();
@@ -43,7 +52,14 @@ public class AtStatesPruner implements Runnable {
 
 				// Prune AT states for all blocks up until our latest minus pruneBlockLimit
 				final int ourLatestHeight = chainTip.getHeight();
-				final int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit();
+				int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit();
+
+				// In archive mode we are only allowed to trim blocks that have already been archived
+				if (archiveMode) {
+					upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1;
+
+					// TODO: validate that the actual archived data exists before pruning it?
+				}
 
 				int upperBatchHeight = pruneStartHeight + Settings.getInstance().getAtStatesPruneBatchSize();
 				int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight);

src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java

@@ -21,8 +21,8 @@ public class AtStatesTrimmer implements Runnable {
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			int trimStartHeight = repository.getATRepository().getAtTrimHeight();
 
+			repository.discardChanges();
 			repository.getATRepository().rebuildLatestAtStates();
-			repository.saveChanges();
 
 			while (!Controller.isStopping()) {
 				repository.discardChanges();

src/main/java/org/qortal/controller/repository/BlockArchiver.java

@@ -0,0 +1,105 @@
+package org.qortal.controller.repository;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
+import org.qortal.data.block.BlockData;
+import org.qortal.repository.*;
+import org.qortal.settings.Settings;
+import org.qortal.transform.TransformationException;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+
+public class BlockArchiver implements Runnable {
+
+	private static final Logger LOGGER = LogManager.getLogger(BlockArchiver.class);
+
+	private static final long INITIAL_SLEEP_PERIOD = 0L; // TODO: 5 * 60 * 1000L + 1234L; // ms
+
+	public void run() {
+		Thread.currentThread().setName("Block archiver");
+
+		if (!Settings.getInstance().isArchiveEnabled()) {
+			return;
+		}
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			int startHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight();
+
+			// Don't even start building until initial rush has ended
+			Thread.sleep(INITIAL_SLEEP_PERIOD);
+
+			LOGGER.info("Starting block archiver...");
+
+			while (!Controller.isStopping()) {
+				repository.discardChanges();
+
+				final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, true);
+
+				Thread.sleep(Settings.getInstance().getArchiveInterval());
+
+				BlockData chainTip = Controller.getInstance().getChainTip();
+				if (chainTip == null || NTP.getTime() == null) {
+					continue;
+				}
+
+				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
+				if (Controller.getInstance().isSynchronizing()) {
+					continue;
+				}
+
+				// Don't attempt to archive if we're not synced yet
+				final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
+				if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) {
+					continue;
+				}
+
+
+				// Build cache of blocks
+				try {
+					BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository);
+					BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
+					switch (result) {
+						case OK:
+							// Increment block archive height
+							startHeight += writer.getWrittenCount();
+							repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight);
+							repository.saveChanges();
+							break;
+
+						case STOPPING:
+							return;
+
+						case BLOCK_LIMIT_REACHED:
+							// We've reached the limit of the blocks we can archive
+							// Sleep for a while to allow more to become available
+						case NOT_ENOUGH_BLOCKS:
+							// We didn't reach our file size target, so that must mean that we don't have enough blocks
+							// yet or something went wrong. Sleep for a while and then try again.
+							Thread.sleep(60 * 60 * 1000L); // 1 hour
+							break;
+
+						case BLOCK_NOT_FOUND:
+							// We tried to archive a block that didn't exist. This is a major failure and likely means
+							// that a bootstrap or re-sync is needed. Try again every minute until then.
+							LOGGER.info("Error: block not found when building archive. If this error persists, " +
+									"a bootstrap or re-sync may be needed.");
+							Thread.sleep( 60 * 1000L); // 1 minute
+							break;
+					}
+
+				} catch (IOException | TransformationException e) {
+					LOGGER.info("Caught exception when creating block cache", e);
+				}
+
+			}
+		} catch (DataException e) {
+			LOGGER.info("Caught exception when creating block cache", e);
+		} catch (InterruptedException e) {
+			// Do nothing
+		}
+
+	}
+
+}

src/main/java/org/qortal/controller/repository/BlockPruner.java

@@ -18,8 +18,17 @@ public class BlockPruner implements Runnable {
 	public void run() {
 		Thread.currentThread().setName("Block pruner");
 
+		boolean archiveMode = false;
 		if (!Settings.getInstance().isPruningEnabled()) {
-			return;
+			// Pruning isn't enabled, but we might want to prune for the purposes of archiving
+			if (!Settings.getInstance().isArchiveEnabled()) {
+				// No pruning or archiving, so we must not prune anything
+				return;
+			}
+			else {
+				// We're allowed to prune blocks that have already been archived
+				archiveMode = true;
+			}
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
@@ -35,12 +44,24 @@ public class BlockPruner implements Runnable {
 					continue;
 
 				// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
-				if (Controller.getInstance().isSynchronizing())
+				if (Controller.getInstance().isSynchronizing()) {
 					continue;
+				}
+
+				// Don't attempt to prune if we're not synced yet
+				final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
+				if (minLatestBlockTimestamp == null || chainTip.getTimestamp() < minLatestBlockTimestamp) {
+					continue;
+				}
 
 				// Prune all blocks up until our latest minus pruneBlockLimit
 				final int ourLatestHeight = chainTip.getHeight();
-				final int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit();
+				int upperPrunableHeight = ourLatestHeight - Settings.getInstance().getPruneBlockLimit();
+
+				// In archive mode we are only allowed to trim blocks that have already been archived
+				if (archiveMode) {
+					upperPrunableHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1;
+				}
 
 				int upperBatchHeight = pruneStartHeight + Settings.getInstance().getBlockPruneBatchSize();
 				int upperPruneHeight = Math.min(upperBatchHeight, upperPrunableHeight);

src/main/java/org/qortal/controller/repository/PruneManager.java

@@ -35,29 +35,70 @@ public class PruneManager {
     public void start() {
         this.executorService = Executors.newCachedThreadPool(new DaemonThreadFactory());
 
-        // Don't allow both the pruner and the trimmer to run at the same time.
+        if (Settings.getInstance().isPruningEnabled() &&
-        // In pruning mode, we are already deleting far more than we would when trimming.
+            !Settings.getInstance().isArchiveEnabled()) {
-        // In non-pruning mode, we still need to trim to keep the non-essential data
+            // Top-only-sync
-        // out of the database. There isn't a case where both are needed at once.
+            this.startTopOnlySyncMode();
-        // If we ever do need to enable both at once, be very careful with the AT state
+        }
-        // trimming, since both currently rely on having exclusive access to the
+        else if (Settings.getInstance().isArchiveEnabled()) {
-        // prepareForAtStateTrimming() method. For both trimming and pruning to take place
+            // Full node with block archive
-        // at once, we would need to synchronize this method in a way that both can't
+            this.startFullNodeWithBlockArchive();
-        // call it at the same time, as otherwise active ATs would be pruned/trimmed when
-        // they should have been kept.
-
-        if (Settings.getInstance().isPruningEnabled()) {
-            // Pruning enabled - start the pruning processes
-            this.executorService.execute(new AtStatesPruner());
-            this.executorService.execute(new BlockPruner());
         }
         else {
-            // Pruning disabled - use trimming instead
+            // Full node with full SQL support
-            this.executorService.execute(new AtStatesTrimmer());
+            this.startFullSQLNode();
-            this.executorService.execute(new OnlineAccountsSignaturesTrimmer());
         }
     }
 
+    /**
+     * Top-only-sync
+     * In this mode, we delete (prune) all blocks except
+     * a small number of recent ones. There is no need for
+     * trimming or archiving, because all relevant blocks
+     * are deleted.
+     */
+    private void startTopOnlySyncMode() {
+        this.startPruning();
+    }
+
+    /**
+     * Full node with block archive
+     * In this mode we archive trimmed blocks, and then
+     * prune archived blocks to keep the database small
+     */
+    private void startFullNodeWithBlockArchive() {
+        this.startTrimming();
+        this.startArchiving();
+        this.startPruning();
+    }
+
+    /**
+     * Full node with full SQL support
+     * In this mode we trim the database but don't prune
+     * or archive any data, because we want to maintain
+     * full SQL support of old blocks. This mode will not
+     * be actively maintained but can be used by those who
+     * need to perform SQL analysis on older blocks.
+     */
+    private void startFullSQLNode() {
+        this.startTrimming();
+    }
+
+
+    private void startPruning() {
+        this.executorService.execute(new AtStatesPruner());
+        this.executorService.execute(new BlockPruner());
+    }
+
+    private void startTrimming() {
+        this.executorService.execute(new AtStatesTrimmer());
+        this.executorService.execute(new OnlineAccountsSignaturesTrimmer());
+    }
+
+    private void startArchiving() {
+        this.executorService.execute(new BlockArchiver());
+    }
+
     public void stop() {
         this.executorService.shutdownNow();
 

src/main/java/org/qortal/data/block/BlockArchiveData.java

@@ -0,0 +1,47 @@
+package org.qortal.data.block;
+
+import org.qortal.block.Block;
+
+public class BlockArchiveData {
+
+	// Properties
+	private byte[] signature;
+	private Integer height;
+	private Long timestamp;
+	private byte[] minterPublicKey;
+
+	// Constructors
+
+	public BlockArchiveData(byte[] signature, Integer height, long timestamp, byte[] minterPublicKey) {
+		this.signature = signature;
+		this.height = height;
+		this.timestamp = timestamp;
+		this.minterPublicKey = minterPublicKey;
+	}
+
+	public BlockArchiveData(BlockData blockData) {
+		this.signature = blockData.getSignature();
+		this.height = blockData.getHeight();
+		this.timestamp = blockData.getTimestamp();
+		this.minterPublicKey = blockData.getMinterPublicKey();
+	}
+
+	// Getters/setters
+
+	public byte[] getSignature() {
+		return this.signature;
+	}
+
+	public Integer getHeight() {
+		return this.height;
+	}
+
+	public Long getTimestamp() {
+		return this.timestamp;
+	}
+
+	public byte[] getMinterPublicKey() {
+		return this.minterPublicKey;
+	}
+
+}

src/main/java/org/qortal/network/message/CachedBlockMessage.java

@@ -23,7 +23,7 @@ public class CachedBlockMessage extends Message {
 		this.block = block;
 	}
 
-	private CachedBlockMessage(byte[] cachedBytes) {
+	public CachedBlockMessage(byte[] cachedBytes) {
 		super(MessageType.BLOCK);
 
 		this.block = null;

src/main/java/org/qortal/repository/ATRepository.java

@@ -113,7 +113,10 @@ public interface ATRepository {
 	public List<ATStateData> getBlockATStatesAtHeight(int height) throws DataException;
 
 
-	/** Rebuild the latest AT states cache, necessary for AT state trimming/pruning. */
+	/** Rebuild the latest AT states cache, necessary for AT state trimming/pruning.
+	 * <p>
+	 * NOTE: performs implicit <tt>repository.saveChanges()</tt>.
+	 */
 	public void rebuildLatestAtStates() throws DataException;
 
 

src/main/java/org/qortal/repository/BlockArchiveReader.java

@@ -0,0 +1,251 @@
+package org.qortal.repository;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.data.at.ATStateData;
+import org.qortal.data.block.BlockArchiveData;
+import org.qortal.data.block.BlockData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.settings.Settings;
+import org.qortal.transform.TransformationException;
+import org.qortal.transform.block.BlockTransformer;
+import org.qortal.utils.Triple;
+
+import static org.qortal.transform.Transformer.INT_LENGTH;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+
+public class BlockArchiveReader {
+
+    private static BlockArchiveReader instance;
+    private Map<String, Triple<Integer, Integer, Integer>> fileListCache = Collections.synchronizedMap(new HashMap<>());
+
+    private static final Logger LOGGER = LogManager.getLogger(BlockArchiveReader.class);
+
+    public BlockArchiveReader() {
+
+    }
+
+    public static synchronized BlockArchiveReader getInstance() {
+        if (instance == null) {
+            instance = new BlockArchiveReader();
+        }
+
+        return instance;
+    }
+
+    private void fetchFileList() {
+        Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath();
+        File archiveDirFile = archivePath.toFile();
+        String[] files = archiveDirFile.list();
+        Map<String, Triple<Integer, Integer, Integer>> map = new HashMap<>();
+
+        for (String file : files) {
+            Path filePath = Paths.get(file);
+            String filename = filePath.getFileName().toString();
+
+            // Parse the filename
+            if (filename == null || !filename.contains("-") || !filename.contains(".")) {
+                // Not a usable file
+                continue;
+            }
+            // Remove the extension and split into two parts
+            String[] parts = filename.substring(0, filename.lastIndexOf('.')).split("-");
+            Integer startHeight = Integer.parseInt(parts[0]);
+            Integer endHeight = Integer.parseInt(parts[1]);
+            Integer range = endHeight - startHeight;
+            map.put(filename, new Triple(startHeight, endHeight, range));
+        }
+        this.fileListCache = map;
+    }
+
+    public Triple<BlockData, List<TransactionData>, List<ATStateData>> fetchBlockAtHeight(int height) {
+        if (this.fileListCache.isEmpty()) {
+            this.fetchFileList();
+        }
+
+        byte[] serializedBytes = this.fetchSerializedBlockBytesForHeight(height);
+        if (serializedBytes == null) {
+            return null;
+        }
+
+        ByteBuffer byteBuffer = ByteBuffer.wrap(serializedBytes);
+        Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = null;
+        try {
+            blockInfo = BlockTransformer.fromByteBuffer(byteBuffer);
+            if (blockInfo != null && blockInfo.getA() != null) {
+                // Block height is stored outside of the main serialized bytes, so it
+                // won't be set automatically.
+                blockInfo.getA().setHeight(height);
+            }
+        } catch (TransformationException e) {
+            return null;
+        }
+        return blockInfo;
+    }
+
+    public Triple<BlockData, List<TransactionData>, List<ATStateData>> fetchBlockWithSignature(
+            byte[] signature, Repository repository) {
+
+        if (this.fileListCache.isEmpty()) {
+            this.fetchFileList();
+        }
+
+        Integer height = this.fetchHeightForSignature(signature, repository);
+        if (height != null) {
+            return this.fetchBlockAtHeight(height);
+        }
+        return null;
+    }
+
+    public Integer fetchHeightForSignature(byte[] signature, Repository repository) {
+        // Lookup the height for the requested signature
+        try {
+            BlockArchiveData archivedBlock = repository.getBlockArchiveRepository().getBlockArchiveDataForSignature(signature);
+            if (archivedBlock.getHeight() == null) {
+                return null;
+            }
+            return archivedBlock.getHeight();
+
+        } catch (DataException e) {
+            return null;
+        }
+    }
+
+    public int fetchHeightForTimestamp(long timestamp, Repository repository) {
+        // Lookup the height for the requested signature
+        try {
+            return repository.getBlockArchiveRepository().getHeightFromTimestamp(timestamp);
+
+        } catch (DataException e) {
+            return 0;
+        }
+    }
+
+    private String getFilenameForHeight(int height) {
+        Iterator it = this.fileListCache.entrySet().iterator();
+        while (it.hasNext()) {
+            Map.Entry pair = (Map.Entry)it.next();
+            if (pair == null && pair.getKey() == null && pair.getValue() == null) {
+                continue;
+            }
+            Triple<Integer, Integer, Integer> heightInfo = (Triple<Integer, Integer, Integer>) pair.getValue();
+            Integer startHeight = heightInfo.getA();
+            Integer endHeight = heightInfo.getB();
+
+            if (height >= startHeight && height <= endHeight) {
+                // Found the correct file
+                String filename = (String) pair.getKey();
+                return filename;
+            }
+        }
+
+        return null;
+    }
+
+    public byte[] fetchSerializedBlockBytesForSignature(byte[] signature, Repository repository) {
+
+        if (this.fileListCache.isEmpty()) {
+            this.fetchFileList();
+        }
+
+        Integer height = this.fetchHeightForSignature(signature, repository);
+        if (height != null) {
+            return this.fetchSerializedBlockBytesForHeight(height);
+        }
+        return null;
+    }
+
+    public byte[] fetchSerializedBlockBytesForHeight(int height) {
+        String filename = this.getFilenameForHeight(height);
+        if (filename == null) {
+            // We don't have this block in the archive
+            // Invalidate the file list cache in case it is out of date
+            this.invalidateFileListCache();
+            return null;
+        }
+
+        Path filePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive", filename).toAbsolutePath();
+        RandomAccessFile file = null;
+        try {
+            file = new RandomAccessFile(filePath.toString(), "r");
+            // Get info about this file (the "fixed length header")
+            final int version = file.readInt(); // Do not remove or comment out, as it is moving the file pointer
+            final int startHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer
+            final int endHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer
+            file.readInt(); // Block count (unused) // Do not remove or comment out, as it is moving the file pointer
+            final int variableHeaderLength = file.readInt(); // Do not remove or comment out, as it is moving the file pointer
+            final int fixedHeaderLength = (int)file.getFilePointer();
+            // End of fixed length header
+
+            // Make sure the version is one we recognize
+            if (version != 1) {
+                LOGGER.info("Error: unknown version in file {}: {}", filename, version);
+                return null;
+            }
+
+            // Verify that the block is within the reported range
+            if (height < startHeight || height > endHeight) {
+                LOGGER.info("Error: requested height {} but the range of file {} is {}-{}",
+                        height, filename, startHeight, endHeight);
+                return null;
+            }
+
+            // Seek to the location of the block index in the variable length header
+            final int locationOfBlockIndexInVariableHeaderSegment = (height - startHeight) * INT_LENGTH;
+            file.seek(fixedHeaderLength + locationOfBlockIndexInVariableHeaderSegment);
+
+            // Read the value to obtain the index of this block in the data segment
+            int locationOfBlockInDataSegment = file.readInt();
+
+            // Now seek to the block data itself
+            int dataSegmentStartIndex = fixedHeaderLength + variableHeaderLength + INT_LENGTH; // Confirmed correct
+            file.seek(dataSegmentStartIndex + locationOfBlockInDataSegment);
+
+            // Read the block metadata
+            int blockHeight = file.readInt(); // Do not remove or comment out, as it is moving the file pointer
+            int blockLength = file.readInt(); // Do not remove or comment out, as it is moving the file pointer
+
+            // Ensure the block height matches the one requested
+            if (blockHeight != height) {
+                LOGGER.info("Error: height {} does not match requested: {}", blockHeight, height);
+                return null;
+            }
+
+            // Now retrieve the block's serialized bytes
+            byte[] blockBytes = new byte[blockLength];
+            file.read(blockBytes);
+
+            return blockBytes;
+
+        } catch (FileNotFoundException e) {
+            LOGGER.info("File {} not found: {}", filename, e.getMessage());
+            return null;
+        } catch (IOException e) {
+            LOGGER.info("Unable to read block {} from archive: {}", height, e.getMessage());
+            return null;
+        }
+        finally {
+            // Close the file
+            if (file != null) {
+                try {
+                    file.close();
+                } catch (IOException e) {
+                    // Failed to close, but no need to handle this
+                }
+            }
+        }
+    }
+
+    public void invalidateFileListCache() {
+        this.fileListCache.clear();
+    }
+
+}

src/main/java/org/qortal/repository/BlockArchiveRepository.java

@@ -0,0 +1,118 @@
+package org.qortal.repository;
+
+import org.qortal.api.model.BlockSignerSummary;
+import org.qortal.data.block.BlockArchiveData;
+import org.qortal.data.block.BlockData;
+import org.qortal.data.block.BlockSummaryData;
+
+import java.util.List;
+
+public interface BlockArchiveRepository {
+
+    /**
+     * Returns BlockData from archive using block signature.
+     *
+     * @param signature
+     * @return block data, or null if not found in archive.
+     * @throws DataException
+     */
+    public BlockData fromSignature(byte[] signature) throws DataException;
+
+    /**
+     * Return height of block in archive using block's signature.
+     *
+     * @param signature
+     * @return height, or 0 if not found in blockchain.
+     * @throws DataException
+     */
+    public int getHeightFromSignature(byte[] signature) throws DataException;
+
+    /**
+     * Returns BlockData from archive using block height.
+     *
+     * @param height
+     * @return block data, or null if not found in blockchain.
+     * @throws DataException
+     */
+    public BlockData fromHeight(int height) throws DataException;
+
+    /**
+     * Returns BlockData from archive using block reference.
+     * Currently relies on a child block being the one block
+     * higher than its parent. This limitation can be removed
+     * by storing the reference in the BlockArchive table, but
+     * this has been avoided to reduce space.
+     *
+     * @param reference
+     * @return block data, or null if either parent or child
+     * not found in the archive.
+     * @throws DataException
+     */
+    public BlockData fromReference(byte[] reference) throws DataException;
+
+    /**
+     * Return height of block with timestamp just before passed timestamp.
+     *
+     * @param timestamp
+     * @return height, or 0 if not found in blockchain.
+     * @throws DataException
+     */
+    public int getHeightFromTimestamp(long timestamp) throws DataException;
+
+    /**
+     * Returns block summaries for blocks signed by passed public key, or reward-share with minter with passed public key.
+     */
+    public List<BlockSummaryData> getBlockSummariesBySigner(byte[] signerPublicKey, Integer limit, Integer offset, Boolean reverse) throws DataException;
+
+    /**
+     * Returns summaries of block signers, optionally limited to passed addresses.
+     * This combines both the BlockArchive and the Blocks data into a single result set.
+     */
+    public List<BlockSignerSummary> getBlockSigners(List<String> addresses, Integer limit, Integer offset, Boolean reverse) throws DataException;
+
+
+    /** Returns height of first unarchived block. */
+    public int getBlockArchiveHeight() throws DataException;
+
+    /** Sets new height for block archiving.
+     * <p>
+     * NOTE: performs implicit <tt>repository.saveChanges()</tt>.
+     */
+    public void setBlockArchiveHeight(int archiveHeight) throws DataException;
+
+
+    /**
+     * Returns the block archive data for a given signature, from the block archive.
+     * <p>
+     * This method will return null if no block archive has been built for the
+     * requested signature. In those cases, the height (and other data) can be
+     * looked up using the Blocks table. This allows a block to be located in
+     * the archive when we only know its signature.
+     * <p>
+     *
+     * @param signature
+     * @throws DataException
+     */
+    public BlockArchiveData getBlockArchiveDataForSignature(byte[] signature) throws DataException;
+
+    /**
+     * Saves a block archive entry into the repository.
+     * <p>
+     * This can be used to find the height of a block by its signature, without
+     * having access to the block data itself.
+     * <p>
+     *
+     * @param blockArchiveData
+     * @throws DataException
+     */
+    public void save(BlockArchiveData blockArchiveData) throws DataException;
+
+    /**
+     * Deletes a block archive entry from the repository.
+     *
+     * @param blockArchiveData
+     * @throws DataException
+     */
+    public void delete(BlockArchiveData blockArchiveData) throws DataException;
+
+}

src/main/java/org/qortal/repository/BlockArchiveWriter.java

@@ -0,0 +1,193 @@
+package org.qortal.repository;
+
+import com.google.common.primitives.Ints;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.block.Block;
+import org.qortal.controller.Controller;
+import org.qortal.data.block.BlockArchiveData;
+import org.qortal.data.block.BlockData;
+import org.qortal.settings.Settings;
+import org.qortal.transform.TransformationException;
+import org.qortal.transform.block.BlockTransformer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+public class BlockArchiveWriter {
+
+    public enum BlockArchiveWriteResult {
+        OK,
+        STOPPING,
+        NOT_ENOUGH_BLOCKS,
+        BLOCK_LIMIT_REACHED,
+        BLOCK_NOT_FOUND
+    }
+
+    private static final Logger LOGGER = LogManager.getLogger(BlockArchiveWriter.class);
+
+    private int startHeight;
+    private final int endHeight;
+    private final Repository repository;
+
+    private int writtenCount;
+
+    public BlockArchiveWriter(int startHeight, int endHeight, Repository repository) {
+        this.startHeight = startHeight;
+        this.endHeight = endHeight;
+        this.repository = repository;
+    }
+
+    public static int getMaxArchiveHeight(Repository repository, boolean useMaximumDuplicatedLimit) throws DataException {
+        // We must only archive trimmed blocks, or the archive will grow far too large
+        final int accountSignaturesTrimStartHeight = repository.getBlockRepository().getOnlineAccountsSignaturesTrimHeight();
+        final int atTrimStartHeight = repository.getATRepository().getAtTrimHeight();
+        final int trimStartHeight = Math.min(accountSignaturesTrimStartHeight, atTrimStartHeight);
+
+        // In some cases we want to restrict the upper height of the archiver to save space
+        if (useMaximumDuplicatedLimit) {
+            // To save on disk space, it's best to not allow the archiver to get too far ahead of the pruner
+            // This reduces the amount of data that is held twice during the transition
+            final int blockPruneStartHeight = repository.getBlockRepository().getBlockPruneHeight();
+            final int atPruneStartHeight = repository.getATRepository().getAtPruneHeight();
+            final int pruneStartHeight = Math.min(blockPruneStartHeight, atPruneStartHeight);
+            final int maximumDuplicatedBlocks = Settings.getInstance().getMaxDuplicatedBlocksWhenArchiving();
+
+            // To summarize the above:
+            // - We must never archive anything greater than or equal to trimStartHeight
+            // - We should avoid archiving anything maximumDuplicatedBlocks higher than pruneStartHeight
+            return Math.min(trimStartHeight, pruneStartHeight + maximumDuplicatedBlocks);
+        }
+        else {
+            // We don't want to apply the maximum duplicated limit
+            return trimStartHeight;
+        }
+    }
+
+    public static boolean isArchiverUpToDate(Repository repository, boolean useMaximumDuplicatedLimit) throws DataException {
+        final int maxArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, useMaximumDuplicatedLimit);
+        final int actualArchiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight();
+        final float progress = (float)actualArchiveHeight / (float) maxArchiveHeight;
+        LOGGER.info(String.format("maxArchiveHeight: %d, actualArchiveHeight: %d, progress: %f",
+                maxArchiveHeight, actualArchiveHeight, progress));
+
+        // If archiver is within 90% of the maximum, treat it as up to date
+        // We need several percent as an allowance because the archiver will only
+        // save files when they reach the target size
+        return (progress >= 0.90);
+    }
+
+    public BlockArchiveWriteResult write() throws DataException, IOException, TransformationException, InterruptedException {
+        // Create the archive folder if it doesn't exist
+        // This is a subfolder of the db directory, to make bootstrapping easier
+        Path archivePath = Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toAbsolutePath();
+        try {
+            Files.createDirectories(archivePath);
+        } catch (IOException e) {
+            LOGGER.info("Unable to create archive folder");
+            throw new DataException("Unable to create archive folder");
+        }
+
+        // Determine start height of blocks to fetch
+        if (startHeight <= 2) {
+            // Skip genesis block, as it's not designed to be transmitted, and we can build that from blockchain.json
+            // TODO: include genesis block if we can
+            startHeight = 2;
+        }
+
+        // Header bytes will store the block indexes
+        ByteArrayOutputStream headerBytes = new ByteArrayOutputStream();
+        // Bytes will store the actual block data
+        ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+        LOGGER.info(String.format("Fetching blocks from height %d...", startHeight));
+        int i = 0;
+        long fileSizeTarget = 100 * 1024 * 1024; // 100MiB
+        while (headerBytes.size() + bytes.size() < fileSizeTarget) {
+            if (Controller.isStopping()) {
+                return BlockArchiveWriteResult.STOPPING;
+            }
+            if (Controller.getInstance().isSynchronizing()) {
+                continue;
+            }
+
+            int currentHeight = startHeight + i;
+            if (currentHeight >= endHeight) {
+                return BlockArchiveWriteResult.BLOCK_LIMIT_REACHED;
+            }
+
+            //LOGGER.info("Fetching block {}...", currentHeight);
+
+            BlockData blockData = repository.getBlockRepository().fromHeight(currentHeight);
+            if (blockData == null) {
+                return BlockArchiveWriteResult.BLOCK_NOT_FOUND;
+            }
+
+            // Write the signature and height into the BlockArchive table
+            BlockArchiveData blockArchiveData = new BlockArchiveData(blockData);
+            repository.getBlockArchiveRepository().save(blockArchiveData);
+            repository.saveChanges();
+
+            // Write the block data to some byte buffers
+            Block block = new Block(repository, blockData);
+            int blockIndex = bytes.size();
+            // Write block index to header
+            headerBytes.write(Ints.toByteArray(blockIndex));
+            // Write block height
+            bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
+            byte[] blockBytes = BlockTransformer.toBytes(block);
+            // Write block length
+            bytes.write(Ints.toByteArray(blockBytes.length));
+            // Write block bytes
+            bytes.write(blockBytes);
+            i++;
+
+        }
+        int totalLength = headerBytes.size() + bytes.size();
+        LOGGER.info(String.format("Total length of %d blocks is %d bytes", i, totalLength));
+
+        // Validate file size, in case something went wrong
+        if (totalLength < fileSizeTarget) {
+            return BlockArchiveWriteResult.NOT_ENOUGH_BLOCKS;
+        }
+
+        // We have enough blocks to create a new file
+        int endHeight = startHeight + i - 1;
+        int version = 1;
+        String filePath = String.format("%s/%d-%d.dat", archivePath.toString(), startHeight, endHeight);
+        FileOutputStream fileOutputStream = new FileOutputStream(filePath);
+        // Write version number
+        fileOutputStream.write(Ints.toByteArray(version));
+        // Write start height
+        fileOutputStream.write(Ints.toByteArray(startHeight));
+        // Write end height
+        fileOutputStream.write(Ints.toByteArray(endHeight));
+        // Write total count
+        fileOutputStream.write(Ints.toByteArray(i));
+        // Write dynamic header (block indexes) segment length
+        fileOutputStream.write(Ints.toByteArray(headerBytes.size()));
+        // Write dynamic header (block indexes) data
+        headerBytes.writeTo(fileOutputStream);
+        // Write data segment (block data) length
+        fileOutputStream.write(Ints.toByteArray(bytes.size()));
+        // Write data
+        bytes.writeTo(fileOutputStream);
+        // Close the file
+        fileOutputStream.close();
+
+        // Invalidate cache so that the rest of the app picks up the new file
+        BlockArchiveReader.getInstance().invalidateFileListCache();
+
+        this.writtenCount = i;
+        return BlockArchiveWriteResult.OK;
+    }
+
+    public int getWrittenCount() {
+        return this.writtenCount;
+    }
+
+}

src/main/java/org/qortal/repository/BlockRepository.java

@@ -137,11 +137,6 @@ public interface BlockRepository {
 	 */
 	public List<BlockSummaryData> getBlockSummaries(int firstBlockHeight, int lastBlockHeight) throws DataException;
 
-	/**
-	 * Returns block summaries for the passed height range, for API use.
-	 */
-	public List<BlockSummaryData> getBlockSummaries(Integer startHeight, Integer endHeight, Integer count) throws DataException;
-
 	/** Returns height of first trimmable online accounts signatures. */
 	public int getOnlineAccountsSignaturesTrimHeight() throws DataException;
 

src/main/java/org/qortal/repository/Repository.java

@@ -12,6 +12,8 @@ public interface Repository extends AutoCloseable {
 
 	public BlockRepository getBlockRepository();
 
+	public BlockArchiveRepository getBlockArchiveRepository();
+
 	public ChatRepository getChatRepository();
 
 	public CrossChainRepository getCrossChainRepository();

src/main/java/org/qortal/repository/RepositoryManager.java

@@ -2,6 +2,7 @@ package org.qortal.repository;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving;
 import org.qortal.repository.hsqldb.HSQLDBDatabasePruning;
 import org.qortal.settings.Settings;
 
@@ -57,9 +58,23 @@ public abstract class RepositoryManager {
 		}
 	}
 
-	public static void prune() {
+	public static boolean archive() {
+		// Bulk archive the database the first time we use archive mode
+		if (Settings.getInstance().isArchiveEnabled()) {
+			try {
+				return HSQLDBDatabaseArchiving.buildBlockArchive();
+
+			} catch (DataException e) {
+				LOGGER.info("Unable to bulk prune AT states. The database may have been left in an inconsistent state.");
+			}
+		}
+		return false;
+	}
+
+	public static boolean prune() {
 		// Bulk prune the database the first time we use pruning mode
-		if (Settings.getInstance().isPruningEnabled()) {
+		if (Settings.getInstance().isPruningEnabled() ||
+			Settings.getInstance().isArchiveEnabled()) {
 			try {
 				boolean prunedATStates = HSQLDBDatabasePruning.pruneATStates();
 				boolean prunedBlocks = HSQLDBDatabasePruning.pruneBlocks();
@@ -67,12 +82,14 @@ public abstract class RepositoryManager {
 				// Perform repository maintenance to shrink the db size down
 				if (prunedATStates && prunedBlocks) {
 					HSQLDBDatabasePruning.performMaintenance();
+					return true;
 				}
 
 			} catch (SQLException | DataException e) {
 				LOGGER.info("Unable to bulk prune AT states. The database may have been left in an inconsistent state.");
 			}
 		}
+		return false;
 	}
 
 	public static void setRequestedCheckpoint(Boolean quick) {

src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java

@@ -608,6 +608,7 @@ public class HSQLDBATRepository implements ATRepository {
 		// that could result in one process using a partial or empty dataset
 		// because it was in the process of being rebuilt by another thread
 		synchronized (this.repository.latestATStatesLock) {
+			LOGGER.trace("Rebuilding latest AT states...");
 
 			// Rebuild cache of latest AT states that we can't trim
 			String deleteSql = "DELETE FROM LatestATStates";
@@ -632,6 +633,8 @@ public class HSQLDBATRepository implements ATRepository {
 				repository.examineException(e);
 				throw new DataException("Unable to populate temporary latest AT states cache in repository", e);
 			}
+			this.repository.saveChanges();
+			LOGGER.trace("Rebuilt latest AT states");
 		}
 	}
 
@@ -661,7 +664,7 @@ public class HSQLDBATRepository implements ATRepository {
 				this.repository.executeCheckedUpdate(updateSql, trimHeight);
 				this.repository.saveChanges();
 			} catch (SQLException e) {
-				repository.examineException(e);
+				this.repository.examineException(e);
 				throw new DataException("Unable to set AT state trim height in repository", e);
 			}
 		}
@@ -689,7 +692,10 @@ public class HSQLDBATRepository implements ATRepository {
 					+ "LIMIT ?";
 
 			try {
-				return this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit);
+				int modifiedRows = this.repository.executeCheckedUpdate(sql, minHeight, maxHeight, limit);
+				this.repository.saveChanges();
+				return modifiedRows;
+
 			} catch (SQLException e) {
 				repository.examineException(e);
 				throw new DataException("Unable to trim AT states in repository", e);
@@ -757,7 +763,7 @@ public class HSQLDBATRepository implements ATRepository {
 						} while (resultSet.next());
 					}
 				} catch (SQLException e) {
-					throw new DataException("Unable to fetch flagged accounts from repository", e);
+					throw new DataException("Unable to fetch latest AT states from repository", e);
 				}
 
 				List<ATStateData> atStates = this.getBlockATStatesAtHeight(height);
@@ -785,6 +791,7 @@ public class HSQLDBATRepository implements ATRepository {
 					}
 				}
 			}
+			this.repository.saveChanges();
 
 			return deletedCount;
 		}

src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java

@@ -0,0 +1,277 @@
+package org.qortal.repository.hsqldb;
+
+import org.qortal.api.ApiError;
+import org.qortal.api.ApiExceptionFactory;
+import org.qortal.api.model.BlockSignerSummary;
+import org.qortal.data.block.BlockArchiveData;
+import org.qortal.data.block.BlockData;
+import org.qortal.data.block.BlockSummaryData;
+import org.qortal.repository.BlockArchiveReader;
+import org.qortal.repository.BlockArchiveRepository;
+import org.qortal.repository.DataException;
+import org.qortal.utils.Triple;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository {
+
+    protected HSQLDBRepository repository;
+
+    public HSQLDBBlockArchiveRepository(HSQLDBRepository repository) {
+        this.repository = repository;
+    }
+
+
+    @Override
+    public BlockData fromSignature(byte[] signature) throws DataException {
+        Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockWithSignature(signature, this.repository);
+        if (blockInfo != null) {
+            return (BlockData) blockInfo.getA();
+        }
+        return null;
+    }
+
+    @Override
+    public int getHeightFromSignature(byte[] signature) throws DataException {
+        Integer height = BlockArchiveReader.getInstance().fetchHeightForSignature(signature, this.repository);
+        if (height == null || height == 0) {
+            return 0;
+        }
+        return height;
+    }
+
+    @Override
+    public BlockData fromHeight(int height) throws DataException {
+        Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height);
+        if (blockInfo != null) {
+            return (BlockData) blockInfo.getA();
+        }
+        return null;
+    }
+
+    @Override
+    public BlockData fromReference(byte[] reference) throws DataException {
+        BlockData referenceBlock = this.repository.getBlockArchiveRepository().fromSignature(reference);
+        if (referenceBlock != null) {
+            int height = referenceBlock.getHeight();
+            if (height > 0) {
+                // Request the block at height + 1
+                Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height + 1);
+                if (blockInfo != null) {
+                    return (BlockData) blockInfo.getA();
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public int getHeightFromTimestamp(long timestamp) throws DataException {
+        String sql = "SELECT height FROM BlockArchive WHERE minted_when <= ? ORDER BY minted_when DESC, height DESC LIMIT 1";
+
+        try (ResultSet resultSet = this.repository.checkedExecute(sql, timestamp)) {
+            if (resultSet == null) {
+                return 0;
+            }
+            return resultSet.getInt(1);
+
+        } catch (SQLException e) {
+            throw new DataException("Error fetching height from BlockArchive repository", e);
+        }
+    }
+
+    @Override
+    public List<BlockSummaryData> getBlockSummariesBySigner(byte[] signerPublicKey, Integer limit, Integer offset, Boolean reverse) throws DataException {
+        StringBuilder sql = new StringBuilder(512);
+        sql.append("SELECT signature, height, BlockArchive.minter FROM ");
+
+        // List of minter account's public key and reward-share public keys with minter's public key
+        sql.append("(SELECT * FROM (VALUES (CAST(? AS QortalPublicKey))) UNION (SELECT reward_share_public_key FROM RewardShares WHERE minter_public_key = ?)) AS PublicKeys (public_key) ");
+
+        // Match BlockArchive blocks signed with public key from above list
+        sql.append("JOIN BlockArchive ON BlockArchive.minter = public_key ");
+
+        sql.append("ORDER BY BlockArchive.height ");
+        if (reverse != null && reverse)
+            sql.append("DESC ");
+
+        HSQLDBRepository.limitOffsetSql(sql, limit, offset);
+
+        List<BlockSummaryData> blockSummaries = new ArrayList<>();
+
+        try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), signerPublicKey, signerPublicKey)) {
+            if (resultSet == null)
+                return blockSummaries;
+
+            do {
+                byte[] signature = resultSet.getBytes(1);
+                int height = resultSet.getInt(2);
+                byte[] blockMinterPublicKey = resultSet.getBytes(3);
+
+                // Fetch additional info from the archive itself
+                int onlineAccountsCount = 0;
+                BlockData blockData = this.fromSignature(signature);
+                if (blockData != null) {
+                    onlineAccountsCount = blockData.getOnlineAccountsCount();
+                }
+
+                BlockSummaryData blockSummary = new BlockSummaryData(height, signature, blockMinterPublicKey, onlineAccountsCount);
+                blockSummaries.add(blockSummary);
+            } while (resultSet.next());
+
+            return blockSummaries;
+        } catch (SQLException e) {
+            throw new DataException("Unable to fetch minter's block summaries from repository", e);
+        }
+    }
+
+    @Override
+    public List<BlockSignerSummary> getBlockSigners(List<String> addresses, Integer limit, Integer offset, Boolean reverse) throws DataException {
+        String subquerySql = "SELECT minter, COUNT(signature) FROM (" +
+                    "(SELECT minter, signature FROM Blocks) UNION ALL (SELECT minter, signature FROM BlockArchive)" +
+                ") GROUP BY minter";
+
+        StringBuilder sql = new StringBuilder(1024);
+        sql.append("SELECT DISTINCT block_minter, n_blocks, minter_public_key, minter, recipient FROM (");
+        sql.append(subquerySql);
+        sql.append(") AS Minters (block_minter, n_blocks) LEFT OUTER JOIN RewardShares ON reward_share_public_key = block_minter ");
+
+        if (addresses != null && !addresses.isEmpty()) {
+            sql.append(" LEFT OUTER JOIN Accounts AS BlockMinterAccounts ON BlockMinterAccounts.public_key = block_minter ");
+            sql.append(" LEFT OUTER JOIN Accounts AS RewardShareMinterAccounts ON RewardShareMinterAccounts.public_key = minter_public_key ");
+            sql.append(" JOIN (VALUES ");
+
+            final int addressesSize = addresses.size();
+            for (int ai = 0; ai < addressesSize; ++ai) {
+                if (ai != 0)
+                    sql.append(", ");
+
+                sql.append("(?)");
+            }
+
+            sql.append(") AS FilterAccounts (account) ");
+            sql.append(" ON FilterAccounts.account IN (recipient, BlockMinterAccounts.account, RewardShareMinterAccounts.account) ");
+        } else {
+            addresses = Collections.emptyList();
+        }
+
+        sql.append("ORDER BY n_blocks ");
+        if (reverse != null && reverse)
+            sql.append("DESC ");
+
+        HSQLDBRepository.limitOffsetSql(sql, limit, offset);
+
+        List<BlockSignerSummary> summaries = new ArrayList<>();
+
+        try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), addresses.toArray())) {
+            if (resultSet == null)
+                return summaries;
+
+            do {
+                byte[] blockMinterPublicKey = resultSet.getBytes(1);
+                int nBlocks = resultSet.getInt(2);
+
+                // May not be present if no reward-share:
+                byte[] mintingAccountPublicKey = resultSet.getBytes(3);
+                String minterAccount = resultSet.getString(4);
+                String recipientAccount = resultSet.getString(5);
+
+                BlockSignerSummary blockSignerSummary;
+                if (recipientAccount == null)
+                    blockSignerSummary = new BlockSignerSummary(blockMinterPublicKey, nBlocks);
+                else
+                    blockSignerSummary = new BlockSignerSummary(blockMinterPublicKey, nBlocks, mintingAccountPublicKey, minterAccount, recipientAccount);
+
+                summaries.add(blockSignerSummary);
+            } while (resultSet.next());
+
+            return summaries;
+        } catch (SQLException e) {
+            throw new DataException("Unable to fetch block minters from repository", e);
+        }
+    }
+
+
+    @Override
+    public int getBlockArchiveHeight() throws DataException {
+        String sql = "SELECT block_archive_height FROM DatabaseInfo";
+
+        try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
+            if (resultSet == null)
+                return 0;
+
+            return resultSet.getInt(1);
+        } catch (SQLException e) {
+            throw new DataException("Unable to fetch block archive height from repository", e);
+        }
+    }
+
+    @Override
+    public void setBlockArchiveHeight(int archiveHeight) throws DataException {
+        // trimHeightsLock is to prevent concurrent update on DatabaseInfo
+        // that could result in "transaction rollback: serialization failure"
+        synchronized (this.repository.trimHeightsLock) {
+            String updateSql = "UPDATE DatabaseInfo SET block_archive_height = ?";
+
+            try {
+                this.repository.executeCheckedUpdate(updateSql, archiveHeight);
+                this.repository.saveChanges();
+            } catch (SQLException e) {
+                repository.examineException(e);
+                throw new DataException("Unable to set block archive height in repository", e);
+            }
+        }
+    }
+
+
+    @Override
+    public BlockArchiveData getBlockArchiveDataForSignature(byte[] signature) throws DataException {
+        String sql = "SELECT height, signature, minted_when, minter FROM BlockArchive WHERE signature = ? LIMIT 1";
+
+        try (ResultSet resultSet = this.repository.checkedExecute(sql, signature)) {
+            if (resultSet == null) {
+                return null;
+            }
+            int height = resultSet.getInt(1);
+            byte[] sig = resultSet.getBytes(2);
+            long timestamp = resultSet.getLong(3);
+            byte[] minterPublicKey = resultSet.getBytes(4);
+            return new BlockArchiveData(sig, height, timestamp, minterPublicKey);
+
+        } catch (SQLException e) {
+            throw new DataException("Error fetching height from BlockArchive repository", e);
+        }
+    }
+
+
+    @Override
+    public void save(BlockArchiveData blockArchiveData) throws DataException {
+        HSQLDBSaver saveHelper = new HSQLDBSaver("BlockArchive");
+
+        saveHelper.bind("signature", blockArchiveData.getSignature())
+                .bind("height", blockArchiveData.getHeight())
+                .bind("minted_when", blockArchiveData.getTimestamp())
+                .bind("minter", blockArchiveData.getMinterPublicKey());
+
+        try {
+            saveHelper.execute(this.repository);
+        } catch (SQLException e) {
+            throw new DataException("Unable to save SimpleBlockData into BlockArchive repository", e);
+        }
+    }
+
+    @Override
+    public void delete(BlockArchiveData blockArchiveData) throws DataException {
+        try {
+            this.repository.delete("BlockArchive",
+                    "block_signature = ?", blockArchiveData.getSignature());
+        } catch (SQLException e) {
+            throw new DataException("Unable to delete SimpleBlockData from BlockArchive repository", e);
+        }
+    }
+
+}

src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java

@@ -10,6 +10,7 @@ import org.qortal.api.model.BlockSignerSummary;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.block.BlockSummaryData;
 import org.qortal.data.block.BlockTransactionData;
+import org.qortal.data.block.BlockArchiveData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.repository.BlockRepository;
 import org.qortal.repository.DataException;
@@ -382,86 +383,6 @@ public class HSQLDBBlockRepository implements BlockRepository {
 		}
 	}
 
-	@Override
-	public List<BlockSummaryData> getBlockSummaries(Integer startHeight, Integer endHeight, Integer count) throws DataException {
-		StringBuilder sql = new StringBuilder(512);
-		List<Object> bindParams = new ArrayList<>();
-
-		sql.append("SELECT signature, height, minter, online_accounts_count, minted_when, transaction_count ");
-
-		/*
-		 * start	end		count		result
-		 * 10		40		null		blocks 10 to 39 (excludes end block, ignore count)
-		 * 
-		 * null		null	null		blocks 1 to 50 (assume count=50, maybe start=1)
-		 * 30		null	null		blocks 30 to 79 (assume count=50)
-		 * 30		null	10			blocks 30 to 39
-		 * 
-		 * null		null	50			last 50 blocks? so if max(blocks.height) is 200, then blocks 151 to 200
-		 * null		200		null		blocks 150 to 199 (excludes end block, assume count=50)
-		 * null		200		10			blocks 190 to 199 (excludes end block)
-		 */
-
-		if (startHeight != null && endHeight != null) {
-			sql.append("FROM Blocks ");
-			sql.append("WHERE height BETWEEN ? AND ?");
-			bindParams.add(startHeight);
-			bindParams.add(Integer.valueOf(endHeight - 1));
-		} else if (endHeight != null || (startHeight == null && count != null)) {
-			// we are going to return blocks from the end of the chain
-			if (count == null)
-				count = 50;
-
-			if (endHeight == null) {
-				sql.append("FROM (SELECT height FROM Blocks ORDER BY height DESC LIMIT 1) AS MaxHeights (max_height) ");
-				sql.append("JOIN Blocks ON height BETWEEN (max_height - ? + 1) AND max_height ");
-				bindParams.add(count);
-			} else {
-				sql.append("FROM Blocks ");
-				sql.append("WHERE height BETWEEN ? AND ?");
-				bindParams.add(Integer.valueOf(endHeight - count));
-				bindParams.add(Integer.valueOf(endHeight - 1));
-			}
-
-		} else {
-			// we are going to return blocks from the start of the chain
-			if (startHeight == null)
-				startHeight = 1;
-
-			if (count == null)
-				count = 50;
-
-			sql.append("FROM Blocks ");
-			sql.append("WHERE height BETWEEN ? AND ?");
-			bindParams.add(startHeight);
-			bindParams.add(Integer.valueOf(startHeight + count - 1));
-		}
-
-		List<BlockSummaryData> blockSummaries = new ArrayList<>();
-
-		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
-			if (resultSet == null)
-				return blockSummaries;
-
-			do {
-				byte[] signature = resultSet.getBytes(1);
-				int height = resultSet.getInt(2);
-				byte[] minterPublicKey = resultSet.getBytes(3);
-				int onlineAccountsCount = resultSet.getInt(4);
-				long timestamp = resultSet.getLong(5);
-				int transactionCount = resultSet.getInt(6);
-
-				BlockSummaryData blockSummary = new BlockSummaryData(height, signature, minterPublicKey, onlineAccountsCount,
-						timestamp, transactionCount);
-				blockSummaries.add(blockSummary);
-			} while (resultSet.next());
-
-			return blockSummaries;
-		} catch (SQLException e) {
-			throw new DataException("Unable to fetch height-ranged block summaries from repository", e);
-		}
-	}
-
 	@Override
 	public int getOnlineAccountsSignaturesTrimHeight() throws DataException {
 		String sql = "SELECT online_signatures_trim_height FROM DatabaseInfo";

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseArchiving.java

@@ -0,0 +1,87 @@
+package org.qortal.repository.hsqldb;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.controller.Controller;
+import org.qortal.repository.BlockArchiveWriter;
+import org.qortal.repository.DataException;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.transform.TransformationException;
+
+import java.io.IOException;
+
+/**
+ *
+ * When switching to an archiving node, we need to archive most of the database contents.
+ * This involves copying its data into flat files.
+ * If we do this entirely as a background process, it is very slow and can interfere with syncing.
+ * However, if we take the approach of doing this in bulk, before starting up the rest of the
+ * processes, this makes it much faster and less invasive.
+ *
+ * From that point, the original background archiving process will run, but can be dialled right down
+ * so not to interfere with syncing.
+ *
+ */
+
+
+public class HSQLDBDatabaseArchiving {
+
+    private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabaseArchiving.class);
+
+
+    public static boolean buildBlockArchive() throws DataException {
+        try (final HSQLDBRepository repository = (HSQLDBRepository)RepositoryManager.getRepository()) {
+
+            // Only build the archive if we have never done so before
+            int archiveHeight = repository.getBlockArchiveRepository().getBlockArchiveHeight();
+            if (archiveHeight > 0) {
+                // Already archived
+                return false;
+            }
+
+            LOGGER.info("Building block archive - this process could take a while... (approx. 15 mins on high spec)");
+
+            final int maximumArchiveHeight = BlockArchiveWriter.getMaxArchiveHeight(repository, false);
+            int startHeight = 0;
+
+            while (!Controller.isStopping()) {
+                try {
+                    BlockArchiveWriter writer = new BlockArchiveWriter(startHeight, maximumArchiveHeight, repository);
+                    BlockArchiveWriter.BlockArchiveWriteResult result = writer.write();
+                    switch (result) {
+                        case OK:
+                            // Increment block archive height
+                            startHeight += writer.getWrittenCount();
+                            repository.getBlockArchiveRepository().setBlockArchiveHeight(startHeight);
+                            repository.saveChanges();
+                            break;
+
+                        case STOPPING:
+                            return false;
+
+                        case BLOCK_LIMIT_REACHED:
+                        case NOT_ENOUGH_BLOCKS:
+                            // We've reached the limit of the blocks we can archive
+                            // Return from the whole method
+                            return true;
+
+                        case BLOCK_NOT_FOUND:
+                            // We tried to archive a block that didn't exist. This is a major failure and likely means
+                            // that a bootstrap or re-sync is needed. Return rom the method
+                            LOGGER.info("Error: block not found when building archive. If this error persists, " +
+                                    "a bootstrap or re-sync may be needed.");
+                            return false;
+                    }
+
+                } catch (IOException | TransformationException | InterruptedException e) {
+                    LOGGER.info("Caught exception when creating block cache", e);
+                    return false;
+                }
+            }
+        }
+
+        // If we got this far then something went wrong (most likely the app is stopping)
+        return false;
+    }
+
+}

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java

@@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.controller.Controller;
 import org.qortal.data.block.BlockData;
+import org.qortal.repository.BlockArchiveWriter;
 import org.qortal.repository.DataException;
 import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
@@ -36,6 +37,7 @@ public class HSQLDBDatabasePruning {
 
     private static final Logger LOGGER = LogManager.getLogger(HSQLDBDatabasePruning.class);
 
+
     public static boolean pruneATStates() throws SQLException, DataException {
         try (final HSQLDBRepository repository = (HSQLDBRepository)RepositoryManager.getRepository()) {
 
@@ -46,7 +48,18 @@ public class HSQLDBDatabasePruning {
                 return false;
             }
 
-            LOGGER.info("Starting bulk prune of AT states - this process could take a while... (approx. 2 mins on high spec)");
+            if (Settings.getInstance().isArchiveEnabled()) {
+                // Only proceed if we can see that the archiver has already finished
+                // This way, if the archiver failed for any reason, we can prune once it has had
+                // some opportunities to try again
+                boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository, false);
+                if (!upToDate) {
+                    return false;
+                }
+            }
+
+            LOGGER.info("Starting bulk prune of AT states - this process could take a while... " +
+                    "(approx. 2 mins on high spec, or upwards of 30 mins in some cases)");
 
             // Create new AT-states table to hold smaller dataset
             repository.executeCheckedUpdate("DROP TABLE IF EXISTS ATStatesNew");
@@ -68,11 +81,17 @@ public class HSQLDBDatabasePruning {
 
             // Calculate some constants for later use
             final int blockchainHeight = latestBlock.getHeight();
-            final int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit();
+            int maximumBlockToTrim = blockchainHeight - Settings.getInstance().getPruneBlockLimit();
+            if (Settings.getInstance().isArchiveEnabled()) {
+                // Archive mode - don't prune anything that hasn't been archived yet
+                maximumBlockToTrim = Math.min(maximumBlockToTrim, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1);
+            }
             final int startHeight = maximumBlockToTrim;
             final int endHeight = blockchainHeight;
             final int blockStep = 10000;
 
+
+
             // Loop through all the LatestATStates and copy them to the new table
             LOGGER.info("Copying AT states...");
             for (int height = 0; height < endHeight; height += blockStep) {
@@ -99,7 +118,7 @@ public class HSQLDBDatabasePruning {
                             }
 
                             if (height >= startHeight) {
-                                // Now copy this AT states for each recent block it is present in
+                                // Now copy this AT's states for each recent block they is present in
                                 for (int i = startHeight; i < endHeight; i++) {
                                     if (latestAtHeight < i) {
                                         // This AT finished before this block so there is nothing to copy
@@ -159,20 +178,25 @@ public class HSQLDBDatabasePruning {
     private static boolean pruneATStateData() throws SQLException, DataException {
         try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) {
 
+            if (Settings.getInstance().isArchiveEnabled()) {
+                // Don't prune ATStatesData in archive mode
+                return true;
+            }
+
             BlockData latestBlock = repository.getBlockRepository().getLastBlock();
             if (latestBlock == null) {
                 LOGGER.info("Unable to determine blockchain height, necessary for bulk ATStatesData pruning");
                 return false;
             }
             final int blockchainHeight = latestBlock.getHeight();
-            final int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit();
+            int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit();
             // ATStateData is already trimmed - so carry on from where we left off in the past
             int pruneStartHeight = repository.getATRepository().getAtTrimHeight();
 
             LOGGER.info("Starting bulk prune of AT states data - this process could take a while... (approx. 3 mins on high spec)");
 
             while (pruneStartHeight < upperPrunableHeight) {
-                // Prune all AT state data up until our latest minus pruneBlockLimit
+                // Prune all AT state data up until our latest minus pruneBlockLimit (or our archive height)
 
                 if (Controller.isStopping()) {
                     return false;
@@ -225,15 +249,30 @@ public class HSQLDBDatabasePruning {
                 return false;
             }
 
+            if (Settings.getInstance().isArchiveEnabled()) {
+                // Only proceed if we can see that the archiver has already finished
+                // This way, if the archiver failed for any reason, we can prune once it has had
+                // some opportunities to try again
+                boolean upToDate = BlockArchiveWriter.isArchiverUpToDate(repository, false);
+                if (!upToDate) {
+                    return false;
+                }
+            }
+
             BlockData latestBlock = repository.getBlockRepository().getLastBlock();
             if (latestBlock == null) {
                 LOGGER.info("Unable to determine blockchain height, necessary for bulk block pruning");
                 return false;
             }
             final int blockchainHeight = latestBlock.getHeight();
-            final int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit();
+            int upperPrunableHeight = blockchainHeight - Settings.getInstance().getPruneBlockLimit();
             int pruneStartHeight = 0;
 
+            if (Settings.getInstance().isArchiveEnabled()) {
+                // Archive mode - don't prune anything that hasn't been archived yet
+                upperPrunableHeight = Math.min(upperPrunableHeight, repository.getBlockArchiveRepository().getBlockArchiveHeight() - 1);
+            }
+
             LOGGER.info("Starting bulk prune of blocks - this process could take a while... (approx. 5 mins on high spec)");
 
             while (pruneStartHeight < upperPrunableHeight) {

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -873,6 +873,25 @@ public class HSQLDBDatabaseUpdates {
 					stmt.execute("ALTER TABLE DatabaseInfo ADD block_prune_height INT NOT NULL DEFAULT 0");
 					break;
 
+				case 36:
+					// Block archive support
+					stmt.execute("ALTER TABLE DatabaseInfo ADD block_archive_height INT NOT NULL DEFAULT 0");
+
+					// Block archive (lookup table to map signature to height)
+					// Actual data is stored in archive files outside of the database
+					stmt.execute("CREATE TABLE BlockArchive (signature BlockSignature, height INTEGER NOT NULL, "
+							+ "minted_when EpochMillis NOT NULL, minter QortalPublicKey NOT NULL, "
+							+ "PRIMARY KEY (signature))");
+					// For finding blocks by height.
+					stmt.execute("CREATE INDEX BlockArchiveHeightIndex ON BlockArchive (height)");
+					// For finding blocks by the account that minted them.
+					stmt.execute("CREATE INDEX BlockArchiveMinterIndex ON BlockArchive (minter)");
+					// For finding blocks by timestamp or finding height of latest block immediately before timestamp, etc.
+					stmt.execute("CREATE INDEX BlockArchiveTimestampHeightIndex ON BlockArchive (minted_when, height)");
+					// Use a separate table space as this table will be very large.
+					stmt.execute("SET TABLE BlockArchive NEW SPACE");
+					break;
+
 				default:
 					// nothing to do
 					return false;

src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java

@@ -31,22 +31,7 @@ import org.qortal.crypto.Crypto;
 import org.qortal.data.crosschain.TradeBotData;
 import org.qortal.globalization.Translator;
 import org.qortal.gui.SysTray;
-import org.qortal.repository.ATRepository;
+import org.qortal.repository.*;
-import org.qortal.repository.AccountRepository;
-import org.qortal.repository.ArbitraryRepository;
-import org.qortal.repository.AssetRepository;
-import org.qortal.repository.BlockRepository;
-import org.qortal.repository.ChatRepository;
-import org.qortal.repository.CrossChainRepository;
-import org.qortal.repository.DataException;
-import org.qortal.repository.GroupRepository;
-import org.qortal.repository.MessageRepository;
-import org.qortal.repository.NameRepository;
-import org.qortal.repository.NetworkRepository;
-import org.qortal.repository.Repository;
-import org.qortal.repository.RepositoryManager;
-import org.qortal.repository.TransactionRepository;
-import org.qortal.repository.VotingRepository;
 import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository;
 import org.qortal.settings.Settings;
 import org.qortal.utils.Base58;
@@ -76,6 +61,7 @@ public class HSQLDBRepository implements Repository {
 	private final ArbitraryRepository arbitraryRepository = new HSQLDBArbitraryRepository(this);
 	private final AssetRepository assetRepository = new HSQLDBAssetRepository(this);
 	private final BlockRepository blockRepository = new HSQLDBBlockRepository(this);
+	private final BlockArchiveRepository blockArchiveRepository = new HSQLDBBlockArchiveRepository(this);
 	private final ChatRepository chatRepository = new HSQLDBChatRepository(this);
 	private final CrossChainRepository crossChainRepository = new HSQLDBCrossChainRepository(this);
 	private final GroupRepository groupRepository = new HSQLDBGroupRepository(this);
@@ -143,6 +129,11 @@ public class HSQLDBRepository implements Repository {
 		return this.blockRepository;
 	}
 
+	@Override
+	public BlockArchiveRepository getBlockArchiveRepository() {
+		return this.blockArchiveRepository;
+	}
+
 	@Override
 	public ChatRepository getChatRepository() {
 		return this.chatRepository;

src/main/java/org/qortal/settings/Settings.java

@@ -129,6 +129,15 @@ public class Settings {
 	private int blockPruneBatchSize = 10000; // blocks
 
 
+	/** Whether we should archive old data to reduce the database size */
+	private boolean archiveEnabled = true;
+	/** How often to attempt archiving (ms). */
+	private long archiveInterval = 7171L; // milliseconds
+	/** The maximum number of blocks that can exist in both the
+	 * database and the archive at the same time */
+	private int maxDuplicatedBlocksWhenArchiving = 100000;
+
+
 	// Peer-to-peer related
 	private boolean isTestNet = false;
 	/** Port number for inbound peer-to-peer connections. */
@@ -574,4 +583,17 @@ public class Settings {
 		return this.blockPruneBatchSize;
 	}
 
+
+	public boolean isArchiveEnabled() {
+		return this.archiveEnabled;
+	}
+
+	public long getArchiveInterval() {
+		return this.archiveInterval;
+	}
+
+	public int getMaxDuplicatedBlocksWhenArchiving() {
+		return this.maxDuplicatedBlocksWhenArchiving;
+	}
+
 }