Added sync from genesis and reindex

src/main/java/org/qortal/api/restricted/resource/AdminResource.java

@@ -35,6 +35,7 @@ import org.qortal.data.account.RewardShareData;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
 import org.qortal.network.PeerAddress;
+import org.qortal.repository.ReindexManager;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
@@ -894,6 +895,50 @@ public class AdminResource {
 		}
 	}
 
+	@POST
+	@Path("/repository/reindex")
+	@Operation(
+			summary = "Reindex repository",
+			description = "Rebuilds all transactions and balances from archived blocks. Warning: takes around 1 week, and the core will not function normally during this time. If 'false' is returned, the database may be left in an inconsistent state, requiring another reindex or a bootstrap to correct it.",
+			responses = {
+					@ApiResponse(
+							description = "\"true\"",
+							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string"))
+					)
+			}
+	)
+	@ApiErrors({ApiError.REPOSITORY_ISSUE, ApiError.BLOCKCHAIN_NEEDS_SYNC})
+	@SecurityRequirement(name = "apiKey")
+	public String reindex(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+		Security.checkApiCallAllowed(request);
+
+		if (Synchronizer.getInstance().isSynchronizing())
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC);
+
+		try {
+			ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+
+			blockchainLock.lockInterruptibly();
+
+			try {
+				ReindexManager reindexManager = new ReindexManager();
+				reindexManager.reindex();
+				return "true";
+
+			} catch (DataException e) {
+				LOGGER.info("DataException when reindexing: {}", e.getMessage());
+
+			} finally {
+				blockchainLock.unlock();
+			}
+		} catch (InterruptedException e) {
+			// We couldn't lock blockchain to perform reindex
+			return "false";
+		}
+
+		return "false";
+	}
+
 	@DELETE
 	@Path("/repository")
 	@Operation(
@@ -966,8 +1011,6 @@ public class AdminResource {
 		}
 	}
 
-
-
 	@POST
 	@Path("/apikey/generate")
 	@Operation(

src/main/java/org/qortal/block/Block.java

@@ -1285,10 +1285,18 @@ public class Block {
 				// Apply fix for block 212937 but fix will be rolled back before we exit method
 				Block212937.processFix(this);
 			}
+			else if (this.blockData.getHeight() == 1333492) {
+				// Apply fix for block 1333492 but fix will be rolled back before we exit method
+				Block1333492.processFix(this);
+			}
 			else if (InvalidNameRegistrationBlocks.isAffectedBlock(this.blockData.getHeight())) {
 				// Apply fix for affected name registration blocks, but fix will be rolled back before we exit method
 				InvalidNameRegistrationBlocks.processFix(this);
 			}
+			else if (InvalidBalanceBlocks.isAffectedBlock(this.blockData.getHeight())) {
+				// Apply fix for affected balance blocks, but fix will be rolled back before we exit method
+				InvalidBalanceBlocks.processFix(this);
+			}
 
 			for (Transaction transaction : this.getTransactions()) {
 				TransactionData transactionData = transaction.getTransactionData();
@@ -1554,16 +1562,21 @@ public class Block {
 				// Apply fix for block 212937
 				Block212937.processFix(this);
 			}
-
+			else if (this.blockData.getHeight() == 1333492) {
-			if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV1Height()) {
+				// Apply fix for block 1333492
+				Block1333492.processFix(this);
+			}
+			else if (InvalidBalanceBlocks.isAffectedBlock(this.blockData.getHeight())) {
+				// Apply fix for affected balance blocks
+				InvalidBalanceBlocks.processFix(this);
+			}
+			else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV1Height()) {
 				SelfSponsorshipAlgoV1Block.processAccountPenalties(this);
 			}
-
+			else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV2Height()) {
-			if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV2Height()) {
 				SelfSponsorshipAlgoV2Block.processAccountPenalties(this);
 			}
-
+			else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV3Height()) {
-			if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV3Height()) {
 				SelfSponsorshipAlgoV3Block.processAccountPenalties(this);
 			}
 		}
@@ -1854,16 +1867,21 @@ public class Block {
 				// Revert fix for block 212937
 				Block212937.orphanFix(this);
 			}
-
+			else if (this.blockData.getHeight() == 1333492) {
-			if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV1Height()) {
+				// Revert fix for block 1333492
+				Block1333492.orphanFix(this);
+			}
+			else if (InvalidBalanceBlocks.isAffectedBlock(this.blockData.getHeight())) {
+				// Revert fix for affected balance blocks
+				InvalidBalanceBlocks.orphanFix(this);
+			}
+			else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV1Height()) {
 				SelfSponsorshipAlgoV1Block.orphanAccountPenalties(this);
 			}
-
+			else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV2Height()) {
-			if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV2Height()) {
 				SelfSponsorshipAlgoV2Block.orphanAccountPenalties(this);
 			}
-
+			else if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV3Height()) {
-			if (this.blockData.getHeight() == BlockChain.getInstance().getSelfSponsorshipAlgoV3Height()) {
 				SelfSponsorshipAlgoV3Block.orphanAccountPenalties(this);
 			}
 			

src/main/java/org/qortal/block/Block1333492.java

@@ -0,0 +1,101 @@
+package org.qortal.block;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.eclipse.persistence.jaxb.JAXBContextFactory;
+import org.eclipse.persistence.jaxb.UnmarshallerProperties;
+import org.qortal.data.account.AccountBalanceData;
+import org.qortal.repository.DataException;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.UnmarshalException;
+import javax.xml.bind.Unmarshaller;
+import javax.xml.transform.stream.StreamSource;
+import java.io.InputStream;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Block 1333492
+ * <p>
+ * As described in InvalidBalanceBlocks.java, legacy bugs caused a small drift in account balances.
+ * This block adjusts any remaining differences between a clean reindex/resync and a recent bootstrap.
+ * <p>
+ * The block height 1333492 isn't significant - it's simply the height of a recent bootstrap at the
+ * time of development, so that the account balances could be accessed and compared against the same
+ * block in a reindexed db.
+ * <p>
+ * As with InvalidBalanceBlocks, the discrepancies are insignificant, except for a single
+ * account which has a 3.03 QORT discrepancy. This was due to the account being the first recipient
+ * of a name sale and encountering an early bug in this area.
+ * <p>
+ * The total offset for this block is 3.02816514 QORT.
+ */
+public final class Block1333492 {
+
+	private static final Logger LOGGER = LogManager.getLogger(Block1333492.class);
+	private static final String ACCOUNT_DELTAS_SOURCE = "block-1333492-deltas.json";
+
+	private static final List<AccountBalanceData> accountDeltas = readAccountDeltas();
+
+	private Block1333492() {
+		/* Do not instantiate */
+	}
+
+	@SuppressWarnings("unchecked")
+	private static List<AccountBalanceData> readAccountDeltas() {
+		Unmarshaller unmarshaller;
+
+		try {
+			// Create JAXB context aware of classes we need to unmarshal
+			JAXBContext jc = JAXBContextFactory.createContext(new Class[] {
+				AccountBalanceData.class
+			}, null);
+
+			// Create unmarshaller
+			unmarshaller = jc.createUnmarshaller();
+
+			// Set the unmarshaller media type to JSON
+			unmarshaller.setProperty(UnmarshallerProperties.MEDIA_TYPE, "application/json");
+
+			// Tell unmarshaller that there's no JSON root element in the JSON input
+			unmarshaller.setProperty(UnmarshallerProperties.JSON_INCLUDE_ROOT, false);
+		} catch (JAXBException e) {
+			String message = "Failed to setup unmarshaller to read block 1333492 deltas";
+			LOGGER.error(message, e);
+			throw new RuntimeException(message, e);
+		}
+
+		ClassLoader classLoader = BlockChain.class.getClassLoader();
+		InputStream in = classLoader.getResourceAsStream(ACCOUNT_DELTAS_SOURCE);
+		StreamSource jsonSource = new StreamSource(in);
+
+		try  {
+			// Attempt to unmarshal JSON stream to BlockChain config
+			return (List<AccountBalanceData>) unmarshaller.unmarshal(jsonSource, AccountBalanceData.class).getValue();
+		} catch (UnmarshalException e) {
+			String message = "Failed to parse block 1333492 deltas";
+			LOGGER.error(message, e);
+			throw new RuntimeException(message, e);
+		} catch (JAXBException e) {
+			String message = "Unexpected JAXB issue while processing block 1333492 deltas";
+			LOGGER.error(message, e);
+			throw new RuntimeException(message, e);
+		}
+	}
+
+	public static void processFix(Block block) throws DataException {
+		block.repository.getAccountRepository().modifyAssetBalances(accountDeltas);
+	}
+
+	public static void orphanFix(Block block) throws DataException {
+		// Create inverse deltas
+		List<AccountBalanceData> inverseDeltas = accountDeltas.stream()
+				.map(delta -> new AccountBalanceData(delta.getAddress(), delta.getAssetId(), 0 - delta.getBalance()))
+				.collect(Collectors.toList());
+
+		block.repository.getAccountRepository().modifyAssetBalances(inverseDeltas);
+	}
+
+}

src/main/java/org/qortal/block/BlockChain.java

@@ -7,6 +7,7 @@ import org.eclipse.persistence.jaxb.JAXBContextFactory;
 import org.eclipse.persistence.jaxb.UnmarshallerProperties;
 import org.qortal.controller.Controller;
 import org.qortal.data.block.BlockData;
+import org.qortal.data.network.PeerData;
 import org.qortal.network.Network;
 import org.qortal.repository.*;
 import org.qortal.settings.Settings;
@@ -24,6 +25,7 @@ import javax.xml.transform.stream.StreamSource;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.InputStream;
+import java.security.SecureRandom;
 import java.sql.SQLException;
 import java.util.*;
 import java.util.concurrent.locks.ReentrantLock;
@@ -80,7 +82,8 @@ public class BlockChain {
 		arbitraryOptionalFeeTimestamp,
 		unconfirmableRewardSharesHeight,
 		disableTransferPrivsTimestamp,
-		enableTransferPrivsTimestamp
+		enableTransferPrivsTimestamp,
+		cancelSellNameValidationTimestamp
 	}
 
 	// Custom transaction fees
@@ -610,6 +613,10 @@ public class BlockChain {
 		return this.featureTriggers.get(FeatureTrigger.enableTransferPrivsTimestamp.name()).longValue();
 	}
 
+	public long getCancelSellNameValidationTimestamp() {
+		return this.featureTriggers.get(FeatureTrigger.cancelSellNameValidationTimestamp.name()).longValue();
+	}
+
 	// More complex getters for aspects that change by height or timestamp
 
 	public long getRewardAtHeight(int ourHeight) {
@@ -805,10 +812,12 @@ public class BlockChain {
 		boolean isLite = Settings.getInstance().isLite();
 		boolean canBootstrap = Settings.getInstance().getBootstrap();
 		boolean needsArchiveRebuild = false;
+		int checkHeight = 0;
 		BlockData chainTip;
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			chainTip = repository.getBlockRepository().getLastBlock();
+			checkHeight = repository.getBlockRepository().getBlockchainHeight();
 
 			// Ensure archive is (at least partially) intact, and force a bootstrap if it isn't
 			if (!isTopOnly && archiveEnabled && canBootstrap) {
@@ -824,6 +833,17 @@ public class BlockChain {
 				}
 			}
 
+			if (!canBootstrap) {
+				if (checkHeight > 2) {
+					LOGGER.info("Retrieved block 2 from archive. Syncing from genesis block resumed!");
+				} else {
+					needsArchiveRebuild = (repository.getBlockArchiveRepository().fromHeight(2) == null);
+					if (needsArchiveRebuild) {
+						LOGGER.info("Couldn't retrieve block 2 from archive. Bootstrapping is disabled. Syncing from genesis block!");
+					}
+				}
+			}
+
 			// Validate checkpoints
 			// Limited to topOnly nodes for now, in order to reduce risk, and to solve a real-world problem with divergent topOnly nodes
 			// TODO: remove the isTopOnly conditional below once this feature has had more testing time
@@ -856,11 +876,12 @@ public class BlockChain {
 
 		// Check first block is Genesis Block
 		if (!isGenesisBlockValid() || needsArchiveRebuild) {
-			try {
+			if (checkHeight < 3) {
-				rebuildBlockchain();
+				try {
-
+					rebuildBlockchain();
-			} catch (InterruptedException e) {
+				} catch (InterruptedException e) {
-				throw new DataException(String.format("Interrupted when trying to rebuild blockchain: %s", e.getMessage()));
+					throw new DataException(String.format("Interrupted when trying to rebuild blockchain: %s", e.getMessage()));
+				}
 			}
 		}
 
@@ -1001,5 +1022,4 @@ public class BlockChain {
 			blockchainLock.unlock();
 		}
 	}
-
 }

src/main/java/org/qortal/block/InvalidBalanceBlocks.java

@@ -0,0 +1,134 @@
+package org.qortal.block;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.eclipse.persistence.jaxb.JAXBContextFactory;
+import org.eclipse.persistence.jaxb.UnmarshallerProperties;
+import org.qortal.data.account.AccountBalanceData;
+import org.qortal.repository.DataException;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.UnmarshalException;
+import javax.xml.bind.Unmarshaller;
+import javax.xml.transform.stream.StreamSource;
+import java.io.InputStream;
+import java.util.*;
+import java.util.stream.Collectors;
+
+
+/**
+ * Due to various bugs - which have been fixed - a small amount of balance drift occurred
+ * in the chainstate of running nodes and bootstraps, when compared with a clean sync from genesis.
+ * This resulted in a significant number of invalid transactions in the chain history due to
+ * subtle balance discrepancies. The sum of all discrepancies that resulted in an invalid
+ * transaction is 0.00198322 QORT, so despite the large quantity of transactions, they
+ * represent an insignificant amount when summed.
+ * <p>
+ * This class is responsible for retroactively fixing all the past transactions which
+ * are invalid due to the balance discrepancies.
+ */
+
+
+public final class InvalidBalanceBlocks {
+
+	private static final Logger LOGGER = LogManager.getLogger(InvalidBalanceBlocks.class);
+
+	private static final String ACCOUNT_DELTAS_SOURCE = "invalid-transaction-balance-deltas.json";
+
+	private static final List<AccountBalanceData> accountDeltas = readAccountDeltas();
+	private static final List<Integer> affectedHeights = getAffectedHeights();
+
+	private InvalidBalanceBlocks() {
+		/* Do not instantiate */
+	}
+
+	@SuppressWarnings("unchecked")
+	private static List<AccountBalanceData> readAccountDeltas() {
+		Unmarshaller unmarshaller;
+
+		try {
+			// Create JAXB context aware of classes we need to unmarshal
+			JAXBContext jc = JAXBContextFactory.createContext(new Class[] {
+					AccountBalanceData.class
+			}, null);
+
+			// Create unmarshaller
+			unmarshaller = jc.createUnmarshaller();
+
+			// Set the unmarshaller media type to JSON
+			unmarshaller.setProperty(UnmarshallerProperties.MEDIA_TYPE, "application/json");
+
+			// Tell unmarshaller that there's no JSON root element in the JSON input
+			unmarshaller.setProperty(UnmarshallerProperties.JSON_INCLUDE_ROOT, false);
+		} catch (JAXBException e) {
+			String message = "Failed to setup unmarshaller to read block 212937 deltas";
+			LOGGER.error(message, e);
+			throw new RuntimeException(message, e);
+		}
+
+		ClassLoader classLoader = BlockChain.class.getClassLoader();
+		InputStream in = classLoader.getResourceAsStream(ACCOUNT_DELTAS_SOURCE);
+		StreamSource jsonSource = new StreamSource(in);
+
+		try  {
+			// Attempt to unmarshal JSON stream to BlockChain config
+			return (List<AccountBalanceData>) unmarshaller.unmarshal(jsonSource, AccountBalanceData.class).getValue();
+		} catch (UnmarshalException e) {
+			String message = "Failed to parse balance deltas";
+			LOGGER.error(message, e);
+			throw new RuntimeException(message, e);
+		} catch (JAXBException e) {
+			String message = "Unexpected JAXB issue while processing balance deltas";
+			LOGGER.error(message, e);
+			throw new RuntimeException(message, e);
+		}
+	}
+
+	private static List<Integer> getAffectedHeights() {
+		List<Integer> heights = new ArrayList<>();
+		for (AccountBalanceData accountBalanceData : accountDeltas) {
+			if (!heights.contains(accountBalanceData.getHeight())) {
+				heights.add(accountBalanceData.getHeight());
+			}
+		}
+		return heights;
+	}
+
+	private static List<AccountBalanceData> getAccountDeltasAtHeight(int height) {
+		return accountDeltas.stream().filter(a -> a.getHeight() == height).collect(Collectors.toList());
+	}
+
+	public static boolean isAffectedBlock(int height) {
+		return affectedHeights.contains(Integer.valueOf(height));
+	}
+
+	public static void processFix(Block block) throws DataException {
+		Integer blockHeight = block.getBlockData().getHeight();
+		List<AccountBalanceData> deltas = getAccountDeltasAtHeight(blockHeight);
+		if (deltas == null) {
+			throw new DataException(String.format("Unable to lookup invalid balance data for block height %d", blockHeight));
+		}
+
+		block.repository.getAccountRepository().modifyAssetBalances(deltas);
+
+		LOGGER.info("Applied balance patch for block {}", blockHeight);
+	}
+
+	public static void orphanFix(Block block) throws DataException {
+		Integer blockHeight = block.getBlockData().getHeight();
+		List<AccountBalanceData> deltas = getAccountDeltasAtHeight(blockHeight);
+		if (deltas == null) {
+			throw new DataException(String.format("Unable to lookup invalid balance data for block height %d", blockHeight));
+		}
+
+		// Create inverse delta(s)
+		for (AccountBalanceData accountBalanceData : deltas) {
+			AccountBalanceData inverseBalanceData = new AccountBalanceData(accountBalanceData.getAddress(), accountBalanceData.getAssetId(), -accountBalanceData.getBalance());
+			block.repository.getAccountRepository().modifyAssetBalances(List.of(inverseBalanceData));
+		}
+
+		LOGGER.info("Reverted balance patch for block {}", blockHeight);
+	}
+
+}

src/main/java/org/qortal/controller/Controller.java

@@ -32,6 +32,7 @@ import org.qortal.gui.Gui;
 import org.qortal.gui.SysTray;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
+import org.qortal.network.PeerAddress;
 import org.qortal.network.message.*;
 import org.qortal.repository.*;
 import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
@@ -48,8 +49,11 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.SecureRandom;
 import java.security.Security;
 import java.time.LocalDateTime;
 import java.time.ZoneOffset;
@@ -592,6 +596,83 @@ public class Controller extends Thread {
 				}
 			}
 		}, 10*60*1000, 10*60*1000);
+
+		// Check if we need sync from genesis and start syncing
+		Timer syncFromGenesis = new Timer();
+		syncFromGenesis.schedule(new TimerTask() {
+			@Override
+			public void run() {
+				LOGGER.debug("Start sync from genesis check.");
+				boolean canBootstrap = Settings.getInstance().getBootstrap();
+				boolean needsArchiveRebuild = false;
+				int checkHeight = 0;
+				Repository repository = null;
+
+				try {
+					repository = RepositoryManager.getRepository();
+					needsArchiveRebuild = (repository.getBlockArchiveRepository().fromHeight(2) == null);
+					checkHeight = repository.getBlockRepository().getBlockchainHeight();
+				} catch (DataException e) {
+					throw new RuntimeException(e);
+				}
+
+				if (canBootstrap) {
+					LOGGER.debug("Bootstrapping is enabled, cancel sync from genesis check.");
+					syncFromGenesis.cancel();
+					return;
+				}
+
+				if (!needsArchiveRebuild) {
+					LOGGER.debug("We have more than 2 blocks, cancel sync from genesis check.");
+					syncFromGenesis.cancel();
+					return;
+				}
+
+				if (checkHeight > 3) {
+					LOGGER.debug("We have more than 2 blocks, cancel sync from genesis check.");
+					syncFromGenesis.cancel();
+					return;
+				}
+
+				if (needsArchiveRebuild && !canBootstrap) {
+					LOGGER.info("Start syncing from genesis!");
+					List<Peer> seeds = new ArrayList<>(Network.getInstance().getImmutableHandshakedPeers());
+
+					if (seeds.isEmpty()) {
+						LOGGER.info("No connected peers, will try again later.");
+						return;
+					}
+
+					int index = new SecureRandom().nextInt(seeds.size());
+					String syncNode = String.valueOf(seeds.get(index));
+					PeerAddress peerAddress = PeerAddress.fromString(syncNode);
+					InetSocketAddress resolvedAddress = null;
+
+					try {
+						resolvedAddress = peerAddress.toSocketAddress();
+					} catch (UnknownHostException e) {
+						throw new RuntimeException(e);
+					}
+
+					InetSocketAddress finalResolvedAddress = resolvedAddress;
+					Peer targetPeer = seeds.stream().filter(peer -> peer.getResolvedAddress().equals(finalResolvedAddress)).findFirst().orElse(null);
+					Synchronizer.SynchronizationResult syncResult;
+
+					try {
+						do {
+							try {
+								syncResult = Synchronizer.getInstance().actuallySynchronize(targetPeer, true);
+							} catch (InterruptedException e) {
+								throw new RuntimeException(e);
+							}
+						}
+						while (syncResult == Synchronizer.SynchronizationResult.OK);
+					} finally {
+						syncFromGenesis.cancel();
+					}
+				}
+			}
+		}, 3*60*1000, 3*60*1000);
 	}
 
 	/** Called by AdvancedInstaller's launch EXE in single-instance mode, when an instance is already running. */

src/main/java/org/qortal/repository/ReindexManager.java

@@ -0,0 +1,213 @@
+package org.qortal.repository;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.block.Block;
+import org.qortal.block.GenesisBlock;
+import org.qortal.controller.Controller;
+import org.qortal.data.block.BlockArchiveData;
+import org.qortal.data.block.BlockData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.settings.Settings;
+import org.qortal.transaction.Transaction;
+import org.qortal.transform.block.BlockTransformation;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+
+import java.util.concurrent.TimeoutException;
+
+public class ReindexManager {
+
+    private static final Logger LOGGER = LogManager.getLogger(ReindexManager.class);
+
+    private Repository repository;
+
+    private final int pruneAndTrimBlockInterval = 2000;
+    private final int maintenanceBlockInterval = 50000;
+
+    private boolean resume = false;
+
+    public ReindexManager() {
+
+    }
+
+    public void reindex() throws DataException {
+        try {
+            this.runPreChecks();
+            this.rebuildRepository();
+
+            try (final Repository repository = RepositoryManager.getRepository()) {
+                this.repository = repository;
+                this.requestCheckpoint();
+                this.processGenesisBlock();
+                this.processBlocks();
+            }
+
+        } catch (InterruptedException e) {
+            throw new DataException("Interrupted before complete");
+        }
+    }
+
+    private void runPreChecks() throws DataException, InterruptedException {
+        LOGGER.info("Running pre-checks...");
+        if (Settings.getInstance().isTopOnly()) {
+            throw new DataException("Reindexing not supported in top-only mode. Please bootstrap or resync from genesis.");
+        }
+        if (Settings.getInstance().isLite()) {
+            throw new DataException("Reindexing not supported in lite mode.");
+        }
+
+        while (NTP.getTime() == null) {
+            LOGGER.info("Waiting for NTP...");
+            Thread.sleep(5000L);
+        }
+    }
+
+    private void rebuildRepository() throws DataException {
+        if (resume) {
+            return;
+        }
+
+        LOGGER.info("Rebuilding repository...");
+        RepositoryManager.rebuild();
+    }
+
+    private void requestCheckpoint() {
+        RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
+    }
+
+    private void processGenesisBlock() throws DataException, InterruptedException {
+        if (resume) {
+            return;
+        }
+
+        LOGGER.info("Processing genesis block...");
+
+        GenesisBlock genesisBlock = GenesisBlock.getInstance(repository);
+
+        // Add Genesis Block to blockchain
+        genesisBlock.process();
+
+        this.repository.saveChanges();
+    }
+
+    private void processBlocks() throws DataException {
+        LOGGER.info("Processing blocks...");
+
+        int height = this.repository.getBlockRepository().getBlockchainHeight();
+        while (true) {
+            height++;
+
+            boolean processed = this.processBlock(height);
+            if (!processed) {
+                LOGGER.info("Block {} couldn't be processed. If this is the last archived block, then the process is complete.", height);
+                break; // TODO: check if complete
+            }
+
+            // Prune and trim regularly, leaving a buffer
+            if (height >= pruneAndTrimBlockInterval*2 && height % pruneAndTrimBlockInterval == 0) {
+                int startHeight = Math.max(height - pruneAndTrimBlockInterval*2, 2);
+                int endHeight = height - pruneAndTrimBlockInterval;
+                LOGGER.info("Pruning and trimming blocks {} to {}...", startHeight, endHeight);
+                this.repository.getATRepository().rebuildLatestAtStates(height - 250);
+                this.repository.saveChanges();
+                this.prune(startHeight, endHeight);
+                this.trim(startHeight, endHeight);
+            }
+
+            // Run repository maintenance regularly, to keep blockchain.data size down
+            if (height % maintenanceBlockInterval == 0) {
+                this.runRepositoryMaintenance();
+            }
+        }
+    }
+
+    private boolean processBlock(int height) throws DataException {
+        Block block = this.fetchBlock(height);
+        if (block == null) {
+            return false;
+        }
+
+        // Transactions are stored without approval status so determine that now
+        for (Transaction transaction : block.getTransactions())
+            transaction.setInitialApprovalStatus();
+
+        // It's best not to run preProcess() until there is a reason to
+        // block.preProcess();
+
+        Block.ValidationResult validationResult = block.isValid();
+        if (validationResult != Block.ValidationResult.OK) {
+            throw new DataException(String.format("Invalid block at height %d: %s", height, validationResult));
+        }
+
+        // Save transactions attached to this block
+        for (Transaction transaction : block.getTransactions()) {
+            TransactionData transactionData = transaction.getTransactionData();
+            this.repository.getTransactionRepository().save(transactionData);
+        }
+
+        block.process();
+
+        LOGGER.info(String.format("Reindexed block height %d, sig %.8s", block.getBlockData().getHeight(), Base58.encode(block.getBlockData().getSignature())));
+
+        // Add to block archive table, since this originated from the archive but the chainstate has to be rebuilt
+        this.addToBlockArchive(block.getBlockData());
+
+        this.repository.saveChanges();
+
+        Controller.getInstance().onNewBlock(block.getBlockData());
+
+        return true;
+    }
+
+    private Block fetchBlock(int height) {
+        BlockTransformation b = BlockArchiveReader.getInstance().fetchBlockAtHeight(height);
+        if (b != null) {
+            if (b.getAtStatesHash() != null) {
+                return new Block(this.repository, b.getBlockData(), b.getTransactions(), b.getAtStatesHash());
+            }
+            else {
+                return new Block(this.repository, b.getBlockData(), b.getTransactions(), b.getAtStates());
+            }
+        }
+
+        return null;
+    }
+
+    private void addToBlockArchive(BlockData blockData) throws DataException {
+        // Write the signature and height into the BlockArchive table
+        BlockArchiveData blockArchiveData = new BlockArchiveData(blockData);
+        this.repository.getBlockArchiveRepository().save(blockArchiveData);
+        this.repository.getBlockArchiveRepository().setBlockArchiveHeight(blockData.getHeight()+1);
+        this.repository.saveChanges();
+    }
+
+    private void prune(int startHeight, int endHeight) throws DataException {
+        this.repository.getBlockRepository().pruneBlocks(startHeight, endHeight);
+        this.repository.getATRepository().pruneAtStates(startHeight, endHeight);
+        this.repository.getATRepository().setAtPruneHeight(endHeight+1);
+        this.repository.saveChanges();
+    }
+
+    private void trim(int startHeight, int endHeight) throws DataException {
+        this.repository.getBlockRepository().trimOldOnlineAccountsSignatures(startHeight, endHeight);
+
+        int count = 1; // Any number greater than 0
+        while (count > 0) {
+            count = this.repository.getATRepository().trimAtStates(startHeight, endHeight, Settings.getInstance().getAtStatesTrimLimit());
+        }
+
+        this.repository.getBlockRepository().setBlockPruneHeight(endHeight+1);
+        this.repository.getATRepository().setAtTrimHeight(endHeight+1);
+        this.repository.saveChanges();
+    }
+
+    private void runRepositoryMaintenance() throws DataException {
+        try {
+            this.repository.performPeriodicMaintenance(1000L);
+        } catch (TimeoutException e) {
+            LOGGER.info("Timed out waiting for repository before running maintenance");
+        }
+    }
+
+}

src/main/java/org/qortal/transaction/CancelSellNameTransaction.java

@@ -3,6 +3,7 @@ package org.qortal.transaction;
 import com.google.common.base.Utf8;
 import org.qortal.account.Account;
 import org.qortal.asset.Asset;
+import org.qortal.block.BlockChain;
 import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
 import org.qortal.data.naming.NameData;
 import org.qortal.data.transaction.CancelSellNameTransactionData;
@@ -63,8 +64,11 @@ public class CancelSellNameTransaction extends Transaction {
 			return ValidationResult.NAME_DOES_NOT_EXIST;
 
 		// Check name is currently for sale
-		if (!nameData.isForSale())
+		if (!nameData.isForSale()) {
-			return ValidationResult.NAME_NOT_FOR_SALE;
+			// Only validate after feature-trigger timestamp, due to a small number of double cancelations in the chain history
+			if (this.cancelSellNameTransactionData.getTimestamp() > BlockChain.getInstance().getCancelSellNameValidationTimestamp())
+				return ValidationResult.NAME_NOT_FOR_SALE;
+		}
 
 		// Check transaction creator matches name's current owner
 		Account owner = getOwner();

src/main/resources/blockchain.json

@@ -103,7 +103,8 @@
 		"arbitraryOptionalFeeTimestamp": 1680278400000,
 		"unconfirmableRewardSharesHeight": 1575500,
 		"disableTransferPrivsTimestamp": 1706745000000,
-		"enableTransferPrivsTimestamp": 1709251200000
+		"enableTransferPrivsTimestamp": 1709251200000,
+		"cancelSellNameValidationTimestamp": 1676986362069
 	},
 	"checkpoints": [
 		{ "height": 1136300, "signature": "3BbwawEF2uN8Ni5ofpJXkukoU8ctAPxYoFB7whq9pKfBnjfZcpfEJT4R95NvBDoTP8WDyWvsUvbfHbcr9qSZuYpSKZjUQTvdFf6eqznHGEwhZApWfvXu6zjGCxYCp65F4jsVYYJjkzbjmkCg5WAwN5voudngA23kMK6PpTNygapCzXt" }