Add block_sequence to Transactions table, and populate all past transactions.

This data was being lost when pruning the BlockTransactions table.

Note: on first run this will reshape the db, which can take several minutes.

src/main/java/org/qortal/block/Block.java

@@ -1682,12 +1682,14 @@ public class Block {
 					transactionData.getSignature());
 			this.repository.getBlockRepository().save(blockTransactionData);
 
-			// Update transaction's height in repository
+			// Update transaction's height in repository and local transactionData
 			transactionRepository.updateBlockHeight(transactionData.getSignature(), this.blockData.getHeight());
-
-			// Update local transactionData's height too
 			transaction.getTransactionData().setBlockHeight(this.blockData.getHeight());
 
+			// Update transaction's sequence in repository and local transactionData
+			transactionRepository.updateBlockSequence(transactionData.getSignature(), sequence);
+			transaction.getTransactionData().setBlockSequence(sequence);
+
 			// No longer unconfirmed
 			transactionRepository.confirmTransaction(transactionData.getSignature());
 
@@ -1774,6 +1776,9 @@ public class Block {
 
 				// Unset height
 				transactionRepository.updateBlockHeight(transactionData.getSignature(), null);
+
+				// Unset sequence
+				transactionRepository.updateBlockSequence(transactionData.getSignature(), null);
 			}
 
 			transactionRepository.deleteParticipants(transactionData);

src/main/java/org/qortal/controller/Controller.java

@@ -404,6 +404,7 @@ public class Controller extends Thread {
 			try (final Repository repository = RepositoryManager.getRepository()) {
 				RepositoryManager.archive(repository);
 				RepositoryManager.prune(repository);
+				RepositoryManager.rebuildTransactionSequences(repository);
 			}
 		} catch (DataException e) {
 			// If exception has no cause then repository is in use by some other process.

src/main/java/org/qortal/data/transaction/TransactionData.java

@@ -76,6 +76,10 @@ public abstract class TransactionData {
 	@Schema(accessMode = AccessMode.READ_ONLY, hidden = true, description = "height of block containing transaction")
 	protected Integer blockHeight;
 
+	// Not always present
+	@Schema(accessMode = AccessMode.READ_ONLY, hidden = true, description = "sequence in block containing transaction")
+	protected Integer blockSequence;
+
 	// Not always present
 	@Schema(accessMode = AccessMode.READ_ONLY, description = "group-approval status")
 	protected ApprovalStatus approvalStatus;
@@ -106,6 +110,7 @@ public abstract class TransactionData {
 		this.fee = baseTransactionData.fee;
 		this.signature = baseTransactionData.signature;
 		this.blockHeight = baseTransactionData.blockHeight;
+		this.blockSequence = baseTransactionData.blockSequence;
 		this.approvalStatus = baseTransactionData.approvalStatus;
 		this.approvalHeight = baseTransactionData.approvalHeight;
 	}
@@ -174,6 +179,15 @@ public abstract class TransactionData {
 		this.blockHeight = blockHeight;
 	}
 
+	public Integer getBlockSequence() {
+		return this.blockSequence;
+	}
+
+	@XmlTransient
+	public void setBlockSequence(Integer blockSequence) {
+		this.blockSequence = blockSequence;
+	}
+
 	public ApprovalStatus getApprovalStatus() {
 		return approvalStatus;
 	}

src/main/java/org/qortal/repository/RepositoryManager.java

@@ -2,13 +2,18 @@ package org.qortal.repository;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.qortal.data.transaction.TransactionData;
 import org.qortal.gui.SplashFrame;
 import org.qortal.repository.hsqldb.HSQLDBDatabaseArchiving;
 import org.qortal.repository.hsqldb.HSQLDBDatabasePruning;
 import org.qortal.repository.hsqldb.HSQLDBRepository;
 import org.qortal.settings.Settings;
+import org.qortal.transaction.Transaction;
 
 import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.concurrent.TimeoutException;
 
 public abstract class RepositoryManager {
@@ -117,6 +122,69 @@ public abstract class RepositoryManager {
 		return false;
 	}
 
+	public static boolean rebuildTransactionSequences(Repository repository) throws DataException {
+		if (Settings.getInstance().isLite()) {
+			// Lite nodes have no blockchain
+			return false;
+		}
+
+		try {
+			// Check if we have any unpopulated block_sequence values for the first 1000 blocks
+			List<byte[]> testSignatures = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
+					null, Arrays.asList("block_height < 1000 AND block_sequence IS NULL"), new ArrayList<>());
+			if (testSignatures.isEmpty()) {
+				// block_sequence already populated for the first 1000 blocks, so assume complete.
+				// We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so
+				// we shouldn't ever be left in a partially rebuilt state.
+				return false;
+			}
+
+			int blockchainHeight = repository.getBlockRepository().getBlockchainHeight();
+			int totalTransactionCount = 0;
+
+			for (int height = 1; height < blockchainHeight; height++) {
+				List<TransactionData> transactions = new ArrayList<>();
+
+				// Fetch transactions for height
+				List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, height, height);
+				for (byte[] signature : signatures) {
+					TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+					if (transactionData != null) {
+						transactions.add(transactionData);
+					}
+				}
+				totalTransactionCount += transactions.size();
+
+				// Sort the transactions for this height
+				transactions.sort(Transaction.getDataComparator());
+
+				// Loop through and update sequences
+				for (int sequence = 0; sequence < transactions.size(); ++sequence) {
+					TransactionData transactionData = transactions.get(sequence);
+
+					// Update transaction's sequence in repository
+					repository.getTransactionRepository().updateBlockSequence(transactionData.getSignature(), sequence);
+				}
+
+				if (height % 10000 == 0) {
+					LOGGER.info("Rebuilt sequences for {} blocks (total transactions: {})", height, totalTransactionCount);
+				}
+
+				repository.saveChanges();
+			}
+
+			LOGGER.info("Completed rebuild of transaction sequences.");
+			return true;
+		}
+		catch (DataException e) {
+			LOGGER.info("Unable to rebuild transaction sequences: {}. The database may have been left in an inconsistent state.", e.getMessage());
+
+			// Throw an exception so that the node startup is halted, allowing for a retry next time.
+			repository.discardChanges();
+			throw new DataException("Rebuild of transaction sequences failed.");
+		}
+	}
+
 	public static void setRequestedCheckpoint(Boolean quick) {
 		quickCheckpointRequested = quick;
 	}

src/main/java/org/qortal/repository/TransactionRepository.java

@@ -309,6 +309,8 @@ public interface TransactionRepository {
 
 	public void updateBlockHeight(byte[] signature, Integer height) throws DataException;
 
+	public void updateBlockSequence(byte[] signature, Integer sequence) throws DataException;
+
 	public void updateApprovalHeight(byte[] signature, Integer approvalHeight) throws DataException;
 
 	/**

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -993,6 +993,17 @@ public class HSQLDBDatabaseUpdates {
 					stmt.execute("ALTER TABLE CancelSellNameTransactions ADD sale_price QortalAmount");
 					break;
 
+				case 47:
+					// Add `block_sequence` to the Transaction table, as the BlockTransactions table is pruned for
+					// older blocks and therefore the sequence becomes unavailable
+					LOGGER.info("Reshaping Transactions table - this can take a while...");
+					stmt.execute("ALTER TABLE Transactions ADD block_sequence INTEGER");
+
+					// For finding transactions by height and sequence
+					LOGGER.info("Adding index to Transactions table - this can take a while...");
+					stmt.execute("CREATE INDEX TransactionHeightSequenceIndex on Transactions (block_height, block_sequence)");
+					break;
+
 				default:
 					// nothing to do
 					return false;

src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java

@@ -657,8 +657,13 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 															List<Object> bindParams) throws DataException {
 		List<byte[]> signatures = new ArrayList<>();
 
+		String txTypeClassName = "";
+		if (txType != null) {
+			txTypeClassName = txType.className;
+		}
+
 		StringBuilder sql = new StringBuilder(1024);
-		sql.append(String.format("SELECT signature FROM %sTransactions", txType.className));
+		sql.append(String.format("SELECT signature FROM %sTransactions", txTypeClassName));
 
 		if (!whereClauses.isEmpty()) {
 			sql.append(" WHERE ");
@@ -1444,6 +1449,19 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 		}
 	}
 
+	@Override
+	public void updateBlockSequence(byte[] signature, Integer blockSequence) throws DataException {
+		HSQLDBSaver saver = new HSQLDBSaver("Transactions");
+
+		saver.bind("signature", signature).bind("block_sequence", blockSequence);
+
+		try {
+			saver.execute(repository);
+		} catch (SQLException e) {
+			throw new DataException("Unable to update transaction's block sequence in repository", e);
+		}
+	}
+
 	@Override
 	public void updateApprovalHeight(byte[] signature, Integer approvalHeight) throws DataException {
 		HSQLDBSaver saver = new HSQLDBSaver("Transactions");