Split AT state storage into two HSQLDB table for better management

This involves a database reshape, but before this happens the node-local
data is exported to local files, giving the user the option to use a
bootstrap file instead of waiting.

src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java

@@ -250,7 +250,8 @@ public class HSQLDBATRepository implements ATRepository {
 	public ATStateData getATStateAtHeight(String atAddress, int height) throws DataException {
 		String sql = "SELECT state_data, state_hash, fees, is_initial "
 				+ "FROM ATStates "
-				+ "WHERE AT_address = ? AND height = ? "
+				+ "LEFT OUTER JOIN ATStatesData USING (AT_address, height) "
+				+ "WHERE ATStates.AT_address = ? AND ATStates.height = ? "
 				+ "LIMIT 1";
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddress, height)) {
@@ -272,10 +273,11 @@ public class HSQLDBATRepository implements ATRepository {
 	public ATStateData getLatestATState(String atAddress) throws DataException {
 		String sql = "SELECT height, state_data, state_hash, fees, is_initial "
 				+ "FROM ATStates "
-				+ "WHERE AT_address = ? "
-				// AT_address then height so the compound primary key is used as an index
-				// Both must be the same direction also
-				+ "ORDER BY AT_address DESC, height DESC "
+				+ "JOIN ATStatesData USING (AT_address, height) "
+				+ "WHERE ATStates.AT_address = ? "
+				// Order by AT_address and height to use compound primary key as index
+				// Both must be the same direction (DESC) also
+				+ "ORDER BY ATStates.AT_address DESC, ATStates.height DESC "
 				+ "LIMIT 1 ";
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql, atAddress)) {
@@ -306,16 +308,17 @@ public class HSQLDBATRepository implements ATRepository {
 				+ "CROSS JOIN LATERAL("
 					+ "SELECT height, state_data, state_hash, fees, is_initial "
 					+ "FROM ATStates "
+					+ "JOIN ATStatesData USING (AT_address, height) "
 					+ "WHERE ATStates.AT_address = ATs.AT_address ");
 
 		if (minimumFinalHeight != null) {
-			sql.append("AND height >= ? ");
+			sql.append("AND ATStates.height >= ? ");
 			bindParams.add(minimumFinalHeight);
 		}
 
-		// AT_address then height so the compound primary key is used as an index
-		// Both must be the same direction also
-		sql.append("ORDER BY AT_address DESC, height DESC "
+		// Order by AT_address and height to use compound primary key as index
+		// Both must be the same direction (DESC) also
+		sql.append("ORDER BY ATStates.AT_address DESC, ATStates.height DESC "
 					+ "LIMIT 1 "
 				+ ") AS FinalATStates "
 				+ "WHERE code_hash = ? ");
@@ -337,7 +340,7 @@ public class HSQLDBATRepository implements ATRepository {
 			bindParams.add(rawExpectedValue);
 		}
 
-		sql.append(" ORDER BY height ");
+		sql.append(" ORDER BY FinalATStates.height ");
 		if (reverse != null && reverse)
 			sql.append("DESC");
 
@@ -431,7 +434,7 @@ public class HSQLDBATRepository implements ATRepository {
 
 	@Override
 	public void prepareForAtStateTrimming() throws DataException {
-		// Rebuild cache of latest, non-finished AT states that we can't trim
+		// Rebuild cache of latest AT states that we can't trim
 		String deleteSql = "DELETE FROM LatestATStates";
 		try {
 			this.repository.executeCheckedUpdate(deleteSql);
@@ -463,13 +466,12 @@ public class HSQLDBATRepository implements ATRepository {
 
 		// We're often called so no need to trim all states in one go.
 		// Limit updates to reduce CPU and memory load.
-		String sql = "UPDATE ATStates SET state_data = NULL "
-				+ "WHERE state_data IS NOT NULL "
-				+ "AND height BETWEEN ? AND ? "
+		String sql = "DELETE FROM ATStatesData "
+				+ "WHERE height BETWEEN ? AND ? "
 				+ "AND NOT EXISTS("
 					+ "SELECT TRUE FROM LatestATStates "
-					+ "WHERE LatestATStates.AT_address = ATStates.AT_address "
-					+ "AND LatestATStates.height = ATStates.height"
+					+ "WHERE LatestATStates.AT_address = ATStatesData.AT_address "
+					+ "AND LatestATStates.height = ATStatesData.height"
 				+ ") "
 				+ "LIMIT ?";
 
@@ -487,23 +489,44 @@ public class HSQLDBATRepository implements ATRepository {
 		if (atStateData.getStateHash() == null || atStateData.getHeight() == null)
 			throw new IllegalArgumentException("Refusing to save partial AT state into repository!");
 
-		HSQLDBSaver saveHelper = new HSQLDBSaver("ATStates");
+		HSQLDBSaver atStatesSaver = new HSQLDBSaver("ATStates");
 
-		saveHelper.bind("AT_address", atStateData.getATAddress()).bind("height", atStateData.getHeight())
-				.bind("state_data", atStateData.getStateData()).bind("state_hash", atStateData.getStateHash())
+		atStatesSaver.bind("AT_address", atStateData.getATAddress()).bind("height", atStateData.getHeight())
+				.bind("state_hash", atStateData.getStateHash())
 				.bind("fees", atStateData.getFees()).bind("is_initial", atStateData.isInitial());
 
 		try {
-			saveHelper.execute(this.repository);
+			atStatesSaver.execute(this.repository);
 		} catch (SQLException e) {
 			throw new DataException("Unable to save AT state into repository", e);
 		}
+
+		if (atStateData.getStateData() != null) {
+			HSQLDBSaver atStatesDataSaver = new HSQLDBSaver("ATStatesData");
+
+			atStatesDataSaver.bind("AT_address", atStateData.getATAddress()).bind("height", atStateData.getHeight())
+					.bind("state_data", atStateData.getStateData());
+
+			try {
+				atStatesDataSaver.execute(this.repository);
+			} catch (SQLException e) {
+				throw new DataException("Unable to save AT state data into repository", e);
+			}
+		} else {
+			try {
+				this.repository.delete("ATStatesData", "AT_address = ? AND height = ?",
+						atStateData.getATAddress(), atStateData.getHeight());
+			} catch (SQLException e) {
+				throw new DataException("Unable to delete AT state data from repository", e);
+			}
+		}
 	}
 
 	@Override
 	public void delete(String atAddress, int height) throws DataException {
 		try {
 			this.repository.delete("ATStates", "AT_address = ? AND height = ?", atAddress, height);
+			this.repository.delete("ATStatesData", "AT_address = ? AND height = ?", atAddress, height);
 		} catch (SQLException e) {
 			throw new DataException("Unable to delete AT state from repository", e);
 		}
@@ -513,6 +536,7 @@ public class HSQLDBATRepository implements ATRepository {
 	public void deleteATStates(int height) throws DataException {
 		try {
 			this.repository.delete("ATStates", "height = ?", height);
+			this.repository.delete("ATStatesData", "height = ?", height);
 		} catch (SQLException e) {
 			throw new DataException("Unable to delete AT states from repository", e);
 		}

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -660,9 +660,10 @@ public class HSQLDBDatabaseUpdates {
 					break;
 
 				case 25:
+					// DISABLED: improved version in case 30!
 					// Remove excess created_when from ATStates
-					stmt.execute("ALTER TABLE ATStates DROP created_when");
-					stmt.execute("CREATE INDEX ATStateHeightIndex on ATStates (height)");
+					// stmt.execute("ALTER TABLE ATStates DROP created_when");
+					// stmt.execute("CREATE INDEX ATStateHeightIndex on ATStates (height)");
 					break;
 
 				case 26:
@@ -691,6 +692,69 @@ public class HSQLDBDatabaseUpdates {
 					stmt.execute("CHECKPOINT");
 					break;
 
+				case 30:
+					// Split AT state data off to new table for better performance/management.
+
+					if (!"mem".equals(HSQLDBRepository.getDbPathname(connection.getMetaData().getURL()))) {
+						// First, backup node-local data in case user wants to avoid long reshape and use bootstrap instead
+						stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'MintingAccounts.script'");
+						stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'TradeBotStates.script'");
+						LOGGER.info("Exported sensitive/node-local data: minting keys and trade bot states");
+						LOGGER.info("If following reshape takes too long, use bootstrap and import node-local data using API's POST /admin/repository");
+					}
+
+					// Create new AT-states table without full state data
+					stmt.execute("CREATE TABLE ATStatesNew ("
+							+ "AT_address QortalAddress, height INTEGER NOT NULL, state_hash ATStateHash NOT NULL, "
+							+ "fees QortalAmount NOT NULL, is_initial BOOLEAN NOT NULL, "
+							+ "PRIMARY KEY (AT_address, height), "
+							+ "FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)");
+					stmt.execute("SET TABLE ATStatesNew NEW SPACE");
+					stmt.execute("CHECKPOINT");
+
+					ResultSet resultSet = stmt.executeQuery("SELECT height FROM Blocks ORDER BY height DESC LIMIT 1");
+					final int blockchainHeight = resultSet.next() ? resultSet.getInt(1) : 0;
+					final int heightStep = 100;
+
+					LOGGER.info("Rebuilding AT state summaries in repository - this might take a while... (approx. 2 mins on high-spec)");
+					for (int minHeight = 1; minHeight < blockchainHeight; minHeight += heightStep) {
+						stmt.execute("INSERT INTO ATStatesNew ("
+								+ "SELECT AT_address, height, state_hash, fees, is_initial "
+								+ "FROM ATStates "
+								+ "WHERE height BETWEEN " + minHeight + " AND " + (minHeight + heightStep - 1)
+								+ ")");
+						stmt.execute("COMMIT");
+					}
+					stmt.execute("CHECKPOINT");
+
+					LOGGER.info("Rebuilding AT states height index in repository - this might take about 3x longer...");
+					stmt.execute("CREATE INDEX ATStatesHeightIndex ON ATStatesNew (height)");
+					stmt.execute("CHECKPOINT");
+
+					stmt.execute("CREATE TABLE ATStatesData ("
+							+ "AT_address QortalAddress, height INTEGER NOT NULL, state_data ATState NOT NULL, "
+							+ "PRIMARY KEY (height, AT_address), "
+							+ "FOREIGN KEY (AT_address) REFERENCES ATs (AT_address) ON DELETE CASCADE)");
+					stmt.execute("SET TABLE ATStatesData NEW SPACE");
+					stmt.execute("CHECKPOINT");
+
+					LOGGER.info("Rebuilding AT state data in repository - this might take a while... (approx. 2 mins on high-spec)");
+					for (int minHeight = 1; minHeight < blockchainHeight; minHeight += heightStep) {
+						stmt.execute("INSERT INTO ATStatesData ("
+								+ "SELECT AT_address, height, state_data "
+								+ "FROM ATstates "
+								+ "WHERE state_data IS NOT NULL "
+								+ "AND height BETWEEN " + minHeight + " AND " + (minHeight + heightStep - 1)
+								+ ")");
+						stmt.execute("COMMIT");
+					}
+					stmt.execute("CHECKPOINT");
+
+					stmt.execute("DROP TABLE ATStates");
+					stmt.execute("ALTER TABLE ATStatesNew RENAME TO ATStates");
+					stmt.execute("CHECKPOINT");
+					break;
+
 				default:
 					// nothing to do
 					return false;

src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java

@@ -426,7 +426,7 @@ public class HSQLDBRepository implements Repository {
 	}
 
 	/** Returns DB pathname from passed connection URL. If memory DB, returns "mem". */
-	private static String getDbPathname(String connectionUrl) {
+	/*package*/ static String getDbPathname(String connectionUrl) {
 		Pattern pattern = Pattern.compile("hsqldb:(mem|file):(.*?)(;|$)");
 		Matcher matcher = pattern.matcher(connectionUrl);