Rework of the repository export and import functions.

The existing HSQL export/import (PERFORM EXPORT SCRIPT and PERFORM IMPORT SCRIPT) have been replaced with a custom JSON import and export. Whilst this is less generic, it has some significant advantages:

- When exporting data, it is now able to combine the exported data with any data that already exists in the backup file. This prevents a backup after a bootstrap from overwriting data from before the bootstrap, and removes the need for all of the "archive" files that we currently create.
- Adds support for partial imports, and updates. Previously an import would fail if any of the data being imported already existed in the db. It will now add new rows and update existing ones.
- The format and contents of the exported trade bot data now matches the output of the /crosschain/tradebot API.
- Data is retrieved without the need for a database lock, and therefore the export process is much faster and less invasive. This should prevent the lockups and other problems seen when using the trade portal.

For now, there are a couple of trade-offs to using this new approach:
- The minting key import/export has been temporarily removed until there is more time to transition it to this new format.
- Existing .script backups can no longer be imported using versions higher than 1.5.1.

Both of these can be solved by temporarily running version 1.5.1, performing the necessary imports/exports, then returning to the latest version. Longer term the minting keys export/import will be reimplemented using the JSON format.

src/main/java/org/qortal/api/resource/AdminResource.java

@@ -542,19 +542,8 @@ public class AdminResource {
 		Security.checkApiCallAllowed(request);
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
-			ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+			repository.exportNodeLocalData();
-
+			return "true";
-			blockchainLock.lockInterruptibly();
-
-			try {
-				repository.exportNodeLocalData(true);
-				return "true";
-			} finally {
-				blockchainLock.unlock();
-			}
-		} catch (InterruptedException e) {
-			// We couldn't lock blockchain to perform export
-			return "false";
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -564,7 +553,7 @@ public class AdminResource {
 	@Path("/repository/data")
 	@Operation(
 		summary = "Import data into repository.",
-		description = "Imports data from file on local machine. Filename is forced to 'import.script' if apiKey is not set.",
+		description = "Imports data from file on local machine. Filename is forced to 'import.json' if apiKey is not set.",
 		requestBody = @RequestBody(
 			required = true,
 			content = @Content(
@@ -588,7 +577,7 @@ public class AdminResource {
 
 		// Hard-coded because it's too dangerous to allow user-supplied filenames in weaker security contexts
 		if (Settings.getInstance().getApiKey() == null)
-			filename = "import.script";
+			filename = "import.json";
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();

src/main/java/org/qortal/controller/tradebot/TradeBot.java

@@ -272,15 +272,9 @@ public class TradeBot implements Listener {
 		// Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure
 		try {
 			LOGGER.info("About to backup trade bot data...");
-			ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+			repository.exportNodeLocalData();
-			blockchainLock.lockInterruptibly();
+		} catch (DataException e) {
-			try {
+			LOGGER.info(String.format("Repository issue when exporting trade bot data: %s", e.getMessage()));
-				repository.exportNodeLocalData(true);
-			} finally {
-				blockchainLock.unlock();
-			}
-		} catch (InterruptedException | DataException e) {
-			LOGGER.info(String.format("Failed to obtain blockchain lock when exporting trade bot data: %s", e.getMessage()));
 		}
 	}
 

src/main/java/org/qortal/data/crosschain/TradeBotData.java

@@ -6,6 +6,9 @@ import javax.xml.bind.annotation.XmlTransient;
 import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
 
 import io.swagger.v3.oas.annotations.media.Schema;
+import org.json.JSONObject;
+
+import org.qortal.utils.Base58;
 
 // All properties to be converted to JSON via JAXB
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -205,6 +208,58 @@ public class TradeBotData {
 		return this.receivingAccountInfo;
 	}
 
+	public JSONObject toJson() {
+		JSONObject jsonObject = new JSONObject();
+		jsonObject.put("tradePrivateKey", Base58.encode(this.getTradePrivateKey()));
+		jsonObject.put("acctName", this.getAcctName());
+		jsonObject.put("tradeState", this.getState());
+		jsonObject.put("tradeStateValue", this.getStateValue());
+		jsonObject.put("creatorAddress", this.getCreatorAddress());
+		jsonObject.put("atAddress", this.getAtAddress());
+		jsonObject.put("timestamp", this.getTimestamp());
+		jsonObject.put("qortAmount", this.getQortAmount());
+		if (this.getTradeNativePublicKey() != null) jsonObject.put("tradeNativePublicKey", Base58.encode(this.getTradeNativePublicKey()));
+		if (this.getTradeNativePublicKeyHash() != null) jsonObject.put("tradeNativePublicKeyHash", Base58.encode(this.getTradeNativePublicKeyHash()));
+		jsonObject.put("tradeNativeAddress", this.getTradeNativeAddress());
+		if (this.getSecret() != null) jsonObject.put("secret", Base58.encode(this.getSecret()));
+		if (this.getHashOfSecret() != null) jsonObject.put("hashOfSecret", Base58.encode(this.getHashOfSecret()));
+		jsonObject.put("foreignBlockchain", this.getForeignBlockchain());
+		if (this.getTradeForeignPublicKey() != null) jsonObject.put("tradeForeignPublicKey", Base58.encode(this.getTradeForeignPublicKey()));
+		if (this.getTradeForeignPublicKeyHash() != null) jsonObject.put("tradeForeignPublicKeyHash", Base58.encode(this.getTradeForeignPublicKeyHash()));
+		jsonObject.put("foreignKey", this.getForeignKey());
+        jsonObject.put("foreignAmount", this.getForeignAmount());
+		if (this.getLastTransactionSignature() != null) jsonObject.put("lastTransactionSignature", Base58.encode(this.getLastTransactionSignature()));
+        jsonObject.put("lockTimeA", this.getLockTimeA());
+		if (this.getReceivingAccountInfo() != null) jsonObject.put("receivingAccountInfo", Base58.encode(this.getReceivingAccountInfo()));
+		return jsonObject;
+	}
+
+	public static TradeBotData fromJson(JSONObject json) {
+		return new TradeBotData(
+				json.isNull("tradePrivateKey") ? null : Base58.decode(json.getString("tradePrivateKey")),
+				json.isNull("acctName") ? null : json.getString("acctName"),
+				json.isNull("tradeState") ? null : json.getString("tradeState"),
+				json.isNull("tradeStateValue") ? null : json.getInt("tradeStateValue"),
+				json.isNull("creatorAddress") ? null : json.getString("creatorAddress"),
+				json.isNull("atAddress") ? null : json.getString("atAddress"),
+				json.isNull("timestamp") ? null : json.getLong("timestamp"),
+				json.isNull("qortAmount") ? null : json.getLong("qortAmount"),
+				json.isNull("tradeNativePublicKey") ? null : Base58.decode(json.getString("tradeNativePublicKey")),
+				json.isNull("tradeNativePublicKeyHash") ? null : Base58.decode(json.getString("tradeNativePublicKeyHash")),
+				json.isNull("tradeNativeAddress") ? null : json.getString("tradeNativeAddress"),
+				json.isNull("secret") ? null : Base58.decode(json.getString("secret")),
+				json.isNull("hashOfSecret") ? null : Base58.decode(json.getString("hashOfSecret")),
+				json.isNull("foreignBlockchain") ? null : json.getString("foreignBlockchain"),
+				json.isNull("tradeForeignPublicKey") ? null : Base58.decode(json.getString("tradeForeignPublicKey")),
+				json.isNull("tradeForeignPublicKeyHash") ? null : Base58.decode(json.getString("tradeForeignPublicKeyHash")),
+				json.isNull("foreignAmount") ? null : json.getLong("foreignAmount"),
+				json.isNull("foreignKey") ? null : json.getString("foreignKey"),
+				json.isNull("lastTransactionSignature") ? null : Base58.decode(json.getString("lastTransactionSignature")),
+				json.isNull("lockTimeA") ? null : json.getInt("lockTimeA"),
+				json.isNull("receivingAccountInfo") ? null : Base58.decode(json.getString("receivingAccountInfo"))
+		);
+	}
+
 	// Mostly for debugging
 	public String toString() {
 		return String.format("%s: %s (%d)", this.atAddress, this.tradeState, this.tradeStateValue);

src/main/java/org/qortal/repository/Repository.java

@@ -49,7 +49,7 @@ public interface Repository extends AutoCloseable {
 
 	public void performPeriodicMaintenance() throws DataException;
 
-	public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException;
+	public void exportNodeLocalData() throws DataException;
 
 	public void importDataFromFile(String filename) throws DataException;
 

src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java

@@ -2,6 +2,7 @@ package org.qortal.repository.hsqldb;
 
 import java.awt.TrayIcon.MessageType;
 import java.io.File;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.nio.file.Files;
@@ -15,23 +16,19 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Savepoint;
 import java.sql.Statement;
-import java.util.ArrayDeque;
+import java.util.*;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Deque;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
 
+import org.json.JSONArray;
+import org.json.JSONObject;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.crypto.Crypto;
+import org.qortal.data.crosschain.TradeBotData;
 import org.qortal.globalization.Translator;
 import org.qortal.gui.SysTray;
 import org.qortal.repository.ATRepository;
@@ -52,7 +49,7 @@ import org.qortal.repository.TransactionRepository;
 import org.qortal.repository.VotingRepository;
 import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository;
 import org.qortal.settings.Settings;
-import org.qortal.utils.NTP;
+import org.qortal.utils.Base58;
 
 public class HSQLDBRepository implements Repository {
 
@@ -460,8 +457,7 @@ public class HSQLDBRepository implements Repository {
 	}
 
 	@Override
-	public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException {
+	public void exportNodeLocalData() throws DataException {
-
 		// Create the qortal-backup folder if it doesn't exist
 		Path backupPath = Paths.get("qortal-backup");
 		try {
@@ -471,52 +467,59 @@ public class HSQLDBRepository implements Repository {
 			throw new DataException("Unable to create backup folder");
 		}
 
-		// We need to rename or delete an existing TradeBotStates backup before creating a new one
+		try {
-		File tradeBotStatesBackupFile = new File("qortal-backup/TradeBotStates.script");
+			// Load trade bot data
-		if (tradeBotStatesBackupFile.exists()) {
+			List<TradeBotData> allTradeBotData = this.getCrossChainRepository().getAllTradeBotData();
-			if (keepArchivedCopy) {
+			JSONArray allTradeBotDataJson = new JSONArray();
-				// Rename existing TradeBotStates backup, to make sure that we're not overwriting any keys
+			for (TradeBotData tradeBotData : allTradeBotData) {
-				File archivedBackupFile = new File(String.format("qortal-backup/TradeBotStates-archive-%d.script", NTP.getTime()));
+				JSONObject tradeBotDataJson = tradeBotData.toJson();
-				if (tradeBotStatesBackupFile.renameTo(archivedBackupFile))
+				allTradeBotDataJson.put(tradeBotDataJson);
-					LOGGER.info(String.format("Moved existing TradeBotStates backup file to %s", archivedBackupFile.getPath()));
-				else
-					throw new DataException("Unable to rename existing TradeBotStates backup");
-			} else {
-				// Delete existing copy
-				LOGGER.info("Deleting existing TradeBotStates backup because it is being replaced with a new one");
-				tradeBotStatesBackupFile.delete();
 			}
-		}
 
-		// There's currently no need to take an archived copy of the MintingAccounts data - just delete the old one if it exists
+			// We need to combine existing TradeBotStates data before overwriting
-		File mintingAccountsBackupFile = new File("qortal-backup/MintingAccounts.script");
+			String fileName = "qortal-backup/TradeBotStates.json";
-		if (mintingAccountsBackupFile.exists()) {
+			File tradeBotStatesBackupFile = new File(fileName);
-			LOGGER.info("Deleting existing MintingAccounts backup because it is being replaced with a new one");
+			if (tradeBotStatesBackupFile.exists()) {
-			mintingAccountsBackupFile.delete();
+				String jsonString = new String(Files.readAllBytes(Paths.get(fileName)));
-		}
+				JSONArray allExistingTradeBotData = new JSONArray(jsonString);
+				Iterator<Object> iterator = allExistingTradeBotData.iterator();
+				while(iterator.hasNext()) {
+					JSONObject existingTradeBotData = (JSONObject)iterator.next();
+					String existingTradePrivateKey = (String) existingTradeBotData.get("tradePrivateKey");
+						// Check if we already have an entry for this trade
+						boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey));
+						if (found == false)
+							// We need to add this to our list
+							allTradeBotDataJson.put(existingTradeBotData);
+				}
+			}
 
-		try (Statement stmt = this.connection.createStatement()) {
+			FileWriter writer = new FileWriter(fileName);
-			stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'qortal-backup/MintingAccounts.script'");
+			writer.write(allTradeBotDataJson.toString());
-			stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'qortal-backup/TradeBotStates.script'");
+			writer.close();
-			LOGGER.info("Exported sensitive/node-local data: minting keys and trade bot states");
+			LOGGER.info("Exported sensitive/node-local data: trade bot states");
-		} catch (SQLException e) {
+
-			throw new DataException("Unable to export sensitive/node-local data from repository");
+		} catch (DataException | IOException e) {
+			throw new DataException("Unable to export trade bot states from repository");
 		}
 	}
 
 	@Override
 	public void importDataFromFile(String filename) throws DataException {
-		try (Statement stmt = this.connection.createStatement()) {
+		LOGGER.info(() -> String.format("Importing data into repository from %s", filename));
-			LOGGER.info(() -> String.format("Importing data into repository from %s", filename));
+		try {
-
+			String jsonString = new String(Files.readAllBytes(Paths.get(filename)));
-			String escapedFilename = stmt.enquoteLiteral(filename);
+			JSONArray tradeBotDataToImport = new JSONArray(jsonString);
-			stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " CONTINUE ON ERROR");
+			Iterator<Object> iterator = tradeBotDataToImport.iterator();
-
+			while(iterator.hasNext()) {
-			LOGGER.info(() -> String.format("Imported data into repository from %s", filename));
+				JSONObject tradeBotDataJson = (JSONObject)iterator.next();
-		} catch (SQLException e) {
+				TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson);
-			LOGGER.info(() -> String.format("Failed to import data into repository from %s: %s", filename, e.getMessage()));
+				this.getCrossChainRepository().save(tradeBotData);
-			throw new DataException("Unable to import sensitive/node-local data to repository: " + e.getMessage());
+			}
+		} catch (IOException e) {
+			throw new DataException("Unable to import sensitive/node-local trade bot states to repository: " + e.getMessage());
 		}
+		LOGGER.info(() -> String.format("Imported trade bot states into repository from %s", filename));
 	}
 
 	@Override