Rework of import/export process.

- Adds support for minting accounts as well as trade bot states
- Includes automatic import of both types on node startup, and automatic export on node shutdown
- Retains legacy trade bot states in a separate "TradeBotStatesArchive.json" file, whilst keeping the current active ones in "TradeBotStates.json". This prevents states being re-imported after they have been removed, but still keeps a copy of the data in case a key is ever needed.
- Uses indentation in the JSON files for easier readability.

src/test/java/org/qortal/test/ImportExportTests.java

@@ -0,0 +1,390 @@
+package org.qortal.test;
+
+import org.apache.commons.io.FileUtils;
+import org.bitcoinj.core.Address;
+import org.bitcoinj.core.AddressFormatException;
+import org.bitcoinj.core.ECKey;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.qortal.account.PublicKeyAccount;
+import org.qortal.controller.tradebot.LitecoinACCTv1TradeBot;
+import org.qortal.controller.tradebot.TradeBot;
+import org.qortal.crosschain.Litecoin;
+import org.qortal.crosschain.LitecoinACCTv1;
+import org.qortal.crosschain.SupportedBlockchain;
+import org.qortal.crypto.Crypto;
+import org.qortal.data.account.MintingAccountData;
+import org.qortal.data.crosschain.TradeBotData;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.repository.hsqldb.HSQLDBImportExport;
+import org.qortal.settings.Settings;
+import org.qortal.test.common.Common;
+import org.qortal.utils.NTP;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class ImportExportTests extends Common {
+
+    @Before
+    public void beforeTest() throws DataException {
+        Common.useDefaultSettings();
+        this.deleteExportDirectory();
+    }
+
+    @After
+    public void afterTest() throws DataException {
+        this.deleteExportDirectory();
+    }
+
+
+    @Test
+    public void testExportAndImportTradeBotStates() throws DataException, IOException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Ensure no trade bots exist
+            assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
+
+            // Create some trade bots
+            List<TradeBotData> tradeBots = new ArrayList<>();
+            for (int i=0; i<10; i++) {
+                TradeBotData tradeBotData = this.createTradeBotData(repository);
+                repository.getCrossChainRepository().save(tradeBotData);
+                tradeBots.add(tradeBotData);
+            }
+
+            // Ensure they have been added
+            assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size());
+
+            // Export them
+            HSQLDBImportExport.backupTradeBotStates(repository);
+
+            // Delete them from the repository
+            for (TradeBotData tradeBotData : tradeBots) {
+                repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
+            }
+
+            // Ensure they have been deleted
+            assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
+
+            // Import them
+            Path exportPath = HSQLDBImportExport.getExportDirectory(false);
+            Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json");
+            HSQLDBImportExport.importDataFromFile(filePath.toString(), repository);
+
+            // Ensure they have been imported
+            assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size());
+
+            // Ensure all the data matches
+            for (TradeBotData tradeBotData : tradeBots) {
+                byte[] tradePrivateKey = tradeBotData.getTradePrivateKey();
+                TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
+                assertNotNull(repositoryTradeBotData);
+                assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString());
+            }
+
+            repository.saveChanges();
+        }
+    }
+
+    @Test
+    public void testExportAndImportCurrentTradeBotStates() throws DataException, IOException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Ensure no trade bots exist
+            assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
+
+            // Create some trade bots
+            List<TradeBotData> tradeBots = new ArrayList<>();
+            for (int i=0; i<10; i++) {
+                TradeBotData tradeBotData = this.createTradeBotData(repository);
+                repository.getCrossChainRepository().save(tradeBotData);
+                tradeBots.add(tradeBotData);
+            }
+
+            // Ensure they have been added
+            assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size());
+
+            // Export them
+            HSQLDBImportExport.backupTradeBotStates(repository);
+
+            // Delete them from the repository
+            for (TradeBotData tradeBotData : tradeBots) {
+                repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
+            }
+
+            // Ensure they have been deleted
+            assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
+
+            // Add some more trade bots
+            List<TradeBotData> additionalTradeBots = new ArrayList<>();
+            for (int i=0; i<5; i++) {
+                TradeBotData tradeBotData = this.createTradeBotData(repository);
+                repository.getCrossChainRepository().save(tradeBotData);
+                additionalTradeBots.add(tradeBotData);
+            }
+
+            // Export again
+            HSQLDBImportExport.backupTradeBotStates(repository);
+
+            // Import current states only
+            Path exportPath = HSQLDBImportExport.getExportDirectory(false);
+            Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json");
+            HSQLDBImportExport.importDataFromFile(filePath.toString(), repository);
+
+            // Ensure they have been imported
+            assertEquals(5, repository.getCrossChainRepository().getAllTradeBotData().size());
+
+            // Ensure that only the additional trade bots have been imported and that the data matches
+            for (TradeBotData tradeBotData : additionalTradeBots) {
+                byte[] tradePrivateKey = tradeBotData.getTradePrivateKey();
+                TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
+                assertNotNull(repositoryTradeBotData);
+                assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString());
+            }
+
+            // None of the original trade bots should exist in the repository
+            for (TradeBotData tradeBotData : tradeBots) {
+                byte[] tradePrivateKey = tradeBotData.getTradePrivateKey();
+                TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
+                assertNull(repositoryTradeBotData);
+            }
+
+            repository.saveChanges();
+        }
+    }
+
+    @Test
+    public void testExportAndImportAllTradeBotStates() throws DataException, IOException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Ensure no trade bots exist
+            assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
+
+            // Create some trade bots
+            List<TradeBotData> tradeBots = new ArrayList<>();
+            for (int i=0; i<10; i++) {
+                TradeBotData tradeBotData = this.createTradeBotData(repository);
+                repository.getCrossChainRepository().save(tradeBotData);
+                tradeBots.add(tradeBotData);
+            }
+
+            // Ensure they have been added
+            assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size());
+
+            // Export them
+            HSQLDBImportExport.backupTradeBotStates(repository);
+
+            // Delete them from the repository
+            for (TradeBotData tradeBotData : tradeBots) {
+                repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
+            }
+
+            // Ensure they have been deleted
+            assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
+
+            // Add some more trade bots
+            List<TradeBotData> additionalTradeBots = new ArrayList<>();
+            for (int i=0; i<5; i++) {
+                TradeBotData tradeBotData = this.createTradeBotData(repository);
+                repository.getCrossChainRepository().save(tradeBotData);
+                additionalTradeBots.add(tradeBotData);
+            }
+
+            // Export again
+            HSQLDBImportExport.backupTradeBotStates(repository);
+
+            // Import all states from the archive
+            Path exportPath = HSQLDBImportExport.getExportDirectory(false);
+            Path filePath = Paths.get(exportPath.toString(), "TradeBotStatesArchive.json");
+            HSQLDBImportExport.importDataFromFile(filePath.toString(), repository);
+
+            // Ensure they have been imported
+            assertEquals(15, repository.getCrossChainRepository().getAllTradeBotData().size());
+
+            // Ensure that all known trade bots have been imported and that the data matches
+            tradeBots.addAll(additionalTradeBots);
+
+            for (TradeBotData tradeBotData : tradeBots) {
+                byte[] tradePrivateKey = tradeBotData.getTradePrivateKey();
+                TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
+                assertNotNull(repositoryTradeBotData);
+                assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString());
+            }
+
+            repository.saveChanges();
+        }
+    }
+
+    @Test
+    public void testExportAndImportLegacyTradeBotStates() throws DataException, IOException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Create some trade bots, but don't save them in the repository
+            List<TradeBotData> tradeBots = new ArrayList<>();
+            for (int i=0; i<10; i++) {
+                TradeBotData tradeBotData = this.createTradeBotData(repository);
+                tradeBots.add(tradeBotData);
+            }
+
+            // Create a legacy format TradeBotStates.json backup file
+            this.exportLegacyTradeBotStatesJson(tradeBots);
+
+            // Ensure no trade bots exist in repository
+            assertTrue(repository.getCrossChainRepository().getAllTradeBotData().isEmpty());
+
+            // Import the legacy format file
+            Path exportPath = HSQLDBImportExport.getExportDirectory(false);
+            Path filePath = Paths.get(exportPath.toString(), "TradeBotStates.json");
+            HSQLDBImportExport.importDataFromFile(filePath.toString(), repository);
+
+            // Ensure they have been imported
+            assertEquals(10, repository.getCrossChainRepository().getAllTradeBotData().size());
+
+            for (TradeBotData tradeBotData : tradeBots) {
+                byte[] tradePrivateKey = tradeBotData.getTradePrivateKey();
+                TradeBotData repositoryTradeBotData = repository.getCrossChainRepository().getTradeBotData(tradePrivateKey);
+                assertNotNull(repositoryTradeBotData);
+                assertEquals(tradeBotData.toJson().toString(), repositoryTradeBotData.toJson().toString());
+            }
+
+            repository.saveChanges();
+        }
+    }
+
+    @Test
+    public void testExportAndImportMintingAccountData() throws DataException, IOException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Ensure no minting accounts exist
+            assertTrue(repository.getAccountRepository().getMintingAccounts().isEmpty());
+
+            // Create some minting accounts
+            List<MintingAccountData> mintingAccounts = new ArrayList<>();
+            for (int i=0; i<10; i++) {
+                MintingAccountData mintingAccountData = this.createMintingAccountData();
+                repository.getAccountRepository().save(mintingAccountData);
+                mintingAccounts.add(mintingAccountData);
+            }
+
+            // Ensure they have been added
+            assertEquals(10, repository.getAccountRepository().getMintingAccounts().size());
+
+            // Export them
+            HSQLDBImportExport.backupMintingAccounts(repository);
+
+            // Delete them from the repository
+            for (MintingAccountData mintingAccountData : mintingAccounts) {
+                repository.getAccountRepository().delete(mintingAccountData.getPrivateKey());
+            }
+
+            // Ensure they have been deleted
+            assertTrue(repository.getAccountRepository().getMintingAccounts().isEmpty());
+
+            // Import them
+            Path exportPath = HSQLDBImportExport.getExportDirectory(false);
+            Path filePath = Paths.get(exportPath.toString(), "MintingAccounts.json");
+            HSQLDBImportExport.importDataFromFile(filePath.toString(), repository);
+
+            // Ensure they have been imported
+            assertEquals(10, repository.getAccountRepository().getMintingAccounts().size());
+
+            // Ensure all the data matches
+            for (MintingAccountData mintingAccountData : mintingAccounts) {
+                byte[] privateKey = mintingAccountData.getPrivateKey();
+                MintingAccountData repositoryMintingAccountData = repository.getAccountRepository().getMintingAccount(privateKey);
+                assertNotNull(repositoryMintingAccountData);
+                assertEquals(mintingAccountData.toJson().toString(), repositoryMintingAccountData.toJson().toString());
+            }
+
+            repository.saveChanges();
+        }
+    }
+
+
+    private TradeBotData createTradeBotData(Repository repository) throws DataException {
+        byte[] tradePrivateKey = TradeBot.generateTradePrivateKey();
+
+        byte[] tradeNativePublicKey = TradeBot.deriveTradeNativePublicKey(tradePrivateKey);
+        byte[] tradeNativePublicKeyHash = Crypto.hash160(tradeNativePublicKey);
+        String tradeNativeAddress = Crypto.toAddress(tradeNativePublicKey);
+
+        byte[] tradeForeignPublicKey = TradeBot.deriveTradeForeignPublicKey(tradePrivateKey);
+        byte[] tradeForeignPublicKeyHash = Crypto.hash160(tradeForeignPublicKey);
+
+        String receivingAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
+
+        // Convert Litecoin receiving address into public key hash (we only support P2PKH at this time)
+        Address litecoinReceivingAddress;
+        try {
+            litecoinReceivingAddress = Address.fromString(Litecoin.getInstance().getNetworkParameters(), receivingAddress);
+        } catch (AddressFormatException e) {
+            throw new DataException("Unsupported Litecoin receiving address: " + receivingAddress);
+        }
+
+        byte[] litecoinReceivingAccountInfo = litecoinReceivingAddress.getHash();
+
+        byte[] creatorPublicKey = new byte[32];
+        PublicKeyAccount creator = new PublicKeyAccount(repository, creatorPublicKey);
+
+        long timestamp = NTP.getTime();
+        String atAddress = "AT_ADDRESS";
+        long foreignAmount = 1234;
+        long qortAmount= 5678;
+
+        TradeBotData tradeBotData =  new TradeBotData(tradePrivateKey, LitecoinACCTv1.NAME,
+                LitecoinACCTv1TradeBot.State.BOB_WAITING_FOR_AT_CONFIRM.name(), LitecoinACCTv1TradeBot.State.BOB_WAITING_FOR_AT_CONFIRM.value,
+                creator.getAddress(), atAddress, timestamp, qortAmount,
+                tradeNativePublicKey, tradeNativePublicKeyHash, tradeNativeAddress,
+                null, null,
+                SupportedBlockchain.LITECOIN.name(),
+                tradeForeignPublicKey, tradeForeignPublicKeyHash,
+                foreignAmount, null, null, null, litecoinReceivingAccountInfo);
+
+        return tradeBotData;
+    }
+
+    private MintingAccountData createMintingAccountData() {
+        // These don't need to be valid keys - just 32 byte strings for the purposes of testing
+        byte[] privateKey = new ECKey().getPrivKeyBytes();
+        byte[] publicKey = new ECKey().getPrivKeyBytes();
+
+        return new MintingAccountData(privateKey, publicKey);
+    }
+
+    private void exportLegacyTradeBotStatesJson(List<TradeBotData> allTradeBotData) throws IOException, DataException {
+        JSONArray allTradeBotDataJson = new JSONArray();
+        for (TradeBotData tradeBotData : allTradeBotData) {
+            JSONObject tradeBotDataJson = tradeBotData.toJson();
+            allTradeBotDataJson.put(tradeBotDataJson);
+        }
+
+        Path backupDirectory = HSQLDBImportExport.getExportDirectory(true);
+        String fileName = Paths.get(backupDirectory.toString(), "TradeBotStates.json").toString();
+        FileWriter writer = new FileWriter(fileName);
+        writer.write(allTradeBotDataJson.toString());
+        writer.close();
+    }
+
+    private void deleteExportDirectory() {
+        // Delete archive directory if exists
+        Path archivePath = Paths.get(Settings.getInstance().getExportPath());
+        try {
+            FileUtils.deleteDirectory(archivePath.toFile());
+        } catch (IOException e) {
+
+        }
+    }
+
+}

src/test/resources/test-settings-v2-bitcoin-regtest.json

@@ -3,6 +3,7 @@
   "litecoinNet": "REGTEST",
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2.json",
+  "exportPath": "qortal-backup-test",
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0

src/test/resources/test-settings-v2-block-archive.json

@@ -3,6 +3,7 @@
   "litecoinNet": "TEST3",
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2.json",
+  "exportPath": "qortal-backup-test",
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0,

src/test/resources/test-settings-v2-founder-rewards.json

@@ -1,6 +1,7 @@
 {
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-founder-rewards.json",
+  "exportPath": "qortal-backup-test",
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0

src/test/resources/test-settings-v2-leftover-reward.json

@@ -1,6 +1,7 @@
 {
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-leftover-reward.json",
+  "exportPath": "qortal-backup-test",
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0

src/test/resources/test-settings-v2-minting.json

@@ -1,6 +1,7 @@
 {
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-minting.json",
+  "exportPath": "qortal-backup-test",
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0

src/test/resources/test-settings-v2-qora-holder-extremes.json

@@ -1,6 +1,7 @@
 {
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder-extremes.json",
+  "exportPath": "qortal-backup-test",
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0

src/test/resources/test-settings-v2-qora-holder.json

@@ -1,6 +1,7 @@
 {
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-qora-holder.json",
+  "exportPath": "qortal-backup-test",
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0

src/test/resources/test-settings-v2-reward-levels.json

@@ -1,6 +1,7 @@
 {
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-reward-levels.json",
+  "exportPath": "qortal-backup-test",
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0

src/test/resources/test-settings-v2-reward-scaling.json

@@ -1,6 +1,7 @@
 {
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2-reward-scaling.json",
+  "exportPath": "qortal-backup-test",
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0

src/test/resources/test-settings-v2.json

@@ -3,6 +3,7 @@
   "litecoinNet": "TEST3",
   "restrictedApi": false,
   "blockchainConfig": "src/test/resources/test-chain-v2.json",
+  "exportPath": "qortal-backup-test",
   "wipeUnconfirmedOnStart": false,
   "testNtpOffset": 0,
   "minPeers": 0