Rework of import/export process.

- Adds support for minting accounts as well as trade bot states
- Includes automatic import of both types on node startup, and automatic export on node shutdown
- Retains legacy trade bot states in a separate "TradeBotStatesArchive.json" file, whilst keeping the current active ones in "TradeBotStates.json". This prevents states being re-imported after they have been removed, but still keeps a copy of the data in case a key is ever needed.
- Uses indentation in the JSON files for easier readability.
This commit is contained in:
CalDescent
2021-09-28 09:24:21 +01:00
parent 114833cf8e
commit 8926d2a73c
21 changed files with 810 additions and 70 deletions

View File

@@ -609,6 +609,10 @@ public class AdminResource {
repository.saveChanges();
return "true";
} catch (IOException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e);
} finally {
blockchainLock.unlock();
}

View File

@@ -424,6 +424,9 @@ public class Controller extends Thread {
return; // Not System.exit() so that GUI can display error
}
// Import current trade bot states and minting accounts if they exist
Controller.importRepositoryData();
// Rebuild Names table and check database integrity
NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck();
namesDatabaseIntegrityCheck.rebuildAllNames();
@@ -601,6 +604,47 @@ public class Controller extends Thread {
}
}
/**
* Import current trade bot states and minting accounts.
* This is needed because the user may have bootstrapped, or there could be a database inconsistency
* if the core crashed when computing the nonce during the start of the trade process.
*/
private static void importRepositoryData() {
try (final Repository repository = RepositoryManager.getRepository()) {
String exportPath = Settings.getInstance().getExportPath();
try {
Path importPath = Paths.get(exportPath, "TradeBotStates.json");
repository.importDataFromFile(importPath.toString());
} catch (FileNotFoundException e) {
// Do nothing, as the files will only exist in certain cases
}
try {
Path importPath = Paths.get(exportPath, "MintingAccounts.json");
repository.importDataFromFile(importPath.toString());
} catch (FileNotFoundException e) {
// Do nothing, as the files will only exist in certain cases
}
repository.saveChanges();
}
catch (DataException | IOException e) {
LOGGER.info("Unable to import data into repository: {}", e.getMessage());
}
}
/**
* Export current trade bot states and minting accounts.
*/
private void exportRepositoryData() {
try (final Repository repository = RepositoryManager.getRepository()) {
repository.exportNodeLocalData();
} catch (DataException e) {
// Fail silently as this is an optional step
}
}
public static final Predicate<Peer> hasMisbehaved = peer -> {
final Long lastMisbehaved = peer.getPeerData().getLastMisbehaved();
return lastMisbehaved != null && lastMisbehaved > NTP.getTime() - MISBEHAVIOUR_COOLOFF;
@@ -952,6 +996,10 @@ public class Controller extends Thread {
}
}
// Export local data
LOGGER.info("Backing up local data");
this.exportRepositoryData();
LOGGER.info("Shutting down networking");
Network.getInstance().shutdown();

View File

@@ -245,17 +245,17 @@ public class TradeBot implements Listener {
}
}
/*package*/ static byte[] generateTradePrivateKey() {
public static byte[] generateTradePrivateKey() {
// The private key is used for both Curve25519 and secp256k1 so needs to be valid for both.
// Curve25519 accepts any seed, so generate a valid secp256k1 key and use that.
return new ECKey().getPrivKeyBytes();
}
/*package*/ static byte[] deriveTradeNativePublicKey(byte[] privateKey) {
public static byte[] deriveTradeNativePublicKey(byte[] privateKey) {
return PrivateKeyAccount.toPublicKey(privateKey);
}
/*package*/ static byte[] deriveTradeForeignPublicKey(byte[] privateKey) {
public static byte[] deriveTradeForeignPublicKey(byte[] privateKey) {
return ECKey.fromPrivate(privateKey).getPubKey();
}

View File

@@ -4,10 +4,12 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlTransient;
import org.json.JSONObject;
import org.qortal.crypto.Crypto;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
import org.qortal.utils.Base58;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
@@ -61,4 +63,21 @@ public class MintingAccountData {
return this.publicKey;
}
// JSON
public JSONObject toJson() {
JSONObject jsonObject = new JSONObject();
jsonObject.put("privateKey", Base58.encode(this.getPrivateKey()));
jsonObject.put("publicKey", Base58.encode(this.getPublicKey()));
return jsonObject;
}
public static MintingAccountData fromJson(JSONObject json) {
return new MintingAccountData(
json.isNull("privateKey") ? null : Base58.decode(json.getString("privateKey")),
json.isNull("publicKey") ? null : Base58.decode(json.getString("publicKey"))
);
}
}

View File

@@ -191,6 +191,8 @@ public interface AccountRepository {
public List<MintingAccountData> getMintingAccounts() throws DataException;
public MintingAccountData getMintingAccount(byte[] mintingAccountKey) throws DataException;
public void save(MintingAccountData mintingAccountData) throws DataException;
/** Delete minting account info, used by BlockMinter, from repository using passed public or private key. */

View File

@@ -1,5 +1,7 @@
package org.qortal.repository;
import java.io.IOException;
public interface Repository extends AutoCloseable {
public ATRepository getATRepository();
@@ -47,14 +49,16 @@ public interface Repository extends AutoCloseable {
public void setDebug(boolean debugState);
public void backup(boolean quick) throws DataException;
public void backup(boolean quick, String name) throws DataException;
public void performPeriodicMaintenance() throws DataException;
public void exportNodeLocalData() throws DataException;
public void importDataFromFile(String filename) throws DataException;
public void importDataFromFile(String filename) throws DataException, IOException;
public void checkConsistency() throws DataException;
public static void attemptRecovery(String connectionUrl, String name) throws DataException {}
}

View File

@@ -904,6 +904,25 @@ public class HSQLDBAccountRepository implements AccountRepository {
}
}
@Override
public MintingAccountData getMintingAccount(byte[] mintingAccountKey) throws DataException {
try (ResultSet resultSet = this.repository.checkedExecute("SELECT minter_private_key, minter_public_key " +
"FROM MintingAccounts WHERE minter_private_key = ? OR minter_public_key = ?",
mintingAccountKey, mintingAccountKey)) {
if (resultSet == null)
return null;
byte[] minterPrivateKey = resultSet.getBytes(1);
byte[] minterPublicKey = resultSet.getBytes(2);
return new MintingAccountData(minterPrivateKey, minterPublicKey);
} catch (SQLException e) {
throw new DataException("Unable to fetch minting accounts from repository", e);
}
}
@Override
public void save(MintingAccountData mintingAccountData) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("MintingAccounts");

View File

@@ -0,0 +1,298 @@
package org.qortal.repository.hsqldb;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.crosschain.TradeBotData;
import org.qortal.repository.Bootstrap;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import org.qortal.utils.Triple;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
import java.util.List;
public class HSQLDBImportExport {
private static final Logger LOGGER = LogManager.getLogger(Bootstrap.class);
public static void backupTradeBotStates(Repository repository) throws DataException {
HSQLDBImportExport.backupCurrentTradeBotStates(repository);
HSQLDBImportExport.backupArchivedTradeBotStates(repository);
LOGGER.info("Exported sensitive/node-local data: trade bot states");
}
public static void backupMintingAccounts(Repository repository) throws DataException {
HSQLDBImportExport.backupCurrentMintingAccounts(repository);
LOGGER.info("Exported sensitive/node-local data: minting accounts");
}
/* Trade bot states */
/**
* Backs up the trade bot states currently in the repository, without combining them with past ones
* @param repository
* @throws DataException
*/
private static void backupCurrentTradeBotStates(Repository repository) throws DataException {
try {
Path backupDirectory = HSQLDBImportExport.getExportDirectory(true);
// Load current trade bot data
List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
JSONArray currentTradeBotDataJson = new JSONArray();
for (TradeBotData tradeBotData : allTradeBotData) {
JSONObject tradeBotDataJson = tradeBotData.toJson();
currentTradeBotDataJson.put(tradeBotDataJson);
}
// Wrap current trade bot data in an object to indicate the type
JSONObject currentTradeBotDataJsonWrapper = new JSONObject();
currentTradeBotDataJsonWrapper.put("type", "tradeBotStates");
currentTradeBotDataJsonWrapper.put("dataset", "current");
currentTradeBotDataJsonWrapper.put("data", currentTradeBotDataJson);
// Write current trade bot data (just the ones currently in the database)
String fileName = Paths.get(backupDirectory.toString(), "TradeBotStates.json").toString();
FileWriter writer = new FileWriter(fileName);
writer.write(currentTradeBotDataJsonWrapper.toString(2));
writer.close();
} catch (DataException | IOException e) {
throw new DataException("Unable to export trade bot states from repository");
}
}
/**
* Backs up the trade bot states currently in the repository to a separate "archive" file,
* making sure to combine them with any unique states already present in the archive.
* @param repository
* @throws DataException
*/
private static void backupArchivedTradeBotStates(Repository repository) throws DataException {
try {
Path backupDirectory = HSQLDBImportExport.getExportDirectory(true);
// Load current trade bot data
List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
JSONArray allTradeBotDataJson = new JSONArray();
for (TradeBotData tradeBotData : allTradeBotData) {
JSONObject tradeBotDataJson = tradeBotData.toJson();
allTradeBotDataJson.put(tradeBotDataJson);
}
// We need to combine existing archived TradeBotStates data before overwriting
String fileName = Paths.get(backupDirectory.toString(), "TradeBotStatesArchive.json").toString();
File tradeBotStatesBackupFile = new File(fileName);
if (tradeBotStatesBackupFile.exists()) {
String jsonString = new String(Files.readAllBytes(Paths.get(fileName)));
Triple<String, String, JSONArray> parsedJSON = HSQLDBImportExport.parseJSONString(jsonString);
if (parsedJSON.getA() == null || parsedJSON.getC() == null) {
throw new DataException("Missing data when exporting archived trade bot states");
}
String type = parsedJSON.getA();
String dataset = parsedJSON.getB();
JSONArray data = parsedJSON.getC();
if (!type.equals("tradeBotStates") || !dataset.equals("archive")) {
throw new DataException("Format mismatch when exporting archived trade bot states");
}
Iterator<Object> iterator = data.iterator();
while(iterator.hasNext()) {
JSONObject existingTradeBotDataItem = (JSONObject)iterator.next();
String existingTradePrivateKey = (String) existingTradeBotDataItem.get("tradePrivateKey");
// Check if we already have an entry for this trade
boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey));
if (found == false)
// Add the data from the backup file to our "allTradeBotDataJson" array as it's not currently in the db
allTradeBotDataJson.put(existingTradeBotDataItem);
}
}
// Wrap all trade bot data in an object to indicate the type
JSONObject allTradeBotDataJsonWrapper = new JSONObject();
allTradeBotDataJsonWrapper.put("type", "tradeBotStates");
allTradeBotDataJsonWrapper.put("dataset", "archive");
allTradeBotDataJsonWrapper.put("data", allTradeBotDataJson);
// Write ALL trade bot data to archive (current plus states that are no longer in the database)
FileWriter writer = new FileWriter(fileName);
writer.write(allTradeBotDataJsonWrapper.toString(2));
writer.close();
} catch (DataException | IOException e) {
throw new DataException("Unable to export trade bot states from repository");
}
}
/* Minting accounts */
/**
* Backs up the minting accounts currently in the repository, without combining them with past ones
* @param repository
* @throws DataException
*/
private static void backupCurrentMintingAccounts(Repository repository) throws DataException {
try {
Path backupDirectory = HSQLDBImportExport.getExportDirectory(true);
// Load current trade bot data
List<MintingAccountData> allMintingAccountData = repository.getAccountRepository().getMintingAccounts();
JSONArray currentMintingAccountJson = new JSONArray();
for (MintingAccountData mintingAccountData : allMintingAccountData) {
JSONObject mintingAccountDataJson = mintingAccountData.toJson();
currentMintingAccountJson.put(mintingAccountDataJson);
}
// Wrap current trade bot data in an object to indicate the type
JSONObject currentMintingAccountDataJsonWrapper = new JSONObject();
currentMintingAccountDataJsonWrapper.put("type", "mintingAccounts");
currentMintingAccountDataJsonWrapper.put("dataset", "current");
currentMintingAccountDataJsonWrapper.put("data", currentMintingAccountJson);
// Write current trade bot data (just the ones currently in the database)
String fileName = Paths.get(backupDirectory.toString(), "MintingAccounts.json").toString();
FileWriter writer = new FileWriter(fileName);
writer.write(currentMintingAccountDataJsonWrapper.toString(2));
writer.close();
} catch (DataException | IOException e) {
throw new DataException("Unable to export minting accounts from repository");
}
}
/* Utils */
/**
* Imports data from supplied file
* Data type is loaded from the file itself, and if missing, TradeBotStates is assumed
*
* @param filename
* @param repository
* @throws DataException
* @throws IOException
*/
public static void importDataFromFile(String filename, Repository repository) throws DataException, IOException {
Path path = Paths.get(filename);
if (!path.toFile().exists()) {
throw new FileNotFoundException(String.format("File doesn't exist: %s", filename));
}
byte[] fileContents = Files.readAllBytes(path);
if (fileContents == null) {
throw new FileNotFoundException(String.format("Unable to read file contents: %s", filename));
}
LOGGER.info(String.format("Importing %s into repository ...", filename));
String jsonString = new String(fileContents);
Triple<String, String, JSONArray> parsedJSON = HSQLDBImportExport.parseJSONString(jsonString);
if (parsedJSON.getA() == null || parsedJSON.getC() == null) {
throw new DataException(String.format("Missing data when importing %s into repository", filename));
}
String type = parsedJSON.getA();
JSONArray data = parsedJSON.getC();
Iterator<Object> iterator = data.iterator();
while(iterator.hasNext()) {
JSONObject dataJsonObject = (JSONObject)iterator.next();
if (type.equals("tradeBotStates")) {
HSQLDBImportExport.importTradeBotDataJSON(dataJsonObject, repository);
}
else if (type.equals("mintingAccounts")) {
HSQLDBImportExport.importMintingAccountDataJSON(dataJsonObject, repository);
}
else {
throw new DataException(String.format("Unrecognized data type when importing %s into repository", filename));
}
}
LOGGER.info(String.format("Imported %s into repository from %s", type, filename));
}
private static void importTradeBotDataJSON(JSONObject tradeBotDataJson, Repository repository) throws DataException {
TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson);
repository.getCrossChainRepository().save(tradeBotData);
}
private static void importMintingAccountDataJSON(JSONObject mintingAccountDataJson, Repository repository) throws DataException {
MintingAccountData mintingAccountData = MintingAccountData.fromJson(mintingAccountDataJson);
repository.getAccountRepository().save(mintingAccountData);
}
public static Path getExportDirectory(boolean createIfNotExists) throws DataException {
Path backupPath = Paths.get(Settings.getInstance().getExportPath());
if (createIfNotExists) {
// Create the qortal-backup folder if it doesn't exist
try {
Files.createDirectories(backupPath);
} catch (IOException e) {
LOGGER.info(String.format("Unable to create %s folder", backupPath.toString()));
throw new DataException(String.format("Unable to create %s folder", backupPath.toString()));
}
}
return backupPath;
}
/**
* Parses a JSON string and returns "data", "type", and "dataset" fields.
* In the case of legacy JSON files with no type, they are assumed to be TradeBotStates archives,
* as we had never implemented this for any other types.
*
* @param jsonString
* @return Triple<String, String, JSONArray> (type, dataset, data)
*/
private static Triple<String, String, JSONArray> parseJSONString(String jsonString) throws DataException {
String type = null;
String dataset = null;
JSONArray data = null;
try {
// Firstly try importing the new format
JSONObject jsonData = new JSONObject(jsonString);
if (jsonData != null && jsonData.getString("type") != null) {
type = jsonData.getString("type");
dataset = jsonData.getString("dataset");
data = jsonData.getJSONArray("data");
}
} catch (JSONException e) {
// Could be a legacy format which didn't contain a type or any other outer keys, so try importing that
// Treat these as TradeBotStates archives, given that this was the only type previously implemented
try {
type = "tradeBotStates";
dataset = "archive";
data = new JSONArray(jsonString);
} catch (JSONException e2) {
// Still failed, so give up
throw new DataException("Couldn't import JSON file");
}
}
return new Triple(type, dataset, data);
}
}

View File

@@ -2,7 +2,6 @@ package org.qortal.repository.hsqldb;
import java.awt.TrayIcon.MessageType;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.math.BigDecimal;
import java.nio.file.Files;
@@ -21,20 +20,15 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import org.json.JSONArray;
import org.json.JSONObject;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.crypto.Crypto;
import org.qortal.data.crosschain.TradeBotData;
import org.qortal.globalization.Translator;
import org.qortal.gui.SysTray;
import org.qortal.repository.*;
import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
public class HSQLDBRepository implements Repository {
@@ -450,68 +444,13 @@ public class HSQLDBRepository implements Repository {
@Override
public void exportNodeLocalData() throws DataException {
// Create the qortal-backup folder if it doesn't exist
Path backupPath = Paths.get("qortal-backup");
try {
Files.createDirectories(backupPath);
} catch (IOException e) {
LOGGER.info("Unable to create backup folder");
throw new DataException("Unable to create backup folder");
}
try {
// Load trade bot data
List<TradeBotData> allTradeBotData = this.getCrossChainRepository().getAllTradeBotData();
JSONArray allTradeBotDataJson = new JSONArray();
for (TradeBotData tradeBotData : allTradeBotData) {
JSONObject tradeBotDataJson = tradeBotData.toJson();
allTradeBotDataJson.put(tradeBotDataJson);
}
// We need to combine existing TradeBotStates data before overwriting
String fileName = "qortal-backup/TradeBotStates.json";
File tradeBotStatesBackupFile = new File(fileName);
if (tradeBotStatesBackupFile.exists()) {
String jsonString = new String(Files.readAllBytes(Paths.get(fileName)));
JSONArray allExistingTradeBotData = new JSONArray(jsonString);
Iterator<Object> iterator = allExistingTradeBotData.iterator();
while(iterator.hasNext()) {
JSONObject existingTradeBotData = (JSONObject)iterator.next();
String existingTradePrivateKey = (String) existingTradeBotData.get("tradePrivateKey");
// Check if we already have an entry for this trade
boolean found = allTradeBotData.stream().anyMatch(tradeBotData -> Base58.encode(tradeBotData.getTradePrivateKey()).equals(existingTradePrivateKey));
if (found == false)
// We need to add this to our list
allTradeBotDataJson.put(existingTradeBotData);
}
}
FileWriter writer = new FileWriter(fileName);
writer.write(allTradeBotDataJson.toString());
writer.close();
LOGGER.info("Exported sensitive/node-local data: trade bot states");
} catch (DataException | IOException e) {
throw new DataException("Unable to export trade bot states from repository");
}
HSQLDBImportExport.backupTradeBotStates(this);
HSQLDBImportExport.backupMintingAccounts(this);
}
@Override
public void importDataFromFile(String filename) throws DataException {
LOGGER.info(() -> String.format("Importing data into repository from %s", filename));
try {
String jsonString = new String(Files.readAllBytes(Paths.get(filename)));
JSONArray tradeBotDataToImport = new JSONArray(jsonString);
Iterator<Object> iterator = tradeBotDataToImport.iterator();
while(iterator.hasNext()) {
JSONObject tradeBotDataJson = (JSONObject)iterator.next();
TradeBotData tradeBotData = TradeBotData.fromJson(tradeBotDataJson);
this.getCrossChainRepository().save(tradeBotData);
}
} catch (IOException e) {
throw new DataException("Unable to import sensitive/node-local trade bot states to repository: " + e.getMessage());
}
LOGGER.info(() -> String.format("Imported trade bot states into repository from %s", filename));
public void importDataFromFile(String filename) throws DataException, IOException {
HSQLDBImportExport.importDataFromFile(filename, this);
}
@Override

View File

@@ -184,6 +184,9 @@ public class Settings {
/** Repository connection pool size. Needs to be a bit bigger than maxNetworkThreadPoolSize */
private int repositoryConnectionPoolSize = 100;
// Export/import
private String exportPath = "qortal-backup";
// Auto-update sources
private String[] autoUpdateRepos = new String[] {
"https://github.com/Qortal/qortal/raw/%s/qortal.update",
@@ -502,6 +505,10 @@ public class Settings {
return this.repositoryConnectionPoolSize;
}
public String getExportPath() {
return this.exportPath;
}
public boolean isAutoUpdateEnabled() {
return this.autoUpdateEnabled;
}