Qortal/qortal
1
0
Fork 1
mirror of https://github.com/Qortal/qortal.git synced 2025-07-23 04:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added bootstrap tests

Browse Source 
This involved adding a feature to the test suite in include the option of using a repository located on disk rather than in memory. Also moved the bootstrap compression/extraction working directories to temporary folders.
This commit is contained in:
CalDescent
2021-10-01 07:44:33 +01:00
parent 347d799d85
commit 7375357b11
18 changed files with 396 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/main/java/org/qortal/api/resource/BootstrapResource.java
View File

@@ -12,6 +12,8 @@ import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.repository.Bootstrap;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
@@ -44,17 +46,18 @@ public class BootstrapResource {
public String createBootstrap() {
Security.checkApiCallAllowed(request);
Bootstrap bootstrap = new Bootstrap();
if (!bootstrap.canBootstrap()) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
}
try (final Repository repository = RepositoryManager.getRepository()) {
boolean isBlockchainValid = bootstrap.validateBlockchain();
if (!isBlockchainValid) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
}
Bootstrap bootstrap = new Bootstrap(repository);
if (!bootstrap.canBootstrap()) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
}
boolean isBlockchainValid = bootstrap.validateBlockchain();
if (!isBlockchainValid) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
}
try {
return bootstrap.create();
} catch (DataException | InterruptedException | IOException e) {
@@ -78,7 +81,13 @@ public class BootstrapResource {
public boolean validateBootstrap() {
Security.checkApiCallAllowed(request);
Bootstrap bootstrap = new Bootstrap();
return bootstrap.validateCompleteBlockchain();
try (final Repository repository = RepositoryManager.getRepository()) {
Bootstrap bootstrap = new Bootstrap(repository);
return bootstrap.validateCompleteBlockchain();
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
}
}
}

32
src/main/java/org/qortal/block/BlockChain.java
View File

@@ -508,6 +508,7 @@ public class BlockChain {
}
boolean pruningEnabled = Settings.getInstance().isPruningEnabled();
boolean archiveEnabled = Settings.getInstance().isArchiveEnabled();
boolean hasBlocks = (chainTip != null && chainTip.getHeight() > 1);
if (pruningEnabled && hasBlocks) {
@@ -527,31 +528,16 @@ public class BlockChain {
// Set the number of blocks to validate based on the pruned state of the chain
// If pruned, subtract an extra 10 to allow room for error
int blocksToValidate = pruningEnabled ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440;
int blocksToValidate = (pruningEnabled || archiveEnabled) ? Settings.getInstance().getPruneBlockLimit() - 10 : 1440;
int startHeight = Math.max(repository.getBlockRepository().getBlockchainHeight() - blocksToValidate, 1);
BlockData detachedBlockData = repository.getBlockRepository().getDetachedBlockSignature(startHeight);
if (detachedBlockData != null) {
LOGGER.error(String.format("Block %d's reference does not match any block's signature", detachedBlockData.getHeight()));
// Orphan if we aren't a pruning node
if (!Settings.getInstance().isPruningEnabled()) {
// Wait for blockchain lock (whereas orphan() only tries to get lock)
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
blockchainLock.lock();
try {
LOGGER.info(String.format("Orphaning back to block %d", detachedBlockData.getHeight() - 1));
orphan(detachedBlockData.getHeight() - 1);
} finally {
blockchainLock.unlock();
}
}
else {
LOGGER.error(String.format("Not orphaning because we are in pruning mode. You may be on an " +
"invalid chain and should consider bootstrapping or re-syncing from genesis."));
}
LOGGER.error(String.format("Block %d's reference does not match any block's signature",
detachedBlockData.getHeight()));
LOGGER.error(String.format("Your chain may be invalid and you should consider bootstrapping" +
" or re-syncing from genesis."));
}
}
}
@@ -618,8 +604,10 @@ public class BlockChain {
boolean shouldBootstrap = Settings.getInstance().getBootstrap();
if (shouldBootstrap) {
// Settings indicate that we should apply a bootstrap rather than rebuilding and syncing from genesis
Bootstrap bootstrap = new Bootstrap();
bootstrap.startImport();
try (final Repository repository = RepositoryManager.getRepository()) {
Bootstrap bootstrap = new Bootstrap(repository);
bootstrap.startImport();
}
return;
}

157
src/main/java/org/qortal/repository/Bootstrap.java
View File

@@ -9,27 +9,25 @@ import org.qortal.data.account.MintingAccountData;
import org.qortal.data.block.BlockData;
import org.qortal.data.crosschain.TradeBotData;
import org.qortal.repository.hsqldb.HSQLDBImportExport;
import org.qortal.repository.hsqldb.HSQLDBRepository;
import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory;
import org.qortal.settings.Settings;
import org.qortal.utils.NTP;
import org.qortal.utils.SevenZ;
import java.io.BufferedInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.*;
import java.util.List;
import java.util.concurrent.locks.ReentrantLock;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
public class Bootstrap {
private Repository repository;
private static final Logger LOGGER = LogManager.getLogger(Bootstrap.class);
/** The maximum number of untrimmed blocks allowed to be included in a bootstrap, beyond the trim threshold */
@@ -39,8 +37,8 @@ public class Bootstrap {
private static final int MAXIMUM_UNPRUNED_BLOCKS = 100;
public Bootstrap() {
public Bootstrap(Repository repository) {
this.repository = repository;
}
/**
@@ -50,9 +48,8 @@ public class Bootstrap {
* All failure reasons are logged
*/
public boolean canBootstrap() {
LOGGER.info("Checking repository state...");
try (final Repository repository = RepositoryManager.getRepository()) {
try {
LOGGER.info("Checking repository state...");
final boolean pruningEnabled = Settings.getInstance().isPruningEnabled();
final boolean archiveEnabled = Settings.getInstance().isArchiveEnabled();
@@ -203,73 +200,80 @@ public class Bootstrap {
}
public String create() throws DataException, InterruptedException, IOException {
try (final HSQLDBRepository repository = (HSQLDBRepository) RepositoryManager.getRepository()) {
LOGGER.info("Acquiring blockchain lock...");
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
blockchainLock.lockInterruptibly();
LOGGER.info("Acquiring blockchain lock...");
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
blockchainLock.lockInterruptibly();
Path inputPath = null;
Path inputPath = null;
Path outputPath = null;
try {
LOGGER.info("Exporting local data...");
repository.exportNodeLocalData();
LOGGER.info("Deleting trade bot states...");
List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
for (TradeBotData tradeBotData : allTradeBotData) {
repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
}
LOGGER.info("Deleting minting accounts...");
List<MintingAccountData> mintingAccounts = repository.getAccountRepository().getMintingAccounts();
for (MintingAccountData mintingAccount : mintingAccounts) {
repository.getAccountRepository().delete(mintingAccount.getPrivateKey());
}
repository.saveChanges();
LOGGER.info("Performing repository maintenance...");
repository.performPeriodicMaintenance();
LOGGER.info("Creating bootstrap...");
repository.backup(true, "bootstrap");
LOGGER.info("Moving files to output directory...");
inputPath = Paths.get(Settings.getInstance().getRepositoryPath(), "bootstrap");
outputPath = Paths.get(Files.createTempDirectory("qortal-bootstrap").toString(), "bootstrap");
// Move the db backup to a "bootstrap" folder in the root directory
Files.move(inputPath, outputPath, REPLACE_EXISTING);
// Copy the archive folder to inside the bootstrap folder
FileUtils.copyDirectory(
Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toFile(),
Paths.get(outputPath.toString(), "archive").toFile()
);
LOGGER.info("Compressing...");
String compressedOutputPath = String.format("%s%s", Settings.getInstance().getBootstrapFilenamePrefix(), "bootstrap.7z");
try {
Files.delete(Paths.get(compressedOutputPath));
} catch (NoSuchFileException e) {
// Doesn't exist, so no need to delete
}
SevenZ.compress(compressedOutputPath, outputPath.toFile());
LOGGER.info("Exporting local data...");
repository.exportNodeLocalData();
// Return the path to the compressed bootstrap file
Path finalPath = Paths.get(outputPath.toString(), compressedOutputPath);
return finalPath.toAbsolutePath().toString();
LOGGER.info("Deleting trade bot states...");
List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
for (TradeBotData tradeBotData : allTradeBotData) {
repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
}
} finally {
LOGGER.info("Re-importing local data...");
Path exportPath = HSQLDBImportExport.getExportDirectory(false);
repository.importDataFromFile(Paths.get(exportPath.toString(), "TradeBotStates.json").toString());
repository.importDataFromFile(Paths.get(exportPath.toString(), "MintingAccounts.json").toString());
LOGGER.info("Deleting minting accounts...");
List<MintingAccountData> mintingAccounts = repository.getAccountRepository().getMintingAccounts();
for (MintingAccountData mintingAccount : mintingAccounts) {
repository.getAccountRepository().delete(mintingAccount.getPrivateKey());
}
blockchainLock.unlock();
repository.saveChanges();
LOGGER.info("Performing repository maintenance...");
repository.performPeriodicMaintenance();
LOGGER.info("Creating bootstrap...");
repository.backup(true, "bootstrap");
LOGGER.info("Moving files to output directory...");
inputPath = Paths.get(Settings.getInstance().getRepositoryPath(), "bootstrap");
Path outputPath = Paths.get("bootstrap");
// Cleanup
if (inputPath != null) {
FileUtils.deleteDirectory(inputPath.toFile());
}
if (outputPath != null) {
FileUtils.deleteDirectory(outputPath.toFile());
// Move the db backup to a "bootstrap" folder in the root directory
Files.move(inputPath, outputPath);
// Copy the archive folder to inside the bootstrap folder
FileUtils.copyDirectory(
Paths.get(Settings.getInstance().getRepositoryPath(), "archive").toFile(),
Paths.get(outputPath.toString(), "archive").toFile()
);
LOGGER.info("Compressing...");
String fileName = "bootstrap.7z";
SevenZ.compress(fileName, outputPath.toFile());
// Return the path to the compressed bootstrap file
Path finalPath = Paths.get(outputPath.toString(), fileName);
return finalPath.toAbsolutePath().toString();
} finally {
LOGGER.info("Re-importing local data...");
Path exportPath = HSQLDBImportExport.getExportDirectory(false);
repository.importDataFromFile(Paths.get(exportPath.toString(), "TradeBotStates.json").toString());
repository.importDataFromFile(Paths.get(exportPath.toString(), "MintingAccounts.json").toString());
blockchainLock.unlock();
// Cleanup
if (inputPath != null) {
FileUtils.deleteDirectory(inputPath.toFile());
}
}
}
}
@@ -305,7 +309,7 @@ public class Bootstrap {
try {
LOGGER.info("Downloading bootstrap...");
InputStream in = new URL(bootstrapUrl).openStream();
Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING);
Files.copy(in, path, REPLACE_EXISTING);
break;
} catch (IOException e) {
@@ -324,7 +328,7 @@ public class Bootstrap {
throw new DataException("Unable to download bootstrap");
}
private void importFromPath(Path path) throws InterruptedException, DataException, IOException {
public void importFromPath(Path path) throws InterruptedException, DataException, IOException {
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
blockchainLock.lockInterruptibly();
@@ -332,13 +336,18 @@ public class Bootstrap {
try {
LOGGER.info("Extracting bootstrap...");
Path input = path.toAbsolutePath();
Path output = path.getParent().toAbsolutePath();
Path output = path.toAbsolutePath().getParent().toAbsolutePath();
SevenZ.decompress(input.toString(), output.toFile());
LOGGER.info("Stopping repository...");
// Close the repository while we are still able to
// Otherwise, the caller will run into difficulties when it tries to close it
repository.discardChanges();
repository.close();
// Now close the repository factory so that we can swap out the database files
RepositoryManager.closeRepositoryFactory();
Path inputPath = Paths.get("bootstrap");
Path inputPath = Paths.get(output.toString(), "bootstrap");
Path outputPath = Paths.get(Settings.getInstance().getRepositoryPath());
if (!inputPath.toFile().exists()) {
throw new DataException("Extracted bootstrap doesn't exist");

2
src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java
View File

@@ -267,7 +267,7 @@ public class HSQLDBRepository implements Repository {
public void close() throws DataException {
// Already closed? No need to do anything but maybe report double-call
if (this.connection == null) {
LOGGER.warn("HSQLDBRepository.close() called when repository already closed", new Exception("Repository already closed"));
LOGGER.warn("HSQLDBRepository.close() called when repository already closed. This is expected when bootstrapping.");
return;
}

7
src/main/java/org/qortal/settings/Settings.java
View File

@@ -191,6 +191,9 @@ public class Settings {
// Export/import
private String exportPath = "qortal-backup";
// Bootstrap
private String bootstrapFilenamePrefix = "";
// Auto-update sources
private String[] autoUpdateRepos = new String[] {
"https://github.com/Qortal/qortal/raw/%s/qortal.update",
@@ -513,6 +516,10 @@ public class Settings {
return this.exportPath;
}
public String getBootstrapFilenamePrefix() {
return this.bootstrapFilenamePrefix;
}
public boolean isAutoUpdateEnabled() {
return this.autoUpdateEnabled;
}

4
src/main/java/org/qortal/utils/SevenZ.java
View File

@@ -18,8 +18,8 @@ public class SevenZ {
}
public static void compress(String name, File... files) throws IOException {
try (SevenZOutputFile out = new SevenZOutputFile(new File(name))){
public static void compress(String outputPath, File... files) throws IOException {
try (SevenZOutputFile out = new SevenZOutputFile(new File(outputPath))){
for (File file : files){
addToArchiveCompression(out, file, ".");
}