greenflame089/qortal
1
0
Fork 0
forked from Qortal-Forker/qortal
Code Pull Requests Releases Activity

Merge branch 'master' into arbitrary-resources-cache

Browse Source 
# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
#	src/main/java/org/qortal/repository/RepositoryManager.java
#	src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
This commit is contained in:
CalDescent
2023-05-26 19:37:13 +02:00
parent f5f82dc3f6 eda6ab5701
commit 74a914367f
23 changed files with 474 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.qortal</groupId>
<artifactId>qortal</artifactId>
<version>4.0.3</version>
<version>4.1.2</version>
<packaging>jar</packaging>
<properties>
<skipTests>true</skipTests>

25
src/main/java/org/qortal/api/resource/BlocksResource.java
View File

@@ -222,14 +222,25 @@ public class BlocksResource {
}
try (final Repository repository = RepositoryManager.getRepository()) {
// Check if the block exists in either the database or archive
if (repository.getBlockRepository().getHeightFromSignature(signature) == 0 &&
repository.getBlockArchiveRepository().getHeightFromSignature(signature) == 0) {
// Not found in either the database or archive
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
}
// Check if the block exists in either the database or archive
int height = repository.getBlockRepository().getHeightFromSignature(signature);
if (height == 0) {
height = repository.getBlockArchiveRepository().getHeightFromSignature(signature);
if (height == 0) {
// Not found in either the database or archive
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
}
}
return repository.getBlockRepository().getTransactionsFromSignature(signature, limit, offset, reverse);
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, height, height);
// Expand signatures to transactions
List<TransactionData> transactions = new ArrayList<>(signatures.size());
for (byte[] s : signatures) {
transactions.add(repository.getTransactionRepository().fromSignature(s));
}
return transactions;
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}

70
src/main/java/org/qortal/api/resource/StatsResource.java Normal file
View File

@@ -0,0 +1,70 @@
package org.qortal.api.resource;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.*;
import org.qortal.block.BlockChain;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.utils.Amounts;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.math.BigDecimal;
import java.util.List;
@Path("/stats")
@Tag(name = "Stats")
public class StatsResource {
private static final Logger LOGGER = LogManager.getLogger(StatsResource.class);
@Context
HttpServletRequest request;
@GET
@Path("/supply/circulating")
@Operation(
summary = "Fetch circulating QORT supply",
responses = {
@ApiResponse(
description = "circulating supply of QORT",
content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "string", format = "number"))
)
}
)
public BigDecimal circulatingSupply() {
long total = 0L;
try (final Repository repository = RepositoryManager.getRepository()) {
int currentHeight = repository.getBlockRepository().getBlockchainHeight();
List<BlockChain.RewardByHeight> rewardsByHeight = BlockChain.getInstance().getBlockRewardsByHeight();
int rewardIndex = rewardsByHeight.size() - 1;
BlockChain.RewardByHeight rewardInfo = rewardsByHeight.get(rewardIndex);
for (int height = currentHeight; height > 1; --height) {
if (height < rewardInfo.height) {
--rewardIndex;
rewardInfo = rewardsByHeight.get(rewardIndex);
}
total += rewardInfo.reward;
}
return Amounts.toBigDecimal(total);
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
}

21
src/main/java/org/qortal/api/resource/TransactionsResource.java
View File

@@ -215,10 +215,25 @@ public class TransactionsResource {
}
try (final Repository repository = RepositoryManager.getRepository()) {
if (repository.getBlockRepository().getHeightFromSignature(signature) == 0)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
// Check if the block exists in either the database or archive
int height = repository.getBlockRepository().getHeightFromSignature(signature);
if (height == 0) {
height = repository.getBlockArchiveRepository().getHeightFromSignature(signature);
if (height == 0) {
// Not found in either the database or archive
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);
}
}
return repository.getBlockRepository().getTransactionsFromSignature(signature, limit, offset, reverse);
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, height, height);
// Expand signatures to transactions
List<TransactionData> transactions = new ArrayList<>(signatures.size());
for (byte[] s : signatures) {
transactions.add(repository.getTransactionRepository().fromSignature(s));
}
return transactions;
} catch (ApiException e) {
throw e;
} catch (DataException e) {

11
src/main/java/org/qortal/block/Block.java
View File

@@ -1686,12 +1686,14 @@ public class Block {
transactionData.getSignature());
this.repository.getBlockRepository().save(blockTransactionData);
// Update transaction's height in repository
// Update transaction's height in repository and local transactionData
transactionRepository.updateBlockHeight(transactionData.getSignature(), this.blockData.getHeight());
// Update local transactionData's height too
transaction.getTransactionData().setBlockHeight(this.blockData.getHeight());
// Update transaction's sequence in repository and local transactionData
transactionRepository.updateBlockSequence(transactionData.getSignature(), sequence);
transaction.getTransactionData().setBlockSequence(sequence);
// No longer unconfirmed
transactionRepository.confirmTransaction(transactionData.getSignature());
@@ -1778,6 +1780,9 @@ public class Block {
// Unset height
transactionRepository.updateBlockHeight(transactionData.getSignature(), null);
// Unset sequence
transactionRepository.updateBlockSequence(transactionData.getSignature(), null);
}
transactionRepository.deleteParticipants(transactionData);

3
src/main/java/org/qortal/block/BlockChain.java
View File

@@ -871,6 +871,9 @@ public class BlockChain {
BlockData orphanBlockData = repository.getBlockRepository().fromHeight(height);
while (height > targetHeight) {
if (Controller.isStopping()) {
return false;
}
LOGGER.info(String.format("Forcably orphaning block %d", height));
Block block = new Block(repository, orphanBlockData);

21
src/main/java/org/qortal/controller/Controller.java
View File

@@ -403,12 +403,12 @@ public class Controller extends Thread {
RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
try (final Repository repository = RepositoryManager.getRepository()) {
RepositoryManager.rebuildTransactionSequences(repository);
ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, false);
}
}
catch (DataException e) {
// If exception has no cause then repository is in use by some other process.
if (e.getCause() == null) {
} catch (DataException e) {
// If exception has no cause or message then repository is in use by some other process.
if (e.getCause() == null && e.getMessage() == null) {
LOGGER.info("Repository in use by another process?");
Gui.getInstance().fatalError("Repository issue", "Repository in use by another process?");
} else {
@@ -442,6 +442,19 @@ public class Controller extends Thread {
}
}
try (Repository repository = RepositoryManager.getRepository()) {
if (RepositoryManager.needsTransactionSequenceRebuild(repository)) {
// Don't allow the node to start if transaction sequences haven't been built yet
// This is needed to handle a case when bootstrapping
LOGGER.error("Database upgrade needed. Please restart the core to complete the upgrade process.");
Gui.getInstance().fatalError("Database upgrade needed", "Please restart the core to complete the upgrade process.");
return;
}
} catch (DataException e) {
LOGGER.error("Error checking transaction sequences in repository", e);
return;
}
// Import current trade bot states and minting accounts if they exist
Controller.importRepositoryData();

6
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@@ -57,6 +57,8 @@ public class ArbitraryDataStorageManager extends Thread {
* This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */
public static final double DELETION_THRESHOLD = 0.98f; // 98%
private static final long PER_NAME_STORAGE_MULTIPLIER = 4L;
public ArbitraryDataStorageManager() {
}
@@ -535,7 +537,9 @@ public class ArbitraryDataStorageManager extends Thread {
}
double maxStorageCapacity = (double)this.storageCapacity * threshold;
long maxStoragePerName = (long)(maxStorageCapacity / (double)followedNamesCount);
// Some names won't need/use much space, so give all names a 4x multiplier to compensate
long maxStoragePerName = (long)(maxStorageCapacity / (double)followedNamesCount) * PER_NAME_STORAGE_MULTIPLIER;
return maxStoragePerName;
}

14
src/main/java/org/qortal/data/transaction/TransactionData.java
View File

@@ -79,6 +79,10 @@ public abstract class TransactionData {
@Schema(accessMode = AccessMode.READ_ONLY, hidden = true, description = "height of block containing transaction")
protected Integer blockHeight;
// Not always present
@Schema(accessMode = AccessMode.READ_ONLY, hidden = true, description = "sequence in block containing transaction")
protected Integer blockSequence;
// Not always present
@Schema(accessMode = AccessMode.READ_ONLY, description = "group-approval status")
protected ApprovalStatus approvalStatus;
@@ -109,6 +113,7 @@ public abstract class TransactionData {
this.fee = baseTransactionData.fee;
this.signature = baseTransactionData.signature;
this.blockHeight = baseTransactionData.blockHeight;
this.blockSequence = baseTransactionData.blockSequence;
this.approvalStatus = baseTransactionData.approvalStatus;
this.approvalHeight = baseTransactionData.approvalHeight;
}
@@ -177,6 +182,15 @@ public abstract class TransactionData {
this.blockHeight = blockHeight;
}
public Integer getBlockSequence() {
return this.blockSequence;
}
@XmlTransient
public void setBlockSequence(Integer blockSequence) {
this.blockSequence = blockSequence;
}
public ApprovalStatus getApprovalStatus() {
return approvalStatus;
}

2
src/main/java/org/qortal/network/Handshake.java
View File

@@ -265,7 +265,7 @@ public enum Handshake {
private static final long PEER_VERSION_131 = 0x0100030001L;
/** Minimum peer version that we are allowed to communicate with */
private static final String MIN_PEER_VERSION = "3.8.2";
private static final String MIN_PEER_VERSION = "4.0.0";
private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes
private static final int POW_DIFFICULTY_PRE_131 = 8; // leading zero bits

175
src/main/java/org/qortal/repository/RepositoryManager.java
View File

@@ -2,18 +2,23 @@ package org.qortal.repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.block.Block;
import org.qortal.crypto.Crypto;
import org.qortal.data.at.ATData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.ATTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.gui.SplashFrame;
import org.qortal.settings.Settings;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.transform.block.BlockTransformation;
import java.sql.SQLException;
import java.util.List;
import java.util.*;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import static org.qortal.transaction.Transaction.TransactionType.AT;
public abstract class RepositoryManager {
private static final Logger LOGGER = LogManager.getLogger(RepositoryManager.class);
@@ -65,6 +70,164 @@ public abstract class RepositoryManager {
}
}
public static boolean needsTransactionSequenceRebuild(Repository repository) throws DataException {
// Check if we have any transactions without a block_sequence
List<byte[]> testSignatures = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
null, Arrays.asList("block_height IS NOT NULL AND block_sequence IS NULL"), new ArrayList<>(), 100);
if (testSignatures.isEmpty()) {
// block_sequence intact, so assume complete
return false;
}
return true;
}
public static boolean rebuildTransactionSequences(Repository repository) throws DataException {
if (Settings.getInstance().isLite()) {
// Lite nodes have no blockchain
return false;
}
if (Settings.getInstance().isTopOnly()) {
// topOnly nodes are unable to perform this reindex, and so are temporarily unsupported
throw new DataException("topOnly nodes are now unsupported, as they are missing data required for a db reshape");
}
try {
// Check if we have any unpopulated block_sequence values for the first 1000 blocks
if (!needsTransactionSequenceRebuild(repository)) {
// block_sequence already populated for the first 1000 blocks, so assume complete.
// We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so
// we shouldn't ever be left in a partially rebuilt state.
return false;
}
LOGGER.info("Rebuilding transaction sequences - this will take a while...");
SplashFrame.getInstance().updateStatus("Rebuilding transactions - please wait...");
int blockchainHeight = repository.getBlockRepository().getBlockchainHeight();
int totalTransactionCount = 0;
for (int height = 1; height <= blockchainHeight; ++height) {
List<TransactionData> inputTransactions = new ArrayList<>();
// Fetch block and transactions
BlockData blockData = repository.getBlockRepository().fromHeight(height);
boolean loadedFromArchive = false;
if (blockData == null) {
// Get (non-AT) transactions from the archive
BlockTransformation blockTransformation = BlockArchiveReader.getInstance().fetchBlockAtHeight(height);
blockData = blockTransformation.getBlockData();
inputTransactions = blockTransformation.getTransactions(); // This doesn't include AT transactions
loadedFromArchive = true;
}
else {
// Get transactions from db
Block block = new Block(repository, blockData);
for (Transaction transaction : block.getTransactions()) {
inputTransactions.add(transaction.getTransactionData());
}
}
if (blockData == null) {
throw new DataException("Missing block data");
}
List<TransactionData> transactions = new ArrayList<>();
if (loadedFromArchive) {
List<TransactionData> transactionDataList = new ArrayList<>(blockData.getTransactionCount());
// Fetch any AT transactions in this block
List<byte[]> atSignatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, height, height);
for (byte[] s : atSignatures) {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(s);
if (transactionData.getType() == AT) {
transactionDataList.add(transactionData);
}
}
List<ATTransactionData> atTransactions = new ArrayList<>();
for (TransactionData transactionData : transactionDataList) {
ATTransactionData atTransactionData = (ATTransactionData) transactionData;
atTransactions.add(atTransactionData);
}
// Create sorted list of ATs by creation time
List<ATData> ats = new ArrayList<>();
for (ATTransactionData atTransactionData : atTransactions) {
ATData atData = repository.getATRepository().fromATAddress(atTransactionData.getATAddress());
boolean hasExistingEntry = ats.stream().anyMatch(a -> Objects.equals(a.getATAddress(), atTransactionData.getATAddress()));
if (!hasExistingEntry) {
ats.add(atData);
}
}
// Sort list of ATs by creation date
ats.sort(Comparator.comparingLong(ATData::getCreation));
// Loop through unique ATs
for (ATData atData : ats) {
List<ATTransactionData> thisAtTransactions = atTransactions.stream()
.filter(t -> Objects.equals(t.getATAddress(), atData.getATAddress()))
.collect(Collectors.toList());
int count = thisAtTransactions.size();
if (count == 1) {
ATTransactionData atTransactionData = thisAtTransactions.get(0);
transactions.add(atTransactionData);
}
else if (count == 2) {
String atCreatorAddress = Crypto.toAddress(atData.getCreatorPublicKey());
ATTransactionData atTransactionData1 = thisAtTransactions.stream()
.filter(t -> !Objects.equals(t.getRecipient(), atCreatorAddress))
.findFirst().orElse(null);
transactions.add(atTransactionData1);
ATTransactionData atTransactionData2 = thisAtTransactions.stream()
.filter(t -> Objects.equals(t.getRecipient(), atCreatorAddress))
.findFirst().orElse(null);
transactions.add(atTransactionData2);
}
else if (count > 2) {
LOGGER.info("Error: AT has more than 2 output transactions");
}
}
}
// Add all the regular transactions now that AT transactions have been handled
transactions.addAll(inputTransactions);
totalTransactionCount += transactions.size();
// Loop through and update sequences
for (int sequence = 0; sequence < transactions.size(); ++sequence) {
TransactionData transactionData = transactions.get(sequence);
// Update transaction's sequence in repository
repository.getTransactionRepository().updateBlockSequence(transactionData.getSignature(), sequence);
}
if (height % 10000 == 0) {
LOGGER.info("Rebuilt sequences for {} blocks (total transactions: {})", height, totalTransactionCount);
}
repository.saveChanges();
}
LOGGER.info("Completed rebuild of transaction sequences.");
return true;
}
catch (DataException e) {
LOGGER.info("Unable to rebuild transaction sequences: {}. The database may have been left in an inconsistent state.", e.getMessage());
// Throw an exception so that the node startup is halted, allowing for a retry next time.
repository.discardChanges();
throw new DataException("Rebuild of transaction sequences failed.");
}
}
public static void setRequestedCheckpoint(Boolean quick) {
quickCheckpointRequested = quick;
}

19
src/main/java/org/qortal/repository/TransactionRepository.java
View File

@@ -125,6 +125,23 @@ public interface TransactionRepository {
public List<byte[]> getSignaturesMatchingCustomCriteria(TransactionType txType, List<String> whereClauses,
List<Object> bindParams) throws DataException;
/**
* Returns signatures for transactions that match search criteria, with optional limit.
* <p>
* Alternate version that allows for custom where clauses and bind params.
* Only use for very specific use cases, such as the names integrity check.
* Not advised to be used otherwise, given that it could be possible for
* unsanitized inputs to be passed in if not careful.
*
* @param txType
* @param whereClauses
* @param bindParams
* @return
* @throws DataException
*/
public List<byte[]> getSignaturesMatchingCustomCriteria(TransactionType txType, List<String> whereClauses,
List<Object> bindParams, Integer limit) throws DataException;
/**
* Returns signature for latest auto-update transaction.
* <p>
@@ -309,6 +326,8 @@ public interface TransactionRepository {
public void updateBlockHeight(byte[] signature, Integer height) throws DataException;
public void updateBlockSequence(byte[] signature, Integer sequence) throws DataException;
public void updateApprovalHeight(byte[] signature, Integer approvalHeight) throws DataException;
/**

16
src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java
View File

@@ -296,10 +296,9 @@ public class HSQLDBATRepository implements ATRepository {
@Override
public Integer getATCreationBlockHeight(String atAddress) throws DataException {
String sql = "SELECT height "
String sql = "SELECT block_height "
+ "FROM DeployATTransactions "
+ "JOIN BlockTransactions ON transaction_signature = signature "
+ "JOIN Blocks ON Blocks.signature = block_signature "
+ "JOIN Transactions USING (signature) "
+ "WHERE AT_address = ? "
+ "LIMIT 1";
@@ -877,18 +876,17 @@ public class HSQLDBATRepository implements ATRepository {
public NextTransactionInfo findNextTransaction(String recipient, int height, int sequence) throws DataException {
// We only need to search for a subset of transaction types: MESSAGE, PAYMENT or AT
String sql = "SELECT height, sequence, Transactions.signature "
String sql = "SELECT block_height, block_sequence, Transactions.signature "
+ "FROM ("
+ "SELECT signature FROM PaymentTransactions WHERE recipient = ? "
+ "UNION "
+ "SELECT signature FROM MessageTransactions WHERE recipient = ? "
+ "UNION "
+ "SELECT signature FROM ATTransactions WHERE recipient = ?"
+ ") AS Transactions "
+ "JOIN BlockTransactions ON BlockTransactions.transaction_signature = Transactions.signature "
+ "JOIN Blocks ON Blocks.signature = BlockTransactions.block_signature "
+ "WHERE (height > ? OR (height = ? AND sequence > ?)) "
+ "ORDER BY height ASC, sequence ASC "
+ ") AS SelectedTransactions "
+ "JOIN Transactions USING (signature)"
+ "WHERE (block_height > ? OR (block_height = ? AND block_sequence > ?)) "
+ "ORDER BY block_height ASC, block_sequence ASC "
+ "LIMIT 1";
Object[] bindParams = new Object[] { recipient, recipient, recipient, height, height, sequence };

11
src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
View File

@@ -994,6 +994,17 @@ public class HSQLDBDatabaseUpdates {
break;
case 47:
// Add `block_sequence` to the Transaction table, as the BlockTransactions table is pruned for
// older blocks and therefore the sequence becomes unavailable
LOGGER.info("Reshaping Transactions table - this can take a while...");
stmt.execute("ALTER TABLE Transactions ADD block_sequence INTEGER");
// For finding transactions by height and sequence
LOGGER.info("Adding index to Transactions table - this can take a while...");
stmt.execute("CREATE INDEX TransactionHeightSequenceIndex on Transactions (block_height, block_sequence)");
break;
case 48:
// We need to keep a local cache of arbitrary resources (items published to QDN), for easier searching.
// IMPORTANT: this is a cache of the last known state of a resource (both confirmed
// and valid unconfirmed). It cannot be assumed that all nodes will contain the same state at a

70
src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java
View File

@@ -194,8 +194,7 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
@Override
public TransactionData fromHeightAndSequence(int height, int sequence) throws DataException {
String sql = "SELECT transaction_signature FROM BlockTransactions JOIN Blocks ON signature = block_signature "
+ "WHERE height = ? AND sequence = ?";
String sql = "SELECT signature FROM Transactions WHERE block_height = ? AND block_sequence = ?";
try (ResultSet resultSet = this.repository.checkedExecute(sql, height, sequence)) {
if (resultSet == null)
@@ -657,8 +656,13 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
List<Object> bindParams) throws DataException {
List<byte[]> signatures = new ArrayList<>();
String txTypeClassName = "";
if (txType != null) {
txTypeClassName = txType.className;
}
StringBuilder sql = new StringBuilder(1024);
sql.append(String.format("SELECT signature FROM %sTransactions", txType.className));
sql.append(String.format("SELECT signature FROM %sTransactions", txTypeClassName));
if (!whereClauses.isEmpty()) {
sql.append(" WHERE ");
@@ -690,6 +694,53 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
}
}
public List<byte[]> getSignaturesMatchingCustomCriteria(TransactionType txType, List<String> whereClauses,
List<Object> bindParams, Integer limit) throws DataException {
List<byte[]> signatures = new ArrayList<>();
String txTypeClassName = "";
if (txType != null) {
txTypeClassName = txType.className;
}
StringBuilder sql = new StringBuilder(1024);
sql.append(String.format("SELECT signature FROM %sTransactions", txTypeClassName));
if (!whereClauses.isEmpty()) {
sql.append(" WHERE ");
final int whereClausesSize = whereClauses.size();
for (int wci = 0; wci < whereClausesSize; ++wci) {
if (wci != 0)
sql.append(" AND ");
sql.append(whereClauses.get(wci));
}
}
if (limit != null) {
sql.append(" LIMIT ?");
bindParams.add(limit);
}
LOGGER.trace(() -> String.format("Transaction search SQL: %s", sql));
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return signatures;
do {
byte[] signature = resultSet.getBytes(1);
signatures.add(signature);
} while (resultSet.next());
return signatures;
} catch (SQLException e) {
throw new DataException("Unable to fetch matching transaction signatures from repository", e);
}
}
@Override
public byte[] getLatestAutoUpdateTransaction(TransactionType txType, int txGroupId, Integer service) throws DataException {
StringBuilder sql = new StringBuilder(1024);
@@ -1444,6 +1495,19 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
}
}
@Override
public void updateBlockSequence(byte[] signature, Integer blockSequence) throws DataException {
HSQLDBSaver saver = new HSQLDBSaver("Transactions");
saver.bind("signature", signature).bind("block_sequence", blockSequence);
try {
saver.execute(repository);
} catch (SQLException e) {
throw new DataException("Unable to update transaction's block sequence in repository", e);
}
}
@Override
public void updateApprovalHeight(byte[] signature, Integer approvalHeight) throws DataException {
HSQLDBSaver saver = new HSQLDBSaver("Transactions");

17
src/main/java/org/qortal/settings/Settings.java
View File

@@ -181,7 +181,7 @@ public class Settings {
/** How often to attempt archiving (ms). */
private long archiveInterval = 7171L; // milliseconds
/** Serialization version to use when building an archive */
private int defaultArchiveVersion = 1;
private int defaultArchiveVersion = 2;
/** Whether to automatically bootstrap instead of syncing from genesis */
@@ -201,25 +201,25 @@ public class Settings {
/** Whether to attempt to open the listen port via UPnP */
private boolean uPnPEnabled = true;
/** Minimum number of peers to allow block minting / synchronization. */
private int minBlockchainPeers = 5;
private int minBlockchainPeers = 3;
/** Target number of outbound connections to peers we should make. */
private int minOutboundPeers = 16;
/** Maximum number of peer connections we allow. */
private int maxPeers = 36;
private int maxPeers = 40;
/** Number of slots to reserve for short-lived QDN data transfers */
private int maxDataPeers = 4;
/** Maximum number of threads for network engine. */
private int maxNetworkThreadPoolSize = 32;
private int maxNetworkThreadPoolSize = 120;
/** Maximum number of threads for network proof-of-work compute, used during handshaking. */
private int networkPoWComputePoolSize = 2;
/** Maximum number of retry attempts if a peer fails to respond with the requested data */
private int maxRetries = 2;
/** The number of seconds of no activity before recovery mode begins */
public long recoveryModeTimeout = 10 * 60 * 1000L;
public long recoveryModeTimeout = 24 * 60 * 60 * 1000L;
/** Minimum peer version number required in order to sync with them */
private String minPeerVersion = "3.8.7";
private String minPeerVersion = "4.1.1";
/** Whether to allow connections with peers below minPeerVersion
* If true, we won't sync with them but they can still sync with us, and will show in the peers list
* If false, sync will be blocked both ways, and they will not appear in the peers list */
@@ -267,7 +267,7 @@ public class Settings {
/** Repository storage path. */
private String repositoryPath = "db";
/** Repository connection pool size. Needs to be a bit bigger than maxNetworkThreadPoolSize */
private int repositoryConnectionPoolSize = 100;
private int repositoryConnectionPoolSize = 240;
private List<String> fixedNetwork;
// Export/import
@@ -508,6 +508,9 @@ public class Settings {
if (this.minBlockchainPeers < 1 && !singleNodeTestnet)
throwValidationError("minBlockchainPeers must be at least 1");
if (this.topOnly)
throwValidationError("topOnly mode is no longer supported");
if (this.apiKey != null && this.apiKey.trim().length() < 8)
throwValidationError("apiKey must be at least 8 characters");

7
src/main/java/org/qortal/utils/BlockArchiveUtils.java
View File

@@ -1,5 +1,7 @@
package org.qortal.utils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
@@ -12,6 +14,8 @@ import java.util.List;
public class BlockArchiveUtils {
private static final Logger LOGGER = LogManager.getLogger(BlockArchiveUtils.class);
/**
* importFromArchive
* <p>
@@ -87,7 +91,8 @@ public class BlockArchiveUtils {
} catch (DataException e) {
repository.discardChanges();
throw new IllegalStateException("Unable to import blocks from archive");
LOGGER.info("Unable to import blocks from archive", e);
throw(e);
}
}
repository.saveChanges();

2
src/test/java/org/qortal/test/BootstrapTests.java
View File

@@ -212,7 +212,7 @@ public class BootstrapTests extends Common {
@Test
public void testBootstrapHosts() throws IOException {
String[] bootstrapHosts = Settings.getInstance().getBootstrapHosts();
String[] bootstrapTypes = { "archive", "toponly" };
String[] bootstrapTypes = { "archive" }; // , "toponly"
for (String host : bootstrapHosts) {
for (String type : bootstrapTypes) {

9
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java
View File

@@ -113,13 +113,16 @@ public class ArbitraryDataStorageCapacityTests extends Common {
assertTrue(resourceListManager.addToList("followedNames", "Test2", false));
assertTrue(resourceListManager.addToList("followedNames", "Test3", false));
assertTrue(resourceListManager.addToList("followedNames", "Test4", false));
assertTrue(resourceListManager.addToList("followedNames", "Test5", false));
assertTrue(resourceListManager.addToList("followedNames", "Test6", false));
// Ensure the followed name count is correct
assertEquals(4, resourceListManager.getItemCountForList("followedNames"));
assertEquals(4, ListUtils.followedNamesCount());
assertEquals(6, resourceListManager.getItemCountForList("followedNames"));
assertEquals(6, ListUtils.followedNamesCount());
// Storage space per name should be the total storage capacity divided by the number of names
long expectedStorageCapacityPerName = (long)(totalStorageCapacity / 4.0f);
// then multiplied by 4, to allow for names that don't use much space
long expectedStorageCapacityPerName = (long)(totalStorageCapacity / 6.0f) * 4L;
assertEquals(expectedStorageCapacityPerName, storageManager.storageCapacityPerName(storageFullThreshold));
}

6
src/test/java/org/qortal/test/crosschain/BitcoinTests.java
View File

@@ -8,6 +8,7 @@ import org.bitcoinj.core.Transaction;
import org.bitcoinj.store.BlockStoreException;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.qortal.crosschain.Bitcoin;
import org.qortal.crosschain.ForeignBlockchainException;
@@ -32,6 +33,7 @@ public class BitcoinTests extends Common {
}
@Test
@Ignore("Often fails due to unreliable BTC testnet ElectrumX servers")
public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException {
System.out.println(String.format("Starting BTC instance..."));
System.out.println(String.format("BTC instance started"));
@@ -53,6 +55,7 @@ public class BitcoinTests extends Common {
}
@Test
@Ignore("Often fails due to unreliable BTC testnet ElectrumX servers")
public void testFindHtlcSecret() throws ForeignBlockchainException {
// This actually exists on TEST3 but can take a while to fetch
String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
@@ -65,6 +68,7 @@ public class BitcoinTests extends Common {
}
@Test
@Ignore("Often fails due to unreliable BTC testnet ElectrumX servers")
public void testBuildSpend() {
String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ";
@@ -81,6 +85,7 @@ public class BitcoinTests extends Common {
}
@Test
@Ignore("Often fails due to unreliable BTC testnet ElectrumX servers")
public void testGetWalletBalance() throws ForeignBlockchainException {
String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ";
@@ -102,6 +107,7 @@ public class BitcoinTests extends Common {
}
@Test
@Ignore("Often fails due to unreliable BTC testnet ElectrumX servers")
public void testGetUnusedReceiveAddress() throws ForeignBlockchainException {
String xprv58 = "tprv8ZgxMBicQKsPdahhFSrCdvC1bsWyzHHZfTneTVqUXN6s1wEtZLwAkZXzFP6TYLg2aQMecZLXLre5bTVGajEB55L1HYJcawpdFG66STVAWPJ";

21
src/test/java/org/qortal/test/crosschain/HtlcTests.java
View File

@@ -8,6 +8,7 @@ import org.junit.Ignore;
import org.junit.Test;
import org.qortal.crosschain.Bitcoin;
import org.qortal.crosschain.ForeignBlockchainException;
import org.qortal.crosschain.Litecoin;
import org.qortal.crypto.Crypto;
import org.qortal.crosschain.BitcoinyHTLC;
import org.qortal.repository.DataException;
@@ -18,17 +19,19 @@ import com.google.common.primitives.Longs;
public class HtlcTests extends Common {
private Bitcoin bitcoin;
private Litecoin litecoin;
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings(); // TestNet3
bitcoin = Bitcoin.getInstance();
litecoin = Litecoin.getInstance();
}
@After
public void afterTest() {
Bitcoin.resetForTesting();
bitcoin = null;
litecoin = null;
}
@Test
@@ -52,12 +55,12 @@ public class HtlcTests extends Common {
do {
// We need to perform fresh setup for 1st test
Bitcoin.resetForTesting();
bitcoin = Bitcoin.getInstance();
litecoin = Litecoin.getInstance();
long now = System.currentTimeMillis();
long timestampBoundary = now / 30_000L;
byte[] secret1 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress);
byte[] secret1 = BitcoinyHTLC.findHtlcSecret(litecoin, p2shAddress);
long executionPeriod1 = System.currentTimeMillis() - now;
assertNotNull(secret1);
@@ -65,7 +68,7 @@ public class HtlcTests extends Common {
assertTrue("1st execution period should not be instant!", executionPeriod1 > 10);
byte[] secret2 = BitcoinyHTLC.findHtlcSecret(bitcoin, p2shAddress);
byte[] secret2 = BitcoinyHTLC.findHtlcSecret(litecoin, p2shAddress);
long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1;
assertNotNull(secret2);
@@ -86,7 +89,7 @@ public class HtlcTests extends Common {
// This actually exists on TEST3 but can take a while to fetch
String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
BitcoinyHTLC.Status htlcStatus = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L);
BitcoinyHTLC.Status htlcStatus = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddress, 1L);
assertNotNull(htlcStatus);
System.out.println(String.format("HTLC %s status: %s", p2shAddress, htlcStatus.name()));
@@ -97,21 +100,21 @@ public class HtlcTests extends Common {
do {
// We need to perform fresh setup for 1st test
Bitcoin.resetForTesting();
bitcoin = Bitcoin.getInstance();
litecoin = Litecoin.getInstance();
long now = System.currentTimeMillis();
long timestampBoundary = now / 30_000L;
// Won't ever exist
String p2shAddress = bitcoin.deriveP2shAddress(Crypto.hash160(Longs.toByteArray(now)));
String p2shAddress = litecoin.deriveP2shAddress(Crypto.hash160(Longs.toByteArray(now)));
BitcoinyHTLC.Status htlcStatus1 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L);
BitcoinyHTLC.Status htlcStatus1 = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddress, 1L);
long executionPeriod1 = System.currentTimeMillis() - now;
assertNotNull(htlcStatus1);
assertTrue("1st execution period should not be instant!", executionPeriod1 > 10);
BitcoinyHTLC.Status htlcStatus2 = BitcoinyHTLC.determineHtlcStatus(bitcoin.getBlockchainProvider(), p2shAddress, 1L);
BitcoinyHTLC.Status htlcStatus2 = BitcoinyHTLC.determineHtlcStatus(litecoin.getBlockchainProvider(), p2shAddress, 1L);
long executionPeriod2 = System.currentTimeMillis() - now - executionPeriod1;
assertNotNull(htlcStatus2);

7
src/test/java/org/qortal/test/crosschain/LitecoinTests.java
View File

@@ -5,7 +5,6 @@ import static org.junit.Assert.*;
import java.util.Arrays;
import org.bitcoinj.core.Transaction;
import org.bitcoinj.store.BlockStoreException;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
@@ -33,12 +32,12 @@ public class LitecoinTests extends Common {
}
@Test
public void testGetMedianBlockTime() throws BlockStoreException, ForeignBlockchainException {
public void testGetMedianBlockTime() throws ForeignBlockchainException {
long before = System.currentTimeMillis();
System.out.println(String.format("Bitcoin median blocktime: %d", litecoin.getMedianBlockTime()));
System.out.println(String.format("Litecoin median blocktime: %d", litecoin.getMedianBlockTime()));
long afterFirst = System.currentTimeMillis();
System.out.println(String.format("Bitcoin median blocktime: %d", litecoin.getMedianBlockTime()));
System.out.println(String.format("Litecoin median blocktime: %d", litecoin.getMedianBlockTime()));
long afterSecond = System.currentTimeMillis();
long firstPeriod = afterFirst - before;

3
start.sh
View File

@@ -33,7 +33,8 @@ fi
# Limits Java JVM stack size and maximum heap usage.
# Comment out for bigger systems, e.g. non-routers
# or when API documentation is enabled
# JVM_MEMORY_ARGS="-Xss256k -Xmx128m"
# Uncomment (remove '#' sign) line below if your system has less than 12GB of RAM for optimal RAM defaults
# JVM_MEMORY_ARGS="-Xss1256k -Xmx3128m"
# Although java.net.preferIPv4Stack is supposed to be false
# by default in Java 11, on some platforms (e.g. FreeBSD 12),