Qortal/qortal
1
0
Fork 1
mirror of https://github.com/Qortal/qortal.git synced 2025-07-23 04:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into qdn-metadata

Browse Source 
This involved a slight rewrite to remove the "includeMetadataOnly" boolean. Metadata is now always excluded, otherwise it complicates the caching too much.

# Conflicts:
#	src/main/java/org/qortal/api/resource/ArbitraryResource.java
#	src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
This commit is contained in:
CalDescent
2022-02-26 14:39:20 +00:00
parent acddf36467 aa2e5cb87b
commit f22ad13fa9
41 changed files with 995 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/main/java/org/qortal/account/Account.java
View File

@@ -272,7 +272,7 @@ public class Account {
/**
* Returns 'effective' minting level, or zero if reward-share does not exist.
* <p>
* For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
* this is being used on src/main/java/org/qortal/api/resource/AddressesResource.java to fulfil the online accounts api call
*
* @param repository
* @param rewardSharePublicKey
@@ -288,5 +288,26 @@ public class Account {
Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
return rewardShareMinter.getEffectiveMintingLevel();
}
/**
* Returns 'effective' minting level, with a fix for the zero level.
* <p>
* For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
*
* @param repository
* @param rewardSharePublicKey
* @return 0+
* @throws DataException
*/
public static int getRewardShareEffectiveMintingLevelIncludingLevelZero(Repository repository, byte[] rewardSharePublicKey) throws DataException {
// Find actual minter and get their effective minting level
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey);
if (rewardShareData == null)
return 0;
else if(!rewardShareData.getMinter().equals(rewardShareData.getRecipient()))//the minter is different than the recipient this means sponsorship
return 0;
Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
return rewardShareMinter.getEffectiveMintingLevel();
}
}

5
src/main/java/org/qortal/api/HTMLParser.java
View File

@@ -28,6 +28,11 @@ public class HTMLParser {
// Add base href tag
String baseElement = String.format("<base href=\"%s\">", baseUrl);
head.get(0).prepend(baseElement);
// Add meta charset tag
String metaCharsetElement = "<meta charset=\"UTF-8\">";
head.get(0).prepend(metaCharsetElement);
}
String html = document.html();
this.data = html.getBytes();

2
src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java
View File

@@ -17,7 +17,7 @@ import java.util.Map;
@Path("/")
@Tag(name = "Gateway")
@Tag(name = "Domain Map")
public class DomainMapResource {
@Context HttpServletRequest request;

2
src/main/java/org/qortal/api/resource/AddressesResource.java
View File

@@ -198,7 +198,7 @@ public class AddressesResource {
for (OnlineAccountData onlineAccountData : onlineAccounts) {
try {
final int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, onlineAccountData.getPublicKey());
final int minterLevel = Account.getRewardShareEffectiveMintingLevelIncludingLevelZero(repository, onlineAccountData.getPublicKey());
OnlineAccountLevel onlineAccountLevel = onlineAccountLevels.stream()
.filter(a -> a.getLevel() == minterLevel)

4
src/main/java/org/qortal/api/resource/AdminResource.java
View File

@@ -315,6 +315,7 @@ public class AdminResource {
repository.getAccountRepository().save(mintingAccountData);
repository.saveChanges();
repository.exportNodeLocalData();//after adding new minting account let's persist it to the backup MintingAccounts.json
} catch (IllegalArgumentException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e);
} catch (DataException e) {
@@ -355,6 +356,7 @@ public class AdminResource {
return "false";
repository.saveChanges();
repository.exportNodeLocalData();//after removing new minting account let's persist it to the backup MintingAccounts.json
} catch (IllegalArgumentException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e);
} catch (DataException e) {
@@ -546,7 +548,7 @@ public class AdminResource {
@Path("/repository/data")
@Operation(
summary = "Export sensitive/node-local data from repository.",
description = "Exports data to .script files on local machine"
description = "Exports data to .json files on local machine"
)
@ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE})
@SecurityRequirement(name = "apiKey")

26
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@@ -430,17 +430,12 @@ public class ArbitraryResource {
@ApiErrors({ApiError.REPOSITORY_ISSUE})
public List<ArbitraryTransactionData> getHostedTransactions(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
@QueryParam("includemetadata") Boolean includeMetadata) {
@Parameter(ref = "offset") @QueryParam("offset") Integer offset) {
Security.checkApiCallAllowed(request);
if (includeMetadata == null) {
includeMetadata = false;
}
try (final Repository repository = RepositoryManager.getRepository()) {
List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset, includeMetadata);
List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset);
return hostedTransactions;
@@ -465,18 +460,21 @@ public class ArbitraryResource {
@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
@QueryParam("includemetadata") Boolean includeMetadata) {
@QueryParam("query") String query) {
Security.checkApiCallAllowed(request);
List<ArbitraryResourceInfo> resources = new ArrayList<>();
if (includeMetadata == null) {
includeMetadata = false;
}
try (final Repository repository = RepositoryManager.getRepository()) {
List<ArbitraryTransactionData> transactionDataList;
if (query == null || query.equals("")) {
transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset);
} else {
transactionDataList = ArbitraryDataStorageManager.getInstance().searchHostedTransactions(repository,query, limit, offset);
}
List<ArbitraryTransactionData> transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset, includeMetadata);
for (ArbitraryTransactionData transactionData : transactionDataList) {
ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
arbitraryResourceInfo.name = transactionData.getName();
@@ -498,6 +496,8 @@ public class ArbitraryResource {
}
}
@DELETE
@Path("/resource/{service}/{name}/{identifier}")
@Operation(

2
src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java
View File

@@ -122,7 +122,7 @@ public class CrossChainBitcoinResource {
@Path("/send")
@Operation(
summary = "Sends BTC from hierarchical, deterministic BIP32 wallet to specific address",
description = "Currently only supports 'legacy' P2PKH Bitcoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
description = "Currently supports 'legacy' P2PKH Bitcoin addresses and Native SegWit (P2WPKH) addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
requestBody = @RequestBody(
required = true,
content = @Content(

2
src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java
View File

@@ -122,7 +122,7 @@ public class CrossChainLitecoinResource {
@Path("/send")
@Operation(
summary = "Sends LTC from hierarchical, deterministic BIP32 wallet to specific address",
description = "Currently only supports 'legacy' P2PKH Litecoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
description = "Currently supports 'legacy' P2PKH Litecoin addresses and Native SegWit (P2WPKH) addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
requestBody = @RequestBody(
required = true,
content = @Content(

2
src/main/java/org/qortal/api/resource/PeersResource.java
View File

@@ -354,7 +354,7 @@ public class PeersResource {
List<Peer> connectedPeers = Network.getInstance().getConnectedPeers().stream().collect(Collectors.toList());
for (Peer peer : connectedPeers) {
if (peer.isOutbound()) {
if (!peer.isOutbound()) {
peersSummary.inboundConnections++;
}
else {

5
src/main/java/org/qortal/api/resource/TransactionsResource.java
View File

@@ -638,7 +638,10 @@ public class TransactionsResource {
ApiError.BLOCKCHAIN_NEEDS_SYNC, ApiError.INVALID_SIGNATURE, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE
})
public String processTransaction(String rawBytes58) {
if (!Controller.getInstance().isUpToDate())
// Only allow a transaction to be processed if our latest block is less than 30 minutes old
// If older than this, we should first wait until the blockchain is synced
final Long minLatestBlockTimestamp = NTP.getTime() - (30 * 60 * 1000L);
if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC);
byte[] rawBytes = Base58.decode(rawBytes58);

7
src/main/java/org/qortal/api/websocket/PresenceWebSocket.java
View File

@@ -20,6 +20,7 @@ import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
import org.eclipse.jetty.websocket.api.annotations.WebSocket;
import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.crypto.Crypto;
import org.qortal.data.transaction.PresenceTransactionData;
import org.qortal.data.transaction.TransactionData;
@@ -99,13 +100,13 @@ public class PresenceWebSocket extends ApiWebSocket implements Listener {
@Override
public void listen(Event event) {
// We use NewBlockEvent as a proxy for 1-minute timer
if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Controller.NewBlockEvent))
// We use Synchronizer.NewChainTipEvent as a proxy for 1-minute timer
if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Synchronizer.NewChainTipEvent))
return;
removeOldEntries();
if (event instanceof Controller.NewBlockEvent)
if (event instanceof Synchronizer.NewChainTipEvent)
// We only wanted a chance to cull old entries
return;

5
src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java
View File

@@ -23,6 +23,7 @@ import org.eclipse.jetty.websocket.api.annotations.WebSocket;
import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
import org.qortal.api.model.CrossChainOfferSummary;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.crosschain.SupportedBlockchain;
import org.qortal.crosschain.ACCT;
import org.qortal.crosschain.AcctMode;
@@ -80,10 +81,10 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener {
@Override
public void listen(Event event) {
if (!(event instanceof Controller.NewBlockEvent))
if (!(event instanceof Synchronizer.NewChainTipEvent))
return;
BlockData blockData = ((Controller.NewBlockEvent) event).getBlockData();
BlockData blockData = ((Synchronizer.NewChainTipEvent) event).getNewChainTip();
// Process any new info

15
src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java
View File

@@ -366,6 +366,21 @@ public class ArbitraryDataFile {
return false;
}
public boolean delete(int attempts) {
// Keep trying to delete the data until it is deleted, or we reach 10 attempts
for (int i=0; i<attempts; i++) {
if (this.delete()) {
return true;
}
try {
Thread.sleep(1000L);
} catch (InterruptedException e) {
// Fall through to exit method
}
}
return false;
}
public boolean deleteAllChunks() {
boolean success = false;

104
src/main/java/org/qortal/controller/Controller.java
View File

@@ -105,6 +105,8 @@ public class Controller extends Thread {
private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L);
/** How many (latest) blocks' worth of online accounts we cache */
private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2;
private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L;
private static volatile boolean isStopping = false;
private static BlockMinter blockMinter = null;
@@ -774,14 +776,14 @@ public class Controller extends Thread {
actionText = Translator.INSTANCE.translate("SysTray", "MINTING_ENABLED");
SysTray.getInstance().setTrayIcon(2);
}
else if (Synchronizer.getInstance().isSynchronizing()) {
actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent());
SysTray.getInstance().setTrayIcon(3);
}
else if (numberOfPeers < Settings.getInstance().getMinBlockchainPeers()) {
actionText = Translator.INSTANCE.translate("SysTray", "CONNECTING");
SysTray.getInstance().setTrayIcon(3);
}
else if (!this.isUpToDate()) {
actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent());
SysTray.getInstance().setTrayIcon(3);
}
else {
actionText = Translator.INSTANCE.translate("SysTray", "MINTING_DISABLED");
SysTray.getInstance().setTrayIcon(4);
@@ -1291,6 +1293,14 @@ public class Controller extends Thread {
onNetworkOnlineAccountsMessage(peer, message);
break;
case GET_ONLINE_ACCOUNTS_V2:
onNetworkGetOnlineAccountsV2Message(peer, message);
break;
case ONLINE_ACCOUNTS_V2:
onNetworkOnlineAccountsV2Message(peer, message);
break;
case GET_ARBITRARY_DATA:
// Not currently supported
break;
@@ -1704,6 +1714,53 @@ public class Controller extends Thread {
}
}
private void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) {
GetOnlineAccountsV2Message getOnlineAccountsMessage = (GetOnlineAccountsV2Message) message;
List<OnlineAccountData> excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts();
// Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts
List<OnlineAccountData> accountsToSend;
synchronized (this.onlineAccounts) {
accountsToSend = new ArrayList<>(this.onlineAccounts);
}
Iterator<OnlineAccountData> iterator = accountsToSend.iterator();
SEND_ITERATOR:
while (iterator.hasNext()) {
OnlineAccountData onlineAccountData = iterator.next();
for (int i = 0; i < excludeAccounts.size(); ++i) {
OnlineAccountData excludeAccountData = excludeAccounts.get(i);
if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) {
iterator.remove();
continue SEND_ITERATOR;
}
}
}
Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend);
peer.sendMessage(onlineAccountsMessage);
LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer));
}
private void onNetworkOnlineAccountsV2Message(Peer peer, Message message) {
OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message;
List<OnlineAccountData> peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts();
LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer));
try (final Repository repository = RepositoryManager.getRepository()) {
for (OnlineAccountData onlineAccountData : peersOnlineAccounts)
this.verifyAndAddAccount(repository, onlineAccountData);
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e);
}
}
// Utilities
private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException {
@@ -1815,11 +1872,17 @@ public class Controller extends Thread {
// Request data from other peers?
if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) {
Message message;
List<OnlineAccountData> safeOnlineAccounts;
synchronized (this.onlineAccounts) {
message = new GetOnlineAccountsMessage(this.onlineAccounts);
safeOnlineAccounts = new ArrayList<>(this.onlineAccounts);
}
Network.getInstance().broadcast(peer -> message);
Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts);
Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts);
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
);
}
// Refresh our online accounts signatures?
@@ -1911,8 +1974,12 @@ public class Controller extends Thread {
if (!hasInfoChanged)
return;
Message message = new OnlineAccountsMessage(ourOnlineAccounts);
Network.getInstance().broadcast(peer -> message);
Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts);
Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts);
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
);
LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp));
}
@@ -1998,10 +2065,13 @@ public class Controller extends Thread {
return peers;
}
/** Returns whether we think our node has up-to-date blockchain based on our info about other peers. */
public boolean isUpToDate() {
/**
* Returns whether we think our node has up-to-date blockchain based on our info about other peers.
* @param minLatestBlockTimestamp - the minimum block timestamp to be considered recent
* @return boolean - whether our node's blockchain is up to date or not
*/
public boolean isUpToDate(Long minLatestBlockTimestamp) {
// Do we even have a vaguely recent block?
final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp();
if (minLatestBlockTimestamp == null)
return false;
@@ -2027,6 +2097,16 @@ public class Controller extends Thread {
return !peers.isEmpty();
}
/**
* Returns whether we think our node has up-to-date blockchain based on our info about other peers.
* Uses the default minLatestBlockTimestamp value.
* @return boolean - whether our node's blockchain is up to date or not
*/
public boolean isUpToDate() {
final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp();
return this.isUpToDate(minLatestBlockTimestamp);
}
/** Returns minimum block timestamp for block to be considered 'recent', or <tt>null</tt> if NTP not synced. */
public static Long getMinimumLatestBlockTimestamp() {
Long now = NTP.getTime();

22
src/main/java/org/qortal/controller/Synchronizer.java
View File

@@ -22,6 +22,8 @@ import org.qortal.data.block.CommonBlockData;
import org.qortal.data.network.PeerChainTipData;
import org.qortal.data.transaction.RewardShareTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.event.Event;
import org.qortal.event.EventBus;
import org.qortal.network.Network;
import org.qortal.network.Peer;
import org.qortal.network.message.BlockMessage;
@@ -96,6 +98,24 @@ public class Synchronizer extends Thread {
OK, NOTHING_TO_DO, GENESIS_ONLY, NO_COMMON_BLOCK, TOO_DIVERGENT, NO_REPLY, INFERIOR_CHAIN, INVALID_DATA, NO_BLOCKCHAIN_LOCK, REPOSITORY_ISSUE, SHUTTING_DOWN;
}
public static class NewChainTipEvent implements Event {
private final BlockData priorChainTip;
private final BlockData newChainTip;
public NewChainTipEvent(BlockData priorChainTip, BlockData newChainTip) {
this.priorChainTip = priorChainTip;
this.newChainTip = newChainTip;
}
public BlockData getPriorChainTip() {
return this.priorChainTip;
}
public BlockData getNewChainTip() {
return this.newChainTip;
}
}
// Constructors
private Synchronizer() {
@@ -338,6 +358,8 @@ public class Synchronizer extends Thread {
Network network = Network.getInstance();
network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip));
EventBus.INSTANCE.notify(new NewChainTipEvent(priorChainTip, newChainTip));
}
return syncResult;

6
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@@ -222,7 +222,11 @@ public class ArbitraryDataCleanupManager extends Thread {
try (final Repository repository = RepositoryManager.getRepository()) {
// Check if there are any hosted files that don't have matching transactions
this.checkForExpiredTransactions(repository);
// UPDATE: This has been disabled for now as it was deleting valid transactions
// and causing chunks to go missing on the network. If ever re-enabled, we MUST
// ensure that original copies of data aren't deleted, and that sufficient time
// is allowed (ideally several hours) before treating a transaction as missing.
// this.checkForExpiredTransactions(repository);
// Delete additional data at random if we're over our storage limit
// Use the DELETION_THRESHOLD so that we only start deleting once the hard limit is reached

57
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
View File

@@ -29,6 +29,7 @@ public class ArbitraryDataFileListManager {
private static ArbitraryDataFileListManager instance;
private static String MIN_PEER_VERSION_FOR_FILE_LIST_STATS = "3.2.0";
/**
* Map of recent incoming requests for ARBITRARY transaction data file lists.
@@ -266,18 +267,16 @@ public class ArbitraryDataFileListManager {
List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
List<byte[]> missingHashes = null;
// // TODO: uncomment after GetArbitraryDataFileListMessage updates are deployed
// // Find hashes that we are missing
// try {
// ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
// arbitraryDataFile.setMetadataHash(metadataHash);
// missingHashes = arbitraryDataFile.missingHashes();
// } catch (DataException e) {
// // Leave missingHashes as null, so that all hashes are requested
// }
// int hashCount = missingHashes != null ? missingHashes.size() : 0;
// Find hashes that we are missing
try {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
arbitraryDataFile.setMetadataHash(metadataHash);
missingHashes = arbitraryDataFile.missingHashes();
} catch (DataException e) {
// Leave missingHashes as null, so that all hashes are requested
}
int hashCount = missingHashes != null ? missingHashes.size() : 0;
int hashCount = 0;
LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size()));
// Build request
@@ -405,6 +404,13 @@ public class ArbitraryDataFileListManager {
ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) {
long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime();
LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}",
totalRequestTime, arbitraryDataFileListMessage.getRequestHops(),
arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
}
// Do we have a pending request for this data?
Triple<String, Peer, Long> request = arbitraryDataFileListRequests.get(message.getId());
if (request == null || request.getA() == null) {
@@ -474,12 +480,26 @@ public class ArbitraryDataFileListManager {
if (!isBlocked) {
Peer requestingPeer = request.getB();
if (requestingPeer != null) {
Long requestTime = arbitraryDataFileListMessage.getRequestTime();
Integer requestHops = arbitraryDataFileListMessage.getRequestHops();
// Add each hash to our local mapping so we know who to ask later
Long now = NTP.getTime();
for (byte[] hash : hashes) {
String hash58 = Base58.encode(hash);
ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now);
ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap);
ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops);
ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo);
}
// Bump requestHops if it exists
if (requestHops != null) {
arbitraryDataFileListMessage.setRequestHops(++requestHops);
}
// Remove optional parameters if the requesting peer doesn't support it yet
// A message with less statistical data is better than no message at all
if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
arbitraryDataFileListMessage.removeOptionalStats();
}
// Forward to requesting peer
@@ -584,8 +604,17 @@ public class ArbitraryDataFileListManager {
arbitraryDataFileListRequests.put(message.getId(), newEntry);
}
ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
String ourAddress = Network.getInstance().getOurExternalIpAddress();
ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature,
hashes, NTP.getTime(), 0, ourAddress, true);
arbitraryDataFileListMessage.setId(message.getId());
// Remove optional parameters if the requesting peer doesn't support it yet
// A message with less statistical data is better than no message at all
if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
arbitraryDataFileListMessage.removeOptionalStats();
}
if (!peer.sendMessage(arbitraryDataFileListMessage)) {
LOGGER.debug("Couldn't send list of hashes");
peer.disconnect("failed to send list of hashes");

44
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java
View File

@@ -37,7 +37,7 @@ public class ArbitraryDataFileManager extends Thread {
/**
* Map to keep track of our in progress (outgoing) arbitrary data file requests
*/
private Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
public Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
/**
* Map to keep track of hashes that we might need to relay
@@ -148,7 +148,7 @@ public class ArbitraryDataFileManager extends Thread {
}
}
else {
LOGGER.trace("Already requesting data file {} for signature {}", arbitraryDataFile, Base58.encode(signature));
LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer);
}
}
else {
@@ -240,16 +240,7 @@ public class ArbitraryDataFileManager extends Thread {
ArbitraryDataFile dataFile = arbitraryDataFileMessage.getArbitraryDataFile();
// Keep trying to delete the data until it is deleted, or we reach 10 attempts
for (int i=0; i<10; i++) {
if (dataFile.delete()) {
break;
}
try {
Thread.sleep(1000L);
} catch (InterruptedException e) {
// Fall through to exit method
}
}
dataFile.delete(10);
}
}
@@ -401,6 +392,33 @@ public class ArbitraryDataFileManager extends Thread {
}
}
private ArbitraryRelayInfo getOptimalRelayInfoEntryForHash(String hash58) {
LOGGER.trace("Fetching relay info for hash: {}", hash58);
List<ArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
if (relayInfoList != null && !relayInfoList.isEmpty()) {
// Remove any with null requestHops
relayInfoList.removeIf(r -> r.getRequestHops() == null);
// If list is now empty, then just return one at random
if (relayInfoList.isEmpty()) {
return this.getRandomRelayInfoEntryForHash(hash58);
}
// Sort by number of hops (lowest first)
relayInfoList.sort(Comparator.comparingInt(ArbitraryRelayInfo::getRequestHops));
// FUTURE: secondary sort by requestTime?
ArbitraryRelayInfo relayInfo = relayInfoList.get(0);
LOGGER.trace("Returning optimal relay info for hash: {} (requestHops {})", hash58, relayInfo.getRequestHops());
return relayInfo;
}
LOGGER.trace("No relay info exists for hash: {}", hash58);
return null;
}
private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) {
LOGGER.trace("Fetching random relay info for hash: {}", hash58);
List<ArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
@@ -451,7 +469,7 @@ public class ArbitraryDataFileManager extends Thread {
try {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
ArbitraryRelayInfo relayInfo = this.getRandomRelayInfoEntryForHash(hash58);
ArbitraryRelayInfo relayInfo = this.getOptimalRelayInfoEntryForHash(hash58);
if (arbitraryDataFile.exists()) {
LOGGER.trace("Hash {} exists", hash58);

11
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java
View File

@@ -31,8 +31,6 @@ public class ArbitraryDataFileRequestThread implements Runnable {
try {
while (!Controller.isStopping()) {
Thread.sleep(1000);
Long now = NTP.getTime();
this.processFileHashes(now);
}
@@ -41,7 +39,7 @@ public class ArbitraryDataFileRequestThread implements Runnable {
}
}
private void processFileHashes(Long now) {
private void processFileHashes(Long now) throws InterruptedException {
if (Controller.isStopping()) {
return;
}
@@ -82,6 +80,12 @@ public class ArbitraryDataFileRequestThread implements Runnable {
continue;
}
// Skip if already requesting, but don't remove, as we might want to retry later
if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) {
// Already requesting - leave this attempt for later
continue;
}
// We want to process this file
shouldProcess = true;
iterator.remove();
@@ -91,6 +95,7 @@ public class ArbitraryDataFileRequestThread implements Runnable {
if (!shouldProcess) {
// Nothing to do
Thread.sleep(1000L);
return;
}

7
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@@ -38,7 +38,7 @@ public class ArbitraryDataManager extends Thread {
private int powDifficulty = 14; // Must not be final, as unit tests need to reduce this value
/** Request timeout when transferring arbitrary data */
public static final long ARBITRARY_REQUEST_TIMEOUT = 10 * 1000L; // ms
public static final long ARBITRARY_REQUEST_TIMEOUT = 12 * 1000L; // ms
/** Maximum time to hold information about an in-progress relay */
public static final long ARBITRARY_RELAY_TIMEOUT = 60 * 1000L; // ms
@@ -80,6 +80,9 @@ public class ArbitraryDataManager extends Thread {
Thread.currentThread().setName("Arbitrary Data Manager");
try {
// Wait for node to finish starting up and making connections
Thread.sleep(2 * 60 * 1000L);
while (!isStopping) {
Thread.sleep(2000);
@@ -370,7 +373,7 @@ public class ArbitraryDataManager extends Thread {
public void broadcastHostedSignatureList() {
try (final Repository repository = RepositoryManager.getRepository()) {
List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, null, null, false);
List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, null, null);
List<byte[]> hostedSignatures = hostedTransactions.stream().map(ArbitraryTransactionData::getSignature).collect(Collectors.toList());
if (!hostedSignatures.isEmpty()) {
// Broadcast the list, using null to represent our peer address

99
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@@ -16,6 +16,7 @@ import org.qortal.utils.FilesystemUtils;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -46,6 +47,9 @@ public class ArbitraryDataStorageManager extends Thread {
private List<ArbitraryTransactionData> hostedTransactions;
private String searchQuery;
private List<ArbitraryTransactionData> searchResultsTransactions;
private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
/** Treat storage as full at 90% usage, to reduce risk of going over the limit.
@@ -257,14 +261,8 @@ public class ArbitraryDataStorageManager extends Thread {
}
// Hosted data
public List<ArbitraryTransactionData> listAllHostedTransactions(Repository repository, Integer limit, Integer offset, boolean includeMetadataOnly) {
// Load from cache if we can, to avoid disk reads
if (this.hostedTransactions != null) {
return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset);
}
public List<ArbitraryTransactionData> loadAllHostedTransactions(Repository repository) {
List<ArbitraryTransactionData> arbitraryTransactionDataList = new ArrayList<>();
// Find all hosted paths
@@ -287,15 +285,13 @@ public class ArbitraryDataStorageManager extends Thread {
}
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
// Make sure to exclude metadata-only resources if requested
if (!includeMetadataOnly) {
if (arbitraryTransactionData.getMetadataHash() != null) {
if (contents.length == 1) {
String metadataHash58 = Base58.encode(arbitraryTransactionData.getMetadataHash());
if (Objects.equals(metadataHash58, contents[0])) {
// We only have the metadata file for this resource, not the actual data, so exclude it
continue;
}
// Make sure to exclude metadata-only resources
if (arbitraryTransactionData.getMetadataHash() != null) {
if (contents.length == 1) {
String metadataHash58 = Base58.encode(arbitraryTransactionData.getMetadataHash());
if (Objects.equals(metadataHash58, contents[0])) {
// We only have the metadata file for this resource, not the actual data, so exclude it
continue;
}
}
}
@@ -311,10 +307,69 @@ public class ArbitraryDataStorageManager extends Thread {
// Sort by newest first
arbitraryTransactionDataList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed());
// Update cache
this.hostedTransactions = arbitraryTransactionDataList;
return arbitraryTransactionDataList;
}
// Hosted data
return ArbitraryTransactionUtils.limitOffsetTransactions(arbitraryTransactionDataList, limit, offset);
public List<ArbitraryTransactionData> listAllHostedTransactions(Repository repository, Integer limit, Integer offset) {
// Load from cache if we can, to avoid disk reads
if (this.hostedTransactions != null) {
return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset);
}
this.hostedTransactions = this.loadAllHostedTransactions(repository);
return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset);
}
/**
* searchHostedTransactions
* Allow to run a query against hosted data names and return matches if there are any
* @param repository
* @param query
* @param limit
* @param offset
* @return
*/
public List<ArbitraryTransactionData> searchHostedTransactions(Repository repository, String query, Integer limit, Integer offset) {
// Load from results cache if we can (results that exists for the same query), to avoid disk reads
if (this.searchResultsTransactions != null && this.searchQuery.equals(query.toLowerCase())) {
return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset);
}
// Using cache if we can, to avoid disk reads
if (this.hostedTransactions == null) {
this.hostedTransactions = this.loadAllHostedTransactions(repository);
}
this.searchQuery = query.toLowerCase(); //set the searchQuery so that it can be checked on the next call
List<ArbitraryTransactionData> searchResultsList = new ArrayList<>();
// Loop through cached hostedTransactions
for (ArbitraryTransactionData atd : this.hostedTransactions) {
try {
if (atd.getName() != null && atd.getName().toLowerCase().contains(this.searchQuery)) {
searchResultsList.add(atd);
}
else if (atd.getIdentifier() != null && atd.getIdentifier().toLowerCase().contains(this.searchQuery)) {
searchResultsList.add(atd);
}
} catch (Exception e) {
continue;
}
}
// Sort by newest first
searchResultsList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed());
// Update cache
this.searchResultsTransactions = searchResultsList;
return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset);
}
/**
@@ -338,7 +393,7 @@ public class ArbitraryDataStorageManager extends Thread {
&& path.getFileName().toString().length() > 32)
.collect(Collectors.toList());
}
catch (IOException e) {
catch (IOException | UncheckedIOException e) {
LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
}
@@ -467,7 +522,7 @@ public class ArbitraryDataStorageManager extends Thread {
long maxStoragePerName = this.storageCapacityPerName(threshold);
// Fetch all hosted transactions
List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository, null, null, true);
List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository, null, null);
for (ArbitraryTransactionData transactionData : hostedTransactions) {
String transactionName = transactionData.getName();
if (!Objects.equals(name, transactionName)) {

3
src/main/java/org/qortal/controller/tradebot/TradeBot.java
View File

@@ -16,6 +16,7 @@ import org.bitcoinj.core.ECKey;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.api.model.crosschain.TradeBotCreateRequest;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult;
import org.qortal.crosschain.*;
import org.qortal.data.at.ATData;
@@ -213,7 +214,7 @@ public class TradeBot implements Listener {
@Override
public void listen(Event event) {
if (!(event instanceof Controller.NewBlockEvent))
if (!(event instanceof Synchronizer.NewChainTipEvent))
return;
synchronized (this) {

39
src/main/java/org/qortal/crosschain/Bitcoiny.java
View File

@@ -58,9 +58,14 @@ public abstract class Bitcoiny implements ForeignBlockchain {
* i.e. keys with transactions but with no unspent outputs. */
protected final Set<ECKey> spentKeys = Collections.synchronizedSet(new HashSet<>());
/** How many bitcoinj wallet keys to generate in each batch. */
/** How many wallet keys to generate in each batch. */
private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3;
/** How many wallet keys to generate when using bitcoinj as the data provider.
* We must use a higher value here since we are unable to request multiple batches of keys.
* Without this, the bitcoinj state can be missing transactions, causing errors such as "insufficient balance". */
private static final int WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ = 50;
/** Byte offset into raw block headers to block timestamp. */
private static final int TIMESTAMP_OFFSET = 4 + 32 + 32;
@@ -99,8 +104,9 @@ public abstract class Bitcoiny implements ForeignBlockchain {
try {
ScriptType addressType = Address.fromString(this.params, address).getOutputScriptType();
return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH;
return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH || addressType == ScriptType.P2WPKH;
} catch (AddressFormatException e) {
LOGGER.error(String.format("Unrecognised address format: %s", address));
return false;
}
}
@@ -404,7 +410,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
Set<String> keySet = new HashSet<>();
// Set the number of consecutive empty batches required before giving up
final int numberOfAdditionalBatchesToSearch = 5;
final int numberOfAdditionalBatchesToSearch = 7;
int unusedCounter = 0;
int ki = 0;
@@ -470,6 +476,9 @@ public abstract class Bitcoiny implements ForeignBlockchain {
List<SimpleTransaction.Input> inputs = new ArrayList<>();
List<SimpleTransaction.Output> outputs = new ArrayList<>();
boolean anyOutputAddressInWallet = false;
boolean transactionInvolvesExternalWallet = false;
for (BitcoinyTransaction.Input input : t.inputs) {
try {
BitcoinyTransaction t2 = getTransaction(input.outputTxHash);
@@ -483,6 +492,9 @@ public abstract class Bitcoiny implements ForeignBlockchain {
total += inputAmount;
addressInWallet = true;
}
else {
transactionInvolvesExternalWallet = true;
}
inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet));
}
}
@@ -496,12 +508,16 @@ public abstract class Bitcoiny implements ForeignBlockchain {
for (String address : output.addresses) {
boolean addressInWallet = false;
if (keySet.contains(address)) {
if (total > 0L) {
if (total > 0L) { // Change returned from sent amount
amount -= (total - output.value);
} else {
} else { // Amount received
amount += output.value;
}
addressInWallet = true;
anyOutputAddressInWallet = true;
}
else {
transactionInvolvesExternalWallet = true;
}
outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet));
}
@@ -510,6 +526,17 @@ public abstract class Bitcoiny implements ForeignBlockchain {
}
}
long fee = totalInputAmount - totalOutputAmount;
if (!anyOutputAddressInWallet) {
// No outputs relate to this wallet - check if any inputs did (which is signified by a positive total)
if (total > 0) {
amount = total * -1;
}
}
else if (!transactionInvolvesExternalWallet) {
// All inputs and outputs relate to this wallet, so the balance should be unaffected
amount = 0;
}
return new SimpleTransaction(t.txHash, t.timestamp, amount, fee, inputs, outputs);
}
@@ -602,7 +629,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
this.keyChain = this.wallet.getActiveKeyChain();
// Set up wallet's key chain
this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT);
this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ);
this.keyChain.maybeLookAhead();
}

27
src/main/java/org/qortal/crosschain/ElectrumX.java
View File

@@ -401,13 +401,36 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
String scriptPubKey = (String) ((JSONObject) outputJson.get("scriptPubKey")).get("hex");
long value = BigDecimal.valueOf((Double) outputJson.get("value")).setScale(8).unscaledValue().longValue();
// address too, if present
// address too, if present in the "addresses" array
List<String> addresses = null;
Object addressesObj = ((JSONObject) outputJson.get("scriptPubKey")).get("addresses");
if (addressesObj instanceof JSONArray) {
addresses = new ArrayList<>();
for (Object addressObj : (JSONArray) addressesObj)
for (Object addressObj : (JSONArray) addressesObj) {
addresses.add((String) addressObj);
}
}
// some peers return a single "address" string
Object addressObj = ((JSONObject) outputJson.get("scriptPubKey")).get("address");
if (addressObj instanceof String) {
if (addresses == null) {
addresses = new ArrayList<>();
}
addresses.add((String) addressObj);
}
// For the purposes of Qortal we require all outputs to contain addresses
// Some servers omit this info, causing problems down the line with balance calculations
// Update: it turns out that they were just using a different key - "address" instead of "addresses"
// The code below can remain in place, just in case a peer returns a missing address in the future
if (addresses == null || addresses.isEmpty()) {
if (this.currentServer != null) {
this.uselessServers.add(this.currentServer);
this.closeServer(this.currentServer);
}
LOGGER.info("No output addresses returned for transaction {}", txHash);
throw new ForeignBlockchainException(String.format("No output addresses returned for transaction %s", txHash));
}
outputs.add(new BitcoinyTransaction.Output(scriptPubKey, value, addresses));

14
src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java
View File

@@ -9,12 +9,16 @@ public class ArbitraryRelayInfo {
private final String signature58;
private final Peer peer;
private final Long timestamp;
private final Long requestTime;
private final Integer requestHops;
public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp) {
public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp, Long requestTime, Integer requestHops) {
this.hash58 = hash58;
this.signature58 = signature58;
this.peer = peer;
this.timestamp = timestamp;
this.requestTime = requestTime;
this.requestHops = requestHops;
}
public boolean isValid() {
@@ -38,6 +42,14 @@ public class ArbitraryRelayInfo {
return timestamp;
}
public Long getRequestTime() {
return this.requestTime;
}
public Integer getRequestHops() {
return this.requestHops;
}
@Override
public String toString() {
return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp);

9
src/main/java/org/qortal/network/Handshake.java
View File

@@ -74,6 +74,12 @@ public enum Handshake {
peer.setPeersConnectionTimestamp(peersConnectionTimestamp);
peer.setPeersVersion(versionString, version);
// Ensure the peer is running at least the version specified in MIN_PEER_VERSION
if (peer.isAtLeastVersion(MIN_PEER_VERSION) == false) {
LOGGER.debug(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString));
return null;
}
if (Settings.getInstance().getAllowConnectionsWithOlderPeerVersions() == false) {
// Ensure the peer is running at least the minimum version allowed for connections
final String minPeerVersion = Settings.getInstance().getMinPeerVersion();
@@ -258,6 +264,9 @@ public enum Handshake {
private static final long PEER_VERSION_131 = 0x0100030001L;
/** Minimum peer version that we are allowed to communicate with */
private static final String MIN_PEER_VERSION = "3.1.0";
private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes
private static final int POW_DIFFICULTY_PRE_131 = 8; // leading zero bits
// Can always be made harder in the future...

39
src/main/java/org/qortal/network/Network.java
View File

@@ -1,5 +1,6 @@
package org.qortal.network;
import com.dosse.upnp.UPnP;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters;
@@ -7,7 +8,6 @@ import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters;
import org.qortal.block.BlockChain;
import org.qortal.controller.Controller;
import org.qortal.controller.arbitrary.ArbitraryDataFileListManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.crypto.Crypto;
import org.qortal.data.block.BlockData;
import org.qortal.data.network.PeerData;
@@ -183,6 +183,14 @@ public class Network {
}
}
// Attempt to set up UPnP. All errors are ignored.
if (Settings.getInstance().isUPnPEnabled()) {
UPnP.openPortTCP(Settings.getInstance().getListenPort());
}
else {
UPnP.closePortTCP(Settings.getInstance().getListenPort());
}
// Start up first networking thread
networkEPC.start();
}
@@ -243,12 +251,15 @@ public class Network {
public boolean requestDataFromPeer(String peerAddressString, byte[] signature) {
if (peerAddressString != null) {
PeerAddress peerAddress = PeerAddress.fromString(peerAddressString);
PeerData peerData = null;
// Reuse an existing PeerData instance if it's already in the known peers list
PeerData peerData = this.allKnownPeers.stream()
.filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress))
.findFirst()
.orElse(null);
synchronized (this.allKnownPeers) {
peerData = this.allKnownPeers.stream()
.filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress))
.findFirst()
.orElse(null);
}
if (peerData == null) {
// Not a known peer, so we need to create one
@@ -263,10 +274,13 @@ public class Network {
}
// Check if we're already connected to and handshaked with this peer
Peer connectedPeer = this.connectedPeers.stream()
.filter(p -> p.getPeerData().getAddress().equals(peerAddress))
.findFirst()
.orElse(null);
Peer connectedPeer = null;
synchronized (this.connectedPeers) {
connectedPeer = this.connectedPeers.stream()
.filter(p -> p.getPeerData().getAddress().equals(peerAddress))
.findFirst()
.orElse(null);
}
boolean isConnected = (connectedPeer != null);
boolean isHandshaked = this.getHandshakedPeers().stream()
@@ -1178,7 +1192,12 @@ public class Network {
public void onExternalIpUpdate(String ipAddress) {
LOGGER.info("External IP address updated to {}", ipAddress);
ArbitraryDataManager.getInstance().broadcastHostedSignatureList();
//ArbitraryDataManager.getInstance().broadcastHostedSignatureList();
}
public String getOurExternalIpAddress() {
// FUTURE: replace port if UPnP is active, as it will be more accurate
return this.ourExternalIpAddress;
}

102
src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java
View File

@@ -1,6 +1,8 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import org.qortal.data.network.PeerData;
import org.qortal.transform.TransformationException;
import org.qortal.transform.Transformer;
import org.qortal.utils.Serialization;
@@ -16,22 +18,38 @@ public class ArbitraryDataFileListMessage extends Message {
private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
private static final int HASH_LENGTH = Transformer.SHA256_LENGTH;
private static final int MAX_PEER_ADDRESS_LENGTH = PeerData.MAX_PEER_ADDRESS_SIZE;
private final byte[] signature;
private final List<byte[]> hashes;
private Long requestTime;
private Integer requestHops;
private String peerAddress;
private Boolean isRelayPossible;
public ArbitraryDataFileListMessage(byte[] signature, List<byte[]> hashes) {
public ArbitraryDataFileListMessage(byte[] signature, List<byte[]> hashes, Long requestTime,
Integer requestHops, String peerAddress, boolean isRelayPossible) {
super(MessageType.ARBITRARY_DATA_FILE_LIST);
this.signature = signature;
this.hashes = hashes;
this.requestTime = requestTime;
this.requestHops = requestHops;
this.peerAddress = peerAddress;
this.isRelayPossible = isRelayPossible;
}
public ArbitraryDataFileListMessage(int id, byte[] signature, List<byte[]> hashes) {
public ArbitraryDataFileListMessage(int id, byte[] signature, List<byte[]> hashes, Long requestTime,
Integer requestHops, String peerAddress, boolean isRelayPossible) {
super(id, MessageType.ARBITRARY_DATA_FILE_LIST);
this.signature = signature;
this.hashes = hashes;
this.requestTime = requestTime;
this.requestHops = requestHops;
this.peerAddress = peerAddress;
this.isRelayPossible = isRelayPossible;
}
public List<byte[]> getHashes() {
@@ -48,9 +66,6 @@ public class ArbitraryDataFileListMessage extends Message {
int count = bytes.getInt();
if (bytes.remaining() != count * HASH_LENGTH)
return null;
List<byte[]> hashes = new ArrayList<>();
for (int i = 0; i < count; ++i) {
@@ -59,7 +74,26 @@ public class ArbitraryDataFileListMessage extends Message {
hashes.add(hash);
}
return new ArbitraryDataFileListMessage(id, signature, hashes);
Long requestTime = null;
Integer requestHops = null;
String peerAddress = null;
boolean isRelayPossible = true; // Legacy versions only send this message when relaying is possible
// The remaining fields are optional
if (bytes.hasRemaining()) {
requestTime = bytes.getLong();
requestHops = bytes.getInt();
peerAddress = Serialization.deserializeSizedStringV2(bytes, MAX_PEER_ADDRESS_LENGTH);
isRelayPossible = bytes.getInt() > 0;
}
return new ArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, peerAddress, isRelayPossible);
}
@Override
@@ -75,6 +109,20 @@ public class ArbitraryDataFileListMessage extends Message {
bytes.write(hash);
}
if (this.requestTime == null) { // To maintain backwards support
return bytes.toByteArray();
}
// The remaining fields are optional
bytes.write(Longs.toByteArray(this.requestTime));
bytes.write(Ints.toByteArray(this.requestHops));
Serialization.serializeSizedStringV2(bytes, this.peerAddress);
bytes.write(Ints.toByteArray(this.isRelayPossible ? 1 : 0));
return bytes.toByteArray();
} catch (IOException e) {
return null;
@@ -82,9 +130,49 @@ public class ArbitraryDataFileListMessage extends Message {
}
public ArbitraryDataFileListMessage cloneWithNewId(int newId) {
ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes);
ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes,
this.requestTime, this.requestHops, this.peerAddress, this.isRelayPossible);
clone.setId(newId);
return clone;
}
public void removeOptionalStats() {
this.requestTime = null;
this.requestHops = null;
this.peerAddress = null;
this.isRelayPossible = null;
}
public Long getRequestTime() {
return this.requestTime;
}
public void setRequestTime(Long requestTime) {
this.requestTime = requestTime;
}
public Integer getRequestHops() {
return this.requestHops;
}
public void setRequestHops(Integer requestHops) {
this.requestHops = requestHops;
}
public String getPeerAddress() {
return this.peerAddress;
}
public void setPeerAddress(String peerAddress) {
this.peerAddress = peerAddress;
}
public Boolean isRelayPossible() {
return this.isRelayPossible;
}
public void setIsRelayPossible(Boolean isRelayPossible) {
this.isRelayPossible = isRelayPossible;
}
}

5
src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java
View File

@@ -1,6 +1,8 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.repository.DataException;
import org.qortal.transform.Transformer;
@@ -12,6 +14,8 @@ import java.nio.ByteBuffer;
public class ArbitraryDataFileMessage extends Message {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileMessage.class);
private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
private final byte[] signature;
@@ -52,6 +56,7 @@ public class ArbitraryDataFileMessage extends Message {
return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile);
}
catch (DataException e) {
LOGGER.info("Unable to process received file: {}", e.getMessage());
return null;
}
}

117
src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java Normal file
View File

@@ -0,0 +1,117 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.transform.Transformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* For requesting online accounts info from remote peer, given our list of online accounts.
*
* Different format to V1:
* V1 is: number of entries, then timestamp + pubkey for each entry
* V2 is: groups of: number of entries, timestamp, then pubkey for each entry
*
* Also V2 only builds online accounts message once!
*/
public class GetOnlineAccountsV2Message extends Message {
private List<OnlineAccountData> onlineAccounts;
private byte[] cachedData;
public GetOnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
this(-1, onlineAccounts);
}
private GetOnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
super(id, MessageType.GET_ONLINE_ACCOUNTS_V2);
this.onlineAccounts = onlineAccounts;
}
public List<OnlineAccountData> getOnlineAccounts() {
return this.onlineAccounts;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
int accountCount = bytes.getInt();
List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
while (accountCount > 0) {
long timestamp = bytes.getLong();
for (int i = 0; i < accountCount; ++i) {
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKey);
onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey));
}
if (bytes.hasRemaining()) {
accountCount = bytes.getInt();
} else {
// we've finished
accountCount = 0;
}
}
return new GetOnlineAccountsV2Message(id, onlineAccounts);
}
@Override
protected synchronized byte[] toData() {
if (this.cachedData != null)
return this.cachedData;
// Shortcut in case we have no online accounts
if (this.onlineAccounts.isEmpty()) {
this.cachedData = Ints.toByteArray(0);
return this.cachedData;
}
// How many of each timestamp
Map<Long, Integer> countByTimestamp = new HashMap<>();
for (int i = 0; i < this.onlineAccounts.size(); ++i) {
OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
Long timestamp = onlineAccountData.getTimestamp();
countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
}
// We should know exactly how many bytes to allocate now
int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+ this.onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH;
try {
ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
for (long timestamp : countByTimestamp.keySet()) {
bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
bytes.write(Longs.toByteArray(timestamp));
for (int i = 0; i < this.onlineAccounts.size(); ++i) {
OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
if (onlineAccountData.getTimestamp() == timestamp)
bytes.write(onlineAccountData.getPublicKey());
}
}
this.cachedData = bytes.toByteArray();
return this.cachedData;
} catch (IOException e) {
return null;
}
}
}

2
src/main/java/org/qortal/network/message/Message.java
View File

@@ -78,6 +78,8 @@ public abstract class Message {
ONLINE_ACCOUNTS(80),
GET_ONLINE_ACCOUNTS(81),
ONLINE_ACCOUNTS_V2(82),
GET_ONLINE_ACCOUNTS_V2(83),
ARBITRARY_DATA(90),
GET_ARBITRARY_DATA(91),

124
src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java Normal file
View File

@@ -0,0 +1,124 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.transform.Transformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* For sending online accounts info to remote peer.
*
* Different format to V1:
* V1 is: number of entries, then timestamp + sig + pubkey for each entry
* V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry
*
* Also V2 only builds online accounts message once!
*/
public class OnlineAccountsV2Message extends Message {
private List<OnlineAccountData> onlineAccounts;
private byte[] cachedData;
public OnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
this(-1, onlineAccounts);
}
private OnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
super(id, MessageType.ONLINE_ACCOUNTS_V2);
this.onlineAccounts = onlineAccounts;
}
public List<OnlineAccountData> getOnlineAccounts() {
return this.onlineAccounts;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
int accountCount = bytes.getInt();
List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
while (accountCount > 0) {
long timestamp = bytes.getLong();
for (int i = 0; i < accountCount; ++i) {
byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
bytes.get(signature);
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKey);
onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey));
}
if (bytes.hasRemaining()) {
accountCount = bytes.getInt();
} else {
// we've finished
accountCount = 0;
}
}
return new OnlineAccountsV2Message(id, onlineAccounts);
}
@Override
protected synchronized byte[] toData() {
if (this.cachedData != null)
return this.cachedData;
// Shortcut in case we have no online accounts
if (this.onlineAccounts.isEmpty()) {
this.cachedData = Ints.toByteArray(0);
return this.cachedData;
}
// How many of each timestamp
Map<Long, Integer> countByTimestamp = new HashMap<>();
for (int i = 0; i < this.onlineAccounts.size(); ++i) {
OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
Long timestamp = onlineAccountData.getTimestamp();
countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
}
// We should know exactly how many bytes to allocate now
int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+ this.onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH);
try {
ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
for (long timestamp : countByTimestamp.keySet()) {
bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
bytes.write(Longs.toByteArray(timestamp));
for (int i = 0; i < this.onlineAccounts.size(); ++i) {
OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
if (onlineAccountData.getTimestamp() == timestamp) {
bytes.write(onlineAccountData.getSignature());
bytes.write(onlineAccountData.getPublicKey());
}
}
}
this.cachedData = bytes.toByteArray();
return this.cachedData;
} catch (IOException e) {
return null;
}
}
}

8
src/main/java/org/qortal/settings/Settings.java
View File

@@ -181,6 +181,8 @@ public class Settings {
private boolean isTestNet = false;
/** Port number for inbound peer-to-peer connections. */
private Integer listenPort;
/** Whether to attempt to open the listen port via UPnP */
private boolean uPnPEnabled = true;
/** Minimum number of peers to allow block minting / synchronization. */
private int minBlockchainPeers = 5;
/** Target number of outbound connections to peers we should make. */
@@ -195,7 +197,7 @@ public class Settings {
private int maxRetries = 2;
/** Minimum peer version number required in order to sync with them */
private String minPeerVersion = "3.0.1";
private String minPeerVersion = "3.1.0";
/** Whether to allow connections with peers below minPeerVersion
* If true, we won't sync with them but they can still sync with us, and will show in the peers list
* If false, sync will be blocked both ways, and they will not appear in the peers list */
@@ -629,6 +631,10 @@ public class Settings {
return this.bindAddress;
}
public boolean isUPnPEnabled() {
return this.uPnPEnabled;
}
public int getMinBlockchainPeers() {
return this.minBlockchainPeers;
}

4
src/main/java/org/qortal/transaction/TransferPrivsTransaction.java
View File

@@ -58,7 +58,9 @@ public class TransferPrivsTransaction extends Transaction {
return ValidationResult.INVALID_ADDRESS;
// Check recipient is new account
if (this.repository.getAccountRepository().accountExists(this.transferPrivsTransactionData.getRecipient()))
AccountData recipientAccountData = this.repository.getAccountRepository().getAccount(this.transferPrivsTransactionData.getRecipient());
// Non-existent account data is OK, but if account data exists then reference needs to be null
if (recipientAccountData != null && recipientAccountData.getReference() != null)
return ValidationResult.ACCOUNT_ALREADY_EXISTS;
// Check sender has funds for fee

114
src/test/java/org/qortal/test/network/OnlineAccountsTests.java Normal file
View File

@@ -0,0 +1,114 @@
package org.qortal.test.network;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.junit.Test;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.network.message.*;
import org.qortal.transform.Transformer;
import java.nio.ByteBuffer;
import java.security.Security;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
public class OnlineAccountsTests {
private static final Random RANDOM = new Random();
static {
// This must go before any calls to LogManager/Logger
System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
Security.insertProviderAt(new BouncyCastleProvider(), 0);
Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
}
@Test
public void testGetOnlineAccountsV2() throws Message.MessageException {
List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(false);
Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut);
byte[] messageBytes = messageOut.toBytes();
ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut);
byte[] oldMessageBytes = oldMessageOut.toBytes();
long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
onlineAccountsOut.size(),
numTimestamps,
numTimestamps != 1 ? "s" : "",
oldMessageBytes.length,
messageBytes.length));
}
@Test
public void testOnlineAccountsV2() throws Message.MessageException {
List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(true);
Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut);
byte[] messageBytes = messageOut.toBytes();
ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut);
byte[] oldMessageBytes = oldMessageOut.toBytes();
long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
onlineAccountsOut.size(),
numTimestamps,
numTimestamps != 1 ? "s" : "",
oldMessageBytes.length,
messageBytes.length));
}
private List<OnlineAccountData> generateOnlineAccounts(boolean withSignatures) {
List<OnlineAccountData> onlineAccounts = new ArrayList<>();
int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2
for (int t = 0; t < numTimestamps; ++t) {
int numAccounts = RANDOM.nextInt(3000);
for (int a = 0; a < numAccounts; ++a) {
byte[] sig = null;
if (withSignatures) {
sig = new byte[Transformer.SIGNATURE_LENGTH];
RANDOM.nextBytes(sig);
}
byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH];
RANDOM.nextBytes(pubkey);
onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey));
}
}
return onlineAccounts;
}
}