Compare commits

...

10 Commits

Author SHA1 Message Date
catbref
0594bdf1c7 Bump to v1.0.5 in pom.xml. Filter out build/test artifacts from shaded JAR. 2020-03-24 11:17:43 +00:00
catbref
72c299a331 Add SysTray notification for DB backup. More translations.
Added a setting "showBackupNotification", which is false by default,
that shows a tray notification when a repository backup occurs.

Above notification, and the auto-update notification, now refer to
the SysTray i18n translation lookup resources.
2020-03-24 09:26:40 +00:00
catbref
0b42a7ad63 Modify minOutboundPeers, maxPeers and maxNetworkThreadPoolSize default settings
Typical users don't need quite so many connections, so minOutboundPeers and
maxPeers reduced accordingly.

maxNetworkThreadPoolSize increased from 10 to 20.
2020-03-24 09:24:22 +00:00
catbref
51e59f6ab7 Change HSQLDB repository log fsync() interval from 500ms to 5s 2020-03-24 09:23:17 +00:00
catbref
38394de661 Reduce peer response timeout from 5s to 2s
5s is way too long, and even 2s might still be considered excessive.
However, reducing the timeout might also reduce the number of
network engine "spawn failures" due to too many threads tied up
waiting for ping responses from overloaded peers.

Does not affect peer handshaking: that has a separate timeout.
2020-03-24 09:20:50 +00:00
catbref
22f9755f4f Performance optimizations. Accounts/NTP/System.currentTimeMillis, etc.
Added Ed25519 private key to public key function accessible from SQL.
Added Ed25519 public key to Qortal address function accessible from SQL.

Used above functions to store minting account public key in SQL to
reduce the number of unnecessarily repeated Ed25519 conversions.

Used above functions to store reward-share minting's accounts address
to reduce the number of unneccessarily repeated PK-to-address conversions.

Reduced the usage of PublicKeyAccount to simply Account where possible,
to reduce the number of Ed25519 conversions.

Account.canMint(), Account.canRewardShare() and Account.getEffectiveMintingLevel()
now only perform 1 repository fetch instead of potentially 2 or more.

Cleaned up NTP main thread to reduce CPU load.
A fixed offset can be applied to NTP.getTime() responses, for both
scenarios when NTP is running or not. Useful for testing or simulating
distant remote peers.

Controller.onNetworkMessage() and Network.onMessage() have both had their
complexity simplified by extracting per-case code to separate methods.

Network's EPC engine's thread pool size no longer hard-coded, but comes
from Settings.maxNetworkThreadPoolSize, which is still 10 by default,
but can be increased for high-availability nodes.

Network's EPC task-producing code streamlined to reduce CPU load.

Generally reduced calls to System.currentTimeMillis(), especially
where the value would only be used in verbose logging situations,
and especially in high-call-volume methods, like within repository.
2020-03-23 11:14:05 +00:00
catbref
4cb2e113cb Log (and discard) duplicate outbound connections to the same peer 2020-03-23 11:07:08 +00:00
catbref
e0f024ef5c Performance improvements in networking ExecuteProduceConsume engine
Keep track of when EPC engine can't spawn a new thread as this
might indicate thread-pool exhaustion and cause some network
messages to be lost.

If logging level is NOT 'trace' (or 'all') then don't call
System.currentTimeMillis() as we'll never use the value.

Similarly, don't set thread names if not logging at 'trace' either.

Update EPC tests, particularly unified per-second/end-of-test stats
reporting.
2020-03-23 11:00:19 +00:00
catbref
f95cb99cdc Add support for logging PROOF network message calculation time 2020-03-23 10:57:23 +00:00
catbref
1f0170bb4b Increase timeout for transaction submission via API POST /transactions/process from 500ms to 30s 2020-03-23 10:54:35 +00:00
30 changed files with 1174 additions and 1094 deletions

View File

@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.qortal</groupId>
<artifactId>qortal</artifactId>
<version>1.0.4</version>
<version>1.0.5</version>
<packaging>jar</packaging>
<properties>
<bitcoin.version>0.15.4</bitcoin.version>
@@ -257,6 +257,8 @@
<!-- Don't include original swagger-UI as we're including our own
modified version -->
<exclude>org.webjars:swagger-ui</exclude>
<!-- Don't include JUnit as it's for testing only! -->
<exclude>junit:junit</exclude>
</excludes>
</artifactSet>
<filters>
@@ -379,12 +381,14 @@
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>3.0.0</version>
<scope>provided</scope><!-- needed for build, not for runtime -->
</dependency>
<!-- https://mvnrepository.com/artifact/com.github.bohnman/package-info-maven-plugin -->
<dependency>
<groupId>com.github.bohnman</groupId>
<artifactId>package-info-maven-plugin</artifactId>
<version>${package-info-maven-plugin.version}</version>
<scope>provided</scope><!-- needed for build, not for runtime -->
</dependency>
<!-- HSQLDB for repository -->
<dependency>

View File

@@ -204,11 +204,15 @@ public class Account {
* @throws DataException
*/
public boolean canMint() throws DataException {
Integer level = this.getLevel();
AccountData accountData = this.repository.getAccountRepository().getAccount(this.address);
if (accountData == null)
return false;
Integer level = accountData.getLevel();
if (level != null && level >= BlockChain.getInstance().getMinAccountLevelToMint())
return true;
if (this.isFounder())
if (Account.isFounder(accountData.getFlags()))
return true;
return false;
@@ -226,11 +230,15 @@ public class Account {
* @throws DataException
*/
public boolean canRewardShare() throws DataException {
Integer level = this.getLevel();
AccountData accountData = this.repository.getAccountRepository().getAccount(this.address);
if (accountData == null)
return false;
Integer level = accountData.getLevel();
if (level != null && level >= BlockChain.getInstance().getMinAccountLevelToRewardShare())
return true;
if (this.isFounder())
if (Account.isFounder(accountData.getFlags()))
return true;
return false;
@@ -264,10 +272,14 @@ public class Account {
* @throws DataException
*/
public int getEffectiveMintingLevel() throws DataException {
if (this.isFounder())
AccountData accountData = this.repository.getAccountRepository().getAccount(this.address);
if (accountData == null)
return 0;
if (Account.isFounder(accountData.getFlags()))
return BlockChain.getInstance().getFounderEffectiveMintingLevel();
Integer level = this.getLevel();
Integer level = accountData.getLevel();
if (level == null)
return 0;
@@ -290,7 +302,7 @@ public class Account {
if (rewardShareData == null)
return 0;
PublicKeyAccount rewardShareMinter = new PublicKeyAccount(repository, rewardShareData.getMinterPublicKey());
Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
return rewardShareMinter.getEffectiveMintingLevel();
}

View File

@@ -32,10 +32,13 @@ public class BlockMinterSummary {
}
/** Constructs BlockMinterSummary in reward-share context. */
public BlockMinterSummary(byte[] rewardSharePublicKey, int blockCount, byte[] mintingAccountPublicKey, String recipientAccount) {
this(mintingAccountPublicKey, blockCount);
public BlockMinterSummary(byte[] rewardSharePublicKey, int blockCount, byte[] mintingAccountPublicKey, String minterAccount, String recipientAccount) {
this.rewardSharePublicKey = rewardSharePublicKey;
this.blockCount = blockCount;
this.mintingAccountPublicKey = mintingAccountPublicKey;
this.mintingAccount = minterAccount;
this.recipientAccount = recipientAccount;
}

View File

@@ -36,8 +36,8 @@ import javax.ws.rs.core.MediaType;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.RollingFileAppender;
import org.qortal.account.Account;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.account.PublicKeyAccount;
import org.qortal.api.ApiError;
import org.qortal.api.ApiErrors;
import org.qortal.api.ApiException;
@@ -240,7 +240,7 @@ public class AdminResource {
// ignore
}
return new MintingAccountData(mintingAccountData.getPrivateKey(), rewardShareData);
return new MintingAccountData(mintingAccountData, rewardShareData);
}).collect(Collectors.toList());
return mintingAccounts;
@@ -284,11 +284,11 @@ public class AdminResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
// Qortal: check reward-share's minting account is still allowed to mint
PublicKeyAccount rewardShareMintingAccount = new PublicKeyAccount(repository, rewardShareData.getMinterPublicKey());
Account rewardShareMintingAccount = new Account(repository, rewardShareData.getMinter());
if (!rewardShareMintingAccount.canMint())
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.CANNOT_MINT);
MintingAccountData mintingAccountData = new MintingAccountData(seed);
MintingAccountData mintingAccountData = new MintingAccountData(mintingAccount.getPrivateKey(), mintingAccount.getPublicKey());
repository.getAccountRepository().save(mintingAccountData);
repository.saveChanges();

View File

@@ -470,7 +470,7 @@ public class TransactionsResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_SIGNATURE);
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (!blockchainLock.tryLock(500, TimeUnit.MILLISECONDS))
if (!blockchainLock.tryLock(30, TimeUnit.SECONDS))
throw createTransactionInvalidException(request, ValidationResult.NO_BLOCKCHAIN_LOCK);
try {

View File

@@ -145,7 +145,7 @@ public class Block {
this.repository = repository;
this.rewardShareData = repository.getAccountRepository().getRewardShareByIndex(accountIndex);
this.mintingAccount = new PublicKeyAccount(repository, this.rewardShareData.getMinterPublicKey());
this.mintingAccount = new Account(repository, this.rewardShareData.getMinter());
this.mintingAccountData = repository.getAccountRepository().getAccount(this.mintingAccount.getAddress());
this.isMinterFounder = Account.isFounder(mintingAccountData.getFlags());

View File

@@ -12,7 +12,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.Account;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.account.PublicKeyAccount;
import org.qortal.block.Block.ValidationResult;
import org.qortal.controller.Controller;
import org.qortal.data.account.MintingAccountData;
@@ -123,7 +122,7 @@ public class BlockMinter extends Thread {
continue;
}
PublicKeyAccount mintingAccount = new PublicKeyAccount(repository, rewardShareData.getMinterPublicKey());
Account mintingAccount = new Account(repository, rewardShareData.getMinter());
if (!mintingAccount.canMint()) {
// Minting-account component of reward-share can no longer mint - disregard
madi.remove();
@@ -158,12 +157,12 @@ public class BlockMinter extends Thread {
newBlocks.clear();
}
// Discard accounts we have already built blocks with
mintingAccountsData.removeIf(mintingAccountData -> newBlocks.stream().anyMatch(newBlock -> Arrays.equals(newBlock.getBlockData().getMinterPublicKey(), mintingAccountData.getPublicKey())));
// Do we need to build any potential new blocks?
List<PrivateKeyAccount> mintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
// Discard accounts we have blocks for
mintingAccounts.removeIf(account -> newBlocks.stream().anyMatch(newBlock -> newBlock.getMinter().getAddress().equals(account.getAddress())));
for (PrivateKeyAccount mintingAccount : mintingAccounts) {
// First block does the AT heavy-lifting
if (newBlocks.isEmpty()) {
@@ -257,11 +256,10 @@ public class BlockMinter extends Thread {
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey());
if (rewardShareData != null) {
PublicKeyAccount mintingAccount = new PublicKeyAccount(repository, rewardShareData.getMinterPublicKey());
LOGGER.info(String.format("Minted block %d, sig %.8s by %s on behalf of %s",
newBlock.getBlockData().getHeight(),
Base58.encode(newBlock.getBlockData().getSignature()),
mintingAccount.getAddress(),
rewardShareData.getMinter(),
rewardShareData.getRecipient()));
} else {
LOGGER.info(String.format("Minted block %d, sig %.8s by %s",

View File

@@ -21,6 +21,7 @@ import org.qortal.ApplyUpdate;
import org.qortal.api.ApiRequest;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.globalization.Translator;
import org.qortal.gui.SysTray;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
@@ -245,7 +246,9 @@ public class AutoUpdate extends Thread {
LOGGER.info(String.format("Applying update with: %s", String.join(" ", javaCmd)));
SysTray.getInstance().showMessage("Auto Update", "Applying automatic update and restarting...", MessageType.INFO);
SysTray.getInstance().showMessage(Translator.INSTANCE.translate("SysTray", "AUTO_UPDATE"),
Translator.INSTANCE.translate("SysTray", "APPLYING_UPDATE_AND_RESTARTING"),
MessageType.INFO);
new ProcessBuilder(javaCmd).start();

File diff suppressed because it is too large Load Diff

View File

@@ -2,10 +2,8 @@ package org.qortal.data.account;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlTransient;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.crypto.Crypto;
import io.swagger.v3.oas.annotations.media.Schema;
@@ -16,14 +14,17 @@ import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
public class MintingAccountData {
// Properties
// Never actually displayed by API
@Schema(hidden = true)
@XmlTransient
protected byte[] privateKey;
// Not always present - used by API if not null
@XmlTransient
@Schema(hidden = true)
// Read-only by API, we never ask for it as input
@Schema(accessMode = AccessMode.READ_ONLY)
protected byte[] publicKey;
// Not always present - used by API if not null
protected String mintingAccount;
protected String recipientAccount;
protected String address;
@@ -34,17 +35,17 @@ public class MintingAccountData {
protected MintingAccountData() {
}
public MintingAccountData(byte[] privateKey) {
public MintingAccountData(byte[] privateKey, byte[] publicKey) {
this.privateKey = privateKey;
this.publicKey = PrivateKeyAccount.toPublicKey(privateKey);
this.publicKey = publicKey;
}
public MintingAccountData(byte[] privateKey, RewardShareData rewardShareData) {
this(privateKey);
public MintingAccountData(MintingAccountData srcMintingAccountData, RewardShareData rewardShareData) {
this(srcMintingAccountData.privateKey, srcMintingAccountData.publicKey);
if (rewardShareData != null) {
this.recipientAccount = rewardShareData.getRecipient();
this.mintingAccount = Crypto.toAddress(rewardShareData.getMinterPublicKey());
this.mintingAccount = rewardShareData.getMinter();
} else {
this.address = Crypto.toAddress(this.publicKey);
}
@@ -56,8 +57,6 @@ public class MintingAccountData {
return this.privateKey;
}
@XmlElement(name = "publicKey")
@Schema(accessMode = AccessMode.READ_ONLY)
public byte[] getPublicKey() {
return this.publicKey;
}

View File

@@ -5,8 +5,9 @@ import java.math.BigDecimal;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlTransient;
import org.qortal.crypto.Crypto;
import io.swagger.v3.oas.annotations.media.Schema;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
@@ -14,6 +15,12 @@ public class RewardShareData {
// Properties
private byte[] minterPublicKey;
// "minter" is called "mintingAccount" instead
@XmlTransient
@Schema(hidden = true)
private String minter;
private String recipient;
private byte[] rewardSharePublicKey;
private BigDecimal sharePercent;
@@ -25,8 +32,9 @@ public class RewardShareData {
}
// Used when fetching from repository
public RewardShareData(byte[] minterPublicKey, String recipient, byte[] rewardSharePublicKey, BigDecimal sharePercent) {
public RewardShareData(byte[] minterPublicKey, String minter, String recipient, byte[] rewardSharePublicKey, BigDecimal sharePercent) {
this.minterPublicKey = minterPublicKey;
this.minter = minter;
this.recipient = recipient;
this.rewardSharePublicKey = rewardSharePublicKey;
this.sharePercent = sharePercent;
@@ -38,6 +46,10 @@ public class RewardShareData {
return this.minterPublicKey;
}
public String getMinter() {
return this.minter;
}
public String getRecipient() {
return this.recipient;
}
@@ -52,7 +64,7 @@ public class RewardShareData {
@XmlElement(name = "mintingAccount")
public String getMintingAccount() {
return Crypto.toAddress(this.minterPublicKey);
return this.minter;
}
}

View File

@@ -62,11 +62,10 @@ public enum Handshake {
// Is this ID already connected inbound or outbound?
Peer otherInboundPeer = Network.getInstance().getInboundPeerWithId(peerId);
Peer otherOutboundPeer = Network.getInstance().getOutboundHandshakedPeerWithId(peerId);
// Extra checks on inbound peers with known IDs, to prevent ID stealing
if (!peer.isOutbound() && otherInboundPeer != null) {
Peer otherOutboundPeer = Network.getInstance().getOutboundHandshakedPeerWithId(peerId);
if (otherOutboundPeer == null) {
// We already have an inbound peer with this ID, but no outgoing peer with which to request verification
LOGGER.trace(String.format("Discarding inbound peer %s with existing ID", peer));
@@ -86,6 +85,11 @@ public enum Handshake {
// Generate verification codes for later
peer.generateVerificationCodes();
}
} else if (peer.isOutbound() && otherOutboundPeer != null) {
// We already have an outbound connection to this peer?
LOGGER.info(String.format("We already have another outbound connection to peer %s - discarding", peer));
// Handshake failure - caller will deal with disconnect
return null;
} else {
// Set peer's ID
peer.setPeerId(peerId);
@@ -231,7 +235,7 @@ public enum Handshake {
private static void sendProof(Peer peer) {
if (peer.isOutbound()) {
// For outbound connections we need to generate real proof
new Proof(peer).start();
new Proof(peer).start(); // Calculate & send in a new thread to free up networking processing
} else {
// For incoming connections we only need to send a fake proof message as confirmation
Message proofMessage = new ProofMessage(peer.getConnectionTimestamp(), 0, 0);

View File

@@ -95,6 +95,8 @@ public class Network {
"node10.qortal.org"
};
private static final long NETWORK_EPC_KEEPALIVE = 10L; // seconds
public static final int MAX_SIGNATURES_PER_REPLY = 500;
public static final int MAX_BLOCK_SUMMARIES_PER_REPLY = 500;
public static final int PEER_ID_LENGTH = 128;
@@ -142,9 +144,10 @@ public class Network {
mergePeersLock = new ReentrantLock();
// We'll use a cached thread pool, max 10 threads, but with more aggressive 10 second timeout.
ExecutorService networkExecutor = new ThreadPoolExecutor(1, 10,
10L, TimeUnit.SECONDS,
// We'll use a cached thread pool but with more aggressive timeout.
ExecutorService networkExecutor = new ThreadPoolExecutor(1,
Settings.getInstance().getMaxNetworkThreadPoolSize(),
NETWORK_EPC_KEEPALIVE, TimeUnit.SECONDS,
new SynchronousQueue<Runnable>());
networkEPC = new NetworkProcessor(networkExecutor);
}
@@ -302,15 +305,17 @@ public class Network {
if (task != null)
return task;
task = maybeProducePeerPingTask();
final Long now = NTP.getTime();
task = maybeProducePeerPingTask(now);
if (task != null)
return task;
task = maybeProduceConnectPeerTask();
task = maybeProduceConnectPeerTask(now);
if (task != null)
return task;
task = maybeProduceBroadcastTask();
task = maybeProduceBroadcastTask(now);
if (task != null)
return task;
@@ -323,6 +328,65 @@ public class Network {
return null;
}
private Task maybeProducePeerMessageTask() {
for (Peer peer : getConnectedPeers()) {
Task peerTask = peer.getMessageTask();
if (peerTask != null)
return peerTask;
}
return null;
}
private Task maybeProducePeerPingTask(Long now) {
// Ask connected peers whether they need a ping
for (Peer peer : getConnectedPeers()) {
Task peerTask = peer.getPingTask(now);
if (peerTask != null)
return peerTask;
}
return null;
}
class PeerConnectTask implements ExecuteProduceConsume.Task {
private final Peer peer;
public PeerConnectTask(Peer peer) {
this.peer = peer;
}
@Override
public void perform() throws InterruptedException {
connectPeer(peer);
}
}
private Task maybeProduceConnectPeerTask(Long now) throws InterruptedException {
if (now == null || now < nextConnectTaskTimestamp)
return null;
if (getOutboundHandshakedPeers().size() >= minOutboundPeers)
return null;
nextConnectTaskTimestamp = now + 1000L;
Peer targetPeer = getConnectablePeer(now);
if (targetPeer == null)
return null;
// Create connection task
return new PeerConnectTask(targetPeer);
}
private Task maybeProduceBroadcastTask(Long now) {
if (now == null || now < nextBroadcastTimestamp)
return null;
nextBroadcastTimestamp = now + BROADCAST_INTERVAL;
return () -> Controller.getInstance().doNetworkBroadcast();
}
class ChannelTask implements ExecuteProduceConsume.Task {
private final SelectionKey selectionKey;
@@ -405,67 +469,6 @@ public class Network {
return new ChannelTask(nextSelectionKey);
}
private Task maybeProducePeerMessageTask() {
for (Peer peer : getConnectedPeers()) {
Task peerTask = peer.getMessageTask();
if (peerTask != null)
return peerTask;
}
return null;
}
private Task maybeProducePeerPingTask() {
// Ask connected peers whether they need a ping
for (Peer peer : getConnectedPeers()) {
Task peerTask = peer.getPingTask();
if (peerTask != null)
return peerTask;
}
return null;
}
class PeerConnectTask implements ExecuteProduceConsume.Task {
private final Peer peer;
public PeerConnectTask(Peer peer) {
this.peer = peer;
}
@Override
public void perform() throws InterruptedException {
connectPeer(peer);
}
}
private Task maybeProduceConnectPeerTask() throws InterruptedException {
if (getOutboundHandshakedPeers().size() >= minOutboundPeers)
return null;
final Long now = NTP.getTime();
if (now == null || now < nextConnectTaskTimestamp)
return null;
nextConnectTaskTimestamp = now + 1000L;
Peer targetPeer = getConnectablePeer();
if (targetPeer == null)
return null;
// Create connection task
return new PeerConnectTask(targetPeer);
}
private Task maybeProduceBroadcastTask() {
final Long now = NTP.getTime();
if (now == null || now < nextBroadcastTimestamp)
return null;
nextBroadcastTimestamp = now + BROADCAST_INTERVAL;
return () -> Controller.getInstance().doNetworkBroadcast();
}
}
private void acceptConnection(ServerSocketChannel serverSocketChannel) throws InterruptedException {
@@ -588,9 +591,27 @@ public class Network {
}
}
private Peer getConnectablePeer() throws InterruptedException {
final long now = NTP.getTime();
private final Predicate<PeerData> isSelfPeer = peerData -> {
PeerAddress peerAddress = peerData.getAddress();
return this.selfPeers.stream().anyMatch(selfPeer -> selfPeer.equals(peerAddress));
};
private final Predicate<PeerData> isConnectedPeer = peerData -> {
PeerAddress peerAddress = peerData.getAddress();
return this.connectedPeers.stream().anyMatch(peer -> peer.getPeerData().getAddress().equals(peerAddress));
};
private final Predicate<PeerData> isResolvedAsConnectedPeer = peerData -> {
try {
InetSocketAddress resolvedSocketAddress = peerData.getAddress().toSocketAddress();
return this.connectedPeers.stream().anyMatch(peer -> peer.getResolvedAddress().equals(resolvedSocketAddress));
} catch (UnknownHostException e) {
// Can't resolve - no point even trying to connect
return true;
}
};
private Peer getConnectablePeer(final Long now) throws InterruptedException {
// We can't block here so use tryRepository(). We don't NEED to connect a new peer.
try (final Repository repository = RepositoryManager.tryRepository()) {
if (repository == null)
@@ -606,36 +627,17 @@ public class Network {
peerData.getLastAttempted() > lastAttemptedThreshold);
// Don't consider peers that we know loop back to ourself
Predicate<PeerData> isSelfPeer = peerData -> {
PeerAddress peerAddress = peerData.getAddress();
return this.selfPeers.stream().anyMatch(selfPeer -> selfPeer.equals(peerAddress));
};
synchronized (this.selfPeers) {
peers.removeIf(isSelfPeer);
}
// Don't consider already connected peers (simple address match)
Predicate<PeerData> isConnectedPeer = peerData -> {
PeerAddress peerAddress = peerData.getAddress();
return this.connectedPeers.stream().anyMatch(peer -> peer.getPeerData().getAddress().equals(peerAddress));
};
synchronized (this.connectedPeers) {
peers.removeIf(isConnectedPeer);
}
// Don't consider already connected peers (resolved address match)
Predicate<PeerData> isResolvedAsConnectedPeer = peerData -> {
try {
InetSocketAddress resolvedSocketAddress = peerData.getAddress().toSocketAddress();
return this.connectedPeers.stream().anyMatch(peer -> peer.getResolvedAddress().equals(resolvedSocketAddress));
} catch (UnknownHostException e) {
// Can't resolve - no point even trying to connect
return true;
}
};
// XXX This might be too slow if we end up waiting a long time for hostnames to resolve via DNS
synchronized (this.connectedPeers) {
peers.removeIf(isResolvedAsConnectedPeer);
}
@@ -735,126 +737,43 @@ public class Network {
Handshake handshakeStatus = peer.getHandshakeStatus();
if (handshakeStatus != Handshake.COMPLETED) {
try {
// Still handshaking
LOGGER.trace(() -> String.format("Handshake status %s, message %s from peer %s", handshakeStatus.name(), (message != null ? message.getType().name() : "null"), peer));
// v1 nodes are keen on sending PINGs early. Send to back of queue so we'll process right after handshake
if (message != null && message.getType() == MessageType.PING) {
peer.queueMessage(message);
return;
}
// Check message type is as expected
if (handshakeStatus.expectedMessageType != null && message.getType() != handshakeStatus.expectedMessageType) {
LOGGER.debug(String.format("Unexpected %s message from %s, expected %s", message.getType().name(), peer, handshakeStatus.expectedMessageType));
peer.disconnect("unexpected message");
return;
}
Handshake newHandshakeStatus = handshakeStatus.onMessage(peer, message);
if (newHandshakeStatus == null) {
// Handshake failure
LOGGER.debug(String.format("Handshake failure with peer %s message %s", peer, message.getType().name()));
peer.disconnect("handshake failure");
return;
}
if (peer.isOutbound())
// If we made outbound connection then we need to act first
newHandshakeStatus.action(peer);
else
// We have inbound connection so we need to respond in kind with what we just received
handshakeStatus.action(peer);
peer.setHandshakeStatus(newHandshakeStatus);
if (newHandshakeStatus == Handshake.COMPLETED)
this.onHandshakeCompleted(peer);
return;
} finally {
peer.resetHandshakeMessagePending();
}
onHandshakingMessage(peer, message, handshakeStatus);
return;
}
// Should be non-handshaking messages from now on
// Ordered by message type value
switch (message.getType()) {
case PEER_VERIFY:
// Remote peer wants extra verification
possibleVerificationResponse(peer);
case GET_PEERS:
onGetPeersMessage(peer, message);
break;
case VERIFICATION_CODES:
VerificationCodesMessage verificationCodesMessage = (VerificationCodesMessage) message;
case PEERS:
onPeersMessage(peer, message);
break;
// Remote peer is sending the code it wants to receive back via our outbound connection to it
Peer ourUnverifiedPeer = Network.getInstance().getInboundPeerWithId(Network.getInstance().getOurPeerId());
ourUnverifiedPeer.setVerificationCodes(verificationCodesMessage.getVerificationCodeSent(), verificationCodesMessage.getVerificationCodeExpected());
possibleVerificationResponse(ourUnverifiedPeer);
case PING:
onPingMessage(peer, message);
break;
case VERSION:
case PEER_ID:
case PROOF:
LOGGER.debug(String.format("Unexpected handshaking message %s from peer %s", message.getType().name(), peer));
LOGGER.debug(() -> String.format("Unexpected handshaking message %s from peer %s", message.getType().name(), peer));
peer.disconnect("unexpected handshaking message");
return;
case PING:
PingMessage pingMessage = (PingMessage) message;
// Generate 'pong' using same ID
PingMessage pongMessage = new PingMessage();
pongMessage.setId(pingMessage.getId());
if (!peer.sendMessage(pongMessage))
peer.disconnect("failed to send ping reply");
break;
case PEERS:
PeersMessage peersMessage = (PeersMessage) message;
List<PeerAddress> peerAddresses = new ArrayList<>();
// v1 PEERS message doesn't support port numbers so we have to add default port
for (InetAddress peerAddress : peersMessage.getPeerAddresses())
// This is always IPv4 so we don't have to worry about bracketing IPv6.
peerAddresses.add(PeerAddress.fromString(peerAddress.getHostAddress()));
// Also add peer's details
peerAddresses.add(PeerAddress.fromString(peer.getPeerData().getAddress().getHost()));
mergePeers(peer.toString(), peerAddresses);
break;
case PEERS_V2:
PeersV2Message peersV2Message = (PeersV2Message) message;
List<PeerAddress> peerV2Addresses = peersV2Message.getPeerAddresses();
// First entry contains remote peer's listen port but empty address.
int peerPort = peerV2Addresses.get(0).getPort();
peerV2Addresses.remove(0);
// If inbound peer, use listen port and socket address to recreate first entry
if (!peer.isOutbound()) {
PeerAddress sendingPeerAddress = PeerAddress.fromString(peer.getPeerData().getAddress().getHost() + ":" + peerPort);
LOGGER.trace(() -> String.format("PEERS_V2 sending peer's listen address: %s", sendingPeerAddress.toString()));
peerV2Addresses.add(0, sendingPeerAddress);
}
mergePeers(peer.toString(), peerV2Addresses);
onPeersV2Message(peer, message);
break;
case GET_PEERS:
// Send our known peers
if (!peer.sendMessage(buildPeersMessage(peer)))
peer.disconnect("failed to send peers list");
case PEER_VERIFY:
onPeerVerifyMessage(peer, message);
break;
case VERIFICATION_CODES:
onVerificationCodesMessage(peer, message);
break;
default:
@@ -864,6 +783,116 @@ public class Network {
}
}
private void onHandshakingMessage(Peer peer, Message message, Handshake handshakeStatus) {
try {
// Still handshaking
LOGGER.trace(() -> String.format("Handshake status %s, message %s from peer %s", handshakeStatus.name(), (message != null ? message.getType().name() : "null"), peer));
// v1 nodes are keen on sending PINGs early. Send to back of queue so we'll process right after handshake
if (message != null && message.getType() == MessageType.PING) {
peer.queueMessage(message);
return;
}
// Check message type is as expected
if (handshakeStatus.expectedMessageType != null && message.getType() != handshakeStatus.expectedMessageType) {
LOGGER.debug(() -> String.format("Unexpected %s message from %s, expected %s", message.getType().name(), peer, handshakeStatus.expectedMessageType));
peer.disconnect("unexpected message");
return;
}
Handshake newHandshakeStatus = handshakeStatus.onMessage(peer, message);
if (newHandshakeStatus == null) {
// Handshake failure
LOGGER.debug(() -> String.format("Handshake failure with peer %s message %s", peer, message.getType().name()));
peer.disconnect("handshake failure");
return;
}
if (peer.isOutbound())
// If we made outbound connection then we need to act first
newHandshakeStatus.action(peer);
else
// We have inbound connection so we need to respond in kind with what we just received
handshakeStatus.action(peer);
peer.setHandshakeStatus(newHandshakeStatus);
if (newHandshakeStatus == Handshake.COMPLETED)
this.onHandshakeCompleted(peer);
} finally {
peer.resetHandshakeMessagePending();
}
}
private void onGetPeersMessage(Peer peer, Message message) {
// Send our known peers
if (!peer.sendMessage(buildPeersMessage(peer)))
peer.disconnect("failed to send peers list");
}
private void onPeersMessage(Peer peer, Message message) {
PeersMessage peersMessage = (PeersMessage) message;
List<PeerAddress> peerAddresses = new ArrayList<>();
// v1 PEERS message doesn't support port numbers so we have to add default port
for (InetAddress peerAddress : peersMessage.getPeerAddresses())
// This is always IPv4 so we don't have to worry about bracketing IPv6.
peerAddresses.add(PeerAddress.fromString(peerAddress.getHostAddress()));
// Also add peer's details
peerAddresses.add(PeerAddress.fromString(peer.getPeerData().getAddress().getHost()));
mergePeers(peer.toString(), peerAddresses);
}
private void onPingMessage(Peer peer, Message message) {
PingMessage pingMessage = (PingMessage) message;
// Generate 'pong' using same ID
PingMessage pongMessage = new PingMessage();
pongMessage.setId(pingMessage.getId());
if (!peer.sendMessage(pongMessage))
peer.disconnect("failed to send ping reply");
}
private void onPeersV2Message(Peer peer, Message message) {
PeersV2Message peersV2Message = (PeersV2Message) message;
List<PeerAddress> peerV2Addresses = peersV2Message.getPeerAddresses();
// First entry contains remote peer's listen port but empty address.
int peerPort = peerV2Addresses.get(0).getPort();
peerV2Addresses.remove(0);
// If inbound peer, use listen port and socket address to recreate first entry
if (!peer.isOutbound()) {
PeerAddress sendingPeerAddress = PeerAddress.fromString(peer.getPeerData().getAddress().getHost() + ":" + peerPort);
LOGGER.trace(() -> String.format("PEERS_V2 sending peer's listen address: %s", sendingPeerAddress.toString()));
peerV2Addresses.add(0, sendingPeerAddress);
}
mergePeers(peer.toString(), peerV2Addresses);
}
private void onPeerVerifyMessage(Peer peer, Message message) {
// Remote peer wants extra verification
possibleVerificationResponse(peer);
}
private void onVerificationCodesMessage(Peer peer, Message message) {
VerificationCodesMessage verificationCodesMessage = (VerificationCodesMessage) message;
// Remote peer is sending the code it wants to receive back via our outbound connection to it
Peer ourUnverifiedPeer = Network.getInstance().getInboundPeerWithId(Network.getInstance().getOurPeerId());
ourUnverifiedPeer.setVerificationCodes(verificationCodesMessage.getVerificationCodeSent(), verificationCodesMessage.getVerificationCodeExpected());
possibleVerificationResponse(ourUnverifiedPeer);
}
private void possibleVerificationResponse(Peer peer) {
// Can't respond if we don't have the codes (yet?)
if (peer.getVerificationCodeExpected() == null)

View File

@@ -43,7 +43,7 @@ public class Peer {
private static final int CONNECT_TIMEOUT = 1000; // ms
/** Maximum time to wait for a message reply to arrive from peer. (ms) */
private static final int RESPONSE_TIMEOUT = 5000; // ms
private static final int RESPONSE_TIMEOUT = 2000; // ms
/**
* Interval between PING messages to a peer. (ms)
@@ -466,16 +466,14 @@ public class Peer {
/* package */ void startPings() {
// Replacing initial null value allows getPingTask() to start sending pings.
LOGGER.trace(() -> String.format("Enabling pings for peer %s", this));
this.lastPingSent = System.currentTimeMillis();
this.lastPingSent = NTP.getTime();
}
/* package */ ExecuteProduceConsume.Task getPingTask() {
/* package */ ExecuteProduceConsume.Task getPingTask(Long now) {
// Pings not enabled yet?
if (this.lastPingSent == null)
if (now == null || this.lastPingSent == null)
return null;
final long now = System.currentTimeMillis();
// Time to send another ping?
if (now < this.lastPingSent + PING_INTERVAL)
return null; // Not yet
@@ -486,7 +484,6 @@ public class Peer {
return () -> {
PingMessage pingMessage = new PingMessage();
Message message = this.getResponse(pingMessage);
final long after = System.currentTimeMillis();
if (message == null || message.getType() != MessageType.PING) {
LOGGER.debug(() -> String.format("Didn't receive reply from %s for PING ID %d", this, pingMessage.getId()));
@@ -494,7 +491,7 @@ public class Peer {
return;
}
this.setLastPing(after - now);
this.setLastPing(NTP.getTime() - now);
};
}

View File

@@ -5,6 +5,8 @@ import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.util.HashSet;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.network.message.ProofMessage;
import com.google.common.primitives.Longs;
@@ -13,6 +15,7 @@ public class Proof extends Thread {
private static final int MIN_PROOF_ZEROS = 2;
private static final HashSet<Long> seenSalts = new HashSet<>();
private static final Logger LOGGER = LogManager.getLogger(Proof.class);
private Peer peer;
@@ -38,6 +41,7 @@ public class Proof extends Thread {
setName("Proof for peer " + this.peer);
// Do proof-of-work calculation to gain acceptance with remote end
final long startTime = LOGGER.isTraceEnabled() ? System.currentTimeMillis() : 0;
// Remote end knows this (approximately)
long timestamp = this.peer.getConnectionTimestamp();
@@ -64,7 +68,7 @@ public class Proof extends Thread {
long nonce;
for (nonce = 0; nonce < Long.MAX_VALUE; ++nonce) {
// Check whether we're shutting down every so often
if ((nonce & 0xff) == 0 && (peer.isStopping() || Thread.currentThread().isInterrupted()))
if ((nonce & 0xff) == 0 && (this.peer.isStopping() || Thread.currentThread().isInterrupted()))
// throw new InterruptedException("Interrupted during peer proof calculation");
return;
@@ -79,6 +83,8 @@ public class Proof extends Thread {
sha256.reset();
}
LOGGER.trace(() -> String.format("Proof for peer %s took %dms", this.peer, System.currentTimeMillis() - startTime));
ProofMessage proofMessage = new ProofMessage(timestamp, salt, nonce);
peer.sendMessage(proofMessage);
}

View File

@@ -620,16 +620,17 @@ public class HSQLDBAccountRepository implements AccountRepository {
@Override
public RewardShareData getRewardShare(byte[] minterPublicKey, String recipient) throws DataException {
String sql = "SELECT reward_share_public_key, share_percent FROM RewardShares WHERE minter_public_key = ? AND recipient = ?";
String sql = "SELECT minter, reward_share_public_key, share_percent FROM RewardShares WHERE minter_public_key = ? AND recipient = ?";
try (ResultSet resultSet = this.repository.checkedExecute(sql, minterPublicKey, recipient)) {
if (resultSet == null)
return null;
byte[] rewardSharePublicKey = resultSet.getBytes(1);
BigDecimal sharePercent = resultSet.getBigDecimal(2);
String minter = resultSet.getString(1);
byte[] rewardSharePublicKey = resultSet.getBytes(2);
BigDecimal sharePercent = resultSet.getBigDecimal(3);
return new RewardShareData(minterPublicKey, recipient, rewardSharePublicKey, sharePercent);
return new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent);
} catch (SQLException e) {
throw new DataException("Unable to fetch reward-share info from repository", e);
}
@@ -637,17 +638,18 @@ public class HSQLDBAccountRepository implements AccountRepository {
@Override
public RewardShareData getRewardShare(byte[] rewardSharePublicKey) throws DataException {
String sql = "SELECT minter_public_key, recipient, share_percent FROM RewardShares WHERE reward_share_public_key = ?";
String sql = "SELECT minter_public_key, minter, recipient, share_percent FROM RewardShares WHERE reward_share_public_key = ?";
try (ResultSet resultSet = this.repository.checkedExecute(sql, rewardSharePublicKey)) {
if (resultSet == null)
return null;
byte[] minterPublicKey = resultSet.getBytes(1);
String recipient = resultSet.getString(2);
BigDecimal sharePercent = resultSet.getBigDecimal(3);
String minter = resultSet.getString(2);
String recipient = resultSet.getString(3);
BigDecimal sharePercent = resultSet.getBigDecimal(4);
return new RewardShareData(minterPublicKey, recipient, rewardSharePublicKey, sharePercent);
return new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent);
} catch (SQLException e) {
throw new DataException("Unable to fetch reward-share info from repository", e);
}
@@ -675,7 +677,7 @@ public class HSQLDBAccountRepository implements AccountRepository {
@Override
public List<RewardShareData> getRewardShares() throws DataException {
String sql = "SELECT minter_public_key, recipient, share_percent, reward_share_public_key FROM RewardShares";
String sql = "SELECT minter_public_key, minter, recipient, share_percent, reward_share_public_key FROM RewardShares";
List<RewardShareData> rewardShares = new ArrayList<>();
@@ -685,11 +687,12 @@ public class HSQLDBAccountRepository implements AccountRepository {
do {
byte[] minterPublicKey = resultSet.getBytes(1);
String recipient = resultSet.getString(2);
BigDecimal sharePercent = resultSet.getBigDecimal(3);
byte[] rewardSharePublicKey = resultSet.getBytes(4);
String minter = resultSet.getString(2);
String recipient = resultSet.getString(3);
BigDecimal sharePercent = resultSet.getBigDecimal(4);
byte[] rewardSharePublicKey = resultSet.getBytes(5);
rewardShares.add(new RewardShareData(minterPublicKey, recipient, rewardSharePublicKey, sharePercent));
rewardShares.add(new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent));
} while (resultSet.next());
return rewardShares;
@@ -702,7 +705,7 @@ public class HSQLDBAccountRepository implements AccountRepository {
public List<RewardShareData> findRewardShares(List<String> minters, List<String> recipients, List<String> involvedAddresses,
Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(1024);
sql.append("SELECT DISTINCT minter_public_key, recipient, share_percent, reward_share_public_key FROM RewardShares ");
sql.append("SELECT DISTINCT minter_public_key, minter, recipient, share_percent, reward_share_public_key FROM RewardShares ");
List<Object> args = new ArrayList<>();
@@ -772,11 +775,12 @@ public class HSQLDBAccountRepository implements AccountRepository {
do {
byte[] minterPublicKey = resultSet.getBytes(1);
String recipient = resultSet.getString(2);
BigDecimal sharePercent = resultSet.getBigDecimal(3);
byte[] rewardSharePublicKey = resultSet.getBytes(4);
String minter = resultSet.getString(2);
String recipient = resultSet.getString(3);
BigDecimal sharePercent = resultSet.getBigDecimal(4);
byte[] rewardSharePublicKey = resultSet.getBytes(5);
rewardShares.add(new RewardShareData(minterPublicKey, recipient, rewardSharePublicKey, sharePercent));
rewardShares.add(new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent));
} while (resultSet.next());
return rewardShares;
@@ -801,7 +805,7 @@ public class HSQLDBAccountRepository implements AccountRepository {
@Override
public RewardShareData getRewardShareByIndex(int index) throws DataException {
String sql = "SELECT minter_public_key, recipient, share_percent, reward_share_public_key FROM RewardShares "
String sql = "SELECT minter_public_key, minter, recipient, share_percent, reward_share_public_key FROM RewardShares "
+ "ORDER BY reward_share_public_key ASC "
+ "OFFSET ? LIMIT 1";
@@ -810,11 +814,12 @@ public class HSQLDBAccountRepository implements AccountRepository {
return null;
byte[] minterPublicKey = resultSet.getBytes(1);
String recipient = resultSet.getString(2);
BigDecimal sharePercent = resultSet.getBigDecimal(3);
byte[] rewardSharePublicKey = resultSet.getBytes(4);
String minter = resultSet.getString(2);
String recipient = resultSet.getString(3);
BigDecimal sharePercent = resultSet.getBigDecimal(4);
byte[] rewardSharePublicKey = resultSet.getBytes(5);
return new RewardShareData(minterPublicKey, recipient, rewardSharePublicKey, sharePercent);
return new RewardShareData(minterPublicKey, minter, recipient, rewardSharePublicKey, sharePercent);
} catch (SQLException e) {
throw new DataException("Unable to fetch reward-share info from repository", e);
}
@@ -824,8 +829,9 @@ public class HSQLDBAccountRepository implements AccountRepository {
public void save(RewardShareData rewardShareData) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("RewardShares");
saveHelper.bind("minter_public_key", rewardShareData.getMinterPublicKey()).bind("recipient", rewardShareData.getRecipient())
.bind("reward_share_public_key", rewardShareData.getRewardSharePublicKey()).bind("share_percent", rewardShareData.getSharePercent());
saveHelper.bind("minter_public_key", rewardShareData.getMinterPublicKey()).bind("minter", rewardShareData.getMinter())
.bind("recipient", rewardShareData.getRecipient()).bind("reward_share_public_key", rewardShareData.getRewardSharePublicKey())
.bind("share_percent", rewardShareData.getSharePercent());
try {
saveHelper.execute(this.repository);
@@ -849,14 +855,15 @@ public class HSQLDBAccountRepository implements AccountRepository {
public List<MintingAccountData> getMintingAccounts() throws DataException {
List<MintingAccountData> mintingAccounts = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute("SELECT minter_private_key FROM MintingAccounts")) {
try (ResultSet resultSet = this.repository.checkedExecute("SELECT minter_private_key, minter_public_key FROM MintingAccounts")) {
if (resultSet == null)
return mintingAccounts;
do {
byte[] minterPrivateKey = resultSet.getBytes(1);
byte[] minterPublicKey = resultSet.getBytes(2);
mintingAccounts.add(new MintingAccountData(minterPrivateKey));
mintingAccounts.add(new MintingAccountData(minterPrivateKey, minterPublicKey));
} while (resultSet.next());
return mintingAccounts;
@@ -869,7 +876,8 @@ public class HSQLDBAccountRepository implements AccountRepository {
public void save(MintingAccountData mintingAccountData) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("MintingAccounts");
saveHelper.bind("minter_private_key", mintingAccountData.getPrivateKey());
saveHelper.bind("minter_private_key", mintingAccountData.getPrivateKey())
.bind("minter_public_key", mintingAccountData.getPublicKey());
try {
saveHelper.execute(this.repository);

View File

@@ -201,7 +201,7 @@ public class HSQLDBBlockRepository implements BlockRepository {
String subquerySql = "SELECT minter, COUNT(signature) FROM Blocks GROUP BY minter";
StringBuilder sql = new StringBuilder(1024);
sql.append("SELECT DISTINCT block_minter, n_blocks, minter_public_key, recipient FROM (");
sql.append("SELECT DISTINCT block_minter, n_blocks, minter_public_key, minter, recipient FROM (");
sql.append(subquerySql);
sql.append(") AS Minters (block_minter, n_blocks) LEFT OUTER JOIN RewardShares ON reward_share_public_key = block_minter ");
@@ -239,14 +239,17 @@ public class HSQLDBBlockRepository implements BlockRepository {
do {
byte[] blockMinterPublicKey = resultSet.getBytes(1);
int nBlocks = resultSet.getInt(2);
// May not be present if no reward-share:
byte[] mintingAccountPublicKey = resultSet.getBytes(3);
String recipientAccount = resultSet.getString(4);
String minterAccount = resultSet.getString(4);
String recipientAccount = resultSet.getString(5);
BlockMinterSummary blockMinterSummary;
if (recipientAccount == null)
blockMinterSummary = new BlockMinterSummary(blockMinterPublicKey, nBlocks);
else
blockMinterSummary = new BlockMinterSummary(blockMinterPublicKey, nBlocks, mintingAccountPublicKey, recipientAccount);
blockMinterSummary = new BlockMinterSummary(blockMinterPublicKey, nBlocks, mintingAccountPublicKey, minterAccount, recipientAccount);
summaries.add(blockMinterSummary);
} while (resultSet.next());
@@ -260,13 +263,13 @@ public class HSQLDBBlockRepository implements BlockRepository {
@Override
public List<BlockSummaryData> getBlockSummariesByMinter(byte[] minterPublicKey, Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(512);
sql.append("SELECT signature, height, minter, online_accounts_count FROM ");
sql.append("SELECT signature, height, Blocks.minter, online_accounts_count FROM ");
// List of minter account's public key and reward-share public keys with minter's public key
sql.append("(SELECT * FROM (VALUES (CAST(? AS QortalPublicKey))) UNION (SELECT reward_share_public_key FROM RewardShares WHERE minter_public_key = ?)) AS PublicKeys (public_key) ");
// Match Blocks signed with public key from above list
sql.append("JOIN Blocks ON minter = public_key ");
sql.append("JOIN Blocks ON Blocks.minter = public_key ");
sql.append("ORDER BY Blocks.height ");
if (reverse != null && reverse)

View File

@@ -933,6 +933,29 @@ public class HSQLDBDatabaseUpdates {
stmt.execute("ALTER TABLE AccountBalances ADD CONSTRAINT CheckBalanceNotNegative CHECK (balance >= 0)");
break;
case 67:
// Provide external function to convert private keys to public keys
stmt.execute("CREATE FUNCTION Ed25519_private_to_public_key (IN privateKey VARBINARY(32)) RETURNS VARBINARY(32) LANGUAGE JAVA DETERMINISTIC NO SQL EXTERNAL NAME 'CLASSPATH:org.qortal.repository.hsqldb.HSQLDBRepository.ed25519PrivateToPublicKey'");
// Cache minting account public keys to save us recalculating them
stmt.execute("ALTER TABLE MintingAccounts ADD minter_public_key QortalPublicKey");
stmt.execute("UPDATE MintingAccounts SET minter_public_key = Ed25519_private_to_public_key(minter_private_key)");
stmt.execute("ALTER TABLE MintingAccounts ALTER COLUMN minter_public_key SET NOT NULL");
// Provide external function to convert public keys to addresses
stmt.execute("CREATE FUNCTION Ed25519_public_key_to_address (IN privateKey VARBINARY(32)) RETURNS VARCHAR(36) LANGUAGE JAVA DETERMINISTIC NO SQL EXTERNAL NAME 'CLASSPATH:org.qortal.repository.hsqldb.HSQLDBRepository.ed25519PublicKeyToAddress'");
// Cache reward-share minting account's address
stmt.execute("ALTER TABLE RewardShares ADD minter QortalAddress BEFORE recipient");
stmt.execute("UPDATE RewardShares SET minter = Ed25519_public_key_to_address(minter_public_key)");
stmt.execute("ALTER TABLE RewardShares ALTER COLUMN minter SET NOT NULL");
break;
case 68:
// Slow down log fsync() calls from every 500ms to reduce I/O load
stmt.execute("SET FILES WRITE DELAY 5"); // only fsync() every 5 seconds
break;
default:
// nothing to do
return false;

View File

@@ -28,6 +28,8 @@ import java.util.regex.Pattern;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.crypto.Crypto;
import org.qortal.repository.ATRepository;
import org.qortal.repository.AccountRepository;
import org.qortal.repository.ArbitraryRepository;
@@ -57,6 +59,8 @@ public class HSQLDBRepository implements Repository {
protected List<String> sqlStatements;
protected long sessionId;
// Constructors
// NB: no visibility modifier so only callable from within same package
/* package */ HSQLDBRepository(Connection connection) throws DataException {
this.connection = connection;
@@ -84,6 +88,8 @@ public class HSQLDBRepository implements Repository {
assertEmptyTransaction("connection creation");
}
// Getters / setters
@Override
public ATRepository getATRepository() {
return new HSQLDBATRepository(this);
@@ -134,6 +140,18 @@ public class HSQLDBRepository implements Repository {
return new HSQLDBVotingRepository(this);
}
@Override
public boolean getDebug() {
return this.debugState;
}
@Override
public void setDebug(boolean debugState) {
this.debugState = debugState;
}
// Transaction COMMIT / ROLLBACK / savepoints
@Override
public void saveChanges() throws DataException {
try {
@@ -203,6 +221,8 @@ public class HSQLDBRepository implements Repository {
}
}
// Close / backup / rebuild / restore
@Override
public void close() throws DataException {
// Already closed? No need to do anything but maybe report double-call
@@ -257,16 +277,6 @@ public class HSQLDBRepository implements Repository {
}
}
@Override
public boolean getDebug() {
return this.debugState;
}
@Override
public void setDebug(boolean debugState) {
this.debugState = debugState;
}
@Override
public void backup(boolean quick) throws DataException {
if (!quick)
@@ -386,6 +396,8 @@ public class HSQLDBRepository implements Repository {
}
}
// SQL statements, etc.
/**
* Returns prepared statement using passed SQL, logging query if necessary.
*/
@@ -399,19 +411,6 @@ public class HSQLDBRepository implements Repository {
return this.connection.prepareStatement(sql);
}
/**
* Logs this transaction's SQL statements, if enabled.
*/
public void logStatements() {
if (this.sqlStatements == null)
return;
LOGGER.info(String.format("HSQLDB SQL statements (session %d) leading up to this were:", this.sessionId));
for (String sql : this.sqlStatements)
LOGGER.info(sql);
}
/**
* Execute SQL and return ResultSet with but added checking.
* <p>
@@ -429,15 +428,18 @@ public class HSQLDBRepository implements Repository {
// We can't use try-with-resources here as closing the PreparedStatement on return would also prematurely close the ResultSet.
preparedStatement.closeOnCompletion();
long beforeQuery = System.currentTimeMillis();
long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis();
ResultSet resultSet = this.checkedExecuteResultSet(preparedStatement, objects);
long queryTime = System.currentTimeMillis() - beforeQuery;
if (this.slowQueryThreshold != null && queryTime > this.slowQueryThreshold) {
LOGGER.info(String.format("HSQLDB query took %d ms: %s", queryTime, sql), new SQLException("slow query"));
if (this.slowQueryThreshold != null) {
long queryTime = System.currentTimeMillis() - beforeQuery;
logStatements();
if (queryTime > this.slowQueryThreshold) {
LOGGER.info(String.format("HSQLDB query took %d ms: %s", queryTime, sql), new SQLException("slow query"));
logStatements();
}
}
return resultSet;
@@ -500,16 +502,19 @@ public class HSQLDBRepository implements Repository {
try (PreparedStatement preparedStatement = this.prepareStatement(sql)) {
prepareExecute(preparedStatement, objects);
long beforeQuery = System.currentTimeMillis();
long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis();
if (preparedStatement.execute())
throw new SQLException("Database produced results, not row count");
long queryTime = System.currentTimeMillis() - beforeQuery;
if (this.slowQueryThreshold != null && queryTime > this.slowQueryThreshold) {
LOGGER.info(String.format("HSQLDB query took %d ms: %s", queryTime, sql), new SQLException("slow query"));
if (this.slowQueryThreshold != null) {
long queryTime = System.currentTimeMillis() - beforeQuery;
logStatements();
if (queryTime > this.slowQueryThreshold) {
LOGGER.info(String.format("HSQLDB query took %d ms: %s", queryTime, sql), new SQLException("slow query"));
logStatements();
}
}
int rowCount = preparedStatement.getUpdateCount();
@@ -670,6 +675,21 @@ public class HSQLDBRepository implements Repository {
stringBuilder.append(") ");
}
// Debugging
/**
* Logs this transaction's SQL statements, if enabled.
*/
public void logStatements() {
if (this.sqlStatements == null)
return;
LOGGER.info(String.format("HSQLDB SQL statements (session %d) leading up to this were:", this.sessionId));
for (String sql : this.sqlStatements)
LOGGER.info(sql);
}
/** Logs other HSQLDB sessions then re-throws passed exception */
public SQLException examineException(SQLException e) throws SQLException {
LOGGER.error(String.format("HSQLDB error (session %d): %s", this.sessionId, e.getMessage()), e);
@@ -726,6 +746,22 @@ public class HSQLDBRepository implements Repository {
}
}
// Utility methods
public static byte[] ed25519PrivateToPublicKey(byte[] privateKey) {
if (privateKey == null)
return null;
return PrivateKeyAccount.toPublicKey(privateKey);
}
public static String ed25519PublicKeyToAddress(byte[] publicKey) {
if (publicKey == null)
return null;
return Crypto.toAddress(publicKey);
}
/** Converts milliseconds from epoch to OffsetDateTime needed for TIMESTAMP WITH TIME ZONE columns. */
/* package */ static OffsetDateTime toOffsetDateTime(Long timestamp) {
if (timestamp == null)

View File

@@ -71,6 +71,8 @@ public class Settings {
private int maxTransactionTimestampFuture = 24 * 60 * 60 * 1000; // milliseconds
/** Whether we check, fetch and install auto-updates */
private boolean autoUpdateEnabled = true;
/** Whether to show a notification when we backup repository. */
private boolean showBackupNotification = false;
// Peer-to-peer related
private boolean isTestNet = false;
@@ -79,9 +81,11 @@ public class Settings {
/** Minimum number of peers to allow block minting / synchronization. */
private int minBlockchainPeers = 10;
/** Target number of outbound connections to peers we should make. */
private int minOutboundPeers = 40;
private int minOutboundPeers = 25;
/** Maximum number of peer connections we allow. */
private int maxPeers = 80;
private int maxPeers = 50;
/** Maximum number of threads for network engine. */
private int maxNetworkThreadPoolSize = 20;
// Which blockchains this node is running
private String blockchainConfig = null; // use default from resources
@@ -113,7 +117,7 @@ public class Settings {
"3.cn.pool.ntp.org"
};
/** Additional offset added to values returned by NTP.getTime() */
private long testNtpOffset = 0;
private Long testNtpOffset = null;
// Constructors
@@ -331,6 +335,10 @@ public class Settings {
return this.maxPeers;
}
public int getMaxNetworkThreadPoolSize() {
return this.maxNetworkThreadPoolSize;
}
public String getBlockchainConfig() {
return this.blockchainConfig;
}
@@ -359,8 +367,12 @@ public class Settings {
return this.ntpServers;
}
public long getTestNtpOffset() {
public Long getTestNtpOffset() {
return this.testNtpOffset;
}
public boolean getShowBackupNotification() {
return this.showBackupNotification;
}
}

View File

@@ -195,7 +195,9 @@ public class RewardShareTransaction extends Transaction {
this.repository.getAccountRepository().delete(mintingAccount.getPublicKey(), rewardShareTransactionData.getRecipient());
} else {
// Save reward-share info
rewardShareData = new RewardShareData(mintingAccount.getPublicKey(), rewardShareTransactionData.getRecipient(), rewardShareTransactionData.getRewardSharePublicKey(), rewardShareTransactionData.getSharePercent());
rewardShareData = new RewardShareData(mintingAccount.getPublicKey(), mintingAccount.getAddress(),
rewardShareTransactionData.getRecipient(), rewardShareTransactionData.getRewardSharePublicKey(),
rewardShareTransactionData.getSharePercent());
this.repository.getAccountRepository().save(rewardShareData);
}
}
@@ -217,8 +219,9 @@ public class RewardShareTransaction extends Transaction {
if (rewardShareTransactionData.getPreviousSharePercent() != null) {
// Revert previous sharing arrangement
RewardShareData rewardShareData = new RewardShareData(mintingAccount.getPublicKey(), rewardShareTransactionData.getRecipient(),
rewardShareTransactionData.getRewardSharePublicKey(), rewardShareTransactionData.getPreviousSharePercent());
RewardShareData rewardShareData = new RewardShareData(mintingAccount.getPublicKey(), mintingAccount.getAddress(),
rewardShareTransactionData.getRecipient(), rewardShareTransactionData.getRewardSharePublicKey(),
rewardShareTransactionData.getPreviousSharePercent());
this.repository.getAccountRepository().save(rewardShareData);
} else {

View File

@@ -20,6 +20,7 @@ public abstract class ExecuteProduceConsume implements Runnable {
public int consumerCount = 0;
public int tasksProduced = 0;
public int tasksConsumed = 0;
public int spawnFailures = 0;
public StatsSnapshot() {
}
@@ -27,6 +28,7 @@ public abstract class ExecuteProduceConsume implements Runnable {
private final String className;
private final Logger logger;
private final boolean isLoggerTraceEnabled;
private ExecutorService executor;
@@ -39,12 +41,14 @@ public abstract class ExecuteProduceConsume implements Runnable {
private volatile int consumerCount = 0;
private volatile int tasksProduced = 0;
private volatile int tasksConsumed = 0;
private volatile int spawnFailures = 0;
private volatile boolean hasThreadPending = false;
public ExecuteProduceConsume(ExecutorService executor) {
this.className = this.getClass().getSimpleName();
this.logger = LogManager.getLogger(this.getClass());
this.isLoggerTraceEnabled = this.logger.isTraceEnabled();
this.executor = executor;
}
@@ -75,6 +79,7 @@ public abstract class ExecuteProduceConsume implements Runnable {
snapshot.consumerCount = this.consumerCount;
snapshot.tasksProduced = this.tasksProduced;
snapshot.tasksConsumed = this.tasksConsumed;
snapshot.spawnFailures = this.spawnFailures;
}
return snapshot;
@@ -96,7 +101,8 @@ public abstract class ExecuteProduceConsume implements Runnable {
@Override
public void run() {
Thread.currentThread().setName(this.className + "-" + Thread.currentThread().getId());
if (this.isLoggerTraceEnabled)
Thread.currentThread().setName(this.className + "-" + Thread.currentThread().getId());
boolean wasThreadPending;
synchronized (this) {
@@ -131,10 +137,9 @@ public abstract class ExecuteProduceConsume implements Runnable {
this.logger.trace(() -> String.format("[%d] producing, activeThreadCount: %d, consumerCount: %d, canBlock is %b...",
Thread.currentThread().getId(), this.activeThreadCount, this.consumerCount, lambdaCanIdle));
final long now = System.currentTimeMillis();
final long beforeProduce = isLoggerTraceEnabled ? System.currentTimeMillis() : 0;
task = produceTask(canBlock);
final long delay = System.currentTimeMillis() - now;
this.logger.trace(() -> String.format("[%d] producing took %dms", Thread.currentThread().getId(), delay));
this.logger.trace(() -> String.format("[%d] producing took %dms", Thread.currentThread().getId(), System.currentTimeMillis() - beforeProduce));
}
if (task == null)
@@ -172,6 +177,7 @@ public abstract class ExecuteProduceConsume implements Runnable {
try {
this.executor.execute(this); // Same object, different thread
} catch (RejectedExecutionException e) {
++this.spawnFailures;
this.hasThreadPending = false;
this.logger.trace(() -> String.format("[%d] failed to spawn another thread", Thread.currentThread().getId()));
}
@@ -198,7 +204,8 @@ public abstract class ExecuteProduceConsume implements Runnable {
} catch (InterruptedException e) {
// We're in shutdown situation so exit
} finally {
Thread.currentThread().setName(this.className + "-dormant");
if (this.isLoggerTraceEnabled)
Thread.currentThread().setName(this.className);
}
}

View File

@@ -11,6 +11,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import org.apache.commons.net.ntp.NTPUDPClient;
import org.apache.commons.net.ntp.NtpV3Packet;
@@ -18,7 +19,6 @@ import org.apache.commons.net.ntp.TimeInfo;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.settings.Settings;
public class NTP implements Runnable {
@@ -53,15 +53,10 @@ public class NTP implements Runnable {
this.remote = remote;
}
public boolean doPoll(NTPUDPClient client) {
public boolean doPoll(NTPUDPClient client, final long now) {
Thread.currentThread().setName(String.format("NTP: %s", this.remote));
try {
final long now = System.currentTimeMillis();
if (now < this.nextPoll)
return false;
boolean isUpdated = false;
try {
TimeInfo timeInfo = client.getTime(InetAddress.getByName(remote));
@@ -110,26 +105,26 @@ public class NTP implements Runnable {
}
private final NTPUDPClient client;
private List<NTPServer> ntpServers = new ArrayList<>();
private final List<NTPServer> ntpServers = new ArrayList<>();
private final ExecutorService serverExecutor;
private NTP() {
private NTP(String[] serverNames) {
client = new NTPUDPClient();
client.setDefaultTimeout(2000);
for (String serverName : Settings.getInstance().getNtpServers())
for (String serverName : serverNames)
ntpServers.add(new NTPServer(serverName));
serverExecutor = Executors.newCachedThreadPool();
}
public static synchronized void start() {
public static synchronized void start(String[] serverNames) {
if (isStarted)
return;
isStarted = true;
instanceExecutor = Executors.newSingleThreadExecutor();
instance = new NTP();
instance = new NTP(serverNames);
instanceExecutor.execute(instance);
}
@@ -137,9 +132,9 @@ public class NTP implements Runnable {
instanceExecutor.shutdownNow();
}
public static synchronized void testMode() {
// Fix offset to match system time
NTP.offset = 0L;
public static synchronized void setFixedOffset(Long offset) {
// Fix offset, e.g. for testing
NTP.offset = offset;
}
/**
@@ -151,7 +146,7 @@ public class NTP implements Runnable {
if (NTP.offset == null)
return null;
return System.currentTimeMillis() + NTP.offset + Settings.getInstance().getTestNtpOffset();
return System.currentTimeMillis() + NTP.offset;
}
public void run() {
@@ -161,103 +156,120 @@ public class NTP implements Runnable {
while (!isStopping) {
Thread.sleep(1000);
CompletionService<Boolean> ecs = new ExecutorCompletionService<>(serverExecutor);
for (NTPServer server : ntpServers)
ecs.submit(() -> server.doPoll(client));
boolean haveUpdates = pollServers();
if (!haveUpdates)
continue;
boolean hasUpdate = false;
for (int i = 0; i < ntpServers.size(); ++i) {
if (isStopping)
return;
try {
hasUpdate = ecs.take().get() || hasUpdate;
} catch (ExecutionException e) {
// skip
}
}
if (hasUpdate) {
double s0 = 0;
double s1 = 0;
double s2 = 0;
for (NTPServer server : ntpServers) {
if (server.offset == null) {
server.usage = ' ';
continue;
}
server.usage = '+';
double value = server.offset * (double) server.stratum;
s0 += 1;
s1 += value;
s2 += value * value;
}
if (s0 < ntpServers.size() / 3 + 1) {
LOGGER.debug(String.format("Not enough replies (%d) to calculate network time", s0));
} else {
double thresholdStddev = Math.sqrt(((s0 * s2) - (s1 * s1)) / (s0 * (s0 - 1)));
double mean = s1 / s0;
// Now only consider offsets within 1 stddev?
s0 = 0;
s1 = 0;
s2 = 0;
for (NTPServer server : ntpServers) {
if (server.offset == null || server.reach == 0)
continue;
if (Math.abs(server.offset * (double)server.stratum - mean) > thresholdStddev)
continue;
server.usage = '*';
s0 += 1;
s1 += server.offset;
s2 += server.offset * server.offset;
}
if (s0 <= 1) {
LOGGER.debug(String.format("Not enough useful values (%d) to calculate network time. (stddev: %7.4f)", s0, thresholdStddev));
} else {
double filteredMean = s1 / s0;
double filteredStddev = Math.sqrt(((s0 * s2) - (s1 * s1)) / (s0 * (s0 - 1)));
LOGGER.trace(String.format("Threshold stddev: %7.3f, mean: %7.3f, stddev: %7.3f, nValues: %.0f / %d",
thresholdStddev, filteredMean, filteredStddev, s0, ntpServers.size()));
NTP.offset = (long) filteredMean;
LOGGER.debug(String.format("New NTP offset: %d", NTP.offset));
}
}
if (LOGGER.getLevel().isMoreSpecificThan(Level.TRACE)) {
LOGGER.trace(String.format("%c%16s %16s %2s %c %4s %4s %3s %7s %7s %7s",
' ', "remote", "refid", "st", 't', "when", "poll", "reach", "delay", "offset", "jitter"
));
for (NTPServer server : ntpServers)
LOGGER.trace(String.format("%c%16.16s %16.16s %2s %c %4s %4d %3o %7s %7s %7s",
server.usage,
server.remote,
formatNull("%s", server.refId, ""),
formatNull("%2d", server.stratum, ""),
server.type,
formatNull("%4d", server.getWhen(), "-"),
server.poll,
server.reach,
formatNull("%5dms", server.delay, ""),
formatNull("% 5.0fms", server.offset, ""),
formatNull("%5.2fms", server.jitter, "")
));
}
}
calculateOffset();
}
} catch (InterruptedException e) {
// Exit
// Interrupted - time to exit
return;
}
}
private boolean pollServers() throws InterruptedException {
final long now = System.currentTimeMillis();
List<NTPServer> pendingServers = ntpServers.stream().filter(ntpServer -> now >= ntpServer.nextPoll).collect(Collectors.toList());
CompletionService<Boolean> ecs = new ExecutorCompletionService<>(serverExecutor);
for (NTPServer server : pendingServers)
ecs.submit(() -> server.doPoll(client, now));
boolean haveUpdate = false;
for (int i = 0; i < pendingServers.size(); ++i) {
if (isStopping)
return false;
try {
haveUpdate = ecs.take().get() || haveUpdate;
} catch (ExecutionException e) {
// skip
}
}
return haveUpdate;
}
private void calculateOffset() {
double s0 = 0;
double s1 = 0;
double s2 = 0;
for (NTPServer server : ntpServers) {
if (server.offset == null) {
server.usage = ' ';
continue;
}
server.usage = '+';
double value = server.offset * (double) server.stratum;
s0 += 1;
s1 += value;
s2 += value * value;
}
if (s0 < ntpServers.size() / 3 + 1) {
final double numberReplies = s0;
LOGGER.debug(() -> String.format("Not enough replies (%d) to calculate network time", numberReplies));
} else {
double thresholdStddev = Math.sqrt(((s0 * s2) - (s1 * s1)) / (s0 * (s0 - 1)));
double mean = s1 / s0;
// Now only consider offsets within 1 stddev?
s0 = 0;
s1 = 0;
s2 = 0;
for (NTPServer server : ntpServers) {
if (server.offset == null || server.reach == 0)
continue;
if (Math.abs(server.offset * (double)server.stratum - mean) > thresholdStddev)
continue;
server.usage = '*';
s0 += 1;
s1 += server.offset;
s2 += server.offset * server.offset;
}
final double numberValues = s0;
if (s0 <= 1) {
LOGGER.debug(() -> String.format("Not enough useful values (%d) to calculate network time. (stddev: %7.4f)", numberValues, thresholdStddev));
} else {
double filteredMean = s1 / s0;
double filteredStddev = Math.sqrt(((s0 * s2) - (s1 * s1)) / (s0 * (s0 - 1)));
LOGGER.trace(() -> String.format("Threshold stddev: %7.3f, mean: %7.3f, stddev: %7.3f, nValues: %.0f / %d",
thresholdStddev, filteredMean, filteredStddev, numberValues, ntpServers.size()));
NTP.offset = (long) filteredMean;
LOGGER.debug(() -> String.format("New NTP offset: %d", NTP.offset));
}
}
if (LOGGER.getLevel().isLessSpecificThan(Level.TRACE)) {
LOGGER.trace(() -> String.format("%c%16s %16s %2s %c %4s %4s %3s %7s %7s %7s",
' ', "remote", "refid", "st", 't', "when", "poll", "reach", "delay", "offset", "jitter"
));
for (NTPServer server : ntpServers)
LOGGER.trace(() -> String.format("%c%16.16s %16.16s %2s %c %4s %4d %3o %7s %7s %7s",
server.usage,
server.remote,
formatNull("%s", server.refId, ""),
formatNull("%2d", server.stratum, ""),
server.type,
formatNull("%4d", server.getWhen(), "-"),
server.poll,
server.reach,
formatNull("%5dms", server.delay, ""),
formatNull("% 5.0fms", server.offset, ""),
formatNull("%5.2fms", server.jitter, "")
));
}
}

View File

@@ -1,6 +1,10 @@
#Generated by ResourceBundle Editor (http://essiembre.github.io/eclipse-rbe/)
# SysTray pop-up menu
APPLYING_UPDATE_AND_RESTARTING = Applying automatic update and restarting...
AUTO_UPDATE = Auto Update
BLOCK_HEIGHT = height
CHECK_TIME_ACCURACY = Check time accuracy
@@ -9,6 +13,10 @@ CONNECTION = connection
CONNECTIONS = connections
CREATING_BACKUP_OF_DB_FILES = Creating backup of database files...
DB_BACKUP = Database Backup
EXIT = Exit
MINTING_DISABLED = NOT minting

View File

@@ -3,6 +3,7 @@ package org.qortal.test;
import org.junit.Test;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.DataException;
import org.qortal.settings.Settings;
import org.qortal.test.common.Common;
import org.qortal.transaction.CreateAssetOrderTransaction;
import org.qortal.transaction.CreatePollTransaction;
@@ -22,7 +23,7 @@ public class CompatibilityTests extends Common {
@Before
public void beforeTest() throws DataException {
Common.useSettings("test-settings-v1.json");
NTP.testMode();
NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset());
}
@Test

View File

@@ -61,12 +61,10 @@ public class EPCTests {
ScheduledExecutorService statusExecutor = Executors.newSingleThreadScheduledExecutor();
statusExecutor.scheduleAtFixedRate(() -> {
StatsSnapshot snapshot = testEPC.getStatsSnapshot();
final StatsSnapshot snapshot = testEPC.getStatsSnapshot();
final long seconds = (System.currentTimeMillis() - start) / 1000L;
System.out.println(String.format("After %d second%s, active threads: %d, greatest thread count: %d, tasks produced: %d, tasks consumed: %d",
seconds, (seconds != 1 ? "s" : ""),
snapshot.activeThreadCount, snapshot.greatestActiveThreadCount,
snapshot.tasksProduced, snapshot.tasksConsumed));
System.out.print(String.format("After %d second%s, ", seconds, (seconds != 1 ? "s" : "")));
printSnapshot(snapshot);
}, 1L, 1L, TimeUnit.SECONDS);
// Let it run for a minute
@@ -79,9 +77,16 @@ public class EPCTests {
System.out.println(String.format("Shutdown took %d milliseconds", after - before));
StatsSnapshot snapshot = testEPC.getStatsSnapshot();
System.out.println(String.format("Greatest thread count: %d, tasks produced: %d, tasks consumed: %d",
snapshot.greatestActiveThreadCount, snapshot.tasksProduced, snapshot.tasksConsumed));
final StatsSnapshot snapshot = testEPC.getStatsSnapshot();
System.out.print("After shutdown, ");
printSnapshot(snapshot);
}
private void printSnapshot(final StatsSnapshot snapshot) {
System.out.println(String.format("threads: %d active (%d max, %d exhaustion%s), tasks: %d produced / %d consumed",
snapshot.activeThreadCount, snapshot.greatestActiveThreadCount,
snapshot.spawnFailures, (snapshot.spawnFailures != 1 ? "s": ""),
snapshot.tasksProduced, snapshot.tasksConsumed));
}
@Test

View File

@@ -13,8 +13,9 @@ import org.qortal.transaction.Transaction.ValidationResult;
public class CheckTranslations {
private static final String[] SUPPORTED_LANGS = new String[] { "en", "de", "zh", "ru" };
private static final Set<String> SYSTRAY_KEYS = Set.of("BLOCK_HEIGHT", "CHECK_TIME_ACCURACY", "CONNECTION", "CONNECTIONS",
"EXIT", "MINTING_DISABLED", "MINTING_ENABLED", "NTP_NAG_CAPTION", "NTP_NAG_TEXT_UNIX", "NTP_NAG_TEXT_WINDOWS",
private static final Set<String> SYSTRAY_KEYS = Set.of("AUTO_UPDATE", "APPLYING_UPDATE_AND_RESTARTING", "BLOCK_HEIGHT",
"CHECK_TIME_ACCURACY", "CONNECTION", "CONNECTIONS", "CREATING_BACKUP_OF_DB_FILES", "DB_BACKUP", "EXIT",
"MINTING_DISABLED", "MINTING_ENABLED", "NTP_NAG_CAPTION", "NTP_NAG_TEXT_UNIX", "NTP_NAG_TEXT_WINDOWS",
"OPEN_UI", "SYNCHRONIZE_CLOCK", "SYNCHRONIZING_BLOCKCHAIN", "SYNCHRONIZING_CLOCK");
private static String failurePrefix;

View File

@@ -1,200 +1,44 @@
package org.qortal.test.apps;
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Deque;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.CompletionService;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.Executors;
import org.apache.commons.net.ntp.NTPUDPClient;
import org.apache.commons.net.ntp.NtpV3Packet;
import org.apache.commons.net.ntp.TimeInfo;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.config.Configuration;
import org.apache.logging.log4j.core.config.LoggerConfig;
import org.apache.logging.log4j.core.LoggerContext;
import org.qortal.utils.NTP;
public class NTPTests {
private static final List<String> CC_TLDS = Arrays.asList("oceania", "europe", "cn", "asia", "africa");
public static void main(String[] args) throws UnknownHostException, IOException, InterruptedException {
NTPUDPClient client = new NTPUDPClient();
client.setDefaultTimeout(2000);
class NTPServer {
private static final int MIN_POLL = 8;
public char usage = ' ';
public String remote;
public String refId;
public Integer stratum;
public char type = 'u'; // unicast
public int poll = MIN_POLL;
public byte reach = 0;
public Long delay;
public Double offset;
public Double jitter;
private Deque<Double> offsets = new LinkedList<>();
private double totalSquareOffsets = 0.0;
private long nextPoll;
private Long lastGood;
public NTPServer(String remote) {
this.remote = remote;
}
public boolean poll(NTPUDPClient client) {
final long now = System.currentTimeMillis();
if (now < this.nextPoll)
return false;
boolean isUpdated = false;
try {
TimeInfo timeInfo = client.getTime(InetAddress.getByName(remote));
timeInfo.computeDetails();
NtpV3Packet ntpMessage = timeInfo.getMessage();
this.refId = ntpMessage.getReferenceIdString();
this.stratum = ntpMessage.getStratum();
this.poll = Math.max(MIN_POLL, 1 << ntpMessage.getPoll());
this.delay = timeInfo.getDelay();
this.offset = (double) timeInfo.getOffset();
if (this.offsets.size() == 8) {
double oldOffset = this.offsets.removeFirst();
this.totalSquareOffsets -= oldOffset * oldOffset;
}
this.offsets.addLast(this.offset);
this.totalSquareOffsets += this.offset * this.offset;
this.jitter = Math.sqrt(this.totalSquareOffsets / this.offsets.size());
this.reach = (byte) ((this.reach << 1) | 1);
this.lastGood = now;
isUpdated = true;
} catch (IOException e) {
this.reach <<= 1;
}
this.nextPoll = now + this.poll * 1000;
return isUpdated;
}
public Integer getWhen() {
if (this.lastGood == null)
return null;
return (int) ((System.currentTimeMillis() - this.lastGood) / 1000);
}
}
List<NTPServer> ntpServers = new ArrayList<>();
public static void main(String[] args) throws InterruptedException {
List<String> ntpServers = new ArrayList<>();
for (String ccTld : CC_TLDS)
for (int subpool = 0; subpool <=3; ++subpool)
ntpServers.add(new NTPServer(subpool + "." + ccTld + ".pool.ntp.org"));
for (int subpool = 0; subpool <= 3; ++subpool)
ntpServers.add(new String(subpool + "." + ccTld + ".pool.ntp.org"));
while (true) {
Thread.sleep(1000);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
NTP.shutdownNow();
}));
CompletionService<Boolean> ecs = new ExecutorCompletionService<Boolean>(Executors.newCachedThreadPool());
for (NTPServer server : ntpServers)
ecs.submit(() -> server.poll(client));
Logger ntpLogger = LogManager.getLogger(NTP.class);
LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false);
Configuration config = loggerContext.getConfiguration();
LoggerConfig loggerConfig = config.getLoggerConfig(ntpLogger.getName());
boolean showReport = false;
for (int i = 0; i < ntpServers.size(); ++i)
try {
showReport = ecs.take().get() || showReport;
} catch (ExecutionException e) {
// skip
}
loggerConfig.setLevel(Level.TRACE);
loggerContext.updateLoggers(config);
if (showReport) {
double s0 = 0;
double s1 = 0;
double s2 = 0;
NTP.start(ntpServers.toArray(new String[0]));
for (NTPServer server : ntpServers) {
if (server.offset == null) {
server.usage = ' ';
continue;
}
server.usage = '+';
double value = server.offset * (double) server.stratum;
s0 += 1;
s1 += value;
s2 += value * value;
}
if (s0 < ntpServers.size() / 3 + 1) {
System.out.println("Not enough replies to calculate network time");
} else {
double filterStddev = Math.sqrt(((s0 * s2) - (s1 * s1)) / (s0 * (s0 - 1)));
double filterMean = s1 / s0;
// Now only consider offsets within 1 stddev?
s0 = 0;
s1 = 0;
s2 = 0;
for (NTPServer server : ntpServers) {
if (server.offset == null || server.reach == 0)
continue;
if (Math.abs(server.offset * (double)server.stratum - filterMean) > filterStddev)
continue;
server.usage = '*';
s0 += 1;
s1 += server.offset;
s2 += server.offset * server.offset;
}
if (s0 <= 1) {
System.out.println(String.format("Not enough values to calculate network time. stddev: %7.4f", filterStddev));
} else {
double mean = s1 / s0;
double newStddev = Math.sqrt(((s0 * s2) - (s1 * s1)) / (s0 * (s0 - 1)));
System.out.println(String.format("filtering stddev: %7.3f, mean: %7.3f, new stddev: %7.3f, nValues: %.0f / %d", filterStddev, mean, newStddev, s0, ntpServers.size()));
}
}
System.out.println(String.format("%c%16s %16s %2s %c %4s %4s %3s %7s %7s %7s",
' ', "remote", "refid", "st", 't', "when", "poll", "reach", "delay", "offset", "jitter"
));
for (NTPServer server : ntpServers)
System.out.println(String.format("%c%16.16s %16.16s %2s %c %4s %4d %3o %7s %7s %7s",
server.usage,
server.remote,
formatNull("%s", server.refId, ""),
formatNull("%2d", server.stratum, ""),
server.type,
formatNull("%4d", server.getWhen(), "-"),
server.poll,
server.reach,
formatNull("%5dms", server.delay, ""),
formatNull("% 5.0fms", server.offset, ""),
formatNull("%5.2fms", server.jitter, "")
));
}
}
}
private static String formatNull(String format, Object arg, String nullOutput) {
return arg != null ? String.format(format, arg) : nullOutput;
// Endless sleep
Thread.sleep(1000000000L);
}
}

View File

@@ -6,6 +6,7 @@ import org.junit.Test;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.test.common.AccountUtils;
import org.qortal.test.common.AssetUtils;
import org.qortal.test.common.Common;
@@ -19,7 +20,7 @@ public class OldTradingTests extends Common {
@Before
public void beforeTest() throws DataException {
Common.useSettings("test-settings-old-asset.json");
NTP.testMode();
NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset());
}
@After

View File

@@ -116,7 +116,7 @@ public class Common {
public static void useDefaultSettings() throws DataException {
useSettings(testSettingsFilename);
NTP.testMode();
NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset());
}
public static void resetBlockchain() throws DataException {