Compare commits

...

15 Commits

Author SHA1 Message Date
CalDescent
b725918df6 Upgraded OnlineAccountsV3Message to new format (and fixed merge issues). Needs re-testing. 2022-05-02 08:47:17 +01:00
CalDescent
ed28405ceb Merge branch 'master' into online-accounts-mempow
# Conflicts:
#	src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java
#	src/main/java/org/qortal/network/message/Message.java
#	src/main/java/org/qortal/network/message/OnlineAccountsMessage.java
2022-05-02 08:41:44 +01:00
CalDescent
6d8329de16 Removed GetOnlineAccountsV3Message
We now use GetOnlineAccountsV2Message in all cases, and the response will be either OnlineAccountsV2Message or OnlineAccountsV3Message depending on the version of the requesting peer.
2022-04-16 20:43:18 +01:00
CalDescent
bb2e52d5e1 Attempt to handle tricky situation where some instances of an online account contain the nonce and recent block signature, whereas other instances do not (due to being sent via an older peer).
Right now, two OnlineAccountData objects are considered equal if they have matching timestamps, signatures, and public keys. This reduces the chance of multiple versions of the same online account data from being sent around the network. The downside is that an instance containing a nonce value can be ignored due to already having an inferior OnlineAccountData instance in the list.

The current approach is this:
- Only allow new duplicate onlineAccountData to be added to the import queue if it's superior to the one we already have.
- Remove the existing, inferior data at the time of import (once the new data is considered valid).

This is only a temporary problem, and can be simplified once the additional fields in OnlineAccountsV3Message become required rather than optional.
2022-04-16 17:47:28 +01:00
CalDescent
14f262d567 Merge branch 'master' into online-accounts-mempow
# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
2022-04-16 13:05:09 +01:00
CalDescent
012cde705a REDUCED_SIGNATURE_LENGTH lowered from 8 to 4
This still gives more than enough uniqueness to prevent mempow nonces from being pre-calculated. Could potentially go even lower still?
2022-04-05 09:03:14 +01:00
CalDescent
abc9cb3958 Compute the nonce for the 'next' timestamp before the 'current' timestamp.
Nodes use each 30 minute period to compute the nonce for the next 30 minute period, so this should be prioritized. Once calculated, the 'current' timestamp is attempted if there is enough time. Doing it in this order avoids falling behind and then struggling to catch up.

We will need to think about how to handle node restarts, since otherwise an auto update could cause a gap in online accounts due to all nodes computing the 'next' timestamp before the 'current' one.
2022-04-05 08:07:19 +01:00
CalDescent
32a0b02ea4 Fixed bug in OnlineAccountsV3Message 2022-04-01 20:06:22 +01:00
CalDescent
5273968619 Fixed log formatting error. 2022-04-01 19:03:28 +01:00
CalDescent
5857929508 Correction to commit #eb876e1
Block serialization is in fact affected by the online accounts serialization, as we have to calculate the expected length ahead of time.
2022-04-01 18:39:34 +01:00
CalDescent
9a1941fac4 Ensure online accounts are computed in recovery mode
Without this, no recovery blocks are minted, because no online accounts are available.
2022-04-01 18:32:51 +01:00
CalDescent
eb876e12c8 Modified block minting and validation to support extended OnlineAccountData.
This doesn't require changes to the transformation of the outer Block components, since the "onlineAccountsSignatures" component is already variable length. It does however affect the encoding of the data within "onlineAccountsSignatures". New encoding becomes active once the block timestamp reaches onlineAccountsMemoryPoWTimestamp.
2022-04-01 12:06:02 +01:00
CalDescent
f993f938f4 Added mempow support in OnlineAccountsManager.
- Removed OnlineAccountsMessage and GetOnlineAccountsMessage (no longer any need for backwards support).
- Added OnlineAccountsV3Message and GetOnlineAccountsV3Message.
- Mempow computations can be opted in early via the "onlineAccountsMemPoWEnabled" setting (although they won't be validated).
- Feature trigger timestamp set to unknown future date.
- Still needs calibration on a testnet.
- Still need to uncomment/finalize code to calculate "next" signature ahead of time.
2022-04-01 12:02:29 +01:00
CalDescent
20d45955e5 Increase ONLINE_ACCOUNTS_MODULUS from 5 to 30 mins at a future undecided timestamp.
Note: it's important that this timestamp is set on a 1-hour boundary (such as 16:00:00) to ensure a clean switchover.
2022-04-01 11:35:32 +01:00
CalDescent
5c607d3367 Added optional "timeout" feature to MemoryPoW.compute2(), to give up after a specified amount of time. 2022-04-01 11:26:54 +01:00
24 changed files with 852 additions and 315 deletions

View File

@ -85,7 +85,8 @@ public class Block {
ONLINE_ACCOUNT_UNKNOWN(71),
ONLINE_ACCOUNT_SIGNATURES_MISSING(72),
ONLINE_ACCOUNT_SIGNATURES_MALFORMED(73),
ONLINE_ACCOUNT_SIGNATURE_INCORRECT(74);
ONLINE_ACCOUNT_SIGNATURE_INCORRECT(74),
ONLINE_ACCOUNT_NONCE_INCORRECT(75);
public final int value;
@ -313,6 +314,15 @@ public class Block {
int version = parentBlock.getNextBlockVersion();
byte[] reference = parentBlockData.getSignature();
// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, minter.getPublicKey());
if (minterLevel == 0) {
LOGGER.error("Minter effective level returned zero?");
return null;
}
long timestamp = calcTimestamp(parentBlockData, minter.getPublicKey(), minterLevel);
// Fetch our list of online accounts
List<OnlineAccountData> onlineAccounts = OnlineAccountsManager.getInstance().getOnlineAccounts();
if (onlineAccounts.isEmpty()) {
@ -355,26 +365,13 @@ public class Block {
byte[] encodedOnlineAccounts = BlockTransformer.encodeOnlineAccounts(onlineAccountsSet);
int onlineAccountsCount = onlineAccountsSet.size();
// Concatenate online account timestamp signatures (in correct order)
byte[] onlineAccountsSignatures = new byte[onlineAccountsCount * Transformer.SIGNATURE_LENGTH];
for (int i = 0; i < onlineAccountsCount; ++i) {
Integer accountIndex = accountIndexes.get(i);
OnlineAccountData onlineAccountData = indexedOnlineAccounts.get(accountIndex);
System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH);
}
// Build the onlineAccountsSignatures byte array
byte[] onlineAccountsSignatures = BlockTransformer.encodeOnlineAccountSignatures(indexedOnlineAccounts,
accountIndexes, onlineAccountsCount, timestamp);
byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData,
minter.getPublicKey(), encodedOnlineAccounts));
// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level
int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, minter.getPublicKey());
if (minterLevel == 0) {
LOGGER.error("Minter effective level returned zero?");
return null;
}
long timestamp = calcTimestamp(parentBlockData, minter.getPublicKey(), minterLevel);
int transactionCount = 0;
byte[] transactionsSignature = null;
int height = parentBlockData.getHeight() + 1;
@ -979,7 +976,10 @@ public class Block {
if (this.blockData.getOnlineAccountsSignatures() == null || this.blockData.getOnlineAccountsSignatures().length == 0)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MISSING;
if (this.blockData.getOnlineAccountsSignatures().length != onlineRewardShares.size() * Transformer.SIGNATURE_LENGTH)
// Verify the online account signatures length
int expectedLength = Block.getExpectedOnlineAccountsSignaturesLength(onlineRewardShares.size(), this.blockData.getTimestamp());
if (this.blockData.getOnlineAccountsSignatures().length != expectedLength)
return ValidationResult.ONLINE_ACCOUNT_SIGNATURES_MALFORMED;
// Check signatures
@ -987,23 +987,31 @@ public class Block {
byte[] onlineTimestampBytes = Longs.toByteArray(onlineTimestamp);
// If this block is much older than current online timestamp, then there's no point checking current online accounts
List<OnlineAccountData> currentOnlineAccounts = onlineTimestamp < NTP.getTime() - OnlineAccountsManager.ONLINE_TIMESTAMP_MODULUS
List<OnlineAccountData> currentOnlineAccounts = onlineTimestamp < NTP.getTime() - OnlineAccountsManager.getOnlineTimestampModulus()
? null
: OnlineAccountsManager.getInstance().getOnlineAccounts();
List<OnlineAccountData> latestBlocksOnlineAccounts = OnlineAccountsManager.getInstance().getLatestBlocksOnlineAccounts();
// Extract online accounts' timestamp signatures from block data
List<byte[]> onlineAccountsSignatures = BlockTransformer.decodeTimestampSignatures(this.blockData.getOnlineAccountsSignatures());
List<OnlineAccountData> onlineAccountsSignatures = BlockTransformer.decodeOnlineAccountSignatures(
this.blockData.getOnlineAccountsSignatures(), onlineRewardShares.size(), this.blockData.getTimestamp());
// We'll build up a list of online accounts to hand over to Controller if block is added to chain
// and this will become latestBlocksOnlineAccounts (above) to reduce CPU load when we process next block...
List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
for (int i = 0; i < onlineAccountsSignatures.size(); ++i) {
byte[] signature = onlineAccountsSignatures.get(i);
// onlineAccountsSignatures will contain OnlineAccountData objects with at least a signature, and
// also a reduced block signature and nonce(s) if the mempow feature is active.
// It won't contain a public key or timestamp, so these must be added below.
OnlineAccountData onlineAccountSignatureData = onlineAccountsSignatures.get(i);
byte[] signature = onlineAccountSignatureData.getSignature();
byte[] reducedBlockSignature = onlineAccountSignatureData.getReducedBlockSignature();
List<Integer> nonces = onlineAccountSignatureData.getNonces();
byte[] publicKey = onlineRewardShares.get(i).getRewardSharePublicKey();
OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, signature, publicKey);
// It's simpler to create a new OnlineAccountData object rather than trying to modify the one we already have
OnlineAccountData onlineAccountData = new OnlineAccountData(onlineTimestamp, signature, publicKey, nonces, reducedBlockSignature);
ourOnlineAccounts.add(onlineAccountData);
// If signature is still current then no need to perform Ed25519 verify
@ -1018,6 +1026,10 @@ public class Block {
if (!Crypto.verify(publicKey, signature, onlineTimestampBytes))
return ValidationResult.ONLINE_ACCOUNT_SIGNATURE_INCORRECT;
if (this.blockData.getTimestamp() >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp())
if (!OnlineAccountsManager.getInstance().verifyMemoryPoW(onlineAccountData))
return ValidationResult.ONLINE_ACCOUNT_NONCE_INCORRECT;
}
// All online accounts valid, so save our list of online accounts for potential later use
@ -2048,6 +2060,29 @@ public class Block {
return null;
}
/**
* Expected length of serialized online accounts
* @param onlineRewardSharesCount the number of reward shares in the serialized data
* @param timestamp the block's timestamp, used for versioning / serialization differences
* @return the number of bytes to expect
*/
public static int getExpectedOnlineAccountsSignaturesLength(int onlineRewardSharesCount, long timestamp) {
int expectedLength;
if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) {
// byte array contains signatures, reduced signatures, and nonces
expectedLength = onlineRewardSharesCount *
(Transformer.SIGNATURE_LENGTH + Transformer.REDUCED_SIGNATURE_LENGTH + Transformer.INT_LENGTH +
(OnlineAccountsManager.MAX_NONCE_COUNT * Transformer.INT_LENGTH));
}
else {
// byte array contains signatures only
expectedLength = onlineRewardSharesCount * Transformer.SIGNATURE_LENGTH;
}
return expectedLength;
}
private void logDebugInfo() {
try {
// Avoid calculations if possible. We have to check against INFO here, since Level.isMoreSpecificThan() confusingly uses <= rather than just <

View File

@ -162,6 +162,14 @@ public class BlockChain {
/** Maximum time to retain online account signatures (ms) for block validity checks, to allow for clock variance. */
private long onlineAccountSignaturesMaxLifetime;
/** Feature trigger timestamp for ONLINE_ACCOUNTS_MODULUS time interval increase. Can't use
* featureTriggers because unit tests need to set this value via Reflection. */
private long onlineAccountsModulusV2Timestamp;
/** Feature trigger timestamp for online accounts mempow verification. Can't use featureTriggers
* because unit tests need to set this value via Reflection. */
private long onlineAccountsMemoryPoWTimestamp;
/** Settings relating to CIYAM AT feature. */
public static class CiyamAtSettings {
/** Fee per step/op-code executed. */
@ -310,6 +318,15 @@ public class BlockChain {
return this.maxBlockSize;
}
// Online accounts
public long getOnlineAccountsModulusV2Timestamp() {
return this.onlineAccountsModulusV2Timestamp;
}
public long getOnlineAccountsMemoryPoWTimestamp() {
return this.onlineAccountsMemoryPoWTimestamp;
}
/** Returns true if approval-needing transaction types require a txGroupId other than NO_GROUP. */
public boolean getRequireGroupForApproval() {
return this.requireGroupForApproval;

View File

@ -1146,14 +1146,6 @@ public class Controller extends Thread {
TransactionImporter.getInstance().onNetworkTransactionSignaturesMessage(peer, message);
break;
case GET_ONLINE_ACCOUNTS:
OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsMessage(peer, message);
break;
case ONLINE_ACCOUNTS:
OnlineAccountsManager.getInstance().onNetworkOnlineAccountsMessage(peer, message);
break;
case GET_ONLINE_ACCOUNTS_V2:
OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV2Message(peer, message);
break;
@ -1162,6 +1154,10 @@ public class Controller extends Thread {
OnlineAccountsManager.getInstance().onNetworkOnlineAccountsV2Message(peer, message);
break;
case ONLINE_ACCOUNTS_V3:
OnlineAccountsManager.getInstance().onNetworkOnlineAccountsV3Message(peer, message);
break;
case GET_ARBITRARY_DATA:
// Not currently supported
break;

View File

@ -7,8 +7,10 @@ import org.qortal.account.Account;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.account.PublicKeyAccount;
import org.qortal.block.BlockChain;
import org.qortal.crypto.MemoryPoW;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.account.RewardShareData;
import org.qortal.data.block.BlockData;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.network.Network;
import org.qortal.network.Peer;
@ -16,12 +18,18 @@ import org.qortal.network.message.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import static org.qortal.transform.Transformer.REDUCED_SIGNATURE_LENGTH;
public class OnlineAccountsManager extends Thread {
private class OurOnlineAccountsThread extends Thread {
@ -47,14 +55,19 @@ public class OnlineAccountsManager extends Thread {
private static OnlineAccountsManager instance;
private volatile boolean isStopping = false;
// MemoryPoW
public final int POW_BUFFER_SIZE = 1 * 1024 * 1024; // bytes
public int POW_DIFFICULTY = 18; // leading zero bits
public static final int MAX_NONCE_COUNT = 1; // Maximum number of nonces to verify
// To do with online accounts list
private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms
private static final long ONLINE_ACCOUNTS_BROADCAST_INTERVAL = 1 * 60 * 1000L; // ms
public static final long ONLINE_TIMESTAMP_MODULUS = 5 * 60 * 1000L;
private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L);
public static final long ONLINE_TIMESTAMP_MODULUS_V1 = 5 * 60 * 1000L;
public static final long ONLINE_TIMESTAMP_MODULUS_V2 = 30 * 60 * 1000L;
/** How many (latest) blocks' worth of online accounts we cache */
private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2;
private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L;
private static final long ONLINE_ACCOUNTS_V3_PEER_VERSION = 0x0300030000L;
private long onlineAccountsTasksTimestamp = Controller.startTime + ONLINE_ACCOUNTS_TASKS_INTERVAL; // ms
@ -116,6 +129,13 @@ public class OnlineAccountsManager extends Thread {
this.interrupt();
}
public static long getOnlineTimestampModulus() {
if (NTP.getTime() >= BlockChain.getInstance().getOnlineAccountsModulusV2Timestamp()) {
return ONLINE_TIMESTAMP_MODULUS_V2;
}
return ONLINE_TIMESTAMP_MODULUS_V1;
}
// Online accounts import queue
@ -159,7 +179,7 @@ public class OnlineAccountsManager extends Thread {
PublicKeyAccount otherAccount = new PublicKeyAccount(repository, onlineAccountData.getPublicKey());
// Check timestamp is 'recent' here
if (Math.abs(onlineAccountData.getTimestamp() - now) > ONLINE_TIMESTAMP_MODULUS * 2) {
if (Math.abs(onlineAccountData.getTimestamp() - now) > getOnlineTimestampModulus() * 2) {
LOGGER.trace(() -> String.format("Rejecting online account %s with out of range timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp()));
return;
}
@ -186,8 +206,16 @@ public class OnlineAccountsManager extends Thread {
return;
}
// Validate mempow if feature trigger is active
if (now >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) {
if (!this.verifyMemoryPoW(onlineAccountData)) {
LOGGER.trace(() -> String.format("Rejecting online reward-share for account %s due to invalid PoW nonce", mintingAccount.getAddress()));
return;
}
}
synchronized (this.onlineAccounts) {
OnlineAccountData existingAccountData = this.onlineAccounts.stream().filter(account -> Arrays.equals(account.getPublicKey(), onlineAccountData.getPublicKey())).findFirst().orElse(null);
OnlineAccountData existingAccountData = this.onlineAccounts.stream().filter(account -> Arrays.equals(account.getPublicKey(), onlineAccountData.getPublicKey())).findFirst().orElse(null); // CME??
if (existingAccountData != null) {
if (existingAccountData.getTimestamp() < onlineAccountData.getTimestamp()) {
@ -203,10 +231,53 @@ public class OnlineAccountsManager extends Thread {
LOGGER.trace(() -> String.format("Added online account %s with timestamp %d", otherAccount.getAddress(), onlineAccountData.getTimestamp()));
}
// Remove existing version of this online account data if the new one is superior
if (isOnlineAccountsDataSuperior(onlineAccountData)) {
this.onlineAccounts.remove(onlineAccountData);
}
this.onlineAccounts.add(onlineAccountData);
}
}
/**
* Check if supplied onlineAccountData is superior (i.e. has a nonce value) than existing record.
* Two entries are considered equal even if the nonce and block signature differ, to prevent
* multiple variations co-existing. For this reason, we need to be able to check
* if a new OnlineAccountData should replace the existing one, which may be missing the nonce.
* @param onlineAccountData
* @return
*/
private boolean isOnlineAccountsDataSuperior(OnlineAccountData onlineAccountData) {
if (onlineAccountData.getNonces() == null || onlineAccountData.getNonces().isEmpty()) {
// New online account data has no nonce value(s), so it won't be better than anything we already have
return false;
}
// New online account data has nonce value(s), so we need to check if the existing one does
OnlineAccountData existingOnlineAccountData = null;
for (OnlineAccountData acc : this.onlineAccounts) {
if (acc.equals(onlineAccountData)) {
// Found existing online account data
existingOnlineAccountData = acc;
break;
}
}
if (existingOnlineAccountData == null) {
// No existing online accounts data, so nothing to compare
return false;
}
if (existingOnlineAccountData.getNonces() == null || existingOnlineAccountData.getNonces().isEmpty()) {
// Existing data has no nonce value(s) so we want to replace it with the new one
return true;
}
// Both new and old data have nonce values so the new data isn't considered superior
return false;
}
public void ensureTestingAccountsOnline(PrivateKeyAccount... onlineAccounts) {
if (!BlockChain.getInstance().isTestChain()) {
LOGGER.warn("Ignoring attempt to ensure test account is online for non-test chain!");
@ -218,21 +289,21 @@ public class OnlineAccountsManager extends Thread {
return;
final long onlineAccountsTimestamp = toOnlineAccountTimestamp(now);
byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
List<MintingAccountData> mintingAccounts = new ArrayList<>();
synchronized (this.onlineAccounts) {
this.onlineAccounts.clear();
for (PrivateKeyAccount onlineAccount : onlineAccounts) {
// Check mintingAccount is actually reward-share?
byte[] signature = onlineAccount.sign(timestampBytes);
byte[] publicKey = onlineAccount.getPublicKey();
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey);
this.onlineAccounts.add(ourOnlineAccountData);
}
}
for (PrivateKeyAccount onlineAccount : onlineAccounts) {
// Check mintingAccount is actually reward-share?
MintingAccountData mintingAccountData = new MintingAccountData(onlineAccount.getPrivateKey(), onlineAccount.getPublicKey());
mintingAccounts.add(mintingAccountData);
}
computeOurAccountsForTimestamp(mintingAccounts, onlineAccountsTimestamp);
}
private void performOnlineAccountsTasks() {
@ -241,7 +312,8 @@ public class OnlineAccountsManager extends Thread {
return;
// Expire old entries
final long cutoffThreshold = now - LAST_SEEN_EXPIRY_PERIOD;
final long lastSeenExpiryPeriod = (getOnlineTimestampModulus() * 2) + (1 * 60 * 1000L);
final long cutoffThreshold = now - lastSeenExpiryPeriod;
synchronized (this.onlineAccounts) {
Iterator<OnlineAccountData> iterator = this.onlineAccounts.iterator();
while (iterator.hasNext()) {
@ -265,12 +337,7 @@ public class OnlineAccountsManager extends Thread {
safeOnlineAccounts = new ArrayList<>(this.onlineAccounts);
}
Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts);
Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts);
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
);
Network.getInstance().broadcast(peer -> new GetOnlineAccountsV2Message(safeOnlineAccounts));
}
}
@ -280,6 +347,12 @@ public class OnlineAccountsManager extends Thread {
return;
}
// If we're not up-to-date, then there's no point in computing anything yet
// The exception being when we are in recovery mode, in which case we need some online accounts!
if (!Controller.getInstance().isUpToDate() && !Synchronizer.getInstance().getRecoveryMode()) {
return;
}
List<MintingAccountData> mintingAccounts;
try (final Repository repository = RepositoryManager.getRepository()) {
mintingAccounts = repository.getAccountRepository().getMintingAccounts();
@ -318,61 +391,206 @@ public class OnlineAccountsManager extends Thread {
return;
}
// 'current' timestamp
// 'next' timestamp (prioritize this as it's the most important)
final long nextOnlineAccountsTimestamp = toOnlineAccountTimestamp(now) + getOnlineTimestampModulus();
boolean success = computeOurAccountsForTimestamp(mintingAccounts, nextOnlineAccountsTimestamp);
if (!success) {
// We didn't compute the required nonce value(s), and so can't proceed until they have been retried
return;
}
// 'current' timestamp (if there's enough time after successfully computing the 'next' timestamps)
final long onlineAccountsTimestamp = toOnlineAccountTimestamp(now);
boolean hasInfoChanged = false;
computeOurAccountsForTimestamp(mintingAccounts, onlineAccountsTimestamp);
}
byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
/**
* Compute a mempow nonce and signature for a given set of accounts and timestamp
* @param mintingAccounts - the online accounts
* @param onlineAccountsTimestamp - the online accounts timestamp
*/
private boolean computeOurAccountsForTimestamp(List<MintingAccountData> mintingAccounts, long onlineAccountsTimestamp) {
try (final Repository repository = RepositoryManager.getRepository()) {
MINTING_ACCOUNTS:
for (MintingAccountData mintingAccountData : mintingAccounts) {
PrivateKeyAccount mintingAccount = new PrivateKeyAccount(null, mintingAccountData.getPrivateKey());
boolean hasInfoChanged = false;
byte[] signature = mintingAccount.sign(timestampBytes);
byte[] publicKey = mintingAccount.getPublicKey();
final long currentOnlineAccountsTimestamp = toOnlineAccountTimestamp(NTP.getTime());
// Our account is online
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey);
synchronized (this.onlineAccounts) {
Iterator<OnlineAccountData> iterator = this.onlineAccounts.iterator();
List<OnlineAccountData> ourOnlineAccounts = new ArrayList<>();
MINTING_ACCOUNTS:
for (MintingAccountData mintingAccountData : mintingAccounts) {
PrivateKeyAccount mintingAccount = new PrivateKeyAccount(null, mintingAccountData.getPrivateKey());
byte[] publicKey = mintingAccount.getPublicKey();
// Our account is online
List<OnlineAccountData> safeOnlineAccounts;
synchronized (this.onlineAccounts) {
safeOnlineAccounts = new ArrayList<>(this.onlineAccounts);
}
Iterator<OnlineAccountData> iterator = safeOnlineAccounts.iterator();
while (iterator.hasNext()) {
OnlineAccountData existingOnlineAccountData = iterator.next();
if (Arrays.equals(existingOnlineAccountData.getPublicKey(), ourOnlineAccountData.getPublicKey())) {
if (Arrays.equals(existingOnlineAccountData.getPublicKey(), publicKey)) {
// If our online account is already present, with same timestamp, then move on to next mintingAccount
if (existingOnlineAccountData.getTimestamp() == onlineAccountsTimestamp)
continue MINTING_ACCOUNTS;
// If our online account is already present, but with older timestamp, then remove it
iterator.remove();
break;
if (existingOnlineAccountData.getTimestamp() < currentOnlineAccountsTimestamp) {
this.onlineAccounts.remove(existingOnlineAccountData); // Safe because we are iterating through a copy
}
}
}
this.onlineAccounts.add(ourOnlineAccountData);
// We need to add a new account
byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
int chainHeight = repository.getBlockRepository().getBlockchainHeight();
int referenceHeight = Math.max(1, chainHeight - 10);
BlockData recentBlockData = repository.getBlockRepository().fromHeight(referenceHeight);
if (recentBlockData == null || recentBlockData.getSignature() == null) {
LOGGER.info("Unable to compute online accounts without having a recent block");
return false;
}
byte[] reducedRecentBlockSignature = Arrays.copyOfRange(recentBlockData.getSignature(), 0, REDUCED_SIGNATURE_LENGTH);
byte[] mempowBytes;
try {
mempowBytes = this.getMemoryPoWBytes(publicKey, onlineAccountsTimestamp, reducedRecentBlockSignature);
}
catch (IOException e) {
LOGGER.info("Unable to create bytes for MemoryPoW. Moving on to next account...");
continue MINTING_ACCOUNTS;
}
Integer nonce;
if (isMemoryPoWActive()) {
try {
nonce = this.computeMemoryPoW(mempowBytes, publicKey, onlineAccountsTimestamp);
if (nonce == null) {
// A nonce is required
return false;
}
} catch (TimeoutException e) {
LOGGER.info(String.format("Timed out computing nonce for account %.8s", Base58.encode(publicKey)));
return false;
}
}
else {
// Send zero if we haven't computed a nonce due to feature trigger timestamp
nonce = 0;
}
byte[] signature = mintingAccount.sign(timestampBytes); // TODO: include nonce and block signature?
OnlineAccountData ourOnlineAccountData = new OnlineAccountData(onlineAccountsTimestamp, signature, publicKey, Arrays.asList(nonce), reducedRecentBlockSignature);
// Make sure to verify before adding
if (verifyMemoryPoW(ourOnlineAccountData)) {
this.onlineAccounts.add(ourOnlineAccountData);
LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", mintingAccount.getAddress(), onlineAccountsTimestamp));
ourOnlineAccounts.add(ourOnlineAccountData);
hasInfoChanged = true;
}
}
LOGGER.trace(() -> String.format("Added our online account %s with timestamp %d", mintingAccount.getAddress(), onlineAccountsTimestamp));
ourOnlineAccounts.add(ourOnlineAccountData);
hasInfoChanged = true;
if (!hasInfoChanged) {
// Nothing to do
return true;
}
Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts);
Message messageV3 = new OnlineAccountsV3Message(ourOnlineAccounts);
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V3_PEER_VERSION ? messageV3 : messageV2
);
LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp));
return true;
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while computing online accounts"), e);
return false;
}
}
private byte[] getMemoryPoWBytes(byte[] publicKey, long onlineAccountsTimestamp, byte[] reducedRecentBlockSignature) throws IOException {
byte[] timestampBytes = Longs.toByteArray(onlineAccountsTimestamp);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
outputStream.write(publicKey);
outputStream.write(timestampBytes);
outputStream.write(reducedRecentBlockSignature);
return outputStream.toByteArray();
}
private Integer computeMemoryPoW(byte[] bytes, byte[] publicKey, long onlineAccountsTimestamp) throws TimeoutException {
if (!isMemoryPoWActive()) {
LOGGER.info("Mempow start timestamp not yet reached, and onlineAccountsMemPoWEnabled not enabled in settings");
return null;
}
if (!hasInfoChanged)
return;
LOGGER.info(String.format("Computing nonce for account %.8s and timestamp %d...", Base58.encode(publicKey), onlineAccountsTimestamp));
Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts);
Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts);
// Calculate the time until the next online timestamp and use it as a timeout when computing the nonce
Long startTime = NTP.getTime();
final long nextOnlineAccountsTimestamp = toOnlineAccountTimestamp(startTime) + getOnlineTimestampModulus();
long timeUntilNextTimestamp = nextOnlineAccountsTimestamp - startTime;
Network.getInstance().broadcast(peer ->
peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
);
Integer nonce = MemoryPoW.compute2(bytes, POW_BUFFER_SIZE, POW_DIFFICULTY, timeUntilNextTimestamp);
LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp));
double totalSeconds = (NTP.getTime() - startTime) / 1000.0f;
int minutes = (int) ((totalSeconds % 3600) / 60);
int seconds = (int) (totalSeconds % 60);
double hashRate = nonce / totalSeconds;
LOGGER.info(String.format("Computed nonce for timestamp %d and account %.8s: %d. Buffer size: %d. Difficulty: %d. " +
"Time taken: %02d:%02d. Hashrate: %f", onlineAccountsTimestamp, Base58.encode(publicKey),
nonce, POW_BUFFER_SIZE, POW_DIFFICULTY, minutes, seconds, hashRate));
return nonce;
}
public boolean verifyMemoryPoW(OnlineAccountData onlineAccountData) {
List<Integer> nonces = onlineAccountData.getNonces();
if (nonces == null || nonces.isEmpty()) {
// Missing required nonce value(s)
return false;
}
if (nonces.size() > MAX_NONCE_COUNT) {
// More than the allowed nonce count
return false;
}
byte[] reducedBlockSignature = onlineAccountData.getReducedBlockSignature();
if (reducedBlockSignature == null) {
// Missing required block signature
return false;
}
byte[] mempowBytes;
try {
mempowBytes = this.getMemoryPoWBytes(onlineAccountData.getPublicKey(), onlineAccountData.getTimestamp(), reducedBlockSignature);
} catch (IOException e) {
return false;
}
// For now, we will only require a single nonce
int nonce = nonces.get(0);
// Verify the nonce
return MemoryPoW.verify2(mempowBytes, POW_BUFFER_SIZE, POW_DIFFICULTY, nonce);
}
public static long toOnlineAccountTimestamp(long timestamp) {
return (timestamp / ONLINE_TIMESTAMP_MODULUS) * ONLINE_TIMESTAMP_MODULUS;
return (timestamp / getOnlineTimestampModulus()) * getOnlineTimestampModulus();
}
/** Returns list of online accounts with timestamp recent enough to be considered currently online. */
@ -411,56 +629,17 @@ public class OnlineAccountsManager extends Thread {
}
}
private boolean isMemoryPoWActive() {
Long now = NTP.getTime();
if (now < BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp() || Settings.getInstance().isOnlineAccountsMemPoWEnabled()) {
return false;
}
return true;
}
// Network handlers
public void onNetworkGetOnlineAccountsMessage(Peer peer, Message message) {
GetOnlineAccountsMessage getOnlineAccountsMessage = (GetOnlineAccountsMessage) message;
List<OnlineAccountData> excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts();
// Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts
List<OnlineAccountData> accountsToSend;
synchronized (this.onlineAccounts) {
accountsToSend = new ArrayList<>(this.onlineAccounts);
}
Iterator<OnlineAccountData> iterator = accountsToSend.iterator();
SEND_ITERATOR:
while (iterator.hasNext()) {
OnlineAccountData onlineAccountData = iterator.next();
for (int i = 0; i < excludeAccounts.size(); ++i) {
OnlineAccountData excludeAccountData = excludeAccounts.get(i);
if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) {
iterator.remove();
continue SEND_ITERATOR;
}
}
}
Message onlineAccountsMessage = new OnlineAccountsMessage(accountsToSend);
peer.sendMessage(onlineAccountsMessage);
LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer));
}
public void onNetworkOnlineAccountsMessage(Peer peer, Message message) {
OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message;
List<OnlineAccountData> peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts();
LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer));
try (final Repository repository = RepositoryManager.getRepository()) {
for (OnlineAccountData onlineAccountData : peersOnlineAccounts)
this.verifyAndAddAccount(repository, onlineAccountData);
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e);
}
}
public void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) {
GetOnlineAccountsV2Message getOnlineAccountsMessage = (GetOnlineAccountsV2Message) message;
@ -488,8 +667,10 @@ public class OnlineAccountsManager extends Thread {
}
}
Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend);
peer.sendMessage(onlineAccountsMessage);
Message messageV2 = new OnlineAccountsV2Message(accountsToSend);
Message messageV3 = new OnlineAccountsV3Message(accountsToSend);
peer.sendMessage(peer.getPeersVersion() >= ONLINE_ACCOUNTS_V3_PEER_VERSION ? messageV3 : messageV2);
LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer));
}
@ -502,6 +683,39 @@ public class OnlineAccountsManager extends Thread {
int importCount = 0;
// Add any online accounts to the queue that aren't already present
for (OnlineAccountData onlineAccountData : peersOnlineAccounts) {
// Do we already know about this online account data?
if (onlineAccounts.contains(onlineAccountData)) {
// Don't import if it's no better than the one we already have
if (!isOnlineAccountsDataSuperior(onlineAccountData)) {
// Do NOT remove the existing online account data - this takes place after validation
continue;
}
}
// Is it already in the import queue?
if (onlineAccountsImportQueue.contains(onlineAccountData)) {
continue;
}
onlineAccountsImportQueue.add(onlineAccountData);
importCount++;
}
LOGGER.debug(String.format("Added %d online accounts to queue", importCount));
}
public void onNetworkOnlineAccountsV3Message(Peer peer, Message message) {
OnlineAccountsV3Message onlineAccountsMessage = (OnlineAccountsV3Message) message;
List<OnlineAccountData> peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts();
LOGGER.debug(String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer));
int importCount = 0;
// Add any online accounts to the queue that aren't already present
for (OnlineAccountData onlineAccountData : peersOnlineAccounts) {

View File

@ -1,10 +1,25 @@
package org.qortal.crypto;
import org.qortal.utils.NTP;
import java.nio.ByteBuffer;
import java.util.concurrent.TimeoutException;
public class MemoryPoW {
public static Integer compute2(byte[] data, int workBufferLength, long difficulty) {
try {
return MemoryPoW.compute2(data, workBufferLength, difficulty, null);
} catch (TimeoutException e) {
// This won't happen, because above timeout is null
return null;
}
}
public static Integer compute2(byte[] data, int workBufferLength, long difficulty, Long timeout) throws TimeoutException {
long startTime = NTP.getTime();
// Hash data with SHA256
byte[] hash = Crypto.digest(data);
@ -33,6 +48,13 @@ public class MemoryPoW {
if (Thread.currentThread().isInterrupted())
return -1;
if (timeout != null) {
long now = NTP.getTime();
if (now > startTime + timeout) {
throw new TimeoutException("Timeout reached");
}
}
seed *= seedMultiplier; // per nonce
state[0] = longHash[0] ^ seed;

View File

@ -1,6 +1,7 @@
package org.qortal.data.network;
import java.util.Arrays;
import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@ -15,6 +16,8 @@ public class OnlineAccountData {
protected long timestamp;
protected byte[] signature;
protected byte[] publicKey;
protected List<Integer> nonces;
protected byte[] reducedBlockSignature;
// Constructors
@ -22,10 +25,16 @@ public class OnlineAccountData {
protected OnlineAccountData() {
}
public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey) {
public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey, List<Integer> nonces, byte[] reducedBlockSignature) {
this.timestamp = timestamp;
this.signature = signature;
this.publicKey = publicKey;
this.nonces = nonces;
this.reducedBlockSignature = reducedBlockSignature;
}
public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey) {
this(timestamp, signature, publicKey, null, null);
}
public long getTimestamp() {
@ -40,6 +49,14 @@ public class OnlineAccountData {
return this.publicKey;
}
public List<Integer> getNonces() {
return this.nonces;
}
public byte[] getReducedBlockSignature() {
return this.reducedBlockSignature;
}
// For JAXB
@XmlElement(name = "address")
protected String getAddress() {
@ -69,6 +86,8 @@ public class OnlineAccountData {
if (!Arrays.equals(otherOnlineAccountData.publicKey, this.publicKey))
return false;
// Best not to consider additional properties for the purposes of uniqueness
return true;
}

View File

@ -1,69 +0,0 @@
package org.qortal.network.message;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.transform.Transformer;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
public class GetOnlineAccountsMessage extends Message {
private static final int MAX_ACCOUNT_COUNT = 5000;
private List<OnlineAccountData> onlineAccounts;
public GetOnlineAccountsMessage(List<OnlineAccountData> onlineAccounts) {
super(MessageType.GET_ONLINE_ACCOUNTS);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try {
bytes.write(Ints.toByteArray(onlineAccounts.size()));
for (OnlineAccountData onlineAccountData : onlineAccounts) {
bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp()));
bytes.write(onlineAccountData.getPublicKey());
}
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private GetOnlineAccountsMessage(int id, List<OnlineAccountData> onlineAccounts) {
super(id, MessageType.GET_ONLINE_ACCOUNTS);
this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList());
}
public List<OnlineAccountData> getOnlineAccounts() {
return this.onlineAccounts;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) {
final int accountCount = bytes.getInt();
List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) {
long timestamp = bytes.getLong();
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKey);
onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey));
}
return new GetOnlineAccountsMessage(id, onlineAccounts);
}
}

View File

@ -38,10 +38,9 @@ public enum MessageType {
BLOCK_SUMMARIES(70, BlockSummariesMessage::fromByteBuffer),
GET_BLOCK_SUMMARIES(71, GetBlockSummariesMessage::fromByteBuffer),
ONLINE_ACCOUNTS(80, OnlineAccountsMessage::fromByteBuffer),
GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer),
ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer),
GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer),
ONLINE_ACCOUNTS_V3(84, OnlineAccountsV3Message::fromByteBuffer),
ARBITRARY_DATA(90, ArbitraryDataMessage::fromByteBuffer),
GET_ARBITRARY_DATA(91, GetArbitraryDataMessage::fromByteBuffer),

View File

@ -1,75 +0,0 @@
package org.qortal.network.message;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.transform.Transformer;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
public class OnlineAccountsMessage extends Message {
private static final int MAX_ACCOUNT_COUNT = 5000;
private List<OnlineAccountData> onlineAccounts;
public OnlineAccountsMessage(List<OnlineAccountData> onlineAccounts) {
super(MessageType.ONLINE_ACCOUNTS);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try {
bytes.write(Ints.toByteArray(onlineAccounts.size()));
for (OnlineAccountData onlineAccountData : onlineAccounts) {
bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp()));
bytes.write(onlineAccountData.getSignature());
bytes.write(onlineAccountData.getPublicKey());
}
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private OnlineAccountsMessage(int id, List<OnlineAccountData> onlineAccounts) {
super(id, MessageType.ONLINE_ACCOUNTS);
this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList());
}
public List<OnlineAccountData> getOnlineAccounts() {
return this.onlineAccounts;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) {
final int accountCount = bytes.getInt();
List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) {
long timestamp = bytes.getLong();
byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
bytes.get(signature);
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKey);
OnlineAccountData onlineAccountData = new OnlineAccountData(timestamp, signature, publicKey);
onlineAccounts.add(onlineAccountData);
}
return new OnlineAccountsMessage(id, onlineAccounts);
}
}

View File

@ -0,0 +1,134 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.transform.Transformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* For sending online accounts info to remote peer.
*
* Same format as V2, but with added support for mempow nonce values and a recent block signature
*/
public class OnlineAccountsV3Message extends Message {
private List<OnlineAccountData> onlineAccounts;
private byte[] cachedData;
public OnlineAccountsV3Message(List<OnlineAccountData> onlineAccounts) {
super(MessageType.ONLINE_ACCOUNTS_V3);
// If we don't have ANY online accounts then it's an easier construction...
if (onlineAccounts.isEmpty()) {
// Always supply a number of accounts
this.dataBytes = Ints.toByteArray(0);
this.checksumBytes = Message.generateChecksum(this.dataBytes);
return;
}
// How many of each timestamp
Map<Long, Integer> countByTimestamp = new HashMap<>();
for (int i = 0; i < onlineAccounts.size(); ++i) {
OnlineAccountData onlineAccountData = onlineAccounts.get(i);
Long timestamp = onlineAccountData.getTimestamp();
countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
}
// We should know exactly how many bytes to allocate now
int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+ onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH);
ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
try {
for (long timestamp : countByTimestamp.keySet()) {
bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
bytes.write(Longs.toByteArray(timestamp));
for (int i = 0; i < onlineAccounts.size(); ++i) {
OnlineAccountData onlineAccountData = onlineAccounts.get(i);
if (onlineAccountData.getTimestamp() == timestamp) {
bytes.write(onlineAccountData.getSignature());
bytes.write(onlineAccountData.getPublicKey());
bytes.write(onlineAccountData.getReducedBlockSignature());
int nonceCount = onlineAccountData.getNonces() != null ? onlineAccountData.getNonces().size() : 0;
bytes.write(Ints.toByteArray(nonceCount));
for (int n = 0; n < nonceCount; ++n) {
int nonce = onlineAccountData.getNonces().get(n);
bytes.write(Ints.toByteArray(nonce));
}
}
}
}
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private OnlineAccountsV3Message(int id, List<OnlineAccountData> onlineAccounts) {
super(id, MessageType.ONLINE_ACCOUNTS_V2);
this.onlineAccounts = onlineAccounts;
}
public List<OnlineAccountData> getOnlineAccounts() {
return this.onlineAccounts;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) {
int accountCount = bytes.getInt();
List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
while (accountCount > 0) {
long timestamp = bytes.getLong();
for (int i = 0; i < accountCount; ++i) {
byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
bytes.get(signature);
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKey);
byte[] reducedBlockSignature = new byte[Transformer.REDUCED_SIGNATURE_LENGTH];
bytes.get(reducedBlockSignature);
int nonceCount = bytes.getInt();
List<Integer> nonces = new ArrayList<>();
for (int n = 0; n < nonceCount; ++n) {
Integer nonce = bytes.getInt();
nonces.add(nonce);
}
onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey, nonces, reducedBlockSignature));
}
if (bytes.hasRemaining()) {
accountCount = bytes.getInt();
} else {
// we've finished
accountCount = 0;
}
}
return new OnlineAccountsV3Message(id, onlineAccounts);
}
}

View File

@ -277,6 +277,11 @@ public class Settings {
/** Additional offset added to values returned by NTP.getTime() */
private Long testNtpOffset = null;
// Online accounts
/** Whether to opt-in to mempow computations for online accounts, ahead of general release */
private boolean onlineAccountsMemPoWEnabled = false;
// Data storage (QDN)
@ -752,6 +757,10 @@ public class Settings {
return this.testNtpOffset;
}
public boolean isOnlineAccountsMemPoWEnabled() {
return this.onlineAccountsMemPoWEnabled;
}
public long getRepositoryBackupInterval() {
return this.repositoryBackupInterval;
}

View File

@ -12,7 +12,6 @@ import java.util.function.Supplier;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.account.Account;
import org.qortal.controller.Controller;
import org.qortal.controller.OnlineAccountsManager;
import org.qortal.controller.tradebot.TradeBot;
import org.qortal.crosschain.ACCT;
@ -49,7 +48,7 @@ public class PresenceTransaction extends Transaction {
REWARD_SHARE(0) {
@Override
public long getLifetime() {
return OnlineAccountsManager.ONLINE_TIMESTAMP_MODULUS;
return OnlineAccountsManager.getOnlineTimestampModulus();
}
},
TRADE_BOT(1) {

View File

@ -18,6 +18,8 @@ public abstract class Transformer {
public static final int SIGNATURE_LENGTH = 64;
public static final int TIMESTAMP_LENGTH = LONG_LENGTH;
public static final int REDUCED_SIGNATURE_LENGTH = 4;
public static final int MD5_LENGTH = 16;
public static final int SHA256_LENGTH = 32;
public static final int AES256_LENGTH = 32;

View File

@ -6,11 +6,13 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.qortal.block.Block;
import org.qortal.block.BlockChain;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.DataException;
import org.qortal.transaction.Transaction;
@ -27,6 +29,8 @@ import com.google.common.primitives.Longs;
import io.druid.extendedset.intset.ConciseSet;
import static org.qortal.controller.OnlineAccountsManager.MAX_NONCE_COUNT;
public class BlockTransformer extends Transformer {
private static final int VERSION_LENGTH = INT_LENGTH;
@ -213,7 +217,7 @@ public class BlockTransformer extends Transformer {
// Online accounts timestamp is only present if there are also signatures
onlineAccountsTimestamp = byteBuffer.getLong();
final int signaturesByteLength = onlineAccountsSignaturesCount * Transformer.SIGNATURE_LENGTH;
final int signaturesByteLength = Block.getExpectedOnlineAccountsSignaturesLength(onlineAccountsSignaturesCount, timestamp);
if (signaturesByteLength > BlockChain.getInstance().getMaxBlockSize())
throw new TransformationException("Byte data too long for online accounts signatures");
@ -416,16 +420,101 @@ public class BlockTransformer extends Transformer {
return encodedSignatures;
}
public static List<byte[]> decodeTimestampSignatures(byte[] encodedSignatures) {
List<byte[]> signatures = new ArrayList<>();
public static byte[] encodeOnlineAccountSignatures(Map<Integer, OnlineAccountData> indexedOnlineAccounts,
List<Integer> accountIndexes,
int onlineAccountsCount,
long timestamp) {
byte[] onlineAccountsSignatures;
for (int i = 0; i < encodedSignatures.length; i += Transformer.SIGNATURE_LENGTH) {
byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
System.arraycopy(encodedSignatures, i, signature, 0, Transformer.SIGNATURE_LENGTH);
signatures.add(signature);
if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) {
// Online accounts must include at least one nonce and a reduced block signature from this time onwards
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
for (int i = 0; i < onlineAccountsCount; ++i) {
Integer accountIndex = accountIndexes.get(i);
OnlineAccountData onlineAccountData = indexedOnlineAccounts.get(accountIndex);
List<Integer> nonces = onlineAccountData.getNonces();
byte[] reducedBlockSignature = onlineAccountData.getReducedBlockSignature();
if (nonces == null || nonces.isEmpty() || nonces.size() > MAX_NONCE_COUNT || reducedBlockSignature == null) {
// Missing or invalid data, so exclude this online account
continue;
}
try {
outputStream.write(onlineAccountData.getSignature());
outputStream.write(reducedBlockSignature);
outputStream.write(Ints.toByteArray(nonces.size()));
for (int n = 0; n < nonces.size(); ++n) {
Integer nonce = nonces.get(n);
outputStream.write(Ints.toByteArray(nonce));
}
} catch (IOException e) {
// Couldn't serialize this online account, so exclude it
continue;
}
}
onlineAccountsSignatures = outputStream.toByteArray();
}
else {
// Exclude nonce and reference block signature from online accounts data
// Concatenate online account timestamp signatures (in correct order)
onlineAccountsSignatures = new byte[onlineAccountsCount * Transformer.SIGNATURE_LENGTH];
for (int i = 0; i < onlineAccountsCount; ++i) {
Integer accountIndex = accountIndexes.get(i);
OnlineAccountData onlineAccountData = indexedOnlineAccounts.get(accountIndex);
System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH);
}
}
return signatures;
return onlineAccountsSignatures;
}
public static List<OnlineAccountData> decodeOnlineAccountSignatures(byte[] encodedSignatures, int count, long timestamp) {
List<OnlineAccountData> onlineAccountSignatures = new ArrayList<>();
if (timestamp >= BlockChain.getInstance().getOnlineAccountsMemoryPoWTimestamp()) {
// byte array contains signatures, reduced signatures, and nonces
ByteBuffer byteBuffer = ByteBuffer.wrap(encodedSignatures);
for (int i = 0; i < count; ++i) {
byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
byteBuffer.get(signature);
byte[] reducedBlockSignature = new byte[Transformer.REDUCED_SIGNATURE_LENGTH];
byteBuffer.get(reducedBlockSignature);
int nonceCount = byteBuffer.getInt();
List<Integer> nonces = new ArrayList<>();
for (int n = 0; n < nonceCount; ++n) { // TODO: check against NONCE_COUNT in block validation
Integer nonce = byteBuffer.getInt();
nonces.add(nonce);
}
// Create an OnlineAccountData wrapper object containing the signature, nonce(s), and reduced block signature
OnlineAccountData onlineAccountDataWrapper = new OnlineAccountData(0, signature, null, nonces, reducedBlockSignature);
onlineAccountSignatures.add(onlineAccountDataWrapper);
}
}
else {
// byte array contains signatures only
for (int i = 0; i < encodedSignatures.length; i += Transformer.SIGNATURE_LENGTH) {
byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
System.arraycopy(encodedSignatures, i, signature, 0, Transformer.SIGNATURE_LENGTH);
// Create an OnlineAccountData wrapper object containing only the signature
OnlineAccountData onlineAccountDataWrapper = new OnlineAccountData(0, signature, null);
onlineAccountSignatures.add(onlineAccountDataWrapper);
}
}
return onlineAccountSignatures;
}
}

View File

@ -19,6 +19,8 @@
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 43200000,
"onlineAccountSignaturesMaxLifetime": 86400000,
"onlineAccountsModulusV2Timestamp": 9999999999999,
"onlineAccountsMemoryPoWTimestamp": 9999999999999,
"rewardsByHeight": [
{ "height": 1, "reward": 5.00 },
{ "height": 259201, "reward": 4.75 },

View File

@ -1,22 +1,36 @@
package org.qortal.test.network;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.block.Block;
import org.qortal.block.BlockChain;
import org.qortal.controller.BlockMinter;
import org.qortal.controller.OnlineAccountsManager;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.network.message.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.test.common.Common;
import org.qortal.transform.Transformer;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.Security;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.*;
public class OnlineAccountsTests {
public class OnlineAccountsTests extends Common {
private static final Random RANDOM = new Random();
static {
@ -27,6 +41,12 @@ public class OnlineAccountsTests {
Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
}
@Before
public void beforeTest() throws DataException, IOException {
Common.useSettingsAndDb(Common.testSettingsFilename, false);
NTP.setFixedOffset(Settings.getInstance().getTestNtpOffset());
}
@Test
public void testGetOnlineAccountsV2() throws MessageException {
@ -43,18 +63,6 @@ public class OnlineAccountsTests {
assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut);
byte[] oldMessageBytes = oldMessageOut.toBytes();
long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
onlineAccountsOut.size(),
numTimestamps,
numTimestamps != 1 ? "s" : "",
oldMessageBytes.length,
messageBytes.length));
}
@Test
@ -72,18 +80,6 @@ public class OnlineAccountsTests {
assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut);
byte[] oldMessageBytes = oldMessageOut.toBytes();
long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
onlineAccountsOut.size(),
numTimestamps,
numTimestamps != 1 ? "s" : "",
oldMessageBytes.length,
messageBytes.length));
}
private List<OnlineAccountData> generateOnlineAccounts(boolean withSignatures) {
@ -111,4 +107,136 @@ public class OnlineAccountsTests {
return onlineAccounts;
}
@Test
public void testOnlineAccountsModulusV1() throws IllegalAccessException, DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Set feature trigger timestamp to MAX long so that it is inactive
FieldUtils.writeField(BlockChain.getInstance(), "onlineAccountsModulusV2Timestamp", Long.MAX_VALUE, true);
List<String> onlineAccountSignatures = new ArrayList<>();
long fakeNTPOffset = 0L;
// Mint a block and store its timestamp
Block block = BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
long lastBlockTimestamp = block.getBlockData().getTimestamp();
// Mint some blocks and keep track of the different online account signatures
for (int i = 0; i < 30; i++) {
block = BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
// Increase NTP fixed offset by the block time, to simulate time passing
long blockTimeDelta = block.getBlockData().getTimestamp() - lastBlockTimestamp;
lastBlockTimestamp = block.getBlockData().getTimestamp();
fakeNTPOffset += blockTimeDelta;
NTP.setFixedOffset(fakeNTPOffset);
String lastOnlineAccountSignatures58 = Base58.encode(block.getBlockData().getOnlineAccountsSignatures());
if (!onlineAccountSignatures.contains(lastOnlineAccountSignatures58)) {
onlineAccountSignatures.add(lastOnlineAccountSignatures58);
}
}
// We expect at least 6 unique signatures over 30 blocks (generally 6-8, but could be higher due to block time differences)
System.out.println(String.format("onlineAccountSignatures count: %d", onlineAccountSignatures.size()));
assertTrue(onlineAccountSignatures.size() >= 6);
}
}
@Test
public void testOnlineAccountsModulusV2() throws IllegalAccessException, DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Set feature trigger timestamp to 0 so that it is active
FieldUtils.writeField(BlockChain.getInstance(), "onlineAccountsModulusV2Timestamp", 0L, true);
List<String> onlineAccountSignatures = new ArrayList<>();
long fakeNTPOffset = 0L;
// Mint a block and store its timestamp
Block block = BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
long lastBlockTimestamp = block.getBlockData().getTimestamp();
// Mint some blocks and keep track of the different online account signatures
for (int i = 0; i < 30; i++) {
block = BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
// Increase NTP fixed offset by the block time, to simulate time passing
long blockTimeDelta = block.getBlockData().getTimestamp() - lastBlockTimestamp;
lastBlockTimestamp = block.getBlockData().getTimestamp();
fakeNTPOffset += blockTimeDelta;
NTP.setFixedOffset(fakeNTPOffset);
String lastOnlineAccountSignatures58 = Base58.encode(block.getBlockData().getOnlineAccountsSignatures());
if (!onlineAccountSignatures.contains(lastOnlineAccountSignatures58)) {
onlineAccountSignatures.add(lastOnlineAccountSignatures58);
}
}
// We expect 1-3 unique signatures over 30 blocks
System.out.println(String.format("onlineAccountSignatures count: %d", onlineAccountSignatures.size()));
assertTrue(onlineAccountSignatures.size() >= 1 && onlineAccountSignatures.size() <= 3);
}
}
@Test
public void testBeforeMemoryPoW() throws IllegalAccessException, DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Set feature trigger timestamp to MAX long so that it is inactive
FieldUtils.writeField(BlockChain.getInstance(), "onlineAccountsMemoryPoWTimestamp", Long.MAX_VALUE, true);
// Mint some blocks
for (int i = 0; i < 10; i++) {
BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
}
}
}
@Test
public void testMemoryPoW() throws IllegalAccessException, DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Set feature trigger timestamp to 0 so that it is active
FieldUtils.writeField(BlockChain.getInstance(), "onlineAccountsMemoryPoWTimestamp", 0L, true);
// Set difficulty to 5, to speed up test
FieldUtils.writeField(OnlineAccountsManager.getInstance(), "POW_DIFFICULTY", 5, true);
// Mint some blocks
for (int i = 0; i < 10; i++) {
BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
}
}
}
@Test
public void testTransitionToMemoryPoW() throws IllegalAccessException, DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Set feature trigger timestamp to now + 5 mins
long featureTriggerTimestamp = NTP.getTime() + (5 * 60 * 1000L);
FieldUtils.writeField(BlockChain.getInstance(), "onlineAccountsMemoryPoWTimestamp", featureTriggerTimestamp, true);
// Set difficulty to 5, to speed up test
FieldUtils.writeField(OnlineAccountsManager.getInstance(), "POW_DIFFICULTY", 5, true);
// Mint a block
Block block = BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
assertEquals(1, block.getBlockData().getOnlineAccountsCount());
// Ensure online accounts signatures are in legacy format (no nonce or reduced block signature)
assertEquals(64, block.getBlockData().getOnlineAccountsSignatures().length);
// Mint some blocks (at least 5 minutes' worth, to allow mempow to kick in)
for (int i = 0; i < 10; i++) {
block = BlockMinter.mintTestingBlock(repository, Common.getTestAccount(repository, "alice-reward-share"));
assertEquals(1, block.getBlockData().getOnlineAccountsCount());
}
// Ensure online accounts signatures are in new format (with 1 nonce and a reduced block signature)
assertEquals(80, block.getBlockData().getOnlineAccountsSignatures().length);
}
}
}

View File

@ -14,6 +14,8 @@
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 3600000,
"onlineAccountSignaturesMaxLifetime": 86400000,
"onlineAccountsModulusV2Timestamp": 9999999999999,
"onlineAccountsMemoryPoWTimestamp": 9999999999999,
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },

View File

@ -14,6 +14,8 @@
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 3600000,
"onlineAccountSignaturesMaxLifetime": 86400000,
"onlineAccountsModulusV2Timestamp": 9999999999999,
"onlineAccountsMemoryPoWTimestamp": 9999999999999,
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },

View File

@ -14,6 +14,8 @@
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 3600000,
"onlineAccountSignaturesMaxLifetime": 86400000,
"onlineAccountsModulusV2Timestamp": 9999999999999,
"onlineAccountsMemoryPoWTimestamp": 9999999999999,
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },

View File

@ -14,6 +14,8 @@
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 3600000,
"onlineAccountSignaturesMaxLifetime": 86400000,
"onlineAccountsModulusV2Timestamp": 9999999999999,
"onlineAccountsMemoryPoWTimestamp": 9999999999999,
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },

View File

@ -14,6 +14,8 @@
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 3600000,
"onlineAccountSignaturesMaxLifetime": 86400000,
"onlineAccountsModulusV2Timestamp": 9999999999999,
"onlineAccountsMemoryPoWTimestamp": 9999999999999,
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },

View File

@ -14,6 +14,8 @@
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 3600000,
"onlineAccountSignaturesMaxLifetime": 86400000,
"onlineAccountsModulusV2Timestamp": 9999999999999,
"onlineAccountsMemoryPoWTimestamp": 9999999999999,
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },

View File

@ -14,6 +14,8 @@
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 3600000,
"onlineAccountSignaturesMaxLifetime": 86400000,
"onlineAccountsModulusV2Timestamp": 9999999999999,
"onlineAccountsMemoryPoWTimestamp": 9999999999999,
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },

View File

@ -14,6 +14,8 @@
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 3600000,
"onlineAccountSignaturesMaxLifetime": 86400000,
"onlineAccountsModulusV2Timestamp": 9999999999999,
"onlineAccountsMemoryPoWTimestamp": 9999999999999,
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },