Qortal/qortal
1
0
Fork 1
mirror of https://github.com/Qortal/qortal.git synced 2025-07-23 04:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

New synchronizer and other improvements

Browse Source 
API call GET /addresses/online reports online accounts,
including both addresses relating to the proxy-forge public key.

New PeerChainTipData class to replace the broken "peer data lock"
that was supposed to make sure peer's last height/blockSig/timestamp
were all in sync. Now peer's chain tip data is a single object
reference that can be replaced in one go.

Removed pointless API calls /blocks/time and /blocks/{generatingbalance}.

Various changes, mostly in Block class, to do with switching to BlockTimingByHeight
from old min/max block time.

New block 'weight' based on number of online accounts
and 'distance' of perturbed generator public key from 'ideal' public key
(for that particular block's height).

New sub-chain 'weight' based on accumulating block weights,
currently by shifting previous accumulator left by 8 bits then
adding next block's weight.

More validation of BlockChain config. Helpful for debugging, probably
not very useful to end-users.

BlockGenerator now uses unified Peer predicates from Controller, like:
Controller.hasMisbehaved, Controller.hasNoRecentBlock, etc.

Controller now keeps a list of chain-tip signatures that are for inferior
chains, so it doesn't try to synchronize with peers with inferior chains.
(This list is wiped when node's blockchain changes/block is generated).

Controller now asks Gui to display error box if it can't parse Settings.

Controller.potentiallySynchronize() does more filtering of potential peers
before calling actuallySynchronize(). (Mostly moved from Synchronizer,
so now we expect actuallySynchronize() to do something rather than bail
out because it doesn't like the peer after all).

If synchronization discovers that peer has an inferior chain,
then Controller notifies that peer of our superior chain, to help keep
the network in sync.

Renamed OnlineAccount to OnlineAccountData, as it is in package org.qora.data
after all...

Synchronizer reworked to request block summaries so it can judge which chain
is better, and hence whether to sync with peer or abort.

Slight optimization of Peer.readChannel() to exit earlier if no more network
messages can be extracted from buffer.

More tests.
Improved documentation and logging.
This commit is contained in:
catbref
2019-09-26 17:43:50 +01:00
parent c889e95da4
commit 4c6656dd17
18 changed files with 725 additions and 449 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
src/test/java/org/qora/test/ChainWeightTests.java Normal file
View File

@@ -0,0 +1,134 @@
package org.qora.test;
import static org.junit.Assert.*;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import org.qora.crypto.Crypto;
import org.qora.data.block.BlockSummaryData;
import org.qora.transform.Transformer;
import org.qora.transform.block.BlockTransformer;
import org.junit.Test;
import com.google.common.primitives.Bytes;
import com.google.common.primitives.Longs;
public class ChainWeightTests {
private static final int ACCOUNTS_COUNT_SHIFT = Transformer.PUBLIC_KEY_LENGTH * 8;
private static final int CHAIN_WEIGHT_SHIFT = 8;
private static final Random RANDOM = new Random();
private static final BigInteger MAX_DISTANCE;
static {
byte[] maxValue = new byte[Transformer.PUBLIC_KEY_LENGTH];
Arrays.fill(maxValue, (byte) 0xFF);
MAX_DISTANCE = new BigInteger(1, maxValue);
}
private static byte[] perturbPublicKey(int height, byte[] publicKey) {
return Crypto.digest(Bytes.concat(Longs.toByteArray(height), publicKey));
}
private static BigInteger calcKeyDistance(int parentHeight, byte[] parentGeneratorKey, byte[] publicKey) {
byte[] idealKey = perturbPublicKey(parentHeight, parentGeneratorKey);
byte[] perturbedKey = perturbPublicKey(parentHeight + 1, publicKey);
BigInteger keyDistance = MAX_DISTANCE.subtract(new BigInteger(idealKey).subtract(new BigInteger(perturbedKey)).abs());
return keyDistance;
}
private static BigInteger calcBlockWeight(int parentHeight, byte[] parentGeneratorKey, BlockSummaryData blockSummaryData) {
BigInteger keyDistance = calcKeyDistance(parentHeight, parentGeneratorKey, blockSummaryData.getGeneratorPublicKey());
BigInteger weight = BigInteger.valueOf(blockSummaryData.getOnlineAccountsCount()).shiftLeft(ACCOUNTS_COUNT_SHIFT).add(keyDistance);
return weight;
}
private static BigInteger calcChainWeight(int commonBlockHeight, byte[] commonBlockGeneratorKey, List<BlockSummaryData> blockSummaries) {
BigInteger cumulativeWeight = BigInteger.ZERO;
int parentHeight = commonBlockHeight;
byte[] parentGeneratorKey = commonBlockGeneratorKey;
for (BlockSummaryData blockSummaryData : blockSummaries) {
cumulativeWeight = cumulativeWeight.shiftLeft(CHAIN_WEIGHT_SHIFT).add(calcBlockWeight(parentHeight, parentGeneratorKey, blockSummaryData));
parentHeight = blockSummaryData.getHeight();
parentGeneratorKey = blockSummaryData.getGeneratorPublicKey();
}
return cumulativeWeight;
}
private static BlockSummaryData genBlockSummary(int height) {
byte[] generatorPublicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
RANDOM.nextBytes(generatorPublicKey);
byte[] signature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH];
RANDOM.nextBytes(signature);
int onlineAccountsCount = RANDOM.nextInt(1000);
return new BlockSummaryData(height, signature, generatorPublicKey, onlineAccountsCount);
}
private static List<BlockSummaryData> genBlockSummaries(int count, BlockSummaryData commonBlockSummary) {
List<BlockSummaryData> blockSummaries = new ArrayList<>();
blockSummaries.add(commonBlockSummary);
final int commonBlockHeight = commonBlockSummary.getHeight();
for (int i = 1; i <= count; ++i)
blockSummaries.add(genBlockSummary(commonBlockHeight + i));
return blockSummaries;
}
// Check that more online accounts beats a better key
@Test
public void testMoreAccountsBlock() {
final int parentHeight = 1;
final byte[] parentGeneratorKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
int betterAccountsCount = 100;
int worseAccountsCount = 20;
byte[] betterKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
betterKey[0] = 0x41;
byte[] worseKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
worseKey[0] = 0x23;
BigInteger betterKeyDistance = calcKeyDistance(parentHeight, parentGeneratorKey, betterKey);
BigInteger worseKeyDistance = calcKeyDistance(parentHeight, parentGeneratorKey, worseKey);
assertEquals("hard-coded keys are wrong", 1, betterKeyDistance.compareTo(worseKeyDistance));
BlockSummaryData betterBlockSummary = new BlockSummaryData(parentHeight + 1, null, worseKey, betterAccountsCount);
BlockSummaryData worseBlockSummary = new BlockSummaryData(parentHeight + 1, null, betterKey, worseAccountsCount);
BigInteger betterBlockWeight = calcBlockWeight(parentHeight, parentGeneratorKey, betterBlockSummary);
BigInteger worseBlockWeight = calcBlockWeight(parentHeight, parentGeneratorKey, worseBlockSummary);
assertEquals("block weights are wrong", 1, betterBlockWeight.compareTo(worseBlockWeight));
}
// Check that a longer chain beats a shorter chain
@Test
public void testLongerChain() {
final int commonBlockHeight = 1;
BlockSummaryData commonBlockSummary = genBlockSummary(commonBlockHeight);
byte[] commonBlockGeneratorKey = commonBlockSummary.getGeneratorPublicKey();
List<BlockSummaryData> shorterChain = genBlockSummaries(3, commonBlockSummary);
List<BlockSummaryData> longerChain = genBlockSummaries(shorterChain.size() + 1, commonBlockSummary);
BigInteger shorterChainWeight = calcChainWeight(commonBlockHeight, commonBlockGeneratorKey, shorterChain);
BigInteger longerChainWeight = calcChainWeight(commonBlockHeight, commonBlockGeneratorKey, longerChain);
assertEquals("longer chain should have greater weight", 1, longerChainWeight.compareTo(shorterChainWeight));
}
}

28
src/test/java/org/qora/test/OnlineTests.java
View File

@@ -13,7 +13,7 @@ import org.junit.Before;
import org.junit.Test;
import org.qora.account.PrivateKeyAccount;
import org.qora.account.PublicKeyAccount;
import org.qora.data.network.OnlineAccount;
import org.qora.data.network.OnlineAccountData;
import org.qora.network.message.GetOnlineAccountsMessage;
import org.qora.network.message.Message;
import org.qora.network.message.OnlineAccountsMessage;
@@ -48,7 +48,7 @@ public class OnlineTests extends Common {
private final PrivateKeyAccount account;
private List<OnlineAccount> onlineAccounts;
private List<OnlineAccountData> onlineAccounts;
private long nextOnlineRefresh = 0;
public OnlinePeer(int id, PrivateKeyAccount account) {
@@ -65,22 +65,22 @@ public class OnlineTests extends Common {
case GET_ONLINE_ACCOUNTS: {
GetOnlineAccountsMessage getOnlineAccountsMessage = (GetOnlineAccountsMessage) message;
List<OnlineAccount> excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts();
List<OnlineAccountData> excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts();
// Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts
List<OnlineAccount> accountsToSend;
List<OnlineAccountData> accountsToSend;
synchronized (this.onlineAccounts) {
accountsToSend = new ArrayList<>(this.onlineAccounts);
}
Iterator<OnlineAccount> iterator = accountsToSend.iterator();
Iterator<OnlineAccountData> iterator = accountsToSend.iterator();
SEND_ITERATOR:
while (iterator.hasNext()) {
OnlineAccount onlineAccount = iterator.next();
OnlineAccountData onlineAccount = iterator.next();
for (int i = 0; i < excludeAccounts.size(); ++i) {
OnlineAccount excludeAccount = excludeAccounts.get(i);
OnlineAccountData excludeAccount = excludeAccounts.get(i);
if (onlineAccount.getTimestamp() == excludeAccount.getTimestamp() && Arrays.equals(onlineAccount.getPublicKey(), excludeAccount.getPublicKey())) {
iterator.remove();
@@ -101,12 +101,12 @@ public class OnlineTests extends Common {
case ONLINE_ACCOUNTS: {
OnlineAccountsMessage onlineAccountsMessage = (OnlineAccountsMessage) message;
List<OnlineAccount> onlineAccounts = onlineAccountsMessage.getOnlineAccounts();
List<OnlineAccountData> onlineAccounts = onlineAccountsMessage.getOnlineAccounts();
if (LOG_ACCOUNT_CHANGES)
System.out.println(String.format("[%d] received %d online accounts from %d", this.getId(), onlineAccounts.size(), peer.getId()));
for (OnlineAccount onlineAccount : onlineAccounts)
for (OnlineAccountData onlineAccount : onlineAccounts)
verifyAndAddAccount(onlineAccount);
break;
@@ -117,7 +117,7 @@ public class OnlineTests extends Common {
}
}
private void verifyAndAddAccount(OnlineAccount onlineAccount) {
private void verifyAndAddAccount(OnlineAccountData onlineAccount) {
// we would check timestamp is 'recent' here
// Verify
@@ -131,7 +131,7 @@ public class OnlineTests extends Common {
ByteArray publicKeyBA = new ByteArray(onlineAccount.getPublicKey());
synchronized (this.onlineAccounts) {
OnlineAccount existingAccount = this.onlineAccounts.stream().filter(account -> new ByteArray(account.getPublicKey()).equals(publicKeyBA)).findFirst().orElse(null);
OnlineAccountData existingAccount = this.onlineAccounts.stream().filter(account -> new ByteArray(account.getPublicKey()).equals(publicKeyBA)).findFirst().orElse(null);
if (existingAccount != null) {
if (existingAccount.getTimestamp() < onlineAccount.getTimestamp()) {
@@ -161,9 +161,9 @@ public class OnlineTests extends Common {
// Expire old entries
final long cutoffThreshold = now - LAST_SEEN_EXPIRY_PERIOD;
synchronized (this.onlineAccounts) {
Iterator<OnlineAccount> iterator = this.onlineAccounts.iterator();
Iterator<OnlineAccountData> iterator = this.onlineAccounts.iterator();
while (iterator.hasNext()) {
OnlineAccount onlineAccount = iterator.next();
OnlineAccountData onlineAccount = iterator.next();
if (onlineAccount.getTimestamp() < cutoffThreshold) {
iterator.remove();
@@ -211,7 +211,7 @@ public class OnlineTests extends Common {
byte[] publicKey = this.account.getPublicKey();
// Our account is online
OnlineAccount onlineAccount = new OnlineAccount(timestamp, signature, publicKey);
OnlineAccountData onlineAccount = new OnlineAccountData(timestamp, signature, publicKey);
synchronized (this.onlineAccounts) {
this.onlineAccounts.removeIf(account -> account.getPublicKey() == this.account.getPublicKey());
this.onlineAccounts.add(onlineAccount);