Merge chain-stall, blocksMinted and other fixes

This commit is contained in:
catbref 2020-05-06 08:01:51 +01:00
commit 74b5401e84
7 changed files with 293 additions and 28 deletions

View File

@ -12,7 +12,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
@ -138,7 +137,6 @@ public class Block {
private final Account recipientAccount;
private final AccountData recipientAccountData;
private final boolean isRecipientFounder;
ExpandedAccount(Repository repository, int accountIndex) throws DataException {
this.rewardShareData = repository.getAccountRepository().getRewardShareByIndex(accountIndex);
@ -154,12 +152,10 @@ public class Block {
// Self-share: minter is also recipient
this.recipientAccount = this.mintingAccount;
this.recipientAccountData = this.mintingAccountData;
this.isRecipientFounder = this.isMinterFounder;
} else {
// Recipient differs from minter
this.recipientAccount = new Account(repository, this.rewardShareData.getRecipient());
this.recipientAccountData = repository.getAccountRepository().getAccount(this.recipientAccount.getAddress());
this.isRecipientFounder = Account.isFounder(recipientAccountData.getFlags());
}
}
@ -319,7 +315,11 @@ public class Block {
if (onlineAccountData.getTimestamp() != onlineAccountsTimestamp)
continue;
int accountIndex = repository.getAccountRepository().getRewardShareIndex(onlineAccountData.getPublicKey());
Integer accountIndex = repository.getAccountRepository().getRewardShareIndex(onlineAccountData.getPublicKey());
if (accountIndex == null)
// Online account (reward-share) with current timestamp but reward-share cancelled
continue;
indexedOnlineAccounts.put(accountIndex, onlineAccountData);
}
List<Integer> accountIndexes = new ArrayList<>(indexedOnlineAccounts.keySet());
@ -1268,14 +1268,13 @@ public class Block {
protected void increaseAccountLevels() throws DataException {
// We need to do this for both minters and recipients
this.increaseAccountLevels(expandedAccount -> expandedAccount.isMinterFounder, expandedAccount -> expandedAccount.mintingAccountData);
this.increaseAccountLevels(expandedAccount -> expandedAccount.isRecipientFounder, expandedAccount -> expandedAccount.recipientAccountData);
this.increaseAccountLevels(false, expandedAccount -> expandedAccount.mintingAccountData);
this.increaseAccountLevels(true, expandedAccount -> expandedAccount.recipientAccountData);
}
private void increaseAccountLevels(Predicate<ExpandedAccount> isFounder, Function<ExpandedAccount, AccountData> getAccountData) throws DataException {
private void increaseAccountLevels(boolean isProcessingRecipients, Function<ExpandedAccount, AccountData> getAccountData) throws DataException {
final List<Integer> cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel();
final List<ExpandedAccount> expandedAccounts = this.getExpandedAccounts();
final boolean isProcessingRecipients = getAccountData.apply(expandedAccounts.get(0)) == expandedAccounts.get(0).recipientAccountData;
// Increase blocks-minted count for all accounts
for (int a = 0; a < expandedAccounts.size(); ++a) {
@ -1594,14 +1593,13 @@ public class Block {
protected void decreaseAccountLevels() throws DataException {
// We need to do this for both minters and recipients
this.decreaseAccountLevels(expandedAccount -> expandedAccount.isMinterFounder, expandedAccount -> expandedAccount.mintingAccountData);
this.decreaseAccountLevels(expandedAccount -> expandedAccount.isRecipientFounder, expandedAccount -> expandedAccount.recipientAccountData);
this.decreaseAccountLevels(false, expandedAccount -> expandedAccount.mintingAccountData);
this.decreaseAccountLevels(true, expandedAccount -> expandedAccount.recipientAccountData);
}
private void decreaseAccountLevels(Predicate<ExpandedAccount> isFounder, Function<ExpandedAccount, AccountData> getAccountData) throws DataException {
private void decreaseAccountLevels(boolean isProcessingRecipients, Function<ExpandedAccount, AccountData> getAccountData) throws DataException {
final List<Integer> cumulativeBlocksByLevel = BlockChain.getInstance().getCumulativeBlocksByLevel();
final List<ExpandedAccount> expandedAccounts = this.getExpandedAccounts();
final boolean isProcessingRecipients = getAccountData.apply(expandedAccounts.get(0)) == expandedAccounts.get(0).recipientAccountData;
// Decrease blocks minted count for all accounts
for (int a = 0; a < expandedAccounts.size(); ++a) {

View File

@ -40,6 +40,8 @@ public class BlockMinter extends Thread {
// Other properties
private static final Logger LOGGER = LogManager.getLogger(BlockMinter.class);
private static Long lastLogTimestamp;
private static Long logTimeout;
// Constructors
@ -151,6 +153,9 @@ public class BlockMinter extends Thread {
if (previousBlock == null || !Arrays.equals(previousBlock.getSignature(), lastBlockData.getSignature())) {
previousBlock = new Block(repository, lastBlockData);
newBlocks.clear();
// Reduce log timeout
logTimeout = 10 * 1000L;
}
// Discard accounts we have already built blocks with
@ -163,19 +168,23 @@ public class BlockMinter extends Thread {
// First block does the AT heavy-lifting
if (newBlocks.isEmpty()) {
Block newBlock = Block.mint(repository, previousBlock.getBlockData(), mintingAccount);
if (newBlock == null)
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't build a to-be-minted block"));
continue;
}
newBlocks.add(newBlock);
} else {
// The blocks for other minters require less effort...
Block newBlock = newBlocks.get(0);
if (newBlock == null)
Block newBlock = newBlocks.get(0).remint(mintingAccount);
if (newBlock == null) {
// For some reason we can't mint right now
moderatedLog(() -> LOGGER.error("Couldn't rebuild a to-be-minted block"));
continue;
}
newBlocks.add(newBlock.remint(mintingAccount));
newBlocks.add(newBlock);
}
}
@ -185,15 +194,22 @@ public class BlockMinter extends Thread {
// Make sure we're the only thread modifying the blockchain
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (!blockchainLock.tryLock(30, TimeUnit.SECONDS))
if (!blockchainLock.tryLock(30, TimeUnit.SECONDS)) {
LOGGER.warn("Couldn't acquire blockchain lock even after waiting 30 seconds");
continue;
}
boolean newBlockMinted = false;
try {
// Clear repository's "in transaction" state so we don't cause a repository deadlock
// Clear repository session state so we have latest view of data
repository.discardChanges();
// Now that we have blockchain lock, do final check that chain hasn't changed
BlockData latestBlockData = blockRepository.getLastBlock();
if (!Arrays.equals(lastBlockData.getSignature(), latestBlockData.getSignature()))
continue;
List<Block> goodBlocks = new ArrayList<>();
for (Block testBlock : newBlocks) {
// Is new block's timestamp valid yet?
@ -202,8 +218,12 @@ public class BlockMinter extends Thread {
continue;
// Is new block valid yet? (Before adding unconfirmed transactions)
if (testBlock.isValid() != ValidationResult.OK)
ValidationResult result = testBlock.isValid();
if (result != ValidationResult.OK) {
moderatedLog(() -> LOGGER.error(String.format("To-be-minted block invalid '%s' before adding transactions?", result.name())));
continue;
}
goodBlocks.add(testBlock);
}
@ -352,10 +372,14 @@ public class BlockMinter extends Thread {
// Ensure mintingAccount is 'online' so blocks can be minted
Controller.getInstance().ensureTestingAccountsOnline(mintingAndOnlineAccounts);
BlockData previousBlockData = repository.getBlockRepository().getLastBlock();
PrivateKeyAccount mintingAccount = mintingAndOnlineAccounts[0];
return mintTestingBlockRetainingTimestamps(repository, mintingAccount);
}
public static Block mintTestingBlockRetainingTimestamps(Repository repository, PrivateKeyAccount mintingAccount) throws DataException {
BlockData previousBlockData = repository.getBlockRepository().getLastBlock();
Block newBlock = Block.mint(repository, previousBlockData, mintingAccount);
// Make sure we're the only thread modifying the blockchain
@ -385,4 +409,15 @@ public class BlockMinter extends Thread {
}
}
private static void moderatedLog(Runnable logFunction) {
// We only log if logging at TRACE or previous log timeout has expired
if (!LOGGER.isTraceEnabled() && lastLogTimestamp != null && lastLogTimestamp + logTimeout > System.currentTimeMillis())
return;
lastLogTimestamp = System.currentTimeMillis();
logTimeout = 2 * 60 * 1000L; // initial timeout, can be reduced if new block appears
logFunction.run();
}
}

View File

@ -141,6 +141,8 @@ public interface AccountRepository {
*/
public RewardShareData getRewardShareByIndex(int index) throws DataException;
public boolean rewardShareExists(byte[] rewardSharePublicKey) throws DataException;
public void save(RewardShareData rewardShareData) throws DataException;
/** Delete reward-share from repository using passed minting account's public key and recipient's address. */

View File

@ -666,13 +666,13 @@ public class HSQLDBAccountRepository implements AccountRepository {
}
@Override
public Integer getRewardShareIndex(byte[] publicKey) throws DataException {
public Integer getRewardShareIndex(byte[] rewardSharePublicKey) throws DataException {
if (!this.rewardShareExists(rewardSharePublicKey))
return null;
String sql = "SELECT COUNT(*) FROM RewardShares WHERE reward_share_public_key < ?";
try (ResultSet resultSet = this.repository.checkedExecute(sql, publicKey)) {
if (resultSet == null)
return null;
try (ResultSet resultSet = this.repository.checkedExecute(sql, rewardSharePublicKey)) {
return resultSet.getInt(1);
} catch (SQLException e) {
throw new DataException("Unable to determine reward-share index in repository", e);
@ -701,6 +701,15 @@ public class HSQLDBAccountRepository implements AccountRepository {
}
}
@Override
public boolean rewardShareExists(byte[] rewardSharePublicKey) throws DataException {
try {
return this.repository.exists("RewardShares", "reward_share_public_key = ?", rewardSharePublicKey);
} catch (SQLException e) {
throw new DataException("Unable to check reward-share exists in repository", e);
}
}
@Override
public void save(RewardShareData rewardShareData) throws DataException {
HSQLDBSaver saveHelper = new HSQLDBSaver("RewardShares");

View File

@ -82,7 +82,7 @@ public class Settings {
/** Port number for inbound peer-to-peer connections. */
private Integer listenPort;
/** Minimum number of peers to allow block minting / synchronization. */
private int minBlockchainPeers = 8;
private int minBlockchainPeers = 5;
/** Target number of outbound connections to peers we should make. */
private int minOutboundPeers = 16;
/** Maximum number of peer connections we allow. */

View File

@ -0,0 +1,96 @@
package org.qortal.test.minting;
import static org.junit.Assert.*;
import java.math.BigDecimal;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.block.BlockMinter;
import org.qortal.data.account.RewardShareData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.AccountUtils;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TestAccount;
public class BlocksMintedCountTests extends Common {
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
}
@After
public void afterTest() throws DataException {
Common.orphanCheck();
}
@Test
public void testNonSelfShare() throws DataException {
final BigDecimal sharePercent = new BigDecimal("12.8");
try (final Repository repository = RepositoryManager.getRepository()) {
// Create reward-share
byte[] testRewardSharePrivateKey = AccountUtils.rewardShare(repository, "alice", "bob", sharePercent);
PrivateKeyAccount testRewardShareAccount = new PrivateKeyAccount(repository, testRewardSharePrivateKey);
// Confirm reward-share info set correctly
RewardShareData testRewardShareData = repository.getAccountRepository().getRewardShare(testRewardShareAccount.getPublicKey());
assertNotNull(testRewardShareData);
testRewardShare(repository, testRewardShareAccount, +1, +1);
}
}
@Test
public void testSelfShare() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount testRewardShareAccount = Common.getTestAccount(repository, "alice-reward-share");
// Confirm reward-share exists
RewardShareData testRewardShareData = repository.getAccountRepository().getRewardShare(testRewardShareAccount.getPublicKey());
assertNotNull(testRewardShareData);
testRewardShare(repository, testRewardShareAccount, +1, 0);
}
}
private void testRewardShare(Repository repository, PrivateKeyAccount mintingAccount, int aliceDelta, int bobDelta) throws DataException {
// Fetch pre-mint blocks minted counts
int alicePreMintCount = getBlocksMinted(repository, "alice");
int bobPreMintCount = getBlocksMinted(repository, "bob");
// Mint another block
BlockMinter.mintTestingBlock(repository, mintingAccount);
// Fetch post-mint blocks minted counts
int alicePostMintCount = getBlocksMinted(repository, "alice");
int bobPostMintCount = getBlocksMinted(repository, "bob");
// Check both accounts
assertEquals("Alice's post-mint blocks-minted count incorrect", alicePreMintCount + aliceDelta, alicePostMintCount);
assertEquals("Bob's post-mint blocks-minted count incorrect", bobPreMintCount + bobDelta, bobPostMintCount);
// Orphan latest block
BlockUtils.orphanLastBlock(repository);
// Fetch post-orphan blocks minted counts
int alicePostOrphanCount = getBlocksMinted(repository, "alice");
int bobPostOrphanCount = getBlocksMinted(repository, "bob");
// Check blocks minted counts reverted correctly
assertEquals("Alice's post-orphan blocks-minted count incorrect", alicePreMintCount, alicePostOrphanCount);
assertEquals("Bob's post-orphan blocks-minted count incorrect", bobPreMintCount, bobPostOrphanCount);
}
private int getBlocksMinted(Repository repository, String name) throws DataException {
TestAccount testAccount = Common.getTestAccount(repository, name);
return repository.getAccountRepository().getAccount(testAccount.getAddress()).getBlocksMinted();
}
}

View File

@ -0,0 +1,125 @@
package org.qortal.test.minting;
import static org.junit.Assert.*;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.block.BlockMinter;
import org.qortal.controller.Controller;
import org.qortal.data.account.RewardShareData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.AccountUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TestAccount;
import org.qortal.test.common.TransactionUtils;
import org.qortal.transform.block.BlockTransformer;
import org.roaringbitmap.IntIterator;
import io.druid.extendedset.intset.ConciseSet;
public class DisagreementTests extends Common {
private static final BigDecimal CANCEL_SHARE_PERCENT = BigDecimal.ONE.negate();
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
}
@After
public void afterTest() throws DataException {
Common.orphanCheck();
}
/**
* Testing minting a block when there is a signed online account timestamp present
* that no longer has a corresponding reward-share in DB.
* <p>
* Something like:
* <ul>
* <li>Mint block, with tx to create reward-share R</li>
* <li>Sign current timestamp with R</li>
* <li>Mint block including R as online account</li>
* <li>Mint block, with tx to cancel reward-share R</li>
* <li>Mint another block: R's timestamp should be excluded</li>
* </ul>
*
* @throws DataException
*/
@Test
public void testOnlineAccounts() throws DataException {
final BigDecimal sharePercent = new BigDecimal("12.8");
try (final Repository repository = RepositoryManager.getRepository()) {
TestAccount mintingAccount = Common.getTestAccount(repository, "alice-reward-share");
TestAccount signingAccount = Common.getTestAccount(repository, "alice");
// Create reward-share
byte[] testRewardSharePrivateKey = AccountUtils.rewardShare(repository, "alice", "bob", sharePercent);
PrivateKeyAccount testRewardShareAccount = new PrivateKeyAccount(repository, testRewardSharePrivateKey);
// Confirm reward-share info set correctly
RewardShareData testRewardShareData = repository.getAccountRepository().getRewardShare(testRewardShareAccount.getPublicKey());
assertNotNull(testRewardShareData);
// Create signed timestamps
Controller.getInstance().ensureTestingAccountsOnline(mintingAccount, testRewardShareAccount);
// Mint another block
BlockMinter.mintTestingBlockRetainingTimestamps(repository, mintingAccount);
// Confirm reward-share's signed timestamp is included
BlockData blockData = repository.getBlockRepository().getLastBlock();
List<RewardShareData> rewardSharesData = fetchRewardSharesForBlock(repository, blockData);
boolean doesContainRewardShare = rewardSharesData.stream().anyMatch(rewardShareData -> Arrays.equals(rewardShareData.getRewardSharePublicKey(), testRewardShareData.getRewardSharePublicKey()));
assertTrue(doesContainRewardShare);
// Cancel reward-share
TransactionData cancelRewardShareTransactionData = AccountUtils.createRewardShare(repository, "alice", "bob", CANCEL_SHARE_PERCENT);
TransactionUtils.signAsUnconfirmed(repository, cancelRewardShareTransactionData, signingAccount);
BlockMinter.mintTestingBlockRetainingTimestamps(repository, mintingAccount);
// Confirm reward-share no longer exists in repository
RewardShareData cancelledRewardShareData = repository.getAccountRepository().getRewardShare(testRewardShareAccount.getPublicKey());
assertNull("Reward-share shouldn't exist", cancelledRewardShareData);
// Attempt to mint with cancelled reward-share
BlockMinter.mintTestingBlockRetainingTimestamps(repository, mintingAccount);
// Confirm reward-share's signed timestamp is NOT included
blockData = repository.getBlockRepository().getLastBlock();
rewardSharesData = fetchRewardSharesForBlock(repository, blockData);
doesContainRewardShare = rewardSharesData.stream().anyMatch(rewardShareData -> Arrays.equals(rewardShareData.getRewardSharePublicKey(), testRewardShareData.getRewardSharePublicKey()));
assertFalse(doesContainRewardShare);
}
}
private List<RewardShareData> fetchRewardSharesForBlock(Repository repository, BlockData blockData) throws DataException {
byte[] encodedOnlineAccounts = blockData.getEncodedOnlineAccounts();
ConciseSet accountIndexes = BlockTransformer.decodeOnlineAccounts(encodedOnlineAccounts);
List<RewardShareData> rewardSharesData = new ArrayList<>();
IntIterator iterator = accountIndexes.iterator();
while (iterator.hasNext()) {
int accountIndex = iterator.next();
RewardShareData rewardShareData = repository.getAccountRepository().getRewardShareByIndex(accountIndex);
rewardSharesData.add(rewardShareData);
}
return rewardSharesData;
}
}