Merge branch 'master' into increase-online-timestamp-modulus

# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
This commit is contained in:
CalDescent
2022-07-30 12:23:25 +01:00
30 changed files with 1005 additions and 126 deletions

View File

@@ -8,6 +8,7 @@ import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.Security;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -18,6 +19,8 @@ import org.qortal.api.ApiRequest;
import org.qortal.controller.AutoUpdate;
import org.qortal.settings.Settings;
import static org.qortal.controller.AutoUpdate.AGENTLIB_JVM_HOLDER_ARG;
public class ApplyUpdate {
static {
@@ -197,6 +200,11 @@ public class ApplyUpdate {
// JVM arguments
javaCmd.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments());
// Reapply any retained, but disabled, -agentlib JVM arg
javaCmd = javaCmd.stream()
.map(arg -> arg.replace(AGENTLIB_JVM_HOLDER_ARG, "-agentlib"))
.collect(Collectors.toList());
// Call mainClass in JAR
javaCmd.addAll(Arrays.asList("-jar", JAR_FILENAME));
@@ -205,7 +213,7 @@ public class ApplyUpdate {
}
try {
LOGGER.info(() -> String.format("Restarting node with: %s", String.join(" ", javaCmd)));
LOGGER.info(String.format("Restarting node with: %s", String.join(" ", javaCmd)));
ProcessBuilder processBuilder = new ProcessBuilder(javaCmd);
@@ -214,8 +222,15 @@ public class ApplyUpdate {
processBuilder.environment().put(JAVA_TOOL_OPTIONS_NAME, JAVA_TOOL_OPTIONS_VALUE);
}
processBuilder.start();
} catch (IOException e) {
// New process will inherit our stdout and stderr
processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
Process process = processBuilder.start();
// Nothing to pipe to new process, so close output stream (process's stdin)
process.getOutputStream().close();
} catch (Exception e) {
LOGGER.error(String.format("Failed to restart node (BAD): %s", e.getMessage()));
}
}

View File

@@ -114,7 +114,7 @@ public class BlocksResource {
@Path("/signature/{signature}/data")
@Operation(
summary = "Fetch serialized, base58 encoded block data using base58 signature",
description = "Returns serialized data for the block that matches the given signature",
description = "Returns serialized data for the block that matches the given signature, and an optional block serialization version",
responses = {
@ApiResponse(
description = "the block data",
@@ -125,7 +125,7 @@ public class BlocksResource {
@ApiErrors({
ApiError.INVALID_SIGNATURE, ApiError.BLOCK_UNKNOWN, ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE
})
public String getSerializedBlockData(@PathParam("signature") String signature58) {
public String getSerializedBlockData(@PathParam("signature") String signature58, @QueryParam("version") Integer version) {
// Decode signature
byte[] signature;
try {
@@ -136,20 +136,41 @@ public class BlocksResource {
try (final Repository repository = RepositoryManager.getRepository()) {
// Default to version 1
if (version == null) {
version = 1;
}
// Check the database first
BlockData blockData = repository.getBlockRepository().fromSignature(signature);
if (blockData != null) {
Block block = new Block(repository, blockData);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
bytes.write(BlockTransformer.toBytes(block));
switch (version) {
case 1:
bytes.write(BlockTransformer.toBytes(block));
break;
case 2:
bytes.write(BlockTransformer.toBytesV2(block));
break;
default:
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
}
return Base58.encode(bytes.toByteArray());
}
// Not found, so try the block archive
byte[] bytes = BlockArchiveReader.getInstance().fetchSerializedBlockBytesForSignature(signature, false, repository);
if (bytes != null) {
return Base58.encode(bytes);
if (version != 1) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Archived blocks require version 1");
}
return Base58.encode(bytes);
}
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCK_UNKNOWN);

View File

@@ -199,6 +199,11 @@ public class Block {
}
public boolean hasShareBin(AccountLevelShareBin shareBin, int blockHeight) {
AccountLevelShareBin ourShareBin = this.getShareBin(blockHeight);
return ourShareBin != null && shareBin.id == ourShareBin.id;
}
public long distribute(long accountAmount, Map<String, Long> balanceChanges) {
if (this.isRecipientAlsoMinter) {
// minter & recipient the same - simpler case
@@ -1891,12 +1896,67 @@ public class Block {
final boolean haveFounders = !onlineFounderAccounts.isEmpty();
// Determine reward candidates based on account level
List<AccountLevelShareBin> accountLevelShareBins = BlockChain.getInstance().getAccountLevelShareBins();
for (int binIndex = 0; binIndex < accountLevelShareBins.size(); ++binIndex) {
// Find all accounts in share bin. getShareBin() returns null for minter accounts that are also founders, so they are effectively filtered out.
// This needs a deep copy, so the shares can be modified when tiers aren't activated yet
List<AccountLevelShareBin> accountLevelShareBins = new ArrayList<>();
for (AccountLevelShareBin accountLevelShareBin : BlockChain.getInstance().getAccountLevelShareBins()) {
accountLevelShareBins.add((AccountLevelShareBin) accountLevelShareBin.clone());
}
Map<Integer, List<ExpandedAccount>> accountsForShareBin = new HashMap<>();
// We might need to combine some share bins if they haven't reached the minimum number of minters yet
for (int binIndex = accountLevelShareBins.size()-1; binIndex >= 0; --binIndex) {
AccountLevelShareBin accountLevelShareBin = accountLevelShareBins.get(binIndex);
// Object reference compare is OK as all references are read-only from blockchain config.
List<ExpandedAccount> binnedAccounts = expandedAccounts.stream().filter(accountInfo -> accountInfo.getShareBin(this.blockData.getHeight()) == accountLevelShareBin).collect(Collectors.toList());
// Find all accounts in share bin. getShareBin() returns null for minter accounts that are also founders, so they are effectively filtered out.
List<ExpandedAccount> binnedAccounts = expandedAccounts.stream().filter(accountInfo -> accountInfo.hasShareBin(accountLevelShareBin, this.blockData.getHeight())).collect(Collectors.toList());
// Add any accounts that have been moved down from a higher tier
List<ExpandedAccount> existingBinnedAccounts = accountsForShareBin.get(binIndex);
if (existingBinnedAccounts != null)
binnedAccounts.addAll(existingBinnedAccounts);
// Logic below may only apply to higher levels, and only for share bins with a specific range of online accounts
if (accountLevelShareBin.levels.get(0) < BlockChain.getInstance().getShareBinActivationMinLevel() ||
binnedAccounts.isEmpty() || binnedAccounts.size() >= BlockChain.getInstance().getMinAccountsToActivateShareBin()) {
// Add all accounts for this share bin to the accountsForShareBin list
accountsForShareBin.put(binIndex, binnedAccounts);
continue;
}
// Share bin contains more than one, but less than the minimum number of minters. We treat this share bin
// as not activated yet. In these cases, the rewards and minters are combined and paid out to the previous
// share bin, to prevent a single or handful of accounts receiving the entire rewards for a share bin.
//
// Example:
//
// - Share bin for levels 5 and 6 has 100 minters
// - Share bin for levels 7 and 8 has 10 minters
//
// This is below the minimum of 30, so share bins are reconstructed as follows:
//
// - Share bin for levels 5 and 6 now contains 110 minters
// - Share bin for levels 7 and 8 now contains 0 minters
// - Share bin for levels 5 and 6 now pays out rewards for levels 5, 6, 7, and 8
// - Share bin for levels 7 and 8 pays zero rewards
//
// This process is iterative, so will combine several tiers if needed.
// Designate this share bin as empty
accountsForShareBin.put(binIndex, new ArrayList<>());
// Move the accounts originally intended for this share bin to the previous one
accountsForShareBin.put(binIndex - 1, binnedAccounts);
// Move the block reward from this share bin to the previous one
AccountLevelShareBin previousShareBin = accountLevelShareBins.get(binIndex - 1);
previousShareBin.share += accountLevelShareBin.share;
accountLevelShareBin.share = 0L;
}
// Now loop through (potentially modified) share bins and determine the reward candidates
for (int binIndex = 0; binIndex < accountLevelShareBins.size(); ++binIndex) {
AccountLevelShareBin accountLevelShareBin = accountLevelShareBins.get(binIndex);
List<ExpandedAccount> binnedAccounts = accountsForShareBin.get(binIndex);
// No online accounts in this bin? Skip to next one
if (binnedAccounts.isEmpty())

View File

@@ -68,6 +68,7 @@ public class BlockChain {
atFindNextTransactionFix,
newBlockSigHeight,
shareBinFix,
rewardShareLimitTimestamp,
calcChainWeightTimestamp,
transactionV5Timestamp,
transactionV6Timestamp,
@@ -103,10 +104,23 @@ public class BlockChain {
private List<RewardByHeight> rewardsByHeight;
/** Share of block reward/fees by account level */
public static class AccountLevelShareBin {
public static class AccountLevelShareBin implements Cloneable {
public int id;
public List<Integer> levels;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
public long share;
public Object clone() {
AccountLevelShareBin shareBinCopy = new AccountLevelShareBin();
List<Integer> levelsCopy = new ArrayList<>();
for (Integer level : this.levels) {
levelsCopy.add(level);
}
shareBinCopy.id = this.id;
shareBinCopy.levels = levelsCopy;
shareBinCopy.share = this.share;
return shareBinCopy;
}
}
private List<AccountLevelShareBin> sharesByLevel;
/** Generated lookup of share-bin by account level */
@@ -120,6 +134,12 @@ public class BlockChain {
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private Long qoraPerQortReward;
/** Minimum number of accounts before a share bin is considered activated */
private int minAccountsToActivateShareBin;
/** Min level at which share bin activation takes place; lower levels allow less than minAccountsPerShareBin */
private int shareBinActivationMinLevel;
/**
* Number of minted blocks required to reach next level from previous.
* <p>
@@ -157,7 +177,7 @@ public class BlockChain {
private int minAccountLevelToMint;
private int minAccountLevelForBlockSubmissions;
private int minAccountLevelToRewardShare;
private int maxRewardSharesPerMintingAccount;
private int maxRewardSharesPerFounderMintingAccount;
private int founderEffectiveMintingLevel;
/** Minimum time to retain online account signatures (ms) for block validity checks. */
@@ -169,6 +189,13 @@ public class BlockChain {
* featureTriggers because unit tests need to set this value via Reflection. */
private long onlineAccountsModulusV2Timestamp;
/** Max reward shares by block height */
public static class MaxRewardSharesByTimestamp {
public long timestamp;
public int maxShares;
}
private List<MaxRewardSharesByTimestamp> maxRewardSharesByTimestamp;
/** Settings relating to CIYAM AT feature. */
public static class CiyamAtSettings {
/** Fee per step/op-code executed. */
@@ -363,6 +390,14 @@ public class BlockChain {
return this.qoraPerQortReward;
}
public int getMinAccountsToActivateShareBin() {
return this.minAccountsToActivateShareBin;
}
public int getShareBinActivationMinLevel() {
return this.shareBinActivationMinLevel;
}
public int getMinAccountLevelToMint() {
return this.minAccountLevelToMint;
}
@@ -375,8 +410,8 @@ public class BlockChain {
return this.minAccountLevelToRewardShare;
}
public int getMaxRewardSharesPerMintingAccount() {
return this.maxRewardSharesPerMintingAccount;
public int getMaxRewardSharesPerFounderMintingAccount() {
return this.maxRewardSharesPerFounderMintingAccount;
}
public int getFounderEffectiveMintingLevel() {
@@ -409,6 +444,10 @@ public class BlockChain {
return this.featureTriggers.get(FeatureTrigger.shareBinFix.name()).intValue();
}
public long getRewardShareLimitTimestamp() {
return this.featureTriggers.get(FeatureTrigger.rewardShareLimitTimestamp.name()).longValue();
}
public long getCalcChainWeightTimestamp() {
return this.featureTriggers.get(FeatureTrigger.calcChainWeightTimestamp.name()).longValue();
}
@@ -457,6 +496,14 @@ public class BlockChain {
return this.getUnitFee();
}
public int getMaxRewardSharesAtTimestamp(long ourTimestamp) {
for (int i = maxRewardSharesByTimestamp.size() - 1; i >= 0; --i)
if (maxRewardSharesByTimestamp.get(i).timestamp <= ourTimestamp)
return maxRewardSharesByTimestamp.get(i).maxShares;
return 0;
}
/** Validate blockchain config read from JSON */
private void validateConfig() {
if (this.genesisInfo == null)

View File

@@ -15,6 +15,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -40,6 +41,7 @@ public class AutoUpdate extends Thread {
public static final String JAR_FILENAME = "qortal.jar";
public static final String NEW_JAR_FILENAME = "new-" + JAR_FILENAME;
public static final String AGENTLIB_JVM_HOLDER_ARG = "-DQORTAL_agentlib=";
private static final Logger LOGGER = LogManager.getLogger(AutoUpdate.class);
private static final long CHECK_INTERVAL = 20 * 60 * 1000L; // ms
@@ -243,6 +245,11 @@ public class AutoUpdate extends Thread {
// JVM arguments
javaCmd.addAll(ManagementFactory.getRuntimeMXBean().getInputArguments());
// Disable, but retain, any -agentlib JVM arg as sub-process might fail if it tries to reuse same port
javaCmd = javaCmd.stream()
.map(arg -> arg.replace("-agentlib", AGENTLIB_JVM_HOLDER_ARG))
.collect(Collectors.toList());
// Remove JNI options as they won't be supported by command-line 'java'
// These are typically added by the AdvancedInstaller Java launcher EXE
javaCmd.removeAll(Arrays.asList("abort", "exit", "vfprintf"));
@@ -261,10 +268,19 @@ public class AutoUpdate extends Thread {
Translator.INSTANCE.translate("SysTray", "APPLYING_UPDATE_AND_RESTARTING"),
MessageType.INFO);
new ProcessBuilder(javaCmd).start();
ProcessBuilder processBuilder = new ProcessBuilder(javaCmd);
// New process will inherit our stdout and stderr
processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
Process process = processBuilder.start();
// Nothing to pipe to new process, so close output stream (process's stdin)
process.getOutputStream().close();
return true; // applying update OK
} catch (IOException e) {
} catch (Exception e) {
LOGGER.error(String.format("Failed to apply update: %s", e.getMessage()));
try {

View File

@@ -90,37 +90,40 @@ public class BlockMinter extends Thread {
List<Block> newBlocks = new ArrayList<>();
// Flags for tracking change in whether minting is possible,
// so we can notify Controller, and further update SysTray, etc.
boolean isMintingPossible = false;
boolean wasMintingPossible = isMintingPossible;
while (running) {
if (isMintingPossible != wasMintingPossible)
Controller.getInstance().onMintingPossibleChange(isMintingPossible);
try (final Repository repository = RepositoryManager.getRepository()) {
// Going to need this a lot...
BlockRepository blockRepository = repository.getBlockRepository();
wasMintingPossible = isMintingPossible;
// Flags for tracking change in whether minting is possible,
// so we can notify Controller, and further update SysTray, etc.
boolean isMintingPossible = false;
boolean wasMintingPossible = isMintingPossible;
while (running) {
if (isMintingPossible != wasMintingPossible)
Controller.getInstance().onMintingPossibleChange(isMintingPossible);
try {
// Sleep for a while
Thread.sleep(1000);
wasMintingPossible = isMintingPossible;
isMintingPossible = false;
try {
// Free up any repository locks
repository.discardChanges();
final Long now = NTP.getTime();
if (now == null)
continue;
// Sleep for a while
Thread.sleep(1000);
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
if (minLatestBlockTimestamp == null)
continue;
isMintingPossible = false;
// No online accounts for current timestamp? (e.g. during startup)
if (!OnlineAccountsManager.getInstance().hasOnlineAccounts())
continue;
final Long now = NTP.getTime();
if (now == null)
continue;
try (final Repository repository = RepositoryManager.getRepository()) {
// Going to need this a lot...
BlockRepository blockRepository = repository.getBlockRepository();
final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
if (minLatestBlockTimestamp == null)
continue;
// No online accounts for current timestamp? (e.g. during startup)
if (!OnlineAccountsManager.getInstance().hasOnlineAccounts())
continue;
List<MintingAccountData> mintingAccountsData = repository.getAccountRepository().getMintingAccounts();
// No minting accounts?
@@ -198,10 +201,6 @@ public class BlockMinter extends Thread {
// so go ahead and mint a block if possible.
isMintingPossible = true;
// Reattach newBlocks to new repository handle
for (Block newBlock : newBlocks)
newBlock.setRepository(repository);
// Check blockchain hasn't changed
if (previousBlockData == null || !Arrays.equals(previousBlockData.getSignature(), lastBlockData.getSignature())) {
previousBlockData = lastBlockData;
@@ -439,13 +438,13 @@ public class BlockMinter extends Thread {
Network network = Network.getInstance();
network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newBlockData));
}
} catch (DataException e) {
LOGGER.warn("Repository issue while running block minter", e);
} catch (InterruptedException e) {
// We've been interrupted - time to exit
return;
}
} catch (InterruptedException e) {
// We've been interrupted - time to exit
return;
}
} catch (DataException e) {
LOGGER.warn("Repository issue while running block minter - NO LONGER MINTING", e);
}
}

View File

@@ -819,7 +819,7 @@ public class Controller extends Thread {
actionText = String.format("%s", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"));
SysTray.getInstance().setTrayIcon(3);
}
else if (OnlineAccountsManager.getInstance().hasOnlineAccounts()) {
else if (OnlineAccountsManager.getInstance().hasActiveOnlineAccountSignatures()) {
actionText = Translator.INSTANCE.translate("SysTray", "MINTING_ENABLED");
SysTray.getInstance().setTrayIcon(2);
}

View File

@@ -480,7 +480,7 @@ public class OnlineAccountsManager {
final Long minLatestBlockTimestamp = NTP.getTime() - (2 * 60 * 60 * 1000L);
boolean isUpToDate = Controller.getInstance().isUpToDate(minLatestBlockTimestamp);
return isUpToDate && hasOnlineAccounts();
return isUpToDate && hasOurOnlineAccounts();
}
public boolean hasOurOnlineAccounts() {

View File

@@ -159,6 +159,9 @@ public interface AccountRepository {
/** Returns number of active reward-shares involving passed public key as the minting account only. */
public int countRewardShares(byte[] mintingAccountPublicKey) throws DataException;
/** Returns number of active self-shares involving passed public key as the minting account only. */
public int countSelfShares(byte[] mintingAccountPublicKey) throws DataException;
public List<RewardShareData> getRewardShares() throws DataException;
public List<RewardShareData> findRewardShares(List<String> mintingAccounts, List<String> recipientAccounts, List<String> involvedAddresses, Integer limit, Integer offset, Boolean reverse) throws DataException;

View File

@@ -688,6 +688,17 @@ public class HSQLDBAccountRepository implements AccountRepository {
}
}
@Override
public int countSelfShares(byte[] minterPublicKey) throws DataException {
String sql = "SELECT COUNT(*) FROM RewardShares WHERE minter_public_key = ? AND minter = recipient";
try (ResultSet resultSet = this.repository.checkedExecute(sql, minterPublicKey)) {
return resultSet.getInt(1);
} catch (SQLException e) {
throw new DataException("Unable to count self-shares in repository", e);
}
}
@Override
public List<RewardShareData> getRewardShares() throws DataException {
String sql = "SELECT minter_public_key, minter, recipient, share_percent, reward_share_public_key FROM RewardShares";

View File

@@ -140,8 +140,21 @@ public class RewardShareTransaction extends Transaction {
// Check the minting account hasn't reach maximum number of reward-shares
int rewardShareCount = this.repository.getAccountRepository().countRewardShares(creator.getPublicKey());
if (rewardShareCount >= BlockChain.getInstance().getMaxRewardSharesPerMintingAccount())
int selfShareCount = this.repository.getAccountRepository().countSelfShares(creator.getPublicKey());
int maxRewardShares = BlockChain.getInstance().getMaxRewardSharesAtTimestamp(this.rewardShareTransactionData.getTimestamp());
if (creator.isFounder())
// Founders have a different limit
maxRewardShares = BlockChain.getInstance().getMaxRewardSharesPerFounderMintingAccount();
if (rewardShareCount >= maxRewardShares)
return ValidationResult.MAXIMUM_REWARD_SHARES;
// When filling all reward share slots, one must be a self share (after feature trigger timestamp)
if (this.rewardShareTransactionData.getTimestamp() >= BlockChain.getInstance().getRewardShareLimitTimestamp())
if (!isRecipientAlsoMinter && rewardShareCount == maxRewardShares-1 && selfShareCount == 0)
return ValidationResult.MAXIMUM_REWARD_SHARES;
} else {
// This transaction intends to modify/terminate an existing reward-share

View File

@@ -15,7 +15,11 @@
"minAccountLevelToMint": 1,
"minAccountLevelForBlockSubmissions": 5,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 6,
"maxRewardSharesPerFounderMintingAccount": 6,
"maxRewardSharesByTimestamp": [
{ "timestamp": 0, "maxShares": 6 },
{ "timestamp": 1657382400000, "maxShares": 3 }
],
"founderEffectiveMintingLevel": 10,
"onlineAccountSignaturesMinLifetime": 43200000,
"onlineAccountSignaturesMaxLifetime": 86400000,
@@ -36,14 +40,16 @@
{ "height": 3110401, "reward": 2.00 }
],
"sharesByLevel": [
{ "levels": [ 1, 2 ], "share": 0.05 },
{ "levels": [ 3, 4 ], "share": 0.10 },
{ "levels": [ 5, 6 ], "share": 0.15 },
{ "levels": [ 7, 8 ], "share": 0.20 },
{ "levels": [ 9, 10 ], "share": 0.25 }
{ "id": 1, "levels": [ 1, 2 ], "share": 0.05 },
{ "id": 2, "levels": [ 3, 4 ], "share": 0.10 },
{ "id": 3, "levels": [ 5, 6 ], "share": 0.15 },
{ "id": 4, "levels": [ 7, 8 ], "share": 0.20 },
{ "id": 5, "levels": [ 9, 10 ], "share": 0.25 }
],
"qoraHoldersShare": 0.20,
"qoraPerQortReward": 250,
"minAccountsToActivateShareBin": 30,
"shareBinActivationMinLevel": 7,
"blocksNeededByLevel": [ 7200, 64800, 129600, 172800, 244000, 345600, 518400, 691200, 864000, 1036800 ],
"blockTimingsByHeight": [
{ "height": 1, "target": 60000, "deviation": 30000, "power": 0.2 }
@@ -58,6 +64,7 @@
"atFindNextTransactionFix": 275000,
"newBlockSigHeight": 320000,
"shareBinFix": 399000,
"rewardShareLimitTimestamp": 1657382400000,
"calcChainWeightTimestamp": 1620579600000,
"transactionV5Timestamp": 1642176000000,
"transactionV6Timestamp": 9999999999999,