Merge branch 'networking' into sync-multiple-blocks

# Conflicts:
#	src/main/java/org/qortal/network/Peer.java
This commit is contained in:
CalDescent 2021-05-30 09:57:35 +01:00
commit 61de7e144e
19 changed files with 2237 additions and 1902 deletions

33
.github/workflows/pr-testing.yml vendored Normal file
View File

@ -0,0 +1,33 @@
name: PR testing
on:
pull_request:
branches: [ master ]
jobs:
mavenTesting:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Cache local Maven repository
uses: actions/cache@v2
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Set up the Java JDK
uses: actions/setup-java@v2
with:
java-version: '11'
distribution: 'adopt'
- name: Run all tests
run: |
mvn -B clean test -DskipTests=false --file pom.xml
if [ -f "target/site/jacoco/index.html" ]; then echo "Total coverage: $(cat target/site/jacoco/index.html | grep -o 'Total[^%]*%' | grep -o '[0-9]*%')"; fi
- name: Log coverage percentage
run: |
if [ ! -f "target/site/jacoco/index.html" ]; then echo "No coverage information available"; fi
if [ -f "target/site/jacoco/index.html" ]; then echo "Total coverage: $(cat target/site/jacoco/index.html | grep -o 'Total[^%]*%' | grep -o '[0-9]*%')"; fi

View File

@ -1,61 +1,74 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.network.PeerChainTipData;
import org.qortal.data.network.PeerData;
import org.qortal.network.Handshake;
import org.qortal.network.Peer;
import io.swagger.v3.oas.annotations.media.Schema;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
@XmlAccessorType(XmlAccessType.FIELD)
public class ConnectedPeer {
public enum Direction {
INBOUND,
OUTBOUND;
}
public Direction direction;
public Handshake handshakeStatus;
public Long lastPing;
public Long connectedWhen;
public Long peersConnectedWhen;
public enum Direction {
INBOUND,
OUTBOUND;
}
public String address;
public String version;
public Direction direction;
public Handshake handshakeStatus;
public Long lastPing;
public Long connectedWhen;
public Long peersConnectedWhen;
public String nodeId;
public String address;
public String version;
public Integer lastHeight;
@Schema(example = "base58")
public byte[] lastBlockSignature;
public Long lastBlockTimestamp;
public String nodeId;
protected ConnectedPeer() {
}
public Integer lastHeight;
@Schema(example = "base58")
public byte[] lastBlockSignature;
public Long lastBlockTimestamp;
public UUID connectionId;
public String age;
public ConnectedPeer(Peer peer) {
this.direction = peer.isOutbound() ? Direction.OUTBOUND : Direction.INBOUND;
this.handshakeStatus = peer.getHandshakeStatus();
this.lastPing = peer.getLastPing();
protected ConnectedPeer() {
}
PeerData peerData = peer.getPeerData();
this.connectedWhen = peer.getConnectionTimestamp();
this.peersConnectedWhen = peer.getPeersConnectionTimestamp();
public ConnectedPeer(Peer peer) {
this.direction = peer.isOutbound() ? Direction.OUTBOUND : Direction.INBOUND;
this.handshakeStatus = peer.getHandshakeStatus();
this.lastPing = peer.getLastPing();
this.address = peerData.getAddress().toString();
PeerData peerData = peer.getPeerData();
this.connectedWhen = peer.getConnectionTimestamp();
this.peersConnectedWhen = peer.getPeersConnectionTimestamp();
this.version = peer.getPeersVersionString();
this.nodeId = peer.getPeersNodeId();
this.address = peerData.getAddress().toString();
PeerChainTipData peerChainTipData = peer.getChainTipData();
if (peerChainTipData != null) {
this.lastHeight = peerChainTipData.getLastHeight();
this.lastBlockSignature = peerChainTipData.getLastBlockSignature();
this.lastBlockTimestamp = peerChainTipData.getLastBlockTimestamp();
}
}
this.version = peer.getPeersVersionString();
this.nodeId = peer.getPeersNodeId();
this.connectionId = peer.getPeerConnectionId();
if (peer.getConnectionEstablishedTime() > 0) {
long age = (System.currentTimeMillis() - peer.getConnectionEstablishedTime());
long minutes = TimeUnit.MILLISECONDS.toMinutes(age);
long seconds = TimeUnit.MILLISECONDS.toSeconds(age) - TimeUnit.MINUTES.toSeconds(minutes);
this.age = String.format("%dm %ds", minutes, seconds);
} else {
this.age = "connecting...";
}
PeerChainTipData peerChainTipData = peer.getChainTipData();
if (peerChainTipData != null) {
this.lastHeight = peerChainTipData.getLastHeight();
this.lastBlockSignature = peerChainTipData.getLastBlockSignature();
this.lastBlockTimestamp = peerChainTipData.getLastBlockTimestamp();
}
}
}

View File

@ -23,8 +23,8 @@ import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.api.model.crosschain.BitcoinSendRequest;
import org.qortal.crosschain.Bitcoin;
import org.qortal.crosschain.BitcoinyTransaction;
import org.qortal.crosschain.ForeignBlockchainException;
import org.qortal.crosschain.SimpleTransaction;
@Path("/crosschain/btc")
@Tag(name = "Cross-Chain (Bitcoin)")
@ -89,12 +89,12 @@ public class CrossChainBitcoinResource {
),
responses = {
@ApiResponse(
content = @Content(array = @ArraySchema( schema = @Schema( implementation = BitcoinyTransaction.class ) ) )
content = @Content(array = @ArraySchema( schema = @Schema( implementation = SimpleTransaction.class ) ) )
)
}
)
@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
public List<BitcoinyTransaction> getBitcoinWalletTransactions(String key58) {
public List<SimpleTransaction> getBitcoinWalletTransactions(String key58) {
Security.checkApiCallAllowed(request);
Bitcoin bitcoin = Bitcoin.getInstance();

View File

@ -22,9 +22,9 @@ import org.qortal.api.ApiErrors;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.api.model.crosschain.LitecoinSendRequest;
import org.qortal.crosschain.BitcoinyTransaction;
import org.qortal.crosschain.ForeignBlockchainException;
import org.qortal.crosschain.Litecoin;
import org.qortal.crosschain.SimpleTransaction;
@Path("/crosschain/ltc")
@Tag(name = "Cross-Chain (Litecoin)")
@ -89,12 +89,12 @@ public class CrossChainLitecoinResource {
),
responses = {
@ApiResponse(
content = @Content(array = @ArraySchema( schema = @Schema( implementation = BitcoinyTransaction.class ) ) )
content = @Content(array = @ArraySchema( schema = @Schema( implementation = SimpleTransaction.class ) ) )
)
}
)
@ApiErrors({ApiError.INVALID_PRIVATE_KEY, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE})
public List<BitcoinyTransaction> getLitecoinWalletTransactions(String key58) {
public List<SimpleTransaction> getLitecoinWalletTransactions(String key58) {
Security.checkApiCallAllowed(request);
Litecoin litecoin = Litecoin.getInstance();

View File

@ -91,7 +91,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
return this.params;
}
// Interface obligations
// Interface obligations
@Override
public boolean isValidAddress(String address) {
@ -171,7 +171,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
/**
* Returns fixed P2SH spending fee, in sats per 1000bytes, optionally for historic timestamp.
*
*
* @param timestamp optional milliseconds since epoch, or null for 'now'
* @return sats per 1000bytes
* @throws ForeignBlockchainException if something went wrong
@ -271,7 +271,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
/**
* Returns bitcoinj transaction sending <tt>amount</tt> to <tt>recipient</tt>.
*
*
* @param xprv58 BIP32 private key
* @param recipient P2PKH address
* @param amount unscaled amount
@ -303,7 +303,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
/**
* Returns bitcoinj transaction sending <tt>amount</tt> to <tt>recipient</tt> using default fees.
*
*
* @param xprv58 BIP32 private key
* @param recipient P2PKH address
* @param amount unscaled amount
@ -332,7 +332,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
return balance.value;
}
public List<BitcoinyTransaction> getWalletTransactions(String key58) throws ForeignBlockchainException {
public List<SimpleTransaction> getWalletTransactions(String key58) throws ForeignBlockchainException {
Context.propagate(bitcoinjContext);
Wallet wallet = walletFromDeterministicKey58(key58);
@ -344,6 +344,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
List<DeterministicKey> keys = new ArrayList<>(keyChain.getLeafKeys());
Set<BitcoinyTransaction> walletTransactions = new HashSet<>();
Set<String> keySet = new HashSet<>();
int ki = 0;
do {
@ -354,6 +355,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
// Check for transactions
Address address = Address.fromKey(this.params, dKey, ScriptType.P2PKH);
keySet.add(address.toString());
byte[] script = ScriptBuilder.createOutputScript(address).getProgram();
// Ask for transaction history - if it's empty then key has never been used
@ -377,9 +379,41 @@ public abstract class Bitcoiny implements ForeignBlockchain {
// Process new keys
} while (true);
Comparator<BitcoinyTransaction> newestTimestampFirstComparator = Comparator.comparingInt((BitcoinyTransaction txn) -> txn.timestamp).reversed();
Comparator<SimpleTransaction> newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed();
return walletTransactions.stream().sorted(newestTimestampFirstComparator).collect(Collectors.toList());
return walletTransactions.stream().map(t -> convertToSimpleTransaction(t, keySet)).sorted(newestTimestampFirstComparator).collect(Collectors.toList());
}
protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set<String> keySet) {
long amount = 0;
long total = 0L;
for (BitcoinyTransaction.Input input : t.inputs) {
try {
BitcoinyTransaction t2 = getTransaction(input.outputTxHash);
List<String> senders = t2.outputs.get(input.outputVout).addresses;
for (String sender : senders) {
if (keySet.contains(sender)) {
total += t2.outputs.get(input.outputVout).value;
}
}
} catch (ForeignBlockchainException e) {
LOGGER.trace("Failed to retrieve transaction information {}", input.outputTxHash);
}
}
if (t.outputs != null && !t.outputs.isEmpty()) {
for (BitcoinyTransaction.Output output : t.outputs) {
for (String address : output.addresses) {
if (keySet.contains(address)) {
if (total > 0L) {
amount -= (total - output.value);
} else {
amount += output.value;
}
}
}
}
}
return new SimpleTransaction(t.txHash, t.timestamp, amount);
}
/**
@ -421,7 +455,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
* If there are no unspent outputs then either:
* a) all the outputs have been spent
* b) address has never been used
*
*
* For case (a) we want to remember not to check this address (key) again.
*/
@ -501,7 +535,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
* If there are no unspent outputs then either:
* a) all the outputs have been spent
* b) address has never been used
*
*
* For case (a) we want to remember not to check this address (key) again.
*/

View File

@ -0,0 +1,32 @@
package org.qortal.crosschain;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class SimpleTransaction {
private String txHash;
private Integer timestamp;
private long totalAmount;
public SimpleTransaction() {
}
public SimpleTransaction(String txHash, Integer timestamp, long totalAmount) {
this.txHash = txHash;
this.timestamp = timestamp;
this.totalAmount = totalAmount;
}
public String getTxHash() {
return txHash;
}
public Integer getTimestamp() {
return timestamp;
}
public long getTotalAmount() {
return totalAmount;
}
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -5,6 +5,7 @@ import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.Reader;
import java.util.List;
import java.util.Locale;
import javax.xml.bind.JAXBContext;
@ -156,6 +157,7 @@ public class Settings {
private String repositoryPath = "db";
/** Repository connection pool size. Needs to be a bit bigger than maxNetworkThreadPoolSize */
private int repositoryConnectionPoolSize = 100;
private List<String> fixedNetwork;
// Auto-update sources
private String[] autoUpdateRepos = new String[] {
@ -528,4 +530,7 @@ public class Settings {
return this.onlineSignaturesTrimBatchSize;
}
public List<String> getFixedNetwork() {
return fixedNetwork;
}
}

View File

@ -7,6 +7,7 @@ import java.util.List;
import java.util.stream.Collectors;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.block.Block;
@ -83,6 +84,7 @@ public class BlockTests extends Common {
}
@Test
@Ignore(value = "Doesn't work, to be fixed later")
public void testBlockSerialization() throws DataException, TransformationException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount signingAccount = Common.getTestAccount(repository, "alice");

View File

@ -2,10 +2,12 @@ package org.qortal.test;
import java.awt.TrayIcon.MessageType;
import org.junit.Ignore;
import org.junit.Test;
import org.qortal.gui.SplashFrame;
import org.qortal.gui.SysTray;
@Ignore
public class GuiTests {
@Test

View File

@ -1,5 +1,6 @@
package org.qortal.test;
import org.junit.Ignore;
import org.junit.Test;
import org.qortal.crypto.MemoryPoW;
@ -7,6 +8,7 @@ import static org.junit.Assert.*;
import java.util.Random;
@Ignore
public class MemoryPoWTests {
private static final int workBufferLength = 8 * 1024 * 1024;

View File

@ -1,5 +1,6 @@
package org.qortal.test;
import org.junit.Ignore;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.data.transaction.TransactionData;
@ -37,6 +38,7 @@ public class SerializationTests extends Common {
}
@Test
@Ignore(value = "Doesn't work, to be fixed later")
public void testTransactions() throws DataException, TransformationException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount signingAccount = Common.getTestAccount(repository, "alice");

View File

@ -2,6 +2,7 @@ package org.qortal.test;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.qortal.account.Account;
import org.qortal.account.PrivateKeyAccount;
@ -30,6 +31,7 @@ import static org.junit.Assert.*;
import java.util.List;
import java.util.Random;
@Ignore(value = "Doesn't work, to be fixed later")
public class TransferPrivsTests extends Common {
private static List<Integer> cumulativeBlocksByLevel;

View File

@ -5,6 +5,7 @@ import static org.junit.Assert.*;
import java.util.Collections;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.qortal.api.resource.AddressesResource;
import org.qortal.test.common.ApiCommon;
@ -24,6 +25,7 @@ public class AddressesApiTests extends ApiCommon {
}
@Test
@Ignore(value = "Doesn't work, to be fixed later")
public void testGetOnlineAccounts() {
assertNotNull(this.addressesResource.getOnlineAccounts());
}

View File

@ -4,6 +4,7 @@ import static org.junit.Assert.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.qortal.crosschain.Bitcoin;
import org.qortal.crosschain.ForeignBlockchainException;
@ -43,6 +44,7 @@ public class HtlcTests extends Common {
}
@Test
@Ignore(value = "Doesn't work, to be fixed later")
public void testHtlcSecretCaching() throws ForeignBlockchainException {
String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";
byte[] expectedSecret = "This string is exactly 32 bytes!".getBytes();

View File

@ -8,6 +8,7 @@ import org.bitcoinj.core.Transaction;
import org.bitcoinj.store.BlockStoreException;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.qortal.crosschain.ForeignBlockchainException;
import org.qortal.crosschain.Litecoin;
@ -50,6 +51,7 @@ public class LitecoinTests extends Common {
}
@Test
@Ignore(value = "Doesn't work, to be fixed later")
public void testFindHtlcSecret() throws ForeignBlockchainException {
// This actually exists on TEST3 but can take a while to fetch
String p2shAddress = "2N8WCg52ULCtDSMjkgVTm5mtPdCsUptkHWE";

View File

@ -8,11 +8,7 @@ import java.util.stream.Collectors;
import org.bitcoinj.core.AddressFormatException;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.crosschain.Bitcoin;
import org.qortal.crosschain.Bitcoiny;
import org.qortal.crosschain.BitcoinyTransaction;
import org.qortal.crosschain.ForeignBlockchainException;
import org.qortal.crosschain.Litecoin;
import org.qortal.crosschain.*;
import org.qortal.settings.Settings;
public class GetWalletTransactions {
@ -69,7 +65,7 @@ public class GetWalletTransactions {
System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId()));
// Grab all outputs from transaction
List<BitcoinyTransaction> transactions = null;
List<SimpleTransaction> transactions = null;
try {
transactions = bitcoiny.getWalletTransactions(key58);
} catch (ForeignBlockchainException e) {
@ -79,7 +75,7 @@ public class GetWalletTransactions {
System.out.println(String.format("Found %d transaction%s", transactions.size(), (transactions.size() != 1 ? "s" : "")));
for (BitcoinyTransaction transaction : transactions.stream().sorted(Comparator.comparingInt(t -> t.timestamp)).collect(Collectors.toList()))
for (SimpleTransaction transaction : transactions.stream().sorted(Comparator.comparingInt(SimpleTransaction::getTimestamp)).collect(Collectors.toList()))
System.out.println(String.format("%s", transaction));
}

View File

@ -7,6 +7,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -27,7 +29,7 @@ import org.qortal.utils.Amounts;
import org.qortal.utils.Base58;
public class RewardTests extends Common {
private static final Logger LOGGER = LogManager.getLogger(RewardTests.class);
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
@ -130,19 +132,19 @@ public class RewardTests extends Common {
/*
* Example:
*
*
* Block reward is 100 QORT, QORA-holders' share is 0.20 (20%) = 20 QORT
*
*
* We hold 100 QORA
* Someone else holds 28 QORA
* Total QORA held: 128 QORA
*
*
* Our portion of that is 100 QORA / 128 QORA * 20 QORT = 15.625 QORT
*
*
* QORA holders earn at most 1 QORT per 250 QORA held.
*
*
* So we can earn at most 100 QORA / 250 QORAperQORT = 0.4 QORT
*
*
* Thus our block earning should be capped to 0.4 QORT.
*/
@ -289,7 +291,7 @@ public class RewardTests extends Common {
* Dilbert is only account 'online'.
* No founders online.
* Some legacy QORA holders.
*
*
* So Dilbert should receive 100% - legacy QORA holder's share.
*/
@ -348,11 +350,16 @@ public class RewardTests extends Common {
// Alice self share online
PrivateKeyAccount aliceSelfShare = Common.getTestAccount(repository, "alice-reward-share");
mintingAndOnlineAccounts.add(aliceSelfShare);
byte[] chloeRewardSharePrivateKey;
// Bob self-share NOT online
// Chloe self share online
byte[] chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0);
try {
chloeRewardSharePrivateKey = AccountUtils.rewardShare(repository, "chloe", "chloe", 0);
} catch (IllegalArgumentException ex) {
LOGGER.error("FAILED {}", ex.getLocalizedMessage(), ex);
throw ex;
}
PrivateKeyAccount chloeRewardShareAccount = new PrivateKeyAccount(repository, chloeRewardSharePrivateKey);
mintingAndOnlineAccounts.add(chloeRewardShareAccount);
@ -486,7 +493,7 @@ public class RewardTests extends Common {
byte[] dilbertRewardSharePrivateKey = AccountUtils.rewardShare(repository, "dilbert", "dilbert", 0);
PrivateKeyAccount dilbertRewardShareAccount = new PrivateKeyAccount(repository, dilbertRewardSharePrivateKey);
mintingAndOnlineAccounts.add(dilbertRewardShareAccount);
// Mint enough blocks to bump testAccount levels to 3 and 4
final int minterBlocksNeeded = cumulativeBlocksByLevel.get(4) - 20; // 20 blocks before level 4, so that the test accounts reach the correct levels
for (int bc = 0; bc < minterBlocksNeeded; ++bc)