crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Merge branch 'master' into arbitrary-resources-cache

Browse Source
This commit is contained in:
CalDescent 2023-08-05 13:01:00 +01:00
commit a3ab5238d3
21 changed files with 151 additions and 587 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.qortal</groupId>
<artifactId>qortal</artifactId>
<version>4.1.3</version>
<version>4.2.2</version>
<packaging>jar</packaging>
<properties>
<skipTests>true</skipTests>

170
src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java
View File

@ -16,7 +16,9 @@ import javax.ws.rs.core.Context;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.ProtocolException;
import java.net.URL;
import java.util.Enumeration;
@ -44,16 +46,6 @@ public class DevProxyServerResource {
try {
String source = DevProxyManager.getInstance().getSourceHostAndPort();
// Convert localhost / 127.0.0.1 to IPv6 [::1]
if (source.startsWith("localhost") || source.startsWith("127.0.0.1")) {
int port = 80;
String[] parts = source.split(":");
if (parts.length > 1) {
port = Integer.parseInt(parts[1]);
}
source = String.format("[::1]:%d", port);
}
if (!inPath.startsWith("/")) {
inPath = "/" + inPath;
}
@ -61,76 +53,37 @@ public class DevProxyServerResource {
String queryString = request.getQueryString() != null ? "?" + request.getQueryString() : "";
// Open URL
String urlString = String.format("http://%s%s%s", source, inPath, queryString);
URL url = new URL(urlString);
HttpURLConnection con = (HttpURLConnection) url.openConnection();
con.setRequestMethod(request.getMethod());
URL url = new URL(String.format("http://%s%s%s", source, inPath, queryString));
HttpURLConnection con = (HttpURLConnection) url.openConnection();
// Proxy the request headers
Enumeration<String> headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
String headerName = headerNames.nextElement();
String headerValue = request.getHeader(headerName);
con.setRequestProperty(headerName, headerValue);
// Proxy the request data
this.proxyRequestToConnection(request, con);
try {
// Make the request and proxy the response code
response.setStatus(con.getResponseCode());
}
catch (ConnectException e) {
// TODO: proxy any POST parameters from "request" to "con"
// Proxy the response code
int responseCode = con.getResponseCode();
response.setStatus(responseCode);
// Proxy the response headers
for (int i = 0; ; i++) {
String headerKey = con.getHeaderFieldKey(i);
String headerValue = con.getHeaderField(i);
if (headerKey != null && headerValue != null) {
response.addHeader(headerKey, headerValue);
continue;
// Tey converting localhost / 127.0.0.1 to IPv6 [::1]
if (source.startsWith("localhost") || source.startsWith("127.0.0.1")) {
int port = 80;
String[] parts = source.split(":");
if (parts.length > 1) {
port = Integer.parseInt(parts[1]);
}
source = String.format("[::1]:%d", port);
}
break;
// Retry connection
url = new URL(String.format("http://%s%s%s", source, inPath, queryString));
con = (HttpURLConnection) url.openConnection();
this.proxyRequestToConnection(request, con);
response.setStatus(con.getResponseCode());
}
// Read the response body
InputStream inputStream = con.getInputStream();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
byte[] data = outputStream.toByteArray(); // TODO: limit file size that can be read into memory
// Close the streams
outputStream.close();
inputStream.close();
// Extract filename
String filename = "";
if (inPath.contains("/")) {
String[] parts = inPath.split("/");
if (parts.length > 0) {
filename = parts[parts.length - 1];
}
}
// Parse and modify output if needed
if (HTMLParser.isHtmlFile(filename)) {
// HTML file - needs to be parsed
HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true);
htmlParser.addAdditionalHeaderTags();
response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;");
response.setContentType(con.getContentType());
response.setContentLength(htmlParser.getData().length);
response.getOutputStream().write(htmlParser.getData());
}
else {
// Regular file - can be streamed directly
response.addHeader("Content-Security-Policy", "default-src 'self'");
response.setContentType(con.getContentType());
response.setContentLength(data.length);
response.getOutputStream().write(data);
}
// Proxy the response data back to the caller
this.proxyConnectionToResponse(con, response, inPath);
} catch (IOException e) {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, e.getMessage());
@ -139,4 +92,73 @@ public class DevProxyServerResource {
return response;
}
private void proxyRequestToConnection(HttpServletRequest request, HttpURLConnection con) throws ProtocolException {
// Proxy the request method
con.setRequestMethod(request.getMethod());
// Proxy the request headers
Enumeration<String> headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
String headerName = headerNames.nextElement();
String headerValue = request.getHeader(headerName);
con.setRequestProperty(headerName, headerValue);
}
// TODO: proxy any POST parameters from "request" to "con"
}
private void proxyConnectionToResponse(HttpURLConnection con, HttpServletResponse response, String inPath) throws IOException {
// Proxy the response headers
for (int i = 0; ; i++) {
String headerKey = con.getHeaderFieldKey(i);
String headerValue = con.getHeaderField(i);
if (headerKey != null && headerValue != null) {
response.addHeader(headerKey, headerValue);
continue;
}
break;
}
// Read the response body
InputStream inputStream = con.getInputStream();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
byte[] data = outputStream.toByteArray(); // TODO: limit file size that can be read into memory
// Close the streams
outputStream.close();
inputStream.close();
// Extract filename
String filename = "";
if (inPath.contains("/")) {
String[] parts = inPath.split("/");
if (parts.length > 0) {
filename = parts[parts.length - 1];
}
}
// Parse and modify output if needed
if (HTMLParser.isHtmlFile(filename)) {
// HTML file - needs to be parsed
HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true);
htmlParser.addAdditionalHeaderTags();
response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;");
response.setContentType(con.getContentType());
response.setContentLength(htmlParser.getData().length);
response.getOutputStream().write(htmlParser.getData());
}
else {
// Regular file - can be streamed directly
response.addHeader("Content-Security-Policy", "default-src 'self'");
response.setContentType(con.getContentType());
response.setContentLength(data.length);
response.getOutputStream().write(data);
}
}
}

13
src/main/java/org/qortal/controller/BlockMinter.java
View File

@ -380,9 +380,13 @@ public class BlockMinter extends Thread {
parentSignatureForLastLowWeightBlock = null;
timeOfLastLowWeightBlock = null;
Long unconfirmedStartTime = NTP.getTime();
// Add unconfirmed transactions
addUnconfirmedTransactions(repository, newBlock);
LOGGER.info(String.format("Adding %d unconfirmed transactions took %d ms", newBlock.getTransactions().size(), (NTP.getTime()-unconfirmedStartTime)));
// Sign to create block's signature
newBlock.sign();
@ -484,6 +488,9 @@ public class BlockMinter extends Thread {
// Sign to create block's signature, needed by Block.isValid()
newBlock.sign();
// User-defined limit per block
int limit = Settings.getInstance().getMaxTransactionsPerBlock();
// Attempt to add transactions until block is full, or we run out
// If a transaction makes the block invalid then skip it and it'll either expire or be in next block.
for (TransactionData transactionData : unconfirmedTransactions) {
@ -496,6 +503,12 @@ public class BlockMinter extends Thread {
LOGGER.debug(() -> String.format("Skipping invalid transaction %s during block minting", Base58.encode(transactionData.getSignature())));
newBlock.deleteTransaction(transactionData);
}
// User-defined limit per block
List<Transaction> transactions = newBlock.getTransactions();
if (transactions != null && transactions.size() >= limit) {
break;
}
}
}

7
src/main/java/org/qortal/controller/Controller.java
View File

@ -1305,13 +1305,6 @@ public class Controller extends Thread {
TransactionImporter.getInstance().onNetworkTransactionSignaturesMessage(peer, message);
break;
case GET_ONLINE_ACCOUNTS:
case ONLINE_ACCOUNTS:
case GET_ONLINE_ACCOUNTS_V2:
case ONLINE_ACCOUNTS_V2:
// No longer supported - to be eventually removed
break;
case GET_ONLINE_ACCOUNTS_V3:
OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV3Message(peer, message);
break;

2
src/main/java/org/qortal/controller/OnlineAccountsManager.java
View File

@ -414,7 +414,7 @@ public class OnlineAccountsManager {
boolean isSuperiorEntry = isOnlineAccountsDataSuperior(onlineAccountData);
if (isSuperiorEntry)
// Remove existing inferior entry so it can be re-added below (it's likely the existing copy is missing a nonce value)
onlineAccounts.remove(onlineAccountData);
onlineAccounts.removeIf(a -> Objects.equals(a.getPublicKey(), onlineAccountData.getPublicKey()));
boolean isNewEntry = onlineAccounts.add(onlineAccountData);

3
src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java
View File

@ -25,7 +25,8 @@ public class NamesDatabaseIntegrityCheck {
TransactionType.REGISTER_NAME,
TransactionType.UPDATE_NAME,
TransactionType.BUY_NAME,
TransactionType.SELL_NAME
TransactionType.SELL_NAME,
TransactionType.CANCEL_SELL_NAME
);
private List<TransactionData> nameTransactions = new ArrayList<>();

14
src/main/java/org/qortal/data/network/OnlineAccountData.java
View File

@ -1,6 +1,7 @@
package org.qortal.data.network;
import java.util.Arrays;
import java.util.Objects;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@ -34,10 +35,6 @@ public class OnlineAccountData {
this.nonce = nonce;
}
public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey) {
this(timestamp, signature, publicKey, null);
}
public long getTimestamp() {
return this.timestamp;
}
@ -76,6 +73,10 @@ public class OnlineAccountData {
if (otherOnlineAccountData.timestamp != this.timestamp)
return false;
// Almost as quick
if (!Objects.equals(otherOnlineAccountData.nonce, this.nonce))
return false;
if (!Arrays.equals(otherOnlineAccountData.publicKey, this.publicKey))
return false;
@ -88,9 +89,10 @@ public class OnlineAccountData {
public int hashCode() {
int h = this.hash;
if (h == 0) {
this.hash = h = Long.hashCode(this.timestamp)
^ Arrays.hashCode(this.publicKey);
h = Objects.hash(timestamp, nonce);
h = 31 * h + Arrays.hashCode(publicKey);
// We don't use signature because newer aggregate signatures use random nonces
this.hash = h;
}
return h;
}

2
src/main/java/org/qortal/network/Network.java
View File

@ -187,7 +187,7 @@ public class Network {
this.bindAddress = bindAddress; // Store the selected address, so that it can be used by other parts of the app
break; // We don't want to bind to more than one address
} catch (UnknownHostException e) {
} catch (UnknownHostException | UnsupportedAddressTypeException e) {
LOGGER.error("Can't bind listen socket to address {}", Settings.getInstance().getBindAddress());
if (i == bindAddresses.size()-1) { // Only throw an exception if all addresses have been tried
throw new IOException("Can't bind listen socket to address", e);

69
src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java
View File

@ -1,69 +0,0 @@
package org.qortal.network.message;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.transform.Transformer;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
public class GetOnlineAccountsMessage extends Message {
private static final int MAX_ACCOUNT_COUNT = 5000;
private List<OnlineAccountData> onlineAccounts;
public GetOnlineAccountsMessage(List<OnlineAccountData> onlineAccounts) {
super(MessageType.GET_ONLINE_ACCOUNTS);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try {
bytes.write(Ints.toByteArray(onlineAccounts.size()));
for (OnlineAccountData onlineAccountData : onlineAccounts) {
bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp()));
bytes.write(onlineAccountData.getPublicKey());
}
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private GetOnlineAccountsMessage(int id, List<OnlineAccountData> onlineAccounts) {
super(id, MessageType.GET_ONLINE_ACCOUNTS);
this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList());
}
public List<OnlineAccountData> getOnlineAccounts() {
return this.onlineAccounts;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) {
final int accountCount = bytes.getInt();
List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) {
long timestamp = bytes.getLong();
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKey);
onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey));
}
return new GetOnlineAccountsMessage(id, onlineAccounts);
}
}

109
src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java
View File

@ -1,109 +0,0 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.transform.Transformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* For requesting online accounts info from remote peer, given our list of online accounts.
*
* Different format to V1:
* V1 is: number of entries, then timestamp + pubkey for each entry
* V2 is: groups of: number of entries, timestamp, then pubkey for each entry
*
* Also V2 only builds online accounts message once!
*/
public class GetOnlineAccountsV2Message extends Message {
private List<OnlineAccountData> onlineAccounts;
public GetOnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
super(MessageType.GET_ONLINE_ACCOUNTS_V2);
// If we don't have ANY online accounts then it's an easier construction...
if (onlineAccounts.isEmpty()) {
// Always supply a number of accounts
this.dataBytes = Ints.toByteArray(0);
this.checksumBytes = Message.generateChecksum(this.dataBytes);
return;
}
// How many of each timestamp
Map<Long, Integer> countByTimestamp = new HashMap<>();
for (OnlineAccountData onlineAccountData : onlineAccounts) {
Long timestamp = onlineAccountData.getTimestamp();
countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
}
// We should know exactly how many bytes to allocate now
int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+ onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH;
ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
try {
for (long timestamp : countByTimestamp.keySet()) {
bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
bytes.write(Longs.toByteArray(timestamp));
for (OnlineAccountData onlineAccountData : onlineAccounts) {
if (onlineAccountData.getTimestamp() == timestamp)
bytes.write(onlineAccountData.getPublicKey());
}
}
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private GetOnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
super(id, MessageType.GET_ONLINE_ACCOUNTS_V2);
this.onlineAccounts = onlineAccounts;
}
public List<OnlineAccountData> getOnlineAccounts() {
return this.onlineAccounts;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) {
int accountCount = bytes.getInt();
List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
while (accountCount > 0) {
long timestamp = bytes.getLong();
for (int i = 0; i < accountCount; ++i) {
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKey);
onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey));
}
if (bytes.hasRemaining()) {
accountCount = bytes.getInt();
} else {
// we've finished
accountCount = 0;
}
}
return new GetOnlineAccountsV2Message(id, onlineAccounts);
}
}

6
src/main/java/org/qortal/network/message/MessageType.java
View File

@ -43,11 +43,7 @@ public enum MessageType {
BLOCK_SUMMARIES(70, BlockSummariesMessage::fromByteBuffer),
GET_BLOCK_SUMMARIES(71, GetBlockSummariesMessage::fromByteBuffer),
BLOCK_SUMMARIES_V2(72, BlockSummariesV2Message::fromByteBuffer),
ONLINE_ACCOUNTS(80, OnlineAccountsMessage::fromByteBuffer),
GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer),
ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer),
GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer),
ONLINE_ACCOUNTS_V3(84, OnlineAccountsV3Message::fromByteBuffer),
GET_ONLINE_ACCOUNTS_V3(85, GetOnlineAccountsV3Message::fromByteBuffer),

75
src/main/java/org/qortal/network/message/OnlineAccountsMessage.java
View File

@ -1,75 +0,0 @@
package org.qortal.network.message;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.transform.Transformer;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
public class OnlineAccountsMessage extends Message {
private static final int MAX_ACCOUNT_COUNT = 5000;
private List<OnlineAccountData> onlineAccounts;
public OnlineAccountsMessage(List<OnlineAccountData> onlineAccounts) {
super(MessageType.ONLINE_ACCOUNTS);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try {
bytes.write(Ints.toByteArray(onlineAccounts.size()));
for (OnlineAccountData onlineAccountData : onlineAccounts) {
bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp()));
bytes.write(onlineAccountData.getSignature());
bytes.write(onlineAccountData.getPublicKey());
}
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private OnlineAccountsMessage(int id, List<OnlineAccountData> onlineAccounts) {
super(id, MessageType.ONLINE_ACCOUNTS);
this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList());
}
public List<OnlineAccountData> getOnlineAccounts() {
return this.onlineAccounts;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) {
final int accountCount = bytes.getInt();
List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) {
long timestamp = bytes.getLong();
byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
bytes.get(signature);
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKey);
OnlineAccountData onlineAccountData = new OnlineAccountData(timestamp, signature, publicKey);
onlineAccounts.add(onlineAccountData);
}
return new OnlineAccountsMessage(id, onlineAccounts);
}
}

113
src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java
View File

@ -1,113 +0,0 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import org.qortal.data.network.OnlineAccountData;
import org.qortal.transform.Transformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* For sending online accounts info to remote peer.
*
* Different format to V1:
* V1 is: number of entries, then timestamp + sig + pubkey for each entry
* V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry
*
* Also V2 only builds online accounts message once!
*/
public class OnlineAccountsV2Message extends Message {
private List<OnlineAccountData> onlineAccounts;
public OnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
super(MessageType.ONLINE_ACCOUNTS_V2);
// Shortcut in case we have no online accounts
if (onlineAccounts.isEmpty()) {
this.dataBytes = Ints.toByteArray(0);
this.checksumBytes = Message.generateChecksum(this.dataBytes);
return;
}
// How many of each timestamp
Map<Long, Integer> countByTimestamp = new HashMap<>();
for (OnlineAccountData onlineAccountData : onlineAccounts) {
Long timestamp = onlineAccountData.getTimestamp();
countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
}
// We should know exactly how many bytes to allocate now
int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+ onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH);
ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
try {
for (long timestamp : countByTimestamp.keySet()) {
bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
bytes.write(Longs.toByteArray(timestamp));
for (OnlineAccountData onlineAccountData : onlineAccounts) {
if (onlineAccountData.getTimestamp() == timestamp) {
bytes.write(onlineAccountData.getSignature());
bytes.write(onlineAccountData.getPublicKey());
}
}
}
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private OnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
super(id, MessageType.ONLINE_ACCOUNTS_V2);
this.onlineAccounts = onlineAccounts;
}
public List<OnlineAccountData> getOnlineAccounts() {
return this.onlineAccounts;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException {
int accountCount = bytes.getInt();
List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
while (accountCount > 0) {
long timestamp = bytes.getLong();
for (int i = 0; i < accountCount; ++i) {
byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
bytes.get(signature);
byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
bytes.get(publicKey);
onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey));
}
if (bytes.hasRemaining()) {
accountCount = bytes.getInt();
} else {
// we've finished
accountCount = 0;
}
}
return new OnlineAccountsV2Message(id, onlineAccounts);
}
}

3
src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java
View File

@ -99,9 +99,10 @@ public class OnlineAccountsV3Message extends Message {
bytes.get(publicKey);
// Nonce is optional - will be -1 if missing
// ... but we should skip/ignore an online account if it has no nonce
Integer nonce = bytes.getInt();
if (nonce < 0) {
nonce = null;
continue;
}
onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey, nonce));

2
src/main/java/org/qortal/repository/TransactionRepository.java
View File

@ -314,7 +314,7 @@ public interface TransactionRepository {
* @return list of transactions, or empty if none.
* @throws DataException
*/
public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes) throws DataException;
public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes, Integer limit) throws DataException;
/**
* Remove transaction from unconfirmed transactions pile.

13
src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java
View File

@ -1429,8 +1429,10 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
}
@Override
public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes) throws DataException {
public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes, Integer limit) throws DataException {
StringBuilder sql = new StringBuilder(1024);
List<Object> bindParams = new ArrayList<>();
sql.append("SELECT signature FROM UnconfirmedTransactions ");
sql.append("JOIN Transactions USING (signature) ");
sql.append("WHERE type NOT IN (");
@ -1446,12 +1448,17 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
}
sql.append(")");
sql.append("ORDER BY created_when, signature");
sql.append("ORDER BY created_when, signature ");
if (limit != null) {
sql.append("LIMIT ?");
bindParams.add(limit);
}
List<TransactionData> transactions = new ArrayList<>();
// Find transactions with no corresponding row in BlockTransactions
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) {
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return transactions;

7
src/main/java/org/qortal/settings/Settings.java
View File

@ -146,6 +146,9 @@ public class Settings {
/* How many blocks to cache locally. Defaulted to 10, which covers a typical Synchronizer request + a few spare */
private int blockCacheSize = 10;
/** Maximum number of transactions for the block minter to include in a block */
private int maxTransactionsPerBlock = 25;
/** How long to keep old, full, AT state data (ms). */
private long atStatesMaxLifetime = 5 * 24 * 60 * 60 * 1000L; // milliseconds
/** How often to attempt AT state trimming (ms). */
@ -693,6 +696,10 @@ public class Settings {
return this.blockCacheSize;
}
public int getMaxTransactionsPerBlock() {
return this.maxTransactionsPerBlock;
}
public boolean isTestNet() {
return this.isTestNet;
}

2
src/main/java/org/qortal/transaction/Transaction.java
View File

@ -641,7 +641,7 @@ public abstract class Transaction {
BlockData latestBlockData = repository.getBlockRepository().getLastBlock();
EnumSet<TransactionType> excludedTxTypes = EnumSet.of(TransactionType.CHAT, TransactionType.PRESENCE);
List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes);
List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes, null);
unconfirmedTransactions.sort(getDataComparator());

4
src/main/resources/q-apps/q-apps.js
View File

@ -454,6 +454,10 @@ function getDefaultTimeout(action) {
// User may take a long time to accept/deny the popup
return 60 * 60 * 1000;
case "SEARCH_QDN_RESOURCES":
// Searching for data can be slow, especially when metadata and statuses are also being included
return 30 * 1000;
case "FETCH_QDN_RESOURCE":
// Fetching data can take a while, especially if the status hasn't been checked first
return 60 * 1000;

83
src/test/java/org/qortal/test/network/OnlineAccountsTests.java
View File

@ -51,89 +51,6 @@ public class OnlineAccountsTests extends Common {
}
@Test
public void testGetOnlineAccountsV2() throws MessageException {
List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(false);
Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut);
byte[] messageBytes = messageOut.toBytes();
ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut);
byte[] oldMessageBytes = oldMessageOut.toBytes();
long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
onlineAccountsOut.size(),
numTimestamps,
numTimestamps != 1 ? "s" : "",
oldMessageBytes.length,
messageBytes.length));
}
@Test
public void testOnlineAccountsV2() throws MessageException {
List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(true);
Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut);
byte[] messageBytes = messageOut.toBytes();
ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut);
byte[] oldMessageBytes = oldMessageOut.toBytes();
long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
onlineAccountsOut.size(),
numTimestamps,
numTimestamps != 1 ? "s" : "",
oldMessageBytes.length,
messageBytes.length));
}
private List<OnlineAccountData> generateOnlineAccounts(boolean withSignatures) {
List<OnlineAccountData> onlineAccounts = new ArrayList<>();
int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2
for (int t = 0; t < numTimestamps; ++t) {
int numAccounts = RANDOM.nextInt(3000);
for (int a = 0; a < numAccounts; ++a) {
byte[] sig = null;
if (withSignatures) {
sig = new byte[Transformer.SIGNATURE_LENGTH];
RANDOM.nextBytes(sig);
}
byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH];
RANDOM.nextBytes(pubkey);
onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey));
}
}
return onlineAccounts;
}
@Test
public void testOnlineAccountsModulusV1() throws IllegalAccessException, DataException {
try (final Repository repository = RepositoryManager.getRepository()) {

39
src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java
View File

@ -26,41 +26,6 @@ public class OnlineAccountsV3Tests {
Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
}
@Ignore("For informational use")
@Test
public void compareV2ToV3() throws MessageException {
List<OnlineAccountData> onlineAccounts = generateOnlineAccounts(false);
// How many of each timestamp and leading byte (of public key)
Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte = convertToHashMaps(onlineAccounts);
byte[] v3DataBytes = new GetOnlineAccountsV3Message(hashesByTimestampThenByte).toBytes();
int v3ByteSize = v3DataBytes.length;
byte[] v2DataBytes = new GetOnlineAccountsV2Message(onlineAccounts).toBytes();
int v2ByteSize = v2DataBytes.length;
int numTimestamps = hashesByTimestampThenByte.size();
System.out.printf("For %d accounts split across %d timestamp%s: V2 size %d vs V3 size %d%n",
onlineAccounts.size(),
numTimestamps,
numTimestamps != 1 ? "s" : "",
v2ByteSize,
v3ByteSize
);
for (var outerMapEntry : hashesByTimestampThenByte.entrySet()) {
long timestamp = outerMapEntry.getKey();
var innerMap = outerMapEntry.getValue();
System.out.printf("For timestamp %d: %d / 256 slots used.%n",
timestamp,
innerMap.size()
);
}
}
private Map<Long, Map<Byte, byte[]>> convertToHashMaps(List<OnlineAccountData> onlineAccounts) {
// How many of each timestamp and leading byte (of public key)
Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte = new HashMap<>();
@ -200,7 +165,9 @@ public class OnlineAccountsV3Tests {
byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH];
RANDOM.nextBytes(pubkey);
onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey));
Integer nonce = RANDOM.nextInt();
onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey, nonce));
}
}