diff --git a/pom.xml b/pom.xml index 7c7ac147..30ad1e04 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 4.1.3 + 4.2.2 jar true diff --git a/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java b/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java index d51e6852..7972c551 100644 --- a/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java +++ b/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java @@ -16,7 +16,9 @@ import javax.ws.rs.core.Context; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.ConnectException; import java.net.HttpURLConnection; +import java.net.ProtocolException; import java.net.URL; import java.util.Enumeration; @@ -44,16 +46,6 @@ public class DevProxyServerResource { try { String source = DevProxyManager.getInstance().getSourceHostAndPort(); - // Convert localhost / 127.0.0.1 to IPv6 [::1] - if (source.startsWith("localhost") || source.startsWith("127.0.0.1")) { - int port = 80; - String[] parts = source.split(":"); - if (parts.length > 1) { - port = Integer.parseInt(parts[1]); - } - source = String.format("[::1]:%d", port); - } - if (!inPath.startsWith("/")) { inPath = "/" + inPath; } @@ -61,76 +53,37 @@ public class DevProxyServerResource { String queryString = request.getQueryString() != null ? "?" + request.getQueryString() : ""; // Open URL - String urlString = String.format("http://%s%s%s", source, inPath, queryString); - URL url = new URL(urlString); - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod(request.getMethod()); + URL url = new URL(String.format("http://%s%s%s", source, inPath, queryString)); + HttpURLConnection con = (HttpURLConnection) url.openConnection(); - // Proxy the request headers - Enumeration headerNames = request.getHeaderNames(); - while (headerNames.hasMoreElements()) { - String headerName = headerNames.nextElement(); - String headerValue = request.getHeader(headerName); - con.setRequestProperty(headerName, headerValue); + // Proxy the request data + this.proxyRequestToConnection(request, con); + + try { + // Make the request and proxy the response code + response.setStatus(con.getResponseCode()); } + catch (ConnectException e) { - // TODO: proxy any POST parameters from "request" to "con" - - // Proxy the response code - int responseCode = con.getResponseCode(); - response.setStatus(responseCode); - - // Proxy the response headers - for (int i = 0; ; i++) { - String headerKey = con.getHeaderFieldKey(i); - String headerValue = con.getHeaderField(i); - if (headerKey != null && headerValue != null) { - response.addHeader(headerKey, headerValue); - continue; + // Tey converting localhost / 127.0.0.1 to IPv6 [::1] + if (source.startsWith("localhost") || source.startsWith("127.0.0.1")) { + int port = 80; + String[] parts = source.split(":"); + if (parts.length > 1) { + port = Integer.parseInt(parts[1]); + } + source = String.format("[::1]:%d", port); } - break; + + // Retry connection + url = new URL(String.format("http://%s%s%s", source, inPath, queryString)); + con = (HttpURLConnection) url.openConnection(); + this.proxyRequestToConnection(request, con); + response.setStatus(con.getResponseCode()); } - // Read the response body - InputStream inputStream = con.getInputStream(); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - byte[] buffer = new byte[4096]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } - byte[] data = outputStream.toByteArray(); // TODO: limit file size that can be read into memory - - // Close the streams - outputStream.close(); - inputStream.close(); - - // Extract filename - String filename = ""; - if (inPath.contains("/")) { - String[] parts = inPath.split("/"); - if (parts.length > 0) { - filename = parts[parts.length - 1]; - } - } - - // Parse and modify output if needed - if (HTMLParser.isHtmlFile(filename)) { - // HTML file - needs to be parsed - HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true); - htmlParser.addAdditionalHeaderTags(); - response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;"); - response.setContentType(con.getContentType()); - response.setContentLength(htmlParser.getData().length); - response.getOutputStream().write(htmlParser.getData()); - } - else { - // Regular file - can be streamed directly - response.addHeader("Content-Security-Policy", "default-src 'self'"); - response.setContentType(con.getContentType()); - response.setContentLength(data.length); - response.getOutputStream().write(data); - } + // Proxy the response data back to the caller + this.proxyConnectionToResponse(con, response, inPath); } catch (IOException e) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, e.getMessage()); @@ -139,4 +92,73 @@ public class DevProxyServerResource { return response; } + private void proxyRequestToConnection(HttpServletRequest request, HttpURLConnection con) throws ProtocolException { + // Proxy the request method + con.setRequestMethod(request.getMethod()); + + // Proxy the request headers + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String headerName = headerNames.nextElement(); + String headerValue = request.getHeader(headerName); + con.setRequestProperty(headerName, headerValue); + } + + // TODO: proxy any POST parameters from "request" to "con" + } + + private void proxyConnectionToResponse(HttpURLConnection con, HttpServletResponse response, String inPath) throws IOException { + // Proxy the response headers + for (int i = 0; ; i++) { + String headerKey = con.getHeaderFieldKey(i); + String headerValue = con.getHeaderField(i); + if (headerKey != null && headerValue != null) { + response.addHeader(headerKey, headerValue); + continue; + } + break; + } + + // Read the response body + InputStream inputStream = con.getInputStream(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + byte[] data = outputStream.toByteArray(); // TODO: limit file size that can be read into memory + + // Close the streams + outputStream.close(); + inputStream.close(); + + // Extract filename + String filename = ""; + if (inPath.contains("/")) { + String[] parts = inPath.split("/"); + if (parts.length > 0) { + filename = parts[parts.length - 1]; + } + } + + // Parse and modify output if needed + if (HTMLParser.isHtmlFile(filename)) { + // HTML file - needs to be parsed + HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true); + htmlParser.addAdditionalHeaderTags(); + response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;"); + response.setContentType(con.getContentType()); + response.setContentLength(htmlParser.getData().length); + response.getOutputStream().write(htmlParser.getData()); + } + else { + // Regular file - can be streamed directly + response.addHeader("Content-Security-Policy", "default-src 'self'"); + response.setContentType(con.getContentType()); + response.setContentLength(data.length); + response.getOutputStream().write(data); + } + } + } diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index bc879f23..b1ed7e3c 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -380,9 +380,13 @@ public class BlockMinter extends Thread { parentSignatureForLastLowWeightBlock = null; timeOfLastLowWeightBlock = null; + Long unconfirmedStartTime = NTP.getTime(); + // Add unconfirmed transactions addUnconfirmedTransactions(repository, newBlock); + LOGGER.info(String.format("Adding %d unconfirmed transactions took %d ms", newBlock.getTransactions().size(), (NTP.getTime()-unconfirmedStartTime))); + // Sign to create block's signature newBlock.sign(); @@ -484,6 +488,9 @@ public class BlockMinter extends Thread { // Sign to create block's signature, needed by Block.isValid() newBlock.sign(); + // User-defined limit per block + int limit = Settings.getInstance().getMaxTransactionsPerBlock(); + // Attempt to add transactions until block is full, or we run out // If a transaction makes the block invalid then skip it and it'll either expire or be in next block. for (TransactionData transactionData : unconfirmedTransactions) { @@ -496,6 +503,12 @@ public class BlockMinter extends Thread { LOGGER.debug(() -> String.format("Skipping invalid transaction %s during block minting", Base58.encode(transactionData.getSignature()))); newBlock.deleteTransaction(transactionData); } + + // User-defined limit per block + List transactions = newBlock.getTransactions(); + if (transactions != null && transactions.size() >= limit) { + break; + } } } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index b6efd26f..bbf72038 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1305,13 +1305,6 @@ public class Controller extends Thread { TransactionImporter.getInstance().onNetworkTransactionSignaturesMessage(peer, message); break; - case GET_ONLINE_ACCOUNTS: - case ONLINE_ACCOUNTS: - case GET_ONLINE_ACCOUNTS_V2: - case ONLINE_ACCOUNTS_V2: - // No longer supported - to be eventually removed - break; - case GET_ONLINE_ACCOUNTS_V3: OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV3Message(peer, message); break; diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 224228b8..25cace2f 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -414,7 +414,7 @@ public class OnlineAccountsManager { boolean isSuperiorEntry = isOnlineAccountsDataSuperior(onlineAccountData); if (isSuperiorEntry) // Remove existing inferior entry so it can be re-added below (it's likely the existing copy is missing a nonce value) - onlineAccounts.remove(onlineAccountData); + onlineAccounts.removeIf(a -> Objects.equals(a.getPublicKey(), onlineAccountData.getPublicKey())); boolean isNewEntry = onlineAccounts.add(onlineAccountData); diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index 99eaf105..698ad487 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -25,7 +25,8 @@ public class NamesDatabaseIntegrityCheck { TransactionType.REGISTER_NAME, TransactionType.UPDATE_NAME, TransactionType.BUY_NAME, - TransactionType.SELL_NAME + TransactionType.SELL_NAME, + TransactionType.CANCEL_SELL_NAME ); private List nameTransactions = new ArrayList<>(); diff --git a/src/main/java/org/qortal/data/network/OnlineAccountData.java b/src/main/java/org/qortal/data/network/OnlineAccountData.java index bd4842db..a1e1b30f 100644 --- a/src/main/java/org/qortal/data/network/OnlineAccountData.java +++ b/src/main/java/org/qortal/data/network/OnlineAccountData.java @@ -1,6 +1,7 @@ package org.qortal.data.network; import java.util.Arrays; +import java.util.Objects; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -34,10 +35,6 @@ public class OnlineAccountData { this.nonce = nonce; } - public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey) { - this(timestamp, signature, publicKey, null); - } - public long getTimestamp() { return this.timestamp; } @@ -76,6 +73,10 @@ public class OnlineAccountData { if (otherOnlineAccountData.timestamp != this.timestamp) return false; + // Almost as quick + if (!Objects.equals(otherOnlineAccountData.nonce, this.nonce)) + return false; + if (!Arrays.equals(otherOnlineAccountData.publicKey, this.publicKey)) return false; @@ -88,9 +89,10 @@ public class OnlineAccountData { public int hashCode() { int h = this.hash; if (h == 0) { - this.hash = h = Long.hashCode(this.timestamp) - ^ Arrays.hashCode(this.publicKey); + h = Objects.hash(timestamp, nonce); + h = 31 * h + Arrays.hashCode(publicKey); // We don't use signature because newer aggregate signatures use random nonces + this.hash = h; } return h; } diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index ca79f367..a3528a66 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -187,7 +187,7 @@ public class Network { this.bindAddress = bindAddress; // Store the selected address, so that it can be used by other parts of the app break; // We don't want to bind to more than one address - } catch (UnknownHostException e) { + } catch (UnknownHostException | UnsupportedAddressTypeException e) { LOGGER.error("Can't bind listen socket to address {}", Settings.getInstance().getBindAddress()); if (i == bindAddresses.size()-1) { // Only throw an exception if all addresses have been tried throw new IOException("Can't bind listen socket to address", e); diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java deleted file mode 100644 index ae98cf40..00000000 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.qortal.network.message; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import org.qortal.data.network.OnlineAccountData; -import org.qortal.transform.Transformer; - -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; - -public class GetOnlineAccountsMessage extends Message { - private static final int MAX_ACCOUNT_COUNT = 5000; - - private List onlineAccounts; - - public GetOnlineAccountsMessage(List onlineAccounts) { - super(MessageType.GET_ONLINE_ACCOUNTS); - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - try { - bytes.write(Ints.toByteArray(onlineAccounts.size())); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - - bytes.write(onlineAccountData.getPublicKey()); - } - } catch (IOException e) { - throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); - } - - this.dataBytes = bytes.toByteArray(); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - } - - private GetOnlineAccountsMessage(int id, List onlineAccounts) { - super(id, MessageType.GET_ONLINE_ACCOUNTS); - - this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList()); - } - - public List getOnlineAccounts() { - return this.onlineAccounts; - } - - public static Message fromByteBuffer(int id, ByteBuffer bytes) { - final int accountCount = bytes.getInt(); - - List onlineAccounts = new ArrayList<>(accountCount); - - for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) { - long timestamp = bytes.getLong(); - - byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - bytes.get(publicKey); - - onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey)); - } - - return new GetOnlineAccountsMessage(id, onlineAccounts); - } - -} diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java deleted file mode 100644 index fe6b5d72..00000000 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java +++ /dev/null @@ -1,109 +0,0 @@ -package org.qortal.network.message; - -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; -import org.qortal.data.network.OnlineAccountData; -import org.qortal.transform.Transformer; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * For requesting online accounts info from remote peer, given our list of online accounts. - * - * Different format to V1: - * V1 is: number of entries, then timestamp + pubkey for each entry - * V2 is: groups of: number of entries, timestamp, then pubkey for each entry - * - * Also V2 only builds online accounts message once! - */ -public class GetOnlineAccountsV2Message extends Message { - - private List onlineAccounts; - - public GetOnlineAccountsV2Message(List onlineAccounts) { - super(MessageType.GET_ONLINE_ACCOUNTS_V2); - - // If we don't have ANY online accounts then it's an easier construction... - if (onlineAccounts.isEmpty()) { - // Always supply a number of accounts - this.dataBytes = Ints.toByteArray(0); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - return; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - Long timestamp = onlineAccountData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH; - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - try { - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - if (onlineAccountData.getTimestamp() == timestamp) - bytes.write(onlineAccountData.getPublicKey()); - } - } - } catch (IOException e) { - throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); - } - - this.dataBytes = bytes.toByteArray(); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - } - - private GetOnlineAccountsV2Message(int id, List onlineAccounts) { - super(id, MessageType.GET_ONLINE_ACCOUNTS_V2); - - this.onlineAccounts = onlineAccounts; - } - - public List getOnlineAccounts() { - return this.onlineAccounts; - } - - public static Message fromByteBuffer(int id, ByteBuffer bytes) { - int accountCount = bytes.getInt(); - - List onlineAccounts = new ArrayList<>(accountCount); - - while (accountCount > 0) { - long timestamp = bytes.getLong(); - - for (int i = 0; i < accountCount; ++i) { - byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - bytes.get(publicKey); - - onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey)); - } - - if (bytes.hasRemaining()) { - accountCount = bytes.getInt(); - } else { - // we've finished - accountCount = 0; - } - } - - return new GetOnlineAccountsV2Message(id, onlineAccounts); - } - -} diff --git a/src/main/java/org/qortal/network/message/MessageType.java b/src/main/java/org/qortal/network/message/MessageType.java index 4dd4a3c8..6b420e2d 100644 --- a/src/main/java/org/qortal/network/message/MessageType.java +++ b/src/main/java/org/qortal/network/message/MessageType.java @@ -43,11 +43,7 @@ public enum MessageType { BLOCK_SUMMARIES(70, BlockSummariesMessage::fromByteBuffer), GET_BLOCK_SUMMARIES(71, GetBlockSummariesMessage::fromByteBuffer), BLOCK_SUMMARIES_V2(72, BlockSummariesV2Message::fromByteBuffer), - - ONLINE_ACCOUNTS(80, OnlineAccountsMessage::fromByteBuffer), - GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer), - ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer), - GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer), + ONLINE_ACCOUNTS_V3(84, OnlineAccountsV3Message::fromByteBuffer), GET_ONLINE_ACCOUNTS_V3(85, GetOnlineAccountsV3Message::fromByteBuffer), diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java deleted file mode 100644 index e7e4c32c..00000000 --- a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.qortal.network.message; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import org.qortal.data.network.OnlineAccountData; -import org.qortal.transform.Transformer; - -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; - -public class OnlineAccountsMessage extends Message { - private static final int MAX_ACCOUNT_COUNT = 5000; - - private List onlineAccounts; - - public OnlineAccountsMessage(List onlineAccounts) { - super(MessageType.ONLINE_ACCOUNTS); - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - try { - bytes.write(Ints.toByteArray(onlineAccounts.size())); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - - bytes.write(onlineAccountData.getSignature()); - - bytes.write(onlineAccountData.getPublicKey()); - } - } catch (IOException e) { - throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); - } - - this.dataBytes = bytes.toByteArray(); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - } - - private OnlineAccountsMessage(int id, List onlineAccounts) { - super(id, MessageType.ONLINE_ACCOUNTS); - - this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList()); - } - - public List getOnlineAccounts() { - return this.onlineAccounts; - } - - public static Message fromByteBuffer(int id, ByteBuffer bytes) { - final int accountCount = bytes.getInt(); - - List onlineAccounts = new ArrayList<>(accountCount); - - for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) { - long timestamp = bytes.getLong(); - - byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; - bytes.get(signature); - - byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - bytes.get(publicKey); - - OnlineAccountData onlineAccountData = new OnlineAccountData(timestamp, signature, publicKey); - onlineAccounts.add(onlineAccountData); - } - - return new OnlineAccountsMessage(id, onlineAccounts); - } - -} diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java deleted file mode 100644 index 6803e3bf..00000000 --- a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.qortal.network.message; - -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; -import org.qortal.data.network.OnlineAccountData; -import org.qortal.transform.Transformer; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * For sending online accounts info to remote peer. - * - * Different format to V1: - * V1 is: number of entries, then timestamp + sig + pubkey for each entry - * V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry - * - * Also V2 only builds online accounts message once! - */ -public class OnlineAccountsV2Message extends Message { - - private List onlineAccounts; - - public OnlineAccountsV2Message(List onlineAccounts) { - super(MessageType.ONLINE_ACCOUNTS_V2); - - // Shortcut in case we have no online accounts - if (onlineAccounts.isEmpty()) { - this.dataBytes = Ints.toByteArray(0); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - return; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - Long timestamp = onlineAccountData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - try { - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - if (onlineAccountData.getTimestamp() == timestamp) { - bytes.write(onlineAccountData.getSignature()); - bytes.write(onlineAccountData.getPublicKey()); - } - } - } - } catch (IOException e) { - throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); - } - - this.dataBytes = bytes.toByteArray(); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - } - - private OnlineAccountsV2Message(int id, List onlineAccounts) { - super(id, MessageType.ONLINE_ACCOUNTS_V2); - - this.onlineAccounts = onlineAccounts; - } - - public List getOnlineAccounts() { - return this.onlineAccounts; - } - - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { - int accountCount = bytes.getInt(); - - List onlineAccounts = new ArrayList<>(accountCount); - - while (accountCount > 0) { - long timestamp = bytes.getLong(); - - for (int i = 0; i < accountCount; ++i) { - byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; - bytes.get(signature); - - byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - bytes.get(publicKey); - - onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey)); - } - - if (bytes.hasRemaining()) { - accountCount = bytes.getInt(); - } else { - // we've finished - accountCount = 0; - } - } - - return new OnlineAccountsV2Message(id, onlineAccounts); - } - -} diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java index d554d96c..c057fbce 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java @@ -99,9 +99,10 @@ public class OnlineAccountsV3Message extends Message { bytes.get(publicKey); // Nonce is optional - will be -1 if missing + // ... but we should skip/ignore an online account if it has no nonce Integer nonce = bytes.getInt(); if (nonce < 0) { - nonce = null; + continue; } onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey, nonce)); diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index 6cc88290..41986cad 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -314,7 +314,7 @@ public interface TransactionRepository { * @return list of transactions, or empty if none. * @throws DataException */ - public List getUnconfirmedTransactions(EnumSet excludedTxTypes) throws DataException; + public List getUnconfirmedTransactions(EnumSet excludedTxTypes, Integer limit) throws DataException; /** * Remove transaction from unconfirmed transactions pile. diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index 740b3e65..60b4e803 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -1429,8 +1429,10 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } @Override - public List getUnconfirmedTransactions(EnumSet excludedTxTypes) throws DataException { + public List getUnconfirmedTransactions(EnumSet excludedTxTypes, Integer limit) throws DataException { StringBuilder sql = new StringBuilder(1024); + List bindParams = new ArrayList<>(); + sql.append("SELECT signature FROM UnconfirmedTransactions "); sql.append("JOIN Transactions USING (signature) "); sql.append("WHERE type NOT IN ("); @@ -1446,12 +1448,17 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } sql.append(")"); - sql.append("ORDER BY created_when, signature"); + sql.append("ORDER BY created_when, signature "); + + if (limit != null) { + sql.append("LIMIT ?"); + bindParams.add(limit); + } List transactions = new ArrayList<>(); // Find transactions with no corresponding row in BlockTransactions - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) { + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { if (resultSet == null) return transactions; diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index c3d5a0c8..ac9b8857 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -146,6 +146,9 @@ public class Settings { /* How many blocks to cache locally. Defaulted to 10, which covers a typical Synchronizer request + a few spare */ private int blockCacheSize = 10; + /** Maximum number of transactions for the block minter to include in a block */ + private int maxTransactionsPerBlock = 25; + /** How long to keep old, full, AT state data (ms). */ private long atStatesMaxLifetime = 5 * 24 * 60 * 60 * 1000L; // milliseconds /** How often to attempt AT state trimming (ms). */ @@ -693,6 +696,10 @@ public class Settings { return this.blockCacheSize; } + public int getMaxTransactionsPerBlock() { + return this.maxTransactionsPerBlock; + } + public boolean isTestNet() { return this.isTestNet; } diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index f0e9b3f6..bd91f25a 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -641,7 +641,7 @@ public abstract class Transaction { BlockData latestBlockData = repository.getBlockRepository().getLastBlock(); EnumSet excludedTxTypes = EnumSet.of(TransactionType.CHAT, TransactionType.PRESENCE); - List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes); + List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes, null); unconfirmedTransactions.sort(getDataComparator()); diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js index 885a9ba6..d586e2e2 100644 --- a/src/main/resources/q-apps/q-apps.js +++ b/src/main/resources/q-apps/q-apps.js @@ -454,6 +454,10 @@ function getDefaultTimeout(action) { // User may take a long time to accept/deny the popup return 60 * 60 * 1000; + case "SEARCH_QDN_RESOURCES": + // Searching for data can be slow, especially when metadata and statuses are also being included + return 30 * 1000; + case "FETCH_QDN_RESOURCE": // Fetching data can take a while, especially if the status hasn't been checked first return 60 * 1000; diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java index c9e646f1..c8220d66 100644 --- a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java +++ b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java @@ -51,89 +51,6 @@ public class OnlineAccountsTests extends Common { } - @Test - public void testGetOnlineAccountsV2() throws MessageException { - List onlineAccountsOut = generateOnlineAccounts(false); - - Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut); - - byte[] messageBytes = messageOut.toBytes(); - ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes); - - GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer); - - List onlineAccountsIn = messageIn.getOnlineAccounts(); - - assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size()); - assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut)); - - Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut); - byte[] oldMessageBytes = oldMessageOut.toBytes(); - - long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count(); - - System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d", - onlineAccountsOut.size(), - numTimestamps, - numTimestamps != 1 ? "s" : "", - oldMessageBytes.length, - messageBytes.length)); - } - - @Test - public void testOnlineAccountsV2() throws MessageException { - List onlineAccountsOut = generateOnlineAccounts(true); - - Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut); - - byte[] messageBytes = messageOut.toBytes(); - ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes); - - OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer); - - List onlineAccountsIn = messageIn.getOnlineAccounts(); - - assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size()); - assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut)); - - Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut); - byte[] oldMessageBytes = oldMessageOut.toBytes(); - - long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count(); - - System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d", - onlineAccountsOut.size(), - numTimestamps, - numTimestamps != 1 ? "s" : "", - oldMessageBytes.length, - messageBytes.length)); - } - - private List generateOnlineAccounts(boolean withSignatures) { - List onlineAccounts = new ArrayList<>(); - - int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2 - - for (int t = 0; t < numTimestamps; ++t) { - int numAccounts = RANDOM.nextInt(3000); - - for (int a = 0; a < numAccounts; ++a) { - byte[] sig = null; - if (withSignatures) { - sig = new byte[Transformer.SIGNATURE_LENGTH]; - RANDOM.nextBytes(sig); - } - - byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - RANDOM.nextBytes(pubkey); - - onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey)); - } - } - - return onlineAccounts; - } - @Test public void testOnlineAccountsModulusV1() throws IllegalAccessException, DataException { try (final Repository repository = RepositoryManager.getRepository()) { diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java b/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java index 6136c1e1..2c3c01ca 100644 --- a/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java +++ b/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java @@ -26,41 +26,6 @@ public class OnlineAccountsV3Tests { Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); } - @Ignore("For informational use") - @Test - public void compareV2ToV3() throws MessageException { - List onlineAccounts = generateOnlineAccounts(false); - - // How many of each timestamp and leading byte (of public key) - Map> hashesByTimestampThenByte = convertToHashMaps(onlineAccounts); - - byte[] v3DataBytes = new GetOnlineAccountsV3Message(hashesByTimestampThenByte).toBytes(); - int v3ByteSize = v3DataBytes.length; - - byte[] v2DataBytes = new GetOnlineAccountsV2Message(onlineAccounts).toBytes(); - int v2ByteSize = v2DataBytes.length; - - int numTimestamps = hashesByTimestampThenByte.size(); - System.out.printf("For %d accounts split across %d timestamp%s: V2 size %d vs V3 size %d%n", - onlineAccounts.size(), - numTimestamps, - numTimestamps != 1 ? "s" : "", - v2ByteSize, - v3ByteSize - ); - - for (var outerMapEntry : hashesByTimestampThenByte.entrySet()) { - long timestamp = outerMapEntry.getKey(); - - var innerMap = outerMapEntry.getValue(); - - System.out.printf("For timestamp %d: %d / 256 slots used.%n", - timestamp, - innerMap.size() - ); - } - } - private Map> convertToHashMaps(List onlineAccounts) { // How many of each timestamp and leading byte (of public key) Map> hashesByTimestampThenByte = new HashMap<>(); @@ -200,7 +165,9 @@ public class OnlineAccountsV3Tests { byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH]; RANDOM.nextBytes(pubkey); - onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey)); + Integer nonce = RANDOM.nextInt(); + + onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey, nonce)); } }