From fd8d720946a5f7e538699ce0a446631c5daf2638 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 23 Jun 2023 13:30:10 +0100 Subject: [PATCH 01/15] Added support for group encryption in service validation. --- .../org/qortal/arbitrary/misc/Service.java | 5 +++-- .../test/arbitrary/ArbitraryServiceTests.java | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/arbitrary/misc/Service.java b/src/main/java/org/qortal/arbitrary/misc/Service.java index 94ca9252..2b8f8d02 100644 --- a/src/main/java/org/qortal/arbitrary/misc/Service.java +++ b/src/main/java/org/qortal/arbitrary/misc/Service.java @@ -186,6 +186,7 @@ public enum Service { private static final ObjectMapper objectMapper = new ObjectMapper(); private static final String encryptedDataPrefix = "qortalEncryptedData"; + private static final String encryptedGroupDataPrefix = "qortalGroupEncryptedData"; Service(int value, boolean requiresValidation, Long maxSize, boolean single, boolean isPrivate, List requiredKeys) { this.value = value; @@ -221,10 +222,10 @@ public enum Service { // Validate private data for single file resources if (this.single) { String dataString = new String(data, StandardCharsets.UTF_8); - if (this.isPrivate && !dataString.startsWith(encryptedDataPrefix)) { + if (this.isPrivate && !dataString.startsWith(encryptedDataPrefix) && !dataString.startsWith(encryptedGroupDataPrefix)) { return ValidationResult.DATA_NOT_ENCRYPTED; } - if (!this.isPrivate && dataString.startsWith(encryptedDataPrefix)) { + if (!this.isPrivate && (dataString.startsWith(encryptedDataPrefix) || dataString.startsWith(encryptedGroupDataPrefix))) { return ValidationResult.DATA_ENCRYPTED; } } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java index 33632b4a..b4c10fac 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryServiceTests.java @@ -456,6 +456,25 @@ public class ArbitraryServiceTests extends Common { assertEquals(ValidationResult.OK, service.validate(filePath)); } + @Test + public void testValidPrivateGroupData() throws IOException { + String dataString = "qortalGroupEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1rfla5UBoEcoxbtjgZ9G7FLPb8V/Qfr0bfKWfvMmN06U/pgUdLuv2mGL2V0D3qYd1011MUzGdNG1qERjaCDz8GAi63+KnHHjfMtPgYt6bcqjs4CNV+ZZ4dIt3xxHYyVEBNc="; + + // Write the data a single file in a temp path + Path path = Files.createTempDirectory("testValidPrivateData"); + Path filePath = Paths.get(path.toString(), "test"); + filePath.toFile().deleteOnExit(); + + BufferedWriter writer = new BufferedWriter(new FileWriter(filePath.toFile())); + writer.write(dataString); + writer.close(); + + Service service = Service.FILE_PRIVATE; + assertTrue(service.isValidationRequired()); + + assertEquals(ValidationResult.OK, service.validate(filePath)); + } + @Test public void testEncryptedData() throws IOException { String dataString = "qortalEncryptedDatabMx4fELNTV+ifJxmv4+GcuOIJOTo+3qAvbWKNY2L1rfla5UBoEcoxbtjgZ9G7FLPb8V/Qfr0bfKWfvMmN06U/pgUdLuv2mGL2V0D3qYd1011MUzGdNG1qERjaCDz8GAi63+KnHHjfMtPgYt6bcqjs4CNV+ZZ4dIt3xxHYyVEBNc="; From c14fca5660c5dc0b833075ef7e978a919965b19d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 8 Jul 2023 11:05:14 +0100 Subject: [PATCH 02/15] Improved filtering of online accounts data. --- .../org/qortal/controller/OnlineAccountsManager.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 224228b8..5e64161d 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -743,8 +743,14 @@ public class OnlineAccountsManager { if (onlineAccounts == null) onlineAccounts = this.latestBlocksOnlineAccounts.get(timestamp); - if (onlineAccounts != null) - blocksOnlineAccounts.removeAll(onlineAccounts); + if (onlineAccounts != null) { + // Remove accounts with matching timestamp, nonce, and public key + final Set finalOnlineAccounts = onlineAccounts; + blocksOnlineAccounts.removeIf(a1 -> finalOnlineAccounts.stream() + .anyMatch(a2 -> a2.getTimestamp() == a1.getTimestamp() && + Objects.equals(a2.getNonce(), a1.getNonce()) && + Arrays.equals(a2.getPublicKey(), a1.getPublicKey()))); + } } /** From fe999a11f4c801dd3713d74b27659c7cbe4bae09 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 1 Jul 2023 14:01:08 +0100 Subject: [PATCH 03/15] Include CANCEL_SELL_NAME transactions when performing a complete rebuild of names. --- .../controller/repository/NamesDatabaseIntegrityCheck.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java index 99eaf105..698ad487 100644 --- a/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java +++ b/src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java @@ -25,7 +25,8 @@ public class NamesDatabaseIntegrityCheck { TransactionType.REGISTER_NAME, TransactionType.UPDATE_NAME, TransactionType.BUY_NAME, - TransactionType.SELL_NAME + TransactionType.SELL_NAME, + TransactionType.CANCEL_SELL_NAME ); private List nameTransactions = new ArrayList<>(); From 5f86ecafd9e36d8037d0539794efc9af1efe24c3 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 9 Jul 2023 12:35:46 +0100 Subject: [PATCH 04/15] Refactored developer proxy, and modified IPv6 fallback so that it only occurs on a connection failure. --- .../resource/DevProxyServerResource.java | 170 ++++++++++-------- 1 file changed, 96 insertions(+), 74 deletions(-) diff --git a/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java b/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java index d51e6852..7972c551 100644 --- a/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java +++ b/src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java @@ -16,7 +16,9 @@ import javax.ws.rs.core.Context; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.ConnectException; import java.net.HttpURLConnection; +import java.net.ProtocolException; import java.net.URL; import java.util.Enumeration; @@ -44,16 +46,6 @@ public class DevProxyServerResource { try { String source = DevProxyManager.getInstance().getSourceHostAndPort(); - // Convert localhost / 127.0.0.1 to IPv6 [::1] - if (source.startsWith("localhost") || source.startsWith("127.0.0.1")) { - int port = 80; - String[] parts = source.split(":"); - if (parts.length > 1) { - port = Integer.parseInt(parts[1]); - } - source = String.format("[::1]:%d", port); - } - if (!inPath.startsWith("/")) { inPath = "/" + inPath; } @@ -61,76 +53,37 @@ public class DevProxyServerResource { String queryString = request.getQueryString() != null ? "?" + request.getQueryString() : ""; // Open URL - String urlString = String.format("http://%s%s%s", source, inPath, queryString); - URL url = new URL(urlString); - HttpURLConnection con = (HttpURLConnection) url.openConnection(); - con.setRequestMethod(request.getMethod()); + URL url = new URL(String.format("http://%s%s%s", source, inPath, queryString)); + HttpURLConnection con = (HttpURLConnection) url.openConnection(); - // Proxy the request headers - Enumeration headerNames = request.getHeaderNames(); - while (headerNames.hasMoreElements()) { - String headerName = headerNames.nextElement(); - String headerValue = request.getHeader(headerName); - con.setRequestProperty(headerName, headerValue); + // Proxy the request data + this.proxyRequestToConnection(request, con); + + try { + // Make the request and proxy the response code + response.setStatus(con.getResponseCode()); } + catch (ConnectException e) { - // TODO: proxy any POST parameters from "request" to "con" - - // Proxy the response code - int responseCode = con.getResponseCode(); - response.setStatus(responseCode); - - // Proxy the response headers - for (int i = 0; ; i++) { - String headerKey = con.getHeaderFieldKey(i); - String headerValue = con.getHeaderField(i); - if (headerKey != null && headerValue != null) { - response.addHeader(headerKey, headerValue); - continue; + // Tey converting localhost / 127.0.0.1 to IPv6 [::1] + if (source.startsWith("localhost") || source.startsWith("127.0.0.1")) { + int port = 80; + String[] parts = source.split(":"); + if (parts.length > 1) { + port = Integer.parseInt(parts[1]); + } + source = String.format("[::1]:%d", port); } - break; + + // Retry connection + url = new URL(String.format("http://%s%s%s", source, inPath, queryString)); + con = (HttpURLConnection) url.openConnection(); + this.proxyRequestToConnection(request, con); + response.setStatus(con.getResponseCode()); } - // Read the response body - InputStream inputStream = con.getInputStream(); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - byte[] buffer = new byte[4096]; - int bytesRead; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } - byte[] data = outputStream.toByteArray(); // TODO: limit file size that can be read into memory - - // Close the streams - outputStream.close(); - inputStream.close(); - - // Extract filename - String filename = ""; - if (inPath.contains("/")) { - String[] parts = inPath.split("/"); - if (parts.length > 0) { - filename = parts[parts.length - 1]; - } - } - - // Parse and modify output if needed - if (HTMLParser.isHtmlFile(filename)) { - // HTML file - needs to be parsed - HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true); - htmlParser.addAdditionalHeaderTags(); - response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;"); - response.setContentType(con.getContentType()); - response.setContentLength(htmlParser.getData().length); - response.getOutputStream().write(htmlParser.getData()); - } - else { - // Regular file - can be streamed directly - response.addHeader("Content-Security-Policy", "default-src 'self'"); - response.setContentType(con.getContentType()); - response.setContentLength(data.length); - response.getOutputStream().write(data); - } + // Proxy the response data back to the caller + this.proxyConnectionToResponse(con, response, inPath); } catch (IOException e) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, e.getMessage()); @@ -139,4 +92,73 @@ public class DevProxyServerResource { return response; } + private void proxyRequestToConnection(HttpServletRequest request, HttpURLConnection con) throws ProtocolException { + // Proxy the request method + con.setRequestMethod(request.getMethod()); + + // Proxy the request headers + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String headerName = headerNames.nextElement(); + String headerValue = request.getHeader(headerName); + con.setRequestProperty(headerName, headerValue); + } + + // TODO: proxy any POST parameters from "request" to "con" + } + + private void proxyConnectionToResponse(HttpURLConnection con, HttpServletResponse response, String inPath) throws IOException { + // Proxy the response headers + for (int i = 0; ; i++) { + String headerKey = con.getHeaderFieldKey(i); + String headerValue = con.getHeaderField(i); + if (headerKey != null && headerValue != null) { + response.addHeader(headerKey, headerValue); + continue; + } + break; + } + + // Read the response body + InputStream inputStream = con.getInputStream(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + byte[] data = outputStream.toByteArray(); // TODO: limit file size that can be read into memory + + // Close the streams + outputStream.close(); + inputStream.close(); + + // Extract filename + String filename = ""; + if (inPath.contains("/")) { + String[] parts = inPath.split("/"); + if (parts.length > 0) { + filename = parts[parts.length - 1]; + } + } + + // Parse and modify output if needed + if (HTMLParser.isHtmlFile(filename)) { + // HTML file - needs to be parsed + HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true); + htmlParser.addAdditionalHeaderTags(); + response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;"); + response.setContentType(con.getContentType()); + response.setContentLength(htmlParser.getData().length); + response.getOutputStream().write(htmlParser.getData()); + } + else { + // Regular file - can be streamed directly + response.addHeader("Content-Security-Policy", "default-src 'self'"); + response.setContentType(con.getContentType()); + response.setContentLength(data.length); + response.getOutputStream().write(data); + } + } + } From 62908f867a322266b22057004735680487785939 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 16 Jul 2023 19:09:08 +0100 Subject: [PATCH 05/15] Bump version to 4.2.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7c7ac147..f236761e 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 4.1.3 + 4.2.0 jar true From 29dcd5300233636df0ddd52cb6c63e63caf6ba25 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 16 Jul 2023 20:04:45 +0100 Subject: [PATCH 06/15] Revert "Improved filtering of online accounts data." This reverts commit c14fca5660c5dc0b833075ef7e978a919965b19d. --- .../org/qortal/controller/OnlineAccountsManager.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 5e64161d..224228b8 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -743,14 +743,8 @@ public class OnlineAccountsManager { if (onlineAccounts == null) onlineAccounts = this.latestBlocksOnlineAccounts.get(timestamp); - if (onlineAccounts != null) { - // Remove accounts with matching timestamp, nonce, and public key - final Set finalOnlineAccounts = onlineAccounts; - blocksOnlineAccounts.removeIf(a1 -> finalOnlineAccounts.stream() - .anyMatch(a2 -> a2.getTimestamp() == a1.getTimestamp() && - Objects.equals(a2.getNonce(), a1.getNonce()) && - Arrays.equals(a2.getPublicKey(), a1.getPublicKey()))); - } + if (onlineAccounts != null) + blocksOnlineAccounts.removeAll(onlineAccounts); } /** From 8ae7a1d65be525bdd864806d8470030534ccde9e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 21 Jul 2023 14:28:47 +0100 Subject: [PATCH 07/15] Removed (Get)OnlineAccountsV1 and V2, as these are no longer used. --- .../org/qortal/controller/Controller.java | 7 -- .../message/GetOnlineAccountsMessage.java | 69 ----------- .../message/GetOnlineAccountsV2Message.java | 109 ----------------- .../qortal/network/message/MessageType.java | 6 +- .../message/OnlineAccountsMessage.java | 75 ------------ .../message/OnlineAccountsV2Message.java | 113 ------------------ .../test/network/OnlineAccountsTests.java | 83 ------------- .../test/network/OnlineAccountsV3Tests.java | 35 ------ 8 files changed, 1 insertion(+), 496 deletions(-) delete mode 100644 src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java delete mode 100644 src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java delete mode 100644 src/main/java/org/qortal/network/message/OnlineAccountsMessage.java delete mode 100644 src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index e1e90486..2cab24ec 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1278,13 +1278,6 @@ public class Controller extends Thread { TransactionImporter.getInstance().onNetworkTransactionSignaturesMessage(peer, message); break; - case GET_ONLINE_ACCOUNTS: - case ONLINE_ACCOUNTS: - case GET_ONLINE_ACCOUNTS_V2: - case ONLINE_ACCOUNTS_V2: - // No longer supported - to be eventually removed - break; - case GET_ONLINE_ACCOUNTS_V3: OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV3Message(peer, message); break; diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java deleted file mode 100644 index ae98cf40..00000000 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.qortal.network.message; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import org.qortal.data.network.OnlineAccountData; -import org.qortal.transform.Transformer; - -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; - -public class GetOnlineAccountsMessage extends Message { - private static final int MAX_ACCOUNT_COUNT = 5000; - - private List onlineAccounts; - - public GetOnlineAccountsMessage(List onlineAccounts) { - super(MessageType.GET_ONLINE_ACCOUNTS); - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - try { - bytes.write(Ints.toByteArray(onlineAccounts.size())); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - - bytes.write(onlineAccountData.getPublicKey()); - } - } catch (IOException e) { - throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); - } - - this.dataBytes = bytes.toByteArray(); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - } - - private GetOnlineAccountsMessage(int id, List onlineAccounts) { - super(id, MessageType.GET_ONLINE_ACCOUNTS); - - this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList()); - } - - public List getOnlineAccounts() { - return this.onlineAccounts; - } - - public static Message fromByteBuffer(int id, ByteBuffer bytes) { - final int accountCount = bytes.getInt(); - - List onlineAccounts = new ArrayList<>(accountCount); - - for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) { - long timestamp = bytes.getLong(); - - byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - bytes.get(publicKey); - - onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey)); - } - - return new GetOnlineAccountsMessage(id, onlineAccounts); - } - -} diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java deleted file mode 100644 index fe6b5d72..00000000 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java +++ /dev/null @@ -1,109 +0,0 @@ -package org.qortal.network.message; - -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; -import org.qortal.data.network.OnlineAccountData; -import org.qortal.transform.Transformer; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * For requesting online accounts info from remote peer, given our list of online accounts. - * - * Different format to V1: - * V1 is: number of entries, then timestamp + pubkey for each entry - * V2 is: groups of: number of entries, timestamp, then pubkey for each entry - * - * Also V2 only builds online accounts message once! - */ -public class GetOnlineAccountsV2Message extends Message { - - private List onlineAccounts; - - public GetOnlineAccountsV2Message(List onlineAccounts) { - super(MessageType.GET_ONLINE_ACCOUNTS_V2); - - // If we don't have ANY online accounts then it's an easier construction... - if (onlineAccounts.isEmpty()) { - // Always supply a number of accounts - this.dataBytes = Ints.toByteArray(0); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - return; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - Long timestamp = onlineAccountData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH; - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - try { - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - if (onlineAccountData.getTimestamp() == timestamp) - bytes.write(onlineAccountData.getPublicKey()); - } - } - } catch (IOException e) { - throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); - } - - this.dataBytes = bytes.toByteArray(); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - } - - private GetOnlineAccountsV2Message(int id, List onlineAccounts) { - super(id, MessageType.GET_ONLINE_ACCOUNTS_V2); - - this.onlineAccounts = onlineAccounts; - } - - public List getOnlineAccounts() { - return this.onlineAccounts; - } - - public static Message fromByteBuffer(int id, ByteBuffer bytes) { - int accountCount = bytes.getInt(); - - List onlineAccounts = new ArrayList<>(accountCount); - - while (accountCount > 0) { - long timestamp = bytes.getLong(); - - for (int i = 0; i < accountCount; ++i) { - byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - bytes.get(publicKey); - - onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey)); - } - - if (bytes.hasRemaining()) { - accountCount = bytes.getInt(); - } else { - // we've finished - accountCount = 0; - } - } - - return new GetOnlineAccountsV2Message(id, onlineAccounts); - } - -} diff --git a/src/main/java/org/qortal/network/message/MessageType.java b/src/main/java/org/qortal/network/message/MessageType.java index 4dd4a3c8..6b420e2d 100644 --- a/src/main/java/org/qortal/network/message/MessageType.java +++ b/src/main/java/org/qortal/network/message/MessageType.java @@ -43,11 +43,7 @@ public enum MessageType { BLOCK_SUMMARIES(70, BlockSummariesMessage::fromByteBuffer), GET_BLOCK_SUMMARIES(71, GetBlockSummariesMessage::fromByteBuffer), BLOCK_SUMMARIES_V2(72, BlockSummariesV2Message::fromByteBuffer), - - ONLINE_ACCOUNTS(80, OnlineAccountsMessage::fromByteBuffer), - GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer), - ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer), - GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer), + ONLINE_ACCOUNTS_V3(84, OnlineAccountsV3Message::fromByteBuffer), GET_ONLINE_ACCOUNTS_V3(85, GetOnlineAccountsV3Message::fromByteBuffer), diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java deleted file mode 100644 index e7e4c32c..00000000 --- a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.qortal.network.message; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import org.qortal.data.network.OnlineAccountData; -import org.qortal.transform.Transformer; - -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; - -public class OnlineAccountsMessage extends Message { - private static final int MAX_ACCOUNT_COUNT = 5000; - - private List onlineAccounts; - - public OnlineAccountsMessage(List onlineAccounts) { - super(MessageType.ONLINE_ACCOUNTS); - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - try { - bytes.write(Ints.toByteArray(onlineAccounts.size())); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - - bytes.write(onlineAccountData.getSignature()); - - bytes.write(onlineAccountData.getPublicKey()); - } - } catch (IOException e) { - throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); - } - - this.dataBytes = bytes.toByteArray(); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - } - - private OnlineAccountsMessage(int id, List onlineAccounts) { - super(id, MessageType.ONLINE_ACCOUNTS); - - this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList()); - } - - public List getOnlineAccounts() { - return this.onlineAccounts; - } - - public static Message fromByteBuffer(int id, ByteBuffer bytes) { - final int accountCount = bytes.getInt(); - - List onlineAccounts = new ArrayList<>(accountCount); - - for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) { - long timestamp = bytes.getLong(); - - byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; - bytes.get(signature); - - byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - bytes.get(publicKey); - - OnlineAccountData onlineAccountData = new OnlineAccountData(timestamp, signature, publicKey); - onlineAccounts.add(onlineAccountData); - } - - return new OnlineAccountsMessage(id, onlineAccounts); - } - -} diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java deleted file mode 100644 index 6803e3bf..00000000 --- a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java +++ /dev/null @@ -1,113 +0,0 @@ -package org.qortal.network.message; - -import com.google.common.primitives.Ints; -import com.google.common.primitives.Longs; -import org.qortal.data.network.OnlineAccountData; -import org.qortal.transform.Transformer; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * For sending online accounts info to remote peer. - * - * Different format to V1: - * V1 is: number of entries, then timestamp + sig + pubkey for each entry - * V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry - * - * Also V2 only builds online accounts message once! - */ -public class OnlineAccountsV2Message extends Message { - - private List onlineAccounts; - - public OnlineAccountsV2Message(List onlineAccounts) { - super(MessageType.ONLINE_ACCOUNTS_V2); - - // Shortcut in case we have no online accounts - if (onlineAccounts.isEmpty()) { - this.dataBytes = Ints.toByteArray(0); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - return; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - Long timestamp = onlineAccountData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); - - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - try { - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (OnlineAccountData onlineAccountData : onlineAccounts) { - if (onlineAccountData.getTimestamp() == timestamp) { - bytes.write(onlineAccountData.getSignature()); - bytes.write(onlineAccountData.getPublicKey()); - } - } - } - } catch (IOException e) { - throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); - } - - this.dataBytes = bytes.toByteArray(); - this.checksumBytes = Message.generateChecksum(this.dataBytes); - } - - private OnlineAccountsV2Message(int id, List onlineAccounts) { - super(id, MessageType.ONLINE_ACCOUNTS_V2); - - this.onlineAccounts = onlineAccounts; - } - - public List getOnlineAccounts() { - return this.onlineAccounts; - } - - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { - int accountCount = bytes.getInt(); - - List onlineAccounts = new ArrayList<>(accountCount); - - while (accountCount > 0) { - long timestamp = bytes.getLong(); - - for (int i = 0; i < accountCount; ++i) { - byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; - bytes.get(signature); - - byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - bytes.get(publicKey); - - onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey)); - } - - if (bytes.hasRemaining()) { - accountCount = bytes.getInt(); - } else { - // we've finished - accountCount = 0; - } - } - - return new OnlineAccountsV2Message(id, onlineAccounts); - } - -} diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java index c9e646f1..c8220d66 100644 --- a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java +++ b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java @@ -51,89 +51,6 @@ public class OnlineAccountsTests extends Common { } - @Test - public void testGetOnlineAccountsV2() throws MessageException { - List onlineAccountsOut = generateOnlineAccounts(false); - - Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut); - - byte[] messageBytes = messageOut.toBytes(); - ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes); - - GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer); - - List onlineAccountsIn = messageIn.getOnlineAccounts(); - - assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size()); - assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut)); - - Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut); - byte[] oldMessageBytes = oldMessageOut.toBytes(); - - long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count(); - - System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d", - onlineAccountsOut.size(), - numTimestamps, - numTimestamps != 1 ? "s" : "", - oldMessageBytes.length, - messageBytes.length)); - } - - @Test - public void testOnlineAccountsV2() throws MessageException { - List onlineAccountsOut = generateOnlineAccounts(true); - - Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut); - - byte[] messageBytes = messageOut.toBytes(); - ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes); - - OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer); - - List onlineAccountsIn = messageIn.getOnlineAccounts(); - - assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size()); - assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut)); - - Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut); - byte[] oldMessageBytes = oldMessageOut.toBytes(); - - long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count(); - - System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d", - onlineAccountsOut.size(), - numTimestamps, - numTimestamps != 1 ? "s" : "", - oldMessageBytes.length, - messageBytes.length)); - } - - private List generateOnlineAccounts(boolean withSignatures) { - List onlineAccounts = new ArrayList<>(); - - int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2 - - for (int t = 0; t < numTimestamps; ++t) { - int numAccounts = RANDOM.nextInt(3000); - - for (int a = 0; a < numAccounts; ++a) { - byte[] sig = null; - if (withSignatures) { - sig = new byte[Transformer.SIGNATURE_LENGTH]; - RANDOM.nextBytes(sig); - } - - byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH]; - RANDOM.nextBytes(pubkey); - - onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey)); - } - } - - return onlineAccounts; - } - @Test public void testOnlineAccountsModulusV1() throws IllegalAccessException, DataException { try (final Repository repository = RepositoryManager.getRepository()) { diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java b/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java index 6136c1e1..cc2a54ff 100644 --- a/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java +++ b/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java @@ -26,41 +26,6 @@ public class OnlineAccountsV3Tests { Security.insertProviderAt(new BouncyCastleJsseProvider(), 1); } - @Ignore("For informational use") - @Test - public void compareV2ToV3() throws MessageException { - List onlineAccounts = generateOnlineAccounts(false); - - // How many of each timestamp and leading byte (of public key) - Map> hashesByTimestampThenByte = convertToHashMaps(onlineAccounts); - - byte[] v3DataBytes = new GetOnlineAccountsV3Message(hashesByTimestampThenByte).toBytes(); - int v3ByteSize = v3DataBytes.length; - - byte[] v2DataBytes = new GetOnlineAccountsV2Message(onlineAccounts).toBytes(); - int v2ByteSize = v2DataBytes.length; - - int numTimestamps = hashesByTimestampThenByte.size(); - System.out.printf("For %d accounts split across %d timestamp%s: V2 size %d vs V3 size %d%n", - onlineAccounts.size(), - numTimestamps, - numTimestamps != 1 ? "s" : "", - v2ByteSize, - v3ByteSize - ); - - for (var outerMapEntry : hashesByTimestampThenByte.entrySet()) { - long timestamp = outerMapEntry.getKey(); - - var innerMap = outerMapEntry.getValue(); - - System.out.printf("For timestamp %d: %d / 256 slots used.%n", - timestamp, - innerMap.size() - ); - } - } - private Map> convertToHashMaps(List onlineAccounts) { // How many of each timestamp and leading byte (of public key) Map> hashesByTimestampThenByte = new HashMap<>(); From 3215bb638d2faa0ae6f04210900a0a7ecb73a7c5 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sat, 22 Jul 2023 10:44:41 +0100 Subject: [PATCH 08/15] More online accounts improvements --- .../qortal/controller/OnlineAccountsManager.java | 2 +- .../org/qortal/data/network/OnlineAccountData.java | 14 ++++++++------ .../network/message/OnlineAccountsV3Message.java | 3 ++- .../qortal/test/network/OnlineAccountsV3Tests.java | 4 +++- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/qortal/controller/OnlineAccountsManager.java b/src/main/java/org/qortal/controller/OnlineAccountsManager.java index 224228b8..25cace2f 100644 --- a/src/main/java/org/qortal/controller/OnlineAccountsManager.java +++ b/src/main/java/org/qortal/controller/OnlineAccountsManager.java @@ -414,7 +414,7 @@ public class OnlineAccountsManager { boolean isSuperiorEntry = isOnlineAccountsDataSuperior(onlineAccountData); if (isSuperiorEntry) // Remove existing inferior entry so it can be re-added below (it's likely the existing copy is missing a nonce value) - onlineAccounts.remove(onlineAccountData); + onlineAccounts.removeIf(a -> Objects.equals(a.getPublicKey(), onlineAccountData.getPublicKey())); boolean isNewEntry = onlineAccounts.add(onlineAccountData); diff --git a/src/main/java/org/qortal/data/network/OnlineAccountData.java b/src/main/java/org/qortal/data/network/OnlineAccountData.java index bd4842db..a1e1b30f 100644 --- a/src/main/java/org/qortal/data/network/OnlineAccountData.java +++ b/src/main/java/org/qortal/data/network/OnlineAccountData.java @@ -1,6 +1,7 @@ package org.qortal.data.network; import java.util.Arrays; +import java.util.Objects; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -34,10 +35,6 @@ public class OnlineAccountData { this.nonce = nonce; } - public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey) { - this(timestamp, signature, publicKey, null); - } - public long getTimestamp() { return this.timestamp; } @@ -76,6 +73,10 @@ public class OnlineAccountData { if (otherOnlineAccountData.timestamp != this.timestamp) return false; + // Almost as quick + if (!Objects.equals(otherOnlineAccountData.nonce, this.nonce)) + return false; + if (!Arrays.equals(otherOnlineAccountData.publicKey, this.publicKey)) return false; @@ -88,9 +89,10 @@ public class OnlineAccountData { public int hashCode() { int h = this.hash; if (h == 0) { - this.hash = h = Long.hashCode(this.timestamp) - ^ Arrays.hashCode(this.publicKey); + h = Objects.hash(timestamp, nonce); + h = 31 * h + Arrays.hashCode(publicKey); // We don't use signature because newer aggregate signatures use random nonces + this.hash = h; } return h; } diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java index d554d96c..c057fbce 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java @@ -99,9 +99,10 @@ public class OnlineAccountsV3Message extends Message { bytes.get(publicKey); // Nonce is optional - will be -1 if missing + // ... but we should skip/ignore an online account if it has no nonce Integer nonce = bytes.getInt(); if (nonce < 0) { - nonce = null; + continue; } onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey, nonce)); diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java b/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java index cc2a54ff..2c3c01ca 100644 --- a/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java +++ b/src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java @@ -165,7 +165,9 @@ public class OnlineAccountsV3Tests { byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH]; RANDOM.nextBytes(pubkey); - onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey)); + Integer nonce = RANDOM.nextInt(); + + onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey, nonce)); } } From 811b647c88f7a72ba9bd27d62004dc8403a3119e Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 28 Jul 2023 18:58:47 +0100 Subject: [PATCH 09/15] Catch UnsupportedAddressTypeException and fall back to IPv4 binding. --- src/main/java/org/qortal/network/Network.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index ca79f367..a3528a66 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -187,7 +187,7 @@ public class Network { this.bindAddress = bindAddress; // Store the selected address, so that it can be used by other parts of the app break; // We don't want to bind to more than one address - } catch (UnknownHostException e) { + } catch (UnknownHostException | UnsupportedAddressTypeException e) { LOGGER.error("Can't bind listen socket to address {}", Settings.getInstance().getBindAddress()); if (i == bindAddresses.size()-1) { // Only throw an exception if all addresses have been tried throw new IOException("Can't bind listen socket to address", e); From f7e1f2fca876d5fa8074e8643070bb7dfb348808 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 28 Jul 2023 21:47:29 +0100 Subject: [PATCH 10/15] Increased timeout for SEARCH_QDN_RESOURCES from 10 to 30 seconds. --- src/main/resources/q-apps/q-apps.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/resources/q-apps/q-apps.js b/src/main/resources/q-apps/q-apps.js index d26b7791..b638c621 100644 --- a/src/main/resources/q-apps/q-apps.js +++ b/src/main/resources/q-apps/q-apps.js @@ -448,6 +448,10 @@ function getDefaultTimeout(action) { // User may take a long time to accept/deny the popup return 60 * 60 * 1000; + case "SEARCH_QDN_RESOURCES": + // Searching for data can be slow, especially when metadata and statuses are also being included + return 30 * 1000; + case "FETCH_QDN_RESOURCE": // Fetching data can take a while, especially if the status hasn't been checked first return 60 * 1000; From f5c8dfe7661a97cf4585af0c9aa56c26542d0038 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 31 Jul 2023 19:25:26 +0100 Subject: [PATCH 11/15] Added maxTransactionsPerBlock setting (default 25) to reduce minting load on slower machines. This is a short term limit, is well above current usage levels, and can be increased substantially in future once the block minter code has been improved. --- .../qortal/repository/TransactionRepository.java | 2 +- .../transaction/HSQLDBTransactionRepository.java | 13 ++++++++++--- src/main/java/org/qortal/settings/Settings.java | 7 +++++++ .../java/org/qortal/transaction/Transaction.java | 3 ++- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/repository/TransactionRepository.java b/src/main/java/org/qortal/repository/TransactionRepository.java index 6cc88290..41986cad 100644 --- a/src/main/java/org/qortal/repository/TransactionRepository.java +++ b/src/main/java/org/qortal/repository/TransactionRepository.java @@ -314,7 +314,7 @@ public interface TransactionRepository { * @return list of transactions, or empty if none. * @throws DataException */ - public List getUnconfirmedTransactions(EnumSet excludedTxTypes) throws DataException; + public List getUnconfirmedTransactions(EnumSet excludedTxTypes, Integer limit) throws DataException; /** * Remove transaction from unconfirmed transactions pile. diff --git a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java index 740b3e65..60b4e803 100644 --- a/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java @@ -1429,8 +1429,10 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } @Override - public List getUnconfirmedTransactions(EnumSet excludedTxTypes) throws DataException { + public List getUnconfirmedTransactions(EnumSet excludedTxTypes, Integer limit) throws DataException { StringBuilder sql = new StringBuilder(1024); + List bindParams = new ArrayList<>(); + sql.append("SELECT signature FROM UnconfirmedTransactions "); sql.append("JOIN Transactions USING (signature) "); sql.append("WHERE type NOT IN ("); @@ -1446,12 +1448,17 @@ public class HSQLDBTransactionRepository implements TransactionRepository { } sql.append(")"); - sql.append("ORDER BY created_when, signature"); + sql.append("ORDER BY created_when, signature "); + + if (limit != null) { + sql.append("LIMIT ?"); + bindParams.add(limit); + } List transactions = new ArrayList<>(); // Find transactions with no corresponding row in BlockTransactions - try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) { + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) { if (resultSet == null) return transactions; diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index c3d5a0c8..ac9b8857 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -146,6 +146,9 @@ public class Settings { /* How many blocks to cache locally. Defaulted to 10, which covers a typical Synchronizer request + a few spare */ private int blockCacheSize = 10; + /** Maximum number of transactions for the block minter to include in a block */ + private int maxTransactionsPerBlock = 25; + /** How long to keep old, full, AT state data (ms). */ private long atStatesMaxLifetime = 5 * 24 * 60 * 60 * 1000L; // milliseconds /** How often to attempt AT state trimming (ms). */ @@ -693,6 +696,10 @@ public class Settings { return this.blockCacheSize; } + public int getMaxTransactionsPerBlock() { + return this.maxTransactionsPerBlock; + } + public boolean isTestNet() { return this.isTestNet; } diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index f0e9b3f6..10834a06 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -641,7 +641,8 @@ public abstract class Transaction { BlockData latestBlockData = repository.getBlockRepository().getLastBlock(); EnumSet excludedTxTypes = EnumSet.of(TransactionType.CHAT, TransactionType.PRESENCE); - List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes); + int limit = Settings.getInstance().getMaxTransactionsPerBlock(); + List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes, limit); unconfirmedTransactions.sort(getDataComparator()); From 94d3664cb092877f8348bea1d6342dbe5a95993d Mon Sep 17 00:00:00 2001 From: CalDescent Date: Mon, 31 Jul 2023 19:30:45 +0100 Subject: [PATCH 12/15] Bump version to 4.2.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f236761e..1a046758 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 4.2.0 + 4.2.1 jar true From 33cfd02c4960f81ea1c2a82986eeab31d146811a Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 2 Aug 2023 21:13:33 +0100 Subject: [PATCH 13/15] Fixed issues in commit f5c8dfe --- src/main/java/org/qortal/controller/BlockMinter.java | 9 +++++++++ src/main/java/org/qortal/transaction/Transaction.java | 3 +-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index bc879f23..deb00587 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -484,6 +484,9 @@ public class BlockMinter extends Thread { // Sign to create block's signature, needed by Block.isValid() newBlock.sign(); + // User-defined limit per block + int limit = Settings.getInstance().getMaxTransactionsPerBlock(); + // Attempt to add transactions until block is full, or we run out // If a transaction makes the block invalid then skip it and it'll either expire or be in next block. for (TransactionData transactionData : unconfirmedTransactions) { @@ -496,6 +499,12 @@ public class BlockMinter extends Thread { LOGGER.debug(() -> String.format("Skipping invalid transaction %s during block minting", Base58.encode(transactionData.getSignature()))); newBlock.deleteTransaction(transactionData); } + + // User-defined limit per block + List transactions = newBlock.getTransactions(); + if (transactions != null && transactions.size() >= limit) { + break; + } } } diff --git a/src/main/java/org/qortal/transaction/Transaction.java b/src/main/java/org/qortal/transaction/Transaction.java index 10834a06..bd91f25a 100644 --- a/src/main/java/org/qortal/transaction/Transaction.java +++ b/src/main/java/org/qortal/transaction/Transaction.java @@ -641,8 +641,7 @@ public abstract class Transaction { BlockData latestBlockData = repository.getBlockRepository().getLastBlock(); EnumSet excludedTxTypes = EnumSet.of(TransactionType.CHAT, TransactionType.PRESENCE); - int limit = Settings.getInstance().getMaxTransactionsPerBlock(); - List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes, limit); + List unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes, null); unconfirmedTransactions.sort(getDataComparator()); From 528583fe381287c71597ed5a123a66512d6c37b4 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 2 Aug 2023 21:32:57 +0100 Subject: [PATCH 14/15] Added logging relating to unconfirmed transactions. --- src/main/java/org/qortal/controller/BlockMinter.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/qortal/controller/BlockMinter.java b/src/main/java/org/qortal/controller/BlockMinter.java index deb00587..b1ed7e3c 100644 --- a/src/main/java/org/qortal/controller/BlockMinter.java +++ b/src/main/java/org/qortal/controller/BlockMinter.java @@ -380,9 +380,13 @@ public class BlockMinter extends Thread { parentSignatureForLastLowWeightBlock = null; timeOfLastLowWeightBlock = null; + Long unconfirmedStartTime = NTP.getTime(); + // Add unconfirmed transactions addUnconfirmedTransactions(repository, newBlock); + LOGGER.info(String.format("Adding %d unconfirmed transactions took %d ms", newBlock.getTransactions().size(), (NTP.getTime()-unconfirmedStartTime))); + // Sign to create block's signature newBlock.sign(); From 9574100a0880b46122b103dd2a6820e43fa66c8b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 2 Aug 2023 21:36:57 +0100 Subject: [PATCH 15/15] Bump version to 4.2.2 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1a046758..30ad1e04 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.qortal qortal - 4.2.1 + 4.2.2 jar true