Merge branch 'master' into arbitrary-resources-cache

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>4.1.3</version>
+	<version>4.2.2</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

src/main/java/org/qortal/api/proxy/resource/DevProxyServerResource.java

@@ -16,7 +16,9 @@ import javax.ws.rs.core.Context;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.ConnectException;
 import java.net.HttpURLConnection;
+import java.net.ProtocolException;
 import java.net.URL;
 import java.util.Enumeration;
 
@@ -44,16 +46,6 @@ public class DevProxyServerResource {
         try {
             String source = DevProxyManager.getInstance().getSourceHostAndPort();
 
-            // Convert localhost / 127.0.0.1 to IPv6 [::1]
-            if (source.startsWith("localhost") || source.startsWith("127.0.0.1")) {
-                int port = 80;
-                String[] parts = source.split(":");
-                if (parts.length > 1) {
-                    port = Integer.parseInt(parts[1]);
-                }
-                source = String.format("[::1]:%d", port);
-            }
-
             if (!inPath.startsWith("/")) {
                 inPath = "/" + inPath;
             }
@@ -61,76 +53,37 @@ public class DevProxyServerResource {
             String queryString = request.getQueryString() != null ? "?" + request.getQueryString() : "";
 
             // Open URL
-            String urlString = String.format("http://%s%s%s", source, inPath, queryString);
+            URL url = new URL(String.format("http://%s%s%s", source, inPath, queryString));
-            URL url = new URL(urlString);
+            HttpURLConnection con = (HttpURLConnection) url.openConnection();
-			HttpURLConnection con = (HttpURLConnection) url.openConnection();
-			con.setRequestMethod(request.getMethod());
 
-            // Proxy the request headers
+            // Proxy the request data
-            Enumeration<String> headerNames = request.getHeaderNames();
+            this.proxyRequestToConnection(request, con);
-            while (headerNames.hasMoreElements()) {
+
-                String headerName = headerNames.nextElement();
+            try {
-                String headerValue = request.getHeader(headerName);
+                // Make the request and proxy the response code
-                con.setRequestProperty(headerName, headerValue);
+                response.setStatus(con.getResponseCode());
             }
+            catch (ConnectException e) {
 
-            // TODO: proxy any POST parameters from "request" to "con"
+                // Tey converting localhost / 127.0.0.1 to IPv6 [::1]
-
+                if (source.startsWith("localhost") || source.startsWith("127.0.0.1")) {
-            // Proxy the response code
+                    int port = 80;
-            int responseCode = con.getResponseCode();
+                    String[] parts = source.split(":");
-            response.setStatus(responseCode);
+                    if (parts.length > 1) {
-
+                        port = Integer.parseInt(parts[1]);
-            // Proxy the response headers
+                    }
-            for (int i = 0; ; i++) {
+                    source = String.format("[::1]:%d", port);
-                String headerKey = con.getHeaderFieldKey(i);
-                String headerValue = con.getHeaderField(i);
-                if (headerKey != null && headerValue != null) {
-                    response.addHeader(headerKey, headerValue);
-                    continue;
                 }
-                break;
+
+                // Retry connection
+                url = new URL(String.format("http://%s%s%s", source, inPath, queryString));
+                con = (HttpURLConnection) url.openConnection();
+                this.proxyRequestToConnection(request, con);
+                response.setStatus(con.getResponseCode());
             }
 
-            // Read the response body
+            // Proxy the response data back to the caller
-            InputStream inputStream = con.getInputStream();
+            this.proxyConnectionToResponse(con, response, inPath);
-            ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-            byte[] buffer = new byte[4096];
-            int bytesRead;
-            while ((bytesRead = inputStream.read(buffer)) != -1) {
-                outputStream.write(buffer, 0, bytesRead);
-            }
-            byte[] data = outputStream.toByteArray(); // TODO: limit file size that can be read into memory
-
-            // Close the streams
-            outputStream.close();
-            inputStream.close();
-
-            // Extract filename
-            String filename = "";
-            if (inPath.contains("/")) {
-                String[] parts = inPath.split("/");
-                if (parts.length > 0) {
-                    filename = parts[parts.length - 1];
-                }
-            }
-
-            // Parse and modify output if needed
-            if (HTMLParser.isHtmlFile(filename)) {
-                // HTML file - needs to be parsed
-                HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true);
-                htmlParser.addAdditionalHeaderTags();
-                response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;");
-                response.setContentType(con.getContentType());
-                response.setContentLength(htmlParser.getData().length);
-                response.getOutputStream().write(htmlParser.getData());
-            }
-            else {
-                // Regular file - can be streamed directly
-                response.addHeader("Content-Security-Policy", "default-src 'self'");
-                response.setContentType(con.getContentType());
-                response.setContentLength(data.length);
-                response.getOutputStream().write(data);
-            }
 
         } catch (IOException e) {
             throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, e.getMessage());
@@ -139,4 +92,73 @@ public class DevProxyServerResource {
         return response;
     }
 
+    private void proxyRequestToConnection(HttpServletRequest request, HttpURLConnection con) throws ProtocolException {
+        // Proxy the request method
+        con.setRequestMethod(request.getMethod());
+
+        // Proxy the request headers
+        Enumeration<String> headerNames = request.getHeaderNames();
+        while (headerNames.hasMoreElements()) {
+            String headerName = headerNames.nextElement();
+            String headerValue = request.getHeader(headerName);
+            con.setRequestProperty(headerName, headerValue);
+        }
+
+        // TODO: proxy any POST parameters from "request" to "con"
+    }
+
+    private void proxyConnectionToResponse(HttpURLConnection con, HttpServletResponse response, String inPath) throws IOException {
+        // Proxy the response headers
+        for (int i = 0; ; i++) {
+            String headerKey = con.getHeaderFieldKey(i);
+            String headerValue = con.getHeaderField(i);
+            if (headerKey != null && headerValue != null) {
+                response.addHeader(headerKey, headerValue);
+                continue;
+            }
+            break;
+        }
+
+        // Read the response body
+        InputStream inputStream = con.getInputStream();
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        byte[] buffer = new byte[4096];
+        int bytesRead;
+        while ((bytesRead = inputStream.read(buffer)) != -1) {
+            outputStream.write(buffer, 0, bytesRead);
+        }
+        byte[] data = outputStream.toByteArray(); // TODO: limit file size that can be read into memory
+
+        // Close the streams
+        outputStream.close();
+        inputStream.close();
+
+        // Extract filename
+        String filename = "";
+        if (inPath.contains("/")) {
+            String[] parts = inPath.split("/");
+            if (parts.length > 0) {
+                filename = parts[parts.length - 1];
+            }
+        }
+
+        // Parse and modify output if needed
+        if (HTMLParser.isHtmlFile(filename)) {
+            // HTML file - needs to be parsed
+            HTMLParser htmlParser = new HTMLParser("", inPath, "", false, data, "proxy", Service.APP, null, "light", true);
+            htmlParser.addAdditionalHeaderTags();
+            response.addHeader("Content-Security-Policy", "default-src 'self' 'unsafe-inline' 'unsafe-eval'; media-src 'self' data: blob:; img-src 'self' data: blob:; connect-src 'self' ws:; font-src 'self' data:;");
+            response.setContentType(con.getContentType());
+            response.setContentLength(htmlParser.getData().length);
+            response.getOutputStream().write(htmlParser.getData());
+        }
+        else {
+            // Regular file - can be streamed directly
+            response.addHeader("Content-Security-Policy", "default-src 'self'");
+            response.setContentType(con.getContentType());
+            response.setContentLength(data.length);
+            response.getOutputStream().write(data);
+        }
+    }
+
 }

src/main/java/org/qortal/controller/BlockMinter.java

@@ -380,9 +380,13 @@ public class BlockMinter extends Thread {
 						parentSignatureForLastLowWeightBlock = null;
 						timeOfLastLowWeightBlock = null;
 
+						Long unconfirmedStartTime = NTP.getTime();
+
 						// Add unconfirmed transactions
 						addUnconfirmedTransactions(repository, newBlock);
 
+						LOGGER.info(String.format("Adding %d unconfirmed transactions took %d ms", newBlock.getTransactions().size(), (NTP.getTime()-unconfirmedStartTime)));
+
 						// Sign to create block's signature
 						newBlock.sign();
 
@@ -484,6 +488,9 @@ public class BlockMinter extends Thread {
 		// Sign to create block's signature, needed by Block.isValid()
 		newBlock.sign();
 
+		// User-defined limit per block
+		int limit = Settings.getInstance().getMaxTransactionsPerBlock();
+
 		// Attempt to add transactions until block is full, or we run out
 		// If a transaction makes the block invalid then skip it and it'll either expire or be in next block.
 		for (TransactionData transactionData : unconfirmedTransactions) {
@@ -496,6 +503,12 @@ public class BlockMinter extends Thread {
 				LOGGER.debug(() -> String.format("Skipping invalid transaction %s during block minting", Base58.encode(transactionData.getSignature())));
 				newBlock.deleteTransaction(transactionData);
 			}
+
+			// User-defined limit per block
+			List<Transaction> transactions = newBlock.getTransactions();
+			if (transactions != null && transactions.size() >= limit) {
+				break;
+			}
 		}
 	}
 

src/main/java/org/qortal/controller/Controller.java

@@ -1305,13 +1305,6 @@ public class Controller extends Thread {
 				TransactionImporter.getInstance().onNetworkTransactionSignaturesMessage(peer, message);
 				break;
 
-			case GET_ONLINE_ACCOUNTS:
-			case ONLINE_ACCOUNTS:
-			case GET_ONLINE_ACCOUNTS_V2:
-			case ONLINE_ACCOUNTS_V2:
-				// No longer supported - to be eventually removed
-				break;
-
 			case GET_ONLINE_ACCOUNTS_V3:
 				OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV3Message(peer, message);
 				break;

src/main/java/org/qortal/controller/OnlineAccountsManager.java

@@ -414,7 +414,7 @@ public class OnlineAccountsManager {
         boolean isSuperiorEntry = isOnlineAccountsDataSuperior(onlineAccountData);
         if (isSuperiorEntry)
             // Remove existing inferior entry so it can be re-added below (it's likely the existing copy is missing a nonce value)
-            onlineAccounts.remove(onlineAccountData);
+            onlineAccounts.removeIf(a -> Objects.equals(a.getPublicKey(), onlineAccountData.getPublicKey()));
 
         boolean isNewEntry = onlineAccounts.add(onlineAccountData);
 

src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java

@@ -25,7 +25,8 @@ public class NamesDatabaseIntegrityCheck {
             TransactionType.REGISTER_NAME,
             TransactionType.UPDATE_NAME,
             TransactionType.BUY_NAME,
-            TransactionType.SELL_NAME
+            TransactionType.SELL_NAME,
+            TransactionType.CANCEL_SELL_NAME
     );
 
     private List<TransactionData> nameTransactions = new ArrayList<>();

src/main/java/org/qortal/data/network/OnlineAccountData.java

@@ -1,6 +1,7 @@
 package org.qortal.data.network;
 
 import java.util.Arrays;
+import java.util.Objects;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -34,10 +35,6 @@ public class OnlineAccountData {
 		this.nonce = nonce;
 	}
 
-	public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey) {
-		this(timestamp, signature, publicKey, null);
-	}
-
 	public long getTimestamp() {
 		return this.timestamp;
 	}
@@ -76,6 +73,10 @@ public class OnlineAccountData {
 		if (otherOnlineAccountData.timestamp != this.timestamp)
 			return false;
 
+		// Almost as quick
+		if (!Objects.equals(otherOnlineAccountData.nonce, this.nonce))
+			return false;
+
 		if (!Arrays.equals(otherOnlineAccountData.publicKey, this.publicKey))
 			return false;
 
@@ -88,9 +89,10 @@ public class OnlineAccountData {
 	public int hashCode() {
 		int h = this.hash;
 		if (h == 0) {
-			this.hash = h = Long.hashCode(this.timestamp)
+			h = Objects.hash(timestamp, nonce);
-					^ Arrays.hashCode(this.publicKey);
+			h = 31 * h + Arrays.hashCode(publicKey);
 			// We don't use signature because newer aggregate signatures use random nonces
+			this.hash = h;
 		}
 		return h;
 	}

src/main/java/org/qortal/network/Network.java

@@ -187,7 +187,7 @@ public class Network {
 
                 this.bindAddress = bindAddress; // Store the selected address, so that it can be used by other parts of the app
                 break; // We don't want to bind to more than one address
-            } catch (UnknownHostException e) {
+            } catch (UnknownHostException | UnsupportedAddressTypeException e) {
                 LOGGER.error("Can't bind listen socket to address {}", Settings.getInstance().getBindAddress());
                 if (i == bindAddresses.size()-1) { // Only throw an exception if all addresses have been tried
                     throw new IOException("Can't bind listen socket to address", e);

src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java

@@ -1,69 +0,0 @@
-package org.qortal.network.message;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.qortal.data.network.OnlineAccountData;
-import org.qortal.transform.Transformer;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-
-public class GetOnlineAccountsMessage extends Message {
-	private static final int MAX_ACCOUNT_COUNT = 5000;
-
-	private List<OnlineAccountData> onlineAccounts;
-
-	public GetOnlineAccountsMessage(List<OnlineAccountData> onlineAccounts) {
-		super(MessageType.GET_ONLINE_ACCOUNTS);
-
-		ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-
-		try {
-			bytes.write(Ints.toByteArray(onlineAccounts.size()));
-
-			for (OnlineAccountData onlineAccountData : onlineAccounts) {
-				bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp()));
-
-				bytes.write(onlineAccountData.getPublicKey());
-			}
-		} catch (IOException e) {
-			throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
-		}
-
-		this.dataBytes = bytes.toByteArray();
-		this.checksumBytes = Message.generateChecksum(this.dataBytes);
-	}
-
-	private GetOnlineAccountsMessage(int id, List<OnlineAccountData> onlineAccounts) {
-		super(id, MessageType.GET_ONLINE_ACCOUNTS);
-
-		this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList());
-	}
-
-	public List<OnlineAccountData> getOnlineAccounts() {
-		return this.onlineAccounts;
-	}
-
-	public static Message fromByteBuffer(int id, ByteBuffer bytes) {
-		final int accountCount = bytes.getInt();
-
-		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
-
-		for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) {
-			long timestamp = bytes.getLong();
-
-			byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
-			bytes.get(publicKey);
-
-			onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey));
-		}
-
-		return new GetOnlineAccountsMessage(id, onlineAccounts);
-	}
-
-}

src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java

@@ -1,109 +0,0 @@
-package org.qortal.network.message;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-import org.qortal.data.network.OnlineAccountData;
-import org.qortal.transform.Transformer;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * For requesting online accounts info from remote peer, given our list of online accounts.
- *
- * Different format to V1:
- * V1 is: number of entries, then timestamp + pubkey for each entry
- * V2 is: groups of: number of entries, timestamp, then pubkey for each entry
- *
- * Also V2 only builds online accounts message once!
- */
-public class GetOnlineAccountsV2Message extends Message {
-
-	private List<OnlineAccountData> onlineAccounts;
-
-	public GetOnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
-		super(MessageType.GET_ONLINE_ACCOUNTS_V2);
-
-		// If we don't have ANY online accounts then it's an easier construction...
-		if (onlineAccounts.isEmpty()) {
-			// Always supply a number of accounts
-			this.dataBytes = Ints.toByteArray(0);
-			this.checksumBytes = Message.generateChecksum(this.dataBytes);
-			return;
-		}
-
-		// How many of each timestamp
-		Map<Long, Integer> countByTimestamp = new HashMap<>();
-
-		for (OnlineAccountData onlineAccountData : onlineAccounts) {
-			Long timestamp = onlineAccountData.getTimestamp();
-			countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
-		}
-
-		// We should know exactly how many bytes to allocate now
-		int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
-				+ onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH;
-
-		ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
-
-		try {
-			for (long timestamp : countByTimestamp.keySet()) {
-				bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
-
-				bytes.write(Longs.toByteArray(timestamp));
-
-				for (OnlineAccountData onlineAccountData : onlineAccounts) {
-					if (onlineAccountData.getTimestamp() == timestamp)
-						bytes.write(onlineAccountData.getPublicKey());
-				}
-			}
-		} catch (IOException e) {
-			throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
-		}
-
-		this.dataBytes = bytes.toByteArray();
-		this.checksumBytes = Message.generateChecksum(this.dataBytes);
-	}
-
-	private GetOnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
-		super(id, MessageType.GET_ONLINE_ACCOUNTS_V2);
-
-		this.onlineAccounts = onlineAccounts;
-	}
-
-	public List<OnlineAccountData> getOnlineAccounts() {
-		return this.onlineAccounts;
-	}
-
-	public static Message fromByteBuffer(int id, ByteBuffer bytes) {
-		int accountCount = bytes.getInt();
-
-		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
-
-		while (accountCount > 0) {
-			long timestamp = bytes.getLong();
-
-			for (int i = 0; i < accountCount; ++i) {
-				byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
-				bytes.get(publicKey);
-
-				onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey));
-			}
-
-			if (bytes.hasRemaining()) {
-				accountCount = bytes.getInt();
-			} else {
-				// we've finished
-				accountCount = 0;
-			}
-		}
-
-		return new GetOnlineAccountsV2Message(id, onlineAccounts);
-	}
-
-}

src/main/java/org/qortal/network/message/MessageType.java

@@ -43,11 +43,7 @@ public enum MessageType {
     BLOCK_SUMMARIES(70, BlockSummariesMessage::fromByteBuffer),
     GET_BLOCK_SUMMARIES(71, GetBlockSummariesMessage::fromByteBuffer),
     BLOCK_SUMMARIES_V2(72, BlockSummariesV2Message::fromByteBuffer),
-
+    
-    ONLINE_ACCOUNTS(80, OnlineAccountsMessage::fromByteBuffer),
-    GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer),
-    ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer),
-    GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer),
     ONLINE_ACCOUNTS_V3(84, OnlineAccountsV3Message::fromByteBuffer),
     GET_ONLINE_ACCOUNTS_V3(85, GetOnlineAccountsV3Message::fromByteBuffer),
 

src/main/java/org/qortal/network/message/OnlineAccountsMessage.java

@@ -1,75 +0,0 @@
-package org.qortal.network.message;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.qortal.data.network.OnlineAccountData;
-import org.qortal.transform.Transformer;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-
-public class OnlineAccountsMessage extends Message {
-	private static final int MAX_ACCOUNT_COUNT = 5000;
-
-	private List<OnlineAccountData> onlineAccounts;
-
-	public OnlineAccountsMessage(List<OnlineAccountData> onlineAccounts) {
-		super(MessageType.ONLINE_ACCOUNTS);
-
-		ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-
-		try {
-			bytes.write(Ints.toByteArray(onlineAccounts.size()));
-
-			for (OnlineAccountData onlineAccountData : onlineAccounts) {
-				bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp()));
-
-				bytes.write(onlineAccountData.getSignature());
-
-				bytes.write(onlineAccountData.getPublicKey());
-			}
-		} catch (IOException e) {
-			throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
-		}
-
-		this.dataBytes = bytes.toByteArray();
-		this.checksumBytes = Message.generateChecksum(this.dataBytes);
-	}
-
-	private OnlineAccountsMessage(int id, List<OnlineAccountData> onlineAccounts) {
-		super(id, MessageType.ONLINE_ACCOUNTS);
-
-		this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList());
-	}
-
-	public List<OnlineAccountData> getOnlineAccounts() {
-		return this.onlineAccounts;
-	}
-
-	public static Message fromByteBuffer(int id, ByteBuffer bytes) {
-		final int accountCount = bytes.getInt();
-
-		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
-
-		for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) {
-			long timestamp = bytes.getLong();
-
-			byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
-			bytes.get(signature);
-
-			byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
-			bytes.get(publicKey);
-
-			OnlineAccountData onlineAccountData = new OnlineAccountData(timestamp, signature, publicKey);
-			onlineAccounts.add(onlineAccountData);
-		}
-
-		return new OnlineAccountsMessage(id, onlineAccounts);
-	}
-
-}

src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java

@@ -1,113 +0,0 @@
-package org.qortal.network.message;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-import org.qortal.data.network.OnlineAccountData;
-import org.qortal.transform.Transformer;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * For sending online accounts info to remote peer.
- *
- * Different format to V1:
- * V1 is: number of entries, then timestamp + sig + pubkey for each entry
- * V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry
- *
- * Also V2 only builds online accounts message once!
- */
-public class OnlineAccountsV2Message extends Message {
-
-	private List<OnlineAccountData> onlineAccounts;
-
-	public OnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
-		super(MessageType.ONLINE_ACCOUNTS_V2);
-
-		// Shortcut in case we have no online accounts
-		if (onlineAccounts.isEmpty()) {
-			this.dataBytes = Ints.toByteArray(0);
-			this.checksumBytes = Message.generateChecksum(this.dataBytes);
-			return;
-		}
-
-		// How many of each timestamp
-		Map<Long, Integer> countByTimestamp = new HashMap<>();
-
-		for (OnlineAccountData onlineAccountData : onlineAccounts) {
-			Long timestamp = onlineAccountData.getTimestamp();
-			countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
-		}
-
-		// We should know exactly how many bytes to allocate now
-		int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
-				+ onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH);
-
-		ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
-
-		try {
-			for (long timestamp : countByTimestamp.keySet()) {
-				bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
-
-				bytes.write(Longs.toByteArray(timestamp));
-
-				for (OnlineAccountData onlineAccountData : onlineAccounts) {
-					if (onlineAccountData.getTimestamp() == timestamp) {
-						bytes.write(onlineAccountData.getSignature());
-						bytes.write(onlineAccountData.getPublicKey());
-					}
-				}
-			}
-		} catch (IOException e) {
-			throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
-		}
-
-		this.dataBytes = bytes.toByteArray();
-		this.checksumBytes = Message.generateChecksum(this.dataBytes);
-	}
-
-	private OnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
-		super(id, MessageType.ONLINE_ACCOUNTS_V2);
-
-		this.onlineAccounts = onlineAccounts;
-	}
-
-	public List<OnlineAccountData> getOnlineAccounts() {
-		return this.onlineAccounts;
-	}
-
-	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException {
-		int accountCount = bytes.getInt();
-
-		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
-
-		while (accountCount > 0) {
-			long timestamp = bytes.getLong();
-
-			for (int i = 0; i < accountCount; ++i) {
-				byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
-				bytes.get(signature);
-
-				byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
-				bytes.get(publicKey);
-
-				onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey));
-			}
-
-			if (bytes.hasRemaining()) {
-				accountCount = bytes.getInt();
-			} else {
-				// we've finished
-				accountCount = 0;
-			}
-		}
-
-		return new OnlineAccountsV2Message(id, onlineAccounts);
-	}
-
-}

src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java

@@ -99,9 +99,10 @@ public class OnlineAccountsV3Message extends Message {
 				bytes.get(publicKey);
 
 				// Nonce is optional - will be -1 if missing
+				// ... but we should skip/ignore an online account if it has no nonce
 				Integer nonce = bytes.getInt();
 				if (nonce < 0) {
-					nonce = null;
+					continue;
 				}
 
 				onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey, nonce));

src/main/java/org/qortal/repository/TransactionRepository.java

@@ -314,7 +314,7 @@ public interface TransactionRepository {
 	 * @return list of transactions, or empty if none.
 	 * @throws DataException
 	 */
-	public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes) throws DataException;
+	public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes, Integer limit) throws DataException;
 
 	/**
 	 * Remove transaction from unconfirmed transactions pile.

src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java

@@ -1429,8 +1429,10 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 	}
 
 	@Override
-	public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes) throws DataException {
+	public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes, Integer limit) throws DataException {
 		StringBuilder sql = new StringBuilder(1024);
+		List<Object> bindParams = new ArrayList<>();
+
 		sql.append("SELECT signature FROM UnconfirmedTransactions ");
 		sql.append("JOIN Transactions USING (signature) ");
 		sql.append("WHERE type NOT IN (");
@@ -1446,12 +1448,17 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 		}
 
 		sql.append(")");
-		sql.append("ORDER BY created_when, signature");
+		sql.append("ORDER BY created_when, signature ");
+
+		if (limit != null) {
+			sql.append("LIMIT ?");
+			bindParams.add(limit);
+		}
 
 		List<TransactionData> transactions = new ArrayList<>();
 
 		// Find transactions with no corresponding row in BlockTransactions
-		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) {
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
 			if (resultSet == null)
 				return transactions;
 

src/main/java/org/qortal/settings/Settings.java

@@ -146,6 +146,9 @@ public class Settings {
 	/* How many blocks to cache locally. Defaulted to 10, which covers a typical Synchronizer request + a few spare */
 	private int blockCacheSize = 10;
 
+	/** Maximum number of transactions for the block minter to include in a block */
+	private int maxTransactionsPerBlock = 25;
+
 	/** How long to keep old, full, AT state data (ms). */
 	private long atStatesMaxLifetime = 5 * 24 * 60 * 60 * 1000L; // milliseconds
 	/** How often to attempt AT state trimming (ms). */
@@ -693,6 +696,10 @@ public class Settings {
 		return this.blockCacheSize;
 	}
 
+	public int getMaxTransactionsPerBlock() {
+		return this.maxTransactionsPerBlock;
+	}
+
 	public boolean isTestNet() {
 		return this.isTestNet;
 	}

src/main/java/org/qortal/transaction/Transaction.java

@@ -641,7 +641,7 @@ public abstract class Transaction {
 		BlockData latestBlockData = repository.getBlockRepository().getLastBlock();
 
 		EnumSet<TransactionType> excludedTxTypes = EnumSet.of(TransactionType.CHAT, TransactionType.PRESENCE);
-		List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes);
+		List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes, null);
 
 		unconfirmedTransactions.sort(getDataComparator());
 

src/main/resources/q-apps/q-apps.js

@@ -454,6 +454,10 @@ function getDefaultTimeout(action) {
                 // User may take a long time to accept/deny the popup
                 return 60 * 60 * 1000;
 
+            case "SEARCH_QDN_RESOURCES":
+                // Searching for data can be slow, especially when metadata and statuses are also being included
+                return 30 * 1000;
+
             case "FETCH_QDN_RESOURCE":
                 // Fetching data can take a while, especially if the status hasn't been checked first
                 return 60 * 1000;

src/test/java/org/qortal/test/network/OnlineAccountsTests.java

@@ -51,89 +51,6 @@ public class OnlineAccountsTests extends Common {
     }
 
 
-    @Test
-    public void testGetOnlineAccountsV2() throws MessageException {
-        List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(false);
-
-        Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut);
-
-        byte[] messageBytes = messageOut.toBytes();
-        ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
-
-        GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
-
-        List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
-
-        assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
-        assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
-
-        Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut);
-        byte[] oldMessageBytes = oldMessageOut.toBytes();
-
-        long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
-
-        System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
-                onlineAccountsOut.size(),
-                numTimestamps,
-                numTimestamps != 1 ? "s" : "",
-                oldMessageBytes.length,
-                messageBytes.length));
-    }
-
-    @Test
-    public void testOnlineAccountsV2() throws MessageException {
-        List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(true);
-
-        Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut);
-
-        byte[] messageBytes = messageOut.toBytes();
-        ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
-
-        OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
-
-        List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
-
-        assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
-        assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
-
-        Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut);
-        byte[] oldMessageBytes = oldMessageOut.toBytes();
-
-        long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
-
-        System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
-                onlineAccountsOut.size(),
-                numTimestamps,
-                numTimestamps != 1 ? "s" : "",
-                oldMessageBytes.length,
-                messageBytes.length));
-    }
-
-    private List<OnlineAccountData> generateOnlineAccounts(boolean withSignatures) {
-        List<OnlineAccountData> onlineAccounts = new ArrayList<>();
-
-        int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2
-
-        for (int t = 0; t < numTimestamps; ++t) {
-            int numAccounts = RANDOM.nextInt(3000);
-
-            for (int a = 0; a < numAccounts; ++a) {
-                byte[] sig = null;
-                if (withSignatures) {
-                    sig = new byte[Transformer.SIGNATURE_LENGTH];
-                    RANDOM.nextBytes(sig);
-                }
-
-                byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH];
-                RANDOM.nextBytes(pubkey);
-
-                onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey));
-            }
-        }
-
-        return onlineAccounts;
-    }
-
     @Test
     public void testOnlineAccountsModulusV1() throws IllegalAccessException, DataException {
         try (final Repository repository = RepositoryManager.getRepository()) {

src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java

@@ -26,41 +26,6 @@ public class OnlineAccountsV3Tests {
         Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
     }
 
-    @Ignore("For informational use")
-    @Test
-    public void compareV2ToV3() throws MessageException {
-        List<OnlineAccountData> onlineAccounts = generateOnlineAccounts(false);
-
-        // How many of each timestamp and leading byte (of public key)
-        Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte = convertToHashMaps(onlineAccounts);
-
-        byte[] v3DataBytes = new GetOnlineAccountsV3Message(hashesByTimestampThenByte).toBytes();
-        int v3ByteSize = v3DataBytes.length;
-
-        byte[] v2DataBytes = new GetOnlineAccountsV2Message(onlineAccounts).toBytes();
-        int v2ByteSize = v2DataBytes.length;
-
-        int numTimestamps = hashesByTimestampThenByte.size();
-        System.out.printf("For %d accounts split across %d timestamp%s: V2 size %d vs V3 size %d%n",
-                onlineAccounts.size(),
-                numTimestamps,
-                numTimestamps != 1 ? "s" : "",
-                v2ByteSize,
-                v3ByteSize
-        );
-
-        for (var outerMapEntry : hashesByTimestampThenByte.entrySet()) {
-            long timestamp = outerMapEntry.getKey();
-
-            var innerMap = outerMapEntry.getValue();
-
-            System.out.printf("For timestamp %d: %d / 256 slots used.%n",
-                    timestamp,
-                    innerMap.size()
-            );
-        }
-    }
-
     private Map<Long, Map<Byte, byte[]>> convertToHashMaps(List<OnlineAccountData> onlineAccounts) {
         // How many of each timestamp and leading byte (of public key)
         Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte = new HashMap<>();
@@ -200,7 +165,9 @@ public class OnlineAccountsV3Tests {
                 byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH];
                 RANDOM.nextBytes(pubkey);
 
-                onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey));
+                Integer nonce = RANDOM.nextInt();
+
+                onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey, nonce));
             }
         }