Merge branch 'chat-reference'

# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2-block-timestamps.json
#	src/test/resources/test-chain-v2-disable-reference.json
#	src/test/resources/test-chain-v2-founder-rewards.json
#	src/test/resources/test-chain-v2-leftover-reward.json
#	src/test/resources/test-chain-v2-minting.json
#	src/test/resources/test-chain-v2-qora-holder-extremes.json
#	src/test/resources/test-chain-v2-qora-holder-reduction.json
#	src/test/resources/test-chain-v2-qora-holder.json
#	src/test/resources/test-chain-v2-reward-levels.json
#	src/test/resources/test-chain-v2-reward-scaling.json
#	src/test/resources/test-chain-v2-reward-shares.json
#	src/test/resources/test-chain-v2.json

src/main/java/org/qortal/api/resource/ChatResource.java

@@ -70,6 +70,8 @@ public class ChatResource {
 			@QueryParam("txGroupId") Integer txGroupId,
 			@QueryParam("involving") List<String> involvingAddresses,
 			@QueryParam("reference") String reference,
+			@QueryParam("chatreference") String chatReference,
+			@QueryParam("haschatreference") Boolean hasChatReference,
 			@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
 			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
 			@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) {
@@ -92,12 +94,18 @@ public class ChatResource {
 		if (reference != null)
 			referenceBytes = Base58.decode(reference);
 
+		byte[] chatReferenceBytes = null;
+		if (chatReference != null)
+			chatReferenceBytes = Base58.decode(chatReference);
+
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			return repository.getChatRepository().getMessagesMatchingCriteria(
 					before,
 					after,
 					txGroupId,
 					referenceBytes,
+					chatReferenceBytes,
+					hasChatReference,
 					involvingAddresses,
 					limit, offset, reverse);
 		} catch (DataException e) {

src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java

@@ -47,6 +47,8 @@ public class ChatMessagesWebSocket extends ApiWebSocket {
 						txGroupId,
 						null,
 						null,
+						null,
+						null,
 						null, null, null);
 
 				sendMessages(session, chatMessages);
@@ -74,6 +76,8 @@ public class ChatMessagesWebSocket extends ApiWebSocket {
 					null,
 					null,
 					null,
+					null,
+					null,
 					involvingAddresses,
 					null, null, null);
 

src/main/java/org/qortal/block/BlockChain.java

@@ -77,7 +77,8 @@ public class BlockChain {
 		increaseOnlineAccountsDifficultyTimestamp,
 		onlineAccountMinterLevelValidationHeight,
 		selfSponsorshipAlgoV1Height,
-		feeValidationFixTimestamp;
+		feeValidationFixTimestamp,
+		chatReferenceTimestamp;
 	}
 
 	// Custom transaction fees
@@ -506,6 +507,10 @@ public class BlockChain {
 		return this.featureTriggers.get(FeatureTrigger.feeValidationFixTimestamp.name()).longValue();
 	}
 
+	public long getChatReferenceTimestamp() {
+		return this.featureTriggers.get(FeatureTrigger.chatReferenceTimestamp.name()).longValue();
+	}
+
 
 	// More complex getters for aspects that change by height or timestamp
 

src/main/java/org/qortal/data/chat/ChatMessage.java

@@ -27,6 +27,8 @@ public class ChatMessage {
 
 	private String recipientName;
 
+	private byte[] chatReference;
+
 	private byte[] data;
 
 	private boolean isText;
@@ -42,8 +44,8 @@ public class ChatMessage {
 
 	// For repository use
 	public ChatMessage(long timestamp, int txGroupId, byte[] reference, byte[] senderPublicKey, String sender,
-			String senderName, String recipient, String recipientName, byte[] data, boolean isText,
-			boolean isEncrypted, byte[] signature) {
+			String senderName, String recipient, String recipientName, byte[] chatReference, byte[] data,
+			boolean isText, boolean isEncrypted, byte[] signature) {
 		this.timestamp = timestamp;
 		this.txGroupId = txGroupId;
 		this.reference = reference;
@@ -52,6 +54,7 @@ public class ChatMessage {
 		this.senderName = senderName;
 		this.recipient = recipient;
 		this.recipientName = recipientName;
+		this.chatReference = chatReference;
 		this.data = data;
 		this.isText = isText;
 		this.isEncrypted = isEncrypted;
@@ -90,6 +93,10 @@ public class ChatMessage {
 		return this.recipientName;
 	}
 
+	public byte[] getChatReference() {
+		return this.chatReference;
+	}
+
 	public byte[] getData() {
 		return this.data;
 	}

src/main/java/org/qortal/data/transaction/ChatTransactionData.java

@@ -26,6 +26,8 @@ public class ChatTransactionData extends TransactionData {
 
 	private String recipient; // can be null
 
+	private byte[] chatReference; // can be null
+
 	@Schema(description = "raw message data, possibly UTF8 text", example = "2yGEbwRFyhPZZckKA")
 	private byte[] data;
 
@@ -44,13 +46,14 @@ public class ChatTransactionData extends TransactionData {
 	}
 
 	public ChatTransactionData(BaseTransactionData baseTransactionData,
-			String sender, int nonce, String recipient, byte[] data, boolean isText, boolean isEncrypted) {
+			String sender, int nonce, String recipient, byte[] chatReference, byte[] data, boolean isText, boolean isEncrypted) {
 		super(TransactionType.CHAT, baseTransactionData);
 
 		this.senderPublicKey = baseTransactionData.creatorPublicKey;
 		this.sender = sender;
 		this.nonce = nonce;
 		this.recipient = recipient;
+		this.chatReference = chatReference;
 		this.data = data;
 		this.isText = isText;
 		this.isEncrypted = isEncrypted;
@@ -78,6 +81,10 @@ public class ChatTransactionData extends TransactionData {
 		return this.recipient;
 	}
 
+	public byte[] getChatReference() {
+		return this.chatReference;
+	}
+
 	public byte[] getData() {
 		return this.data;
 	}

src/main/java/org/qortal/repository/ChatRepository.java

@@ -14,8 +14,8 @@ public interface ChatRepository {
 	 * Expects EITHER non-null txGroupID OR non-null sender and recipient addresses.
 	 */
 	public List<ChatMessage> getMessagesMatchingCriteria(Long before, Long after,
-			Integer txGroupId, byte[] reference, List<String> involving,
-			Integer limit, Integer offset, Boolean reverse) throws DataException;
+			Integer txGroupId, byte[] reference, byte[] chatReferenceBytes, Boolean hasChatReference,
+			List<String> involving, Integer limit, Integer offset, Boolean reverse) throws DataException;
 
 	public ChatMessage toChatMessage(ChatTransactionData chatTransactionData) throws DataException;
 

src/main/java/org/qortal/repository/hsqldb/HSQLDBChatRepository.java

@@ -24,8 +24,8 @@ public class HSQLDBChatRepository implements ChatRepository {
 
 	@Override
 	public List<ChatMessage> getMessagesMatchingCriteria(Long before, Long after, Integer txGroupId, byte[] referenceBytes,
-			List<String> involving, Integer limit, Integer offset, Boolean reverse)
-			throws DataException {
+			byte[] chatReferenceBytes, Boolean hasChatReference, List<String> involving,
+			Integer limit, Integer offset, Boolean reverse) throws DataException {
 		// Check args meet expectations
 		if ((txGroupId != null && involving != null && !involving.isEmpty())
 				|| (txGroupId == null && (involving == null || involving.size() != 2)))
@@ -35,7 +35,7 @@ public class HSQLDBChatRepository implements ChatRepository {
 
 		sql.append("SELECT created_when, tx_group_id, Transactions.reference, creator, "
 				+ "sender, SenderNames.name, recipient, RecipientNames.name, "
-				+ "data, is_text, is_encrypted, signature "
+				+ "chat_reference, data, is_text, is_encrypted, signature "
 				+ "FROM ChatTransactions "
 				+ "JOIN Transactions USING (signature) "
 				+ "LEFT OUTER JOIN Names AS SenderNames ON SenderNames.owner = sender "
@@ -62,6 +62,18 @@ public class HSQLDBChatRepository implements ChatRepository {
 			bindParams.add(referenceBytes);
 		}
 
+		if (chatReferenceBytes != null) {
+			whereClauses.add("chat_reference = ?");
+			bindParams.add(chatReferenceBytes);
+		}
+
+		if (hasChatReference != null && hasChatReference == true) {
+			whereClauses.add("chat_reference IS NOT NULL");
+		}
+		else if (hasChatReference != null && hasChatReference == false) {
+			whereClauses.add("chat_reference IS NULL");
+		}
+
 		if (txGroupId != null) {
 			whereClauses.add("tx_group_id = " + txGroupId); // int safe to use literally
 			whereClauses.add("recipient IS NULL");
@@ -103,13 +115,14 @@ public class HSQLDBChatRepository implements ChatRepository {
 				String senderName = resultSet.getString(6);
 				String recipient = resultSet.getString(7);
 				String recipientName = resultSet.getString(8);
-				byte[] data = resultSet.getBytes(9);
-				boolean isText = resultSet.getBoolean(10);
-				boolean isEncrypted = resultSet.getBoolean(11);
-				byte[] signature = resultSet.getBytes(12);
+				byte[] chatReference = resultSet.getBytes(9);
+				byte[] data = resultSet.getBytes(10);
+				boolean isText = resultSet.getBoolean(11);
+				boolean isEncrypted = resultSet.getBoolean(12);
+				byte[] signature = resultSet.getBytes(13);
 
 				ChatMessage chatMessage = new ChatMessage(timestamp, groupId, reference, senderPublicKey, sender,
-						senderName, recipient, recipientName, data, isText, isEncrypted, signature);
+						senderName, recipient, recipientName, chatReference, data, isText, isEncrypted, signature);
 
 				chatMessages.add(chatMessage);
 			} while (resultSet.next());
@@ -141,13 +154,14 @@ public class HSQLDBChatRepository implements ChatRepository {
 			byte[] senderPublicKey = chatTransactionData.getSenderPublicKey();
 			String sender = chatTransactionData.getSender();
 			String recipient = chatTransactionData.getRecipient();
+			byte[] chatReference = chatTransactionData.getChatReference();
 			byte[] data = chatTransactionData.getData();
 			boolean isText = chatTransactionData.getIsText();
 			boolean isEncrypted = chatTransactionData.getIsEncrypted();
 			byte[] signature = chatTransactionData.getSignature();
 
 			return new ChatMessage(timestamp, groupId, reference, senderPublicKey, sender,
-					senderName, recipient, recipientName, data, isText, isEncrypted, signature);
+					senderName, recipient, recipientName, chatReference, data, isText, isEncrypted, signature);
 		} catch (SQLException e) {
 			throw new DataException("Unable to fetch convert chat transaction from repository", e);
 		}

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -980,6 +980,12 @@ public class HSQLDBDatabaseUpdates {
 					stmt.execute("ALTER TABLE Accounts ADD blocks_minted_penalty INTEGER NOT NULL DEFAULT 0");
 					break;
 
+				case 45:
+					// Add a chat reference, to allow one message to reference another, and for this to be easily
+					// searchable. Null values are allowed as most transactions won't have a reference.
+					stmt.execute("ALTER TABLE ChatTransactions ADD chat_reference Signature");
+					break;
+
 				default:
 					// nothing to do
 					return false;

src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBChatTransactionRepository.java

@@ -17,7 +17,7 @@ public class HSQLDBChatTransactionRepository extends HSQLDBTransactionRepository
 	}
 
 	TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException {
-		String sql = "SELECT sender, nonce, recipient, is_text, is_encrypted, data FROM ChatTransactions WHERE signature = ?";
+		String sql = "SELECT sender, nonce, recipient, is_text, is_encrypted, data, chat_reference FROM ChatTransactions WHERE signature = ?";
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) {
 			if (resultSet == null)
@@ -29,8 +29,9 @@ public class HSQLDBChatTransactionRepository extends HSQLDBTransactionRepository
 			boolean isText = resultSet.getBoolean(4);
 			boolean isEncrypted = resultSet.getBoolean(5);
 			byte[] data = resultSet.getBytes(6);
+			byte[] chatReference = resultSet.getBytes(7);
 
-			return new ChatTransactionData(baseTransactionData, sender, nonce, recipient, data, isText, isEncrypted);
+			return new ChatTransactionData(baseTransactionData, sender, nonce, recipient, chatReference, data, isText, isEncrypted);
 		} catch (SQLException e) {
 			throw new DataException("Unable to fetch chat transaction from repository", e);
 		}
@@ -45,7 +46,7 @@ public class HSQLDBChatTransactionRepository extends HSQLDBTransactionRepository
 		saveHelper.bind("signature", chatTransactionData.getSignature()).bind("nonce", chatTransactionData.getNonce())
 				.bind("sender", chatTransactionData.getSender()).bind("recipient", chatTransactionData.getRecipient())
 				.bind("is_text", chatTransactionData.getIsText()).bind("is_encrypted", chatTransactionData.getIsEncrypted())
-				.bind("data", chatTransactionData.getData());
+				.bind("data", chatTransactionData.getData()).bind("chat_reference", chatTransactionData.getChatReference());
 
 		try {
 			saveHelper.execute(this.repository);

src/main/java/org/qortal/transform/transaction/ChatTransactionTransformer.java

@@ -4,6 +4,7 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
+import org.qortal.block.BlockChain;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.transaction.BaseTransactionData;
 import org.qortal.data.transaction.ChatTransactionData;
@@ -22,11 +23,13 @@ public class ChatTransactionTransformer extends TransactionTransformer {
 	private static final int NONCE_LENGTH = INT_LENGTH;
 	private static final int HAS_RECIPIENT_LENGTH = BOOLEAN_LENGTH;
 	private static final int RECIPIENT_LENGTH = ADDRESS_LENGTH;
+	private static final int HAS_CHAT_REFERENCE_LENGTH = BOOLEAN_LENGTH;
+	private static final int CHAT_REFERENCE_LENGTH = SIGNATURE_LENGTH;
 	private static final int DATA_SIZE_LENGTH = INT_LENGTH;
 	private static final int IS_TEXT_LENGTH = BOOLEAN_LENGTH;
 	private static final int IS_ENCRYPTED_LENGTH = BOOLEAN_LENGTH;
 
-	private static final int EXTRAS_LENGTH = NONCE_LENGTH + HAS_RECIPIENT_LENGTH + DATA_SIZE_LENGTH + IS_ENCRYPTED_LENGTH + IS_TEXT_LENGTH;
+	private static final int EXTRAS_LENGTH = NONCE_LENGTH + HAS_RECIPIENT_LENGTH + DATA_SIZE_LENGTH + IS_ENCRYPTED_LENGTH + IS_TEXT_LENGTH + HAS_CHAT_REFERENCE_LENGTH;
 
 	protected static final TransactionLayout layout;
 
@@ -77,13 +80,24 @@ public class ChatTransactionTransformer extends TransactionTransformer {
 
 		long fee = byteBuffer.getLong();
 
+		byte[] chatReference = null;
+
+		if (timestamp >= BlockChain.getInstance().getChatReferenceTimestamp()) {
+			boolean hasChatReference = byteBuffer.get() != 0;
+
+			if (hasChatReference) {
+				chatReference = new byte[CHAT_REFERENCE_LENGTH];
+				byteBuffer.get(chatReference);
+			}
+		}
+
 		byte[] signature = new byte[SIGNATURE_LENGTH];
 		byteBuffer.get(signature);
 
 		BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature);
 
 		String sender = Crypto.toAddress(senderPublicKey);
-		return new ChatTransactionData(baseTransactionData, sender, nonce, recipient, data, isText, isEncrypted);
+		return new ChatTransactionData(baseTransactionData, sender, nonce, recipient, chatReference, data, isText, isEncrypted);
 	}
 
 	public static int getDataLength(TransactionData transactionData) {
@@ -94,6 +108,9 @@ public class ChatTransactionTransformer extends TransactionTransformer {
 		if (chatTransactionData.getRecipient() != null)
 			dataLength += RECIPIENT_LENGTH;
 
+		if (chatTransactionData.getChatReference() != null)
+			dataLength += CHAT_REFERENCE_LENGTH;
+
 		return dataLength;
 	}
 
@@ -124,6 +141,16 @@ public class ChatTransactionTransformer extends TransactionTransformer {
 
 			bytes.write(Longs.toByteArray(chatTransactionData.getFee()));
 
+			if (transactionData.getTimestamp() >= BlockChain.getInstance().getChatReferenceTimestamp()) {
+				// Include chat reference if it's not null
+				if (chatTransactionData.getChatReference() != null) {
+					bytes.write((byte) 1);
+					bytes.write(chatTransactionData.getChatReference());
+				} else {
+					bytes.write((byte) 0);
+				}
+			}
+
 			if (chatTransactionData.getSignature() != null)
 				bytes.write(chatTransactionData.getSignature());
 

src/main/resources/blockchain.json

@@ -84,7 +84,8 @@
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 1092000,
 		"selfSponsorshipAlgoV1Height": 1092400,
-		"feeValidationFixTimestamp": 1671918000000
+		"feeValidationFixTimestamp": 1671918000000,
+		"chatReferenceTimestamp": 9999999999999
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-block-timestamps.json

@@ -73,7 +73,8 @@
 		"disableReferenceTimestamp": 9999999999999,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-disable-reference.json

@@ -76,7 +76,8 @@
 		"disableReferenceTimestamp": 0,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-founder-rewards.json

@@ -77,7 +77,8 @@
 		"disableReferenceTimestamp": 9999999999999,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-leftover-reward.json

@@ -77,7 +77,8 @@
 		"disableReferenceTimestamp": 9999999999999,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-minting.json

@@ -77,7 +77,8 @@
 		"disableReferenceTimestamp": 9999999999999,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-qora-holder-extremes.json

@@ -77,7 +77,8 @@
 		"disableReferenceTimestamp": 9999999999999,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-qora-holder-reduction.json

@@ -78,7 +78,8 @@
 		"aggregateSignatureTimestamp": 0,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-qora-holder.json

@@ -77,7 +77,8 @@
 		"disableReferenceTimestamp": 9999999999999,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-reward-levels.json

@@ -77,7 +77,8 @@
 		"disableReferenceTimestamp": 9999999999999,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-reward-scaling.json

@@ -77,7 +77,8 @@
 		"disableReferenceTimestamp": 9999999999999,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2-reward-shares.json

@@ -77,7 +77,8 @@
 		"disableReferenceTimestamp": 9999999999999,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,

src/test/resources/test-chain-v2.json

@@ -77,7 +77,8 @@
 		"disableReferenceTimestamp": 9999999999999,
 		"increaseOnlineAccountsDifficultyTimestamp": 9999999999999,
 		"onlineAccountMinterLevelValidationHeight": 0,
-		"selfSponsorshipAlgoV1Height": 999999999
+		"selfSponsorshipAlgoV1Height": 999999999,
+		"chatReferenceTimestamp": 0
 	},
 	"genesisInfo": {
 		"version": 4,