Redoing account balances with block height.

*** WARNING ***
Possible block reward bug in this commit. Further investigation needed.

Reverted AccountBalances back to height-less form.
Added HistoricAccountBalances table that is populated via trigger on AccountBalances.
This saves work when performing common requests for latest/confirmed balances,
shunting the extra work to when requesting height-related account balances.

Unified API call GET /addresses/balance/{address} by having address/assetId/height as
query params.

Simpler call for fetching legacy QORA holders during block rewarding.

Improved SQL for fetching asset balances, in all conditions,
e.g. with/without filtering addresses, with/without filtering assetIds,
etc.

Unit test for above to make sure query execution is fast enough.
(At one point, some SQL query was taking 6 seconds!)

Added optional 'height' Integer to AccountBalanceData, but this
is not populated/used very often.

HSQLDBAccountRepository.save(AccountBalanceData) now checks zero balance saves
to see if the row can be deleted instead. This fixes a lot of unhappy tests
that complain that there are residual account balance rows left after
post-test orphaning back to genesis block.

Yet more tests.
Removed very old 'TransactionTests' which are mostly covered in more specific tests elsewhere.
Added cancel-sell-name test from above.

Fixed AssetsApiTests to check for QORT not QORA!

Changed hard-coded assetIDs in test.common.AssetUtils in light of new LEGACY_QORA & QORT_FROM_QORA genesis assets.

Some test blockchain config changes.

src/test/java/org/qora/test/AccountBalanceTests.java

@@ -1,10 +1,14 @@
 package org.qora.test;
 
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
 
 import java.math.BigDecimal;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Random;
+import java.util.stream.Collectors;
 
 import org.junit.After;
 import org.junit.Before;
@@ -15,9 +19,11 @@ import org.qora.account.PublicKeyAccount;
 import org.qora.asset.Asset;
 import org.qora.block.BlockChain;
 import org.qora.data.account.AccountBalanceData;
+import org.qora.data.account.AccountData;
 import org.qora.data.transaction.BaseTransactionData;
 import org.qora.data.transaction.PaymentTransactionData;
 import org.qora.data.transaction.TransactionData;
+import org.qora.repository.AccountRepository.BalanceOrdering;
 import org.qora.repository.DataException;
 import org.qora.repository.Repository;
 import org.qora.repository.RepositoryManager;
@@ -78,7 +84,7 @@ public class AccountBalanceTests extends Common {
 			BigDecimal orphanedBalance = alice.getConfirmedBalance(Asset.QORT);
 
 			// Confirm post-orphan balance is same as initial
-			assertTrue("Post-orphan balance should match initial", orphanedBalance.equals(initialBalance));
+			assertEqualBigDecimals("Post-orphan balance should match initial", initialBalance, orphanedBalance);
 		}
 	}
 
@@ -95,7 +101,7 @@ public class AccountBalanceTests extends Common {
 			BigDecimal genesisBalance = accountBalanceData.getBalance();
 
 			// Confirm genesis balance is same as initial
-			assertTrue("Genesis balance should match initial", genesisBalance.equals(initialBalance));
+			assertEqualBigDecimals("Genesis balance should match initial", initialBalance, genesisBalance);
 		}
 	}
 
@@ -116,7 +122,7 @@ public class AccountBalanceTests extends Common {
 
 			// Confirm recipient balance is zero
 			BigDecimal balance = recipientAccount.getConfirmedBalance(Asset.QORT);
-			assertTrue("recipient's balance should be zero", balance.signum() == 0);
+			assertEqualBigDecimals("recipient's balance should be zero", BigDecimal.ZERO, balance);
 
 			// Send 1 QORT to recipient
 			TestAccount sendingAccount = Common.getTestAccount(repository, "alice");
@@ -129,24 +135,28 @@ public class AccountBalanceTests extends Common {
 			// Send more QORT to recipient
 			BigDecimal amount = BigDecimal.valueOf(random.nextInt(123456));
 			pay(repository, sendingAccount, recipientAccount, amount);
+			BigDecimal totalAmount = BigDecimal.ONE.add(amount);
 
 			// Mint some more blocks
 			for (int i = 0; i < 10; ++i)
 				BlockUtils.mintBlock(repository);
 
 			// Confirm recipient balance is as expected
-			BigDecimal totalAmount = amount.add(BigDecimal.ONE);
 			balance = recipientAccount.getConfirmedBalance(Asset.QORT);
-			assertTrue("recipient's balance incorrect", balance.compareTo(totalAmount) == 0);
+			assertEqualBigDecimals("recipient's balance incorrect", totalAmount, balance);
+
+			List<AccountBalanceData> historicBalances = repository.getAccountRepository().getHistoricBalances(recipientAccount.getAddress(), Asset.QORT);
+			for (AccountBalanceData historicBalance : historicBalances)
+				System.out.println(String.format("Block %d: %s", historicBalance.getHeight(), historicBalance.getBalance().toPlainString()));
 
 			// Confirm balance as of 2 blocks ago
 			int height = repository.getBlockRepository().getBlockchainHeight();
 			balance = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 2).getBalance();
-			assertTrue("recipient's historic balance incorrect", balance.compareTo(totalAmount) == 0);
+			assertEqualBigDecimals("recipient's historic balance incorrect", totalAmount, balance);
 
 			// Confirm balance prior to last payment
 			balance = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 15).getBalance();
-			assertTrue("recipient's historic balance incorrect", balance.compareTo(BigDecimal.ONE) == 0);
+			assertEqualBigDecimals("recipient's historic balance incorrect", BigDecimal.ONE, balance);
 
 			// Orphan blocks to before last payment
 			BlockUtils.orphanBlocks(repository, 10 + 5);
@@ -154,7 +164,7 @@ public class AccountBalanceTests extends Common {
 			// Re-check balance from (now) invalid height
 			AccountBalanceData accountBalanceData = repository.getAccountRepository().getBalance(recipientAccount.getAddress(), Asset.QORT, height - 2);
 			balance = accountBalanceData.getBalance();
-			assertTrue("recipient's invalid-height balance should be one", balance.compareTo(BigDecimal.ONE) == 0);
+			assertEqualBigDecimals("recipient's invalid-height balance should be one", BigDecimal.ONE, balance);
 
 			// Orphan blocks to before initial 1 QORT payment
 			BlockUtils.orphanBlocks(repository, 10 + 5);
@@ -177,4 +187,94 @@ public class AccountBalanceTests extends Common {
 		TransactionUtils.signAndMint(repository, transactionData, sendingAccount);
 	}
 
+	/** Tests SQL query speed for account balance fetches. */
+	@Test
+	public void testRepositorySpeed() throws DataException, SQLException {
+		Random random = new Random();
+		final long MAX_QUERY_TIME = 100L; // ms
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			System.out.println("Creating random accounts...");
+
+			// Generate some random accounts
+			List<Account> accounts = new ArrayList<>();
+			for (int ai = 0; ai < 20; ++ai) {
+				byte[] publicKey = new byte[32];
+				random.nextBytes(publicKey);
+
+				PublicKeyAccount account = new PublicKeyAccount(repository, publicKey);
+				accounts.add(account);
+
+				AccountData accountData = new AccountData(account.getAddress());
+				repository.getAccountRepository().ensureAccount(accountData);
+			}
+			repository.saveChanges();
+
+			System.out.println("Creating random balances...");
+
+			// Fill with lots of random balances
+			for (int i = 0; i < 100000; ++i) {
+				Account account = accounts.get(random.nextInt(accounts.size()));
+				int assetId = random.nextInt(2);
+				BigDecimal balance = BigDecimal.valueOf(random.nextInt(100000));
+
+				AccountBalanceData accountBalanceData = new AccountBalanceData(account.getAddress(), assetId, balance);
+				repository.getAccountRepository().save(accountBalanceData);
+
+				// Maybe mint a block to change height
+				if (i > 0 && (i % 1000) == 0)
+					BlockUtils.mintBlock(repository);
+			}
+			repository.saveChanges();
+
+			// Address filtering test cases
+			List<String> testAddresses = accounts.stream().limit(3).map(account -> account.getAddress()).collect(Collectors.toList());
+			List<List<String>> addressFilteringCases = Arrays.asList(null, testAddresses);
+
+			// AssetID filtering test cases
+			List<List<Long>> assetIdFilteringCases = Arrays.asList(null, Arrays.asList(0L, 1L, 2L));
+
+			// Results ordering test cases
+			List<BalanceOrdering> orderingCases = new ArrayList<>();
+			orderingCases.add(null);
+			orderingCases.addAll(Arrays.asList(BalanceOrdering.values()));
+
+			// Zero exclusion test cases
+			List<Boolean> zeroExclusionCases = Arrays.asList(null, true, false);
+
+			// Limit test cases
+			List<Integer> limitCases = Arrays.asList(null, 10);
+
+			// Offset test cases
+			List<Integer> offsetCases = Arrays.asList(null, 10);
+
+			// Reverse results cases
+			List<Boolean> reverseCases = Arrays.asList(null, true, false);
+
+			repository.setDebug(true);
+
+			// Test all cases
+			for (List<String> addresses : addressFilteringCases)
+				for (List<Long> assetIds : assetIdFilteringCases)
+					for (BalanceOrdering balanceOrdering : orderingCases)
+						for (Boolean excludeZero : zeroExclusionCases)
+							for (Integer limit : limitCases)
+								for (Integer offset : offsetCases)
+									for (Boolean reverse : reverseCases) {
+										repository.discardChanges();
+
+										System.out.println(String.format("Testing query: %s addresses, %s assetIDs, %s ordering, %b zero-exclusion, %d limit, %d offset, %b reverse",
+												(addresses == null ? "no" : "with"), (assetIds == null ? "no" : "with"), balanceOrdering, excludeZero, limit, offset, reverse));
+
+										long before = System.currentTimeMillis();
+										repository.getAccountRepository().getAssetBalances(addresses, assetIds, balanceOrdering, excludeZero, limit, offset, reverse);
+										final long period = System.currentTimeMillis() - before;
+										assertTrue(String.format("Query too slow: %dms", period), period < MAX_QUERY_TIME);
+									}
+		}
+
+		// Rebuild repository to avoid orphan check
+		Common.useDefaultSettings();
+	}
+
 }

src/test/java/org/qora/test/api/AssetsApiTests.java

@@ -86,7 +86,7 @@ public class AssetsApiTests extends ApiCommon {
 	@Test
 	public void testGetAssetInfo() {
 		assertNotNull(this.assetsResource.getAssetInfo((int) 0L, null));
-		assertNotNull(this.assetsResource.getAssetInfo(null, "QORA"));
+		assertNotNull(this.assetsResource.getAssetInfo(null, "QORT"));
 	}
 
 	@Test

src/test/java/org/qora/test/common/AssetUtils.java

@@ -28,9 +28,10 @@ public class AssetUtils {
 	public static final int txGroupId = Group.NO_GROUP;
 	public static final BigDecimal fee = BigDecimal.ONE.setScale(8);
 
-	public static final long testAssetId = 1L; // Owned by Alice
-	public static final long otherAssetId = 2L; // Owned by Bob
-	public static final long goldAssetId = 3L; // Owned by Alice
+	// QORT: 0, LEGACY_QORA: 1, QORT_FROM_QORA: 2
+	public static final long testAssetId = 3L; // Owned by Alice
+	public static final long otherAssetId = 4L; // Owned by Bob
+	public static final long goldAssetId = 5L; // Owned by Alice
 
 	public static long issueAsset(Repository repository, String issuerAccountName, String assetName, long quantity, boolean isDivisible) throws DataException {
 		PrivateKeyAccount account = Common.getTestAccount(repository, issuerAccountName);

src/test/java/org/qora/test/common/Common.java

@@ -146,13 +146,13 @@ public class Common {
 			}
 
 			List<AssetData> remainingAssets = repository.getAssetRepository().getAllAssets();
-			checkOrphanedLists("asset", initialAssets, remainingAssets, AssetData::getAssetId);
+			checkOrphanedLists("asset", initialAssets, remainingAssets, AssetData::getAssetId, AssetData::getAssetId);
 
 			List<GroupData> remainingGroups = repository.getGroupRepository().getAllGroups();
-			checkOrphanedLists("group", initialGroups, remainingGroups, GroupData::getGroupId);
+			checkOrphanedLists("group", initialGroups, remainingGroups, GroupData::getGroupId, GroupData::getGroupId);
 
 			List<AccountBalanceData> remainingBalances = repository.getAccountRepository().getAssetBalances(Collections.emptyList(), Collections.emptyList(), BalanceOrdering.ASSET_ACCOUNT, false, null, null, null);
-			checkOrphanedLists("account balance", initialBalances, remainingBalances, entry -> entry.getAssetName() + "-" + entry.getAddress());
+			checkOrphanedLists("account balance", initialBalances, remainingBalances, entry -> entry.getAddress() + " [" + entry.getAssetName() + "]", entry -> entry.getBalance().toPlainString());
 
 			assertEquals("remainingBalances is different size", initialBalances.size(), remainingBalances.size());
 			// Actually compare balances
@@ -168,7 +168,7 @@ public class Common {
 		}
 	}
 
-	private static <T> void checkOrphanedLists(String typeName, List<T> initial, List<T> remaining, Function<T, ? extends Object> keyExtractor) {
+	private static <T> void checkOrphanedLists(String typeName, List<T> initial, List<T> remaining, Function<T, ? extends Object> keyExtractor, Function<T, ? extends Object> valueExtractor) {
 		Predicate<T> isInitial = entry -> initial.stream().anyMatch(initialEntry -> keyExtractor.apply(initialEntry).equals(keyExtractor.apply(entry)));
 		Predicate<T> isRemaining = entry -> remaining.stream().anyMatch(remainingEntry -> keyExtractor.apply(remainingEntry).equals(keyExtractor.apply(entry)));
 
@@ -181,7 +181,7 @@ public class Common {
 		remainingClone.removeIf(isInitial);
 
 		for (T remainingEntry : remainingClone)
-			LOGGER.info(String.format("Non-genesis remaining entry: %s", keyExtractor.apply(remainingEntry)));
+			LOGGER.info(String.format("Non-genesis remaining entry: %s = %s", keyExtractor.apply(remainingEntry), valueExtractor.apply(remainingEntry)));
 
 		assertTrue(String.format("Non-genesis %s remains", typeName), remainingClone.isEmpty());
 	}

src/test/java/org/qora/test/common/TransactionUtils.java

@@ -25,7 +25,7 @@ public class TransactionUtils {
 		// Add to unconfirmed
 		assertTrue("Transaction's signature should be valid", transaction.isSignatureValid());
 
-		// We might need to wait until transaction's timestamp is valid for the block we're about to generate
+		// We might need to wait until transaction's timestamp is valid for the block we're about to mint
 		try {
 			Thread.sleep(1L);
 		} catch (InterruptedException e) {

src/test/java/org/qora/test/naming/BuySellTests.java

@@ -11,6 +11,7 @@ import org.junit.Test;
 import org.qora.account.PrivateKeyAccount;
 import org.qora.data.naming.NameData;
 import org.qora.data.transaction.BuyNameTransactionData;
+import org.qora.data.transaction.CancelSellNameTransactionData;
 import org.qora.data.transaction.RegisterNameTransactionData;
 import org.qora.data.transaction.SellNameTransactionData;
 import org.qora.repository.DataException;
@@ -21,7 +22,7 @@ import org.qora.test.common.Common;
 import org.qora.test.common.TransactionUtils;
 import org.qora.test.common.transaction.TestTransaction;
 
-public class OrphaningTests extends Common {
+public class BuySellTests extends Common {
 
 	protected static final Random random = new Random();
 
@@ -136,6 +137,31 @@ public class OrphaningTests extends Common {
 		assertEqualBigDecimals("price incorrect", price, nameData.getSalePrice());
 	}
 
+	@Test
+	public void testCancelSellName() throws DataException {
+		// Register-name and sell-name
+		testSellName();
+
+		// Cancel Sell-name
+		CancelSellNameTransactionData transactionData = new CancelSellNameTransactionData(TestTransaction.generateBase(alice), name);
+		TransactionUtils.signAndMint(repository, transactionData, alice);
+
+		NameData nameData;
+
+		// Check name is no longer for sale
+		nameData = repository.getNameRepository().fromName(name);
+		assertFalse(nameData.getIsForSale());
+		// Not concerned about price
+
+		// Orphan cancel sell-name
+		BlockUtils.orphanLastBlock(repository);
+
+		// Check name is for sale
+		nameData = repository.getNameRepository().fromName(name);
+		assertTrue(nameData.getIsForSale());
+		assertEqualBigDecimals("price incorrect", price, nameData.getSalePrice());
+	}
+
 	@Test
 	public void testBuyName() throws DataException {
 		// Register-name and sell-name

src/test/resources/test-chain-old-asset.json

@@ -8,6 +8,7 @@
 	"requireGroupForApproval": false,
 	"minAccountLevelToRewardShare": 5,
 	"maxRewardSharesPerMintingAccount": 20,
+	"founderEffectiveMintingLevel": 10,
 	"onlineAccountSignaturesMinLifetime": 3600000,
 	"onlineAccountSignaturesMaxLifetime": 86400000,
 	"rewardsByHeight": [
@@ -23,6 +24,7 @@
 		{ "levels": [ 9, 10 ], "share": 0.25 }
 	],
 	"qoraHoldersShare": 0.20,
+	"qoraPerQortReward": 250,
 	"blocksNeededByLevel": [ 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 ],
 	"blockTimingsByHeight": [
 		{ "height": 1, "target": 60000, "deviation": 30000, "power": 0.2 }
@@ -42,14 +44,19 @@
 		"version": 4,
 		"timestamp": 0,
 		"transactions": [
-			{ "type": "ISSUE_ASSET", "owner": "QcFmNxSArv5tWEzCtTKb2Lqc5QkKuQ7RNs", "assetName": "QORA", "description": "QORA native coin", "data": "", "quantity": 10000000000, "isDivisible": true, "fee": 0, "reference": "3Verk6ZKBJc3WTTVfxFC9icSjKdM8b92eeJEpJP8qNizG4ZszNFq8wdDYdSjJXq2iogDFR1njyhsBdVpbvDfjzU7" },
+			{ "type": "ISSUE_ASSET", "owner": "QUwGVHPPxJNJ2dq95abQNe79EyBN2K26zM", "assetName": "QORT", "description": "QORT native coin", "data": "", "quantity": 0, "isDivisible": true, "fee": 0, "reference": "3Verk6ZKBJc3WTTVfxFC9icSjKdM8b92eeJEpJP8qNizG4ZszNFq8wdDYdSjJXq2iogDFR1njyhsBdVpbvDfjzU7" },
+			{ "type": "ISSUE_ASSET", "owner": "QUwGVHPPxJNJ2dq95abQNe79EyBN2K26zM", "assetName": "Legacy-QORA", "description": "Representative legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true },
+			{ "type": "ISSUE_ASSET", "owner": "QUwGVHPPxJNJ2dq95abQNe79EyBN2K26zM", "assetName": "QORT-from-QORA", "description": "QORT gained from holding legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true },
+
 			{ "type": "GENESIS", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "amount": "1000000000", "fee": 0 },
 			{ "type": "GENESIS", "recipient": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "amount": "1000000", "fee": 0 },
 			{ "type": "GENESIS", "recipient": "QaUpHNhT3Ygx6avRiKobuLdusppR5biXjL", "amount": "1000000", "fee": 0 },
 			{ "type": "GENESIS", "recipient": "Qci5m9k4rcwe4ruKrZZQKka4FzUUMut3er", "amount": "1000000", "fee": 0 },
+
 			{ "type": "ISSUE_ASSET", "owner": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "assetName": "TEST", "description": "test asset", "data": "", "quantity": 1000000, "isDivisible": true, "fee": 0 },
 			{ "type": "ISSUE_ASSET", "owner": "QixPbJUwsaHsVEofJdozU9zgVqkK6aYhrK", "assetName": "OTHER", "description": "other test asset", "data": "", "quantity": 1000000, "isDivisible": true, "fee": 0 },
 			{ "type": "ISSUE_ASSET", "owner": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "assetName": "GOLD", "description": "gold test asset", "data": "", "quantity": 1000000, "isDivisible": true, "fee": 0 },
+
 			{ "type": "ACCOUNT_FLAGS", "target": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "andMask": -1, "orMask": 1, "xorMask": 0 },
 			{ "type": "REWARD_SHARE", "minterPublicKey": "2tiMr5LTpaWCgbRvkPK8TFd7k63DyHJMMFFsz9uBf1ZP", "recipient": "QgV4s3xnzLhVBEJxcYui4u4q11yhUHsd9v", "rewardSharePublicKey": "7PpfnvLSG7y4HPh8hE7KoqAjLCkv7Ui6xw4mKAkbZtox", "sharePercent": 100 }
 		]

src/test/resources/test-chain-v2.json

@@ -44,7 +44,7 @@
 		"version": 4,
 		"timestamp": 0,
 		"transactions": [
-			{ "type": "ISSUE_ASSET", "owner": "QcFmNxSArv5tWEzCtTKb2Lqc5QkKuQ7RNs", "assetName": "QORT", "description": "QORT native coin", "data": "", "quantity": 0, "isDivisible": true, "fee": 0, "reference": "3Verk6ZKBJc3WTTVfxFC9icSjKdM8b92eeJEpJP8qNizG4ZszNFq8wdDYdSjJXq2iogDFR1njyhsBdVpbvDfjzU7" },
+			{ "type": "ISSUE_ASSET", "owner": "QUwGVHPPxJNJ2dq95abQNe79EyBN2K26zM", "assetName": "QORT", "description": "QORT native coin", "data": "", "quantity": 0, "isDivisible": true, "fee": 0, "reference": "3Verk6ZKBJc3WTTVfxFC9icSjKdM8b92eeJEpJP8qNizG4ZszNFq8wdDYdSjJXq2iogDFR1njyhsBdVpbvDfjzU7" },
 			{ "type": "ISSUE_ASSET", "owner": "QUwGVHPPxJNJ2dq95abQNe79EyBN2K26zM", "assetName": "Legacy-QORA", "description": "Representative legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true },
 			{ "type": "ISSUE_ASSET", "owner": "QUwGVHPPxJNJ2dq95abQNe79EyBN2K26zM", "assetName": "QORT-from-QORA", "description": "QORT gained from holding legacy QORA", "quantity": 0, "isDivisible": true, "data": "{}", "isUnspendable": true },