Qortal/qortal
1
0
Fork 1
mirror of https://github.com/Qortal/qortal.git synced 2025-07-23 04:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Major rework of chunk hashes

Browse Source 
Chunk hashes are now stored off chain in a metadata file. The metadata file's hash is then included in the transaction.

The main benefits of this approach are:
1. We no longer need to limit the total file size, because adding more chunks doesn't increase the transaction size.
2. This increases the chain capacity by a huge amount - a 512MB file would have previously increased the transaction size by 16kB, whereas it now requires only an additional 32 bytes.
3. We no longer need to use variable difficulty; every transaction is the same size and so the difficulty can be constant no matter how large the files are.
4. Additional metadata (such as title, description, and tags) can ultimately be stored in the metadata file, as apposed to using a separate transaction & resource.
5. There is also scope for adding hashes of individual files into the metadata file, if we ever wanted to allow single files to be requested without having to download and build the entire resource. Although this is unlikely to be available in the short term.

The only real negative is that we now how to fetch the metadata file before we know anything about the chunks for a transaction. This seems to be quite a small trade off by comparison.

Since we're not live yet, there is no backwards support for on-chain hashes, so a new data testchain will be required. This hasn't been tested outside of unit tests yet, so there will likely be several fixes needed before it is stable.
This commit is contained in:
CalDescent
2021-12-01 12:37:21 +00:00
parent 7c16a90221
commit a2cac003a4
22 changed files with 809 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java Normal file
View File

@@ -0,0 +1,136 @@
package org.qortal.test.arbitrary;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.arbitrary.ArbitraryDataDigest;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.ArbitraryDataTransactionBuilder;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.RegisterNameTransactionData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.utils.Base58;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Random;
import static org.junit.Assert.*;
public class ArbitraryTransactionMetadataTests extends Common {
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
}
@Test
public void testMultipleChunks() throws DataException, IOException, MissingDataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());
String name = "TEST"; // Can be anything for this test
String identifier = null; // Not used for this test
Service service = Service.WEBSITE; // Can be anything for this test
int chunkSize = 100;
int dataLength = 900; // Actual data length will be longer due to encryption
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
Path path1 = generateRandomDataPath(dataLength);
ArbitraryDataFile arbitraryDataFile = this.createAndMintTxn(repository, publicKey58, path1, name, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize);
// Check the chunk count is correct
assertEquals(10, arbitraryDataFile.chunkCount());
// Now build the latest data state for this name
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
arbitraryDataReader.loadSynchronously(true);
Path initialLayerPath = arbitraryDataReader.getFilePath();
ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath);
initialLayerDigest.compute();
// Its directory hash should match the original directory hash
ArbitraryDataDigest path1Digest = new ArbitraryDataDigest(path1);
path1Digest.compute();
assertEquals(path1Digest.getHash58(), initialLayerDigest.getHash58());
}
}
private Path generateRandomDataPath(int length) throws IOException {
// Create a file in a random temp directory
Path tempDir = Files.createTempDirectory("generateRandomDataPath");
File file = new File(Paths.get(tempDir.toString(), "file.txt").toString());
file.deleteOnExit();
// Write a random string to the file
BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file));
String initialString = this.generateRandomString(length - 1); // -1 due to newline at EOF
// Add a newline every 50 chars
// initialString = initialString.replaceAll("(.{50})", "$1\n");
file1Writer.write(initialString);
file1Writer.newLine();
file1Writer.close();
return tempDir;
}
private String generateRandomString(int length) {
int leftLimit = 48; // numeral '0'
int rightLimit = 122; // letter 'z'
Random random = new Random();
return random.ints(leftLimit, rightLimit + 1)
.filter(i -> (i <= 57 || i >= 65) && (i <= 90 || i >= 97))
.limit(length)
.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append)
.toString();
}
private ArbitraryDataFile createAndMintTxn(Repository repository, String publicKey58, Path path, String name, String identifier,
ArbitraryTransactionData.Method method, Service service, PrivateKeyAccount account,
int chunkSize) throws DataException {
ArbitraryDataTransactionBuilder txnBuilder = new ArbitraryDataTransactionBuilder(
repository, publicKey58, path, name, method, service, identifier);
txnBuilder.setChunkSize(chunkSize);
txnBuilder.build();
txnBuilder.computeNonce();
ArbitraryTransactionData transactionData = txnBuilder.getArbitraryTransactionData();
Transaction.ValidationResult result = TransactionUtils.signAndImport(repository, transactionData, account);
assertEquals(Transaction.ValidationResult.OK, result);
BlockUtils.mintBlock(repository);
// We need a new ArbitraryDataFile instance because the files will have been moved to the signature's folder
byte[] hash = txnBuilder.getArbitraryDataFile().getHash();
byte[] signature = transactionData.getSignature();
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
arbitraryDataFile.setMetadataHash(transactionData.getMetadataHash());
return arbitraryDataFile;
}
}

70
src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java
View File

@@ -1,70 +0,0 @@
package org.qortal.test.arbitrary;
import org.junit.Before;
import org.junit.Test;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.PaymentData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.*;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction;
import java.util.ArrayList;
import java.util.List;
import static org.junit.Assert.*;
public class ArbitraryTransactionTests extends Common {
private static final int version = 4;
private static final String recipient = Common.getTestAccount(null, "bob").getAddress();
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
}
@Test
public void testDifficultyCalculation() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
TestAccount alice = Common.getTestAccount(repository, "alice");
ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
Service service = Service.ARBITRARY_DATA;
ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT;
ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.NONE;
List<PaymentData> payments = new ArrayList<>();
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(TestTransaction.generateBase(alice),
5, service, 0, 0, null, null, method,
null, compression, null, dataType, null, payments);
ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData);
assertEquals(12, transaction.difficultyForFileSize(1));
assertEquals(12, transaction.difficultyForFileSize(5123456));
assertEquals(12, transaction.difficultyForFileSize(74 * 1024 * 1024));
assertEquals(13, transaction.difficultyForFileSize(75 * 1024 * 1024));
assertEquals(13, transaction.difficultyForFileSize(144 * 1024 * 1024));
assertEquals(14, transaction.difficultyForFileSize(145 * 1024 * 1024));
assertEquals(14, transaction.difficultyForFileSize(214 * 1024 * 1024));
assertEquals(15, transaction.difficultyForFileSize(215 * 1024 * 1024));
assertEquals(15, transaction.difficultyForFileSize(289 * 1024 * 1024));
assertEquals(16, transaction.difficultyForFileSize(290 * 1024 * 1024));
assertEquals(16, transaction.difficultyForFileSize(359 * 1024 * 1024));
assertEquals(17, transaction.difficultyForFileSize(360 * 1024 * 1024));
assertEquals(17, transaction.difficultyForFileSize(429 * 1024 * 1024));
assertEquals(18, transaction.difficultyForFileSize(430 * 1024 * 1024));
assertEquals(18, transaction.difficultyForFileSize(499 * 1024 * 1024));
assertEquals(19, transaction.difficultyForFileSize(500 * 1024 * 1024));
}
}
}

6
src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java
View File

@@ -30,8 +30,8 @@ public class ArbitraryTestTransaction extends TestTransaction {
final ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP;
final byte[] chunkHashes = new byte[128];
random.nextBytes(chunkHashes);
final byte[] metadataHash = new byte[32];
random.nextBytes(metadataHash);
byte[] data = new byte[1024];
random.nextBytes(data);
@@ -46,7 +46,7 @@ public class ArbitraryTestTransaction extends TestTransaction {
payments.add(new PaymentData(recipient, assetId, amount));
return new ArbitraryTransactionData(generateBase(account), version, service, nonce, size,name, identifier,
method, secret, compression, data, dataType, chunkHashes, payments);
method, secret, compression, data, dataType, metadataHash, payments);
}
}