AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Added "original copy indicator file", which prevents the node from deleting its own published content when storage space runs out.

Browse Source 
Since some files won't have any mirrors, this prevents the cleanup manager from deleting the only copy in existence when freeing up space. This feature can be disabled by setting "originalCopyIndicatorFileEnabled": false in settings.json (or by deleting the ".original" files). The trade off is that the only copy in existence could be deleted if space gets low.

This will also allow for better reporting of own vs third party files in the local UI (not yet implemented).
This commit is contained in:
CalDescent 2022-01-01 14:52:09 +00:00
parent cf2c8d6c67
commit 7aa8f115ce
5 changed files with 96 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@ -18,6 +18,7 @@ import org.qortal.utils.NTP;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.SecureRandom;
@ -358,6 +359,16 @@ public class ArbitraryDataCleanupManager extends Thread {
// If it's a file, we might be able to delete it
if (randomItem.isFile()) {
// If the parent directory contains an ".original" file, don't delete anything
// This indicates that the content was originally updated by this node and so
// could be the only copy that exists.
Path originalCopyIndicatorPath = Paths.get(randomItem.getParent(), ".original");
if (Files.exists(originalCopyIndicatorPath)) {
// This is an original seed copy and so shouldn't be deleted
return false;
}
if (name != null) {
// A name has been specified, so we need to make sure this file relates to
// the name we want to delete. The signature should be the name of parent directory.

8
src/main/java/org/qortal/settings/Settings.java
View File

@ -289,6 +289,10 @@ public class Settings {
/** Whether to allow data outside of the storage policy to be relayed between other peers */
private boolean relayModeEnabled = false;
/** Whether to remember which data was originally uploaded using this node.
* This prevents auto deletion of own files when storage limits are reached. */
private boolean originalCopyIndicatorFileEnabled = true;
/** Whether to make connections directly with peers that have the required data */
private boolean directDataRetrievalEnabled = true;
@ -861,6 +865,10 @@ public class Settings {
return this.directDataRetrievalEnabled;
}
public boolean isOriginalCopyIndicatorFileEnabled() {
return this.originalCopyIndicatorFileEnabled;
}
public Long getBuiltDataExpiryInterval() {
return this.builtDataExpiryInterval;
}

16
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@ -9,10 +9,13 @@ import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.settings.Settings;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
@ -327,6 +330,19 @@ public class ArbitraryTransactionUtils {
// Delete empty parent directories
FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath);
}
// If at least one file was relocated, we can assume that the data from this transaction
// originated from this node
if (filesRelocatedCount > 0) {
if (Settings.getInstance().isOriginalCopyIndicatorFileEnabled()) {
// Create a file in the same directory, to indicate that this is the original copy
LOGGER.info("Creating original copy indicator file...");
ArbitraryDataFile completeFile = ArbitraryDataFile.fromHash(arbitraryDataFile.getHash(), signature);
Path parentDirectory = completeFile.getFilePath().getParent();
File file = Paths.get(parentDirectory.toString(), ".original").toFile();
file.createNewFile();
}
}
}
catch (DataException | IOException e) {
LOGGER.info("Unable to check and relocate all files for signature {}: {}",

5
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java
View File

@ -139,13 +139,16 @@ public class ArbitraryDataStorageCapacityTests extends Common {
}
@Test
public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException {
public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException, IllegalAccessException {
try (final Repository repository = RepositoryManager.getRepository()) {
String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA;
int chunkSize = 100;
int dataLength = 900; // Actual data length will be longer due to encryption
// Set originalCopyIndicatorFileEnabled to false, otherwise nothing will be deleted as it all originates from this node
FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true);
// Alice hosts some data (with 10 chunks)
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String aliceName = "alice";

57
src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java
View File

@ -5,6 +5,7 @@ import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.arbitrary.ArbitraryDataDigest;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.exception.MissingDataException;
@ -17,6 +18,7 @@ import org.qortal.data.transaction.RegisterNameTransactionData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.test.common.ArbitraryUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
@ -367,4 +369,59 @@ public class ArbitraryDataTests extends Common {
}
}
@Test
public void testOriginalCopyIndicatorFile() throws DataException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());
String name = "TEST"; // Can be anything for this test
String identifier = "test1"; // Blank, not null
Service service = Service.DOCUMENT; // Can be anything for this test
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
Path path1 = Paths.get("src/test/resources/arbitrary/demo1/lorem1.txt");
ArbitraryDataDigest path1DirectoryDigest = new ArbitraryDataDigest(path1.getParent());
path1DirectoryDigest.compute();
ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice);
// Ensure that an ".original" file exists
Path parentPath = arbitraryDataFile.getFilePath().getParent();
Path originalCopyIndicatorFile = Paths.get(parentPath.toString(), ".original");
assertTrue(Files.exists(originalCopyIndicatorFile));
}
}
@Test
public void testOriginalCopyIndicatorFileDisabled() throws DataException, IOException, IllegalAccessException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());
String name = "TEST"; // Can be anything for this test
String identifier = "test1"; // Blank, not null
Service service = Service.DOCUMENT; // Can be anything for this test
// Set originalCopyIndicatorFileEnabled to false
FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true);
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
Path path1 = Paths.get("src/test/resources/arbitrary/demo1/lorem1.txt");
ArbitraryDataDigest path1DirectoryDigest = new ArbitraryDataDigest(path1.getParent());
path1DirectoryDigest.compute();
ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice);
// Ensure that an ".original" file exists
Path parentPath = arbitraryDataFile.getFilePath().getParent();
Path originalCopyIndicatorFile = Paths.get(parentPath.toString(), ".original");
assertFalse(Files.exists(originalCopyIndicatorFile));
}
}
}