diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 084a91f7..8c263568 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -18,6 +18,7 @@ import org.qortal.utils.NTP; import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.security.SecureRandom; @@ -358,6 +359,16 @@ public class ArbitraryDataCleanupManager extends Thread { // If it's a file, we might be able to delete it if (randomItem.isFile()) { + + // If the parent directory contains an ".original" file, don't delete anything + // This indicates that the content was originally updated by this node and so + // could be the only copy that exists. + Path originalCopyIndicatorPath = Paths.get(randomItem.getParent(), ".original"); + if (Files.exists(originalCopyIndicatorPath)) { + // This is an original seed copy and so shouldn't be deleted + return false; + } + if (name != null) { // A name has been specified, so we need to make sure this file relates to // the name we want to delete. The signature should be the name of parent directory. diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 627c29ba..d05cf392 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -289,6 +289,10 @@ public class Settings { /** Whether to allow data outside of the storage policy to be relayed between other peers */ private boolean relayModeEnabled = false; + /** Whether to remember which data was originally uploaded using this node. + * This prevents auto deletion of own files when storage limits are reached. */ + private boolean originalCopyIndicatorFileEnabled = true; + /** Whether to make connections directly with peers that have the required data */ private boolean directDataRetrievalEnabled = true; @@ -861,6 +865,10 @@ public class Settings { return this.directDataRetrievalEnabled; } + public boolean isOriginalCopyIndicatorFileEnabled() { + return this.originalCopyIndicatorFileEnabled; + } + public Long getBuiltDataExpiryInterval() { return this.builtDataExpiryInterval; } diff --git a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java index 23a30354..51c2b2d3 100644 --- a/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java +++ b/src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java @@ -9,10 +9,13 @@ import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import org.qortal.settings.Settings; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; @@ -327,6 +330,19 @@ public class ArbitraryTransactionUtils { // Delete empty parent directories FilesystemUtils.safeDeleteEmptyParentDirectories(oldPath); } + + // If at least one file was relocated, we can assume that the data from this transaction + // originated from this node + if (filesRelocatedCount > 0) { + if (Settings.getInstance().isOriginalCopyIndicatorFileEnabled()) { + // Create a file in the same directory, to indicate that this is the original copy + LOGGER.info("Creating original copy indicator file..."); + ArbitraryDataFile completeFile = ArbitraryDataFile.fromHash(arbitraryDataFile.getHash(), signature); + Path parentDirectory = completeFile.getFilePath().getParent(); + File file = Paths.get(parentDirectory.toString(), ".original").toFile(); + file.createNewFile(); + } + } } catch (DataException | IOException e) { LOGGER.info("Unable to check and relocate all files for signature {}: {}", diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java index 62928855..c38327c3 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java @@ -139,13 +139,16 @@ public class ArbitraryDataStorageCapacityTests extends Common { } @Test - public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException { + public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException, IllegalAccessException { try (final Repository repository = RepositoryManager.getRepository()) { String identifier = null; // Not used for this test Service service = Service.ARBITRARY_DATA; int chunkSize = 100; int dataLength = 900; // Actual data length will be longer due to encryption + // Set originalCopyIndicatorFileEnabled to false, otherwise nothing will be deleted as it all originates from this node + FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true); + // Alice hosts some data (with 10 chunks) PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); String aliceName = "alice"; diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java index 58c798de..b3af7786 100644 --- a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java @@ -5,6 +5,7 @@ import org.junit.Before; import org.junit.Test; import org.qortal.account.PrivateKeyAccount; import org.qortal.arbitrary.ArbitraryDataDigest; +import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.arbitrary.ArbitraryDataReader; import org.qortal.arbitrary.exception.MissingDataException; @@ -17,6 +18,7 @@ import org.qortal.data.transaction.RegisterNameTransactionData; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; +import org.qortal.settings.Settings; import org.qortal.test.common.ArbitraryUtils; import org.qortal.test.common.Common; import org.qortal.test.common.TransactionUtils; @@ -367,4 +369,59 @@ public class ArbitraryDataTests extends Common { } } + @Test + public void testOriginalCopyIndicatorFile() throws DataException, IOException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = "test1"; // Blank, not null + Service service = Service.DOCUMENT; // Can be anything for this test + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1/lorem1.txt"); + ArbitraryDataDigest path1DirectoryDigest = new ArbitraryDataDigest(path1.getParent()); + path1DirectoryDigest.compute(); + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Ensure that an ".original" file exists + Path parentPath = arbitraryDataFile.getFilePath().getParent(); + Path originalCopyIndicatorFile = Paths.get(parentPath.toString(), ".original"); + assertTrue(Files.exists(originalCopyIndicatorFile)); + } + } + + @Test + public void testOriginalCopyIndicatorFileDisabled() throws DataException, IOException, IllegalAccessException { + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount alice = Common.getTestAccount(repository, "alice"); + String publicKey58 = Base58.encode(alice.getPublicKey()); + String name = "TEST"; // Can be anything for this test + String identifier = "test1"; // Blank, not null + Service service = Service.DOCUMENT; // Can be anything for this test + + // Set originalCopyIndicatorFileEnabled to false + FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true); + + // Register the name to Alice + RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""); + TransactionUtils.signAndMint(repository, transactionData, alice); + + // Create PUT transaction + Path path1 = Paths.get("src/test/resources/arbitrary/demo1/lorem1.txt"); + ArbitraryDataDigest path1DirectoryDigest = new ArbitraryDataDigest(path1.getParent()); + path1DirectoryDigest.compute(); + ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name, identifier, Method.PUT, service, alice); + + // Ensure that an ".original" file exists + Path parentPath = arbitraryDataFile.getFilePath().getParent(); + Path originalCopyIndicatorFile = Paths.get(parentPath.toString(), ".original"); + assertFalse(Files.exists(originalCopyIndicatorFile)); + } + } + }