Major rework of local data directory structure

Files are now keyed by signature, in the format:
data/si/gn/signature/hash

For times when there is no signature available (i.e. at the time of initial upload), files are keyed by hash, in the format:
data/_misc/ha/sh/hash

Files in the _misc folder are subsequently relocated to a path that is keyed by the resulting signature.

The end result is that chunks are now grouped on the filesystem by signature. This allows more transparency as to what is being hosted, and will also help simplify the reporting and management of local files.
This commit is contained in:
CalDescent
2021-11-27 13:00:32 +00:00
parent 0dd43d5c9a
commit bc38184ebf
10 changed files with 210 additions and 86 deletions

View File

@@ -20,7 +20,7 @@ public class ArbitraryDataFileTests extends Common {
@Test
public void testSplitAndJoin() throws DataException {
String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes());
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes(), null);
assertTrue(arbitraryDataFile.exists());
assertEquals(62, arbitraryDataFile.size());
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58());
@@ -50,7 +50,7 @@ public class ArbitraryDataFileTests extends Common {
byte[] randomData = new byte[fileSize];
new Random().nextBytes(randomData); // No need for SecureRandom here
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData);
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData, null);
assertTrue(arbitraryDataFile.exists());
assertEquals(fileSize, arbitraryDataFile.size());
String originalFileDigest = arbitraryDataFile.digest58();