diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 96bdcdb5..045846aa 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -177,7 +177,7 @@ public class ArbitraryDataFile { File file = path.toFile(); if (file.exists()) { try { - byte[] digest = Crypto.digest(file); + byte[] digest = Crypto.digestFileStream(file); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature); // Copy file to data directory if needed @@ -352,7 +352,7 @@ public class ArbitraryDataFile { return this.chunks.size(); } - public boolean join() { + public boolean join() { // Ensure we have chunks if (this.chunks != null && !this.chunks.isEmpty()) { @@ -373,7 +373,7 @@ public class ArbitraryDataFile { for (ArbitraryDataFileChunk chunk : this.chunks) { File sourceFile = chunk.filePath.toFile(); BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); - byte[] buffer = new byte[2048]; + byte[] buffer = new byte[8192]; int inSize; while ((inSize = in.read(buffer)) != -1) { out.write(buffer, 0, inSize); @@ -398,6 +398,8 @@ public class ArbitraryDataFile { return false; } + + public boolean delete() { // Delete the complete file // ... but only if it's inside the Qortal data or temp directory @@ -667,6 +669,9 @@ public class ArbitraryDataFile { } } + + + public boolean containsChunk(byte[] hash) { for (ArbitraryDataFileChunk chunk : this.chunks) { if (Arrays.equals(hash, chunk.getHash())) { @@ -781,18 +786,17 @@ public class ArbitraryDataFile { return this.filePath; } - public byte[] digest() { - File file = this.getFile(); - if (file != null && file.exists()) { - try { - return Crypto.digest(file); - - } catch (IOException e) { - LOGGER.error("Couldn't compute digest for ArbitraryDataFile"); - } + public byte[] digest() { + File file = this.getFile(); + if (file != null && file.exists()) { + try { + return Crypto.digestFileStream(file); + } catch (IOException e) { + LOGGER.error("Couldn't compute digest for ArbitraryDataFile"); } - return null; } + return null; +} public String digest58() { if (this.digest() != null) { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 6d7e0e23..f3a80be3 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -437,16 +437,24 @@ public class ArbitraryDataReader { throw new IOException(String.format("File doesn't exist: %s", arbitraryDataFile)); } // Ensure the complete hash matches the joined chunks - if (!Arrays.equals(arbitraryDataFile.digest(), transactionData.getData())) { - // Delete the invalid file - LOGGER.info("Deleting invalid file: path = " + arbitraryDataFile.getFilePath()); - if( arbitraryDataFile.delete() ) { - LOGGER.info("Deleted invalid file successfully: path = " + arbitraryDataFile.getFilePath()); - } - else { - LOGGER.warn("Could not delete invalid file: path = " + arbitraryDataFile.getFilePath()); - } + if (!Arrays.equals(arbitraryDataFile.digest(), transactionData.getData())) { + + + // Delete the invalid file + LOGGER.info("Deleting invalid file: path = {}", arbitraryDataFile.getFilePath()); + if (arbitraryDataFile.delete()) { + LOGGER.info("Deleted invalid file successfully: path = {}", arbitraryDataFile.getFilePath()); + } else { + LOGGER.warn("Could not delete invalid file: path = {}", arbitraryDataFile.getFilePath()); + } + + // Also delete its chunks + if (arbitraryDataFile.deleteAllChunks()) { + LOGGER.info("Deleted all chunks associated with invalid file: {}", arbitraryDataFile.getFilePath()); + } else { + LOGGER.warn("Failed to delete one or more chunks for invalid file: {}", arbitraryDataFile.getFilePath()); + } throw new DataException("Unable to validate complete file hash"); } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 71368082..0a51fa49 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -32,6 +32,8 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import org.qortal.crypto.Crypto; + public class ArbitraryDataFileManager extends Thread { public static final int SEND_TIMEOUT_MS = 500; @@ -249,6 +251,18 @@ public class ArbitraryDataFileManager extends Thread { ArbitraryDataFileMessage peersArbitraryDataFileMessage = (ArbitraryDataFileMessage) response; arbitraryDataFile = peersArbitraryDataFileMessage.getArbitraryDataFile(); + byte[] fileBytes = arbitraryDataFile.getBytes(); + if (fileBytes == null) { + LOGGER.debug(String.format("Failed to read bytes for file hash %s", hash58)); + return null; + } + + byte[] actualHash = Crypto.digest(fileBytes); + if (!Arrays.equals(hash, actualHash)) { + LOGGER.debug(String.format("Hash mismatch for chunk: expected %s but got %s", + hash58, Base58.encode(actualHash))); + return null; + } } else { LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58)); arbitraryDataFile = existingFile; diff --git a/src/main/java/org/qortal/crypto/Crypto.java b/src/main/java/org/qortal/crypto/Crypto.java index 3beb7abd..a626394e 100644 --- a/src/main/java/org/qortal/crypto/Crypto.java +++ b/src/main/java/org/qortal/crypto/Crypto.java @@ -1,6 +1,7 @@ package org.qortal.crypto; import com.google.common.primitives.Bytes; + import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters; import org.bouncycastle.crypto.params.X25519PrivateKeyParameters; import org.bouncycastle.crypto.params.X25519PublicKeyParameters; @@ -11,6 +12,7 @@ import org.qortal.utils.Base58; import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -66,6 +68,20 @@ public abstract class Crypto { } } + public static byte[] digestFileStream(File file) throws IOException { + try (InputStream fis = new FileInputStream(file)) { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] buffer = new byte[8192]; // 8 KB buffer + int bytesRead; + while ((bytesRead = fis.read(buffer)) != -1) { + digest.update(buffer, 0, bytesRead); + } + return digest.digest(); + } catch (NoSuchAlgorithmException e) { + throw new IOException("SHA-256 algorithm not available", e); + } + } + /** * Returns 32-byte digest of two rounds of SHA-256 on message passed in input. *