Encrypt websites with AES.

This ensures that nodes are storing unreadable files, outside of the context of Qortal. For public data, the decryption keys themselves are on-chain, included in the "secret" field of arbitrary transactions. When we introduce the concept of private data, we can simply exclude the secret key from the transaction so that only the owner can decrypt it.

When encrypting the file, I have added the 16 byte initialization vector as a prefix to the cyphertext, and it is then automatically extracted back out when decrypting. This gives us the option to encrypt more than one file with the same key, if we ever need it. Right now, we are using a unique key per file, so it's not actually needed, but it's good to have support.

src/main/java/org/qortal/api/resource/WebsiteResource.java

@@ -1,5 +1,10 @@
 package org.qortal.api.resource;
 
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
 import javax.servlet.ServletContext;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -9,6 +14,9 @@ import javax.ws.rs.core.MediaType;
 import java.io.*;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -28,6 +36,7 @@ import org.qortal.api.ApiExceptionFactory;
 import org.qortal.api.HTMLParser;
 import org.qortal.api.Security;
 import org.qortal.block.BlockChain;
+import org.qortal.crypto.AES;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.PaymentData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
@@ -41,6 +50,7 @@ import org.qortal.storage.DataFile;
 import org.qortal.transaction.ArbitraryTransaction;
 import org.qortal.transaction.Transaction;
 import org.qortal.transform.TransformationException;
+import org.qortal.transform.Transformer;
 import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
 import org.qortal.utils.Base58;
 import org.qortal.utils.NTP;
@@ -96,7 +106,6 @@ public class WebsiteResource {
         byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58);
 
         String name = null;
-        byte[] secret = null;
         ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT;
         ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE;
         ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP;
@@ -122,13 +131,14 @@ public class WebsiteResource {
             final int size = (int)dataFile.size();
             final int version = 5;
             final int nonce = 0;
+            byte[] secret = dataFile.getSecret();
             final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
             final byte[] digest = dataFile.digest();
             final byte[] chunkHashes = dataFile.chunkHashes();
             final List<PaymentData> payments = new ArrayList<>();
 
             ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
-                    5, service, nonce, size, name, method,
+                    version, service, nonce, size, name, method,
                     secret, compression, digest, dataType, chunkHashes, payments);
 
             ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData);
@@ -214,16 +224,29 @@ public class WebsiteResource {
         }
 
         // Firstly zip up the directory
-        String outputFilePath = tempDir.toString() + File.separator + "zipped.zip";
+        String zipOutputFilePath = tempDir.toString() + File.separator + "zipped.zip";
         try {
-            ZipUtils.zip(directoryPath, outputFilePath, "data");
+            ZipUtils.zip(directoryPath, zipOutputFilePath, "data");
         } catch (IOException e) {
             LOGGER.info("Unable to zip directory", e);
             throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
         }
 
+        // Next, encrypt the file with AES
+        String encryptedFilePath = tempDir.toString() + File.separator + "zipped_encrypted.zip";
+        SecretKey aesKey;
         try {
-            DataFile dataFile = DataFile.fromPath(outputFilePath);
+            aesKey = AES.generateKey(256);
+            AES.encryptFile("AES", aesKey, zipOutputFilePath, encryptedFilePath);
+            Files.delete(Paths.get(zipOutputFilePath));
+        } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException
+                | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) {
+            throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR);
+        }
+
+        try {
+            DataFile dataFile = DataFile.fromPath(encryptedFilePath);
+            dataFile.setSecret(aesKey.getEncoded());
             DataFile.ValidationResult validationResult = dataFile.isValid();
             if (validationResult != DataFile.ValidationResult.OK) {
                 LOGGER.error("Invalid file: {}", validationResult);
@@ -241,11 +264,15 @@ public class WebsiteResource {
             return null;
         }
         finally {
-            // Clean up by deleting the zipped file
+            // Clean up
-            File zippedFile = new File(outputFilePath);
+            File zippedFile = new File(zipOutputFilePath);
             if (zippedFile.exists()) {
                 zippedFile.delete();
             }
+            File encryptedFile = new File(encryptedFilePath);
+            if (encryptedFile.exists()) {
+                encryptedFile.delete();
+            }
         }
     }
 
@@ -288,6 +315,7 @@ public class WebsiteResource {
 
         String tempDirectory = System.getProperty("java.io.tmpdir");
         String destPath = tempDirectory + File.separator  + "qortal-sites" + File.separator + resourceId;
+        String unencryptedPath = destPath + File.separator + "zipped.zip";
         String unzippedPath = destPath + File.separator + "data";
 
         if (!Files.exists(Paths.get(unzippedPath))) {
@@ -304,6 +332,9 @@ public class WebsiteResource {
                 byte[] digest = transactionData.getData();
                 byte[] chunkHashes = transactionData.getChunkHashes();
 
+                // Load secret
+                byte[] secret = transactionData.getSecret();
+
                 // Load data file(s)
                 DataFile dataFile = DataFile.fromHash(digest);
                 if (!dataFile.exists()) {
@@ -326,10 +357,26 @@ public class WebsiteResource {
                     return this.get404Response();
                 }
 
+                // Decrypt if we have the secret key.
+                if (secret != null && secret.length == Transformer.AES256_LENGTH) {
+                    try {
+                        SecretKey aesKey = new SecretKeySpec(secret, 0, secret.length, "AES");
+                        AES.decryptFile("AES", aesKey, dataFile.getFilePath(), unencryptedPath);
+                    } catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException
+                            | BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) {
+                        return this.get404Response();
+                    }
+                }
+                else {
+                    // Assume it is unencrypted. We may block this.
+                    unencryptedPath = dataFile.getFilePath();
+                }
+
+                // Unzip
                 try {
                     // TODO: compression types
                     //if (transactionData.getCompression() == ArbitraryTransactionData.Compression.ZIP) {
-                        ZipUtils.unzip(dataFile.getFilePath(), destPath);
+                        ZipUtils.unzip(unencryptedPath, destPath);
                     //}
                 } catch (IOException e) {
                     LOGGER.info("Unable to unzip file");

src/main/java/org/qortal/crypto/AES.java

@@ -2,6 +2,7 @@
  * MIT License
  *
  * Copyright (c) 2017 Eugen Paraschiv
+ * Modified in 2021 by CalDescent
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -93,15 +94,24 @@ public class AES {
         return new IvParameterSpec(iv);
     }
 
-    public static void encryptFile(String algorithm, SecretKey key, IvParameterSpec iv,
+    public static void encryptFile(String algorithm, SecretKey key,
-                                   File inputFile, File outputFile) throws IOException, NoSuchPaddingException,
+                                   String inputFilePath, String outputFilePath) throws IOException,
-            NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException,
+            NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException,
             BadPaddingException, IllegalBlockSizeException {
+
+        File inputFile = new File(inputFilePath);
+        File outputFile = new File(outputFilePath);
+
+        IvParameterSpec iv = AES.generateIv();
         Cipher cipher = Cipher.getInstance(algorithm);
         cipher.init(Cipher.ENCRYPT_MODE, key, iv);
         FileInputStream inputStream = new FileInputStream(inputFile);
         FileOutputStream outputStream = new FileOutputStream(outputFile);
-        byte[] buffer = new byte[64];
+
+        // Prepend the output stream with the 16 byte initialization vector
+        outputStream.write(iv.getIV());
+
+        byte[] buffer = new byte[1024];
         int bytesRead;
         while ((bytesRead = inputStream.read(buffer)) != -1) {
             byte[] output = cipher.update(buffer, 0, bytesRead);
@@ -117,14 +127,28 @@ public class AES {
         outputStream.close();
     }
 
-    public static void decryptFile(String algorithm, SecretKey key, IvParameterSpec iv,
+    public static void decryptFile(String algorithm, SecretKey key, String encryptedFilePath,
-                                   File encryptedFile, File decryptedFile) throws IOException, NoSuchPaddingException,
+                                   String decryptedFilePath) throws IOException, NoSuchPaddingException,
             NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException,
             BadPaddingException, IllegalBlockSizeException {
-        Cipher cipher = Cipher.getInstance(algorithm);
+
-        cipher.init(Cipher.DECRYPT_MODE, key, iv);
+        File encryptedFile = new File(encryptedFilePath);
+        File decryptedFile = new File(decryptedFilePath);
+
+        File parent = decryptedFile.getParentFile();
+        if (!parent.isDirectory() && !parent.mkdirs()) {
+            throw new IOException("Failed to create directory " + parent);
+        }
+
         FileInputStream inputStream = new FileInputStream(encryptedFile);
         FileOutputStream outputStream = new FileOutputStream(decryptedFile);
+
+        // Read the initialization vector from the first 16 bytes of the file
+        byte[] iv = new byte[16];
+        inputStream.read(iv);
+        Cipher cipher = Cipher.getInstance(algorithm);
+        cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
+
         byte[] buffer = new byte[64];
         int bytesRead;
         while ((bytesRead = inputStream.read(buffer)) != -1) {

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -276,7 +276,6 @@ public class HSQLDBDatabaseUpdates {
 							+ "service SMALLINT NOT NULL, is_data_raw BOOLEAN NOT NULL, data ArbitraryData NOT NULL, "
 							+ TRANSACTION_KEYS + ")");
 					// NB: Actual data payload stored elsewhere
-					// For the future: data payload should be encrypted, at the very least with transaction's reference as the seed for the encryption key
 					break;
 
 				case 8:

src/main/java/org/qortal/storage/DataFile.java

@@ -50,6 +50,7 @@ public class DataFile {
     protected String filePath;
     protected String hash58;
     private ArrayList<DataFileChunk> chunks;
+    private byte[] secret;
 
     public DataFile() {
     }
@@ -500,6 +501,14 @@ public class DataFile {
         return outputString;
     }
 
+    public void setSecret(byte[] secret) {
+        this.secret = secret;
+    }
+
+    public byte[] getSecret() {
+        return this.secret;
+    }
+
     @Override
     public String toString() {
         return this.shortHash58();

src/test/java/org/qortal/test/CryptoTests.java

@@ -3,6 +3,7 @@ package org.qortal.test;
 import org.junit.Test;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.block.BlockChain;
+import org.qortal.crypto.AES;
 import org.qortal.crypto.BouncyCastle25519;
 import org.qortal.crypto.Crypto;
 import org.qortal.test.common.Common;
@@ -10,7 +11,17 @@ import org.qortal.utils.Base58;
 
 import static org.junit.Assert.*;
 
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.InvalidKeyException;
+import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
+import java.util.Arrays;
+import java.util.Random;
 
 import org.bouncycastle.crypto.agreement.X25519Agreement;
 import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters;
@@ -20,6 +31,11 @@ import org.bouncycastle.crypto.params.X25519PublicKeyParameters;
 
 import com.google.common.hash.HashCode;
 
+import javax.crypto.BadPaddingException;
+import javax.crypto.IllegalBlockSizeException;
+import javax.crypto.NoSuchPaddingException;
+import javax.crypto.SecretKey;
+
 public class CryptoTests extends Common {
 
 	@Test
@@ -255,4 +271,68 @@ public class CryptoTests extends Common {
 		assertEquals(expectedProxyPrivateKey, Base58.encode(proxyPrivateKey));
 	}
 
+
+	@Test
+	public void testAESFileEncryption() throws NoSuchAlgorithmException, IOException, IllegalBlockSizeException,
+			InvalidKeyException, BadPaddingException, InvalidAlgorithmParameterException, NoSuchPaddingException {
+
+		// Create temporary directory and file paths
+		java.nio.file.Path tempDir = Files.createTempDirectory("qortal-tests");
+		String inputFilePath = tempDir.toString() + File.separator + "inputFile";
+		String outputFilePath = tempDir.toString() + File.separator + "outputFile";
+		String decryptedFilePath = tempDir.toString() + File.separator + "decryptedFile";
+		String reencryptedFilePath = tempDir.toString() + File.separator + "reencryptedFile";
+
+		// Generate some dummy data
+		byte[] randomBytes = new byte[1024];
+		new Random().nextBytes(randomBytes);
+
+		// Write it to the input file
+		FileOutputStream outputStream = new FileOutputStream(inputFilePath);
+		outputStream.write(randomBytes);
+
+		// Make sure only the input file exists
+		assertTrue(Files.exists(Paths.get(inputFilePath)));
+		assertFalse(Files.exists(Paths.get(outputFilePath)));
+
+		// Encrypt
+		SecretKey aesKey = AES.generateKey(256);
+		AES.encryptFile("AES", aesKey, inputFilePath, outputFilePath);
+		assertTrue(Files.exists(Paths.get(outputFilePath)));
+		byte[] encryptedBytes = Files.readAllBytes(Paths.get(outputFilePath));
+
+		// Delete the input file
+		Files.delete(Paths.get(inputFilePath));
+		assertFalse(Files.exists(Paths.get(inputFilePath)));
+
+		// Decrypt
+		String encryptedFilePath = outputFilePath;
+		assertFalse(Files.exists(Paths.get(decryptedFilePath)));
+		AES.decryptFile("AES", aesKey, encryptedFilePath, decryptedFilePath);
+		assertTrue(Files.exists(Paths.get(decryptedFilePath)));
+
+		// Delete the output file
+		Files.delete(Paths.get(outputFilePath));
+		assertFalse(Files.exists(Paths.get(outputFilePath)));
+
+		// Check that the decrypted file contents matches the original data
+		byte[] decryptedBytes = Files.readAllBytes(Paths.get(decryptedFilePath));
+		assertTrue(Arrays.equals(decryptedBytes, randomBytes));
+		assertEquals(1024, decryptedBytes.length);
+
+		// Write the original data back to the input file
+		outputStream = new FileOutputStream(inputFilePath);
+		outputStream.write(randomBytes);
+
+		// Now encrypt the data one more time using the same key
+		// This is to ensure the initialization vector produces a different result
+		AES.encryptFile("AES", aesKey, inputFilePath, reencryptedFilePath);
+		assertTrue(Files.exists(Paths.get(reencryptedFilePath)));
+
+		// Make sure the ciphertexts do not match
+		byte[] reencryptedBytes = Files.readAllBytes(Paths.get(reencryptedFilePath));
+		assertFalse(Arrays.equals(encryptedBytes, reencryptedBytes));
+
+	}
+
 }