Added support for uploading an entire directory via POST /data/upload/path

If a directory is specified instead of a file, the directory is automatically zipped before being split into chunks.

src/main/java/org/qortal/api/resource/DataResource.java

@@ -20,14 +20,19 @@ import org.qortal.settings.Settings;
 import org.qortal.storage.DataFile;
 import org.qortal.storage.DataFile.ValidationResult;
 import org.qortal.utils.Base58;
+import org.qortal.utils.ZipUtils;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import java.io.File;
+import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.net.UnknownHostException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.List;
 
 
@@ -66,7 +71,7 @@ public class DataResource {
 		}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
-	public String uploadFile(String filePath) {
+	public String uploadDataAtPath(String path) {
 		Security.checkApiCallAllowed(request);
 
 		// It's too dangerous to allow user-supplied filenames in weaker security contexts
@@ -75,6 +80,28 @@ public class DataResource {
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 
+			// Check if a file or directory has been supplied
+			File file = new File(path);
+			if (file.isFile()) {
+				return this.uploadFile(path);
+			}
+			else if (file.isDirectory()) {
+				return this.uploadDirectory(path);
+			}
+
+			LOGGER.info("No file or folder found at path: {}", path);
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
+
+		} catch (DataException e) {
+			LOGGER.error("Repository issue when uploading data", e);
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		} catch (IllegalStateException e) {
+			LOGGER.error("Invalid upload data", e);
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e);
+		}
+	}
+
+	private String uploadFile(String filePath) {
 		DataFile dataFile = new DataFile(filePath);
 		ValidationResult validationResult = dataFile.isValid();
 		if (validationResult != DataFile.ValidationResult.OK) {
@@ -90,13 +117,49 @@ public class DataResource {
 		}
 
 		return "false";
+	}
 
-		} catch (DataException e) {
+	private String uploadDirectory(String directoryPath) {
-			LOGGER.error("Repository issue when uploading data", e);
+		// Ensure temp folder exists
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		try {
-		} catch (IllegalStateException e) {
+			Files.createDirectories(Paths.get("temp"));
-			LOGGER.error("Invalid upload data", e);
+		} catch (IOException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e);
+			LOGGER.error("Unable to create temp directory");
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
+		}
+
+		// Firstly zip up the directory
+		String outputFilePath = "temp/zipped.zip";
+		try {
+			ZipUtils.zip(directoryPath, outputFilePath);
+		} catch (IOException e) {
+			LOGGER.info("Unable to zip directory", e);
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+		}
+
+		try {
+			DataFile dataFile = new DataFile(outputFilePath);
+			ValidationResult validationResult = dataFile.isValid();
+			if (validationResult != DataFile.ValidationResult.OK) {
+				LOGGER.error("Invalid file: {}", validationResult);
+				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+			}
+			LOGGER.info("Whole file digest: {}", dataFile.base58Digest());
+
+			int chunkCount = dataFile.split();
+			if (chunkCount > 0) {
+				LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s")));
+				return "true";
+			}
+
+			return "false";
+		}
+		finally {
+			// Clean up by deleting the zipped file
+			File zippedFile = new File(outputFilePath);
+			if (zippedFile.exists()) {
+				zippedFile.delete();
+			}
 		}
 	}
 

src/main/java/org/qortal/utils/ZipUtils.java

@@ -0,0 +1,105 @@
+/*
+ * MIT License
+ * Copyright (c) 2017 Eugen Paraschiv
+ *
+ * Based on code taken from: https://github.com/eugenp/tutorials/tree/master/core-java-modules/core-java-io/src/main/java/com/baeldung
+ *
+ */
+
+package org.qortal.utils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+public class ZipUtils {
+
+    public static void zip(String sourcePath, String destFilePath) throws IOException {
+        File sourceFile = new File(sourcePath);
+        FileOutputStream fileOutputStream = new FileOutputStream(destFilePath);
+        ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream);
+        ZipUtils.zip(sourceFile, sourceFile.getName(), zipOutputStream);
+        zipOutputStream.close();
+        fileOutputStream.close();
+    }
+
+    public static void zip(final File fileToZip, final String fileName, final ZipOutputStream zipOut) throws IOException {
+        if (fileToZip.isHidden()) {
+            return;
+        }
+        if (fileToZip.isDirectory()) {
+            if (fileName.endsWith("/")) {
+                zipOut.putNextEntry(new ZipEntry(fileName));
+                zipOut.closeEntry();
+            } else {
+                zipOut.putNextEntry(new ZipEntry(fileName + "/"));
+                zipOut.closeEntry();
+            }
+            final File[] children = fileToZip.listFiles();
+            for (final File childFile : children) {
+                ZipUtils.zip(childFile, fileName + "/" + childFile.getName(), zipOut);
+            }
+            return;
+        }
+        final FileInputStream fis = new FileInputStream(fileToZip);
+        final ZipEntry zipEntry = new ZipEntry(fileName);
+        zipOut.putNextEntry(zipEntry);
+        final byte[] bytes = new byte[1024];
+        int length;
+        while ((length = fis.read(bytes)) >= 0) {
+            zipOut.write(bytes, 0, length);
+        }
+        fis.close();
+    }
+
+    public static void unzip(String sourcePath, String destPath) throws IOException {
+        final File destDir = new File(destPath);
+        final byte[] buffer = new byte[1024];
+        final ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath));
+        ZipEntry zipEntry = zis.getNextEntry();
+        while (zipEntry != null) {
+            final File newFile = ZipUtils.newFile(destDir, zipEntry);
+            if (zipEntry.isDirectory()) {
+                if (!newFile.isDirectory() && !newFile.mkdirs()) {
+                    throw new IOException("Failed to create directory " + newFile);
+                }
+            } else {
+                File parent = newFile.getParentFile();
+                if (!parent.isDirectory() && !parent.mkdirs()) {
+                    throw new IOException("Failed to create directory " + parent);
+                }
+
+                final FileOutputStream fos = new FileOutputStream(newFile);
+                int len;
+                while ((len = zis.read(buffer)) > 0) {
+                    fos.write(buffer, 0, len);
+                }
+                fos.close();
+            }
+            zipEntry = zis.getNextEntry();
+        }
+        zis.closeEntry();
+        zis.close();
+    }
+
+    /**
+     * See: https://snyk.io/research/zip-slip-vulnerability
+     */
+    public static File newFile(File destinationDir, ZipEntry zipEntry) throws IOException {
+        File destFile = new File(destinationDir, zipEntry.getName());
+
+        String destDirPath = destinationDir.getCanonicalPath();
+        String destFilePath = destFile.getCanonicalPath();
+
+        if (!destFilePath.startsWith(destDirPath + File.separator)) {
+            throw new IOException("Entry is outside of the target dir: " + zipEntry.getName());
+        }
+
+        return destFile;
+    }
+
+}