Merge pull request #7 from Philreact/feature/allow-for-unlimited-size-publishes

Feature/allow for unlimited size publishes

pom.xml

@@ -796,5 +796,10 @@
 			<artifactId>jaxb-runtime</artifactId>
 			<version>${jaxb-runtime.version}</version>
 		</dependency>
+		<dependency>
+<groupId>org.apache.tika</groupId>
+<artifactId>tika-core</artifactId>
+<version>3.1.0</version> 
+</dependency>
 	</dependencies>
 </project>

src/main/java/org/qortal/api/ApiService.java

@@ -46,6 +46,7 @@ public class ApiService {
 	private ApiService() {
 		this.config = new ResourceConfig();
 		this.config.packages("org.qortal.api.resource", "org.qortal.api.restricted.resource");
+		this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
 		this.config.register(OpenApiResource.class);
 		this.config.register(ApiDefinition.class);
 		this.config.register(AnnotationPostProcessor.class);

src/main/java/org/qortal/api/DevProxyService.java

@@ -40,6 +40,7 @@ public class DevProxyService {
 	private DevProxyService() {
 		this.config = new ResourceConfig();
 		this.config.packages("org.qortal.api.proxy.resource", "org.qortal.api.resource");
+		this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
 		this.config.register(OpenApiResource.class);
 		this.config.register(ApiDefinition.class);
 		this.config.register(AnnotationPostProcessor.class);

src/main/java/org/qortal/api/DomainMapService.java

@@ -39,6 +39,7 @@ public class DomainMapService {
 	private DomainMapService() {
 		this.config = new ResourceConfig();
 		this.config.packages("org.qortal.api.resource", "org.qortal.api.domainmap.resource");
+		this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
 		this.config.register(OpenApiResource.class);
 		this.config.register(ApiDefinition.class);
 		this.config.register(AnnotationPostProcessor.class);

src/main/java/org/qortal/api/GatewayService.java

@@ -39,6 +39,7 @@ public class GatewayService {
 	private GatewayService() {
 		this.config = new ResourceConfig();
 		this.config.packages("org.qortal.api.resource", "org.qortal.api.gateway.resource");
+		this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class);
 		this.config.register(OpenApiResource.class);
 		this.config.register(ApiDefinition.class);
 		this.config.register(AnnotationPostProcessor.class);

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -3,6 +3,7 @@ package org.qortal.api.resource;
 import com.google.common.primitives.Bytes;
 import com.j256.simplemagic.ContentInfo;
 import com.j256.simplemagic.ContentInfoUtil;
+
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.Parameter;
 import io.swagger.v3.oas.annotations.media.ArraySchema;
@@ -12,6 +13,7 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.logging.log4j.LogManager;
@@ -63,14 +65,19 @@ import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
+
 import java.io.BufferedWriter;
 import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.net.FileNameMap;
 import java.net.URLConnection;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
@@ -78,6 +85,16 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.stream.Collectors;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.tika.Tika;
+import org.apache.tika.mime.MimeTypeException;
+import org.apache.tika.mime.MimeTypes;
+
+import javax.ws.rs.core.Response;
+
+import org.glassfish.jersey.media.multipart.FormDataParam;
+import static org.qortal.api.ApiError.REPOSITORY_ISSUE;
 
 @Path("/arbitrary")
 @Tag(name = "Arbitrary")
@@ -686,20 +703,20 @@ public class ArbitraryResource {
 					)
 			}
 	)
-	public HttpServletResponse get(@PathParam("service") Service service,
+	public void get(@PathParam("service") Service service,
 								   @PathParam("name") String name,
 								   @QueryParam("filepath") String filepath,
 								   @QueryParam("encoding") String encoding,
 								   @QueryParam("rebuild") boolean rebuild,
 								   @QueryParam("async") boolean async,
-								   @QueryParam("attempts") Integer attempts) {
+								   @QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) {
 
 		// Authentication can be bypassed in the settings, for those running public QDN nodes
 		if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
 			Security.checkApiCallAllowed(request);
 		}
 
-		return this.download(service, name, null, filepath, encoding, rebuild, async, attempts);
+		 this.download(service, name, null, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename);
 	}
 
 	@GET
@@ -719,21 +736,21 @@ public class ArbitraryResource {
 					)
 			}
 	)
-	public HttpServletResponse get(@PathParam("service") Service service,
+	public void get(@PathParam("service") Service service,
 								   @PathParam("name") String name,
 								   @PathParam("identifier") String identifier,
 								   @QueryParam("filepath") String filepath,
 								   @QueryParam("encoding") String encoding,
 								   @QueryParam("rebuild") boolean rebuild,
 								   @QueryParam("async") boolean async,
-								   @QueryParam("attempts") Integer attempts) {
+								   @QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) {
 
 		// Authentication can be bypassed in the settings, for those running public QDN nodes
 		if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
 			Security.checkApiCallAllowed(request, null);
 		}
 
-		return this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts);
+		this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename);
 	}
 
 
@@ -878,6 +895,464 @@ public class ArbitraryResource {
 	}
 
 
+	@GET
+	@Path("/check/tmp")
+	@Produces(MediaType.TEXT_PLAIN)
+	@Operation(
+		summary = "Check if the disk has enough disk space for an upcoming upload",
+		responses = {
+			@ApiResponse(description = "OK if sufficient space", responseCode = "200"),
+			@ApiResponse(description = "Insufficient space", responseCode = "507") // 507 = Insufficient Storage
+		}
+	)
+	@SecurityRequirement(name = "apiKey")
+	public Response checkUploadSpace(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
+									 @QueryParam("totalSize") Long totalSize) {
+		Security.checkApiCallAllowed(request);
+	
+		if (totalSize == null || totalSize <= 0) {
+			return Response.status(Response.Status.BAD_REQUEST)
+					.entity("Missing or invalid totalSize parameter").build();
+		}
+	
+		File uploadDir = new File("uploads-temp");
+		if (!uploadDir.exists()) {
+			uploadDir.mkdirs(); // ensure the folder exists
+		}
+
+		long usableSpace = uploadDir.getUsableSpace();
+		long requiredSpace = (long)(((double)totalSize) * 2.2); // estimate for chunks + merge
+
+		if (usableSpace < requiredSpace) {
+			return Response.status(507).entity("Insufficient disk space").build();
+		}
+
+		return Response.ok("Sufficient disk space").build();
+	}
+
+	@POST
+@Path("/{service}/{name}/chunk")
+@Consumes(MediaType.MULTIPART_FORM_DATA)
+@Operation(
+    summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource (no identifier)",
+    requestBody = @RequestBody(
+        required = true,
+        content = @Content(
+            mediaType = MediaType.MULTIPART_FORM_DATA,
+            schema = @Schema(
+                implementation = Object.class
+            )
+        )
+    ),
+    responses = {
+        @ApiResponse(
+            description = "Chunk uploaded successfully",
+            responseCode = "200"
+        ),
+        @ApiResponse(
+            description = "Error writing chunk",
+            responseCode = "500"
+        )
+    }
+)
+@SecurityRequirement(name = "apiKey")
+public Response uploadChunkNoIdentifier(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
+                                        @PathParam("service") String serviceString,
+                                        @PathParam("name") String name,
+                                        @FormDataParam("chunk") InputStream chunkStream,
+                                        @FormDataParam("index") int index) {
+    Security.checkApiCallAllowed(request);
+
+    try {
+		String safeService = Paths.get(serviceString).getFileName().toString();
+        String safeName = Paths.get(name).getFileName().toString();
+
+		java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName);
+		Files.createDirectories(tempDir);
+		
+        
+
+        java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index);
+        Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING);
+
+        return Response.ok("Chunk " + index + " received").build();
+    } catch (IOException e) {
+		LOGGER.error("Failed to write chunk {} for service '{}' and name '{}'", index, serviceString, name, e);
+        return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build();
+    }
+}
+
+@POST
+@Path("/{service}/{name}/finalize")
+@Produces(MediaType.TEXT_PLAIN)
+@Operation(
+    summary = "Finalize a chunked upload (no identifier) and build a raw, unsigned, ARBITRARY transaction",
+    responses = {
+        @ApiResponse(
+            description = "raw, unsigned, ARBITRARY transaction encoded in Base58",
+            content = @Content(mediaType = MediaType.TEXT_PLAIN)
+        )
+    }
+)
+@SecurityRequirement(name = "apiKey")
+public String finalizeUploadNoIdentifier(
+    @HeaderParam(Security.API_KEY_HEADER) String apiKey,
+    @PathParam("service") String serviceString,
+    @PathParam("name") String name,
+    @QueryParam("title") String title,
+    @QueryParam("description") String description,
+    @QueryParam("tags") List<String> tags,
+    @QueryParam("category") Category category,
+    @QueryParam("filename") String filename,
+    @QueryParam("fee") Long fee,
+    @QueryParam("preview") Boolean preview,
+    @QueryParam("isZip") Boolean isZip
+) {
+    Security.checkApiCallAllowed(request);
+    java.nio.file.Path tempFile = null;
+    java.nio.file.Path tempDir = null;
+	java.nio.file.Path chunkDir = null;
+    String safeService = Paths.get(serviceString).getFileName().toString();
+	String safeName = Paths.get(name).getFileName().toString();
+
+
+
+    try {
+		chunkDir = Paths.get("uploads-temp", safeService, safeName);
+
+        if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) {
+            throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload");
+        }
+
+        String safeFilename = (filename == null || filename.isBlank()) ? "qortal-" + NTP.getTime() : filename;
+        tempDir = Files.createTempDirectory("qortal-");
+        String sanitizedFilename = Paths.get(safeFilename).getFileName().toString();
+		tempFile = tempDir.resolve(sanitizedFilename);
+
+        try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
+            byte[] buffer = new byte[65536];
+            for (java.nio.file.Path chunk : Files.list(chunkDir)
+                    .filter(path -> path.getFileName().toString().startsWith("chunk_"))
+                    .sorted(Comparator.comparingInt(path -> {
+                        String name2 = path.getFileName().toString();
+                        String numberPart = name2.substring("chunk_".length());
+                        return Integer.parseInt(numberPart);
+                    })).collect(Collectors.toList())) {
+                try (InputStream in = Files.newInputStream(chunk)) {
+                    int bytesRead;
+                    while ((bytesRead = in.read(buffer)) != -1) {
+                        out.write(buffer, 0, bytesRead);
+                    }
+                }
+            }
+        }
+
+        String detectedExtension = "";
+        String uploadFilename = null;
+        boolean extensionIsValid = false;
+
+        if (filename != null && !filename.isBlank()) {
+            int lastDot = filename.lastIndexOf('.');
+            if (lastDot > 0 && lastDot < filename.length() - 1) {
+                extensionIsValid = true;
+                uploadFilename = filename;
+            }
+        }
+
+        if (!extensionIsValid) {
+            Tika tika = new Tika();
+            String mimeType = tika.detect(tempFile.toFile());
+            try {
+                MimeTypes allTypes = MimeTypes.getDefaultMimeTypes();
+                org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType);
+                detectedExtension = mime.getExtension();
+            } catch (MimeTypeException e) {
+                LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e);
+            }
+
+            if (filename != null && !filename.isBlank()) {
+                int lastDot = filename.lastIndexOf('.');
+                String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename;
+                uploadFilename = baseName + (detectedExtension != null ? detectedExtension : "");
+            } else {
+                uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : "");
+            }
+        }
+
+		Boolean isZipBoolean = false;
+
+		if (isZip != null && isZip) {
+			isZipBoolean = true;
+		}
+        
+
+        // ✅ Call upload with `null` as identifier
+        return this.upload(
+            Service.valueOf(serviceString),
+            name,
+            null, // no identifier
+            tempFile.toString(),
+            null,
+            null,
+            isZipBoolean,
+            fee,
+            uploadFilename,
+            title,
+            description,
+            tags,
+            category,
+            preview
+        );
+
+    } catch (IOException e) {
+		LOGGER.error("Failed to merge chunks for service='{}', name='{}'", serviceString, name, e);
+
+        throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage());
+    } finally {
+        if (tempDir != null) {
+            try {
+                Files.walk(tempDir)
+                    .sorted(Comparator.reverseOrder())
+                    .map(java.nio.file.Path::toFile)
+                    .forEach(File::delete);
+            } catch (IOException e) {
+                LOGGER.warn("Failed to delete temp directory: {}", tempDir, e);
+            }
+        }
+
+        try {
+            Files.walk(chunkDir)
+                .sorted(Comparator.reverseOrder())
+                .map(java.nio.file.Path::toFile)
+                .forEach(File::delete);
+        } catch (IOException e) {
+            LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e);
+        }
+    }
+}
+
+
+
+	@POST
+@Path("/{service}/{name}/{identifier}/chunk")
+@Consumes(MediaType.MULTIPART_FORM_DATA)
+@Operation(
+    summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource",
+    requestBody = @RequestBody(
+        required = true,
+        content = @Content(
+            mediaType = MediaType.MULTIPART_FORM_DATA,
+            schema = @Schema(
+                implementation = Object.class
+            )
+        )
+    ),
+    responses = {
+        @ApiResponse(
+            description = "Chunk uploaded successfully",
+            responseCode = "200"
+        ),
+        @ApiResponse(
+            description = "Error writing chunk",
+            responseCode = "500"
+        )
+    }
+)
+@SecurityRequirement(name = "apiKey")
+public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
+                            @PathParam("service") String serviceString,
+                            @PathParam("name") String name,
+                            @PathParam("identifier") String identifier,
+                            @FormDataParam("chunk") InputStream chunkStream,
+                            @FormDataParam("index") int index) {
+    Security.checkApiCallAllowed(request);
+
+    try {
+        String safeService = Paths.get(serviceString).getFileName().toString();
+        String safeName = Paths.get(name).getFileName().toString();
+        String safeIdentifier = Paths.get(identifier).getFileName().toString();
+
+        java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName, safeIdentifier);
+
+        Files.createDirectories(tempDir);
+
+        java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index);
+        Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING);
+
+        return Response.ok("Chunk " + index + " received").build();
+    } catch (IOException e) {
+		LOGGER.error("Failed to write chunk {} for service='{}', name='{}', identifier='{}'", index, serviceString, name, identifier, e);
+        return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build();
+    }
+}
+
+@POST
+@Path("/{service}/{name}/{identifier}/finalize")
+@Produces(MediaType.TEXT_PLAIN)
+@Operation(
+    summary = "Finalize a chunked upload and build a raw, unsigned, ARBITRARY transaction",
+    responses = {
+        @ApiResponse(
+            description = "raw, unsigned, ARBITRARY transaction encoded in Base58",
+            content = @Content(mediaType = MediaType.TEXT_PLAIN)
+        )
+    }
+)
+@SecurityRequirement(name = "apiKey")
+public String finalizeUpload(
+    @HeaderParam(Security.API_KEY_HEADER) String apiKey,
+    @PathParam("service") String serviceString,
+    @PathParam("name") String name,
+    @PathParam("identifier") String identifier,
+    @QueryParam("title") String title,
+    @QueryParam("description") String description,
+    @QueryParam("tags") List<String> tags,
+    @QueryParam("category") Category category,
+    @QueryParam("filename") String filename,
+    @QueryParam("fee") Long fee,
+    @QueryParam("preview") Boolean preview,
+	@QueryParam("isZip") Boolean isZip
+) {
+    Security.checkApiCallAllowed(request);
+    java.nio.file.Path tempFile = null;
+    java.nio.file.Path tempDir = null;
+	java.nio.file.Path chunkDir = null;
+
+	
+	
+	
+
+    try {
+		String safeService = Paths.get(serviceString).getFileName().toString();
+	String safeName = Paths.get(name).getFileName().toString();
+	String safeIdentifier = Paths.get(identifier).getFileName().toString();
+	java.nio.file.Path baseUploadsDir = Paths.get("uploads-temp"); // relative to Qortal working dir
+	chunkDir = baseUploadsDir.resolve(safeService).resolve(safeName).resolve(safeIdentifier);
+		
+        if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) {
+            throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload");
+        }
+
+        // Step 1: Determine a safe filename for disk temp file (regardless of extension correctness)
+        String safeFilename = filename;
+        if (filename == null || filename.isBlank()) {
+			safeFilename = "qortal-" + NTP.getTime();
+		} 
+
+        tempDir = Files.createTempDirectory("qortal-");
+        String sanitizedFilename = Paths.get(safeFilename).getFileName().toString();
+		tempFile = tempDir.resolve(sanitizedFilename);
+
+
+        // Step 2: Merge chunks
+  
+        try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) {
+            byte[] buffer = new byte[65536];
+            for (java.nio.file.Path chunk : Files.list(chunkDir)
+                    .filter(path -> path.getFileName().toString().startsWith("chunk_"))
+                    .sorted(Comparator.comparingInt(path -> {
+                        String name2 = path.getFileName().toString();
+                        String numberPart = name2.substring("chunk_".length());
+                        return Integer.parseInt(numberPart);
+                    })).collect(Collectors.toList())) {
+                try (InputStream in = Files.newInputStream(chunk)) {
+                    int bytesRead;
+                    while ((bytesRead = in.read(buffer)) != -1) {
+                        out.write(buffer, 0, bytesRead);
+                    }
+                }
+            }
+        }
+       
+
+        // Step 3: Determine correct extension
+        String detectedExtension = "";
+        String uploadFilename  = null;
+        boolean extensionIsValid = false;
+
+        if (filename != null && !filename.isBlank()) {
+            int lastDot = filename.lastIndexOf('.');
+            if (lastDot > 0 && lastDot < filename.length() - 1) {
+                extensionIsValid = true;
+                uploadFilename = filename;
+            }
+        }
+
+        if (!extensionIsValid) {
+            Tika tika = new Tika();
+            String mimeType = tika.detect(tempFile.toFile());
+            try {
+                MimeTypes allTypes = MimeTypes.getDefaultMimeTypes();
+                org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType);
+                detectedExtension = mime.getExtension();
+            } catch (MimeTypeException e) {
+                LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e);
+            }
+
+            if (filename != null && !filename.isBlank()) {
+                int lastDot = filename.lastIndexOf('.');
+                String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename;
+                uploadFilename = baseName + (detectedExtension != null ? detectedExtension : "");
+            } else {
+                uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : "");
+            }
+        }
+
+
+		Boolean isZipBoolean = false;
+
+		if (isZip != null && isZip) {
+			isZipBoolean = true;
+		}
+        
+
+        return this.upload(
+            Service.valueOf(serviceString),
+            name,
+            identifier,
+            tempFile.toString(),
+            null,
+            null,
+            isZipBoolean,
+            fee,
+            uploadFilename,
+            title,
+            description,
+            tags,
+            category,
+            preview
+        );
+
+    } catch (IOException e) {
+		LOGGER.error("Unexpected error in finalizeUpload for service='{}', name='{}', name='{}'", serviceString, name, identifier, e);
+
+        throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage());
+    } finally {
+        if (tempDir != null) {
+            try {
+                Files.walk(tempDir)
+                    .sorted(Comparator.reverseOrder())
+                    .map(java.nio.file.Path::toFile)
+                    .forEach(File::delete);
+            } catch (IOException e) {
+                LOGGER.warn("Failed to delete temp directory: {}", tempDir, e);
+            }
+        }
+
+        try {
+            Files.walk(chunkDir)
+                .sorted(Comparator.reverseOrder())
+                .map(java.nio.file.Path::toFile)
+                .forEach(File::delete);
+        } catch (IOException e) {
+            LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e);
+        }
+    }
+}
+
+
+
+
 
 	// Upload base64-encoded data
 
@@ -1343,7 +1818,7 @@ public class ArbitraryResource {
 			if (path == null) {
 				// See if we have a string instead
 				if (string != null) {
-					if (filename == null) {
+					if (filename == null || filename.isBlank()) {
 						// Use current time as filename
 						filename = String.format("qortal-%d", NTP.getTime());
 					}
@@ -1358,7 +1833,7 @@ public class ArbitraryResource {
 				}
 				// ... or base64 encoded raw data
 				else if (base64 != null) {
-					if (filename == null) {
+					if (filename == null || filename.isBlank()) {
 						// Use current time as filename
 						filename = String.format("qortal-%d", NTP.getTime());
 					}
@@ -1409,6 +1884,7 @@ public class ArbitraryResource {
 				);
 
 				transactionBuilder.build();
+
 				// Don't compute nonce - this is done by the client (or via POST /arbitrary/compute)
 				ArbitraryTransactionData transactionData = transactionBuilder.getArbitraryTransactionData();
 				return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData));
@@ -1424,22 +1900,21 @@ public class ArbitraryResource {
 		}
 	}
 
-	private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) {
+	private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts, boolean attachment, String attachmentFilename) {
-
 		try {
+			
 			ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
-
+	
 			int attempts = 0;
 			if (maxAttempts == null) {
 				maxAttempts = 5;
 			}
-
+	
 			// Loop until we have data
 			if (async) {
 				// Asynchronous
 				arbitraryDataReader.loadAsynchronously(false, 1);
-			}
+			} else {
-			else {
 				// Synchronous
 				while (!Controller.isStopping()) {
 					attempts++;
@@ -1449,7 +1924,6 @@ public class ArbitraryResource {
 							break;
 						} catch (MissingDataException e) {
 							if (attempts > maxAttempts) {
-								// Give up after 5 attempts
 								throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later.");
 							}
 						}
@@ -1457,80 +1931,167 @@ public class ArbitraryResource {
 					Thread.sleep(3000L);
 				}
 			}
-
+	
 			java.nio.file.Path outputPath = arbitraryDataReader.getFilePath();
 			if (outputPath == null) {
 				// Assume the resource doesn't exist
 				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found");
 			}
-
+	
 			if (filepath == null || filepath.isEmpty()) {
 				// No file path supplied - so check if this is a single file resource
 				String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal");
 				if (files != null && files.length == 1) {
 					// This is a single file resource
 					filepath = files[0];
-				}
+				} else {
-				else {
+					throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "filepath is required for resources containing more than one file");
-					throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA,
-							"filepath is required for resources containing more than one file");
 				}
 			}
-
+	
 			java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
 			if (!Files.exists(path)) {
-				String message = String.format("No file exists at filepath: %s", filepath);
+				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No file exists at filepath: " + filepath);
-				throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message);
 			}
+	
+			if (attachment) {
+				String rawFilename;
 
-			byte[] data;
+				if (attachmentFilename != null && !attachmentFilename.isEmpty()) {
-			int fileSize = (int)path.toFile().length();
+					// 1. Sanitize first
-			int length = fileSize;
+					String safeAttachmentFilename = attachmentFilename.replaceAll("[\\\\/:*?\"<>|]", "_");
-
+				
-			// Parse "Range" header
+					// 2. Check for a valid extension (3–5 alphanumeric chars)
-			Integer rangeStart = null;
+					if (!safeAttachmentFilename.matches(".*\\.[a-zA-Z0-9]{2,5}$")) {
-			Integer rangeEnd = null;
+						safeAttachmentFilename += ".bin";
-			String range = request.getHeader("Range");
+					}
-			if (range != null) {
+				
-				range = range.replace("bytes=", "");
+					rawFilename = safeAttachmentFilename;
-				String[] parts = range.split("-");
+				} else {
-				rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null;
+					// Fallback if no filename is provided
-				rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize;
+					String baseFilename = (identifier != null && !identifier.isEmpty())
+						? name + "-" + identifier
+						: name;
+					rawFilename = baseFilename.replaceAll("[\\\\/:*?\"<>|]", "_") + ".bin";
+				}
+				
+				// Optional: trim length
+				rawFilename = rawFilename.length() > 100 ? rawFilename.substring(0, 100) : rawFilename;
+				
+				// 3. Set Content-Disposition header
+				response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\"");
 			}
+			// Determine the total size of the requested file
+				long fileSize = Files.size(path);
+				String mimeType = context.getMimeType(path.toString());
 
-			if (rangeStart != null && rangeEnd != null) {
+				// Attempt to read the "Range" header from the request to support partial content delivery (e.g., for video streaming or resumable downloads)
-				// We have a range, so update the requested length
+				String range = request.getHeader("Range");
-				length = rangeEnd - rangeStart;
+
+				long rangeStart = 0;
+				long rangeEnd = fileSize - 1;
+				boolean isPartial = false;
+
+				// If a Range header is present and no base64 encoding is requested, parse the range values
+				if (range != null && encoding == null) {
+					range = range.replace("bytes=", ""); // Remove the "bytes=" prefix
+					String[] parts = range.split("-"); // Split the range into start and end
+
+					// Parse range start
+					if (parts.length > 0 && !parts[0].isEmpty()) {
+						rangeStart = Long.parseLong(parts[0]);
+					}
+
+					// Parse range end, if present
+					if (parts.length > 1 && !parts[1].isEmpty()) {
+						rangeEnd = Long.parseLong(parts[1]);
+					}
+
+					isPartial = true; // Indicate that this is a partial content request
+				}
+
+				// Calculate how many bytes should be sent in the response
+				long contentLength = rangeEnd - rangeStart + 1;
+
+				// Inform the client that byte ranges are supported
+				response.setHeader("Accept-Ranges", "bytes");
+
+				if (isPartial) {
+					// If partial content was requested, return 206 Partial Content with appropriate headers
+					response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
+					response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize));
+				} else {
+					// Otherwise, return the entire file with status 200 OK
+					response.setStatus(HttpServletResponse.SC_OK);
+				}
+
+				// Initialize output streams for writing the file to the response
+				OutputStream rawOut = response.getOutputStream();
+				OutputStream base64Out = null;
+				OutputStream gzipOut = null;
+
+				if (encoding != null && "base64".equalsIgnoreCase(encoding)) {
+					// If base64 encoding is requested, override content type
+					response.setContentType("text/plain");
+
+					// Check if the client accepts gzip encoding
+					String acceptEncoding = request.getHeader("Accept-Encoding");
+					boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip");
+
+					if (wantsGzip) {
+						// Wrap output in GZIP and Base64 streams if gzip is accepted
+						response.setHeader("Content-Encoding", "gzip");
+						gzipOut = new GZIPOutputStream(rawOut);
+						base64Out = java.util.Base64.getEncoder().wrap(gzipOut);
+					} else {
+						// Wrap output in Base64 only
+						base64Out = java.util.Base64.getEncoder().wrap(rawOut);
+					}
+
+					rawOut = base64Out; // Use the wrapped stream for writing
+				} else {
+					// For raw binary output, set the content type and length
+					response.setContentType(mimeType != null ? mimeType : "application/octet-stream");
+					response.setContentLength((int) contentLength);
+				}
+			// Stream file content
+			try (InputStream inputStream = Files.newInputStream(path)) {
+				if (rangeStart > 0) {
+					inputStream.skip(rangeStart);
+				}
+	
+				byte[] buffer = new byte[65536];
+				long bytesRemaining = contentLength;
+				int bytesRead;
+	
+				while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) {
+					rawOut.write(buffer, 0, bytesRead);
+					bytesRemaining -= bytesRead;
+				}
 			}
+// Stream finished
+if (base64Out != null) {
+    base64Out.close(); // Also flushes and closes the wrapped gzipOut
+} else if (gzipOut != null) {
+    gzipOut.close(); // Only close gzipOut if it wasn't wrapped by base64Out
+} else {
+    rawOut.flush(); // Flush only the base output stream if nothing was wrapped
+}
+if (!response.isCommitted()) {
+    response.setStatus(HttpServletResponse.SC_OK);
+    response.getWriter().write(" ");
+}
 
-			if (length < fileSize && encoding == null) {
+		} catch (IOException | InterruptedException  | ApiException | DataException e) {
-				// Partial content requested, and not encoding the data
+			LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e);
-				response.setStatus(206);
-				response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize));
-				data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length);
-			}
-			else {
-				// Full content requested (or encoded data)
-				response.setStatus(200);
-				data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory
-			}
-
-			// Encode the data if requested
-			if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) {
-				data = Base64.encode(data);
-			}
-
-			response.addHeader("Accept-Ranges", "bytes");
-			response.setContentType(context.getMimeType(path.toString()));
-			response.setContentLength(data.length);
-			response.getOutputStream().write(data);
-
-			return response;
-		} catch (Exception e) {
-			LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage()));
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage());
 		}
+		catch ( NumberFormatException e) {
+			LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e);
+			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage());
+		}
 	}
+	
 
 	private FileProperties getFileProperties(Service service, String name, String identifier) {
 		try {

src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java

@@ -52,7 +52,7 @@ public class ArbitraryDataFile {
 
     private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class);
 
-    public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB
+    public static final long MAX_FILE_SIZE = 2L * 1024 * 1024 * 1024; // 2 GiB
     protected static final int MAX_CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB
     public static final int CHUNK_SIZE = 512 * 1024; // 0.5MiB
     public static int SHORT_DIGEST_LENGTH = 8;

src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java

@@ -29,6 +29,7 @@ import org.qortal.utils.FilesystemUtils;
 import org.qortal.utils.NTP;
 
 import java.io.IOException;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
@@ -197,7 +198,7 @@ public class ArbitraryDataTransactionBuilder {
 
             // We can't use PATCH for on-chain data because this requires the .qortal directory, which can't be put on chain
             final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(this.path, false);
-            final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE);
+            final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE);
             if (shouldUseOnChainData) {
                 LOGGER.info("Data size is small enough to go on chain - using PUT");
                 return Method.PUT;
@@ -245,7 +246,7 @@ public class ArbitraryDataTransactionBuilder {
 
             // Single file resources are handled differently, especially for very small data payloads, as these go on chain
             final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(path, false);
-            final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE);
+            final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE);
 
             // Use zip compression if data isn't going on chain
             Compression compression = shouldUseOnChainData ? Compression.NONE : Compression.ZIP;

src/main/java/org/qortal/crypto/AES.java

@@ -100,7 +100,7 @@ public class AES {
         // Prepend the output stream with the 16 byte initialization vector
         outputStream.write(iv.getIV());
 
-        byte[] buffer = new byte[1024];
+        byte[] buffer = new byte[65536];
         int bytesRead;
         while ((bytesRead = inputStream.read(buffer)) != -1) {
             byte[] output = cipher.update(buffer, 0, bytesRead);
@@ -138,7 +138,7 @@ public class AES {
         Cipher cipher = Cipher.getInstance(algorithm);
         cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
 
-        byte[] buffer = new byte[64];
+        byte[] buffer = new byte[65536];
         int bytesRead;
         while ((bytesRead = inputStream.read(buffer)) != -1) {
             byte[] output = cipher.update(buffer, 0, bytesRead);

src/main/java/org/qortal/utils/FilesystemUtils.java

@@ -6,6 +6,7 @@ import org.qortal.settings.Settings;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.RandomAccessFile;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.*;
@@ -232,31 +233,37 @@ public class FilesystemUtils {
     }
 
     public static byte[] getSingleFileContents(Path path, Integer maxLength) throws IOException {
-        byte[] data = null;
+        Path filePath = null;
-        // TODO: limit the file size that can be loaded into memory
+    
-
+        if (Files.isRegularFile(path)) {
-        // If the path is a file, read the contents directly
+            filePath = path;
-        if (path.toFile().isFile()) {
+        } else if (Files.isDirectory(path)) {
-            int fileSize = (int)path.toFile().length();
-            maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize;
-            data = FilesystemUtils.readFromFile(path.toString(), 0, maxLength);
-        }
-
-        // Or if it's a directory, only load file contents if there is a single file inside it
-        else if (path.toFile().isDirectory()) {
             String[] files = ArrayUtils.removeElement(path.toFile().list(), ".qortal");
             if (files.length == 1) {
-                Path filePath = Paths.get(path.toString(), files[0]);
+                filePath = path.resolve(files[0]);
-                if (filePath.toFile().isFile()) {
-                    int fileSize = (int)filePath.toFile().length();
-                    maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize;
-                    data = FilesystemUtils.readFromFile(filePath.toString(), 0, maxLength);
-                }
             }
         }
-
+    
-        return data;
+        if (filePath == null || !Files.exists(filePath)) {
+            return null;
+        }
+    
+        long fileSize = Files.size(filePath);
+        int length = (maxLength != null) ? Math.min(maxLength, (int) Math.min(fileSize, Integer.MAX_VALUE)) : (int) Math.min(fileSize, Integer.MAX_VALUE);
+    
+        try (InputStream in = Files.newInputStream(filePath)) {
+            byte[] buffer = new byte[length];
+            int bytesRead = in.read(buffer);
+            if (bytesRead < length) {
+                // Resize buffer to actual read size
+                byte[] trimmed = new byte[bytesRead];
+                System.arraycopy(buffer, 0, trimmed, 0, bytesRead);
+                return trimmed;
+            }
+            return buffer;
+        }
     }
+    
 
     /**
      * isSingleFileResource

src/main/java/org/qortal/utils/ZipUtils.java

@@ -27,6 +27,8 @@
 
 package org.qortal.utils;
 
+import java.io.BufferedOutputStream;
+
 import org.qortal.controller.Controller;
 
 import java.io.File;
@@ -44,11 +46,15 @@ public class ZipUtils {
         File sourceFile = new File(sourcePath);
         boolean isSingleFile = Paths.get(sourcePath).toFile().isFile();
         FileOutputStream fileOutputStream = new FileOutputStream(destFilePath);
-        ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream);
+        
+        // 🔧 Use best speed compression level
+        ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream);        
         ZipUtils.zip(sourceFile, enclosingFolderName, zipOutputStream, isSingleFile);
+        
         zipOutputStream.close();
         fileOutputStream.close();
     }
+    
 
     public static void zip(final File fileToZip, final String enclosingFolderName, final ZipOutputStream zipOut, boolean isSingleFile) throws IOException, InterruptedException {
         if (Controller.isStopping()) {
@@ -82,7 +88,7 @@ public class ZipUtils {
         final FileInputStream fis = new FileInputStream(fileToZip);
         final ZipEntry zipEntry = new ZipEntry(enclosingFolderName);
         zipOut.putNextEntry(zipEntry);
-        final byte[] bytes = new byte[1024];
+        final byte[] bytes = new byte[65536];
         int length;
         while ((length = fis.read(bytes)) >= 0) {
             zipOut.write(bytes, 0, length);
@@ -92,33 +98,34 @@ public class ZipUtils {
 
     public static void unzip(String sourcePath, String destPath) throws IOException {
         final File destDir = new File(destPath);
-        final byte[] buffer = new byte[1024];
+        final byte[] buffer = new byte[65536];
-        final ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath));
+        try (ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath))) {
-        ZipEntry zipEntry = zis.getNextEntry();
+            ZipEntry zipEntry = zis.getNextEntry();
-        while (zipEntry != null) {
+            while (zipEntry != null) {
-            final File newFile = ZipUtils.newFile(destDir, zipEntry);
+                final File newFile = ZipUtils.newFile(destDir, zipEntry);
-            if (zipEntry.isDirectory()) {
+                if (zipEntry.isDirectory()) {
-                if (!newFile.isDirectory() && !newFile.mkdirs()) {
+                    if (!newFile.isDirectory() && !newFile.mkdirs()) {
-                    throw new IOException("Failed to create directory " + newFile);
+                        throw new IOException("Failed to create directory " + newFile);
+                    }
+                } else {
+                    File parent = newFile.getParentFile();
+                    if (!parent.isDirectory() && !parent.mkdirs()) {
+                        throw new IOException("Failed to create directory " + parent);
+                    }
+    
+                    try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(newFile), buffer.length)) {
+                        int len;
+                        while ((len = zis.read(buffer)) > 0) {
+                            bos.write(buffer, 0, len);
+                        }
+                    }
                 }
-            } else {
+                zipEntry = zis.getNextEntry();
-                File parent = newFile.getParentFile();
-                if (!parent.isDirectory() && !parent.mkdirs()) {
-                    throw new IOException("Failed to create directory " + parent);
-                }
-
-                final FileOutputStream fos = new FileOutputStream(newFile);
-                int len;
-                while ((len = zis.read(buffer)) > 0) {
-                    fos.write(buffer, 0, len);
-                }
-                fos.close();
             }
-            zipEntry = zis.getNextEntry();
+            zis.closeEntry();
         }
-        zis.closeEntry();
-        zis.close();
     }
+    
 
     /**
      * See: https://snyk.io/research/zip-slip-vulnerability