From 8c811ef1efda57563b4875747305aec9b778691d Mon Sep 17 00:00:00 2001 From: PhilReact Date: Wed, 14 May 2025 20:00:04 +0300 Subject: [PATCH 01/12] initial --- pom.xml | 5 + src/main/java/org/qortal/api/ApiService.java | 1 + .../api/resource/ArbitraryResource.java | 473 +++++++++++++++--- .../qortal/arbitrary/ArbitraryDataFile.java | 2 +- .../ArbitraryDataTransactionBuilder.java | 5 +- src/main/java/org/qortal/crypto/AES.java | 4 +- src/main/java/org/qortal/utils/ZipUtils.java | 57 ++- 7 files changed, 460 insertions(+), 87 deletions(-) diff --git a/pom.xml b/pom.xml index 13ad4807..c4236d24 100644 --- a/pom.xml +++ b/pom.xml @@ -796,5 +796,10 @@ jaxb-runtime ${jaxb-runtime.version} + +org.apache.tika +tika-core +3.1.0 + diff --git a/src/main/java/org/qortal/api/ApiService.java b/src/main/java/org/qortal/api/ApiService.java index 00ab29e0..1cfab1da 100644 --- a/src/main/java/org/qortal/api/ApiService.java +++ b/src/main/java/org/qortal/api/ApiService.java @@ -46,6 +46,7 @@ public class ApiService { private ApiService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.resource", "org.qortal.api.restricted.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index a6f44373..a49dc7f5 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -3,6 +3,7 @@ package org.qortal.api.resource; import com.google.common.primitives.Bytes; import com.j256.simplemagic.ContentInfo; import com.j256.simplemagic.ContentInfoUtil; + import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -12,6 +13,7 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; @@ -63,14 +65,19 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; + import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.FileNameMap; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -79,6 +86,14 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import org.apache.tika.Tika; +import org.apache.tika.mime.MimeTypeException; +import org.apache.tika.mime.MimeTypes; + +import javax.ws.rs.core.Response; + +import org.glassfish.jersey.media.multipart.FormDataParam; + @Path("/arbitrary") @Tag(name = "Arbitrary") public class ArbitraryResource { @@ -878,6 +893,230 @@ public class ArbitraryResource { } + @GET + @Path("/check-tmp-space") + @Produces(MediaType.TEXT_PLAIN) + @Operation( + summary = "Check if the disk has enough disk space for an upcoming upload", + responses = { + @ApiResponse(description = "OK if sufficient space", responseCode = "200"), + @ApiResponse(description = "Insufficient space", responseCode = "507") // 507 = Insufficient Storage + } + ) + @SecurityRequirement(name = "apiKey") + public Response checkUploadSpace(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @QueryParam("totalSize") Long totalSize) { + Security.checkApiCallAllowed(request); + + if (totalSize == null || totalSize <= 0) { + return Response.status(Response.Status.BAD_REQUEST) + .entity("Missing or invalid totalSize parameter").build(); + } + + File baseTmp = new File(System.getProperty("java.io.tmpdir")); + long usableSpace = baseTmp.getUsableSpace(); + long requiredSpace = totalSize * 2; // chunked + merged file estimate + + if (usableSpace < requiredSpace) { + return Response.status(507).entity("Insufficient disk space").build(); // 507 = Insufficient Storage + } + + return Response.ok("Sufficient disk space").build(); + } + + @POST +@Path("/{service}/{name}/{identifier}/chunk") +@Consumes(MediaType.MULTIPART_FORM_DATA) +@Operation( + summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.MULTIPART_FORM_DATA, + schema = @Schema( + implementation = Object.class + ) + ) + ), + responses = { + @ApiResponse( + description = "Chunk uploaded successfully", + responseCode = "200" + ), + @ApiResponse( + description = "Error writing chunk", + responseCode = "500" + ) + } +) +@SecurityRequirement(name = "apiKey") +public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @FormDataParam("chunk") InputStream chunkStream, + @FormDataParam("index") int index) { + Security.checkApiCallAllowed(request); + + try { + java.nio.file.Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name, identifier); + Files.createDirectories(tempDir); + + java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); + Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING); + + return Response.ok("Chunk " + index + " received").build(); + } catch (IOException e) { + return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); + } +} + +@POST +@Path("/{service}/{name}/{identifier}/finalize") +@Produces(MediaType.TEXT_PLAIN) +@Operation( + summary = "Finalize a chunked upload and build a raw, unsigned, ARBITRARY transaction", + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content(mediaType = MediaType.TEXT_PLAIN) + ) + } +) +@SecurityRequirement(name = "apiKey") +public String finalizeUpload( + @HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") List tags, + @QueryParam("category") Category category, + @QueryParam("filename") String filename, + @QueryParam("fee") Long fee, + @QueryParam("preview") Boolean preview +) { + Security.checkApiCallAllowed(request); + java.nio.file.Path tempFile = null; + java.nio.file.Path tempDir = null; + java.nio.file.Path chunkDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name, identifier); + + try { + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); + } + + // Step 1: Determine a safe filename for disk temp file (regardless of extension correctness) + String safeFilename = filename; + if (filename == null || filename.isBlank()) { + safeFilename = "qortal-" + NTP.getTime(); + } + + tempDir = Files.createTempDirectory("qortal-"); + tempFile = tempDir.resolve(safeFilename); + + // Step 2: Merge chunks + + try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + byte[] buffer = new byte[65536]; + for (java.nio.file.Path chunk : Files.list(chunkDir) + .filter(path -> path.getFileName().toString().startsWith("chunk_")) + .sorted(Comparator.comparingInt(path -> { + String name2 = path.getFileName().toString(); + String numberPart = name2.substring("chunk_".length()); + return Integer.parseInt(numberPart); + })).collect(Collectors.toList())) { + try (InputStream in = Files.newInputStream(chunk)) { + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + } + } + + + // Step 3: Determine correct extension + String detectedExtension = ""; + String uploadFilename = null; + boolean extensionIsValid = false; + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + if (lastDot > 0 && lastDot < filename.length() - 1) { + extensionIsValid = true; + uploadFilename = filename; + } + } + + if (!extensionIsValid) { + Tika tika = new Tika(); + String mimeType = tika.detect(tempFile.toFile()); + try { + MimeTypes allTypes = MimeTypes.getDefaultMimeTypes(); + org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType); + detectedExtension = mime.getExtension(); + } catch (MimeTypeException e) { + LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e); + } + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename; + uploadFilename = baseName + (detectedExtension != null ? detectedExtension : ""); + } else { + uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : ""); + } + } + + + + return this.upload( + Service.valueOf(serviceString), + name, + identifier, + tempFile.toString(), + null, + null, + false, + fee, + uploadFilename, + title, + description, + tags, + category, + preview + ); + + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); + } finally { + if (tempDir != null) { + try { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete temp directory: {}", tempDir, e); + } + } + + try { + Files.walk(chunkDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e); + } + } +} + + + + // Upload base64-encoded data @@ -1409,6 +1648,7 @@ public class ArbitraryResource { ); transactionBuilder.build(); + // Don't compute nonce - this is done by the client (or via POST /arbitrary/compute) ArbitraryTransactionData transactionData = transactionBuilder.getArbitraryTransactionData(); return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData)); @@ -1424,23 +1664,127 @@ public class ArbitraryResource { } } - private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { + // private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { + // try { + // ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); + + // int attempts = 0; + // if (maxAttempts == null) { + // maxAttempts = 5; + // } + + // // Loop until we have data + // if (async) { + // // Asynchronous + // arbitraryDataReader.loadAsynchronously(false, 1); + // } + // else { + // // Synchronous + // while (!Controller.isStopping()) { + // attempts++; + // if (!arbitraryDataReader.isBuilding()) { + // try { + // arbitraryDataReader.loadSynchronously(rebuild); + // break; + // } catch (MissingDataException e) { + // if (attempts > maxAttempts) { + // // Give up after 5 attempts + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); + // } + // } + // } + // Thread.sleep(3000L); + // } + // } + + // java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); + // if (outputPath == null) { + // // Assume the resource doesn't exist + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); + // } + + // if (filepath == null || filepath.isEmpty()) { + // // No file path supplied - so check if this is a single file resource + // String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); + // if (files != null && files.length == 1) { + // // This is a single file resource + // filepath = files[0]; + // } + // else { + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, + // "filepath is required for resources containing more than one file"); + // } + // } + + // java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); + // if (!Files.exists(path)) { + // String message = String.format("No file exists at filepath: %s", filepath); + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message); + // } + + // byte[] data; + // int fileSize = (int)path.toFile().length(); + // int length = fileSize; + + // // Parse "Range" header + // Integer rangeStart = null; + // Integer rangeEnd = null; + // String range = request.getHeader("Range"); + // if (range != null) { + // range = range.replace("bytes=", ""); + // String[] parts = range.split("-"); + // rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null; + // rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize; + // } + + // if (rangeStart != null && rangeEnd != null) { + // // We have a range, so update the requested length + // length = rangeEnd - rangeStart; + // } + + // if (length < fileSize && encoding == null) { + // // Partial content requested, and not encoding the data + // response.setStatus(206); + // response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize)); + // data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length); + // } + // else { + // // Full content requested (or encoded data) + // response.setStatus(200); + // data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory + // } + + // // Encode the data if requested + // if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) { + // data = Base64.encode(data); + // } + + // response.addHeader("Accept-Ranges", "bytes"); + // response.setContentType(context.getMimeType(path.toString())); + // response.setContentLength(data.length); + // response.getOutputStream().write(data); + + // return response; + // } catch (Exception e) { + // LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); + // } + // } + + private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { try { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); - + int attempts = 0; if (maxAttempts == null) { maxAttempts = 5; } - - // Loop until we have data + + // Load the file if (async) { - // Asynchronous arbitraryDataReader.loadAsynchronously(false, 1); - } - else { - // Synchronous + } else { while (!Controller.isStopping()) { attempts++; if (!arbitraryDataReader.isBuilding()) { @@ -1449,7 +1793,6 @@ public class ArbitraryResource { break; } catch (MissingDataException e) { if (attempts > maxAttempts) { - // Give up after 5 attempts throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); } } @@ -1457,80 +1800,94 @@ public class ArbitraryResource { Thread.sleep(3000L); } } - + java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); if (outputPath == null) { - // Assume the resource doesn't exist throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); } - + if (filepath == null || filepath.isEmpty()) { - // No file path supplied - so check if this is a single file resource String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); if (files != null && files.length == 1) { - // This is a single file resource filepath = files[0]; - } - else { - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, - "filepath is required for resources containing more than one file"); + } else { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "filepath is required for resources containing more than one file"); } } - + java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); if (!Files.exists(path)) { - String message = String.format("No file exists at filepath: %s", filepath); - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No file exists at filepath: " + filepath); } - - byte[] data; - int fileSize = (int)path.toFile().length(); - int length = fileSize; - - // Parse "Range" header - Integer rangeStart = null; - Integer rangeEnd = null; + + long fileSize = Files.size(path); + String mimeType = context.getMimeType(path.toString()); String range = request.getHeader("Range"); - if (range != null) { + + long rangeStart = 0; + long rangeEnd = fileSize - 1; + boolean isPartial = false; + + if (range != null && encoding == null) { range = range.replace("bytes=", ""); String[] parts = range.split("-"); - rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null; - rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize; + if (parts.length > 0 && !parts[0].isEmpty()) { + rangeStart = Long.parseLong(parts[0]); + } + if (parts.length > 1 && !parts[1].isEmpty()) { + rangeEnd = Long.parseLong(parts[1]); + } + isPartial = true; } - - if (rangeStart != null && rangeEnd != null) { - // We have a range, so update the requested length - length = rangeEnd - rangeStart; + + long contentLength = rangeEnd - rangeStart + 1; + + // Set headers + response.setContentType(mimeType); + response.setHeader("Accept-Ranges", "bytes"); + + if (isPartial) { + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize)); + } else { + response.setStatus(HttpServletResponse.SC_OK); } - - if (length < fileSize && encoding == null) { - // Partial content requested, and not encoding the data - response.setStatus(206); - response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize)); - data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length); + + OutputStream rawOut = response.getOutputStream(); + + if (encoding != null && "base64".equalsIgnoreCase(encoding)) { + // Stream Base64-encoded output + java.util.Base64.Encoder encoder = java.util.Base64.getEncoder(); + rawOut = encoder.wrap(rawOut); + } else { + // Set Content-Length only when not Base64 + response.setContentLength((int) contentLength); } - else { - // Full content requested (or encoded data) - response.setStatus(200); - data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory + + // Stream file content + try (InputStream inputStream = Files.newInputStream(path)) { + if (rangeStart > 0) { + inputStream.skip(rangeStart); + } + + byte[] buffer = new byte[65536]; + long bytesRemaining = contentLength; + int bytesRead; + + while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) { + rawOut.write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } } - - // Encode the data if requested - if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) { - data = Base64.encode(data); - } - - response.addHeader("Accept-Ranges", "bytes"); - response.setContentType(context.getMimeType(path.toString())); - response.setContentLength(data.length); - response.getOutputStream().write(data); - + return response; - } catch (Exception e) { + + } catch (IOException | InterruptedException | NumberFormatException | ApiException | DataException e) { LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); } } + private FileProperties getFileProperties(Service service, String name, String identifier) { try { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 6e1ca0b9..518e27d0 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -52,7 +52,7 @@ public class ArbitraryDataFile { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class); - public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB + public static final long MAX_FILE_SIZE = 3L * 1024 * 1024 * 1024; // 3 GiB protected static final int MAX_CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static final int CHUNK_SIZE = 512 * 1024; // 0.5MiB public static int SHORT_DIGEST_LENGTH = 8; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java index a77442ec..b40f72c7 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java @@ -29,6 +29,7 @@ import org.qortal.utils.FilesystemUtils; import org.qortal.utils.NTP; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -197,7 +198,7 @@ public class ArbitraryDataTransactionBuilder { // We can't use PATCH for on-chain data because this requires the .qortal directory, which can't be put on chain final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(this.path, false); - final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE); + final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE); if (shouldUseOnChainData) { LOGGER.info("Data size is small enough to go on chain - using PUT"); return Method.PUT; @@ -245,7 +246,7 @@ public class ArbitraryDataTransactionBuilder { // Single file resources are handled differently, especially for very small data payloads, as these go on chain final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(path, false); - final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE); + final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE); // Use zip compression if data isn't going on chain Compression compression = shouldUseOnChainData ? Compression.NONE : Compression.ZIP; diff --git a/src/main/java/org/qortal/crypto/AES.java b/src/main/java/org/qortal/crypto/AES.java index d42e22f9..9bfa172a 100644 --- a/src/main/java/org/qortal/crypto/AES.java +++ b/src/main/java/org/qortal/crypto/AES.java @@ -100,7 +100,7 @@ public class AES { // Prepend the output stream with the 16 byte initialization vector outputStream.write(iv.getIV()); - byte[] buffer = new byte[1024]; + byte[] buffer = new byte[65536]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { byte[] output = cipher.update(buffer, 0, bytesRead); @@ -138,7 +138,7 @@ public class AES { Cipher cipher = Cipher.getInstance(algorithm); cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv)); - byte[] buffer = new byte[64]; + byte[] buffer = new byte[65536]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { byte[] output = cipher.update(buffer, 0, bytesRead); diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index c61723e7..8ef2960b 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -27,6 +27,8 @@ package org.qortal.utils; +import java.io.BufferedOutputStream; + import org.qortal.controller.Controller; import java.io.File; @@ -44,11 +46,17 @@ public class ZipUtils { File sourceFile = new File(sourcePath); boolean isSingleFile = Paths.get(sourcePath).toFile().isFile(); FileOutputStream fileOutputStream = new FileOutputStream(destFilePath); + + // 🔧 Use best speed compression level ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); + zipOutputStream.setLevel(java.util.zip.Deflater.BEST_SPEED); + ZipUtils.zip(sourceFile, enclosingFolderName, zipOutputStream, isSingleFile); + zipOutputStream.close(); fileOutputStream.close(); } + public static void zip(final File fileToZip, final String enclosingFolderName, final ZipOutputStream zipOut, boolean isSingleFile) throws IOException, InterruptedException { if (Controller.isStopping()) { @@ -82,7 +90,7 @@ public class ZipUtils { final FileInputStream fis = new FileInputStream(fileToZip); final ZipEntry zipEntry = new ZipEntry(enclosingFolderName); zipOut.putNextEntry(zipEntry); - final byte[] bytes = new byte[1024]; + final byte[] bytes = new byte[65536]; int length; while ((length = fis.read(bytes)) >= 0) { zipOut.write(bytes, 0, length); @@ -92,33 +100,34 @@ public class ZipUtils { public static void unzip(String sourcePath, String destPath) throws IOException { final File destDir = new File(destPath); - final byte[] buffer = new byte[1024]; - final ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath)); - ZipEntry zipEntry = zis.getNextEntry(); - while (zipEntry != null) { - final File newFile = ZipUtils.newFile(destDir, zipEntry); - if (zipEntry.isDirectory()) { - if (!newFile.isDirectory() && !newFile.mkdirs()) { - throw new IOException("Failed to create directory " + newFile); + final byte[] buffer = new byte[65536]; + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath))) { + ZipEntry zipEntry = zis.getNextEntry(); + while (zipEntry != null) { + final File newFile = ZipUtils.newFile(destDir, zipEntry); + if (zipEntry.isDirectory()) { + if (!newFile.isDirectory() && !newFile.mkdirs()) { + throw new IOException("Failed to create directory " + newFile); + } + } else { + File parent = newFile.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) { + throw new IOException("Failed to create directory " + parent); + } + + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(newFile), buffer.length)) { + int len; + while ((len = zis.read(buffer)) > 0) { + bos.write(buffer, 0, len); + } + } } - } else { - File parent = newFile.getParentFile(); - if (!parent.isDirectory() && !parent.mkdirs()) { - throw new IOException("Failed to create directory " + parent); - } - - final FileOutputStream fos = new FileOutputStream(newFile); - int len; - while ((len = zis.read(buffer)) > 0) { - fos.write(buffer, 0, len); - } - fos.close(); + zipEntry = zis.getNextEntry(); } - zipEntry = zis.getNextEntry(); + zis.closeEntry(); } - zis.closeEntry(); - zis.close(); } + /** * See: https://snyk.io/research/zip-slip-vulnerability From 5780a6de7d648e71007c9a4af1a34f3ff63625bc Mon Sep 17 00:00:00 2001 From: PhilReact Date: Wed, 14 May 2025 20:21:13 +0300 Subject: [PATCH 02/12] remove zip best speed --- src/main/java/org/qortal/utils/ZipUtils.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index 8ef2960b..e747e175 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -48,9 +48,7 @@ public class ZipUtils { FileOutputStream fileOutputStream = new FileOutputStream(destFilePath); // 🔧 Use best speed compression level - ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); - zipOutputStream.setLevel(java.util.zip.Deflater.BEST_SPEED); - + ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); ZipUtils.zip(sourceFile, enclosingFolderName, zipOutputStream, isSingleFile); zipOutputStream.close(); From 994761a87ec97beb8799174f4e5e22af00c9db3d Mon Sep 17 00:00:00 2001 From: PhilReact Date: Thu, 15 May 2025 01:20:40 +0300 Subject: [PATCH 03/12] added missing requires --- src/main/java/org/qortal/api/DevProxyService.java | 1 + src/main/java/org/qortal/api/DomainMapService.java | 1 + src/main/java/org/qortal/api/GatewayService.java | 1 + 3 files changed, 3 insertions(+) diff --git a/src/main/java/org/qortal/api/DevProxyService.java b/src/main/java/org/qortal/api/DevProxyService.java index e0bf02db..c0c4e224 100644 --- a/src/main/java/org/qortal/api/DevProxyService.java +++ b/src/main/java/org/qortal/api/DevProxyService.java @@ -40,6 +40,7 @@ public class DevProxyService { private DevProxyService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.proxy.resource", "org.qortal.api.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); diff --git a/src/main/java/org/qortal/api/DomainMapService.java b/src/main/java/org/qortal/api/DomainMapService.java index 8b791121..5b4a6bbe 100644 --- a/src/main/java/org/qortal/api/DomainMapService.java +++ b/src/main/java/org/qortal/api/DomainMapService.java @@ -39,6 +39,7 @@ public class DomainMapService { private DomainMapService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.resource", "org.qortal.api.domainmap.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); diff --git a/src/main/java/org/qortal/api/GatewayService.java b/src/main/java/org/qortal/api/GatewayService.java index 24a7b7c9..3ac77799 100644 --- a/src/main/java/org/qortal/api/GatewayService.java +++ b/src/main/java/org/qortal/api/GatewayService.java @@ -39,6 +39,7 @@ public class GatewayService { private GatewayService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.resource", "org.qortal.api.gateway.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); From bc4e0716db7b8c2b12c67efd81a68380dff24a9e Mon Sep 17 00:00:00 2001 From: PhilReact Date: Thu, 15 May 2025 16:56:53 +0300 Subject: [PATCH 04/12] fix streaming for base64 --- .../api/resource/ArbitraryResource.java | 158 ++++-------------- 1 file changed, 36 insertions(+), 122 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index a49dc7f5..97146b8f 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -85,6 +85,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import java.util.zip.GZIPOutputStream; import org.apache.tika.Tika; import org.apache.tika.mime.MimeTypeException; @@ -701,7 +702,7 @@ public class ArbitraryResource { ) } ) - public HttpServletResponse get(@PathParam("service") Service service, + public void get(@PathParam("service") Service service, @PathParam("name") String name, @QueryParam("filepath") String filepath, @QueryParam("encoding") String encoding, @@ -714,7 +715,7 @@ public class ArbitraryResource { Security.checkApiCallAllowed(request); } - return this.download(service, name, null, filepath, encoding, rebuild, async, attempts); + this.download(service, name, null, filepath, encoding, rebuild, async, attempts); } @GET @@ -734,7 +735,7 @@ public class ArbitraryResource { ) } ) - public HttpServletResponse get(@PathParam("service") Service service, + public void get(@PathParam("service") Service service, @PathParam("name") String name, @PathParam("identifier") String identifier, @QueryParam("filepath") String filepath, @@ -748,7 +749,7 @@ public class ArbitraryResource { Security.checkApiCallAllowed(request, null); } - return this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts); + this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts); } @@ -1664,115 +1665,7 @@ public String finalizeUpload( } } - // private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { - - // try { - // ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); - - // int attempts = 0; - // if (maxAttempts == null) { - // maxAttempts = 5; - // } - - // // Loop until we have data - // if (async) { - // // Asynchronous - // arbitraryDataReader.loadAsynchronously(false, 1); - // } - // else { - // // Synchronous - // while (!Controller.isStopping()) { - // attempts++; - // if (!arbitraryDataReader.isBuilding()) { - // try { - // arbitraryDataReader.loadSynchronously(rebuild); - // break; - // } catch (MissingDataException e) { - // if (attempts > maxAttempts) { - // // Give up after 5 attempts - // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); - // } - // } - // } - // Thread.sleep(3000L); - // } - // } - - // java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); - // if (outputPath == null) { - // // Assume the resource doesn't exist - // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); - // } - - // if (filepath == null || filepath.isEmpty()) { - // // No file path supplied - so check if this is a single file resource - // String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); - // if (files != null && files.length == 1) { - // // This is a single file resource - // filepath = files[0]; - // } - // else { - // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, - // "filepath is required for resources containing more than one file"); - // } - // } - - // java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); - // if (!Files.exists(path)) { - // String message = String.format("No file exists at filepath: %s", filepath); - // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message); - // } - - // byte[] data; - // int fileSize = (int)path.toFile().length(); - // int length = fileSize; - - // // Parse "Range" header - // Integer rangeStart = null; - // Integer rangeEnd = null; - // String range = request.getHeader("Range"); - // if (range != null) { - // range = range.replace("bytes=", ""); - // String[] parts = range.split("-"); - // rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null; - // rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize; - // } - - // if (rangeStart != null && rangeEnd != null) { - // // We have a range, so update the requested length - // length = rangeEnd - rangeStart; - // } - - // if (length < fileSize && encoding == null) { - // // Partial content requested, and not encoding the data - // response.setStatus(206); - // response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize)); - // data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length); - // } - // else { - // // Full content requested (or encoded data) - // response.setStatus(200); - // data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory - // } - - // // Encode the data if requested - // if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) { - // data = Base64.encode(data); - // } - - // response.addHeader("Accept-Ranges", "bytes"); - // response.setContentType(context.getMimeType(path.toString())); - // response.setContentLength(data.length); - // response.getOutputStream().write(data); - - // return response; - // } catch (Exception e) { - // LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); - // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); - // } - // } - - private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { + private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { try { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); @@ -1843,7 +1736,6 @@ public String finalizeUpload( long contentLength = rangeEnd - rangeStart + 1; // Set headers - response.setContentType(mimeType); response.setHeader("Accept-Ranges", "bytes"); if (isPartial) { @@ -1854,13 +1746,25 @@ public String finalizeUpload( } OutputStream rawOut = response.getOutputStream(); - + OutputStream base64Out = null; + OutputStream gzipOut = null; if (encoding != null && "base64".equalsIgnoreCase(encoding)) { - // Stream Base64-encoded output - java.util.Base64.Encoder encoder = java.util.Base64.getEncoder(); - rawOut = encoder.wrap(rawOut); + response.setContentType("text/plain"); + + String acceptEncoding = request.getHeader("Accept-Encoding"); + boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip"); + + if (wantsGzip) { + response.setHeader("Content-Encoding", "gzip"); + gzipOut = new GZIPOutputStream(rawOut); + base64Out = java.util.Base64.getEncoder().wrap(gzipOut); + } else { + base64Out = java.util.Base64.getEncoder().wrap(rawOut); + } + + rawOut = base64Out; } else { - // Set Content-Length only when not Base64 + response.setContentType(mimeType != null ? mimeType : "application/octet-stream"); response.setContentLength((int) contentLength); } @@ -1879,9 +1783,19 @@ public String finalizeUpload( bytesRemaining -= bytesRead; } } - - return response; - +// Stream finished +if (base64Out != null) { + base64Out.close(); // Also flushes and closes the wrapped gzipOut +} else if (gzipOut != null) { + gzipOut.close(); // Only close gzipOut if it wasn't wrapped by base64Out +} else { + rawOut.flush(); // Flush only the base output stream if nothing was wrapped +} +if (!response.isCommitted()) { + response.setStatus(HttpServletResponse.SC_OK); + response.getWriter().write(" "); +} + } catch (IOException | InterruptedException | NumberFormatException | ApiException | DataException e) { LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); From f2b5802d9c11beccf25e6196761eae0d257dc4ca Mon Sep 17 00:00:00 2001 From: PhilReact Date: Fri, 16 May 2025 01:17:01 +0300 Subject: [PATCH 05/12] change to streaming --- .../org/qortal/utils/FilesystemUtils.java | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index e9921561..5a0f676c 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -6,6 +6,7 @@ import org.qortal.settings.Settings; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.RandomAccessFile; import java.nio.charset.StandardCharsets; import java.nio.file.*; @@ -232,31 +233,37 @@ public class FilesystemUtils { } public static byte[] getSingleFileContents(Path path, Integer maxLength) throws IOException { - byte[] data = null; - // TODO: limit the file size that can be loaded into memory - - // If the path is a file, read the contents directly - if (path.toFile().isFile()) { - int fileSize = (int)path.toFile().length(); - maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize; - data = FilesystemUtils.readFromFile(path.toString(), 0, maxLength); - } - - // Or if it's a directory, only load file contents if there is a single file inside it - else if (path.toFile().isDirectory()) { + Path filePath = null; + + if (Files.isRegularFile(path)) { + filePath = path; + } else if (Files.isDirectory(path)) { String[] files = ArrayUtils.removeElement(path.toFile().list(), ".qortal"); if (files.length == 1) { - Path filePath = Paths.get(path.toString(), files[0]); - if (filePath.toFile().isFile()) { - int fileSize = (int)filePath.toFile().length(); - maxLength = maxLength != null ? Math.min(maxLength, fileSize) : fileSize; - data = FilesystemUtils.readFromFile(filePath.toString(), 0, maxLength); - } + filePath = path.resolve(files[0]); } } - - return data; + + if (filePath == null || !Files.exists(filePath)) { + return null; + } + + long fileSize = Files.size(filePath); + int length = (maxLength != null) ? Math.min(maxLength, (int) Math.min(fileSize, Integer.MAX_VALUE)) : (int) Math.min(fileSize, Integer.MAX_VALUE); + + try (InputStream in = Files.newInputStream(filePath)) { + byte[] buffer = new byte[length]; + int bytesRead = in.read(buffer); + if (bytesRead < length) { + // Resize buffer to actual read size + byte[] trimmed = new byte[bytesRead]; + System.arraycopy(buffer, 0, trimmed, 0, bytesRead); + return trimmed; + } + return buffer; + } } + /** * isSingleFileResource From 2cd5f9e4cd5ee5f8df7336703160fc0a9ced2c2a Mon Sep 17 00:00:00 2001 From: PhilReact Date: Fri, 16 May 2025 01:18:02 +0300 Subject: [PATCH 06/12] change limit --- src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 518e27d0..96bdcdb5 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -52,7 +52,7 @@ public class ArbitraryDataFile { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class); - public static final long MAX_FILE_SIZE = 3L * 1024 * 1024 * 1024; // 3 GiB + public static final long MAX_FILE_SIZE = 2L * 1024 * 1024 * 1024; // 2 GiB protected static final int MAX_CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static final int CHUNK_SIZE = 512 * 1024; // 0.5MiB public static int SHORT_DIGEST_LENGTH = 8; From 1c52c18d3208ae1d25ace32293981e028c8cbfd1 Mon Sep 17 00:00:00 2001 From: PhilReact Date: Fri, 16 May 2025 15:49:47 +0300 Subject: [PATCH 07/12] added endpoints --- .../api/resource/ArbitraryResource.java | 186 +++++++++++++++++- 1 file changed, 184 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 97146b8f..3ca3ce75 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -925,6 +925,187 @@ public class ArbitraryResource { return Response.ok("Sufficient disk space").build(); } + @POST +@Path("/{service}/{name}/chunk") +@Consumes(MediaType.MULTIPART_FORM_DATA) +@Operation( + summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource (no identifier)", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.MULTIPART_FORM_DATA, + schema = @Schema( + implementation = Object.class + ) + ) + ), + responses = { + @ApiResponse( + description = "Chunk uploaded successfully", + responseCode = "200" + ), + @ApiResponse( + description = "Error writing chunk", + responseCode = "500" + ) + } +) +@SecurityRequirement(name = "apiKey") +public Response uploadChunkNoIdentifier(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @FormDataParam("chunk") InputStream chunkStream, + @FormDataParam("index") int index) { + Security.checkApiCallAllowed(request); + + try { + java.nio.file.Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name); + Files.createDirectories(tempDir); + + java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); + Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING); + + return Response.ok("Chunk " + index + " received").build(); + } catch (IOException e) { + return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); + } +} + +@POST +@Path("/{service}/{name}/finalize") +@Produces(MediaType.TEXT_PLAIN) +@Operation( + summary = "Finalize a chunked upload (no identifier) and build a raw, unsigned, ARBITRARY transaction", + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content(mediaType = MediaType.TEXT_PLAIN) + ) + } +) +@SecurityRequirement(name = "apiKey") +public String finalizeUploadNoIdentifier( + @HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") List tags, + @QueryParam("category") Category category, + @QueryParam("filename") String filename, + @QueryParam("fee") Long fee, + @QueryParam("preview") Boolean preview, + @QueryParam("isZip") Boolean isZip +) { + Security.checkApiCallAllowed(request); + java.nio.file.Path tempFile = null; + java.nio.file.Path tempDir = null; + java.nio.file.Path chunkDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name); + + try { + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); + } + + String safeFilename = (filename == null || filename.isBlank()) ? "qortal-" + NTP.getTime() : filename; + tempDir = Files.createTempDirectory("qortal-"); + tempFile = tempDir.resolve(safeFilename); + + try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + byte[] buffer = new byte[65536]; + for (java.nio.file.Path chunk : Files.list(chunkDir) + .filter(path -> path.getFileName().toString().startsWith("chunk_")) + .sorted(Comparator.comparingInt(path -> { + String name2 = path.getFileName().toString(); + String numberPart = name2.substring("chunk_".length()); + return Integer.parseInt(numberPart); + })).collect(Collectors.toList())) { + try (InputStream in = Files.newInputStream(chunk)) { + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + } + } + + String detectedExtension = ""; + String uploadFilename = null; + boolean extensionIsValid = false; + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + if (lastDot > 0 && lastDot < filename.length() - 1) { + extensionIsValid = true; + uploadFilename = filename; + } + } + + if (!extensionIsValid) { + Tika tika = new Tika(); + String mimeType = tika.detect(tempFile.toFile()); + try { + MimeTypes allTypes = MimeTypes.getDefaultMimeTypes(); + org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType); + detectedExtension = mime.getExtension(); + } catch (MimeTypeException e) { + LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e); + } + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename; + uploadFilename = baseName + (detectedExtension != null ? detectedExtension : ""); + } else { + uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : ""); + } + } + + // ✅ Call upload with `null` as identifier + return this.upload( + Service.valueOf(serviceString), + name, + null, // no identifier + tempFile.toString(), + null, + null, + isZip, + fee, + uploadFilename, + title, + description, + tags, + category, + preview + ); + + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); + } finally { + if (tempDir != null) { + try { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete temp directory: {}", tempDir, e); + } + } + + try { + Files.walk(chunkDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e); + } + } +} + + + @POST @Path("/{service}/{name}/{identifier}/chunk") @Consumes(MediaType.MULTIPART_FORM_DATA) @@ -996,7 +1177,8 @@ public String finalizeUpload( @QueryParam("category") Category category, @QueryParam("filename") String filename, @QueryParam("fee") Long fee, - @QueryParam("preview") Boolean preview + @QueryParam("preview") Boolean preview, + @QueryParam("isZip") Boolean isZip ) { Security.checkApiCallAllowed(request); java.nio.file.Path tempFile = null; @@ -1080,7 +1262,7 @@ public String finalizeUpload( tempFile.toString(), null, null, - false, + isZip, fee, uploadFilename, title, From e1ea8d65f8638af8e794f2dd920fbefc41d2339c Mon Sep 17 00:00:00 2001 From: PhilReact Date: Fri, 16 May 2025 23:39:32 +0300 Subject: [PATCH 08/12] fix blank filename issue --- src/main/java/org/qortal/api/resource/ArbitraryResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 3ca3ce75..a187aeb5 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1765,7 +1765,7 @@ public String finalizeUpload( if (path == null) { // See if we have a string instead if (string != null) { - if (filename == null) { + if (filename == null || filename.isBlank()) { // Use current time as filename filename = String.format("qortal-%d", NTP.getTime()); } @@ -1780,7 +1780,7 @@ public String finalizeUpload( } // ... or base64 encoded raw data else if (base64 != null) { - if (filename == null) { + if (filename == null || filename.isBlank()) { // Use current time as filename filename = String.format("qortal-%d", NTP.getTime()); } From 58ab02c4f0c81bec88ed401c5ad4d92b9370085e Mon Sep 17 00:00:00 2001 From: PhilReact Date: Sun, 18 May 2025 23:21:49 +0300 Subject: [PATCH 09/12] fix to temp dir --- .../api/resource/ArbitraryResource.java | 85 +++++++++++++++---- 1 file changed, 69 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index a187aeb5..42621582 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -94,6 +94,7 @@ import org.apache.tika.mime.MimeTypes; import javax.ws.rs.core.Response; import org.glassfish.jersey.media.multipart.FormDataParam; +import static org.qortal.api.ApiError.REPOSITORY_ISSUE; @Path("/arbitrary") @Tag(name = "Arbitrary") @@ -914,14 +915,18 @@ public class ArbitraryResource { .entity("Missing or invalid totalSize parameter").build(); } - File baseTmp = new File(System.getProperty("java.io.tmpdir")); - long usableSpace = baseTmp.getUsableSpace(); - long requiredSpace = totalSize * 2; // chunked + merged file estimate - - if (usableSpace < requiredSpace) { - return Response.status(507).entity("Insufficient disk space").build(); // 507 = Insufficient Storage + File uploadDir = new File("uploads-temp"); + if (!uploadDir.exists()) { + uploadDir.mkdirs(); // ensure the folder exists } - + + long usableSpace = uploadDir.getUsableSpace(); + long requiredSpace = totalSize * 2; // estimate for chunks + merge + + if (usableSpace < requiredSpace) { + return Response.status(507).entity("Insufficient disk space").build(); + } + return Response.ok("Sufficient disk space").build(); } @@ -959,14 +964,20 @@ public Response uploadChunkNoIdentifier(@HeaderParam(Security.API_KEY_HEADER) St Security.checkApiCallAllowed(request); try { - java.nio.file.Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name); - Files.createDirectories(tempDir); + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + + java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName); + Files.createDirectories(tempDir); + + java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING); return Response.ok("Chunk " + index + " received").build(); } catch (IOException e) { + LOGGER.error("Failed to write chunk {} for service '{}' and name '{}'", index, serviceString, name, e); return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); } } @@ -1000,16 +1011,23 @@ public String finalizeUploadNoIdentifier( Security.checkApiCallAllowed(request); java.nio.file.Path tempFile = null; java.nio.file.Path tempDir = null; - java.nio.file.Path chunkDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name); + java.nio.file.Path chunkDir = null; + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + + try { + chunkDir = Paths.get("uploads-temp", safeService, safeName); + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); } String safeFilename = (filename == null || filename.isBlank()) ? "qortal-" + NTP.getTime() : filename; tempDir = Files.createTempDirectory("qortal-"); - tempFile = tempDir.resolve(safeFilename); + String sanitizedFilename = Paths.get(safeFilename).getFileName().toString(); + tempFile = tempDir.resolve(sanitizedFilename); try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { byte[] buffer = new byte[65536]; @@ -1061,6 +1079,13 @@ public String finalizeUploadNoIdentifier( } } + Boolean isZipBoolean = false; + + if (isZip != null && isZip) { + isZipBoolean = true; + } + + // ✅ Call upload with `null` as identifier return this.upload( Service.valueOf(serviceString), @@ -1069,7 +1094,7 @@ public String finalizeUploadNoIdentifier( tempFile.toString(), null, null, - isZip, + isZipBoolean, fee, uploadFilename, title, @@ -1080,6 +1105,8 @@ public String finalizeUploadNoIdentifier( ); } catch (IOException e) { + LOGGER.error("Failed to merge chunks for service='{}', name='{}'", serviceString, name, e); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); } finally { if (tempDir != null) { @@ -1141,7 +1168,12 @@ public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey, Security.checkApiCallAllowed(request); try { - java.nio.file.Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name, identifier); + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + String safeIdentifier = Paths.get(identifier).getFileName().toString(); + + java.nio.file.Path tempDir = Paths.get("uploads-temp", safeService, safeName, safeIdentifier); + Files.createDirectories(tempDir); java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); @@ -1149,6 +1181,7 @@ public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey, return Response.ok("Chunk " + index + " received").build(); } catch (IOException e) { + LOGGER.error("Failed to write chunk {} for service='{}', name='{}', identifier='{}'", index, serviceString, name, identifier, e); return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); } } @@ -1183,9 +1216,19 @@ public String finalizeUpload( Security.checkApiCallAllowed(request); java.nio.file.Path tempFile = null; java.nio.file.Path tempDir = null; - java.nio.file.Path chunkDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name, identifier); + java.nio.file.Path chunkDir = null; + + + + try { + String safeService = Paths.get(serviceString).getFileName().toString(); + String safeName = Paths.get(name).getFileName().toString(); + String safeIdentifier = Paths.get(identifier).getFileName().toString(); + java.nio.file.Path baseUploadsDir = Paths.get("uploads-temp"); // relative to Qortal working dir + chunkDir = baseUploadsDir.resolve(safeService).resolve(safeName).resolve(safeIdentifier); + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); } @@ -1197,7 +1240,9 @@ public String finalizeUpload( } tempDir = Files.createTempDirectory("qortal-"); - tempFile = tempDir.resolve(safeFilename); + String sanitizedFilename = Paths.get(safeFilename).getFileName().toString(); + tempFile = tempDir.resolve(sanitizedFilename); + // Step 2: Merge chunks @@ -1253,6 +1298,12 @@ public String finalizeUpload( } } + + Boolean isZipBoolean = false; + + if (isZip != null && isZip) { + isZipBoolean = true; + } return this.upload( @@ -1262,7 +1313,7 @@ public String finalizeUpload( tempFile.toString(), null, null, - isZip, + isZipBoolean, fee, uploadFilename, title, @@ -1273,6 +1324,8 @@ public String finalizeUpload( ); } catch (IOException e) { + LOGGER.error("Unexpected error in finalizeUpload for service='{}', name='{}', name='{}'", serviceString, name, identifier, e); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); } finally { if (tempDir != null) { From ca88cb1f887a899b34d541ea6000cfa07507b867 Mon Sep 17 00:00:00 2001 From: PhilReact Date: Mon, 19 May 2025 16:55:12 +0300 Subject: [PATCH 10/12] allow downloads --- .../api/resource/ArbitraryResource.java | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 42621582..2b912e0b 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -709,14 +709,14 @@ public class ArbitraryResource { @QueryParam("encoding") String encoding, @QueryParam("rebuild") boolean rebuild, @QueryParam("async") boolean async, - @QueryParam("attempts") Integer attempts) { + @QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) { // Authentication can be bypassed in the settings, for those running public QDN nodes if (!Settings.getInstance().isQDNAuthBypassEnabled()) { Security.checkApiCallAllowed(request); } - this.download(service, name, null, filepath, encoding, rebuild, async, attempts); + this.download(service, name, null, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename); } @GET @@ -743,14 +743,14 @@ public class ArbitraryResource { @QueryParam("encoding") String encoding, @QueryParam("rebuild") boolean rebuild, @QueryParam("async") boolean async, - @QueryParam("attempts") Integer attempts) { + @QueryParam("attempts") Integer attempts, @QueryParam("attachment") boolean attachment, @QueryParam("attachmentFilename") String attachmentFilename) { // Authentication can be bypassed in the settings, for those running public QDN nodes if (!Settings.getInstance().isQDNAuthBypassEnabled()) { Security.checkApiCallAllowed(request, null); } - this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts); + this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts, attachment, attachmentFilename); } @@ -1900,8 +1900,9 @@ public String finalizeUpload( } } - private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { + private void download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts, boolean attachment, String attachmentFilename) { try { + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); int attempts = 0; @@ -1948,6 +1949,33 @@ public String finalizeUpload( throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No file exists at filepath: " + filepath); } + if (attachment) { + String rawFilename; + + if (attachmentFilename != null && !attachmentFilename.isEmpty()) { + // 1. Sanitize first + String safeAttachmentFilename = attachmentFilename.replaceAll("[\\\\/:*?\"<>|]", "_"); + + // 2. Check for a valid extension (3–5 alphanumeric chars) + if (!safeAttachmentFilename.matches(".*\\.[a-zA-Z0-9]{2,5}$")) { + safeAttachmentFilename += ".bin"; + } + + rawFilename = safeAttachmentFilename; + } else { + // Fallback if no filename is provided + String baseFilename = (identifier != null && !identifier.isEmpty()) + ? name + "-" + identifier + : name; + rawFilename = baseFilename.replaceAll("[\\\\/:*?\"<>|]", "_") + ".bin"; + } + + // Optional: trim length + rawFilename = rawFilename.length() > 100 ? rawFilename.substring(0, 100) : rawFilename; + + // 3. Set Content-Disposition header + response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\""); + } long fileSize = Files.size(path); String mimeType = context.getMimeType(path.toString()); String range = request.getHeader("Range"); From 9e4925c8dda3f2a1fe41ae8f99ca6d6371b2d7b0 Mon Sep 17 00:00:00 2001 From: PhilReact Date: Sat, 24 May 2025 19:15:36 +0300 Subject: [PATCH 11/12] added back comments --- .../java/org/qortal/api/resource/ArbitraryResource.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 2b912e0b..6adb9e90 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -1910,10 +1910,12 @@ public String finalizeUpload( maxAttempts = 5; } - // Load the file + // Loop until we have data if (async) { + // Asynchronous arbitraryDataReader.loadAsynchronously(false, 1); } else { + // Synchronous while (!Controller.isStopping()) { attempts++; if (!arbitraryDataReader.isBuilding()) { @@ -1932,12 +1934,15 @@ public String finalizeUpload( java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); if (outputPath == null) { + // Assume the resource doesn't exist throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); } if (filepath == null || filepath.isEmpty()) { + // No file path supplied - so check if this is a single file resource String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); if (files != null && files.length == 1) { + // This is a single file resource filepath = files[0]; } else { throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "filepath is required for resources containing more than one file"); From 140d86e209c1edb84a7158a8e5ec691694d0c92c Mon Sep 17 00:00:00 2001 From: PhilReact Date: Sat, 24 May 2025 22:29:33 +0300 Subject: [PATCH 12/12] added comments --- .../api/resource/ArbitraryResource.java | 136 ++++++++++-------- 1 file changed, 79 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 6adb9e90..c453d7b0 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -896,7 +896,7 @@ public class ArbitraryResource { @GET - @Path("/check-tmp-space") + @Path("/check/tmp") @Produces(MediaType.TEXT_PLAIN) @Operation( summary = "Check if the disk has enough disk space for an upcoming upload", @@ -921,7 +921,7 @@ public class ArbitraryResource { } long usableSpace = uploadDir.getUsableSpace(); - long requiredSpace = totalSize * 2; // estimate for chunks + merge + long requiredSpace = (long)(((double)totalSize) * 2.2); // estimate for chunks + merge if (usableSpace < requiredSpace) { return Response.status(507).entity("Insufficient disk space").build(); @@ -1981,61 +1981,79 @@ public String finalizeUpload( // 3. Set Content-Disposition header response.setHeader("Content-Disposition", "attachment; filename=\"" + rawFilename + "\""); } - long fileSize = Files.size(path); - String mimeType = context.getMimeType(path.toString()); - String range = request.getHeader("Range"); - - long rangeStart = 0; - long rangeEnd = fileSize - 1; - boolean isPartial = false; - - if (range != null && encoding == null) { - range = range.replace("bytes=", ""); - String[] parts = range.split("-"); - if (parts.length > 0 && !parts[0].isEmpty()) { - rangeStart = Long.parseLong(parts[0]); - } - if (parts.length > 1 && !parts[1].isEmpty()) { - rangeEnd = Long.parseLong(parts[1]); - } - isPartial = true; - } - - long contentLength = rangeEnd - rangeStart + 1; - - // Set headers - response.setHeader("Accept-Ranges", "bytes"); - - if (isPartial) { - response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); - response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize)); - } else { - response.setStatus(HttpServletResponse.SC_OK); - } - - OutputStream rawOut = response.getOutputStream(); - OutputStream base64Out = null; - OutputStream gzipOut = null; - if (encoding != null && "base64".equalsIgnoreCase(encoding)) { - response.setContentType("text/plain"); + // Determine the total size of the requested file + long fileSize = Files.size(path); + String mimeType = context.getMimeType(path.toString()); - String acceptEncoding = request.getHeader("Accept-Encoding"); - boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip"); - - if (wantsGzip) { - response.setHeader("Content-Encoding", "gzip"); - gzipOut = new GZIPOutputStream(rawOut); - base64Out = java.util.Base64.getEncoder().wrap(gzipOut); - } else { - base64Out = java.util.Base64.getEncoder().wrap(rawOut); + // Attempt to read the "Range" header from the request to support partial content delivery (e.g., for video streaming or resumable downloads) + String range = request.getHeader("Range"); + + long rangeStart = 0; + long rangeEnd = fileSize - 1; + boolean isPartial = false; + + // If a Range header is present and no base64 encoding is requested, parse the range values + if (range != null && encoding == null) { + range = range.replace("bytes=", ""); // Remove the "bytes=" prefix + String[] parts = range.split("-"); // Split the range into start and end + + // Parse range start + if (parts.length > 0 && !parts[0].isEmpty()) { + rangeStart = Long.parseLong(parts[0]); + } + + // Parse range end, if present + if (parts.length > 1 && !parts[1].isEmpty()) { + rangeEnd = Long.parseLong(parts[1]); + } + + isPartial = true; // Indicate that this is a partial content request + } + + // Calculate how many bytes should be sent in the response + long contentLength = rangeEnd - rangeStart + 1; + + // Inform the client that byte ranges are supported + response.setHeader("Accept-Ranges", "bytes"); + + if (isPartial) { + // If partial content was requested, return 206 Partial Content with appropriate headers + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize)); + } else { + // Otherwise, return the entire file with status 200 OK + response.setStatus(HttpServletResponse.SC_OK); + } + + // Initialize output streams for writing the file to the response + OutputStream rawOut = response.getOutputStream(); + OutputStream base64Out = null; + OutputStream gzipOut = null; + + if (encoding != null && "base64".equalsIgnoreCase(encoding)) { + // If base64 encoding is requested, override content type + response.setContentType("text/plain"); + + // Check if the client accepts gzip encoding + String acceptEncoding = request.getHeader("Accept-Encoding"); + boolean wantsGzip = acceptEncoding != null && acceptEncoding.contains("gzip"); + + if (wantsGzip) { + // Wrap output in GZIP and Base64 streams if gzip is accepted + response.setHeader("Content-Encoding", "gzip"); + gzipOut = new GZIPOutputStream(rawOut); + base64Out = java.util.Base64.getEncoder().wrap(gzipOut); + } else { + // Wrap output in Base64 only + base64Out = java.util.Base64.getEncoder().wrap(rawOut); + } + + rawOut = base64Out; // Use the wrapped stream for writing + } else { + // For raw binary output, set the content type and length + response.setContentType(mimeType != null ? mimeType : "application/octet-stream"); + response.setContentLength((int) contentLength); } - - rawOut = base64Out; - } else { - response.setContentType(mimeType != null ? mimeType : "application/octet-stream"); - response.setContentLength((int) contentLength); - } - // Stream file content try (InputStream inputStream = Files.newInputStream(path)) { if (rangeStart > 0) { @@ -2064,10 +2082,14 @@ if (!response.isCommitted()) { response.getWriter().write(" "); } - } catch (IOException | InterruptedException | NumberFormatException | ApiException | DataException e) { - LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); + } catch (IOException | InterruptedException | ApiException | DataException e) { + LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); } + catch ( NumberFormatException e) { + LOGGER.error(String.format("Unable to load %s %s: %s", service, name, e.getMessage()), e); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_DATA, e.getMessage()); + } }