diff --git a/pom.xml b/pom.xml index 13ad4807..c4236d24 100644 --- a/pom.xml +++ b/pom.xml @@ -796,5 +796,10 @@ jaxb-runtime ${jaxb-runtime.version} + +org.apache.tika +tika-core +3.1.0 + diff --git a/src/main/java/org/qortal/api/ApiService.java b/src/main/java/org/qortal/api/ApiService.java index 00ab29e0..1cfab1da 100644 --- a/src/main/java/org/qortal/api/ApiService.java +++ b/src/main/java/org/qortal/api/ApiService.java @@ -46,6 +46,7 @@ public class ApiService { private ApiService() { this.config = new ResourceConfig(); this.config.packages("org.qortal.api.resource", "org.qortal.api.restricted.resource"); + this.config.register(org.glassfish.jersey.media.multipart.MultiPartFeature.class); this.config.register(OpenApiResource.class); this.config.register(ApiDefinition.class); this.config.register(AnnotationPostProcessor.class); diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index a6f44373..a49dc7f5 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -3,6 +3,7 @@ package org.qortal.api.resource; import com.google.common.primitives.Bytes; import com.j256.simplemagic.ContentInfo; import com.j256.simplemagic.ContentInfoUtil; + import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.ArraySchema; @@ -12,6 +13,7 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; + import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; @@ -63,14 +65,19 @@ import javax.servlet.http.HttpServletResponse; import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; + import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.net.FileNameMap; import java.net.URLConnection; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -79,6 +86,14 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import org.apache.tika.Tika; +import org.apache.tika.mime.MimeTypeException; +import org.apache.tika.mime.MimeTypes; + +import javax.ws.rs.core.Response; + +import org.glassfish.jersey.media.multipart.FormDataParam; + @Path("/arbitrary") @Tag(name = "Arbitrary") public class ArbitraryResource { @@ -878,6 +893,230 @@ public class ArbitraryResource { } + @GET + @Path("/check-tmp-space") + @Produces(MediaType.TEXT_PLAIN) + @Operation( + summary = "Check if the disk has enough disk space for an upcoming upload", + responses = { + @ApiResponse(description = "OK if sufficient space", responseCode = "200"), + @ApiResponse(description = "Insufficient space", responseCode = "507") // 507 = Insufficient Storage + } + ) + @SecurityRequirement(name = "apiKey") + public Response checkUploadSpace(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @QueryParam("totalSize") Long totalSize) { + Security.checkApiCallAllowed(request); + + if (totalSize == null || totalSize <= 0) { + return Response.status(Response.Status.BAD_REQUEST) + .entity("Missing or invalid totalSize parameter").build(); + } + + File baseTmp = new File(System.getProperty("java.io.tmpdir")); + long usableSpace = baseTmp.getUsableSpace(); + long requiredSpace = totalSize * 2; // chunked + merged file estimate + + if (usableSpace < requiredSpace) { + return Response.status(507).entity("Insufficient disk space").build(); // 507 = Insufficient Storage + } + + return Response.ok("Sufficient disk space").build(); + } + + @POST +@Path("/{service}/{name}/{identifier}/chunk") +@Consumes(MediaType.MULTIPART_FORM_DATA) +@Operation( + summary = "Upload a single file chunk to be later assembled into a complete arbitrary resource", + requestBody = @RequestBody( + required = true, + content = @Content( + mediaType = MediaType.MULTIPART_FORM_DATA, + schema = @Schema( + implementation = Object.class + ) + ) + ), + responses = { + @ApiResponse( + description = "Chunk uploaded successfully", + responseCode = "200" + ), + @ApiResponse( + description = "Error writing chunk", + responseCode = "500" + ) + } +) +@SecurityRequirement(name = "apiKey") +public Response uploadChunk(@HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @FormDataParam("chunk") InputStream chunkStream, + @FormDataParam("index") int index) { + Security.checkApiCallAllowed(request); + + try { + java.nio.file.Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name, identifier); + Files.createDirectories(tempDir); + + java.nio.file.Path chunkFile = tempDir.resolve("chunk_" + index); + Files.copy(chunkStream, chunkFile, StandardCopyOption.REPLACE_EXISTING); + + return Response.ok("Chunk " + index + " received").build(); + } catch (IOException e) { + return Response.serverError().entity("Failed to write chunk: " + e.getMessage()).build(); + } +} + +@POST +@Path("/{service}/{name}/{identifier}/finalize") +@Produces(MediaType.TEXT_PLAIN) +@Operation( + summary = "Finalize a chunked upload and build a raw, unsigned, ARBITRARY transaction", + responses = { + @ApiResponse( + description = "raw, unsigned, ARBITRARY transaction encoded in Base58", + content = @Content(mediaType = MediaType.TEXT_PLAIN) + ) + } +) +@SecurityRequirement(name = "apiKey") +public String finalizeUpload( + @HeaderParam(Security.API_KEY_HEADER) String apiKey, + @PathParam("service") String serviceString, + @PathParam("name") String name, + @PathParam("identifier") String identifier, + @QueryParam("title") String title, + @QueryParam("description") String description, + @QueryParam("tags") List tags, + @QueryParam("category") Category category, + @QueryParam("filename") String filename, + @QueryParam("fee") Long fee, + @QueryParam("preview") Boolean preview +) { + Security.checkApiCallAllowed(request); + java.nio.file.Path tempFile = null; + java.nio.file.Path tempDir = null; + java.nio.file.Path chunkDir = Paths.get(System.getProperty("java.io.tmpdir"), "qortal-uploads", serviceString, name, identifier); + + try { + if (!Files.exists(chunkDir) || !Files.isDirectory(chunkDir)) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No chunks found for upload"); + } + + // Step 1: Determine a safe filename for disk temp file (regardless of extension correctness) + String safeFilename = filename; + if (filename == null || filename.isBlank()) { + safeFilename = "qortal-" + NTP.getTime(); + } + + tempDir = Files.createTempDirectory("qortal-"); + tempFile = tempDir.resolve(safeFilename); + + // Step 2: Merge chunks + + try (OutputStream out = Files.newOutputStream(tempFile, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING)) { + byte[] buffer = new byte[65536]; + for (java.nio.file.Path chunk : Files.list(chunkDir) + .filter(path -> path.getFileName().toString().startsWith("chunk_")) + .sorted(Comparator.comparingInt(path -> { + String name2 = path.getFileName().toString(); + String numberPart = name2.substring("chunk_".length()); + return Integer.parseInt(numberPart); + })).collect(Collectors.toList())) { + try (InputStream in = Files.newInputStream(chunk)) { + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + } + } + + + // Step 3: Determine correct extension + String detectedExtension = ""; + String uploadFilename = null; + boolean extensionIsValid = false; + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + if (lastDot > 0 && lastDot < filename.length() - 1) { + extensionIsValid = true; + uploadFilename = filename; + } + } + + if (!extensionIsValid) { + Tika tika = new Tika(); + String mimeType = tika.detect(tempFile.toFile()); + try { + MimeTypes allTypes = MimeTypes.getDefaultMimeTypes(); + org.apache.tika.mime.MimeType mime = allTypes.forName(mimeType); + detectedExtension = mime.getExtension(); + } catch (MimeTypeException e) { + LOGGER.warn("Could not determine file extension for MIME type: {}", mimeType, e); + } + + if (filename != null && !filename.isBlank()) { + int lastDot = filename.lastIndexOf('.'); + String baseName = (lastDot > 0) ? filename.substring(0, lastDot) : filename; + uploadFilename = baseName + (detectedExtension != null ? detectedExtension : ""); + } else { + uploadFilename = "qortal-" + NTP.getTime() + (detectedExtension != null ? detectedExtension : ""); + } + } + + + + return this.upload( + Service.valueOf(serviceString), + name, + identifier, + tempFile.toString(), + null, + null, + false, + fee, + uploadFilename, + title, + description, + tags, + category, + preview + ); + + } catch (IOException e) { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, "Failed to merge chunks: " + e.getMessage()); + } finally { + if (tempDir != null) { + try { + Files.walk(tempDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete temp directory: {}", tempDir, e); + } + } + + try { + Files.walk(chunkDir) + .sorted(Comparator.reverseOrder()) + .map(java.nio.file.Path::toFile) + .forEach(File::delete); + } catch (IOException e) { + LOGGER.warn("Failed to delete chunk directory: {}", chunkDir, e); + } + } +} + + + + // Upload base64-encoded data @@ -1409,6 +1648,7 @@ public class ArbitraryResource { ); transactionBuilder.build(); + // Don't compute nonce - this is done by the client (or via POST /arbitrary/compute) ArbitraryTransactionData transactionData = transactionBuilder.getArbitraryTransactionData(); return Base58.encode(ArbitraryTransactionTransformer.toBytes(transactionData)); @@ -1424,23 +1664,127 @@ public class ArbitraryResource { } } - private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { + // private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { + // try { + // ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); + + // int attempts = 0; + // if (maxAttempts == null) { + // maxAttempts = 5; + // } + + // // Loop until we have data + // if (async) { + // // Asynchronous + // arbitraryDataReader.loadAsynchronously(false, 1); + // } + // else { + // // Synchronous + // while (!Controller.isStopping()) { + // attempts++; + // if (!arbitraryDataReader.isBuilding()) { + // try { + // arbitraryDataReader.loadSynchronously(rebuild); + // break; + // } catch (MissingDataException e) { + // if (attempts > maxAttempts) { + // // Give up after 5 attempts + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); + // } + // } + // } + // Thread.sleep(3000L); + // } + // } + + // java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); + // if (outputPath == null) { + // // Assume the resource doesn't exist + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); + // } + + // if (filepath == null || filepath.isEmpty()) { + // // No file path supplied - so check if this is a single file resource + // String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); + // if (files != null && files.length == 1) { + // // This is a single file resource + // filepath = files[0]; + // } + // else { + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, + // "filepath is required for resources containing more than one file"); + // } + // } + + // java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); + // if (!Files.exists(path)) { + // String message = String.format("No file exists at filepath: %s", filepath); + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message); + // } + + // byte[] data; + // int fileSize = (int)path.toFile().length(); + // int length = fileSize; + + // // Parse "Range" header + // Integer rangeStart = null; + // Integer rangeEnd = null; + // String range = request.getHeader("Range"); + // if (range != null) { + // range = range.replace("bytes=", ""); + // String[] parts = range.split("-"); + // rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null; + // rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize; + // } + + // if (rangeStart != null && rangeEnd != null) { + // // We have a range, so update the requested length + // length = rangeEnd - rangeStart; + // } + + // if (length < fileSize && encoding == null) { + // // Partial content requested, and not encoding the data + // response.setStatus(206); + // response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize)); + // data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length); + // } + // else { + // // Full content requested (or encoded data) + // response.setStatus(200); + // data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory + // } + + // // Encode the data if requested + // if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) { + // data = Base64.encode(data); + // } + + // response.addHeader("Accept-Ranges", "bytes"); + // response.setContentType(context.getMimeType(path.toString())); + // response.setContentLength(data.length); + // response.getOutputStream().write(data); + + // return response; + // } catch (Exception e) { + // LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); + // throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); + // } + // } + + private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) { try { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier); - + int attempts = 0; if (maxAttempts == null) { maxAttempts = 5; } - - // Loop until we have data + + // Load the file if (async) { - // Asynchronous arbitraryDataReader.loadAsynchronously(false, 1); - } - else { - // Synchronous + } else { while (!Controller.isStopping()) { attempts++; if (!arbitraryDataReader.isBuilding()) { @@ -1449,7 +1793,6 @@ public class ArbitraryResource { break; } catch (MissingDataException e) { if (attempts > maxAttempts) { - // Give up after 5 attempts throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later."); } } @@ -1457,80 +1800,94 @@ public class ArbitraryResource { Thread.sleep(3000L); } } - + java.nio.file.Path outputPath = arbitraryDataReader.getFilePath(); if (outputPath == null) { - // Assume the resource doesn't exist throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found"); } - + if (filepath == null || filepath.isEmpty()) { - // No file path supplied - so check if this is a single file resource String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal"); if (files != null && files.length == 1) { - // This is a single file resource filepath = files[0]; - } - else { - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, - "filepath is required for resources containing more than one file"); + } else { + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "filepath is required for resources containing more than one file"); } } - + java.nio.file.Path path = Paths.get(outputPath.toString(), filepath); if (!Files.exists(path)) { - String message = String.format("No file exists at filepath: %s", filepath); - throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, message); + throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "No file exists at filepath: " + filepath); } - - byte[] data; - int fileSize = (int)path.toFile().length(); - int length = fileSize; - - // Parse "Range" header - Integer rangeStart = null; - Integer rangeEnd = null; + + long fileSize = Files.size(path); + String mimeType = context.getMimeType(path.toString()); String range = request.getHeader("Range"); - if (range != null) { + + long rangeStart = 0; + long rangeEnd = fileSize - 1; + boolean isPartial = false; + + if (range != null && encoding == null) { range = range.replace("bytes=", ""); String[] parts = range.split("-"); - rangeStart = (parts != null && parts.length > 0) ? Integer.parseInt(parts[0]) : null; - rangeEnd = (parts != null && parts.length > 1) ? Integer.parseInt(parts[1]) : fileSize; + if (parts.length > 0 && !parts[0].isEmpty()) { + rangeStart = Long.parseLong(parts[0]); + } + if (parts.length > 1 && !parts[1].isEmpty()) { + rangeEnd = Long.parseLong(parts[1]); + } + isPartial = true; } - - if (rangeStart != null && rangeEnd != null) { - // We have a range, so update the requested length - length = rangeEnd - rangeStart; + + long contentLength = rangeEnd - rangeStart + 1; + + // Set headers + response.setContentType(mimeType); + response.setHeader("Accept-Ranges", "bytes"); + + if (isPartial) { + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + response.setHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd, fileSize)); + } else { + response.setStatus(HttpServletResponse.SC_OK); } - - if (length < fileSize && encoding == null) { - // Partial content requested, and not encoding the data - response.setStatus(206); - response.addHeader("Content-Range", String.format("bytes %d-%d/%d", rangeStart, rangeEnd-1, fileSize)); - data = FilesystemUtils.readFromFile(path.toString(), rangeStart, length); + + OutputStream rawOut = response.getOutputStream(); + + if (encoding != null && "base64".equalsIgnoreCase(encoding)) { + // Stream Base64-encoded output + java.util.Base64.Encoder encoder = java.util.Base64.getEncoder(); + rawOut = encoder.wrap(rawOut); + } else { + // Set Content-Length only when not Base64 + response.setContentLength((int) contentLength); } - else { - // Full content requested (or encoded data) - response.setStatus(200); - data = Files.readAllBytes(path); // TODO: limit file size that can be read into memory + + // Stream file content + try (InputStream inputStream = Files.newInputStream(path)) { + if (rangeStart > 0) { + inputStream.skip(rangeStart); + } + + byte[] buffer = new byte[65536]; + long bytesRemaining = contentLength; + int bytesRead; + + while (bytesRemaining > 0 && (bytesRead = inputStream.read(buffer, 0, (int) Math.min(buffer.length, bytesRemaining))) != -1) { + rawOut.write(buffer, 0, bytesRead); + bytesRemaining -= bytesRead; + } } - - // Encode the data if requested - if (encoding != null && Objects.equals(encoding.toLowerCase(), "base64")) { - data = Base64.encode(data); - } - - response.addHeader("Accept-Ranges", "bytes"); - response.setContentType(context.getMimeType(path.toString())); - response.setContentLength(data.length); - response.getOutputStream().write(data); - + return response; - } catch (Exception e) { + + } catch (IOException | InterruptedException | NumberFormatException | ApiException | DataException e) { LOGGER.debug(String.format("Unable to load %s %s: %s", service, name, e.getMessage())); throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage()); } } + private FileProperties getFileProperties(Service service, String name, String identifier) { try { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java index 6e1ca0b9..518e27d0 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java @@ -52,7 +52,7 @@ public class ArbitraryDataFile { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class); - public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB + public static final long MAX_FILE_SIZE = 3L * 1024 * 1024 * 1024; // 3 GiB protected static final int MAX_CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static final int CHUNK_SIZE = 512 * 1024; // 0.5MiB public static int SHORT_DIGEST_LENGTH = 8; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java index a77442ec..b40f72c7 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java @@ -29,6 +29,7 @@ import org.qortal.utils.FilesystemUtils; import org.qortal.utils.NTP; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; @@ -197,7 +198,7 @@ public class ArbitraryDataTransactionBuilder { // We can't use PATCH for on-chain data because this requires the .qortal directory, which can't be put on chain final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(this.path, false); - final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE); + final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE); if (shouldUseOnChainData) { LOGGER.info("Data size is small enough to go on chain - using PUT"); return Method.PUT; @@ -245,7 +246,7 @@ public class ArbitraryDataTransactionBuilder { // Single file resources are handled differently, especially for very small data payloads, as these go on chain final boolean isSingleFileResource = FilesystemUtils.isSingleFileResource(path, false); - final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(FilesystemUtils.getSingleFileContents(path).length) <= ArbitraryTransaction.MAX_DATA_SIZE); + final boolean shouldUseOnChainData = (isSingleFileResource && AES.getEncryptedFileSize(Files.size(path)) <= ArbitraryTransaction.MAX_DATA_SIZE); // Use zip compression if data isn't going on chain Compression compression = shouldUseOnChainData ? Compression.NONE : Compression.ZIP; diff --git a/src/main/java/org/qortal/crypto/AES.java b/src/main/java/org/qortal/crypto/AES.java index d42e22f9..9bfa172a 100644 --- a/src/main/java/org/qortal/crypto/AES.java +++ b/src/main/java/org/qortal/crypto/AES.java @@ -100,7 +100,7 @@ public class AES { // Prepend the output stream with the 16 byte initialization vector outputStream.write(iv.getIV()); - byte[] buffer = new byte[1024]; + byte[] buffer = new byte[65536]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { byte[] output = cipher.update(buffer, 0, bytesRead); @@ -138,7 +138,7 @@ public class AES { Cipher cipher = Cipher.getInstance(algorithm); cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv)); - byte[] buffer = new byte[64]; + byte[] buffer = new byte[65536]; int bytesRead; while ((bytesRead = inputStream.read(buffer)) != -1) { byte[] output = cipher.update(buffer, 0, bytesRead); diff --git a/src/main/java/org/qortal/utils/ZipUtils.java b/src/main/java/org/qortal/utils/ZipUtils.java index c61723e7..8ef2960b 100644 --- a/src/main/java/org/qortal/utils/ZipUtils.java +++ b/src/main/java/org/qortal/utils/ZipUtils.java @@ -27,6 +27,8 @@ package org.qortal.utils; +import java.io.BufferedOutputStream; + import org.qortal.controller.Controller; import java.io.File; @@ -44,11 +46,17 @@ public class ZipUtils { File sourceFile = new File(sourcePath); boolean isSingleFile = Paths.get(sourcePath).toFile().isFile(); FileOutputStream fileOutputStream = new FileOutputStream(destFilePath); + + // 🔧 Use best speed compression level ZipOutputStream zipOutputStream = new ZipOutputStream(fileOutputStream); + zipOutputStream.setLevel(java.util.zip.Deflater.BEST_SPEED); + ZipUtils.zip(sourceFile, enclosingFolderName, zipOutputStream, isSingleFile); + zipOutputStream.close(); fileOutputStream.close(); } + public static void zip(final File fileToZip, final String enclosingFolderName, final ZipOutputStream zipOut, boolean isSingleFile) throws IOException, InterruptedException { if (Controller.isStopping()) { @@ -82,7 +90,7 @@ public class ZipUtils { final FileInputStream fis = new FileInputStream(fileToZip); final ZipEntry zipEntry = new ZipEntry(enclosingFolderName); zipOut.putNextEntry(zipEntry); - final byte[] bytes = new byte[1024]; + final byte[] bytes = new byte[65536]; int length; while ((length = fis.read(bytes)) >= 0) { zipOut.write(bytes, 0, length); @@ -92,33 +100,34 @@ public class ZipUtils { public static void unzip(String sourcePath, String destPath) throws IOException { final File destDir = new File(destPath); - final byte[] buffer = new byte[1024]; - final ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath)); - ZipEntry zipEntry = zis.getNextEntry(); - while (zipEntry != null) { - final File newFile = ZipUtils.newFile(destDir, zipEntry); - if (zipEntry.isDirectory()) { - if (!newFile.isDirectory() && !newFile.mkdirs()) { - throw new IOException("Failed to create directory " + newFile); + final byte[] buffer = new byte[65536]; + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(sourcePath))) { + ZipEntry zipEntry = zis.getNextEntry(); + while (zipEntry != null) { + final File newFile = ZipUtils.newFile(destDir, zipEntry); + if (zipEntry.isDirectory()) { + if (!newFile.isDirectory() && !newFile.mkdirs()) { + throw new IOException("Failed to create directory " + newFile); + } + } else { + File parent = newFile.getParentFile(); + if (!parent.isDirectory() && !parent.mkdirs()) { + throw new IOException("Failed to create directory " + parent); + } + + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(newFile), buffer.length)) { + int len; + while ((len = zis.read(buffer)) > 0) { + bos.write(buffer, 0, len); + } + } } - } else { - File parent = newFile.getParentFile(); - if (!parent.isDirectory() && !parent.mkdirs()) { - throw new IOException("Failed to create directory " + parent); - } - - final FileOutputStream fos = new FileOutputStream(newFile); - int len; - while ((len = zis.read(buffer)) > 0) { - fos.write(buffer, 0, len); - } - fos.close(); + zipEntry = zis.getNextEntry(); } - zipEntry = zis.getNextEntry(); + zis.closeEntry(); } - zis.closeEntry(); - zis.close(); } + /** * See: https://snyk.io/research/zip-slip-vulnerability