AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Cleanup temporary files used when building data directories. Also added a safety net to only delete files that are without our data or temp directories.

Browse Source
This commit is contained in:
CalDescent 2021-08-14 15:43:08 +01:00
parent 63f5946527
commit 4b5bd6eed7
7 changed files with 115 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java
View File

@ -1,8 +1,11 @@
package org.qortal.arbitrary;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.utils.FilesystemUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@ -30,6 +33,35 @@ public class ArbitraryDataCombiner {
}
}
public void cleanup() {
this.cleanupPath(this.pathBefore);
this.cleanupPath(this.pathAfter);
}
private void cleanupPath(Path path) {
// Delete pathBefore, if it exists in our data/temp directory
if (FilesystemUtils.pathInsideDataOrTempPath(path)) {
File directory = new File(path.toString());
try {
FileUtils.deleteDirectory(directory);
} catch (IOException e) {
// This will eventually be cleaned up by a maintenance process, so log the error and continue
LOGGER.info("Unable to cleanup directory {}", directory.toString());
}
}
// Delete the parent directory of pathBefore if it is empty (and exists in our data/temp directory)
Path parentDirectory = path.getParent();
if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) {
try {
Files.deleteIfExists(parentDirectory);
} catch (IOException e) {
// This will eventually be cleaned up by a maintenance process, so log the error and continue
LOGGER.info("Unable to cleanup parent directory {}", parentDirectory.toString());
}
}
}
private void preExecute() {
if (this.pathBefore == null || this.pathAfter == null) {
throw new IllegalStateException(String.format("No paths available to build patch"));

35
src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java
View File

@ -6,6 +6,7 @@ import org.qortal.crypto.Crypto;
import org.qortal.settings.Settings;
import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Base58;
import org.qortal.utils.FilesystemUtils;
import java.io.*;
import java.nio.ByteBuffer;
@ -282,7 +283,9 @@ public class ArbitraryDataFile {
// Copy temporary file to data directory
this.filePath = this.copyToDataDirectory(tempDir);
Files.delete(tempDir);
if (FilesystemUtils.pathInsideDataOrTempPath(tempDir)) {
Files.delete(tempDir);
}
return true;
} catch (FileNotFoundException e) {
@ -296,22 +299,18 @@ public class ArbitraryDataFile {
public boolean delete() {
// Delete the complete file
// ... but only if it's inside the Qortal data directory
// ... but only if it's inside the Qortal data or temp directory
Path path = Paths.get(this.filePath);
String dataPath = Settings.getInstance().getDataPath();
Path dataDirectory = Paths.get(dataPath);
if (!path.toAbsolutePath().startsWith(dataDirectory.toAbsolutePath())) {
return false;
}
if (Files.exists(path)) {
try {
Files.delete(path);
this.cleanupFilesystem();
LOGGER.debug("Deleted file {}", path.toString());
return true;
} catch (IOException e) {
LOGGER.warn("Couldn't delete DataFileChunk at path {}", this.filePath);
if (FilesystemUtils.pathInsideDataOrTempPath(path)) {
if (Files.exists(path)) {
try {
Files.delete(path);
this.cleanupFilesystem();
LOGGER.debug("Deleted file {}", path.toString());
return true;
} catch (IOException e) {
LOGGER.warn("Couldn't delete DataFileChunk at path {}", this.filePath);
}
}
}
return false;
@ -343,7 +342,9 @@ public class ArbitraryDataFile {
try (Stream<Path> files = Files.list(directory)) {
final long count = files.count();
if (count == 0) {
Files.delete(directory);
if (FilesystemUtils.pathInsideDataOrTempPath(directory)) {
Files.delete(directory);
}
}
} catch (IOException e) {
LOGGER.warn("Unable to count files in directory", e);

14
src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java
View File

@ -176,19 +176,23 @@ public class ArbitraryDataMerge {
Path dest = Paths.get(base.toString(), relativePath.toString());
LOGGER.trace("Copying {} to {}", source, dest);
FilesystemUtils.copyDirectory(source.toString(), dest.toString());
FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString());
}
private static void deletePathInBaseDir(Path base, Path relativePath) throws IOException {
Path dest = Paths.get(base.toString(), relativePath.toString());
File file = new File(dest.toString());
if (file.exists() && file.isFile()) {
LOGGER.trace("Deleting file {}", dest);
Files.delete(dest);
if (FilesystemUtils.pathInsideDataOrTempPath(dest)) {
LOGGER.trace("Deleting file {}", dest);
Files.delete(dest);
}
}
if (file.exists() && file.isDirectory()) {
LOGGER.trace("Deleting directory {}", dest);
FileUtils.deleteDirectory(file);
if (FilesystemUtils.pathInsideDataOrTempPath(dest)) {
LOGGER.trace("Deleting directory {}", dest);
FileUtils.deleteDirectory(file);
}
}
}

3
src/main/java/org/qortal/arbitrary/ArbitraryDataPatches.java
View File

@ -54,7 +54,8 @@ public class ArbitraryDataPatches {
Path pathAfter = this.paths.get(i);
ArbitraryDataCombiner combiner = new ArbitraryDataCombiner(pathBefore, pathAfter);
combiner.combine();
pathBefore = combiner.getFinalPath(); // TODO: cleanup
combiner.cleanup();
pathBefore = combiner.getFinalPath();
}
this.finalPath = pathBefore;
}

44
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@ -1,5 +1,6 @@
package org.qortal.arbitrary;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.crypto.AES;
@ -77,8 +78,8 @@ public class ArbitraryDataReader {
this.createUncompressedDirectory();
}
private void postExecute() throws IOException {
this.cleanupFilesystem();
private void postExecute() {
}
private void createWorkingDirectory() {
@ -104,7 +105,7 @@ public class ArbitraryDataReader {
private void deleteExistingFiles() {
final Path uncompressedPath = this.uncompressedPath;
if (uncompressedPath != null) {
if (FilesystemUtils.pathInsideDataOrTempPath(uncompressedPath)) {
if (Files.exists(uncompressedPath)) {
LOGGER.trace("Attempting to delete path {}", this.uncompressedPath);
try {
@ -268,7 +269,7 @@ public class ArbitraryDataReader {
if (file.isDirectory()) {
// Already a directory - nothing to uncompress
// We still need to copy the directory to its final destination if it's not already there
this.copyFilePathToFinalDestination();
this.moveFilePathToFinalDestination();
return;
}
@ -282,11 +283,13 @@ public class ArbitraryDataReader {
}
// Replace filePath pointer with the uncompressed file path
Files.delete(this.filePath);
if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
Files.delete(this.filePath);
}
this.filePath = this.uncompressedPath;
}
private void copyFilePathToFinalDestination() throws IOException {
private void moveFilePathToFinalDestination() throws IOException {
if (this.filePath.compareTo(this.uncompressedPath) != 0) {
File source = new File(this.filePath.toString());
File dest = new File(this.uncompressedPath.toString());
@ -296,17 +299,28 @@ public class ArbitraryDataReader {
if (dest == null || !dest.exists()) {
throw new IllegalStateException("Destination directory doesn't exist");
}
FilesystemUtils.copyDirectory(source.toString(), dest.toString());
}
}
FilesystemUtils.copyAndReplaceDirectory(source.toString(), dest.toString());
private void cleanupFilesystem() throws IOException {
// Clean up
if (this.uncompressedPath != null) {
File unzippedFile = new File(this.uncompressedPath.toString());
if (unzippedFile.exists()) {
unzippedFile.delete();
try {
// Delete existing
if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
File directory = new File(this.filePath.toString());
FileUtils.deleteDirectory(directory);
}
// ... and its parent directory if empty
Path parentDirectory = this.filePath.getParent();
if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) {
Files.deleteIfExists(parentDirectory);
}
} catch (IOException e) {
// This will eventually be cleaned up by a maintenance process, so log the error and continue
LOGGER.info("Unable to cleanup directories: {}", e.getMessage());
}
// Finally, update filePath to point to uncompressedPath
this.filePath = this.uncompressedPath;
}
}

12
src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java
View File

@ -9,7 +9,7 @@ import org.qortal.crypto.AES;
import org.qortal.repository.DataException;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import org.qortal.utils.FilesystemUtils;
import org.qortal.utils.ZipUtils;
import javax.crypto.BadPaddingException;
@ -155,7 +155,9 @@ public class ArbitraryDataWriter {
AES.encryptFile("AES", this.aesKey, this.filePath.toString(), this.encryptedPath.toString());
// Replace filePath pointer with the encrypted file path
Files.delete(this.filePath);
if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
Files.delete(this.filePath);
}
this.filePath = this.encryptedPath;
} catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException
@ -205,19 +207,19 @@ public class ArbitraryDataWriter {
private void cleanupFilesystem() throws IOException {
// Clean up
if (this.compressedPath != null) {
if (FilesystemUtils.pathInsideDataOrTempPath(this.compressedPath)) {
File zippedFile = new File(this.compressedPath.toString());
if (zippedFile.exists()) {
zippedFile.delete();
}
}
if (this.encryptedPath != null) {
if (FilesystemUtils.pathInsideDataOrTempPath(this.encryptedPath)) {
File encryptedFile = new File(this.encryptedPath.toString());
if (encryptedFile.exists()) {
encryptedFile.delete();
}
}
if (this.workingPath != null) {
if (FilesystemUtils.pathInsideDataOrTempPath(this.workingPath)) {
FileUtils.deleteDirectory(new File(this.workingPath.toString()));
}
}

24
src/main/java/org/qortal/utils/FilesystemUtils.java
View File

@ -1,10 +1,9 @@
package org.qortal.utils;
import org.qortal.settings.Settings;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.*;
public class FilesystemUtils {
@ -18,17 +17,30 @@ public class FilesystemUtils {
return false;
}
public static void copyDirectory(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException {
public static void copyAndReplaceDirectory(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException {
Files.walk(Paths.get(sourceDirectoryLocation))
.forEach(source -> {
Path destination = Paths.get(destinationDirectoryLocation, source.toString()
.substring(sourceDirectoryLocation.length()));
try {
Files.copy(source, destination);
Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
e.printStackTrace();
}
});
}
public static boolean pathInsideDataOrTempPath(Path path) {
if (path == null) {
return false;
}
Path dataPath = Paths.get(Settings.getInstance().getDataPath()).toAbsolutePath();
Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath()).toAbsolutePath();
Path absolutePath = path.toAbsolutePath();
if (absolutePath.startsWith(dataPath) || absolutePath.startsWith(tempDataPath)) {
return true;
}
return false;
}
}