Merge branch 'master' of github.com:Qortal/qortal

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>3.0.2</version>
+	<version>3.0.4</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

src/main/java/org/qortal/ApplyUpdate.java

@@ -7,14 +7,13 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
 import java.security.Security;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
+import org.qortal.api.ApiKey;
 import org.qortal.api.ApiRequest;
 import org.qortal.controller.AutoUpdate;
 import org.qortal.settings.Settings;
@@ -70,14 +69,40 @@ public class ApplyUpdate {
 		String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
 		LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri));
 
+		// The /admin/stop endpoint requires an API key, which may or may not be already generated
+		boolean apiKeyNewlyGenerated = false;
+		ApiKey apiKey = null;
+		try {
+			apiKey = new ApiKey();
+			if (!apiKey.generated()) {
+				apiKey.generate();
+				apiKeyNewlyGenerated = true;
+				LOGGER.info("Generated API key");
+			}
+		} catch (IOException e) {
+			LOGGER.info("Error loading API key: {}", e.getMessage());
+		}
+
+		// Create GET params
+		Map<String, String> params = new HashMap<>();
+		if (apiKey != null) {
+			params.put("apiKey", apiKey.toString());
+		}
+
+		// Attempt to stop the node
 		int attempt;
 		for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) {
 			final int attemptForLogging = attempt;
 			LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS));
-			String response = ApiRequest.perform(baseUri + "admin/stop", null);
-			if (response == null)
+			String response = ApiRequest.perform(baseUri + "admin/stop", params);
+			if (response == null) {
 				// No response - consider node shut down
+				if (apiKeyNewlyGenerated) {
+					// API key was newly generated for this auto update, so we need to remove it
+					ApplyUpdate.removeGeneratedApiKey();
+				}
 				return true;
+			}
 
 			LOGGER.info(() -> String.format("Response from API: %s", response));
 
@@ -89,6 +114,11 @@ public class ApplyUpdate {
 			}
 		}
 
+		if (apiKeyNewlyGenerated) {
+			// API key was newly generated for this auto update, so we need to remove it
+			ApplyUpdate.removeGeneratedApiKey();
+		}
+
 		if (attempt == MAX_ATTEMPTS) {
 			LOGGER.error("Failed to shutdown node - giving up");
 			return false;
@@ -97,6 +127,19 @@ public class ApplyUpdate {
 		return true;
 	}
 
+	private static void removeGeneratedApiKey() {
+		try {
+			LOGGER.info("Removing newly generated API key...");
+
+			// Delete the API key since it was only generated for this auto update
+			ApiKey apiKey = new ApiKey();
+			apiKey.delete();
+
+		} catch (IOException e) {
+			LOGGER.info("Error loading or deleting API key: {}", e.getMessage());
+		}
+	}
+
 	private static void replaceJar() {
 		// Assuming current working directory contains the JAR files
 		Path realJar = Paths.get(JAR_FILENAME);

src/main/java/org/qortal/api/ApiKey.java

@@ -81,6 +81,15 @@ public class ApiKey {
         writer.close();
     }
 
+    public void delete() throws IOException {
+        this.apiKey = null;
+
+        Path filePath = this.getFilePath();
+        if (Files.exists(filePath)) {
+            Files.delete(filePath);
+        }
+    }
+
 
     public boolean generated() {
         return (this.apiKey != null);

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -575,7 +575,11 @@ public class ArbitraryResource {
 								   @PathParam("name") String name,
 								   @QueryParam("filepath") String filepath,
 								   @QueryParam("rebuild") boolean rebuild) {
+
+		// Authentication can be bypassed in the settings, for those running public QDN nodes
+		if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
 			Security.checkApiCallAllowed(request);
+		}
 
 		return this.download(service, name, null, filepath, rebuild);
 	}
@@ -604,7 +608,11 @@ public class ArbitraryResource {
 								   @PathParam("identifier") String identifier,
 								   @QueryParam("filepath") String filepath,
 								   @QueryParam("rebuild") boolean rebuild) {
+
+		// Authentication can be bypassed in the settings, for those running public QDN nodes
+		if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
 			Security.checkApiCallAllowed(request);
+		}
 
 		return this.download(service, name, identifier, filepath, rebuild);
 	}
@@ -1049,6 +1057,10 @@ public class ArbitraryResource {
 					// This is a single file resource
 					filepath = files[0];
 				}
+				else {
+					throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA,
+							"filepath is required for resources containing more than one file");
+				}
 			}
 
 			// TODO: limit file size that can be read into memory

src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java

@@ -6,6 +6,7 @@ import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
+import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
 import org.qortal.data.arbitrary.ArbitraryResourceStatus;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.list.ResourceListManager;
@@ -116,6 +117,9 @@ public class ArbitraryDataResource {
             // Also delete cached data for the entire resource
             this.deleteCache();
 
+            // Invalidate the hosted transactions cache as we have removed an item
+            ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache();
+
             return true;
 
         } catch (DataException | IOException e) {

src/main/java/org/qortal/controller/BlockMinter.java

@@ -51,7 +51,7 @@ public class BlockMinter extends Thread {
 	// Min account level to submit blocks
 	// This is an unvalidated version of Blockchain.minAccountLevelToMint
 	// and exists only to reduce block candidates by default.
-	private static int MIN_LEVEL_FOR_BLOCK_SUBMISSION = 3;
+	private static int MIN_LEVEL_FOR_BLOCK_SUBMISSION = 6;
 
 	// Constructors
 

src/main/java/org/qortal/list/ResourceList.java

@@ -13,6 +13,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 
 public class ResourceList {
@@ -20,7 +21,7 @@ public class ResourceList {
     private static final Logger LOGGER = LogManager.getLogger(ResourceList.class);
 
     private String name;
-    private List<String> list = new ArrayList<>();
+    private List<String> list = Collections.synchronizedList(new ArrayList<>());
 
     /**
      * ResourceList

src/main/java/org/qortal/list/ResourceListManager.java

@@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -13,7 +14,7 @@ public class ResourceListManager {
     private static final Logger LOGGER = LogManager.getLogger(ResourceListManager.class);
 
     private static ResourceListManager instance;
-    private List<ResourceList> lists = new ArrayList<>();
+    private List<ResourceList> lists = Collections.synchronizedList(new ArrayList<>());
 
 
     public ResourceListManager() {

src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java

@@ -330,7 +330,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			bindParams.add(name);
 		}
 
-		sql.append(" GROUP BY name, service, identifier ORDER BY name");
+		sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
 
 		if (reverse != null && reverse) {
 			sql.append(" DESC");
@@ -401,7 +401,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			bindParams.add(queryWildcard);
 		}
 
-		sql.append(" GROUP BY name, service, identifier ORDER BY name");
+		sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
 
 		if (reverse != null && reverse) {
 			sql.append(" DESC");
@@ -465,7 +465,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			sql.append(" AND (identifier = ? OR (? IS NULL))");
 		}
 
-		sql.append(" GROUP BY name ORDER BY name");
+		sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
 
 		if (reverse != null && reverse) {
 			sql.append(" DESC");

src/main/java/org/qortal/settings/Settings.java

@@ -308,6 +308,9 @@ public class Settings {
 	/** Maximum total size of hosted data, in bytes. Unlimited if null */
 	private Long maxStorageCapacity = null;
 
+	/** Whether to serve QDN data without authentication */
+	private boolean qdnAuthBypassEnabled = false;
+
 	// Domain mapping
 	public static class DomainMap {
 		private String domain;
@@ -884,4 +887,8 @@ public class Settings {
 	public Long getMaxStorageCapacity() {
 		return this.maxStorageCapacity;
 	}
+
+	public boolean isQDNAuthBypassEnabled() {
+		return this.qdnAuthBypassEnabled;
+	}
 }

src/test/java/org/qortal/test/api/AdminApiTests.java

@@ -2,10 +2,12 @@ package org.qortal.test.api;
 
 import static org.junit.Assert.*;
 
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.junit.Before;
 import org.junit.Test;
 import org.qortal.api.resource.AdminResource;
 import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
 import org.qortal.test.common.ApiCommon;
 import org.qortal.test.common.Common;
 
@@ -29,7 +31,10 @@ public class AdminApiTests extends ApiCommon {
 	}
 
 	@Test
-	public void testSummary() {
+	public void testSummary() throws IllegalAccessException {
+		// Set localAuthBypassEnabled to true, since we don't need to test authentication here
+		FieldUtils.writeField(Settings.getInstance(), "localAuthBypassEnabled", true, true);
+
 		assertNotNull(this.adminResource.summary("testApiKey"));
 	}
 

src/test/resources/test-settings-v2.json

@@ -15,6 +15,5 @@
   "tempDataPath": "data-test/_temp",
   "listsPath": "lists-test",
   "storagePolicy": "FOLLOWED_OR_VIEWED",
-  "maxStorageCapacity": 104857600,
-  "localAuthBypassEnabled": true
+  "maxStorageCapacity": 104857600
 }