crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Merge branch 'master' into network-online-accounts-v2

Browse Source
This commit is contained in:
CalDescent 2022-02-18 17:12:44 +00:00 committed by GitHub
commit becb0b37e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
82 changed files with 2577 additions and 911 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
Dockerfile Normal file
View File

@ -0,0 +1,26 @@
FROM maven:3-openjdk-11 as builder
WORKDIR /work
COPY ./ /work/
RUN mvn clean package
###
FROM openjdk:11
RUN useradd -r -u 1000 -g users qortal && \
mkdir /usr/local/qortal /qortal && \
chown 1000:100 /qortal
COPY --from=builder /work/log4j2.properties /usr/local/qortal/
COPY --from=builder /work/target/qortal*.jar /usr/local/qortal/qortal.jar
USER 1000:100
EXPOSE 12391 12392
HEALTHCHECK --start-period=5m CMD curl -sf http://127.0.0.1:12391/admin/info || exit 1
WORKDIR /qortal
VOLUME /qortal
ENTRYPOINT ["java"]
CMD ["-Djava.net.preferIPv4Stack=false", "-jar", "/usr/local/qortal/qortal.jar"]

8
README.md
View File

@ -1,10 +1,4 @@
# Qortal Data Node
## Important
This code is unfinished, and we haven't had the official genesis block for the data chain yet.
Therefore it is only possible to use this code if you first create your own test chain. I would
highly recommend waiting until the code is in a more complete state before trying to run this.
# Qortal Project - Official Repo
## Build / run

8
WindowsInstaller/Qortal.aip
View File

@ -17,10 +17,10 @@
<ROW Property="Manufacturer" Value="Qortal"/>
<ROW Property="MsiLogging" MultiBuildValue="DefaultBuild:vp"/>
<ROW Property="NTP_GOOD" Value="false"/>
<ROW Property="ProductCode" Value="1033:{1FB9DC61-308D-4726-B993-82B2D51AD453} 1049:{0EAB0862-4F57-4D79-B8B4-C3E1D84484A2} 2052:{28F67872-0D8B-48D1-9DE1-7F11D5919CB8} 2057:{725A8624-CB8A-426D-8162-68980ED45BCB} " Type="16"/>
<ROW Property="ProductCode" Value="1033:{5FC8DCC3-BF9C-4D72-8C6D-940340ACD1B8} 1049:{1DEF14AB-2397-4517-B3C8-13221B921753} 2052:{B9E3C1DF-C92D-440A-9A21-869582F8585F} 2057:{91D69E7B-CA7D-4449-8E8A-F22DCEA546FC} " Type="16"/>
<ROW Property="ProductLanguage" Value="2057"/>
<ROW Property="ProductName" Value="Qortal"/>
<ROW Property="ProductVersion" Value="3.0.1" Type="32"/>
<ROW Property="ProductVersion" Value="3.1.1" Type="32"/>
<ROW Property="RECONFIG_NTP" Value="true"/>
<ROW Property="REMOVE_BLOCKCHAIN" Value="YES" Type="4"/>
<ROW Property="REPAIR_BLOCKCHAIN" Value="YES" Type="4"/>
@ -212,7 +212,7 @@
<ROW Component="ADDITIONAL_LICENSE_INFO_71" ComponentId="{12A3ADBE-BB7A-496C-8869-410681E6232F}" Directory_="jdk.zipfs_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_71" Type="0"/>
<ROW Component="ADDITIONAL_LICENSE_INFO_8" ComponentId="{D53AD95E-CF96-4999-80FC-5812277A7456}" Directory_="java.naming_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_8" Type="0"/>
<ROW Component="ADDITIONAL_LICENSE_INFO_9" ComponentId="{6B7EA9B0-5D17-47A8-B78C-FACE86D15E01}" Directory_="java.net.http_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_9" Type="0"/>
<ROW Component="AI_CustomARPName" ComponentId="{163A0AFB-3694-4E1B-ABB8-7C5F28F61305}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
<ROW Component="AI_CustomARPName" ComponentId="{42F5EC19-E46F-4299-B9F7-6E1112F6E4FB}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
<ROW Component="AI_ExePath" ComponentId="{3644948D-AE0B-41BB-9FAF-A79E70490A08}" Directory_="APPDIR" Attributes="260" KeyPath="AI_ExePath"/>
<ROW Component="APPDIR" ComponentId="{680DFDDE-3FB4-47A5-8FF5-934F576C6F91}" Directory_="APPDIR" Attributes="0"/>
<ROW Component="AccessBridgeCallbacks.h" ComponentId="{288055D1-1062-47A3-AA44-5601B4E38AED}" Directory_="bridge_Dir" Attributes="0" KeyPath="AccessBridgeCallbacks.h" Type="0"/>
@ -1173,7 +1173,7 @@
<ROW Action="AI_STORE_LOCATION" Type="51" Source="ARPINSTALLLOCATION" Target="[APPDIR]"/>
<ROW Action="AI_SetPermissions" Type="11265" Source="userAccounts.dll" Target="OnSetPermissions" WithoutSeq="true"/>
<ROW Action="CustomizeLog4j2PropertiesScript" Type="3109" Target="Script Text" TargetUnformatted="var actionData = Session.Property(&quot;CustomActionData&quot;);&#13;&#10;var actionDataArray = actionData.split(&quot;|&quot;);&#13;&#10;var appDir = actionDataArray[0];&#13;&#10;var dataFolder = actionDataArray[1] + actionDataArray[2] + &quot;\\&quot;;&#13;&#10;&#13;&#10;var ForReading = 1, ForWriting = 2, ForAppending = 8;&#13;&#10;var fso = new ActiveXObject(&quot;Scripting.FileSystemObject&quot;);&#13;&#10;&#13;&#10;// Make copy&#13;&#10;fso.CopyFile(appDir + &quot;log4j2.properties&quot;, appDir + &quot;log4j2-orig.properties&quot;, true); // overwrite&#13;&#10;&#13;&#10;// Rewrite %AppDir%\log4j2.properties to update logfile storage path&#13;&#10;var fin = fso.OpenTextFile(appDir + &quot;log4j2-orig.properties&quot;, ForReading, false); // no create&#13;&#10;var fout = fso.OpenTextFile(appDir + &quot;log4j2.properties&quot;, ForWriting, true); // can create&#13;&#10;&#13;&#10;// Copy lines with rewriting where necessary&#13;&#10;while( !fin.AtEndOfStream ) {&#13;&#10;&#9;var line = fin.ReadLine();&#13;&#10;&#13;&#10;&#9;var start = line.indexOf(&quot;property.dirname&quot;);&#13;&#10;&#9;if (start &gt; 0) {&#13;&#10;&#9;&#9;// line: # property.dirname = ...appdata...&#13;&#10;&#9;&#9;// uncomment/replace this line for Windows&#13;&#10;&#9;&#9;fout.WriteLine( &quot;property.dirname = &quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) );&#13;&#10;&#9;} else {&#13;&#10;&#9;&#9;// not found - output verbatim&#13;&#10;&#9;&#9;fout.WriteLine( line );&#13;&#10;&#9;}&#13;&#10;}&#13;&#10;&#13;&#10;fin.Close();&#13;&#10;fout.Close();&#13;&#10;" AdditionalSeq="AI_DATA_SETTER_4"/>
<ROW Action="CustomizeSettingsJsonScript" Type="3109" Target="Script Text" TargetUnformatted="var actionData = Session.Property(&quot;CustomActionData&quot;);&#13;&#10;var actionDataArray = actionData.split(&quot;|&quot;);&#13;&#10;var appDir = actionDataArray[0];&#13;&#10;var dataFolder = actionDataArray[1] + actionDataArray[2] + &quot;\\&quot;;&#13;&#10;&#13;&#10;var ForReading = 1, ForWriting = 2, ForAppending = 8;&#13;&#10;var fso = new ActiveXObject(&quot;Scripting.FileSystemObject&quot;);&#13;&#10;&#13;&#10;// Create basic %APPDIR%\settings.json with path to real settings.json in dataFolder&#13;&#10;var fts = fso.OpenTextFile(appDir + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;fts.WriteLine( &quot;{&quot; );&#13;&#10;// We need to escape Windows path backslashes to keep JSON valid&#13;&#10;fts.WriteLine( &quot; \&quot;userPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;\&quot;&quot; );&#13;&#10;fts.WriteLine( &quot;}&quot; );&#13;&#10;&#13;&#10;fts.Close();&#13;&#10;&#13;&#10;// Make copy&#13;&#10;fso.CopyFile(dataFolder + &quot;settings.json&quot;, dataFolder + &quot;settings-orig.json&quot;, true); // overwrite&#13;&#10;&#13;&#10;// Rewrite settings.json to update repository path&#13;&#10;var fin = fso.OpenTextFile(dataFolder + &quot;settings-orig.json&quot;, ForReading, false);&#13;&#10;var fout = fso.OpenTextFile(dataFolder + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;// First line should contain opening brace&#13;&#10;fout.WriteLine( fin.ReadLine() );&#13;&#10;&#13;&#10;// Append our entries&#13;&#10;fout.WriteLine( &quot; \&quot;repositoryPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;db\&quot;,&quot; );&#13;&#10;&#13;&#10;// copy rest of settings&#13;&#10;while( !fin.AtEndOfStream ) {&#13;&#10;&#9;fout.WriteLine( fin.ReadLine() );&#13;&#10;}&#13;&#10;&#13;&#10;fin.Close();&#13;&#10;fout.Close();&#13;&#10;" AdditionalSeq="AI_DATA_SETTER_3"/>
<ROW Action="CustomizeSettingsJsonScript" Type="3109" Target="Script Text" TargetUnformatted="var actionData = Session.Property(&quot;CustomActionData&quot;);&#13;&#10;var actionDataArray = actionData.split(&quot;|&quot;);&#13;&#10;var appDir = actionDataArray[0];&#13;&#10;var dataFolder = actionDataArray[1] + actionDataArray[2] + &quot;\\&quot;;&#13;&#10;&#13;&#10;var ForReading = 1, ForWriting = 2, ForAppending = 8;&#13;&#10;var fso = new ActiveXObject(&quot;Scripting.FileSystemObject&quot;);&#13;&#10;&#13;&#10;// Create basic %APPDIR%\settings.json with path to real settings.json in dataFolder&#13;&#10;var fts = fso.OpenTextFile(appDir + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;fts.WriteLine( &quot;{&quot; );&#13;&#10;// We need to escape Windows path backslashes to keep JSON valid&#13;&#10;fts.WriteLine( &quot; \&quot;userPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;\&quot;&quot; );&#13;&#10;fts.WriteLine( &quot;}&quot; );&#13;&#10;&#13;&#10;fts.Close();&#13;&#10;&#13;&#10;// Make copy&#13;&#10;fso.CopyFile(dataFolder + &quot;settings.json&quot;, dataFolder + &quot;settings-orig.json&quot;, true); // overwrite&#13;&#10;&#13;&#10;// Rewrite settings.json to update repository path&#13;&#10;var fin = fso.OpenTextFile(dataFolder + &quot;settings-orig.json&quot;, ForReading, false);&#13;&#10;var fout = fso.OpenTextFile(dataFolder + &quot;settings.json&quot;, ForWriting, true);&#13;&#10;&#13;&#10;// First line should contain opening brace&#13;&#10;fout.WriteLine( fin.ReadLine() );&#13;&#10;&#13;&#10;// Append our entries&#13;&#10;fout.WriteLine( &quot; \&quot;repositoryPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;db\&quot;,&quot; );&#13;&#10;fout.WriteLine( &quot; \&quot;dataPath\&quot;: \&quot;&quot; + dataFolder.split(&apos;\\&apos;).join(&apos;\\\\&apos;) + &quot;data\&quot;,&quot; );&#13;&#10;&#13;&#10;// copy rest of settings&#13;&#10;while( !fin.AtEndOfStream ) {&#13;&#10;&#9;fout.WriteLine( fin.ReadLine() );&#13;&#10;}&#13;&#10;&#13;&#10;fin.Close();&#13;&#10;fout.Close();&#13;&#10;" AdditionalSeq="AI_DATA_SETTER_3"/>
<ROW Action="DetectRunningProcess" Type="1" Source="aicustact.dll" Target="DetectProcess" Options="3" AdditionalSeq="AI_DATA_SETTER_8"/>
<ROW Action="DetectW32Time" Type="1" Source="aicustact.dll" Target="DetectService" Options="3" AdditionalSeq="AI_DATA_SETTER_11"/>
<ROW Action="NTP_config" Type="3090" Source="ntpcfg.bat"/>

2
pom.xml
View File

@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>org.qortal</groupId>
<artifactId>qortal</artifactId>
<version>3.0.2</version>
<version>3.1.1</version>
<packaging>jar</packaging>
<properties>
<skipTests>true</skipTests>

53
src/main/java/org/qortal/ApplyUpdate.java
View File

@ -7,14 +7,13 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.Security;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
import org.qortal.api.ApiKey;
import org.qortal.api.ApiRequest;
import org.qortal.controller.AutoUpdate;
import org.qortal.settings.Settings;
@ -70,14 +69,40 @@ public class ApplyUpdate {
String baseUri = "http://localhost:" + Settings.getInstance().getApiPort() + "/";
LOGGER.info(() -> String.format("Shutting down node using API via %s", baseUri));
// The /admin/stop endpoint requires an API key, which may or may not be already generated
boolean apiKeyNewlyGenerated = false;
ApiKey apiKey = null;
try {
apiKey = new ApiKey();
if (!apiKey.generated()) {
apiKey.generate();
apiKeyNewlyGenerated = true;
LOGGER.info("Generated API key");
}
} catch (IOException e) {
LOGGER.info("Error loading API key: {}", e.getMessage());
}
// Create GET params
Map<String, String> params = new HashMap<>();
if (apiKey != null) {
params.put("apiKey", apiKey.toString());
}
// Attempt to stop the node
int attempt;
for (attempt = 0; attempt < MAX_ATTEMPTS; ++attempt) {
final int attemptForLogging = attempt;
LOGGER.info(() -> String.format("Attempt #%d out of %d to shutdown node", attemptForLogging + 1, MAX_ATTEMPTS));
String response = ApiRequest.perform(baseUri + "admin/stop", null);
if (response == null)
String response = ApiRequest.perform(baseUri + "admin/stop", params);
if (response == null) {
// No response - consider node shut down
if (apiKeyNewlyGenerated) {
// API key was newly generated for this auto update, so we need to remove it
ApplyUpdate.removeGeneratedApiKey();
}
return true;
}
LOGGER.info(() -> String.format("Response from API: %s", response));
@ -89,6 +114,11 @@ public class ApplyUpdate {
}
}
if (apiKeyNewlyGenerated) {
// API key was newly generated for this auto update, so we need to remove it
ApplyUpdate.removeGeneratedApiKey();
}
if (attempt == MAX_ATTEMPTS) {
LOGGER.error("Failed to shutdown node - giving up");
return false;
@ -97,6 +127,19 @@ public class ApplyUpdate {
return true;
}
private static void removeGeneratedApiKey() {
try {
LOGGER.info("Removing newly generated API key...");
// Delete the API key since it was only generated for this auto update
ApiKey apiKey = new ApiKey();
apiKey.delete();
} catch (IOException e) {
LOGGER.info("Error loading or deleting API key: {}", e.getMessage());
}
}
private static void replaceJar() {
// Assuming current working directory contains the JAR files
Path realJar = Paths.get(JAR_FILENAME);

9
src/main/java/org/qortal/api/ApiKey.java
View File

@ -81,6 +81,15 @@ public class ApiKey {
writer.close();
}
public void delete() throws IOException {
this.apiKey = null;
Path filePath = this.getFilePath();
if (Files.exists(filePath)) {
Files.delete(filePath);
}
}
public boolean generated() {
return (this.apiKey != null);

2
src/main/java/org/qortal/api/gateway/resource/GatewayResource.java
View File

@ -65,7 +65,7 @@ public class GatewayResource {
}
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.getStatus();
return resource.getStatus(false);
}

3
src/main/java/org/qortal/api/model/NodeStatus.java
View File

@ -4,6 +4,7 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.network.Network;
@XmlAccessorType(XmlAccessType.FIELD)
@ -22,7 +23,7 @@ public class NodeStatus {
public NodeStatus() {
this.isMintingPossible = Controller.getInstance().isMintingPossible();
this.syncPercent = Controller.getInstance().getSyncPercent();
this.syncPercent = Synchronizer.getInstance().getSyncPercent();
this.isSynchronizing = this.syncPercent != null;
this.numberOfConnections = Network.getInstance().getHandshakedPeers().size();

3
src/main/java/org/qortal/api/resource/AdminResource.java
View File

@ -44,6 +44,7 @@ import org.qortal.api.model.NodeInfo;
import org.qortal.api.model.NodeStatus;
import org.qortal.block.BlockChain;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.controller.Synchronizer.SynchronizationResult;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.account.RewardShareData;
@ -525,7 +526,7 @@ public class AdminResource {
SynchronizationResult syncResult;
try {
do {
syncResult = Controller.getInstance().actuallySynchronize(targetPeer, true);
syncResult = Synchronizer.getInstance().actuallySynchronize(targetPeer, true);
} while (syncResult == SynchronizationResult.OK);
} finally {
blockchainLock.unlock();

57
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -232,7 +232,8 @@ public class ArbitraryResource {
}
)
@SecurityRequirement(name = "apiKey")
public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service,
public ArbitraryResourceStatus getDefaultResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") Service service,
@PathParam("name") String name,
@QueryParam("build") Boolean build) {
@ -252,7 +253,8 @@ public class ArbitraryResource {
}
)
@SecurityRequirement(name = "apiKey")
public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service,
public ArbitraryResourceStatus getResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("build") Boolean build) {
@ -574,10 +576,16 @@ public class ArbitraryResource {
@PathParam("service") Service service,
@PathParam("name") String name,
@QueryParam("filepath") String filepath,
@QueryParam("rebuild") boolean rebuild) {
Security.checkApiCallAllowed(request);
@QueryParam("rebuild") boolean rebuild,
@QueryParam("async") boolean async,
@QueryParam("attempts") Integer attempts) {
return this.download(service, name, null, filepath, rebuild);
// Authentication can be bypassed in the settings, for those running public QDN nodes
if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
Security.checkApiCallAllowed(request);
}
return this.download(service, name, null, filepath, rebuild, async, attempts);
}
@GET
@ -603,10 +611,16 @@ public class ArbitraryResource {
@PathParam("name") String name,
@PathParam("identifier") String identifier,
@QueryParam("filepath") String filepath,
@QueryParam("rebuild") boolean rebuild) {
Security.checkApiCallAllowed(request);
@QueryParam("rebuild") boolean rebuild,
@QueryParam("async") boolean async,
@QueryParam("attempts") Integer attempts) {
return this.download(service, name, identifier, filepath, rebuild);
// Authentication can be bypassed in the settings, for those running public QDN nodes
if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
Security.checkApiCallAllowed(request);
}
return this.download(service, name, identifier, filepath, rebuild, async, attempts);
}
@ -1017,14 +1031,23 @@ public class ArbitraryResource {
}
}
private HttpServletResponse download(Service service, String name, String identifier, String filepath, boolean rebuild) {
private HttpServletResponse download(Service service, String name, String identifier, String filepath, boolean rebuild, boolean async, Integer maxAttempts) {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
try {
int attempts = 0;
if (maxAttempts == null) {
maxAttempts = 5;
}
// Loop until we have data
if (async) {
// Asynchronous
arbitraryDataReader.loadAsynchronously(false, 1);
}
else {
// Synchronous
while (!Controller.isStopping()) {
attempts++;
if (!arbitraryDataReader.isBuilding()) {
@ -1032,7 +1055,7 @@ public class ArbitraryResource {
arbitraryDataReader.loadSynchronously(rebuild);
break;
} catch (MissingDataException e) {
if (attempts > 5) {
if (attempts > maxAttempts) {
// Give up after 5 attempts
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data unavailable. Please try again later.");
}
@ -1040,7 +1063,13 @@ public class ArbitraryResource {
}
Thread.sleep(3000L);
}
}
java.nio.file.Path outputPath = arbitraryDataReader.getFilePath();
if (outputPath == null) {
// Assume the resource doesn't exist
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, "File not found");
}
if (filepath == null || filepath.isEmpty()) {
// No file path supplied - so check if this is a single file resource
@ -1049,6 +1078,10 @@ public class ArbitraryResource {
// This is a single file resource
filepath = files[0];
}
else {
throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA,
"filepath is required for resources containing more than one file");
}
}
// TODO: limit file size that can be read into memory
@ -1085,7 +1118,7 @@ public class ArbitraryResource {
}
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.getStatus();
return resource.getStatus(false);
}
private List<ArbitraryResourceInfo> addStatusToResources(List<ArbitraryResourceInfo> resources) {
@ -1094,7 +1127,7 @@ public class ArbitraryResource {
for (ArbitraryResourceInfo resourceInfo : resources) {
ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ResourceIdType.NAME,
resourceInfo.service, resourceInfo.identifier);
ArbitraryResourceStatus status = resource.getStatus();
ArbitraryResourceStatus status = resource.getStatus(true);
if (status != null) {
resourceInfo.status = status;
}

7
src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java
View File

@ -67,11 +67,16 @@ public class CrossChainBitcoinResource {
if (!bitcoin.isValidDeterministicKey(key58))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
Long balance = bitcoin.getWalletBalance(key58);
try {
Long balance = bitcoin.getWalletBalanceFromTransactions(key58);
if (balance == null)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
return balance.toString();
} catch (ForeignBlockchainException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
}
}
@POST

7
src/main/java/org/qortal/api/resource/CrossChainDogecoinResource.java
View File

@ -65,11 +65,16 @@ public class CrossChainDogecoinResource {
if (!dogecoin.isValidDeterministicKey(key58))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
Long balance = dogecoin.getWalletBalance(key58);
try {
Long balance = dogecoin.getWalletBalanceFromTransactions(key58);
if (balance == null)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
return balance.toString();
} catch (ForeignBlockchainException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
}
}
@POST

7
src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java
View File

@ -67,11 +67,16 @@ public class CrossChainLitecoinResource {
if (!litecoin.isValidDeterministicKey(key58))
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY);
Long balance = litecoin.getWalletBalance(key58);
try {
Long balance = litecoin.getWalletBalanceFromTransactions(key58);
if (balance == null)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
return balance.toString();
} catch (ForeignBlockchainException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FOREIGN_BLOCKCHAIN_NETWORK_ISSUE);
}
}
@POST

80
src/main/java/org/qortal/api/resource/TransactionsResource.java
View File

@ -9,6 +9,8 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.tags.Tag;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@ -44,6 +46,7 @@ import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Base58;
import com.google.common.primitives.Bytes;
import org.qortal.utils.NTP;
@Path("/transactions")
@Tag(name = "Transactions")
@ -363,6 +366,83 @@ public class TransactionsResource {
}
}
@GET
@Path("/unitfee")
@Operation(
summary = "Get transaction unit fee",
responses = {
@ApiResponse(
content = @Content(
mediaType = MediaType.TEXT_PLAIN,
schema = @Schema(
type = "number"
)
)
)
}
)
@ApiErrors({
ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE
})
public long getTransactionUnitFee(@QueryParam("txType") TransactionType txType,
@QueryParam("timestamp") Long timestamp,
@QueryParam("level") Integer accountLevel) {
try {
if (timestamp == null) {
timestamp = NTP.getTime();
}
Constructor<?> constructor = txType.constructor;
Transaction transaction = (Transaction) constructor.newInstance(null, null);
// FUTURE: add accountLevel parameter to transaction.getUnitFee() if needed
return transaction.getUnitFee(timestamp);
} catch (InstantiationException | IllegalAccessException | InvocationTargetException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA, e);
}
}
@POST
@Path("/fee")
@Operation(
summary = "Get recommended fee for supplied transaction data",
requestBody = @RequestBody(
required = true,
content = @Content(
mediaType = MediaType.TEXT_PLAIN,
schema = @Schema(
type = "string"
)
)
)
)
@ApiErrors({
ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE
})
public long getRecommendedTransactionFee(String rawInputBytes58) {
byte[] rawInputBytes = Base58.decode(rawInputBytes58);
if (rawInputBytes.length == 0)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.JSON);
try (final Repository repository = RepositoryManager.getRepository()) {
// Append null signature on the end before transformation
byte[] rawBytes = Bytes.concat(rawInputBytes, new byte[TransactionTransformer.SIGNATURE_LENGTH]);
TransactionData transactionData = TransactionTransformer.fromBytes(rawBytes);
if (transactionData == null)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
Transaction transaction = Transaction.fromData(repository, transactionData);
return transaction.calcRecommendedFee();
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
} catch (TransformationException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e);
}
}
@GET
@Path("/creator/{publickey}")
@Operation(

25
src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
View File

@ -13,8 +13,11 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
private final Long creationTimestamp;
private Long buildStartTimestamp = null;
private Long buildEndTimestamp = null;
private Integer priority = 0;
private boolean failed = false;
private static int HIGH_PRIORITY_THRESHOLD = 5;
/* The maximum amount of time to spend on a single build */
// TODO: interrupt an in-progress build
public static long BUILD_TIMEOUT = 60*1000L; // 60 seconds
@ -27,13 +30,20 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
this.creationTimestamp = NTP.getTime();
}
public void prepareForBuild() {
this.buildStartTimestamp = NTP.getTime();
}
public void build() throws IOException, DataException, MissingDataException {
Long now = NTP.getTime();
if (now == null) {
this.buildStartTimestamp = null;
throw new DataException("NTP time hasn't synced yet");
}
if (this.buildStartTimestamp == null) {
this.buildStartTimestamp = now;
}
ArbitraryDataReader arbitraryDataReader =
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);
@ -70,6 +80,21 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
return this.buildStartTimestamp;
}
public Integer getPriority() {
if (this.priority != null) {
return this.priority;
}
return 0;
}
public void setPriority(Integer priority) {
this.priority = priority;
}
public boolean isHighPriority() {
return this.priority >= HIGH_PRIORITY_THRESHOLD;
}
public void setFailed(boolean failed) {
this.failed = failed;
}

12
src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java
View File

@ -61,6 +61,9 @@ public class ArbitraryDataCache {
}
// No need to invalidate the cache
// Remember that it's up to date, so that we won't check again for a while
ArbitraryDataManager.getInstance().addResourceToCache(this.getArbitraryDataResource());
return false;
}
@ -84,14 +87,7 @@ public class ArbitraryDataCache {
// If the state's sig doesn't match the latest transaction's sig, we need to invalidate
// This means that an updated layer is available
if (this.shouldInvalidateDueToSignatureMismatch()) {
// Add to the in-memory cache first, so that we won't check again for a while
ArbitraryDataManager.getInstance().addResourceToCache(this.getArbitraryDataResource());
return true;
}
return false;
return this.shouldInvalidateDueToSignatureMismatch();
}
/**

70
src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java
View File

@ -366,6 +366,21 @@ public class ArbitraryDataFile {
return false;
}
public boolean delete(int attempts) {
// Keep trying to delete the data until it is deleted, or we reach 10 attempts
for (int i=0; i<attempts; i++) {
if (this.delete()) {
return true;
}
try {
Thread.sleep(1000L);
} catch (InterruptedException e) {
// Fall through to exit method
}
}
return false;
}
public boolean deleteAllChunks() {
boolean success = false;
@ -439,6 +454,11 @@ public class ArbitraryDataFile {
return chunk.exists();
}
}
if (Arrays.equals(hash, this.metadataHash)) {
if (this.metadataFile != null) {
return this.metadataFile.exists();
}
}
if (Arrays.equals(this.getHash(), hash)) {
return this.exists();
}
@ -455,9 +475,6 @@ public class ArbitraryDataFile {
if (this.metadataFile == null) {
this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature);
if (!metadataFile.exists()) {
return false;
}
}
// If the metadata file doesn't exist, we can't check if we have the chunks
@ -496,9 +513,6 @@ public class ArbitraryDataFile {
if (this.metadataFile == null) {
this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature);
if (!metadataFile.exists()) {
return false;
}
}
// If the metadata file doesn't exist, we can't check if we have any chunks
@ -540,6 +554,50 @@ public class ArbitraryDataFile {
return false;
}
/**
* Retrieve a list of file hashes for this transaction that we do not hold locally
*
* @return a List of chunk hashes, or null if we are unable to determine what is missing
*/
public List<byte[]> missingHashes() {
List<byte[]> missingHashes = new ArrayList<>();
try {
if (this.metadataHash == null) {
// We don't have any metadata so can't check if we have the chunks
// Even if this transaction has no chunks, we don't have the file either (already checked above)
return null;
}
if (this.metadataFile == null) {
this.metadataFile = ArbitraryDataFile.fromHash(this.metadataHash, this.signature);
}
// If the metadata file doesn't exist, we can't check if we have the chunks
if (!metadataFile.getFilePath().toFile().exists()) {
return null;
}
if (this.metadata == null) {
this.setMetadata(new ArbitraryDataTransactionMetadata(this.metadataFile.getFilePath()));
}
// Read the metadata
List<byte[]> chunks = metadata.getChunks();
for (byte[] chunkHash : chunks) {
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash, this.signature);
if (!chunk.exists()) {
missingHashes.add(chunkHash);
}
}
return missingHashes;
} catch (DataException e) {
// Something went wrong, so we can't make a sensible decision
return null;
}
}
public boolean containsChunk(byte[] hash) {
for (ArbitraryDataFileChunk chunk : this.chunks) {
if (Arrays.equals(hash, chunk.getHash())) {

26
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@ -122,10 +122,22 @@ public class ArbitraryDataReader {
* This adds the build task to a queue, and the result will be cached when complete
* To check the status of the build, periodically call isCachedDataAvailable()
* Once it returns true, you can then use getFilePath() to access the data itself.
*
* @param overwrite - set to true to force rebuild an existing cache
* @return true if added or already present in queue; false if not
*/
public boolean loadAsynchronously() {
return ArbitraryDataBuildManager.getInstance().addToBuildQueue(this.createQueueItem());
public boolean loadAsynchronously(boolean overwrite, int priority) {
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
this.resourceId, this.resourceIdType, this.service, this.identifier);
if (cache.isCachedDataAvailable()) {
// Use cached data
this.filePath = this.uncompressedPath;
return true;
}
ArbitraryDataBuildQueueItem item = this.createQueueItem();
item.setPriority(priority);
return ArbitraryDataBuildManager.getInstance().addToBuildQueue(item);
}
/**
@ -363,7 +375,7 @@ public class ArbitraryDataReader {
}
// Throw a missing data exception, which allows subsequent layers to fetch data
LOGGER.debug(message);
LOGGER.trace(message);
throw new MissingDataException(message);
}
}
@ -458,12 +470,18 @@ public class ArbitraryDataReader {
throw new DataException(String.format("Unable to unzip file: %s", e.getMessage()));
}
// Replace filePath pointer with the uncompressed file path
if (!this.uncompressedPath.toFile().exists()) {
throw new DataException(String.format("Unable to unzip file: %s", this.filePath));
}
// Delete original compressed file
if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
if (Files.exists(this.filePath)) {
Files.delete(this.filePath);
}
}
// Replace filePath pointer with the uncompressed file path
this.filePath = this.uncompressedPath;
}

2
src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java
View File

@ -76,7 +76,7 @@ public class ArbitraryDataRenderer {
if (!arbitraryDataReader.isCachedDataAvailable()) {
// If async is requested, show a loading screen whilst build is in progress
if (async) {
arbitraryDataReader.loadAsynchronously();
arbitraryDataReader.loadAsynchronously(false, 10);
return this.getLoadingResponse(service, resourceId);
}

84
src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java
View File

@ -6,6 +6,7 @@ import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.list.ResourceListManager;
@ -37,6 +38,8 @@ public class ArbitraryDataResource {
private List<ArbitraryTransactionData> transactions;
private ArbitraryTransactionData latestPutTransaction;
private int layerCount;
private Integer localChunkCount = null;
private Integer totalChunkCount = null;
public ArbitraryDataResource(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
this.resourceId = resourceId.toLowerCase();
@ -50,50 +53,56 @@ public class ArbitraryDataResource {
this.identifier = identifier;
}
public ArbitraryResourceStatus getStatus() {
public ArbitraryResourceStatus getStatus(boolean quick) {
// Calculate the chunk counts
// Avoid this for "quick" statuses, to speed things up
if (!quick) {
this.calculateChunkCounts();
}
if (resourceIdType != ResourceIdType.NAME) {
// We only support statuses for resources with a name
return new ArbitraryResourceStatus(Status.UNSUPPORTED);
return new ArbitraryResourceStatus(Status.UNSUPPORTED, this.localChunkCount, this.totalChunkCount);
}
// Check if the name is blocked
if (ResourceListManager.getInstance()
.listContains("blockedNames", this.resourceId, false)) {
return new ArbitraryResourceStatus(Status.BLOCKED);
return new ArbitraryResourceStatus(Status.BLOCKED, this.localChunkCount, this.totalChunkCount);
}
// Check if a build has failed
ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier);
if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) {
return new ArbitraryResourceStatus(Status.BUILD_FAILED, this.localChunkCount, this.totalChunkCount);
}
// Firstly check the cache to see if it's already built
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(
resourceId, resourceIdType, service, identifier);
if (arbitraryDataReader.isCachedDataAvailable()) {
return new ArbitraryResourceStatus(Status.READY);
}
// Next check if there's a build in progress
ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier);
if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
return new ArbitraryResourceStatus(Status.BUILDING);
}
// Check if a build has failed
if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) {
return new ArbitraryResourceStatus(Status.BUILD_FAILED);
return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount);
}
// Check if we have all data locally for this resource
if (!this.allFilesDownloaded()) {
if (this.isDownloading()) {
return new ArbitraryResourceStatus(Status.DOWNLOADING);
return new ArbitraryResourceStatus(Status.DOWNLOADING, this.localChunkCount, this.totalChunkCount);
}
else if (this.isDataPotentiallyAvailable()) {
return new ArbitraryResourceStatus(Status.PUBLISHED);
return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount);
}
return new ArbitraryResourceStatus(Status.MISSING_DATA);
return new ArbitraryResourceStatus(Status.MISSING_DATA, this.localChunkCount, this.totalChunkCount);
}
// Check if there's a build in progress
if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
return new ArbitraryResourceStatus(Status.BUILDING, this.localChunkCount, this.totalChunkCount);
}
// We have all data locally
return new ArbitraryResourceStatus(Status.DOWNLOADED);
return new ArbitraryResourceStatus(Status.DOWNLOADED, this.localChunkCount, this.totalChunkCount);
}
public boolean delete() {
@ -116,6 +125,9 @@ public class ArbitraryDataResource {
// Also delete cached data for the entire resource
this.deleteCache();
// Invalidate the hosted transactions cache as we have removed an item
ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache();
return true;
} catch (DataException | IOException e) {
@ -124,6 +136,13 @@ public class ArbitraryDataResource {
}
public void deleteCache() throws IOException {
// Don't delete anything if there's a build in progress
ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier);
if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
return;
}
String baseDir = Settings.getInstance().getTempDataPath();
String identifier = this.identifier != null ? this.identifier : "default";
Path cachePath = Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier);
@ -136,6 +155,12 @@ public class ArbitraryDataResource {
}
private boolean allFilesDownloaded() {
// Use chunk counts to speed things up if we can
if (this.localChunkCount != null && this.totalChunkCount != null &&
this.localChunkCount >= this.totalChunkCount) {
return true;
}
try {
this.fetchTransactions();
@ -154,6 +179,25 @@ public class ArbitraryDataResource {
}
}
private void calculateChunkCounts() {
try {
this.fetchTransactions();
List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
int localChunkCount = 0;
int totalChunkCount = 0;
for (ArbitraryTransactionData transactionData : transactionDataList) {
localChunkCount += ArbitraryTransactionUtils.ourChunkCount(transactionData);
totalChunkCount += ArbitraryTransactionUtils.totalChunkCount(transactionData);
}
this.localChunkCount = localChunkCount;
this.totalChunkCount = totalChunkCount;
} catch (DataException e) {}
}
private boolean isRateLimited() {
try {
this.fetchTransactions();

29
src/main/java/org/qortal/block/BlockChain.java
View File

@ -68,9 +68,15 @@ public class BlockChain {
atFindNextTransactionFix,
newBlockSigHeight,
shareBinFix,
calcChainWeightTimestamp;
calcChainWeightTimestamp,
transactionV5Timestamp;
}
// Custom transaction fees
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long nameRegistrationUnitFee;
private long nameRegistrationUnitFeeTimestamp;
/** Map of which blockchain features are enabled when (height/timestamp) */
@XmlJavaTypeAdapter(StringLongMapXmlAdapter.class)
private Map<String, Long> featureTriggers;
@ -141,7 +147,8 @@ public class BlockChain {
}
private List<BlockTimingByHeight> blockTimingsByHeight;
private int minAccountLevelToMint = 1;
private int minAccountLevelToMint;
private int minAccountLevelForBlockSubmissions;
private int minAccountLevelToRewardShare;
private int maxRewardSharesPerMintingAccount;
private int founderEffectiveMintingLevel;
@ -299,6 +306,16 @@ public class BlockChain {
return this.maxBlockSize;
}
// Custom transaction fees
public long getNameRegistrationUnitFee() {
return this.nameRegistrationUnitFee;
}
public long getNameRegistrationUnitFeeTimestamp() {
// FUTURE: we could use a separate structure to indicate fee adjustments for different transaction types
return this.nameRegistrationUnitFeeTimestamp;
}
/** Returns true if approval-needing transaction types require a txGroupId other than NO_GROUP. */
public boolean getRequireGroupForApproval() {
return this.requireGroupForApproval;
@ -344,6 +361,10 @@ public class BlockChain {
return this.minAccountLevelToMint;
}
public int getMinAccountLevelForBlockSubmissions() {
return this.minAccountLevelForBlockSubmissions;
}
public int getMinAccountLevelToRewardShare() {
return this.minAccountLevelToRewardShare;
}
@ -386,6 +407,10 @@ public class BlockChain {
return this.featureTriggers.get(FeatureTrigger.calcChainWeightTimestamp.name()).longValue();
}
public long getTransactionV5Timestamp() {
return this.featureTriggers.get(FeatureTrigger.transactionV5Timestamp.name()).longValue();
}
// More complex getters for aspects that change by height or timestamp
public long getRewardAtHeight(int ourHeight) {

132
src/main/java/org/qortal/controller/BlockMinter.java
View File

@ -1,6 +1,8 @@
package org.qortal.controller;
import java.math.BigInteger;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
@ -16,11 +18,11 @@ import org.qortal.account.PrivateKeyAccount;
import org.qortal.block.Block;
import org.qortal.block.Block.ValidationResult;
import org.qortal.block.BlockChain;
import org.qortal.data.account.AccountData;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.account.RewardShareData;
import org.qortal.data.block.BlockData;
import org.qortal.data.block.BlockSummaryData;
import org.qortal.data.block.CommonBlockData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.network.Network;
import org.qortal.network.Peer;
@ -48,11 +50,6 @@ public class BlockMinter extends Thread {
// Recovery
public static final long INVALID_BLOCK_RECOVERY_TIMEOUT = 10 * 60 * 1000L; // ms
// Min account level to submit blocks
// This is an unvalidated version of Blockchain.minAccountLevelToMint
// and exists only to reduce block candidates by default.
private static int MIN_LEVEL_FOR_BLOCK_SUBMISSION = 3;
// Constructors
public BlockMinter() {
@ -81,6 +78,10 @@ public class BlockMinter extends Thread {
BlockRepository blockRepository = repository.getBlockRepository();
BlockData previousBlockData = null;
// Vars to keep track of blocks that were skipped due to chain weight
byte[] parentSignatureForLastLowWeightBlock = null;
Long timeOfLastLowWeightBlock = null;
List<Block> newBlocks = new ArrayList<>();
// Flags for tracking change in whether minting is possible,
@ -137,16 +138,15 @@ public class BlockMinter extends Thread {
continue;
}
// Optional (non-validated) prevention of block submissions below a defined level
AccountData accountData = repository.getAccountRepository().getAccount(mintingAccount.getAddress());
if (accountData != null) {
Integer level = accountData.getLevel();
if (level != null && level < MIN_LEVEL_FOR_BLOCK_SUBMISSION) {
// Optional (non-validated) prevention of block submissions below a defined level.
// This is an unvalidated version of Blockchain.minAccountLevelToMint
// and exists only to reduce block candidates by default.
int level = mintingAccount.getEffectiveMintingLevel();
if (level < BlockChain.getInstance().getMinAccountLevelForBlockSubmissions()) {
madi.remove();
continue;
}
}
}
List<Peer> peers = Network.getInstance().getHandshakedPeers();
BlockData lastBlockData = blockRepository.getLastBlock();
@ -156,7 +156,7 @@ public class BlockMinter extends Thread {
// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
if (Controller.getInstance().getRecoveryMode() == false)
if (Synchronizer.getInstance().getRecoveryMode() == false)
peers.removeIf(Controller.hasNoRecentBlock);
// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
@ -181,7 +181,7 @@ public class BlockMinter extends Thread {
// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
if (Controller.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false)
if (Synchronizer.getInstance().getRecoveryMode() == false && recoverInvalidBlock == false)
continue;
// There are enough peers with a recent block and our latest block is recent
@ -195,6 +195,9 @@ public class BlockMinter extends Thread {
// Reduce log timeout
logTimeout = 10 * 1000L;
// Last low weight block is no longer valid
parentSignatureForLastLowWeightBlock = null;
}
// Discard accounts we have already built blocks with
@ -211,6 +214,14 @@ public class BlockMinter extends Thread {
continue;
}
if (parentSignatureForLastLowWeightBlock != null) {
// The last iteration found a higher weight block in the network, so sleep for a while
// to allow is to sync the higher weight chain. We are sleeping here rather than when
// detected as we don't want to hold the blockchain lock open.
LOGGER.info("Sleeping for 10 seconds...");
Thread.sleep(10 * 1000L);
}
for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
// First block does the AT heavy-lifting
if (newBlocks.isEmpty()) {
@ -302,6 +313,44 @@ public class BlockMinter extends Thread {
}
}
try {
if (this.higherWeightChainExists(repository, bestWeight)) {
// Check if the base block has updated since the last time we were here
if (parentSignatureForLastLowWeightBlock == null || timeOfLastLowWeightBlock == null ||
!Arrays.equals(parentSignatureForLastLowWeightBlock, previousBlockData.getSignature())) {
// We've switched to a different chain, so reset the timer
timeOfLastLowWeightBlock = NTP.getTime();
}
parentSignatureForLastLowWeightBlock = previousBlockData.getSignature();
// If less than 30 seconds has passed since first detection the higher weight chain,
// we should skip our block submission to give us the opportunity to sync to the better chain
if (NTP.getTime() - timeOfLastLowWeightBlock < 30*1000L) {
LOGGER.info("Higher weight chain found in peers, so not signing a block this round");
LOGGER.info("Time since detected: {}", NTP.getTime() - timeOfLastLowWeightBlock);
continue;
}
else {
// More than 30 seconds have passed, so we should submit our block candidate anyway.
LOGGER.info("More than 30 seconds passed, so proceeding to submit block candidate...");
}
}
else {
LOGGER.debug("No higher weight chain found in peers");
}
} catch (DataException e) {
LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
}
// Discard any uncommitted changes as a result of the higher weight chain detection
repository.discardChanges();
// Clear variables that track low weight blocks
parentSignatureForLastLowWeightBlock = null;
timeOfLastLowWeightBlock = null;
// Add unconfirmed transactions
addUnconfirmedTransactions(repository, newBlock);
@ -469,6 +518,61 @@ public class BlockMinter extends Thread {
}
}
private BigInteger getOurChainWeightSinceBlock(Repository repository, BlockSummaryData commonBlock, List<BlockSummaryData> peerBlockSummaries) throws DataException {
final int commonBlockHeight = commonBlock.getHeight();
final byte[] commonBlockSig = commonBlock.getSignature();
int mutualHeight = commonBlockHeight;
// Fetch our corresponding block summaries
final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
List<BlockSummaryData> ourBlockSummaries = repository.getBlockRepository()
.getBlockSummaries(commonBlockHeight + 1, ourLatestBlockData.getHeight());
if (!ourBlockSummaries.isEmpty()) {
Synchronizer.getInstance().populateBlockSummariesMinterLevels(repository, ourBlockSummaries);
}
if (ourBlockSummaries != null && peerBlockSummaries != null) {
mutualHeight += Math.min(ourBlockSummaries.size(), peerBlockSummaries.size());
}
return Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight);
}
private boolean higherWeightChainExists(Repository repository, BigInteger blockCandidateWeight) throws DataException {
if (blockCandidateWeight == null) {
// Can't make decisions without knowing the block candidate weight
return false;
}
NumberFormat formatter = new DecimalFormat("0.###E0");
List<Peer> peers = Network.getInstance().getHandshakedPeers();
// Loop through handshaked peers and check for any new block candidates
for (Peer peer : peers) {
if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) {
// This peer has common block data
CommonBlockData commonBlockData = peer.getCommonBlockData();
BlockSummaryData commonBlockSummaryData = commonBlockData.getCommonBlockSummary();
if (commonBlockData.getChainWeight() != null) {
// The synchronizer has calculated this peer's chain weight
BigInteger ourChainWeightSinceCommonBlock = this.getOurChainWeightSinceBlock(repository, commonBlockSummaryData, commonBlockData.getBlockSummariesAfterCommonBlock());
BigInteger ourChainWeight = ourChainWeightSinceCommonBlock.add(blockCandidateWeight);
BigInteger peerChainWeight = commonBlockData.getChainWeight();
if (peerChainWeight.compareTo(ourChainWeight) >= 0) {
// This peer has a higher weight chain than ours
LOGGER.debug("Peer {} is on a higher weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight));
return true;
} else {
LOGGER.debug("Peer {} is on a lower weight chain ({}) than ours ({})", peer, formatter.format(peerChainWeight), formatter.format(ourChainWeight));
}
} else {
LOGGER.debug("Peer {} has no chain weight", peer);
}
} else {
LOGGER.debug("Peer {} has no common block data", peer);
}
}
return false;
}
private static void moderatedLog(Runnable logFunction) {
// We only log if logging at TRACE or previous log timeout has expired
if (!LOGGER.isTraceEnabled() && lastLogTimestamp != null && lastLogTimestamp + logTimeout > System.currentTimeMillis())

523
src/main/java/org/qortal/controller/Controller.java
View File

@ -7,25 +7,14 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.SecureRandom;
import java.security.Security;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Deque;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantLock;
@ -51,7 +40,6 @@ import org.qortal.block.Block;
import org.qortal.block.BlockChain;
import org.qortal.block.BlockChain.BlockTimingByHeight;
import org.qortal.controller.arbitrary.*;
import org.qortal.controller.Synchronizer.SynchronizationResult;
import org.qortal.controller.repository.PruneManager;
import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
import org.qortal.controller.tradebot.TradeBot;
@ -93,14 +81,22 @@ public class Controller extends Thread {
public static final String VERSION_PREFIX = "qortal-";
private static final Logger LOGGER = LogManager.getLogger(Controller.class);
private static final long MISBEHAVIOUR_COOLOFF = 10 * 60 * 1000L; // ms
public static final long MISBEHAVIOUR_COOLOFF = 10 * 60 * 1000L; // ms
private static final int MAX_BLOCKCHAIN_TIP_AGE = 5; // blocks
private static final Object shutdownLock = new Object();
private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s" + File.separator + "blockchain;create=true;hsqldb.full_log_replay=true";
private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms
private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms
private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms
private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms
private static final int MAX_INCOMING_TRANSACTIONS = 5000;
/** Minimum time before considering an invalid unconfirmed transaction as "stale" */
public static final long INVALID_TRANSACTION_STALE_TIMEOUT = 30 * 60 * 1000L; // ms
/** Minimum frequency to re-request stale unconfirmed transactions from peers, to recheck validity */
public static final long INVALID_TRANSACTION_RECHECK_INTERVAL = 60 * 60 * 1000L; // ms\
/** Minimum frequency to re-request expired unconfirmed transactions from peers, to recheck validity
* This mainly exists to stop expired transactions from bloating the list */
public static final long EXPIRED_TRANSACTION_RECHECK_INTERVAL = 10 * 60 * 1000L; // ms
// To do with online accounts list
private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms
@ -114,7 +110,6 @@ public class Controller extends Thread {
private static volatile boolean isStopping = false;
private static BlockMinter blockMinter = null;
private static volatile boolean requestSync = false;
private static volatile boolean requestSysTrayUpdate = true;
private static Controller instance;
@ -148,20 +143,11 @@ public class Controller extends Thread {
/** Whether we can mint new blocks, as reported by BlockMinter. */
private volatile boolean isMintingPossible = false;
/** Synchronization object for sync variables below */
private final Object syncLock = new Object();
/** Whether we are attempting to synchronize. */
private volatile boolean isSynchronizing = false;
/** Temporary estimate of synchronization progress for SysTray use. */
private volatile int syncPercent = 0;
/** List of incoming transaction that are in the import queue */
private List<TransactionData> incomingTransactions = Collections.synchronizedList(new ArrayList<>());
/** Latest block signatures from other peers that we know are on inferior chains. */
List<ByteArray> inferiorChainSignatures = new ArrayList<>();
/** Recovery mode, which is used to bring back a stalled network */
private boolean recoveryMode = false;
private boolean peersAvailable = true; // peersAvailable must default to true
private long timePeersLastAvailable = 0;
/** List of recent invalid unconfirmed transactions */
private Map<String, Long> invalidUnconfirmedTransactions = Collections.synchronizedMap(new HashMap<>());
/** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly minted block. */
private final ReentrantLock blockchainLock = new ReentrantLock();
@ -351,20 +337,6 @@ public class Controller extends Thread {
return this.isMintingPossible;
}
public boolean isSynchronizing() {
return this.isSynchronizing;
}
public Integer getSyncPercent() {
synchronized (this.syncLock) {
return this.isSynchronizing ? this.syncPercent : null;
}
}
public boolean getRecoveryMode() {
return this.recoveryMode;
}
// Entry point
public static void main(String[] args) {
@ -469,6 +441,9 @@ public class Controller extends Thread {
}
});
LOGGER.info("Starting synchronizer");
Synchronizer.getInstance().start();
LOGGER.info("Starting block minter");
blockMinter = new BlockMinter();
blockMinter.start();
@ -479,6 +454,7 @@ public class Controller extends Thread {
// Arbitrary data controllers
LOGGER.info("Starting arbitrary-transaction controllers");
ArbitraryDataManager.getInstance().start();
ArbitraryDataFileManager.getInstance().start();
ArbitraryDataBuildManager.getInstance().start();
ArbitraryDataCleanupManager.getInstance().start();
ArbitraryDataStorageManager.getInstance().start();
@ -541,7 +517,7 @@ public class Controller extends Thread {
@Override
public void run() {
Thread.currentThread().setName("Controller");
Thread.currentThread().setName("Qortal");
final long repositoryBackupInterval = Settings.getInstance().getRepositoryBackupInterval();
final long repositoryCheckpointInterval = Settings.getInstance().getRepositoryCheckpointInterval();
@ -581,10 +557,10 @@ public class Controller extends Thread {
}
}
if (requestSync) {
requestSync = false;
potentiallySynchronize();
}
// Process incoming transactions queue
processIncomingTransactionsQueue();
// Clean up invalid incoming transactions list
cleanupInvalidTransactionsList(now);
// Clean up arbitrary data request cache
ArbitraryDataManager.getInstance().cleanupRequestCache(now);
@ -710,27 +686,6 @@ public class Controller extends Thread {
}
}
private long getRandomRepositoryMaintenanceInterval() {
final long minInterval = Settings.getInstance().getRepositoryMaintenanceMinInterval();
final long maxInterval = Settings.getInstance().getRepositoryMaintenanceMaxInterval();
if (maxInterval == 0) {
return 0;
}
return (new Random().nextLong() % (maxInterval - minInterval)) + minInterval;
}
/**
* Export current trade bot states and minting accounts.
*/
public void exportRepositoryData() {
try (final Repository repository = RepositoryManager.getRepository()) {
repository.exportNodeLocalData();
} catch (DataException e) {
// Fail silently as this is an optional step
}
}
public static final Predicate<Peer> hasMisbehaved = peer -> {
final Long lastMisbehaved = peer.getPeerData().getLastMisbehaved();
return lastMisbehaved != null && lastMisbehaved > NTP.getTime() - MISBEHAVIOUR_COOLOFF;
@ -755,7 +710,7 @@ public class Controller extends Thread {
public static final Predicate<Peer> hasInferiorChainTip = peer -> {
final PeerChainTipData peerChainTipData = peer.getChainTipData();
final List<ByteArray> inferiorChainTips = getInstance().inferiorChainSignatures;
final List<ByteArray> inferiorChainTips = Synchronizer.getInstance().inferiorChainSignatures;
return peerChainTipData == null || peerChainTipData.getLastBlockSignature() == null || inferiorChainTips.contains(new ByteArray(peerChainTipData.getLastBlockSignature()));
};
@ -764,218 +719,34 @@ public class Controller extends Thread {
return peer.isAtLeastVersion(minPeerVersion) == false;
};
private void potentiallySynchronize() throws InterruptedException {
// Already synchronizing via another thread?
if (this.isSynchronizing)
return;
List<Peer> peers = Network.getInstance().getHandshakedPeers();
// Disregard peers that have "misbehaved" recently
peers.removeIf(hasMisbehaved);
// Disregard peers that only have genesis block
peers.removeIf(hasOnlyGenesisBlock);
// Disregard peers that don't have a recent block
peers.removeIf(hasNoRecentBlock);
// Disregard peers that are on an old version
peers.removeIf(hasOldVersion);
checkRecoveryModeForPeers(peers);
if (recoveryMode) {
peers = Network.getInstance().getHandshakedPeers();
peers.removeIf(hasOnlyGenesisBlock);
peers.removeIf(hasMisbehaved);
peers.removeIf(hasOldVersion);
private long getRandomRepositoryMaintenanceInterval() {
final long minInterval = Settings.getInstance().getRepositoryMaintenanceMinInterval();
final long maxInterval = Settings.getInstance().getRepositoryMaintenanceMaxInterval();
if (maxInterval == 0) {
return 0;
}
return (new Random().nextLong() % (maxInterval - minInterval)) + minInterval;
}
// Check we have enough peers to potentially synchronize
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
return;
// Disregard peers that have no block signature or the same block signature as us
peers.removeIf(hasNoOrSameBlock);
// Disregard peers that are on the same block as last sync attempt and we didn't like their chain
peers.removeIf(hasInferiorChainTip);
final int peersBeforeComparison = peers.size();
// Request recent block summaries from the remaining peers, and locate our common block with each
Synchronizer.getInstance().findCommonBlocksWithPeers(peers);
// Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks
peers = Synchronizer.getInstance().comparePeers(peers);
// We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains
peers.removeIf(hasInferiorChainTip);
final int peersRemoved = peersBeforeComparison - peers.size();
if (peersRemoved > 0 && peers.size() > 0)
LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size()));
if (peers.isEmpty())
return;
if (peers.size() > 1) {
StringBuilder finalPeersString = new StringBuilder();
for (Peer peer : peers)
finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer);
LOGGER.debug(String.format("Choosing random peer from: [%s]", finalPeersString.toString()));
}
// Pick random peer to sync with
int index = new SecureRandom().nextInt(peers.size());
Peer peer = peers.get(index);
actuallySynchronize(peer, false);
}
public SynchronizationResult actuallySynchronize(Peer peer, boolean force) throws InterruptedException {
boolean hasStatusChanged = false;
BlockData priorChainTip = this.getChainTip();
synchronized (this.syncLock) {
this.syncPercent = (priorChainTip.getHeight() * 100) / peer.getChainTipData().getLastHeight();
// Only update SysTray if we're potentially changing height
if (this.syncPercent < 100) {
this.isSynchronizing = true;
hasStatusChanged = true;
}
}
peer.setSyncInProgress(true);
if (hasStatusChanged)
updateSysTray();
try {
SynchronizationResult syncResult = Synchronizer.getInstance().synchronize(peer, force);
switch (syncResult) {
case GENESIS_ONLY:
case NO_COMMON_BLOCK:
case TOO_DIVERGENT:
case INVALID_DATA: {
// These are more serious results that warrant a cool-off
LOGGER.info(String.format("Failed to synchronize with peer %s (%s) - cooling off", peer, syncResult.name()));
// Don't use this peer again for a while
Network.getInstance().peerMisbehaved(peer);
break;
}
case INFERIOR_CHAIN: {
// Update our list of inferior chain tips
ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature());
if (!inferiorChainSignatures.contains(inferiorChainSignature))
inferiorChainSignatures.add(inferiorChainSignature);
// These are minor failure results so fine to try again
LOGGER.debug(() -> String.format("Refused to synchronize with peer %s (%s)", peer, syncResult.name()));
// Notify peer of our superior chain
if (!peer.sendMessage(Network.getInstance().buildHeightMessage(peer, priorChainTip)))
peer.disconnect("failed to notify peer of our superior chain");
break;
}
case NO_REPLY:
case NO_BLOCKCHAIN_LOCK:
case REPOSITORY_ISSUE:
// These are minor failure results so fine to try again
LOGGER.debug(() -> String.format("Failed to synchronize with peer %s (%s)", peer, syncResult.name()));
break;
case SHUTTING_DOWN:
// Just quietly exit
break;
case OK:
// fall-through...
case NOTHING_TO_DO: {
// Update our list of inferior chain tips
ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature());
if (!inferiorChainSignatures.contains(inferiorChainSignature))
inferiorChainSignatures.add(inferiorChainSignature);
LOGGER.debug(() -> String.format("Synchronized with peer %s (%s)", peer, syncResult.name()));
break;
}
}
// Has our chain tip changed?
BlockData newChainTip;
/**
* Export current trade bot states and minting accounts.
*/
public void exportRepositoryData() {
try (final Repository repository = RepositoryManager.getRepository()) {
newChainTip = repository.getBlockRepository().getLastBlock();
repository.exportNodeLocalData();
} catch (DataException e) {
LOGGER.warn(String.format("Repository issue when trying to fetch post-synchronization chain tip: %s", e.getMessage()));
return syncResult;
}
if (!Arrays.equals(newChainTip.getSignature(), priorChainTip.getSignature())) {
// Reset our cache of inferior chains
inferiorChainSignatures.clear();
Network network = Network.getInstance();
network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip));
}
return syncResult;
} finally {
isSynchronizing = false;
peer.setSyncInProgress(false);
// Fail silently as this is an optional step
}
}
private boolean checkRecoveryModeForPeers(List<Peer> qualifiedPeers) {
List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
if (handshakedPeers.size() > 0) {
// There is at least one handshaked peer
if (qualifiedPeers.isEmpty()) {
// There are no 'qualified' peers - i.e. peers that have a recent block we can sync to
boolean werePeersAvailable = peersAvailable;
peersAvailable = false;
// If peers only just became unavailable, update our record of the time they were last available
if (werePeersAvailable)
timePeersLastAvailable = NTP.getTime();
// If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint
if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) {
if (recoveryMode == false) {
LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000));
recoveryMode = true;
}
}
} else {
// We now have at least one peer with a recent block, so we can exit recovery mode and sync normally
peersAvailable = true;
if (recoveryMode) {
LOGGER.info("Peers have become available again. Exiting recovery mode...");
recoveryMode = false;
}
}
}
return recoveryMode;
}
public void addInferiorChainSignature(byte[] inferiorSignature) {
// Update our list of inferior chain tips
ByteArray inferiorChainSignature = new ByteArray(inferiorSignature);
if (!inferiorChainSignatures.contains(inferiorChainSignature))
inferiorChainSignatures.add(inferiorChainSignature);
}
public static class StatusChangeEvent implements Event {
public StatusChangeEvent() {
}
}
private void updateSysTray() {
public void updateSysTray() {
if (NTP.getTime() == null) {
SysTray.getInstance().setToolTipText(Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_CLOCK"));
SysTray.getInstance().setTrayIcon(1);
@ -991,13 +762,13 @@ public class Controller extends Thread {
String actionText;
synchronized (this.syncLock) {
synchronized (Synchronizer.getInstance().syncLock) {
if (this.isMintingPossible) {
actionText = Translator.INSTANCE.translate("SysTray", "MINTING_ENABLED");
SysTray.getInstance().setTrayIcon(2);
}
else if (this.isSynchronizing) {
actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), this.syncPercent);
else if (Synchronizer.getInstance().isSynchronizing()) {
actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent());
SysTray.getInstance().setTrayIcon(3);
}
else if (numberOfPeers < Settings.getInstance().getMinBlockchainPeers()) {
@ -1053,6 +824,121 @@ public class Controller extends Thread {
}
}
// Incoming transactions queue
private boolean incomingTransactionQueueContains(byte[] signature) {
synchronized (incomingTransactions) {
return incomingTransactions.stream().anyMatch(t -> Arrays.equals(t.getSignature(), signature));
}
}
private void removeIncomingTransaction(byte[] signature) {
incomingTransactions.removeIf(t -> Arrays.equals(t.getSignature(), signature));
}
private void processIncomingTransactionsQueue() {
if (this.incomingTransactions.size() == 0) {
// Don't bother locking if there are no new transactions to process
return;
}
if (Synchronizer.getInstance().isSyncRequested() || Synchronizer.getInstance().isSynchronizing()) {
// Prioritize syncing, and don't attempt to lock
return;
}
try {
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (!blockchainLock.tryLock(2, TimeUnit.SECONDS)) {
LOGGER.trace(() -> String.format("Too busy to process incoming transactions queue"));
return;
}
} catch (InterruptedException e) {
LOGGER.info("Interrupted when trying to acquire blockchain lock");
return;
}
try (final Repository repository = RepositoryManager.getRepository()) {
LOGGER.debug("Processing incoming transactions queue (size {})...", this.incomingTransactions.size());
// Take a copy of incomingTransactions so we can release the lock
List<TransactionData>incomingTransactionsCopy = new ArrayList<>(this.incomingTransactions);
// Iterate through incoming transactions list
Iterator iterator = incomingTransactionsCopy.iterator();
while (iterator.hasNext()) {
if (isStopping) {
return;
}
if (Synchronizer.getInstance().isSyncRequestPending()) {
LOGGER.debug("Breaking out of transaction processing loop with {} remaining, because a sync request is pending", incomingTransactionsCopy.size());
return;
}
TransactionData transactionData = (TransactionData) iterator.next();
Transaction transaction = Transaction.fromData(repository, transactionData);
// Check signature
if (!transaction.isSignatureValid()) {
LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature", transactionData.getType().name(), Base58.encode(transactionData.getSignature())));
removeIncomingTransaction(transactionData.getSignature());
continue;
}
ValidationResult validationResult = transaction.importAsUnconfirmed();
if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) {
LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature())));
removeIncomingTransaction(transactionData.getSignature());
continue;
}
if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) {
LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction", Base58.encode(transactionData.getSignature())));
removeIncomingTransaction(transactionData.getSignature());
continue;
}
if (validationResult != ValidationResult.OK) {
final String signature58 = Base58.encode(transactionData.getSignature());
LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58));
Long now = NTP.getTime();
if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) {
Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL;
if (validationResult == ValidationResult.TIMESTAMP_TOO_OLD) {
// Use shorter recheck interval for expired transactions
expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL;
}
Long expiry = now + expiryLength;
LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58);
// Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it
invalidUnconfirmedTransactions.put(signature58, expiry);
}
removeIncomingTransaction(transactionData.getSignature());
continue;
}
LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature())));
removeIncomingTransaction(transactionData.getSignature());
}
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while processing incoming transactions", e));
} finally {
LOGGER.debug("Finished processing incoming transactions queue");
blockchainLock.unlock();
}
}
private void cleanupInvalidTransactionsList(Long now) {
if (now == null) {
return;
}
// Periodically remove invalid unconfirmed transactions from the list, so that they can be fetched again
invalidUnconfirmedTransactions.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < now);
}
// Shutdown
public void shutdown() {
@ -1060,6 +946,9 @@ public class Controller extends Thread {
if (!isStopping) {
isStopping = true;
LOGGER.info("Shutting down synchronizer");
Synchronizer.getInstance().shutdown();
LOGGER.info("Shutting down API");
ApiService.getInstance().stop();
@ -1071,6 +960,7 @@ public class Controller extends Thread {
// Arbitrary data controllers
LOGGER.info("Shutting down arbitrary-transaction controllers");
ArbitraryDataManager.getInstance().shutdown();
ArbitraryDataFileManager.getInstance().shutdown();
ArbitraryDataBuildManager.getInstance().shutdown();
ArbitraryDataCleanupManager.getInstance().shutdown();
ArbitraryDataStorageManager.getInstance().shutdown();
@ -1101,6 +991,17 @@ public class Controller extends Thread {
// We were interrupted while waiting for thread to join
}
// Make sure we're the only thread modifying the blockchain when shutting down the repository
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
try {
if (!blockchainLock.tryLock(5, TimeUnit.SECONDS)) {
LOGGER.debug("Couldn't acquire blockchain lock even after waiting 5 seconds");
// Proceed anyway, as we have to shut down
}
} catch (InterruptedException e) {
LOGGER.info("Interrupted when waiting for blockchain lock");
}
try {
LOGGER.info("Shutting down repository");
RepositoryManager.closeRepositoryFactory();
@ -1108,6 +1009,11 @@ public class Controller extends Thread {
LOGGER.error("Error occurred while shutting down repository", e);
}
// Release the lock if we acquired it
if (blockchainLock.isHeldByCurrentThread()) {
blockchainLock.unlock();
}
LOGGER.info("Shutting down NTP");
NTP.shutdownNow();
@ -1507,50 +1413,10 @@ public class Controller extends Thread {
private void onNetworkTransactionMessage(Peer peer, Message message) {
TransactionMessage transactionMessage = (TransactionMessage) message;
TransactionData transactionData = transactionMessage.getTransactionData();
/*
* If we can't obtain blockchain lock immediately,
* e.g. Synchronizer is active, or another transaction is taking a while to validate,
* then we're using up a network thread for ages and clogging things up
* so bail out early
*/
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (!blockchainLock.tryLock()) {
LOGGER.trace(() -> String.format("Too busy to import %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer));
return;
if (this.incomingTransactions.size() < MAX_INCOMING_TRANSACTIONS) {
if (!this.incomingTransactions.contains(transactionData)) {
this.incomingTransactions.add(transactionData);
}
try (final Repository repository = RepositoryManager.getRepository()) {
Transaction transaction = Transaction.fromData(repository, transactionData);
// Check signature
if (!transaction.isSignatureValid()) {
LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer));
return;
}
ValidationResult validationResult = transaction.importAsUnconfirmed();
if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) {
LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer));
return;
}
if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) {
LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer));
return;
}
if (validationResult != ValidationResult.OK) {
LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s from peer %s", validationResult.name(), transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer));
return;
}
LOGGER.debug(() -> String.format("Imported %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer));
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while processing transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer), e);
} finally {
blockchainLock.unlock();
}
}
@ -1702,7 +1568,7 @@ public class Controller extends Thread {
peer.setChainTipData(newChainTipData);
// Potentially synchronize
requestSync = true;
Synchronizer.getInstance().requestSync();
}
private void onNetworkGetTransactionMessage(Peer peer, Message message) {
@ -1749,6 +1615,19 @@ public class Controller extends Thread {
try (final Repository repository = RepositoryManager.getRepository()) {
for (byte[] signature : signatures) {
String signature58 = Base58.encode(signature);
if (invalidUnconfirmedTransactions.containsKey(signature58)) {
// Previously invalid transaction - don't keep requesting it
// It will be periodically removed from invalidUnconfirmedTransactions to allow for rechecks
continue;
}
// Ignore if this transaction is in the queue
if (incomingTransactionQueueContains(signature)) {
LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peer));
continue;
}
// Do we have it already? (Before requesting transaction data itself)
if (repository.getTransactionRepository().exists(signature)) {
LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer));
@ -1995,8 +1874,7 @@ public class Controller extends Thread {
private void sendOurOnlineAccountsInfo() {
final Long now = NTP.getTime();
if (now == null)
return;
if (now != null) {
List<MintingAccountData> mintingAccounts;
try (final Repository repository = RepositoryManager.getRepository()) {
@ -2008,6 +1886,7 @@ public class Controller extends Thread {
// Only reward-share accounts allowed
Iterator<MintingAccountData> iterator = mintingAccounts.iterator();
int i = 0;
while (iterator.hasNext()) {
MintingAccountData mintingAccountData = iterator.next();
@ -2024,6 +1903,11 @@ public class Controller extends Thread {
iterator.remove();
continue;
}
if (++i > 2) {
iterator.remove();
continue;
}
}
} catch (DataException e) {
LOGGER.warn(String.format("Repository issue trying to fetch minting accounts: %s", e.getMessage()));
@ -2082,6 +1966,7 @@ public class Controller extends Thread {
LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp));
}
}
public static long toOnlineAccountTimestamp(long timestamp) {
return (timestamp / ONLINE_TIMESTAMP_MODULUS) * ONLINE_TIMESTAMP_MODULUS;

330
src/main/java/org/qortal/controller/Synchronizer.java
View File

@ -1,9 +1,11 @@
package org.qortal.controller;
import java.math.BigInteger;
import java.security.SecureRandom;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;
@ -20,6 +22,7 @@ import org.qortal.data.block.CommonBlockData;
import org.qortal.data.network.PeerChainTipData;
import org.qortal.data.transaction.RewardShareTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.network.Network;
import org.qortal.network.Peer;
import org.qortal.network.message.BlockMessage;
import org.qortal.network.message.BlockSummariesMessage;
@ -35,11 +38,10 @@ import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.transaction.Transaction;
import org.qortal.utils.Base58;
import org.qortal.utils.ByteArray;
import org.qortal.utils.NTP;
import static org.qortal.network.Peer.FETCH_BLOCKS_TIMEOUT;
public class Synchronizer {
public class Synchronizer extends Thread {
private static final Logger LOGGER = LogManager.getLogger(Synchronizer.class);
@ -57,12 +59,32 @@ public class Synchronizer {
/** Maximum number of block signatures we ask from peer in one go */
private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings?
private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms
private boolean running;
/** Latest block signatures from other peers that we know are on inferior chains. */
List<ByteArray> inferiorChainSignatures = new ArrayList<>();
/** Recovery mode, which is used to bring back a stalled network */
private boolean recoveryMode = false;
private boolean peersAvailable = true; // peersAvailable must default to true
private long timePeersLastAvailable = 0;
// Keep track of the size of the last re-org, so it can be logged
private int lastReorgSize;
/** Synchronization object for sync variables below */
public final Object syncLock = new Object();
/** Whether we are attempting to synchronize. */
private volatile boolean isSynchronizing = false;
/** Temporary estimate of synchronization progress for SysTray use. */
private volatile int syncPercent = 0;
private static volatile boolean requestSync = false;
private boolean syncRequestPending = false;
// Keep track of invalid blocks so that we don't keep trying to sync them
private Map<String, Long> invalidBlockSignatures = Collections.synchronizedMap(new HashMap<>());
public Long timeValidBlockLastReceived = null;
@ -77,6 +99,7 @@ public class Synchronizer {
// Constructors
private Synchronizer() {
this.running = true;
}
public static Synchronizer getInstance() {
@ -87,6 +110,284 @@ public class Synchronizer {
}
@Override
public void run() {
Thread.currentThread().setName("Synchronizer");
try {
while (running && !Controller.isStopping()) {
Thread.sleep(1000);
if (requestSync) {
requestSync = false;
boolean success = Synchronizer.getInstance().potentiallySynchronize();
if (!success) {
// Something went wrong, so try again next time
requestSync = true;
}
// Remember that we have a pending sync request if this attempt failed
syncRequestPending = !success;
}
}
} catch (InterruptedException e) {
// Clear interrupted flag so we can shutdown trim threads
Thread.interrupted();
// Fall-through to exit
}
}
public void shutdown() {
this.running = false;
this.interrupt();
}
public boolean isSynchronizing() {
return this.isSynchronizing;
}
public boolean isSyncRequestPending() {
return this.syncRequestPending;
}
public Integer getSyncPercent() {
synchronized (this.syncLock) {
return this.isSynchronizing ? this.syncPercent : null;
}
}
public void requestSync() {
requestSync = true;
}
public boolean isSyncRequested() {
return requestSync;
}
public boolean getRecoveryMode() {
return this.recoveryMode;
}
public boolean potentiallySynchronize() throws InterruptedException {
// Already synchronizing via another thread?
if (this.isSynchronizing)
return true;
List<Peer> peers = Network.getInstance().getHandshakedPeers();
// Disregard peers that have "misbehaved" recently
peers.removeIf(Controller.hasMisbehaved);
// Disregard peers that only have genesis block
peers.removeIf(Controller.hasOnlyGenesisBlock);
// Disregard peers that don't have a recent block
peers.removeIf(Controller.hasNoRecentBlock);
// Disregard peers that are on an old version
peers.removeIf(Controller.hasOldVersion);
checkRecoveryModeForPeers(peers);
if (recoveryMode) {
peers = Network.getInstance().getHandshakedPeers();
peers.removeIf(Controller.hasOnlyGenesisBlock);
peers.removeIf(Controller.hasMisbehaved);
peers.removeIf(Controller.hasOldVersion);
}
// Check we have enough peers to potentially synchronize
if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
return true;
// Disregard peers that have no block signature or the same block signature as us
peers.removeIf(Controller.hasNoOrSameBlock);
// Disregard peers that are on the same block as last sync attempt and we didn't like their chain
peers.removeIf(Controller.hasInferiorChainTip);
final int peersBeforeComparison = peers.size();
// Request recent block summaries from the remaining peers, and locate our common block with each
Synchronizer.getInstance().findCommonBlocksWithPeers(peers);
// Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks
peers = Synchronizer.getInstance().comparePeers(peers);
// We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains
peers.removeIf(Controller.hasInferiorChainTip);
final int peersRemoved = peersBeforeComparison - peers.size();
if (peersRemoved > 0 && peers.size() > 0)
LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size()));
if (peers.isEmpty())
return true;
if (peers.size() > 1) {
StringBuilder finalPeersString = new StringBuilder();
for (Peer peer : peers)
finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer);
LOGGER.debug(String.format("Choosing random peer from: [%s]", finalPeersString.toString()));
}
// Pick random peer to sync with
int index = new SecureRandom().nextInt(peers.size());
Peer peer = peers.get(index);
SynchronizationResult syncResult = actuallySynchronize(peer, false);
if (syncResult == SynchronizationResult.NO_BLOCKCHAIN_LOCK) {
// No blockchain lock - force a retry by returning false
return false;
}
return true;
}
public SynchronizationResult actuallySynchronize(Peer peer, boolean force) throws InterruptedException {
boolean hasStatusChanged = false;
BlockData priorChainTip = Controller.getInstance().getChainTip();
synchronized (this.syncLock) {
this.syncPercent = (priorChainTip.getHeight() * 100) / peer.getChainTipData().getLastHeight();
// Only update SysTray if we're potentially changing height
if (this.syncPercent < 100) {
this.isSynchronizing = true;
hasStatusChanged = true;
}
}
peer.setSyncInProgress(true);
if (hasStatusChanged)
Controller.getInstance().updateSysTray();
try {
SynchronizationResult syncResult = Synchronizer.getInstance().synchronize(peer, force);
switch (syncResult) {
case GENESIS_ONLY:
case NO_COMMON_BLOCK:
case TOO_DIVERGENT:
case INVALID_DATA: {
// These are more serious results that warrant a cool-off
LOGGER.info(String.format("Failed to synchronize with peer %s (%s) - cooling off", peer, syncResult.name()));
// Don't use this peer again for a while
Network.getInstance().peerMisbehaved(peer);
break;
}
case INFERIOR_CHAIN: {
// Update our list of inferior chain tips
ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature());
if (!inferiorChainSignatures.contains(inferiorChainSignature))
inferiorChainSignatures.add(inferiorChainSignature);
// These are minor failure results so fine to try again
LOGGER.debug(() -> String.format("Refused to synchronize with peer %s (%s)", peer, syncResult.name()));
// Notify peer of our superior chain
if (!peer.sendMessage(Network.getInstance().buildHeightMessage(peer, priorChainTip)))
peer.disconnect("failed to notify peer of our superior chain");
break;
}
case NO_REPLY:
case NO_BLOCKCHAIN_LOCK:
case REPOSITORY_ISSUE:
// These are minor failure results so fine to try again
LOGGER.debug(() -> String.format("Failed to synchronize with peer %s (%s)", peer, syncResult.name()));
break;
case SHUTTING_DOWN:
// Just quietly exit
break;
case OK:
// fall-through...
case NOTHING_TO_DO: {
// Update our list of inferior chain tips
ByteArray inferiorChainSignature = new ByteArray(peer.getChainTipData().getLastBlockSignature());
if (!inferiorChainSignatures.contains(inferiorChainSignature))
inferiorChainSignatures.add(inferiorChainSignature);
LOGGER.debug(() -> String.format("Synchronized with peer %s (%s)", peer, syncResult.name()));
break;
}
}
if (!running) {
// We've stopped
return SynchronizationResult.SHUTTING_DOWN;
}
// Has our chain tip changed?
BlockData newChainTip;
try (final Repository repository = RepositoryManager.getRepository()) {
newChainTip = repository.getBlockRepository().getLastBlock();
} catch (DataException e) {
LOGGER.warn(String.format("Repository issue when trying to fetch post-synchronization chain tip: %s", e.getMessage()));
return syncResult;
}
if (!Arrays.equals(newChainTip.getSignature(), priorChainTip.getSignature())) {
// Reset our cache of inferior chains
inferiorChainSignatures.clear();
Network network = Network.getInstance();
network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip));
}
return syncResult;
} finally {
this.isSynchronizing = false;
peer.setSyncInProgress(false);
}
}
private boolean checkRecoveryModeForPeers(List<Peer> qualifiedPeers) {
List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
if (handshakedPeers.size() > 0) {
// There is at least one handshaked peer
if (qualifiedPeers.isEmpty()) {
// There are no 'qualified' peers - i.e. peers that have a recent block we can sync to
boolean werePeersAvailable = peersAvailable;
peersAvailable = false;
// If peers only just became unavailable, update our record of the time they were last available
if (werePeersAvailable)
timePeersLastAvailable = NTP.getTime();
// If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint
if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) {
if (recoveryMode == false) {
LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000));
recoveryMode = true;
}
}
} else {
// We now have at least one peer with a recent block, so we can exit recovery mode and sync normally
peersAvailable = true;
if (recoveryMode) {
LOGGER.info("Peers have become available again. Exiting recovery mode...");
recoveryMode = false;
}
}
}
return recoveryMode;
}
public void addInferiorChainSignature(byte[] inferiorSignature) {
// Update our list of inferior chain tips
ByteArray inferiorChainSignature = new ByteArray(inferiorSignature);
if (!inferiorChainSignatures.contains(inferiorChainSignature))
inferiorChainSignatures.add(inferiorChainSignature);
}
/**
* Iterate through a list of supplied peers, and attempt to find our common block with each.
* If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later.
@ -259,6 +560,8 @@ public class Synchronizer {
// Create a placeholder to track of common blocks that we can discard due to being inferior chains
int dropPeersAfterCommonBlockHeight = 0;
NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
// Remove peers with no common block data
Iterator iterator = peers.iterator();
while (iterator.hasNext()) {
@ -279,7 +582,7 @@ public class Synchronizer {
// We have already determined that the correct chain diverged from a lower height. We are safe to skip these peers.
for (Peer peer : peersSharingCommonBlock) {
LOGGER.debug(String.format("Peer %s has common block at height %d but the superior chain is at height %d. Removing it from this round.", peer, commonBlockSummary.getHeight(), dropPeersAfterCommonBlockHeight));
Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature());
this.addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature());
}
continue;
}
@ -381,9 +684,7 @@ public class Synchronizer {
if (ourBlockSummaries.size() > 0)
ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, maxHeightForChainWeightComparisons);
NumberFormat formatter = new DecimalFormat("0.###E0");
NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", (usingSameLengthChainWeight ? minChainLength : ourBlockSummaries.size()), formatter.format(ourChainWeight)));
LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", (usingSameLengthChainWeight ? minChainLength : ourBlockSummaries.size()), accurateFormatter.format(ourChainWeight)));
LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature())));
for (Peer peer : peersSharingCommonBlock) {
@ -405,7 +706,7 @@ public class Synchronizer {
LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock));
BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, maxHeightForChainWeightComparisons);
peer.getCommonBlockData().setChainWeight(peerChainWeight);
LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight)));
LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, (usingSameLengthChainWeight ? minChainLength : peerBlockSummariesAfterCommonBlock.size()), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), accurateFormatter.format(peerChainWeight)));
// Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group)
if (ourChainWeight.compareTo(peerChainWeight) > 0) {
@ -571,9 +872,11 @@ public class Synchronizer {
// Make sure we're the only thread modifying the blockchain
// If we're already synchronizing with another peer then this will also return fast
ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
if (!blockchainLock.tryLock())
if (!blockchainLock.tryLock(3, TimeUnit.SECONDS)) {
// Wasn't peer's fault we couldn't sync
LOGGER.info("Synchronizer couldn't acquire blockchain lock");
return SynchronizationResult.NO_BLOCKCHAIN_LOCK;
}
try {
try (final Repository repository = RepositoryManager.getRepository()) {
@ -793,7 +1096,7 @@ public class Synchronizer {
return SynchronizationResult.REPOSITORY_ISSUE;
if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
LOGGER.info(String.format("Ditching our chain after height %d as our latest block is very old", commonBlockHeight));
LOGGER.info(String.format("Ditching our chain after height %d", commonBlockHeight));
} else {
// Compare chain weights
@ -853,8 +1156,9 @@ public class Synchronizer {
BigInteger ourChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight);
BigInteger peerChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, peerBlockSummaries, mutualHeight);
NumberFormat formatter = new DecimalFormat("0.###E0");
LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", formatter.format(ourChainWeight), formatter.format(peerChainWeight)));
NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
LOGGER.debug(String.format("commonBlockHeight: %d, commonBlockSig: %.8s, ourBlockSummaries.size(): %d, peerBlockSummaries.size(): %d", commonBlockHeight, Base58.encode(commonBlockSig), ourBlockSummaries.size(), peerBlockSummaries.size()));
LOGGER.debug(String.format("Our chain weight: %s, peer's chain weight: %s (higher is better)", accurateFormatter.format(ourChainWeight), accurateFormatter.format(peerChainWeight)));
// If our blockchain has greater weight then don't synchronize with peer
if (ourChainWeight.compareTo(peerChainWeight) >= 0) {
@ -1222,7 +1526,7 @@ public class Synchronizer {
return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates());
}
private void populateBlockSummariesMinterLevels(Repository repository, List<BlockSummaryData> blockSummaries) throws DataException {
public void populateBlockSummariesMinterLevels(Repository repository, List<BlockSummaryData> blockSummaries) throws DataException {
final int firstBlockHeight = blockSummaries.get(0).getHeight();
for (int i = 0; i < blockSummaries.size(); ++i) {

24
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java
View File

@ -37,11 +37,16 @@ public class ArbitraryDataBuildManager extends Thread {
@Override
public void run() {
Thread.currentThread().setName("Arbitrary Data Build Manager");
try {
// Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
// This can be expanded to have multiple threads processing the build queue when needed
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
int threadCount = 5;
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(threadCount);
for (int i = 0; i < threadCount; i++) {
arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
}
while (!isStopping) {
// Nothing to do yet
@ -101,7 +106,7 @@ public class ArbitraryDataBuildManager extends Thread {
return true;
}
LOGGER.info("Added {} to build queue", queueItem);
log(queueItem, String.format("Added %s to build queue", queueItem));
// Added to queue
return true;
@ -149,7 +154,7 @@ public class ArbitraryDataBuildManager extends Thread {
return true;
}
LOGGER.info("Added {} to failed builds list", queueItem);
log(queueItem, String.format("Added %s to failed builds list", queueItem));
// Added to queue
return true;
@ -182,4 +187,17 @@ public class ArbitraryDataBuildManager extends Thread {
public boolean getBuildInProgress() {
return this.buildInProgress;
}
private void log(ArbitraryDataBuildQueueItem queueItem, String message) {
if (queueItem == null) {
return;
}
if (queueItem.isHighPriority()) {
LOGGER.info(message);
}
else {
LOGGER.debug(message);
}
}
}

58
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java
View File

@ -9,6 +9,7 @@ import org.qortal.repository.DataException;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.util.Comparator;
import java.util.Map;
@ -20,13 +21,14 @@ public class ArbitraryDataBuilderThread implements Runnable {
}
@Override
public void run() {
Thread.currentThread().setName("Arbitrary Data Build Manager");
Thread.currentThread().setName("Arbitrary Data Builder Thread");
ArbitraryDataBuildManager buildManager = ArbitraryDataBuildManager.getInstance();
while (!Controller.isStopping()) {
try {
Thread.sleep(1000);
Thread.sleep(100);
if (buildManager.arbitraryDataBuildQueue == null) {
continue;
@ -35,48 +37,57 @@ public class ArbitraryDataBuilderThread implements Runnable {
continue;
}
// Find resources that are queued for building
Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
.entrySet().stream()
.filter(e -> e.getValue().isQueued())
.findFirst().get();
if (next == null) {
continue;
}
Long now = NTP.getTime();
if (now == null) {
continue;
}
ArbitraryDataBuildQueueItem queueItem = next.getValue();
ArbitraryDataBuildQueueItem queueItem = null;
// Find resources that are queued for building (sorted by highest priority first)
synchronized (buildManager.arbitraryDataBuildQueue) {
Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
.entrySet().stream()
.filter(e -> e.getValue().isQueued())
.sorted(Comparator.comparing(item -> item.getValue().getPriority()))
.reduce((first, second) -> second).orElse(null);
if (next == null) {
continue;
}
queueItem = next.getValue();
if (queueItem == null) {
this.removeFromQueue(queueItem);
continue;
}
// Ignore builds that have failed recently
if (buildManager.isInFailedBuildsList(queueItem)) {
this.removeFromQueue(queueItem);
continue;
}
// Set the start timestamp, to prevent other threads from building it at the same time
queueItem.prepareForBuild();
}
try {
// Perform the build
LOGGER.info("Building {}...", queueItem);
log(queueItem, String.format("Building %s... priority: %d", queueItem, queueItem.getPriority()));
queueItem.build();
this.removeFromQueue(queueItem);
LOGGER.info("Finished building {}", queueItem);
log(queueItem, String.format("Finished building %s", queueItem));
} catch (MissingDataException e) {
LOGGER.info("Missing data for {}: {}", queueItem, e.getMessage());
log(queueItem, String.format("Missing data for %s: %s", queueItem, e.getMessage()));
queueItem.setFailed(true);
this.removeFromQueue(queueItem);
// Don't add to the failed builds list, as we may want to retry sooner
} catch (IOException | DataException | RuntimeException e) {
LOGGER.info("Error building {}: {}", queueItem, e.getMessage());
log(queueItem, String.format("Error building %s: %s", queueItem, e.getMessage()));
// Something went wrong - so remove it from the queue, and add to failed builds list
queueItem.setFailed(true);
buildManager.addToFailedBuildsList(queueItem);
@ -95,4 +106,17 @@ public class ArbitraryDataBuilderThread implements Runnable {
}
ArbitraryDataBuildManager.getInstance().arbitraryDataBuildQueue.remove(queueItem.getUniqueKey());
}
private void log(ArbitraryDataBuildQueueItem queueItem, String message) {
if (queueItem == null) {
return;
}
if (queueItem.isHighPriority()) {
LOGGER.info(message);
}
else {
LOGGER.debug(message);
}
}
}

32
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@ -108,6 +108,10 @@ public class ArbitraryDataCleanupManager extends Thread {
try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
// LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
if (isStopping) {
return;
}
if (signatures == null || signatures.isEmpty()) {
offset = 0;
continue;
@ -117,6 +121,10 @@ public class ArbitraryDataCleanupManager extends Thread {
// Loop through the signatures in this batch
for (int i=0; i<signatures.size(); i++) {
if (isStopping) {
return;
}
byte[] signature = signatures.get(i);
if (signature == null) {
continue;
@ -231,6 +239,9 @@ public class ArbitraryDataCleanupManager extends Thread {
// Delete random data associated with name if we're over our storage limit for this name
// Use the DELETION_THRESHOLD, for the same reasons as above
for (String followedName : storageManager.followedNames()) {
if (isStopping) {
return;
}
if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) {
this.storageLimitReachedForName(repository, followedName);
}
@ -253,6 +264,9 @@ public class ArbitraryDataCleanupManager extends Thread {
// Loop through each path and find those without matching signatures
for (Path path : allPaths) {
if (isStopping) {
break;
}
try {
String[] contents = path.toFile().list();
if (contents == null || contents.length == 0) {
@ -279,6 +293,9 @@ public class ArbitraryDataCleanupManager extends Thread {
private void checkForExpiredTransactions(Repository repository) {
List<Path> expiredPaths = this.findPathsWithNoAssociatedTransaction(repository);
for (Path expiredPath : expiredPaths) {
if (isStopping) {
return;
}
LOGGER.info("Found path with no associated transaction: {}", expiredPath.toString());
this.safeDeleteDirectory(expiredPath.toFile(), "no matching transaction");
}
@ -300,6 +317,9 @@ public class ArbitraryDataCleanupManager extends Thread {
// when they reach their storage limit
Path dataPath = Paths.get(Settings.getInstance().getDataPath());
for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
if (isStopping) {
return;
}
this.deleteRandomFile(repository, dataPath.toFile(), null);
}
@ -318,6 +338,9 @@ public class ArbitraryDataCleanupManager extends Thread {
// when they reach their storage limit
Path dataPath = Paths.get(Settings.getInstance().getDataPath());
for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
if (isStopping) {
return;
}
this.deleteRandomFile(repository, dataPath.toFile(), name);
}
}
@ -429,6 +452,9 @@ public class ArbitraryDataCleanupManager extends Thread {
final File[] directories = tempDir.toFile().listFiles();
if (directories != null) {
for (final File directory : directories) {
if (isStopping) {
return;
}
contentsCount++;
// We're expecting the contents of each subfolder to be a directory
@ -464,6 +490,9 @@ public class ArbitraryDataCleanupManager extends Thread {
final File[] directories = readerCacheNamesPath.toFile().listFiles();
if (directories != null) {
for (final File directory : directories) {
if (isStopping) {
return;
}
// Delete data relating to blocked names
String name = directory.getName();
@ -489,6 +518,9 @@ public class ArbitraryDataCleanupManager extends Thread {
final File[] directories = readerNameCachePath.toFile().listFiles();
if (directories != null) {
for (final File directory : directories) {
if (isStopping) {
return;
}
// Each directory is a "service" type
String service = directory.getName();
this.cleanupReaderCacheForNameAndService(name, service, now);

137
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
View File

@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataFileChunk;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryRelayInfo;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.network.Network;
@ -59,7 +60,7 @@ public class ArbitraryDataFileListManager {
/** Maximum number of seconds that a file list relay request is able to exist on the network */
private static long RELAY_REQUEST_MAX_DURATION = 5000L;
/** Maximum number of hops that a file list relay request is allowed to make */
private static int RELAY_REQUEST_MAX_HOPS = 3;
private static int RELAY_REQUEST_MAX_HOPS = 4;
private ArbitraryDataFileListManager() {
@ -236,9 +237,11 @@ public class ArbitraryDataFileListManager {
}
// Lookup file lists by signature
// Lookup file lists by signature (and optionally hashes)
public boolean fetchArbitraryDataFileList(ArbitraryTransactionData arbitraryTransactionData) {
byte[] digest = arbitraryTransactionData.getData();
byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
byte[] signature = arbitraryTransactionData.getSignature();
String signature58 = Base58.encode(signature);
@ -261,10 +264,22 @@ public class ArbitraryDataFileListManager {
this.addToSignatureRequests(signature58, true, false);
List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
LOGGER.debug(String.format("Sending data file list request for signature %s to %d peers...", signature58, handshakedPeers.size()));
List<byte[]> missingHashes = null;
// Find hashes that we are missing
try {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
arbitraryDataFile.setMetadataHash(metadataHash);
missingHashes = arbitraryDataFile.missingHashes();
} catch (DataException e) {
// Leave missingHashes as null, so that all hashes are requested
}
int hashCount = missingHashes != null ? missingHashes.size() : 0;
LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size()));
// Build request
Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, now, 0);
Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, missingHashes, now, 0);
// Save our request into requests map
Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
@ -304,6 +319,64 @@ public class ArbitraryDataFileListManager {
return true;
}
public boolean fetchArbitraryDataFileList(Peer peer, byte[] signature) {
String signature58 = Base58.encode(signature);
// Require an NTP sync
Long now = NTP.getTime();
if (now == null) {
return false;
}
int hashCount = 0;
LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to peer %s...", signature58, hashCount, peer));
// Build request
// Use a time in the past, so that the recipient peer doesn't try and relay it
// Also, set hashes to null since it's easier to request all hashes than it is to determine which ones we need
// This could be optimized in the future
long timestamp = now - 60000L;
List<byte[]> hashes = null;
Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, timestamp, 0);
// Save our request into requests map
Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
// Assign random ID to this message
int id;
do {
id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
// Put queue into map (keyed by message ID) so we can poll for a response
// If putIfAbsent() doesn't return null, then this ID is already taken
} while (arbitraryDataFileListRequests.put(id, requestEntry) != null);
getArbitraryDataFileListMessage.setId(id);
// Send the request
peer.sendMessage(getArbitraryDataFileListMessage);
// Poll to see if data has arrived
final long singleWait = 100;
long totalWait = 0;
while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
try {
Thread.sleep(singleWait);
} catch (InterruptedException e) {
break;
}
requestEntry = arbitraryDataFileListRequests.get(id);
if (requestEntry == null)
return false;
if (requestEntry.getA() == null)
break;
totalWait += singleWait;
}
return true;
}
public void deleteFileListRequestsForSignature(byte[] signature) {
String signature58 = Base58.encode(signature);
for (Iterator<Map.Entry<Integer, Triple<String, Peer, Long>>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) {
@ -377,6 +450,14 @@ public class ArbitraryDataFileListManager {
// }
if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) {
// Keep track of the hashes this peer reports to have access to
Long now = NTP.getTime();
for (byte[] hash : hashes) {
String hash58 = Base58.encode(hash);
String sig58 = Base58.encode(signature);
ArbitraryDataFileManager.getInstance().arbitraryDataFileHashResponses.put(hash58, new Triple<>(peer, sig58, now));
}
// Go and fetch the actual data, since this isn't a relay request
arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, hashes);
}
@ -395,10 +476,8 @@ public class ArbitraryDataFileListManager {
Long now = NTP.getTime();
for (byte[] hash : hashes) {
String hash58 = Base58.encode(hash);
Triple<String, Peer, Long> value = new Triple<>(signature58, peer, now);
if (arbitraryDataFileManager.arbitraryRelayMap.putIfAbsent(hash58, value) == null) {
LOGGER.debug("Added {} to relay map: {}, {}, {}", hash58, signature58, peer, now);
}
ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now);
ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap);
}
// Forward to requesting peer
@ -422,6 +501,7 @@ public class ArbitraryDataFileListManager {
GetArbitraryDataFileListMessage getArbitraryDataFileListMessage = (GetArbitraryDataFileListMessage) message;
byte[] signature = getArbitraryDataFileListMessage.getSignature();
String signature58 = Base58.encode(signature);
List<byte[]> requestedHashes = getArbitraryDataFileListMessage.getHashes();
Long now = NTP.getTime();
Triple<String, Peer, Long> newEntry = new Triple<>(signature58, peer, now);
@ -451,21 +531,32 @@ public class ArbitraryDataFileListManager {
// Load file(s) and add any that exist to the list of hashes
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
if (metadataHash != null) {
arbitraryDataFile.setMetadataHash(metadataHash);
// If the peer didn't supply a hash list, we need to return all hashes for this transaction
if (requestedHashes == null || requestedHashes.isEmpty()) {
requestedHashes = new ArrayList<>();
// Add the metadata file
if (arbitraryDataFile.getMetadataHash() != null) {
requestedHashes.add(arbitraryDataFile.getMetadataHash());
}
// Add the chunk hashes
if (arbitraryDataFile.getChunkHashes().size() > 0) {
requestedHashes.addAll(arbitraryDataFile.getChunkHashes());
}
// Add complete file if there are no hashes
else {
requestedHashes.add(arbitraryDataFile.getHash());
}
}
// Assume all chunks exists, unless one can't be found below
allChunksExist = true;
// If we have the metadata file, add its hash
if (arbitraryDataFile.getMetadataFile().exists()) {
hashes.add(arbitraryDataFile.getMetadataHash());
}
else {
allChunksExist = false;
}
for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) {
for (byte[] requestedHash : requestedHashes) {
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(requestedHash, signature);
if (chunk.exists()) {
hashes.add(chunk.getHash());
//LOGGER.trace("Added hash {}", chunk.getHash58());
@ -474,16 +565,6 @@ public class ArbitraryDataFileListManager {
allChunksExist = false;
}
}
} else {
// This transaction has no chunks, so include the complete file if we have it
if (arbitraryDataFile.exists()) {
hashes.add(arbitraryDataFile.getHash());
allChunksExist = true;
}
else {
allChunksExist = false;
}
}
}
}

180
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java
View File

@ -4,10 +4,10 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryRelayInfo;
import org.qortal.data.network.ArbitraryPeerData;
import org.qortal.data.network.PeerData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.network.Network;
import org.qortal.network.Peer;
import org.qortal.network.message.*;
@ -22,13 +22,16 @@ import org.qortal.utils.Triple;
import java.security.SecureRandom;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
public class ArbitraryDataFileManager {
public class ArbitraryDataFileManager extends Thread {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileManager.class);
private static ArbitraryDataFileManager instance;
private volatile boolean isStopping = false;
/**
@ -37,10 +40,16 @@ public class ArbitraryDataFileManager {
private Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
/**
* Map to keep track of hashes that we might need to relay, keyed by the hash of the file (base58 encoded).
* Value is comprised of the base58-encoded signature, the peer that is hosting it, and the timestamp that it was added
* Map to keep track of hashes that we might need to relay
*/
public Map<String, Triple<String, Peer, Long>> arbitraryRelayMap = Collections.synchronizedMap(new HashMap<>());
public List<ArbitraryRelayInfo> arbitraryRelayMap = Collections.synchronizedList(new ArrayList<>());
/**
* Map to keep track of any arbitrary data file hash responses
* Key: string - the hash encoded in base58
* Value: Triple<respondingPeer, signature58, timeResponded>
*/
public Map<String, Triple<Peer, String, Long>> arbitraryDataFileHashResponses = Collections.synchronizedMap(new HashMap<>());
private ArbitraryDataFileManager() {
@ -53,6 +62,32 @@ public class ArbitraryDataFileManager {
return instance;
}
@Override
public void run() {
Thread.currentThread().setName("Arbitrary Data File Manager");
try {
// Use a fixed thread pool to execute the arbitrary data file requests
int threadCount = 10;
ExecutorService arbitraryDataFileRequestExecutor = Executors.newFixedThreadPool(threadCount);
for (int i = 0; i < threadCount; i++) {
arbitraryDataFileRequestExecutor.execute(new ArbitraryDataFileRequestThread());
}
while (!isStopping) {
// Nothing to do yet
Thread.sleep(1000);
}
} catch (InterruptedException e) {
// Fall-through to exit thread...
}
}
public void shutdown() {
isStopping = true;
this.interrupt();
}
public void cleanupRequestCache(Long now) {
if (now == null) {
@ -62,29 +97,14 @@ public class ArbitraryDataFileManager {
arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() == null || entry.getValue() < requestMinimumTimestamp);
final long relayMinimumTimestamp = now - ArbitraryDataManager.getInstance().ARBITRARY_RELAY_TIMEOUT;
arbitraryRelayMap.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp);
arbitraryRelayMap.removeIf(entry -> entry == null || entry.getTimestamp() == null || entry.getTimestamp() < relayMinimumTimestamp);
arbitraryDataFileHashResponses.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < relayMinimumTimestamp);
}
// Fetch data files by hash
public boolean fetchAllArbitraryDataFiles(Repository repository, Peer peer, byte[] signature) {
try {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (!(transactionData instanceof ArbitraryTransactionData))
return false;
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
// We use null to represent all hashes associated with this transaction
return this.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, null);
} catch (DataException e) {}
return false;
}
public boolean fetchArbitraryDataFiles(Repository repository,
Peer peer,
byte[] signature,
@ -95,43 +115,46 @@ public class ArbitraryDataFileManager {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData(), signature);
byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
arbitraryDataFile.setMetadataHash(metadataHash);
// If hashes are null, we will treat this to mean all data hashes associated with this file
if (hashes == null) {
if (metadataHash == null) {
// This transaction has no metadata/chunks, so use the main file hash
hashes = Arrays.asList(arbitraryDataFile.getHash());
}
else if (!arbitraryDataFile.getMetadataFile().exists()) {
// We don't have the metadata file yet, so request it
hashes = Arrays.asList(arbitraryDataFile.getMetadataFile().getHash());
}
else {
// Add the chunk hashes
hashes = arbitraryDataFile.getChunkHashes();
}
}
boolean receivedAtLeastOneFile = false;
// Now fetch actual data from this peer
for (byte[] hash : hashes) {
if (isStopping) {
return false;
}
String hash58 = Base58.encode(hash);
if (!arbitraryDataFile.chunkExists(hash)) {
// Only request the file if we aren't already requesting it from someone else
if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
LOGGER.debug("Requesting data file {} from peer {}", hash58, peer);
Long startTime = NTP.getTime();
ArbitraryDataFileMessage receivedArbitraryDataFileMessage = fetchArbitraryDataFile(peer, null, signature, hash, null);
Long endTime = NTP.getTime();
if (receivedArbitraryDataFileMessage != null) {
LOGGER.debug("Received data file {} from peer {}", receivedArbitraryDataFileMessage.getArbitraryDataFile().getHash58(), peer);
LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFileMessage.getArbitraryDataFile().getHash58(), peer, (endTime-startTime));
receivedAtLeastOneFile = true;
// Remove this hash from arbitraryDataFileHashResponses now that we have received it
arbitraryDataFileHashResponses.remove(hash58);
}
else {
LOGGER.debug("Peer {} didn't respond with data file {} for signature {}", peer, Base58.encode(hash), Base58.encode(signature));
LOGGER.debug("Peer {} didn't respond with data file {} for signature {}. Time taken: {} ms", peer, Base58.encode(hash), Base58.encode(signature), (endTime-startTime));
// Remove this hash from arbitraryDataFileHashResponses now that we have failed to receive it
arbitraryDataFileHashResponses.remove(hash58);
// Stop asking for files from this peer
break;
}
}
else {
LOGGER.trace("Already requesting data file {} for signature {}", arbitraryDataFile, Base58.encode(signature));
}
}
else {
// Remove this hash from arbitraryDataFileHashResponses because we have a local copy
arbitraryDataFileHashResponses.remove(hash58);
}
}
if (receivedAtLeastOneFile) {
@ -147,7 +170,6 @@ public class ArbitraryDataFileManager {
// Invalidate the hosted transactions cache as we are now hosting something new
ArbitraryDataStorageManager.getInstance().invalidateHostedTransactionsCache();
}
// Check if we have all the files we need for this transaction
if (arbitraryDataFile.allFilesExist()) {
@ -165,17 +187,19 @@ public class ArbitraryDataFileManager {
}
}
}
return receivedAtLeastOneFile;
}
private ArbitraryDataFileMessage fetchArbitraryDataFile(Peer peer, Peer requestingPeer, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature);
boolean fileAlreadyExists = existingFile.exists();
String hash58 = Base58.encode(hash);
Message message = null;
// Fetch the file if it doesn't exist locally
if (!fileAlreadyExists) {
String hash58 = Base58.encode(hash);
LOGGER.debug(String.format("Fetching data file %.8s from peer %s", hash58, peer));
arbitraryDataFileRequests.put(hash58, NTP.getTime());
Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash);
@ -191,9 +215,17 @@ public class ArbitraryDataFileManager {
// We may need to remove the file list request, if we have all the files for this transaction
this.handleFileListRequests(signature);
if (message == null || message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) {
if (message == null) {
LOGGER.debug("Received null message from peer {}", peer);
return null;
}
if (message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) {
LOGGER.debug("Received message with invalid type: {} from peer {}", message.getType(), peer);
return null;
}
}
else {
LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58));
}
ArbitraryDataFileMessage arbitraryDataFileMessage = (ArbitraryDataFileMessage) message;
@ -208,16 +240,7 @@ public class ArbitraryDataFileManager {
ArbitraryDataFile dataFile = arbitraryDataFileMessage.getArbitraryDataFile();
// Keep trying to delete the data until it is deleted, or we reach 10 attempts
for (int i=0; i<10; i++) {
if (dataFile.delete()) {
break;
}
try {
Thread.sleep(1000L);
} catch (InterruptedException e) {
// Fall through to exit method
}
}
dataFile.delete(10);
}
}
@ -359,6 +382,48 @@ public class ArbitraryDataFileManager {
}
// Relays
private List<ArbitraryRelayInfo> getRelayInfoListForHash(String hash58) {
synchronized (arbitraryRelayMap) {
return arbitraryRelayMap.stream()
.filter(relayInfo -> Objects.equals(relayInfo.getHash58(), hash58))
.collect(Collectors.toList());
}
}
private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) {
LOGGER.trace("Fetching random relay info for hash: {}", hash58);
List<ArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
if (relayInfoList != null && !relayInfoList.isEmpty()) {
// Pick random item
int index = new SecureRandom().nextInt(relayInfoList.size());
LOGGER.trace("Returning random relay info for hash: {} (index {})", hash58, index);
return relayInfoList.get(index);
}
LOGGER.trace("No relay info exists for hash: {}", hash58);
return null;
}
public void addToRelayMap(ArbitraryRelayInfo newEntry) {
if (newEntry == null || !newEntry.isValid()) {
return;
}
// Remove existing entry for this peer if it exists, to renew the timestamp
this.removeFromRelayMap(newEntry);
// Re-add
arbitraryRelayMap.add(newEntry);
LOGGER.debug("Added entry to relay map: {}", newEntry);
}
private void removeFromRelayMap(ArbitraryRelayInfo entry) {
arbitraryRelayMap.removeIf(relayInfo -> relayInfo.equals(entry));
}
// Network handlers
public void onNetworkGetArbitraryDataFileMessage(Peer peer, Message message) {
@ -377,7 +442,7 @@ public class ArbitraryDataFileManager {
try {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
Triple<String, Peer, Long> relayInfo = this.arbitraryRelayMap.get(hash58);
ArbitraryRelayInfo relayInfo = this.getRandomRelayInfoEntryForHash(hash58);
if (arbitraryDataFile.exists()) {
LOGGER.trace("Hash {} exists", hash58);
@ -394,15 +459,12 @@ public class ArbitraryDataFileManager {
else if (relayInfo != null) {
LOGGER.debug("We have relay info for hash {}", Base58.encode(hash));
// We need to ask this peer for the file
Peer peerToAsk = relayInfo.getB();
Peer peerToAsk = relayInfo.getPeer();
if (peerToAsk != null) {
// Forward the message to this peer
LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58);
this.fetchArbitraryDataFile(peerToAsk, peer, signature, hash, message);
// Remove from the map regardless of outcome, as the relay attempt is now considered complete
arbitraryRelayMap.remove(hash58);
}
else {
LOGGER.debug("Peer {} not found in relay info", peer);

117
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java Normal file
View File

@ -0,0 +1,117 @@
package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.network.Peer;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.utils.ArbitraryTransactionUtils;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import org.qortal.utils.Triple;
import java.util.Arrays;
import java.util.Iterator;
import java.util.Map;
public class ArbitraryDataFileRequestThread implements Runnable {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileRequestThread.class);
public ArbitraryDataFileRequestThread() {
}
@Override
public void run() {
Thread.currentThread().setName("Arbitrary Data File Request Thread");
try {
while (!Controller.isStopping()) {
Long now = NTP.getTime();
this.processFileHashes(now);
}
} catch (InterruptedException e) {
// Fall-through to exit thread...
}
}
private void processFileHashes(Long now) throws InterruptedException {
if (Controller.isStopping()) {
return;
}
ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance();
String signature58 = null;
String hash58 = null;
Peer peer = null;
boolean shouldProcess = false;
synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) {
Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator();
while (iterator.hasNext()) {
if (Controller.isStopping()) {
return;
}
Map.Entry entry = (Map.Entry) iterator.next();
if (entry == null || entry.getKey() == null || entry.getValue() == null) {
iterator.remove();
continue;
}
hash58 = (String) entry.getKey();
Triple<Peer, String, Long> value = (Triple<Peer, String, Long>) entry.getValue();
if (value == null) {
iterator.remove();
continue;
}
peer = value.getA();
signature58 = value.getB();
Long timestamp = value.getC();
if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) {
// Ignore - to be deleted
iterator.remove();
continue;
}
// We want to process this file
shouldProcess = true;
iterator.remove();
break;
}
}
if (!shouldProcess) {
// Nothing to do
Thread.sleep(1000L);
return;
}
byte[] hash = Base58.decode(hash58);
byte[] signature = Base58.decode(signature58);
// Fetch the transaction data
try (final Repository repository = RepositoryManager.getRepository()) {
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
if (arbitraryTransactionData == null) {
return;
}
if (signature == null || hash == null || peer == null || arbitraryTransactionData == null) {
return;
}
LOGGER.debug("Fetching file {} from peer {} via request thread...", hash58, peer);
arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash));
} catch (DataException e) {
LOGGER.debug("Unable to process file hashes: {}", e.getMessage());
}
}
}

7
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -38,10 +38,10 @@ public class ArbitraryDataManager extends Thread {
private int powDifficulty = 14; // Must not be final, as unit tests need to reduce this value
/** Request timeout when transferring arbitrary data */
public static final long ARBITRARY_REQUEST_TIMEOUT = 10 * 1000L; // ms
public static final long ARBITRARY_REQUEST_TIMEOUT = 12 * 1000L; // ms
/** Maximum time to hold information about an in-progress relay */
public static final long ARBITRARY_RELAY_TIMEOUT = 30 * 1000L; // ms
public static final long ARBITRARY_RELAY_TIMEOUT = 60 * 1000L; // ms
/** Maximum number of hops that an arbitrary signatures request is allowed to make */
private static int ARBITRARY_SIGNATURES_REQUEST_MAX_HOPS = 3;
@ -80,6 +80,9 @@ public class ArbitraryDataManager extends Thread {
Thread.currentThread().setName("Arbitrary Data Manager");
try {
// Wait for node to finish starting up and making connections
Thread.sleep(2 * 60 * 1000L);
while (!isStopping) {
Thread.sleep(2000);

2
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java
View File

@ -32,7 +32,7 @@ public class ArbitraryDataRenderManager extends Thread {
@Override
public void run() {
Thread.currentThread().setName("Arbitrary Data Manager");
Thread.currentThread().setName("Arbitrary Data Render Manager");
try {
while (!isStopping) {

3
src/main/java/org/qortal/controller/repository/AtStatesPruner.java
View File

@ -3,6 +3,7 @@ package org.qortal.controller.repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.data.block.BlockData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
@ -47,7 +48,7 @@ public class AtStatesPruner implements Runnable {
continue;
// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
if (Controller.getInstance().isSynchronizing())
if (Synchronizer.getInstance().isSynchronizing())
continue;
// Prune AT states for all blocks up until our latest minus pruneBlockLimit

3
src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java
View File

@ -3,6 +3,7 @@ package org.qortal.controller.repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.data.block.BlockData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
@ -34,7 +35,7 @@ public class AtStatesTrimmer implements Runnable {
continue;
// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
if (Controller.getInstance().isSynchronizing())
if (Synchronizer.getInstance().isSynchronizing())
continue;
long currentTrimmableTimestamp = NTP.getTime() - Settings.getInstance().getAtStatesMaxLifetime();

3
src/main/java/org/qortal/controller/repository/BlockArchiver.java
View File

@ -3,6 +3,7 @@ package org.qortal.controller.repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.data.block.BlockData;
import org.qortal.repository.*;
import org.qortal.settings.Settings;
@ -51,7 +52,7 @@ public class BlockArchiver implements Runnable {
}
// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
if (Controller.getInstance().isSynchronizing()) {
if (Synchronizer.getInstance().isSynchronizing()) {
continue;
}

3
src/main/java/org/qortal/controller/repository/BlockPruner.java
View File

@ -3,6 +3,7 @@ package org.qortal.controller.repository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.data.block.BlockData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
@ -51,7 +52,7 @@ public class BlockPruner implements Runnable {
continue;
// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
if (Controller.getInstance().isSynchronizing()) {
if (Synchronizer.getInstance().isSynchronizing()) {
continue;
}

89
src/main/java/org/qortal/controller/repository/NamesDatabaseIntegrityCheck.java
View File

@ -268,42 +268,6 @@ public class NamesDatabaseIntegrityCheck {
return registerNameTransactions;
}
private List<UpdateNameTransactionData> fetchUpdateNameTransactions() {
List<UpdateNameTransactionData> updateNameTransactions = new ArrayList<>();
for (TransactionData transactionData : this.nameTransactions) {
if (transactionData.getType() == TransactionType.UPDATE_NAME) {
UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData;
updateNameTransactions.add(updateNameTransactionData);
}
}
return updateNameTransactions;
}
private List<SellNameTransactionData> fetchSellNameTransactions() {
List<SellNameTransactionData> sellNameTransactions = new ArrayList<>();
for (TransactionData transactionData : this.nameTransactions) {
if (transactionData.getType() == TransactionType.SELL_NAME) {
SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData;
sellNameTransactions.add(sellNameTransactionData);
}
}
return sellNameTransactions;
}
private List<BuyNameTransactionData> fetchBuyNameTransactions() {
List<BuyNameTransactionData> buyNameTransactions = new ArrayList<>();
for (TransactionData transactionData : this.nameTransactions) {
if (transactionData.getType() == TransactionType.BUY_NAME) {
BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData;
buyNameTransactions.add(buyNameTransactionData);
}
}
return buyNameTransactions;
}
private void fetchAllNameTransactions(Repository repository) throws DataException {
List<TransactionData> nameTransactions = new ArrayList<>();
@ -319,43 +283,36 @@ public class NamesDatabaseIntegrityCheck {
this.nameTransactions = nameTransactions;
}
private List<TransactionData> fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException {
List<TransactionData> transactions = new ArrayList<>();
public List<TransactionData> fetchAllTransactionsInvolvingName(String name, Repository repository) throws DataException {
List<byte[]> signatures = new ArrayList<>();
String reducedName = Unicode.sanitize(name);
// Fetch all the confirmed name-modification transactions
if (this.nameTransactions.isEmpty()) {
this.fetchAllNameTransactions(repository);
}
List<byte[]> registerNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
TransactionType.REGISTER_NAME, Arrays.asList("(name = ? OR reduced_name = ?)"), Arrays.asList(name, reducedName));
signatures.addAll(registerNameTransactions);
for (TransactionData transactionData : this.nameTransactions) {
List<byte[]> updateNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
TransactionType.UPDATE_NAME,
Arrays.asList("(name = ? OR new_name = ? OR (reduced_new_name != '' AND reduced_new_name = ?))"),
Arrays.asList(name, name, reducedName));
signatures.addAll(updateNameTransactions);
if ((transactionData instanceof RegisterNameTransactionData)) {
RegisterNameTransactionData registerNameTransactionData = (RegisterNameTransactionData) transactionData;
if (Objects.equals(registerNameTransactionData.getReducedName(), reducedName)) {
List<byte[]> sellNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
TransactionType.SELL_NAME, Arrays.asList("name = ?"), Arrays.asList(name));
signatures.addAll(sellNameTransactions);
List<byte[]> buyNameTransactions = repository.getTransactionRepository().getSignaturesMatchingCustomCriteria(
TransactionType.BUY_NAME, Arrays.asList("name = ?"), Arrays.asList(name));
signatures.addAll(buyNameTransactions);
List<TransactionData> transactions = new ArrayList<>();
for (byte[] signature : signatures) {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
// Filter out any unconfirmed transactions
if (transactionData.getBlockHeight() != null && transactionData.getBlockHeight() > 0) {
transactions.add(transactionData);
}
}
if ((transactionData instanceof UpdateNameTransactionData)) {
UpdateNameTransactionData updateNameTransactionData = (UpdateNameTransactionData) transactionData;
if (Objects.equals(updateNameTransactionData.getName(), name) ||
Objects.equals(updateNameTransactionData.getReducedNewName(), reducedName)) {
transactions.add(transactionData);
}
}
if ((transactionData instanceof BuyNameTransactionData)) {
BuyNameTransactionData buyNameTransactionData = (BuyNameTransactionData) transactionData;
if (Objects.equals(buyNameTransactionData.getName(), name)) {
transactions.add(transactionData);
}
}
if ((transactionData instanceof SellNameTransactionData)) {
SellNameTransactionData sellNameTransactionData = (SellNameTransactionData) transactionData;
if (Objects.equals(sellNameTransactionData.getName(), name)) {
transactions.add(transactionData);
}
}
}
return transactions;
}

3
src/main/java/org/qortal/controller/repository/OnlineAccountsSignaturesTrimmer.java
View File

@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.block.BlockChain;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.data.block.BlockData;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
@ -36,7 +37,7 @@ public class OnlineAccountsSignaturesTrimmer implements Runnable {
continue;
// Don't even attempt if we're mid-sync as our repository requests will be delayed for ages
if (Controller.getInstance().isSynchronizing())
if (Synchronizer.getInstance().isSynchronizing())
continue;
// Trim blockchain by removing 'old' online accounts signatures

97
src/main/java/org/qortal/crosschain/Bitcoiny.java
View File

@ -1,12 +1,6 @@
package org.qortal.crosschain;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.logging.log4j.LogManager;
@ -39,6 +33,7 @@ import org.qortal.utils.Amounts;
import org.qortal.utils.BitTwiddling;
import com.google.common.hash.HashCode;
import org.qortal.utils.NTP;
/** Bitcoin-like (Bitcoin, Litecoin, etc.) support */
public abstract class Bitcoiny implements ForeignBlockchain {
@ -53,6 +48,12 @@ public abstract class Bitcoiny implements ForeignBlockchain {
protected final NetworkParameters params;
/** Cache recent transactions to speed up subsequent lookups */
protected List<SimpleTransaction> transactionsCache;
protected Long transactionsCacheTimestamp;
protected String transactionsCacheXpub;
protected static long TRANSACTIONS_CACHE_TIMEOUT = 2 * 60 * 1000L; // 2 minutes
/** Keys that have been previously marked as fully spent,<br>
* i.e. keys with transactions but with no unspent outputs. */
protected final Set<ECKey> spentKeys = Collections.synchronizedSet(new HashSet<>());
@ -228,6 +229,25 @@ public abstract class Bitcoiny implements ForeignBlockchain {
return transaction.getOutputs();
}
/**
* Returns transactions for passed script
* <p>
* @throws ForeignBlockchainException if error occurs
*/
public List<TransactionHash> getAddressTransactions(byte[] scriptPubKey, boolean includeUnconfirmed) throws ForeignBlockchainException {
int retries = 0;
ForeignBlockchainException e2 = null;
while (retries <= 3) {
try {
return this.blockchain.getAddressTransactions(scriptPubKey, includeUnconfirmed);
} catch (ForeignBlockchainException e) {
e2 = e;
retries++;
}
}
throw(e2);
}
/**
* Returns list of transaction hashes pertaining to passed address.
* <p>
@ -262,7 +282,17 @@ public abstract class Bitcoiny implements ForeignBlockchain {
* @throws ForeignBlockchainException if error occurs
*/
public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException {
int retries = 0;
ForeignBlockchainException e2 = null;
while (retries <= 3) {
try {
return this.blockchain.getTransaction(txHash);
} catch (ForeignBlockchainException e) {
e2 = e;
retries++;
}
}
throw(e2);
}
/**
@ -337,7 +367,29 @@ public abstract class Bitcoiny implements ForeignBlockchain {
return balance.value;
}
public Long getWalletBalanceFromTransactions(String key58) throws ForeignBlockchainException {
long balance = 0;
Comparator<SimpleTransaction> oldestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp);
List<SimpleTransaction> transactions = getWalletTransactions(key58).stream().sorted(oldestTimestampFirstComparator).collect(Collectors.toList());
for (SimpleTransaction transaction : transactions) {
balance += transaction.getTotalAmount();
}
return balance;
}
public List<SimpleTransaction> getWalletTransactions(String key58) throws ForeignBlockchainException {
synchronized (this) {
// Serve from the cache if it's recent, and matches this xpub
if (Objects.equals(transactionsCacheXpub, key58)) {
if (transactionsCache != null && transactionsCacheTimestamp != null) {
Long now = NTP.getTime();
boolean isCacheStale = (now != null && now - transactionsCacheTimestamp >= TRANSACTIONS_CACHE_TIMEOUT);
if (!isCacheStale) {
return transactionsCache;
}
}
}
Context.propagate(bitcoinjContext);
Wallet wallet = walletFromDeterministicKey58(key58);
@ -352,7 +404,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
Set<String> keySet = new HashSet<>();
// Set the number of consecutive empty batches required before giving up
final int numberOfAdditionalBatchesToSearch = 5;
final int numberOfAdditionalBatchesToSearch = 7;
int unusedCounter = 0;
int ki = 0;
@ -368,7 +420,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
byte[] script = ScriptBuilder.createOutputScript(address).getProgram();
// Ask for transaction history - if it's empty then key has never been used
List<TransactionHash> historicTransactionHashes = this.blockchain.getAddressTransactions(script, false);
List<TransactionHash> historicTransactionHashes = this.getAddressTransactions(script, false);
if (!historicTransactionHashes.isEmpty()) {
areAllKeysUnused = false;
@ -386,8 +438,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
}
// We haven't hit our search limit yet so increment the counter and keep looking
unusedCounter++;
}
else {
} else {
// Some keys in this batch were used, so reset the counter
unusedCounter = 0;
}
@ -400,7 +451,15 @@ public abstract class Bitcoiny implements ForeignBlockchain {
Comparator<SimpleTransaction> newestTimestampFirstComparator = Comparator.comparingInt(SimpleTransaction::getTimestamp).reversed();
return walletTransactions.stream().map(t -> convertToSimpleTransaction(t, keySet)).sorted(newestTimestampFirstComparator).collect(Collectors.toList());
// Update cache and return
transactionsCacheTimestamp = NTP.getTime();
transactionsCacheXpub = key58;
transactionsCache = walletTransactions.stream()
.map(t -> convertToSimpleTransaction(t, keySet))
.sorted(newestTimestampFirstComparator).collect(Collectors.toList());
return transactionsCache;
}
}
protected SimpleTransaction convertToSimpleTransaction(BitcoinyTransaction t, Set<String> keySet) {
@ -411,12 +470,15 @@ public abstract class Bitcoiny implements ForeignBlockchain {
List<SimpleTransaction.Input> inputs = new ArrayList<>();
List<SimpleTransaction.Output> outputs = new ArrayList<>();
boolean anyOutputAddressInWallet = false;
for (BitcoinyTransaction.Input input : t.inputs) {
try {
BitcoinyTransaction t2 = getTransaction(input.outputTxHash);
List<String> senders = t2.outputs.get(input.outputVout).addresses;
long inputAmount = t2.outputs.get(input.outputVout).value;
totalInputAmount += inputAmount;
if (senders != null) {
for (String sender : senders) {
boolean addressInWallet = false;
if (keySet.contains(sender)) {
@ -425,12 +487,14 @@ public abstract class Bitcoiny implements ForeignBlockchain {
}
inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet));
}
}
} catch (ForeignBlockchainException e) {
LOGGER.trace("Failed to retrieve transaction information {}", input.outputTxHash);
}
}
if (t.outputs != null && !t.outputs.isEmpty()) {
for (BitcoinyTransaction.Output output : t.outputs) {
if (output.addresses != null) {
for (String address : output.addresses) {
boolean addressInWallet = false;
if (keySet.contains(address)) {
@ -440,13 +504,22 @@ public abstract class Bitcoiny implements ForeignBlockchain {
amount += output.value;
}
addressInWallet = true;
anyOutputAddressInWallet = true;
}
outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet));
}
}
totalOutputAmount += output.value;
}
}
long fee = totalInputAmount - totalOutputAmount;
if (!anyOutputAddressInWallet) {
// No outputs relate to this wallet - check if any inputs did (which is signified by a positive total)
if (total > 0) {
amount = total * -1;
}
}
return new SimpleTransaction(t.txHash, t.timestamp, amount, fee, inputs, outputs);
}

7
src/main/java/org/qortal/crosschain/Dogecoin.java
View File

@ -19,12 +19,13 @@ public class Dogecoin extends Bitcoiny {
public static final String CURRENCY_CODE = "DOGE";
private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(500000000); // 5 DOGE per 1000 bytes
private static final Coin DEFAULT_FEE_PER_KB = Coin.valueOf(1000000); // 0.01 DOGE per 1000 bytes
private static final long MINIMUM_ORDER_AMOUNT = 300000000L; // 3 DOGE minimum order. The RPC dust threshold is around 2 DOGE
private static final long MINIMUM_ORDER_AMOUNT = 100000000L; // 1 DOGE minimum order. See recommendations:
// https://github.com/dogecoin/dogecoin/blob/master/doc/fee-recommendation.md
// Temporary values until a dynamic fee system is written.
private static final long MAINNET_FEE = 110000000L;
private static final long MAINNET_FEE = 100000L;
private static final long NON_MAINNET_FEE = 10000L; // TODO: calibrate this
private static final Map<ConnectionType, Integer> DEFAULT_ELECTRUMX_PORTS = new EnumMap<>(ConnectionType.class);

60
src/main/java/org/qortal/crosschain/ElectrumX.java
View File

@ -5,19 +5,7 @@ import java.math.BigDecimal;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Random;
import java.util.Scanner;
import java.util.Set;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@ -50,6 +38,9 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
/** Error message sent by some ElectrumX servers when they don't support returning verbose transactions. */
private static final String VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE = "verbose transactions are currently unsupported";
private static final int RESPONSE_TIME_READINGS = 5;
private static final long MAX_AVG_RESPONSE_TIME = 500L; // ms
public static class Server {
String hostname;
@ -57,6 +48,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
ConnectionType connectionType;
int port;
private List<Long> responseTimes = new ArrayList<>();
public Server(String hostname, ConnectionType connectionType, int port) {
this.hostname = hostname;
@ -64,6 +56,25 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
this.port = port;
}
public void addResponseTime(long responseTime) {
while (this.responseTimes.size() > RESPONSE_TIME_READINGS) {
this.responseTimes.remove(0);
}
this.responseTimes.add(responseTime);
}
public long averageResponseTime() {
if (this.responseTimes.size() < RESPONSE_TIME_READINGS) {
// Not enough readings yet
return 0L;
}
OptionalDouble average = this.responseTimes.stream().mapToDouble(a -> a).average();
if (average.isPresent()) {
return Double.valueOf(average.getAsDouble()).longValue();
}
return 0L;
}
@Override
public boolean equals(Object other) {
if (other == this)
@ -103,7 +114,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
private Scanner scanner;
private int nextId = 1;
private static final int TX_CACHE_SIZE = 200;
private static final int TX_CACHE_SIZE = 1000;
@SuppressWarnings("serial")
private final Map<String, BitcoinyTransaction> transactionCache = Collections.synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) {
// This method is called just after a new entry has been added
@ -539,6 +550,17 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
while (haveConnection()) {
Object response = connectedRpc(method, params);
// If we have more servers and this one replied slowly, try another
if (!this.remainingServers.isEmpty()) {
long averageResponseTime = this.currentServer.averageResponseTime();
if (averageResponseTime > MAX_AVG_RESPONSE_TIME) {
LOGGER.info("Slow average response time {}ms from {} - trying another server...", averageResponseTime, this.currentServer.hostname);
this.closeServer();
break;
}
}
if (response != null)
return response;
@ -628,6 +650,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
String request = requestJson.toJSONString() + "\n";
LOGGER.trace(() -> String.format("Request: %s", request));
long startTime = System.currentTimeMillis();
final String response;
try {
@ -638,7 +661,11 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
return null;
}
long endTime = System.currentTimeMillis();
long responseTime = endTime-startTime;
LOGGER.trace(() -> String.format("Response: %s", response));
LOGGER.trace(() -> String.format("Time taken: %dms", endTime-startTime));
if (response.isEmpty())
// Empty response - try another server?
@ -649,6 +676,11 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
// Unexpected response - try another server?
return null;
// Keep track of response times
if (this.currentServer != null) {
this.currentServer.addResponseTime(responseTime);
}
JSONObject responseJson = (JSONObject) responseObj;
Object errorObj = responseJson.get("error");

8
src/main/java/org/qortal/crosschain/Litecoin.java
View File

@ -50,8 +50,12 @@ public class Litecoin extends Bitcoiny {
new Server("electrum.ltc.xurious.com", Server.ConnectionType.TCP, 50001),
new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002),
new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 50002),
new Server("ltc.rentonisk.com", Server.ConnectionType.TCP, 50001),
new Server("ltc.rentonisk.com", Server.ConnectionType.SSL, 50002),
new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20063),
new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20063),
new Server("electrum3.cipig.net", ConnectionType.TCP, 10063),
new Server("electrum2.cipig.net", Server.ConnectionType.TCP, 10063),
new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20063),
new Server("electrum1.cipig.net", Server.ConnectionType.TCP, 10063),
new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002),
new Server("ltc.litepay.ch", Server.ConnectionType.SSL, 50022),
new Server("electrum-ltc-bysh.me", Server.ConnectionType.TCP, 50002),

60
src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java Normal file
View File

@ -0,0 +1,60 @@
package org.qortal.data.arbitrary;
import org.qortal.network.Peer;
import java.util.Objects;
public class ArbitraryRelayInfo {
private final String hash58;
private final String signature58;
private final Peer peer;
private final Long timestamp;
public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp) {
this.hash58 = hash58;
this.signature58 = signature58;
this.peer = peer;
this.timestamp = timestamp;
}
public boolean isValid() {
return this.getHash58() != null && this.getSignature58() != null
&& this.getPeer() != null && this.getTimestamp() != null;
}
public String getHash58() {
return this.hash58;
}
public String getSignature58() {
return signature58;
}
public Peer getPeer() {
return peer;
}
public Long getTimestamp() {
return timestamp;
}
@Override
public String toString() {
return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp);
}
@Override
public boolean equals(Object other) {
if (other == this)
return true;
if (!(other instanceof ArbitraryRelayInfo))
return false;
ArbitraryRelayInfo otherRelayInfo = (ArbitraryRelayInfo) other;
return this.peer == otherRelayInfo.getPeer()
&& Objects.equals(this.hash58, otherRelayInfo.getHash58())
&& Objects.equals(this.signature58, otherRelayInfo.getSignature58());
}
}

10
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java
View File

@ -30,13 +30,21 @@ public class ArbitraryResourceStatus {
private String title;
private String description;
private Integer localChunkCount;
private Integer totalChunkCount;
public ArbitraryResourceStatus() {
}
public ArbitraryResourceStatus(Status status) {
public ArbitraryResourceStatus(Status status, Integer localChunkCount, Integer totalChunkCount) {
this.id = status.toString();
this.title = status.title;
this.description = status.description;
this.localChunkCount = localChunkCount;
this.totalChunkCount = totalChunkCount;
}
public ArbitraryResourceStatus(Status status) {
this(status, null, null);
}
}

3
src/main/java/org/qortal/list/ResourceList.java
View File

@ -13,6 +13,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class ResourceList {
@ -20,7 +21,7 @@ public class ResourceList {
private static final Logger LOGGER = LogManager.getLogger(ResourceList.class);
private String name;
private List<String> list = new ArrayList<>();
private List<String> list = Collections.synchronizedList(new ArrayList<>());
/**
* ResourceList

3
src/main/java/org/qortal/list/ResourceListManager.java
View File

@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@ -13,7 +14,7 @@ public class ResourceListManager {
private static final Logger LOGGER = LogManager.getLogger(ResourceListManager.class);
private static ResourceListManager instance;
private List<ResourceList> lists = new ArrayList<>();
private List<ResourceList> lists = Collections.synchronizedList(new ArrayList<>());
public ResourceListManager() {

9
src/main/java/org/qortal/network/Handshake.java
View File

@ -74,6 +74,12 @@ public enum Handshake {
peer.setPeersConnectionTimestamp(peersConnectionTimestamp);
peer.setPeersVersion(versionString, version);
// Ensure the peer is running at least the version specified in MIN_PEER_VERSION
if (peer.isAtLeastVersion(MIN_PEER_VERSION) == false) {
LOGGER.debug(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString));
return null;
}
if (Settings.getInstance().getAllowConnectionsWithOlderPeerVersions() == false) {
// Ensure the peer is running at least the minimum version allowed for connections
final String minPeerVersion = Settings.getInstance().getMinPeerVersion();
@ -258,6 +264,9 @@ public enum Handshake {
private static final long PEER_VERSION_131 = 0x0100030001L;
/** Minimum peer version that we are allowed to communicate with */
private static final String MIN_PEER_VERSION = "3.1.0";
private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes
private static final int POW_DIFFICULTY_PRE_131 = 8; // leading zero bits
// Can always be made harder in the future...

13
src/main/java/org/qortal/network/Network.java
View File

@ -6,7 +6,7 @@ import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters;
import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters;
import org.qortal.block.BlockChain;
import org.qortal.controller.Controller;
import org.qortal.controller.arbitrary.ArbitraryDataFileManager;
import org.qortal.controller.arbitrary.ArbitraryDataFileListManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.crypto.Crypto;
import org.qortal.data.block.BlockData;
@ -307,12 +307,7 @@ public class Network {
return false;
}
try (final Repository repository = RepositoryManager.getRepository()) {
return ArbitraryDataFileManager.getInstance().fetchAllArbitraryDataFiles(repository, connectedPeer, signature);
} catch (DataException e) {
LOGGER.info("Unable to fetch arbitrary data files");
}
return false;
return ArbitraryDataFileListManager.getInstance().fetchArbitraryDataFileList(connectedPeer, signature);
}
/**
@ -1169,6 +1164,7 @@ public class Network {
if (consecutiveReadings >= consecutiveReadingsRequired) {
// Last 10 readings were the same - i.e. more than one peer agreed on the new IP address...
String ip = ipAddressHistory.get(size - 1);
if (ip != null && !Objects.equals(ip, "null")) {
if (!Objects.equals(ip, this.ourExternalIpAddress)) {
// ... and the readings were different to our current recorded value, so
// update our external IP address value
@ -1177,11 +1173,12 @@ public class Network {
}
}
}
}
public void onExternalIpUpdate(String ipAddress) {
LOGGER.info("External IP address updated to {}", ipAddress);
ArbitraryDataManager.getInstance().broadcastHostedSignatureList();
//ArbitraryDataManager.getInstance().broadcastHostedSignatureList();
}

5
src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java
View File

@ -1,6 +1,8 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.repository.DataException;
import org.qortal.transform.Transformer;
@ -12,6 +14,8 @@ import java.nio.ByteBuffer;
public class ArbitraryDataFileMessage extends Message {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileMessage.class);
private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
private final byte[] signature;
@ -52,6 +56,7 @@ public class ArbitraryDataFileMessage extends Message {
return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile);
}
catch (DataException e) {
LOGGER.info("Unable to process received file: {}", e.getMessage());
return null;
}
}

45
src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java
View File

@ -3,11 +3,14 @@ package org.qortal.network.message;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
import org.qortal.transform.Transformer;
import org.qortal.transform.transaction.TransactionTransformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
import static org.qortal.transform.Transformer.INT_LENGTH;
import static org.qortal.transform.Transformer.LONG_LENGTH;
@ -15,19 +18,22 @@ import static org.qortal.transform.Transformer.LONG_LENGTH;
public class GetArbitraryDataFileListMessage extends Message {
private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH;
private final byte[] signature;
private List<byte[]> hashes;
private final long requestTime;
private int requestHops;
public GetArbitraryDataFileListMessage(byte[] signature, long requestTime, int requestHops) {
this(-1, signature, requestTime, requestHops);
public GetArbitraryDataFileListMessage(byte[] signature, List<byte[]> hashes, long requestTime, int requestHops) {
this(-1, signature, hashes, requestTime, requestHops);
}
private GetArbitraryDataFileListMessage(int id, byte[] signature, long requestTime, int requestHops) {
private GetArbitraryDataFileListMessage(int id, byte[] signature, List<byte[]> hashes, long requestTime, int requestHops) {
super(id, MessageType.GET_ARBITRARY_DATA_FILE_LIST);
this.signature = signature;
this.hashes = hashes;
this.requestTime = requestTime;
this.requestHops = requestHops;
}
@ -36,10 +42,11 @@ public class GetArbitraryDataFileListMessage extends Message {
return this.signature;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
if (bytes.remaining() != SIGNATURE_LENGTH + LONG_LENGTH + INT_LENGTH)
return null;
public List<byte[]> getHashes() {
return this.hashes;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
byte[] signature = new byte[SIGNATURE_LENGTH];
bytes.get(signature);
@ -48,7 +55,23 @@ public class GetArbitraryDataFileListMessage extends Message {
int requestHops = bytes.getInt();
return new GetArbitraryDataFileListMessage(id, signature, requestTime, requestHops);
List<byte[]> hashes = null;
if (bytes.hasRemaining()) {
int hashCount = bytes.getInt();
if (bytes.remaining() != hashCount * HASH_LENGTH) {
return null;
}
hashes = new ArrayList<>();
for (int i = 0; i < hashCount; ++i) {
byte[] hash = new byte[HASH_LENGTH];
bytes.get(hash);
hashes.add(hash);
}
}
return new GetArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops);
}
@Override
@ -62,6 +85,14 @@ public class GetArbitraryDataFileListMessage extends Message {
bytes.write(Ints.toByteArray(this.requestHops));
if (this.hashes != null) {
bytes.write(Ints.toByteArray(this.hashes.size()));
for (byte[] hash : this.hashes) {
bytes.write(hash);
}
}
return bytes.toByteArray();
} catch (IOException e) {
return null;

2
src/main/java/org/qortal/repository/BlockArchiveReader.java
View File

@ -145,6 +145,7 @@ public class BlockArchiveReader {
}
private String getFilenameForHeight(int height) {
synchronized (this.fileListCache) {
Iterator it = this.fileListCache.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pair = (Map.Entry) it.next();
@ -161,6 +162,7 @@ public class BlockArchiveReader {
return filename;
}
}
}
return null;
}

3
src/main/java/org/qortal/repository/BlockArchiveWriter.java
View File

@ -5,6 +5,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.block.Block;
import org.qortal.controller.Controller;
import org.qortal.controller.Synchronizer;
import org.qortal.data.block.BlockArchiveData;
import org.qortal.data.block.BlockData;
import org.qortal.settings.Settings;
@ -100,7 +101,7 @@ public class BlockArchiveWriter {
if (Controller.isStopping()) {
return BlockArchiveWriteResult.STOPPING;
}
if (Controller.getInstance().isSynchronizing()) {
if (Synchronizer.getInstance().isSynchronizing()) {
continue;
}

28
src/main/java/org/qortal/repository/TransactionRepository.java
View File

@ -108,6 +108,23 @@ public interface TransactionRepository {
public List<byte[]> getSignaturesMatchingCriteria(TransactionType txType, byte[] publicKey,
Integer minBlockHeight, Integer maxBlockHeight) throws DataException;
/**
* Returns signatures for transactions that match search criteria.
* <p>
* Alternate version that allows for custom where clauses and bind params.
* Only use for very specific use cases, such as the names integrity check.
* Not advised to be used otherwise, given that it could be possible for
* unsanitized inputs to be passed in if not careful.
*
* @param txType
* @param whereClauses
* @param bindParams
* @return
* @throws DataException
*/
public List<byte[]> getSignaturesMatchingCustomCriteria(TransactionType txType, List<String> whereClauses,
List<Object> bindParams) throws DataException;
/**
* Returns signature for latest auto-update transaction.
* <p>
@ -125,6 +142,17 @@ public interface TransactionRepository {
*/
public byte[] getLatestAutoUpdateTransaction(TransactionType txType, int txGroupId, Integer service) throws DataException;
/**
* Returns signatures for all name-registration related transactions relating to supplied name.
* Note: this does not currently include ARBITRARY data relating to the name.
*
* @param name
* @param confirmationStatus
* @return
* @throws DataException
*/
public List<TransactionData> getTransactionsInvolvingName(String name, ConfirmationStatus confirmationStatus) throws DataException;
/**
* Returns list of transactions relating to specific asset ID.
*

6
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@ -330,7 +330,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
bindParams.add(name);
}
sql.append(" GROUP BY name, service, identifier ORDER BY name");
sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
if (reverse != null && reverse) {
sql.append(" DESC");
@ -401,7 +401,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
bindParams.add(queryWildcard);
}
sql.append(" GROUP BY name, service, identifier ORDER BY name");
sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
if (reverse != null && reverse) {
sql.append(" DESC");
@ -465,7 +465,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
sql.append(" AND (identifier = ? OR (? IS NULL))");
}
sql.append(" GROUP BY name ORDER BY name");
sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
if (reverse != null && reverse) {
sql.append(" DESC");

14
src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java
View File

@ -945,6 +945,20 @@ public class HSQLDBDatabaseUpdates {
stmt.execute("CREATE INDEX ArbitraryPeersHashIndex ON ArbitraryPeers (hash)");
break;
case 40:
// For looking up name registration transactions based on name or reduced name
stmt.execute("CREATE INDEX RegisterNameNameIndex ON RegisterNameTransactions (name)");
stmt.execute("CREATE INDEX RegisterNameReducedNameIndex ON RegisterNameTransactions (reduced_name)");
// For looking up update name transactions based on name, new name, or new reduced name
stmt.execute("CREATE INDEX UpdateNameNameIndex ON UpdateNameTransactions (name)");
stmt.execute("CREATE INDEX UpdateNameNewNameIndex ON UpdateNameTransactions (new_name)");
stmt.execute("CREATE INDEX UpdateNameReducedNewNameIndex ON UpdateNameTransactions (reduced_new_name)");
// For looking up buy name transactions based on name
stmt.execute("CREATE INDEX BuyNameNameIndex ON BuyNameTransactions (name)");
// For looking up sell name transactions based on name
stmt.execute("CREATE INDEX SellNameNameIndex ON SellNameTransactions (name)");
break;
default:
// nothing to do
return false;

121
src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java
View File

@ -30,6 +30,7 @@ import org.qortal.repository.hsqldb.HSQLDBSaver;
import org.qortal.transaction.Transaction.ApprovalStatus;
import org.qortal.transaction.Transaction.TransactionType;
import org.qortal.utils.Base58;
import org.qortal.utils.Unicode;
public class HSQLDBTransactionRepository implements TransactionRepository {
@ -655,6 +656,44 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
}
}
public List<byte[]> getSignaturesMatchingCustomCriteria(TransactionType txType, List<String> whereClauses,
List<Object> bindParams) throws DataException {
List<byte[]> signatures = new ArrayList<>();
StringBuilder sql = new StringBuilder(1024);
sql.append(String.format("SELECT signature FROM %sTransactions", txType.className));
if (!whereClauses.isEmpty()) {
sql.append(" WHERE ");
final int whereClausesSize = whereClauses.size();
for (int wci = 0; wci < whereClausesSize; ++wci) {
if (wci != 0)
sql.append(" AND ");
sql.append(whereClauses.get(wci));
}
}
LOGGER.trace(() -> String.format("Transaction search SQL: %s", sql));
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return signatures;
do {
byte[] signature = resultSet.getBytes(1);
signatures.add(signature);
} while (resultSet.next());
return signatures;
} catch (SQLException e) {
throw new DataException("Unable to fetch matching transaction signatures from repository", e);
}
}
@Override
public byte[] getLatestAutoUpdateTransaction(TransactionType txType, int txGroupId, Integer service) throws DataException {
StringBuilder sql = new StringBuilder(1024);
@ -700,6 +739,88 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
}
}
@Override
public List<TransactionData> getTransactionsInvolvingName(String name, ConfirmationStatus confirmationStatus) throws DataException {
TransactionType[] transactionTypes = new TransactionType[] {
REGISTER_NAME, UPDATE_NAME, BUY_NAME, SELL_NAME
}; // TODO: CancelSellNameTransaction?
String reducedName = Unicode.sanitize(name);
StringBuilder sql = new StringBuilder(1024);
List<Object> bindParams = new ArrayList<>();
sql.append("SELECT Transactions.signature FROM Transactions");
for (int ti = 0; ti < transactionTypes.length; ++ti) {
sql.append(" LEFT OUTER JOIN ");
sql.append(transactionTypes[ti].className);
sql.append("Transactions USING (signature)");
}
sql.append(" WHERE Transactions.type IN (");
for (int ti = 0; ti < transactionTypes.length; ++ti) {
if (ti != 0)
sql.append(", ");
sql.append(transactionTypes[ti].value);
}
sql.append(")");
// Confirmation status
switch (confirmationStatus) {
case BOTH:
break;
case CONFIRMED:
sql.append(" AND Transactions.block_height IS NOT NULL");
break;
case UNCONFIRMED:
sql.append(" AND Transactions.block_height IS NULL");
break;
}
sql.append(" AND (RegisterNameTransactions.name = ?");
bindParams.add(name);
sql.append(" OR RegisterNameTransactions.reduced_name = ?");
bindParams.add(reducedName);
sql.append(" OR UpdateNameTransactions.name = ?");
bindParams.add(name);
sql.append(" OR (UpdateNameTransactions.reduced_new_name != '' AND UpdateNameTransactions.reduced_new_name = ?)");
bindParams.add(reducedName);
sql.append(" OR UpdateNameTransactions.new_name = ?");
bindParams.add(name);
sql.append(" OR SellNameTransactions.name = ?");
bindParams.add(name);
sql.append(" OR BuyNameTransactions.name = ?");
bindParams.add(name);
sql.append(") GROUP BY Transactions.signature, Transactions.created_when ORDER BY Transactions.created_when");
List<TransactionData> transactions = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
if (resultSet == null)
return transactions;
do {
byte[] signature = resultSet.getBytes(1);
TransactionData transactionData = this.fromSignature(signature);
if (transactionData == null)
// Something inconsistent with the repository
throw new DataException("Unable to fetch name-related transaction from repository?");
transactions.add(transactionData);
} while (resultSet.next());
return transactions;
} catch (SQLException | DataException e) {
throw new DataException("Unable to fetch name-related transactions from repository", e);
}
}
@Override
public List<TransactionData> getAssetTransactions(long assetId, ConfirmationStatus confirmationStatus, Integer limit, Integer offset, Boolean reverse)
throws DataException {

16
src/main/java/org/qortal/settings/Settings.java
View File

@ -195,16 +195,16 @@ public class Settings {
private int maxRetries = 2;
/** Minimum peer version number required in order to sync with them */
private String minPeerVersion = "3.0.1";
private String minPeerVersion = "3.1.0";
/** Whether to allow connections with peers below minPeerVersion
* If true, we won't sync with them but they can still sync with us, and will show in the peers list
* If false, sync will be blocked both ways, and they will not appear in the peers list */
private boolean allowConnectionsWithOlderPeerVersions = true;
/** Minimum time (in seconds) that we should attempt to remain connected to a peer for */
private int minPeerConnectionTime = 2 * 60; // seconds
private int minPeerConnectionTime = 5 * 60; // seconds
/** Maximum time (in seconds) that we should attempt to remain connected to a peer for */
private int maxPeerConnectionTime = 20 * 60; // seconds
private int maxPeerConnectionTime = 60 * 60; // seconds
/** Whether to sync multiple blocks at once in normal operation */
private boolean fastSyncEnabled = true;
@ -243,7 +243,8 @@ public class Settings {
private String[] bootstrapHosts = new String[] {
"http://bootstrap.qortal.org",
"http://bootstrap2.qortal.org",
"http://cinfu1.crowetic.com"
"http://81.169.136.59",
"http://62.171.190.193"
};
// Auto-update sources
@ -308,6 +309,9 @@ public class Settings {
/** Maximum total size of hosted data, in bytes. Unlimited if null */
private Long maxStorageCapacity = null;
/** Whether to serve QDN data without authentication */
private boolean qdnAuthBypassEnabled = false;
// Domain mapping
public static class DomainMap {
private String domain;
@ -884,4 +888,8 @@ public class Settings {
public Long getMaxStorageCapacity() {
return this.maxStorageCapacity;
}
public boolean isQDNAuthBypassEnabled() {
return this.qdnAuthBypassEnabled;
}
}

9
src/main/java/org/qortal/transaction/RegisterNameTransaction.java
View File

@ -37,6 +37,15 @@ public class RegisterNameTransaction extends Transaction {
return Collections.emptyList();
}
@Override
public long getUnitFee(Long timestamp) {
// Use a higher unit fee after the fee increase timestamp
if (timestamp > BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp()) {
return BlockChain.getInstance().getNameRegistrationUnitFee();
}
return BlockChain.getInstance().getUnitFee();
}
// Navigation
public Account getRegistrant() {

19
src/main/java/org/qortal/transaction/Transaction.java
View File

@ -334,7 +334,7 @@ public abstract class Transaction {
/** Returns whether transaction's fee is at least minimum unit fee as specified in blockchain config. */
public boolean hasMinimumFee() {
return this.transactionData.getFee() >= BlockChain.getInstance().getUnitFee();
return this.transactionData.getFee() >= this.getUnitFee(this.transactionData.getTimestamp());
}
public long feePerByte() {
@ -347,7 +347,7 @@ public abstract class Transaction {
/** Returns whether transaction's fee is at least amount needed to cover byte-length of transaction. */
public boolean hasMinimumFeePerByte() {
long unitFee = BlockChain.getInstance().getUnitFee();
long unitFee = this.getUnitFee(this.transactionData.getTimestamp());
int maxBytePerUnitFee = BlockChain.getInstance().getMaxBytesPerUnitFee();
// If the unit fee is zero, any fee is enough to cover the byte-length of the transaction
@ -369,7 +369,18 @@ public abstract class Transaction {
int unitFeeCount = ((dataLength - 1) / maxBytePerUnitFee) + 1;
return BlockChain.getInstance().getUnitFee() * unitFeeCount;
return this.getUnitFee(this.transactionData.getTimestamp()) * unitFeeCount;
}
/**
* Calculate unit fee for a given transaction type
*
* FUTURE: add "accountLevel" parameter if needed - the level of the transaction creator
* @param timestamp - the transaction's timestamp (currently not used)
* @return
*/
public long getUnitFee(Long timestamp) {
return BlockChain.getInstance().getUnitFee();
}
/**
@ -382,7 +393,7 @@ public abstract class Transaction {
* @return transaction version number
*/
public static int getVersionByTimestamp(long timestamp) {
if (timestamp >= 1642176000000L) {
if (timestamp >= BlockChain.getInstance().getTransactionV5Timestamp()) {
return 5;
}
return 4;

40
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@ -1,5 +1,6 @@
package org.qortal.utils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataFile;
@ -147,16 +148,49 @@ public class ArbitraryTransactionUtils {
byte[] metadataHash = transactionData.getMetadataHash();
byte[] signature = transactionData.getSignature();
if (metadataHash == null) {
// This file doesn't have any metadata, therefore it has no chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
arbitraryDataFile.setMetadataHash(metadataHash);
// Find the folder containing the files
Path parentPath = arbitraryDataFile.getFilePath().getParent();
String[] files = parentPath.toFile().list();
if (files == null) {
return 0;
}
// Remove the original copy indicator file if it exists
files = ArrayUtils.removeElement(files, ".original");
int count = files.length;
// If the complete file exists (and this transaction has chunks), subtract it from the count
if (arbitraryDataFile.chunkCount() > 0 && arbitraryDataFile.exists()) {
// We are only measuring the individual chunks, not the joined file
count -= 1;
}
return count;
}
public static int totalChunkCount(ArbitraryTransactionData transactionData) throws DataException {
if (transactionData == null) {
return 0;
}
byte[] digest = transactionData.getData();
byte[] metadataHash = transactionData.getMetadataHash();
byte[] signature = transactionData.getSignature();
if (metadataHash == null) {
// This file doesn't have any metadata, therefore it has a single (complete) chunk
return 1;
}
// Load complete file and chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
arbitraryDataFile.setMetadataHash(metadataHash);
return arbitraryDataFile.chunkCount();
return arbitraryDataFile.chunkCount() + 1; // +1 for the metadata file
}
public static boolean isFileRecent(Path filePath, long now, long cleanupAfter) {

7
src/main/resources/blockchain.json
View File

@ -4,10 +4,14 @@
"maxBlockSize": 2097152,
"maxBytesPerUnitFee": 1024,
"unitFee": "0.001",
"nameRegistrationUnitFee": "5",
"nameRegistrationUnitFeeTimestamp": 1645372800000,
"useBrokenMD160ForAddresses": false,
"requireGroupForApproval": false,
"defaultGroupId": 0,
"oneNamePerAccount": true,
"minAccountLevelToMint": 1,
"minAccountLevelForBlockSubmissions": 5,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 6,
"founderEffectiveMintingLevel": 10,
@ -51,7 +55,8 @@
"atFindNextTransactionFix": 275000,
"newBlockSigHeight": 320000,
"shareBinFix": 399000,
"calcChainWeightTimestamp": 1620579600000
"calcChainWeightTimestamp": 1620579600000,
"transactionV5Timestamp": 1642176000000
},
"genesisInfo": {
"version": 4,

9
src/main/resources/loading/index.html
View File

@ -46,6 +46,7 @@
var url = host + '/arbitrary/resource/status/' + service + '/' + name + '?build=true';
var textStatus = "Loading...";
var textProgress = "";
var retryInterval = 2500;
const response = await fetch(url, {
@ -82,7 +83,7 @@
textStatus = status.description;
retryInterval = 1000;
}
else if (status.status == "DOWNLOADING") {
else if (status.id == "DOWNLOADING") {
textStatus = status.description;
retryInterval = 1000;
}
@ -96,7 +97,12 @@
textStatus = status.description;
}
if (status.localChunkCount != null && status.totalChunkCount != null) {
textProgress = "Files downloaded: " + status.localChunkCount + " / " + status.totalChunkCount;
}
document.getElementById("status").innerHTML = textStatus;
document.getElementById("progress").innerHTML = textProgress;
setTimeout(checkStatus, retryInterval);
}
@ -260,6 +266,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
This page will refresh automatically when the content becomes available.
</p>
<p><span id="status">Loading...</span></p>
<p><span id="progress"></span></p>
</div>
</div>

7
src/test/java/org/qortal/test/api/AdminApiTests.java
View File

@ -2,10 +2,12 @@ package org.qortal.test.api;
import static org.junit.Assert.*;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.junit.Before;
import org.junit.Test;
import org.qortal.api.resource.AdminResource;
import org.qortal.repository.DataException;
import org.qortal.settings.Settings;
import org.qortal.test.common.ApiCommon;
import org.qortal.test.common.Common;
@ -29,7 +31,10 @@ public class AdminApiTests extends ApiCommon {
}
@Test
public void testSummary() {
public void testSummary() throws IllegalAccessException {
// Set localAuthBypassEnabled to true, since we don't need to test authentication here
FieldUtils.writeField(Settings.getInstance(), "localAuthBypassEnabled", true, true);
assertNotNull(this.adminResource.summary("testApiKey"));
}

5
src/test/java/org/qortal/test/api/NamesApiTests.java
View File

@ -16,6 +16,8 @@ import org.qortal.test.common.ApiCommon;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.utils.NTP;
public class NamesApiTests extends ApiCommon {
@ -47,6 +49,7 @@ public class NamesApiTests extends ApiCommon {
String name = "test-name";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
assertNotNull(this.namesResource.getNamesByAddress(alice.getAddress(), null, null, null));
@ -62,6 +65,7 @@ public class NamesApiTests extends ApiCommon {
String name = "test-name";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
assertNotNull(this.namesResource.getName(name));
@ -77,6 +81,7 @@ public class NamesApiTests extends ApiCommon {
long price = 1_23456789L;
TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Sell-name

3
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java
View File

@ -22,6 +22,7 @@ import org.qortal.test.common.ArbitraryUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
@ -153,6 +154,7 @@ public class ArbitraryDataStorageCapacityTests extends Common {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String aliceName = "alice";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength);
ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize);
@ -161,6 +163,7 @@ public class ArbitraryDataStorageCapacityTests extends Common {
PrivateKeyAccount bob = Common.getTestAccount(repository, "bob");
String bobName = "bob";
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, bob);
Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength);
ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize);

72
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java
View File

@ -21,7 +21,9 @@ import org.qortal.settings.Settings;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.nio.file.Path;
@ -59,25 +61,27 @@ public class ArbitraryDataStoragePolicyTests extends Common {
String name = "Test";
// Register the name to Alice
TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice);
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create transaction
ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name);
ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name);
// Add name to followed list
assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false));
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(transactionData));
assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
}
}
@ -92,25 +96,27 @@ public class ArbitraryDataStoragePolicyTests extends Common {
FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "FOLLOWED", true);
// Register the name to Alice
TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice);
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create transaction
ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name);
ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name);
// Add name to followed list
assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false));
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(transactionData));
assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We shouldn't store or pre-fetch data for this transaction
assertFalse(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
}
}
@ -125,25 +131,27 @@ public class ArbitraryDataStoragePolicyTests extends Common {
FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "VIEWED", true);
// Register the name to Alice
TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice);
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create transaction
ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name);
ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name);
// Add name to followed list
assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false));
// We should store but not pre-fetch data for this transaction
assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
}
}
@ -158,25 +166,27 @@ public class ArbitraryDataStoragePolicyTests extends Common {
FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "ALL", true);
// Register the name to Alice
TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice);
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create transaction
ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name);
ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name);
// Add name to followed list
assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false));
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(transactionData));
assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store and pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(transactionData));
assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
}
}
@ -191,25 +201,27 @@ public class ArbitraryDataStoragePolicyTests extends Common {
FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "NONE", true);
// Register the name to Alice
TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice);
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create transaction
ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name);
ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name);
// Add name to followed list
assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false));
// We shouldn't store or pre-fetch data for this transaction
assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy());
assertFalse(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We shouldn't store or pre-fetch data for this transaction
assertFalse(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
}
}

11
src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java
View File

@ -23,7 +23,9 @@ import org.qortal.test.common.ArbitraryUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.nio.file.Files;
@ -55,6 +57,7 @@ public class ArbitraryDataTests extends Common {
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
@ -149,6 +152,7 @@ public class ArbitraryDataTests extends Common {
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
@ -181,6 +185,7 @@ public class ArbitraryDataTests extends Common {
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
@ -226,6 +231,7 @@ public class ArbitraryDataTests extends Common {
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
@ -294,6 +300,7 @@ public class ArbitraryDataTests extends Common {
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
@ -343,6 +350,7 @@ public class ArbitraryDataTests extends Common {
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
@ -380,6 +388,7 @@ public class ArbitraryDataTests extends Common {
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
@ -409,6 +418,7 @@ public class ArbitraryDataTests extends Common {
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction
@ -435,6 +445,7 @@ public class ArbitraryDataTests extends Common {
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction

3
src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java
View File

@ -20,7 +20,9 @@ import org.qortal.test.common.ArbitraryUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.nio.file.Path;
@ -50,6 +52,7 @@ public class ArbitraryTransactionMetadataTests extends Common {
// Register the name to Alice
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Create PUT transaction

3
src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java
View File

@ -19,7 +19,9 @@ import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.nio.file.Path;
@ -46,6 +48,7 @@ public class ArbitraryTransactionTests extends Common {
// Register the name to Alice
RegisterNameTransactionData registerNameTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
registerNameTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(registerNameTransactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, registerNameTransactionData, alice);
// Set difficulty to 1

3
src/test/java/org/qortal/test/naming/BuySellTests.java
View File

@ -20,7 +20,9 @@ import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.utils.Amounts;
import org.qortal.utils.NTP;
public class BuySellTests extends Common {
@ -62,6 +64,7 @@ public class BuySellTests extends Common {
public void testRegisterName() throws DataException {
// Register-name
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
String name = transactionData.getName();

106
src/test/java/org/qortal/test/naming/IntegrityTests.java
View File

@ -11,7 +11,11 @@ import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.utils.NTP;
import java.util.List;
import static org.junit.Assert.*;
@ -31,6 +35,7 @@ public class IntegrityTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -45,6 +50,96 @@ public class IntegrityTests extends Common {
}
}
@Test
public void testBlankReducedName() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Register-name
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String name = "\uD83E\uDD73"; // Translates to a reducedName of ""
String data = "\uD83E\uDD73";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
assertEquals(data, repository.getNameRepository().fromName(name).getData());
// Ensure the reducedName is blank
assertEquals("", repository.getNameRepository().fromName(name).getReducedName());
// Run the database integrity check for this name
NamesDatabaseIntegrityCheck integrityCheck = new NamesDatabaseIntegrityCheck();
assertEquals(1, integrityCheck.rebuildName(name, repository));
// Ensure the name still exists and the data is still correct
assertEquals(data, repository.getNameRepository().fromName(name).getData());
assertEquals("", repository.getNameRepository().fromName(name).getReducedName());
}
}
@Test
public void testUpdateWithBlankNewName() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Register-name to Alice
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String name = "initial_name";
String data = "initial_data";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Update the name, but keep the new name blank
String newName = "";
String newData = "updated_data";
UpdateNameTransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), name, newName, newData);
TransactionUtils.signAndMint(repository, updateTransactionData, alice);
// Ensure the original name exists and the data is correct
assertEquals(name, repository.getNameRepository().fromName(name).getName());
assertEquals(newData, repository.getNameRepository().fromName(name).getData());
// Run the database integrity check for this name
NamesDatabaseIntegrityCheck integrityCheck = new NamesDatabaseIntegrityCheck();
assertEquals(2, integrityCheck.rebuildName(name, repository));
// Ensure the name still exists and the data is still correct
assertEquals(name, repository.getNameRepository().fromName(name).getName());
assertEquals(newData, repository.getNameRepository().fromName(name).getData());
}
}
@Test
public void testUpdateWithBlankNewNameAndBlankEmojiName() throws DataException {
// Attempt to simulate a real world problem where an emoji with blank reducedName
// confused the integrity check by associating it with previous UPDATE_NAME transactions
// due to them also having a blank "newReducedName"
try (final Repository repository = RepositoryManager.getRepository()) {
// Register-name to Alice
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String name = "initial_name";
String data = "initial_data";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Update the name, but keep the new name blank
String newName = "";
String newData = "updated_data";
UpdateNameTransactionData updateTransactionData = new UpdateNameTransactionData(TestTransaction.generateBase(alice), name, newName, newData);
TransactionUtils.signAndMint(repository, updateTransactionData, alice);
// Register emoji name
String emojiName = "\uD83E\uDD73"; // Translates to a reducedName of ""
// Ensure that the initial_name isn't associated with the emoji name
NamesDatabaseIntegrityCheck namesDatabaseIntegrityCheck = new NamesDatabaseIntegrityCheck();
List<TransactionData> transactions = namesDatabaseIntegrityCheck.fetchAllTransactionsInvolvingName(emojiName, repository);
assertEquals(0, transactions.size());
}
}
@Test
public void testMissingName() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
@ -54,6 +149,7 @@ public class IntegrityTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -83,6 +179,7 @@ public class IntegrityTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -121,6 +218,7 @@ public class IntegrityTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -146,6 +244,7 @@ public class IntegrityTests extends Common {
// Attempt to register the new name
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), newName, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
Transaction transaction = Transaction.fromData(repository, transactionData);
transaction.sign(alice);
@ -165,6 +264,7 @@ public class IntegrityTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -179,6 +279,7 @@ public class IntegrityTests extends Common {
// Attempt to register the name again
String duplicateName = "TEST-nÁme";
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), duplicateName, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
Transaction transaction = Transaction.fromData(repository, transactionData);
transaction.sign(alice);
@ -198,6 +299,7 @@ public class IntegrityTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -231,6 +333,7 @@ public class IntegrityTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -240,6 +343,7 @@ public class IntegrityTests extends Common {
String secondName = "new-missing-name";
String secondNameData = "{\"data2\":true}";
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), secondName, secondNameData);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the second name exists and the data is correct
@ -273,6 +377,7 @@ public class IntegrityTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -304,6 +409,7 @@ public class IntegrityTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct

69
src/test/java/org/qortal/test/naming/MiscTests.java
View File

@ -3,20 +3,26 @@ package org.qortal.test.naming;
import static org.junit.Assert.*;
import java.util.List;
import java.util.Optional;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.block.BlockChain;
import org.qortal.controller.BlockMinter;
import org.qortal.data.transaction.*;
import org.qortal.naming.Name;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.test.common.*;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.transaction.Transaction.ValidationResult;
import org.qortal.utils.NTP;
public class MiscTests extends Common {
@ -34,6 +40,7 @@ public class MiscTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
List<String> recentNames = repository.getNameRepository().getRecentNames(0L);
@ -53,11 +60,13 @@ public class MiscTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// duplicate
String duplicateName = "TEST-nÁme";
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), duplicateName, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
Transaction transaction = Transaction.fromData(repository, transactionData);
transaction.sign(alice);
@ -76,12 +85,14 @@ public class MiscTests extends Common {
String data = "{}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// duplicate (this time registered by Bob)
PrivateKeyAccount bob = Common.getTestAccount(repository, "bob");
String duplicateName = "TEST-nÁme";
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), duplicateName, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
Transaction transaction = Transaction.fromData(repository, transactionData);
transaction.sign(alice);
@ -100,12 +111,14 @@ public class MiscTests extends Common {
String data = "{\"age\":30}";
TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Register another name that we will later attempt to rename to first name (above)
String otherName = "new-name";
String otherData = "";
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), otherName, otherData);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// we shouldn't be able to update name to existing name
@ -129,6 +142,7 @@ public class MiscTests extends Common {
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
Transaction transaction = Transaction.fromData(repository, transactionData);
transaction.sign(alice);
@ -147,6 +161,7 @@ public class MiscTests extends Common {
String data = "{\"age\":30}";
TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// we shouldn't be able to update name to an address
@ -175,6 +190,7 @@ public class MiscTests extends Common {
// Register the name
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -201,6 +217,7 @@ public class MiscTests extends Common {
// Register the name
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -252,6 +269,7 @@ public class MiscTests extends Common {
// Register the name
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Ensure the name exists and the data is correct
@ -283,6 +301,7 @@ public class MiscTests extends Common {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
// Ensure the name doesn't exist
assertNull(repository.getNameRepository().fromName(name));
@ -304,4 +323,54 @@ public class MiscTests extends Common {
}
}
// test name registration fee increase
@Test
public void testRegisterNameFeeIncrease() throws DataException, IllegalAccessException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Set nameRegistrationUnitFeeTimestamp to a time far in the future
long futureTimestamp = 9999999999999L; // 20 Nov 2286
FieldUtils.writeField(BlockChain.getInstance(), "nameRegistrationUnitFeeTimestamp", futureTimestamp, true);
assertEquals(futureTimestamp, BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp());
// Validate unit fees pre and post timestamp
assertEquals(10000000, BlockChain.getInstance().getUnitFee()); // 0.1 QORT
assertEquals(500000000, BlockChain.getInstance().getNameRegistrationUnitFee()); // 5 QORT
// Register-name
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String name = "test-name";
String data = "{\"age\":30}";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
assertEquals(10000000L, transactionData.getFee().longValue());
TransactionUtils.signAndMint(repository, transactionData, alice);
// Set nameRegistrationUnitFeeTimestamp to a time in the past
Long now = NTP.getTime();
FieldUtils.writeField(BlockChain.getInstance(), "nameRegistrationUnitFeeTimestamp", now - 1000L, true);
assertEquals(now - 1000L, BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp());
// Register a different name
// First try with the default unit fee
String name2 = "test-name-2";
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, data);
assertEquals(10000000L, transactionData.getFee().longValue());
Transaction transaction = Transaction.fromData(repository, transactionData);
transaction.sign(alice);
ValidationResult result = transaction.importAsUnconfirmed();
assertTrue("Transaction should be invalid", ValidationResult.INSUFFICIENT_FEE == result);
// Now try using correct fee (this is specified by the UI, via the /transaction/unitfee API endpoint)
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, data);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
assertEquals(500000000L, transactionData.getFee().longValue());
transaction = Transaction.fromData(repository, transactionData);
transaction.sign(alice);
result = transaction.importAsUnconfirmed();
assertTrue("Transaction should be valid", ValidationResult.OK == result);
}
}
}

9
src/test/java/org/qortal/test/naming/UpdateTests.java
View File

@ -16,6 +16,8 @@ import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
import org.qortal.transaction.RegisterNameTransaction;
import org.qortal.utils.NTP;
public class UpdateTests extends Common {
@ -34,6 +36,7 @@ public class UpdateTests extends Common {
String initialData = "{\"age\":30}";
TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, initialTransactionData, alice);
// Check name, reduced name, and data exist
@ -100,6 +103,7 @@ public class UpdateTests extends Common {
String constantReducedName = "initia1-name";
TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, initialTransactionData, alice);
// Check initial name exists
@ -147,6 +151,7 @@ public class UpdateTests extends Common {
String initialData = "{\"age\":30}";
TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, initialTransactionData, alice);
// Check initial name exists
@ -225,6 +230,7 @@ public class UpdateTests extends Common {
String initialData = "{\"age\":30}";
TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Check initial name exists
@ -282,6 +288,7 @@ public class UpdateTests extends Common {
String initialData = "{\"age\":30}";
TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Check initial name exists
@ -323,6 +330,7 @@ public class UpdateTests extends Common {
String initialData = "{\"age\":30}";
TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Check initial name exists
@ -385,6 +393,7 @@ public class UpdateTests extends Common {
String initialData = "{\"age\":30}";
TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));;
TransactionUtils.signAndMint(repository, transactionData, alice);
// Check initial name exists

4
src/test/resources/test-chain-v2-founder-rewards.json
View File

@ -5,6 +5,7 @@
"maxBlockSize": 2097152,
"maxBytesPerUnitFee": 1024,
"unitFee": "0.1",
"nameRegistrationUnitFee": "5",
"requireGroupForApproval": false,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 20,
@ -48,7 +49,8 @@
"atFindNextTransactionFix": 0,
"newBlockSigHeight": 999999,
"shareBinFix": 999999,
"calcChainWeightTimestamp": 0
"calcChainWeightTimestamp": 0,
"transactionV5Timestamp": 0
},
"genesisInfo": {
"version": 4,

4
src/test/resources/test-chain-v2-leftover-reward.json
View File

@ -5,6 +5,7 @@
"maxBlockSize": 2097152,
"maxBytesPerUnitFee": 1024,
"unitFee": "0.1",
"nameRegistrationUnitFee": "5",
"requireGroupForApproval": false,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 20,
@ -48,7 +49,8 @@
"atFindNextTransactionFix": 0,
"newBlockSigHeight": 999999,
"shareBinFix": 999999,
"calcChainWeightTimestamp": 0
"calcChainWeightTimestamp": 0,
"transactionV5Timestamp": 0
},
"genesisInfo": {
"version": 4,

4
src/test/resources/test-chain-v2-minting.json
View File

@ -5,6 +5,7 @@
"maxBlockSize": 2097152,
"maxBytesPerUnitFee": 1024,
"unitFee": "0.1",
"nameRegistrationUnitFee": "5",
"requireGroupForApproval": false,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 20,
@ -48,7 +49,8 @@
"atFindNextTransactionFix": 0,
"newBlockSigHeight": 999999,
"shareBinFix": 999999,
"calcChainWeightTimestamp": 0
"calcChainWeightTimestamp": 0,
"transactionV5Timestamp": 0
},
"genesisInfo": {
"version": 4,

4
src/test/resources/test-chain-v2-qora-holder-extremes.json
View File

@ -5,6 +5,7 @@
"maxBlockSize": 2097152,
"maxBytesPerUnitFee": 1024,
"unitFee": "0.1",
"nameRegistrationUnitFee": "5",
"requireGroupForApproval": false,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 20,
@ -48,7 +49,8 @@
"atFindNextTransactionFix": 0,
"newBlockSigHeight": 999999,
"shareBinFix": 999999,
"calcChainWeightTimestamp": 0
"calcChainWeightTimestamp": 0,
"transactionV5Timestamp": 0
},
"genesisInfo": {
"version": 4,

4
src/test/resources/test-chain-v2-qora-holder.json
View File

@ -5,6 +5,7 @@
"maxBlockSize": 2097152,
"maxBytesPerUnitFee": 1024,
"unitFee": "0.1",
"nameRegistrationUnitFee": "5",
"requireGroupForApproval": false,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 20,
@ -48,7 +49,8 @@
"atFindNextTransactionFix": 0,
"newBlockSigHeight": 999999,
"shareBinFix": 999999,
"calcChainWeightTimestamp": 0
"calcChainWeightTimestamp": 0,
"transactionV5Timestamp": 0
},
"genesisInfo": {
"version": 4,

4
src/test/resources/test-chain-v2-reward-levels.json
View File

@ -5,6 +5,7 @@
"maxBlockSize": 2097152,
"maxBytesPerUnitFee": 1024,
"unitFee": "0.1",
"nameRegistrationUnitFee": "5",
"requireGroupForApproval": false,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 20,
@ -48,7 +49,8 @@
"atFindNextTransactionFix": 0,
"newBlockSigHeight": 999999,
"shareBinFix": 6,
"calcChainWeightTimestamp": 0
"calcChainWeightTimestamp": 0,
"transactionV5Timestamp": 0
},
"genesisInfo": {
"version": 4,

4
src/test/resources/test-chain-v2-reward-scaling.json
View File

@ -5,6 +5,7 @@
"maxBlockSize": 2097152,
"maxBytesPerUnitFee": 1024,
"unitFee": "0.1",
"nameRegistrationUnitFee": "5",
"requireGroupForApproval": false,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 20,
@ -48,7 +49,8 @@
"atFindNextTransactionFix": 0,
"newBlockSigHeight": 999999,
"shareBinFix": 999999,
"calcChainWeightTimestamp": 0
"calcChainWeightTimestamp": 0,
"transactionV5Timestamp": 0
},
"genesisInfo": {
"version": 4,

4
src/test/resources/test-chain-v2.json
View File

@ -5,6 +5,7 @@
"maxBlockSize": 2097152,
"maxBytesPerUnitFee": 1024,
"unitFee": "0.1",
"nameRegistrationUnitFee": "5",
"requireGroupForApproval": false,
"minAccountLevelToRewardShare": 5,
"maxRewardSharesPerMintingAccount": 20,
@ -48,7 +49,8 @@
"atFindNextTransactionFix": 0,
"newBlockSigHeight": 999999,
"shareBinFix": 999999,
"calcChainWeightTimestamp": 0
"calcChainWeightTimestamp": 0,
"transactionV5Timestamp": 0
},
"genesisInfo": {
"version": 4,

3
src/test/resources/test-settings-v2.json
View File

@ -15,6 +15,5 @@
"tempDataPath": "data-test/_temp",
"listsPath": "lists-test",
"storagePolicy": "FOLLOWED_OR_VIEWED",
"maxStorageCapacity": 104857600,
"localAuthBypassEnabled": true
"maxStorageCapacity": 104857600
}