greenflame089/qortal
1
0
Fork 0
forked from Qortal-Forker/qortal
Code Pull Requests Releases Activity

Merge pull request #250 from kennycud/master

Browse Source 
Merging current 'test release' from kennycud repo after extensive testing by community.
This commit is contained in:
crowetic
2025-04-08 08:34:07 -07:00
committed by GitHub
parent faee7c8f6a bcf3538d18
commit 0b37666d2b
51 changed files with 2678 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/org/qortal/account/Account.java
View File

@@ -14,6 +14,7 @@ import org.qortal.repository.NameRepository;
import org.qortal.repository.Repository;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import org.qortal.utils.Groups;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -227,7 +228,7 @@ public class Account {
}
int level = accountData.getLevel();
int groupIdToMint = BlockChain.getInstance().getMintingGroupId();
List<Integer> groupIdsToMint = Groups.getGroupIdsToMint( BlockChain.getInstance(), blockchainHeight );
int nameCheckHeight = BlockChain.getInstance().getOnlyMintWithNameHeight();
int groupCheckHeight = BlockChain.getInstance().getGroupMemberCheckHeight();
int removeNameCheckHeight = BlockChain.getInstance().getRemoveOnlyMintWithNameHeight();
@@ -261,9 +262,9 @@ public class Account {
if (blockchainHeight >= groupCheckHeight && blockchainHeight < removeNameCheckHeight) {
List<NameData> myName = nameRepository.getNamesByOwner(myAddress);
if (Account.isFounder(accountData.getFlags())) {
return accountData.getBlocksMintedPenalty() == 0 && !myName.isEmpty() && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress));
return accountData.getBlocksMintedPenalty() == 0 && !myName.isEmpty() && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
} else {
return level >= levelToMint && !myName.isEmpty() && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress));
return level >= levelToMint && !myName.isEmpty() && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
}
}
@@ -272,9 +273,9 @@ public class Account {
// Account's address is a member of the minter group
if (blockchainHeight >= removeNameCheckHeight) {
if (Account.isFounder(accountData.getFlags())) {
return accountData.getBlocksMintedPenalty() == 0 && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress));
return accountData.getBlocksMintedPenalty() == 0 && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
} else {
return level >= levelToMint && (isGroupValidated || groupRepository.memberExists(groupIdToMint, myAddress));
return level >= levelToMint && (isGroupValidated || Groups.memberExistsInAnyGroup(groupRepository, groupIdsToMint, myAddress));
}
}

1
src/main/java/org/qortal/api/ApiService.java
View File

@@ -194,6 +194,7 @@ public class ApiService {
context.addServlet(AdminStatusWebSocket.class, "/websockets/admin/status");
context.addServlet(BlocksWebSocket.class, "/websockets/blocks");
context.addServlet(DataMonitorSocket.class, "/websockets/datamonitor");
context.addServlet(ActiveChatsWebSocket.class, "/websockets/chat/active/*");
context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages");
context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers");

72
src/main/java/org/qortal/api/model/CrossChainTradeLedgerEntry.java Normal file
View File

@@ -0,0 +1,72 @@
package org.qortal.api.model;
import io.swagger.v3.oas.annotations.media.Schema;
import org.qortal.data.crosschain.CrossChainTradeData;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
public class CrossChainTradeLedgerEntry {
private String market;
private String currency;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long quantity;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long feeAmount;
private String feeCurrency;
@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
private long totalPrice;
private long tradeTimestamp;
protected CrossChainTradeLedgerEntry() {
/* For JAXB */
}
public CrossChainTradeLedgerEntry(String market, String currency, long quantity, long feeAmount, String feeCurrency, long totalPrice, long tradeTimestamp) {
this.market = market;
this.currency = currency;
this.quantity = quantity;
this.feeAmount = feeAmount;
this.feeCurrency = feeCurrency;
this.totalPrice = totalPrice;
this.tradeTimestamp = tradeTimestamp;
}
public String getMarket() {
return market;
}
public String getCurrency() {
return currency;
}
public long getQuantity() {
return quantity;
}
public long getFeeAmount() {
return feeAmount;
}
public String getFeeCurrency() {
return feeCurrency;
}
public long getTotalPrice() {
return totalPrice;
}
public long getTradeTimestamp() {
return tradeTimestamp;
}
}

94
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@@ -33,9 +33,13 @@ import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.controller.arbitrary.ArbitraryMetadataManager;
import org.qortal.data.account.AccountData;
import org.qortal.data.arbitrary.ArbitraryCategoryInfo;
import org.qortal.data.arbitrary.ArbitraryDataIndexDetail;
import org.qortal.data.arbitrary.ArbitraryDataIndexScoreKey;
import org.qortal.data.arbitrary.ArbitraryDataIndexScorecard;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
import org.qortal.data.arbitrary.ArbitraryResourceStatus;
import org.qortal.data.arbitrary.IndexCache;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
@@ -69,8 +73,11 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@Path("/arbitrary")
@Tag(name = "Arbitrary")
@@ -172,6 +179,7 @@ public class ArbitraryResource {
@Parameter(description = "Name (searches name field only)") @QueryParam("name") List<String> names,
@Parameter(description = "Title (searches title metadata field only)") @QueryParam("title") String title,
@Parameter(description = "Description (searches description metadata field only)") @QueryParam("description") String description,
@Parameter(description = "Keyword (searches description metadata field by keywords)") @QueryParam("keywords") List<String> keywords,
@Parameter(description = "Prefix only (if true, only the beginning of fields are matched)") @QueryParam("prefix") Boolean prefixOnly,
@Parameter(description = "Exact match names only (if true, partial name matches are excluded)") @QueryParam("exactmatchnames") Boolean exactMatchNamesOnly,
@Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource,
@@ -212,7 +220,7 @@ public class ArbitraryResource {
}
List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
.searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly,
.searchArbitraryResources(service, query, identifier, names, title, description, keywords, usePrefixOnly,
exactMatchNames, defaultRes, mode, minLevel, followedOnly, excludeBlocked, includeMetadata, includeStatus,
before, after, limit, offset, reverse);
@@ -1185,6 +1193,90 @@ public class ArbitraryResource {
}
}
@GET
@Path("/indices")
@Operation(
summary = "Find matching arbitrary resource indices",
description = "",
responses = {
@ApiResponse(
description = "indices",
content = @Content(
array = @ArraySchema(
schema = @Schema(
implementation = ArbitraryDataIndexScorecard.class
)
)
)
)
}
)
public List<ArbitraryDataIndexScorecard> searchIndices(@QueryParam("terms") String[] terms) {
List<ArbitraryDataIndexDetail> indices = new ArrayList<>();
// get index details for each term
for( String term : terms ) {
List<ArbitraryDataIndexDetail> details = IndexCache.getInstance().getIndicesByTerm().get(term);
if( details != null ) {
indices.addAll(details);
}
}
// sum up the scores for each index with identical attributes
Map<ArbitraryDataIndexScoreKey, Double> scoreForKey
= indices.stream()
.collect(
Collectors.groupingBy(
index -> new ArbitraryDataIndexScoreKey(index.name, index.category, index.link),
Collectors.summingDouble(detail -> 1.0 / detail.rank)
)
);
// create scorecards for each index group and put them in descending order by score
List<ArbitraryDataIndexScorecard> scorecards
= scoreForKey.entrySet().stream().map(
entry
->
new ArbitraryDataIndexScorecard(
entry.getValue(),
entry.getKey().name,
entry.getKey().category,
entry.getKey().link)
)
.sorted(Comparator.comparingDouble(ArbitraryDataIndexScorecard::getScore).reversed())
.collect(Collectors.toList());
return scorecards;
}
@GET
@Path("/indices/{name}/{idPrefix}")
@Operation(
summary = "Find matching arbitrary resource indices for a registered name and identifier prefix",
description = "",
responses = {
@ApiResponse(
description = "indices",
content = @Content(
array = @ArraySchema(
schema = @Schema(
implementation = ArbitraryDataIndexDetail.class
)
)
)
)
}
)
public List<ArbitraryDataIndexDetail> searchIndicesByName(@PathParam("name") String name, @PathParam("idPrefix") String idPrefix) {
return
IndexCache.getInstance().getIndicesByIssuer()
.getOrDefault(name, new ArrayList<>(0)).stream()
.filter( indexDetail -> indexDetail.indexIdentifer.startsWith(idPrefix))
.collect(Collectors.toList());
}
// Shared methods

141
src/main/java/org/qortal/api/resource/CrossChainResource.java
View File

@@ -10,11 +10,13 @@ import io.swagger.v3.oas.annotations.parameters.RequestBody;
import io.swagger.v3.oas.annotations.responses.ApiResponse;
import io.swagger.v3.oas.annotations.security.SecurityRequirement;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.glassfish.jersey.media.multipart.ContentDisposition;
import org.qortal.api.ApiError;
import org.qortal.api.ApiErrors;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.Security;
import org.qortal.api.model.CrossChainCancelRequest;
import org.qortal.api.model.CrossChainTradeLedgerEntry;
import org.qortal.api.model.CrossChainTradeSummary;
import org.qortal.controller.tradebot.TradeBot;
import org.qortal.crosschain.ACCT;
@@ -44,10 +46,14 @@ import org.qortal.utils.Base58;
import org.qortal.utils.ByteArray;
import org.qortal.utils.NTP;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import java.io.IOException;
import java.util.*;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@@ -61,6 +67,13 @@ public class CrossChainResource {
@Context
HttpServletRequest request;
@Context
HttpServletResponse response;
@Context
ServletContext context;
@GET
@Path("/tradeoffers")
@Operation(
@@ -258,11 +271,11 @@ public class CrossChainResource {
example = "1597310000000"
) @QueryParam("minimumTimestamp") Long minimumTimestamp,
@Parameter(
description = "Optionally filter by buyer Qortal address"
) @QueryParam("buyerAddress") String buyerAddress,
description = "Optionally filter by buyer Qortal public key"
) @QueryParam("buyerPublicKey") String buyerPublicKey58,
@Parameter(
description = "Optionally filter by seller Qortal address"
) @QueryParam("sellerAddress") String sellerAddress,
description = "Optionally filter by seller Qortal public key"
) @QueryParam("sellerPublicKey") String sellerPublicKey58,
@Parameter( ref = "limit") @QueryParam("limit") Integer limit,
@Parameter( ref = "offset" ) @QueryParam("offset") Integer offset,
@Parameter( ref = "reverse" ) @QueryParam("reverse") Boolean reverse) {
@@ -274,6 +287,10 @@ public class CrossChainResource {
if (minimumTimestamp != null && minimumTimestamp <= 0)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
// Decode public keys
byte[] buyerPublicKey = decodePublicKey(buyerPublicKey58);
byte[] sellerPublicKey = decodePublicKey(sellerPublicKey58);
final Boolean isFinished = Boolean.TRUE;
try (final Repository repository = RepositoryManager.getRepository()) {
@@ -304,7 +321,7 @@ public class CrossChainResource {
byte[] codeHash = acctInfo.getKey().value;
ACCT acct = acctInfo.getValue().get();
List<ATStateData> atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, buyerAddress, sellerAddress,
List<ATStateData> atStates = repository.getATRepository().getMatchingFinalATStates(codeHash, buyerPublicKey, sellerPublicKey,
isFinished, acct.getModeByteOffset(), (long) AcctMode.REDEEMED.value, minimumFinalHeight,
limit, offset, reverse);
@@ -343,6 +360,120 @@ public class CrossChainResource {
}
}
/**
* Decode Public Key
*
* @param publicKey58 the public key in a string
*
* @return the public key in bytes
*/
private byte[] decodePublicKey(String publicKey58) {
if( publicKey58 == null ) return null;
if( publicKey58.isEmpty() ) return new byte[0];
byte[] publicKey;
try {
publicKey = Base58.decode(publicKey58);
} catch (NumberFormatException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY, e);
}
// Correct size for public key?
if (publicKey.length != Transformer.PUBLIC_KEY_LENGTH)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PUBLIC_KEY);
return publicKey;
}
@GET
@Path("/ledger/{publicKey}")
@Operation(
summary = "Accounting entries for all trades.",
description = "Returns accounting entries for all completed cross-chain trades",
responses = {
@ApiResponse(
content = @Content(
schema = @Schema(
type = "string",
format = "byte"
)
)
)
}
)
@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.REPOSITORY_ISSUE})
public HttpServletResponse getLedgerEntries(
@PathParam("publicKey") String publicKey58,
@Parameter(
description = "Only return trades that completed on/after this timestamp (milliseconds since epoch)",
example = "1597310000000"
) @QueryParam("minimumTimestamp") Long minimumTimestamp) {
byte[] publicKey = decodePublicKey(publicKey58);
// minimumTimestamp (if given) needs to be positive
if (minimumTimestamp != null && minimumTimestamp <= 0)
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
try (final Repository repository = RepositoryManager.getRepository()) {
Integer minimumFinalHeight = null;
if (minimumTimestamp != null) {
minimumFinalHeight = repository.getBlockRepository().getHeightFromTimestamp(minimumTimestamp);
// If not found in the block repository it will return either 0 or 1
if (minimumFinalHeight == 0 || minimumFinalHeight == 1) {
// Try the archive
minimumFinalHeight = repository.getBlockArchiveRepository().getHeightFromTimestamp(minimumTimestamp);
}
if (minimumFinalHeight == 0)
// We don't have any blocks since minimumTimestamp, let alone trades, so nothing to return
return response;
// height returned from repository is for block BEFORE timestamp
// but we want trades AFTER timestamp so bump height accordingly
minimumFinalHeight++;
}
List<CrossChainTradeLedgerEntry> crossChainTradeLedgerEntries = new ArrayList<>();
Map<ByteArray, Supplier<ACCT>> acctsByCodeHash = SupportedBlockchain.getAcctMap();
// collect ledger entries for each ACCT
for (Map.Entry<ByteArray, Supplier<ACCT>> acctInfo : acctsByCodeHash.entrySet()) {
byte[] codeHash = acctInfo.getKey().value;
ACCT acct = acctInfo.getValue().get();
// collect buys and sells
CrossChainUtils.collectLedgerEntries(publicKey, repository, minimumFinalHeight, crossChainTradeLedgerEntries, codeHash, acct, true);
CrossChainUtils.collectLedgerEntries(publicKey, repository, minimumFinalHeight, crossChainTradeLedgerEntries, codeHash, acct, false);
}
crossChainTradeLedgerEntries.sort((a, b) -> Longs.compare(a.getTradeTimestamp(), b.getTradeTimestamp()));
response.setStatus(HttpServletResponse.SC_OK);
response.setContentType("text/csv");
response.setHeader(
HttpHeaders.CONTENT_DISPOSITION,
ContentDisposition
.type("attachment")
.fileName(CrossChainUtils.createLedgerFileName(Crypto.toAddress(publicKey)))
.build()
.toString()
);
CrossChainUtils.writeToLedger( response.getWriter(), crossChainTradeLedgerEntries);
return response;
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
} catch (IOException e) {
response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
return response;
}
}
@GET
@Path("/price/{blockchain}")
@Operation(

139
src/main/java/org/qortal/api/resource/CrossChainUtils.java
View File

@@ -10,21 +10,36 @@ import org.bitcoinj.script.ScriptBuilder;
import org.bouncycastle.util.Strings;
import org.json.simple.JSONObject;
import org.qortal.api.model.CrossChainTradeLedgerEntry;
import org.qortal.api.model.crosschain.BitcoinyTBDRequest;
import org.qortal.crosschain.*;
import org.qortal.data.at.ATData;
import org.qortal.data.at.ATStateData;
import org.qortal.data.crosschain.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.utils.Amounts;
import org.qortal.utils.BitTwiddling;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Writer;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.*;
import java.util.stream.Collectors;
public class CrossChainUtils {
public static final String QORT_CURRENCY_CODE = "QORT";
private static final Logger LOGGER = LogManager.getLogger(CrossChainUtils.class);
public static final String CORE_API_CALL = "Core API Call";
public static final String QORTAL_EXCHANGE_LABEL = "Qortal";
public static ServerConfigurationInfo buildServerConfigurationInfo(Bitcoiny blockchain) {
@@ -632,4 +647,128 @@ public class CrossChainUtils {
byte[] lockTimeABytes = BitTwiddling.toBEByteArray((long) lockTimeA);
return Bytes.concat(partnerBitcoinPKH, hashOfSecretA, lockTimeABytes);
}
/**
* Write To Ledger
*
* @param writer the writer to the ledger
* @param entries the entries to write to the ledger
*
* @throws IOException
*/
public static void writeToLedger(Writer writer, List<CrossChainTradeLedgerEntry> entries) throws IOException {
BufferedWriter bufferedWriter = new BufferedWriter(writer);
StringJoiner header = new StringJoiner(",");
header.add("Market");
header.add("Currency");
header.add("Quantity");
header.add("Commission Paid");
header.add("Commission Currency");
header.add("Total Price");
header.add("Date Time");
header.add("Exchange");
bufferedWriter.append(header.toString());
DateFormat dateFormatter = new SimpleDateFormat("yyyyMMdd HH:mm");
dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC"));
for( CrossChainTradeLedgerEntry entry : entries ) {
StringJoiner joiner = new StringJoiner(",");
joiner.add(entry.getMarket());
joiner.add(entry.getCurrency());
joiner.add(String.valueOf(Amounts.prettyAmount(entry.getQuantity())));
joiner.add(String.valueOf(Amounts.prettyAmount(entry.getFeeAmount())));
joiner.add(entry.getFeeCurrency());
joiner.add(String.valueOf(Amounts.prettyAmount(entry.getTotalPrice())));
joiner.add(dateFormatter.format(new Date(entry.getTradeTimestamp())));
joiner.add(QORTAL_EXCHANGE_LABEL);
bufferedWriter.newLine();
bufferedWriter.append(joiner.toString());
}
bufferedWriter.newLine();
bufferedWriter.flush();
}
/**
* Create Ledger File Name
*
* Create a file name the includes timestamp and address.
*
* @param address the address
*
* @return the file name created
*/
public static String createLedgerFileName(String address) {
DateFormat dateFormatter = new SimpleDateFormat("yyyyMMddHHmmss");
String fileName = "ledger-" + address + "-" + dateFormatter.format(new Date());
return fileName;
}
/**
* Collect Ledger Entries
*
* @param publicKey the public key for the ledger entries, buy and sell
* @param repository the data repository
* @param minimumFinalHeight the minimum block height for entries to be collected
* @param entries the ledger entries to add to
* @param codeHash code hash for the entry blockchain
* @param acct the ACCT for the entry blockchain
* @param isBuy true collecting entries for a buy, otherwise false
*
* @throws DataException
*/
public static void collectLedgerEntries(
byte[] publicKey,
Repository repository,
Integer minimumFinalHeight,
List<CrossChainTradeLedgerEntry> entries,
byte[] codeHash,
ACCT acct,
boolean isBuy) throws DataException {
// get all the final AT states for the code hash (foreign coin)
List<ATStateData> atStates
= repository.getATRepository().getMatchingFinalATStates(
codeHash,
isBuy ? publicKey : null,
!isBuy ? publicKey : null,
Boolean.TRUE, acct.getModeByteOffset(),
(long) AcctMode.REDEEMED.value,
minimumFinalHeight,
null, null, false
);
String foreignBlockchainCurrencyCode = acct.getBlockchain().getCurrencyCode();
// for each trade, build ledger entry, collect ledger entry
for (ATStateData atState : atStates) {
CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atState);
// We also need block timestamp for use as trade timestamp
long localTimestamp = repository.getBlockRepository().getTimestampFromHeight(atState.getHeight());
if (localTimestamp == 0) {
// Try the archive
localTimestamp = repository.getBlockArchiveRepository().getTimestampFromHeight(atState.getHeight());
}
CrossChainTradeLedgerEntry ledgerEntry
= new CrossChainTradeLedgerEntry(
isBuy ? QORT_CURRENCY_CODE : foreignBlockchainCurrencyCode,
isBuy ? foreignBlockchainCurrencyCode : QORT_CURRENCY_CODE,
isBuy ? crossChainTradeData.qortAmount : crossChainTradeData.expectedForeignAmount,
0,
foreignBlockchainCurrencyCode,
isBuy ? crossChainTradeData.expectedForeignAmount : crossChainTradeData.qortAmount,
localTimestamp);
entries.add(ledgerEntry);
}
}
}

102
src/main/java/org/qortal/api/websocket/DataMonitorSocket.java Normal file
View File

@@ -0,0 +1,102 @@
package org.qortal.api.websocket;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.eclipse.jetty.websocket.api.Session;
import org.eclipse.jetty.websocket.api.WebSocketException;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketClose;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketConnect;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketError;
import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
import org.eclipse.jetty.websocket.api.annotations.WebSocket;
import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
import org.qortal.api.ApiError;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.DataMonitorInfo;
import org.qortal.event.DataMonitorEvent;
import org.qortal.event.Event;
import org.qortal.event.EventBus;
import org.qortal.event.Listener;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.utils.Base58;
import java.io.IOException;
import java.io.StringWriter;
import java.util.List;
@WebSocket
@SuppressWarnings("serial")
public class DataMonitorSocket extends ApiWebSocket implements Listener {
private static final Logger LOGGER = LogManager.getLogger(DataMonitorSocket.class);
@Override
public void configure(WebSocketServletFactory factory) {
LOGGER.info("configure");
factory.register(DataMonitorSocket.class);
EventBus.INSTANCE.addListener(this);
}
@Override
public void listen(Event event) {
if (!(event instanceof DataMonitorEvent))
return;
DataMonitorEvent dataMonitorEvent = (DataMonitorEvent) event;
for (Session session : getSessions())
sendDataEventSummary(session, buildInfo(dataMonitorEvent));
}
private DataMonitorInfo buildInfo(DataMonitorEvent dataMonitorEvent) {
return new DataMonitorInfo(
dataMonitorEvent.getTimestamp(),
dataMonitorEvent.getIdentifier(),
dataMonitorEvent.getName(),
dataMonitorEvent.getService(),
dataMonitorEvent.getDescription(),
dataMonitorEvent.getTransactionTimestamp(),
dataMonitorEvent.getLatestPutTimestamp()
);
}
@OnWebSocketConnect
@Override
public void onWebSocketConnect(Session session) {
super.onWebSocketConnect(session);
}
@OnWebSocketClose
@Override
public void onWebSocketClose(Session session, int statusCode, String reason) {
super.onWebSocketClose(session, statusCode, reason);
}
@OnWebSocketError
public void onWebSocketError(Session session, Throwable throwable) {
/* We ignore errors for now, but method here to silence log spam */
}
@OnWebSocketMessage
public void onWebSocketMessage(Session session, String message) {
LOGGER.info("onWebSocketMessage: message = " + message);
}
private void sendDataEventSummary(Session session, DataMonitorInfo dataMonitorInfo) {
StringWriter stringWriter = new StringWriter();
try {
marshall(stringWriter, dataMonitorInfo);
session.getRemote().sendStringByFuture(stringWriter.toString());
} catch (IOException | WebSocketException e) {
// No output this time
}
}
}

10
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@@ -439,7 +439,15 @@ public class ArbitraryDataReader {
// Ensure the complete hash matches the joined chunks
if (!Arrays.equals(arbitraryDataFile.digest(), transactionData.getData())) {
// Delete the invalid file
arbitraryDataFile.delete();
LOGGER.info("Deleting invalid file: path = " + arbitraryDataFile.getFilePath());
if( arbitraryDataFile.delete() ) {
LOGGER.info("Deleted invalid file successfully: path = " + arbitraryDataFile.getFilePath());
}
else {
LOGGER.warn("Could not delete invalid file: path = " + arbitraryDataFile.getFilePath());
}
throw new DataException("Unable to validate complete file hash");
}
}

23
src/main/java/org/qortal/block/Block.java
View File

@@ -39,6 +39,7 @@ import org.qortal.transform.block.BlockTransformer;
import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Amounts;
import org.qortal.utils.Base58;
import org.qortal.utils.Groups;
import org.qortal.utils.NTP;
import java.io.ByteArrayOutputStream;
@@ -150,7 +151,7 @@ public class Block {
final BlockChain blockChain = BlockChain.getInstance();
ExpandedAccount(Repository repository, RewardShareData rewardShareData) throws DataException {
ExpandedAccount(Repository repository, RewardShareData rewardShareData, int blockHeight) throws DataException {
this.rewardShareData = rewardShareData;
this.sharePercent = this.rewardShareData.getSharePercent();
@@ -159,7 +160,12 @@ public class Block {
this.isMinterFounder = Account.isFounder(mintingAccountData.getFlags());
this.isRecipientAlsoMinter = this.rewardShareData.getRecipient().equals(this.mintingAccount.getAddress());
this.isMinterMember = repository.getGroupRepository().memberExists(BlockChain.getInstance().getMintingGroupId(), this.mintingAccount.getAddress());
this.isMinterMember
= Groups.memberExistsInAnyGroup(
repository.getGroupRepository(),
Groups.getGroupIdsToMint(BlockChain.getInstance(), blockHeight),
this.mintingAccount.getAddress()
);
if (this.isRecipientAlsoMinter) {
// Self-share: minter is also recipient
@@ -435,9 +441,9 @@ public class Block {
if (height >= BlockChain.getInstance().getGroupMemberCheckHeight()) {
onlineAccounts.removeIf(a -> {
try {
int groupId = BlockChain.getInstance().getMintingGroupId();
List<Integer> groupIdsToMint = Groups.getGroupIdsToMint(BlockChain.getInstance(), height);
String address = Account.getRewardShareMintingAddress(repository, a.getPublicKey());
boolean isMinterGroupMember = repository.getGroupRepository().memberExists(groupId, address);
boolean isMinterGroupMember = Groups.memberExistsInAnyGroup(repository.getGroupRepository(), groupIdsToMint, address);
return !isMinterGroupMember;
} catch (DataException e) {
// Something went wrong, so remove the account
@@ -753,7 +759,7 @@ public class Block {
List<ExpandedAccount> expandedAccounts = new ArrayList<>();
for (RewardShareData rewardShare : this.cachedOnlineRewardShares) {
expandedAccounts.add(new ExpandedAccount(repository, rewardShare));
expandedAccounts.add(new ExpandedAccount(repository, rewardShare, this.blockData.getHeight()));
}
this.cachedExpandedAccounts = expandedAccounts;
@@ -2485,11 +2491,10 @@ public class Block {
try (final Repository repository = RepositoryManager.getRepository()) {
GroupRepository groupRepository = repository.getGroupRepository();
List<Integer> mintingGroupIds = Groups.getGroupIdsToMint(BlockChain.getInstance(), this.blockData.getHeight());
// all minter admins
List<String> minterAdmins
= groupRepository.getGroupAdmins(BlockChain.getInstance().getMintingGroupId()).stream()
.map(GroupAdminData::getAdmin)
.collect(Collectors.toList());
List<String> minterAdmins = Groups.getAllAdmins(groupRepository, mintingGroupIds);
// all minter admins that are online
List<ExpandedAccount> onlineMinterAdminAccounts

12
src/main/java/org/qortal/block/BlockChain.java
View File

@@ -212,7 +212,13 @@ public class BlockChain {
private int minAccountLevelToRewardShare;
private int maxRewardSharesPerFounderMintingAccount;
private int founderEffectiveMintingLevel;
private int mintingGroupId;
public static class IdsForHeight {
public int height;
public List<Integer> ids;
}
private List<IdsForHeight> mintingGroupIds;
/** Minimum time to retain online account signatures (ms) for block validity checks. */
private long onlineAccountSignaturesMinLifetime;
@@ -544,8 +550,8 @@ public class BlockChain {
return this.onlineAccountSignaturesMaxLifetime;
}
public int getMintingGroupId() {
return this.mintingGroupId;
public List<IdsForHeight> getMintingGroupIds() {
return mintingGroupIds;
}
public CiyamAtSettings getCiyamAtSettings() {

16
src/main/java/org/qortal/controller/Controller.java
View File

@@ -423,6 +423,12 @@ public class Controller extends Thread {
LOGGER.info("Db Cache Disabled");
}
LOGGER.info("Arbitrary Indexing Starting ...");
ArbitraryIndexUtils.startCaching(
Settings.getInstance().getArbitraryIndexingPriority(),
Settings.getInstance().getArbitraryIndexingFrequency()
);
if( Settings.getInstance().isBalanceRecorderEnabled() ) {
Optional<HSQLDBBalanceRecorder> recorder = HSQLDBBalanceRecorder.getInstance();
@@ -541,6 +547,16 @@ public class Controller extends Thread {
ArbitraryDataStorageManager.getInstance().start();
ArbitraryDataRenderManager.getInstance().start();
// start rebuild arbitrary resource cache timer task
if( Settings.getInstance().isRebuildArbitraryResourceCacheTaskEnabled() ) {
new Timer().schedule(
new RebuildArbitraryResourceCacheTask(),
Settings.getInstance().getRebuildArbitraryResourceCacheTaskDelay() * RebuildArbitraryResourceCacheTask.MILLIS_IN_MINUTE,
Settings.getInstance().getRebuildArbitraryResourceCacheTaskPeriod() * RebuildArbitraryResourceCacheTask.MILLIS_IN_HOUR
);
}
LOGGER.info("Starting online accounts manager");
OnlineAccountsManager.getInstance().start();

12
src/main/java/org/qortal/controller/OnlineAccountsManager.java
View File

@@ -25,6 +25,7 @@ import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.utils.Base58;
import org.qortal.utils.Groups;
import org.qortal.utils.NTP;
import org.qortal.utils.NamedThreadFactory;
@@ -225,11 +226,14 @@ public class OnlineAccountsManager {
Set<OnlineAccountData> onlineAccountsToAdd = new HashSet<>();
Set<OnlineAccountData> onlineAccountsToRemove = new HashSet<>();
try (final Repository repository = RepositoryManager.getRepository()) {
int blockHeight = repository.getBlockRepository().getBlockchainHeight();
List<String> mintingGroupMemberAddresses
= repository.getGroupRepository()
.getGroupMembers(BlockChain.getInstance().getMintingGroupId()).stream()
.map(GroupMemberData::getMember)
.collect(Collectors.toList());
= Groups.getAllMembers(
repository.getGroupRepository(),
Groups.getGroupIdsToMint(BlockChain.getInstance(), blockHeight)
);
for (OnlineAccountData onlineAccountData : this.onlineAccountsImportQueue) {
if (isStopping)

167
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java
View File

@@ -2,22 +2,30 @@ package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.event.DataMonitorEvent;
import org.qortal.event.EventBus;
import org.qortal.gui.SplashFrame;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.utils.Base58;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
public class ArbitraryDataCacheManager extends Thread {
@@ -29,6 +37,11 @@ public class ArbitraryDataCacheManager extends Thread {
/** Queue of arbitrary transactions that require cache updates */
private final List<ArbitraryTransactionData> updateQueue = Collections.synchronizedList(new ArrayList<>());
private static final NumberFormat FORMATTER = NumberFormat.getNumberInstance();
static {
FORMATTER.setGroupingUsed(true);
}
public static synchronized ArbitraryDataCacheManager getInstance() {
if (instance == null) {
@@ -45,17 +58,22 @@ public class ArbitraryDataCacheManager extends Thread {
try {
while (!Controller.isStopping()) {
Thread.sleep(500L);
try {
Thread.sleep(500L);
// Process queue
processResourceQueue();
// Process queue
processResourceQueue();
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
Thread.sleep(600_000L); // wait 10 minutes to continue
}
}
} catch (InterruptedException e) {
// Fall through to exit thread
}
// Clear queue before terminating thread
processResourceQueue();
// Clear queue before terminating thread
processResourceQueue();
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
public void shutdown() {
@@ -85,14 +103,25 @@ public class ArbitraryDataCacheManager extends Thread {
// Update arbitrary resource caches
try {
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCache(repository);
arbitraryTransaction.updateArbitraryMetadataCache(repository);
arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, new HashSet<>(0), new HashMap<>(0));
repository.saveChanges();
// Update status as separate commit, as this is more prone to failure
arbitraryTransaction.updateArbitraryResourceStatus(repository);
repository.saveChanges();
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
transactionData.getIdentifier(),
transactionData.getName(),
transactionData.getService().name(),
"updated resource cache and status, queue",
transactionData.getTimestamp(),
transactionData.getTimestamp()
)
);
LOGGER.debug(() -> String.format("Finished processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature())));
} catch (DataException e) {
@@ -103,6 +132,9 @@ public class ArbitraryDataCacheManager extends Thread {
} catch (DataException e) {
LOGGER.error("Repository issue while processing arbitrary resource cache updates", e);
}
catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
public void addToUpdateQueue(ArbitraryTransactionData transactionData) {
@@ -148,34 +180,66 @@ public class ArbitraryDataCacheManager extends Thread {
LOGGER.info("Building arbitrary resources cache...");
SplashFrame.getInstance().updateStatus("Building QDN cache - please wait...");
final int batchSize = 100;
final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository().getLatestArbitraryTransactions();
LOGGER.info("arbitrary transactions: count = " + allArbitraryTransactionsInDescendingOrder.size());
List<ArbitraryResourceData> resources = repository.getArbitraryRepository().getArbitraryResources(null, null, true);
Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper = new HashMap<>(resources.size());
for( ArbitraryResourceData resource : resources ) {
resourceByWrapper.put(
new ArbitraryTransactionDataHashWrapper(resource.service.value, resource.name, resource.identifier),
resource
);
}
LOGGER.info("arbitrary resources: count = " + resourceByWrapper.size());
Set<ArbitraryTransactionDataHashWrapper> latestTransactionsWrapped = new HashSet<>(allArbitraryTransactionsInDescendingOrder.size());
// Loop through all ARBITRARY transactions, and determine latest state
while (!Controller.isStopping()) {
LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
LOGGER.info(
"Fetching arbitrary transactions {} - {} / {} Total",
FORMATTER.format(offset),
FORMATTER.format(offset+batchSize-1),
FORMATTER.format(allArbitraryTransactionsInDescendingOrder.size())
);
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false);
if (signatures.isEmpty()) {
List<ArbitraryTransactionData> transactionsToProcess
= allArbitraryTransactionsInDescendingOrder.stream()
.skip(offset)
.limit(batchSize)
.collect(Collectors.toList());
if (transactionsToProcess.isEmpty()) {
// Complete
break;
}
// Expand signatures to transactions
for (byte[] signature : signatures) {
ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository
.getTransactionRepository().fromSignature(signature);
try {
for( ArbitraryTransactionData transactionData : transactionsToProcess) {
if (transactionData.getService() == null) {
// Unsupported service - ignore this resource
continue;
}
if (transactionData.getService() == null) {
// Unsupported service - ignore this resource
continue;
latestTransactionsWrapped.add(new ArbitraryTransactionDataHashWrapper(transactionData));
// Update arbitrary resource caches
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCacheIncludingMetadata(repository, latestTransactionsWrapped, resourceByWrapper);
}
// Update arbitrary resource caches
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceCache(repository);
arbitraryTransaction.updateArbitraryMetadataCache(repository);
repository.saveChanges();
} catch (DataException e) {
repository.discardChanges();
LOGGER.error(e.getMessage(), e);
}
offset += batchSize;
}
@@ -193,6 +257,11 @@ public class ArbitraryDataCacheManager extends Thread {
repository.discardChanges();
throw new DataException("Build of arbitrary resources cache failed.");
}
catch (Exception e) {
LOGGER.error(e.getMessage(), e);
return false;
}
}
private boolean refreshArbitraryStatuses(Repository repository) throws DataException {
@@ -200,27 +269,48 @@ public class ArbitraryDataCacheManager extends Thread {
LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions...");
SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait...");
final int batchSize = 100;
final int batchSize = Settings.getInstance().getBuildArbitraryResourcesBatchSize();
int offset = 0;
List<ArbitraryTransactionData> allHostedTransactions
= ArbitraryDataStorageManager.getInstance()
.listAllHostedTransactions(repository, null, null);
// Loop through all ARBITRARY transactions, and determine latest state
while (!Controller.isStopping()) {
LOGGER.info("Fetching hosted transactions {} - {}", offset, offset+batchSize-1);
LOGGER.info(
"Fetching hosted transactions {} - {} / {} Total",
FORMATTER.format(offset),
FORMATTER.format(offset+batchSize-1),
FORMATTER.format(allHostedTransactions.size())
);
List<ArbitraryTransactionData> hostedTransactions
= allHostedTransactions.stream()
.skip(offset)
.limit(batchSize)
.collect(Collectors.toList());
List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, batchSize, offset);
if (hostedTransactions.isEmpty()) {
// Complete
break;
}
// Loop through hosted transactions
for (ArbitraryTransactionData transactionData : hostedTransactions) {
try {
// Loop through hosted transactions
for (ArbitraryTransactionData transactionData : hostedTransactions) {
// Determine status and update cache
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceStatus(repository);
// Determine status and update cache
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
arbitraryTransaction.updateArbitraryResourceStatus(repository);
}
repository.saveChanges();
} catch (DataException e) {
repository.discardChanges();
LOGGER.error(e.getMessage(), e);
}
offset += batchSize;
}
@@ -234,6 +324,11 @@ public class ArbitraryDataCacheManager extends Thread {
repository.discardChanges();
throw new DataException("Refresh of arbitrary resource statuses failed.");
}
catch (Exception e) {
LOGGER.error(e.getMessage(), e);
return false;
}
}
}

160
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@@ -2,9 +2,10 @@ package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.event.DataMonitorEvent;
import org.qortal.event.EventBus;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
@@ -21,8 +22,12 @@ import java.nio.file.Paths;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD;
@@ -77,6 +82,19 @@ public class ArbitraryDataCleanupManager extends Thread {
final int limit = 100;
int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
try (final Repository repository = RepositoryManager.getRepository()) {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
} catch( Exception e) {
LOGGER.error(e.getMessage(), e);
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
}
Set<ArbitraryTransactionData> processedTransactions = new HashSet<>();
try {
while (!isStopping) {
Thread.sleep(30000);
@@ -107,27 +125,31 @@ public class ArbitraryDataCleanupManager extends Thread {
// Any arbitrary transactions we want to fetch data for?
try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
// LOGGER.info("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
List<ArbitraryTransactionData> transactions = allArbitraryTransactionsInDescendingOrder.stream().skip(offset).limit(limit).collect(Collectors.toList());
if (isStopping) {
return;
}
if (signatures == null || signatures.isEmpty()) {
if (transactions == null || transactions.isEmpty()) {
offset = 0;
continue;
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
transactions = allArbitraryTransactionsInDescendingOrder.stream().limit(limit).collect(Collectors.toList());
processedTransactions.clear();
}
offset += limit;
now = NTP.getTime();
// Loop through the signatures in this batch
for (int i=0; i<signatures.size(); i++) {
for (int i=0; i<transactions.size(); i++) {
if (isStopping) {
return;
}
byte[] signature = signatures.get(i);
if (signature == null) {
ArbitraryTransactionData arbitraryTransactionData = transactions.get(i);
if (arbitraryTransactionData == null) {
continue;
}
@@ -136,9 +158,7 @@ public class ArbitraryDataCleanupManager extends Thread {
Thread.sleep(5000);
}
// Fetch the transaction data
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
if (arbitraryTransactionData == null || arbitraryTransactionData.getService() == null) {
if (arbitraryTransactionData.getService() == null) {
continue;
}
@@ -147,6 +167,8 @@ public class ArbitraryDataCleanupManager extends Thread {
continue;
}
boolean mostRecentTransaction = processedTransactions.add(arbitraryTransactionData);
// Check if we have the complete file
boolean completeFileExists = ArbitraryTransactionUtils.completeFileExists(arbitraryTransactionData);
@@ -167,20 +189,54 @@ public class ArbitraryDataCleanupManager extends Thread {
LOGGER.info("Deleting transaction {} because we can't host its data",
Base58.encode(arbitraryTransactionData.getSignature()));
ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"can't store data, deleting",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
continue;
}
// Check to see if we have had a more recent PUT
boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
if (hasMoreRecentPutTransaction) {
if (!mostRecentTransaction) {
// There is a more recent PUT transaction than the one we are currently processing.
// When a PUT is issued, it replaces any layers that would have been there before.
// Therefore any data relating to this older transaction is no longer needed.
LOGGER.info(String.format("Newer PUT found for %s %s since transaction %s. " +
"Deleting all files associated with the earlier transaction.", arbitraryTransactionData.getService(),
arbitraryTransactionData.getName(), Base58.encode(signature)));
arbitraryTransactionData.getName(), Base58.encode(arbitraryTransactionData.getSignature())));
ArbitraryTransactionUtils.deleteCompleteFileAndChunks(arbitraryTransactionData);
Optional<ArbitraryTransactionData> moreRecentPutTransaction
= processedTransactions.stream()
.filter(data -> data.equals(arbitraryTransactionData))
.findAny();
if( moreRecentPutTransaction.isPresent() ) {
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"deleting data due to replacement",
arbitraryTransactionData.getTimestamp(),
moreRecentPutTransaction.get().getTimestamp()
)
);
}
else {
LOGGER.warn("Something went wrong with the most recent put transaction determination!");
}
continue;
}
@@ -199,7 +255,21 @@ public class ArbitraryDataCleanupManager extends Thread {
LOGGER.debug(String.format("Transaction %s has complete file and all chunks",
Base58.encode(arbitraryTransactionData.getSignature())));
ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
boolean wasDeleted = ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, STALE_FILE_TIMEOUT);
if( wasDeleted ) {
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"deleting file, retaining chunks",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
}
continue;
}
@@ -237,17 +307,6 @@ public class ArbitraryDataCleanupManager extends Thread {
this.storageLimitReached(repository);
}
// Delete random data associated with name if we're over our storage limit for this name
// Use the DELETION_THRESHOLD, for the same reasons as above
for (String followedName : ListUtils.followedNames()) {
if (isStopping) {
return;
}
if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) {
this.storageLimitReachedForName(repository, followedName);
}
}
} catch (DataException e) {
LOGGER.error("Repository issue when cleaning up arbitrary transaction data", e);
}
@@ -326,25 +385,6 @@ public class ArbitraryDataCleanupManager extends Thread {
// FUTURE: consider reducing the expiry time of the reader cache
}
public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException {
// We think that the storage limit has been reached for supplied name - but we should double check
if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) {
// We have space available for this name, so don't delete anything
return;
}
// Delete a batch of random chunks associated with this name
// This reduces the chance of too many nodes deleting the same chunk
// when they reach their storage limit
Path dataPath = Paths.get(Settings.getInstance().getDataPath());
for (int i=0; i<CHUNK_DELETION_BATCH_SIZE; i++) {
if (isStopping) {
return;
}
this.deleteRandomFile(repository, dataPath.toFile(), name);
}
}
/**
* Iteratively walk through given directory and delete a single random file
*
@@ -423,6 +463,7 @@ public class ArbitraryDataCleanupManager extends Thread {
}
LOGGER.info("Deleting random file {} because we have reached max storage capacity...", randomItem.toString());
fireRandomItemDeletionNotification(randomItem, repository, "Deleting random file, because we have reached max storage capacity");
boolean success = randomItem.delete();
if (success) {
try {
@@ -437,6 +478,35 @@ public class ArbitraryDataCleanupManager extends Thread {
return false;
}
private void fireRandomItemDeletionNotification(File randomItem, Repository repository, String reason) {
try {
Path parentFileNamePath = randomItem.toPath().toAbsolutePath().getParent().getFileName();
if (parentFileNamePath != null) {
String signature58 = parentFileNamePath.toString();
byte[] signature = Base58.decode(signature58);
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData != null && transactionData.getType() == Transaction.TransactionType.ARBITRARY) {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
reason,
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
}
}
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
private void cleanupTempDirectory(String folder, long now, long minAge) {
String baseDir = Settings.getInstance().getTempDataPath();
Path tempDir = Paths.get(baseDir, folder);

21
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataExamination.java Normal file
View File

@@ -0,0 +1,21 @@
package org.qortal.controller.arbitrary;
public class ArbitraryDataExamination {
private boolean pass;
private String notes;
public ArbitraryDataExamination(boolean pass, String notes) {
this.pass = pass;
this.notes = notes;
}
public boolean isPass() {
return pass;
}
public String getNotes() {
return notes;
}
}

2
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java
View File

@@ -5,6 +5,8 @@ import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.event.DataMonitorEvent;
import org.qortal.event.EventBus;
import org.qortal.network.Peer;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;

180
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@@ -10,6 +10,8 @@ import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.event.DataMonitorEvent;
import org.qortal.event.EventBus;
import org.qortal.network.Network;
import org.qortal.network.Peer;
import org.qortal.repository.DataException;
@@ -28,6 +30,7 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
public class ArbitraryDataManager extends Thread {
@@ -195,13 +198,35 @@ public class ArbitraryDataManager extends Thread {
final int limit = 100;
int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
try (final Repository repository = RepositoryManager.getRepository()) {
if( name == null ) {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
}
else {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactionsByName(name);
}
} catch( Exception e) {
LOGGER.error(e.getMessage(), e);
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
}
// collect processed transactions in a set to ensure outdated data transactions do not get fetched
Set<ArbitraryTransactionDataHashWrapper> processedTransactions = new HashSet<>();
while (!isStopping) {
Thread.sleep(1000L);
// Any arbitrary transactions we want to fetch data for?
try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, name, null, ConfirmationStatus.BOTH, limit, offset, true);
// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
List<byte[]> signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions);
if (signatures == null || signatures.isEmpty()) {
offset = 0;
break;
@@ -223,14 +248,38 @@ public class ArbitraryDataManager extends Thread {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
// Skip transactions that we don't need to proactively store data for
if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) {
ArbitraryDataExamination arbitraryDataExamination = storageManager.shouldPreFetchData(repository, arbitraryTransactionData);
if (!arbitraryDataExamination.isPass()) {
iterator.remove();
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
arbitraryDataExamination.getNotes(),
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
continue;
}
// Remove transactions that we already have local data for
if (hasLocalData(arbitraryTransaction)) {
iterator.remove();
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"already have local data, skipping",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
}
}
@@ -248,8 +297,21 @@ public class ArbitraryDataManager extends Thread {
// Check to see if we have had a more recent PUT
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
if (hasMoreRecentPutTransaction) {
Optional<ArbitraryTransactionData> moreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
if (moreRecentPutTransaction.isPresent()) {
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"not fetching old data",
arbitraryTransactionData.getTimestamp(),
moreRecentPutTransaction.get().getTimestamp()
)
);
// There is a more recent PUT transaction than the one we are currently processing.
// When a PUT is issued, it replaces any layers that would have been there before.
// Therefore any data relating to this older transaction is no longer needed and we
@@ -257,10 +319,34 @@ public class ArbitraryDataManager extends Thread {
continue;
}
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"fetching data",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
// Ask our connected peers if they have files for this signature
// This process automatically then fetches the files themselves if a peer is found
fetchData(arbitraryTransactionData);
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"fetched data",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
} catch (DataException e) {
LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
}
@@ -274,6 +360,20 @@ public class ArbitraryDataManager extends Thread {
final int limit = 100;
int offset = 0;
List<ArbitraryTransactionData> allArbitraryTransactionsInDescendingOrder;
try (final Repository repository = RepositoryManager.getRepository()) {
allArbitraryTransactionsInDescendingOrder
= repository.getArbitraryRepository()
.getLatestArbitraryTransactions();
} catch( Exception e) {
LOGGER.error(e.getMessage(), e);
allArbitraryTransactionsInDescendingOrder = new ArrayList<>(0);
}
// collect processed transactions in a set to ensure outdated data transactions do not get fetched
Set<ArbitraryTransactionDataHashWrapper> processedTransactions = new HashSet<>();
while (!isStopping) {
final int minSeconds = 3;
final int maxSeconds = 10;
@@ -282,8 +382,8 @@ public class ArbitraryDataManager extends Thread {
// Any arbitrary transactions we want to fetch data for?
try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
List<byte[]> signatures = processTransactionsForSignatures(limit, offset, allArbitraryTransactionsInDescendingOrder, processedTransactions);
if (signatures == null || signatures.isEmpty()) {
offset = 0;
break;
@@ -328,26 +428,74 @@ public class ArbitraryDataManager extends Thread {
continue;
}
// Check to see if we have had a more recent PUT
// No longer need to see if we have had a more recent PUT since we compared the transactions to process
// to the transactions previously processed, so we can fetch the transactiondata, notify the event bus,
// fetch the metadata and notify the event bus again
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
if (hasMoreRecentPutTransaction) {
// There is a more recent PUT transaction than the one we are currently processing.
// When a PUT is issued, it replaces any layers that would have been there before.
// Therefore any data relating to this older transaction is no longer needed and we
// shouldn't fetch it from the network.
continue;
}
// Ask our connected peers if they have metadata for this signature
fetchMetadata(arbitraryTransactionData);
EventBus.INSTANCE.notify(
new DataMonitorEvent(
System.currentTimeMillis(),
arbitraryTransactionData.getIdentifier(),
arbitraryTransactionData.getName(),
arbitraryTransactionData.getService().name(),
"fetched metadata",
arbitraryTransactionData.getTimestamp(),
arbitraryTransactionData.getTimestamp()
)
);
} catch (DataException e) {
LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
}
}
}
private static List<byte[]> processTransactionsForSignatures(
int limit,
int offset,
List<ArbitraryTransactionData> transactionsInDescendingOrder,
Set<ArbitraryTransactionDataHashWrapper> processedTransactions) {
// these transactions are in descending order, latest transactions come first
List<ArbitraryTransactionData> transactions
= transactionsInDescendingOrder.stream()
.skip(offset)
.limit(limit)
.collect(Collectors.toList());
// wrap the transactions, so they can be used for hashing and comparing
// Class ArbitraryTransactionDataHashWrapper supports hashCode() and equals(...) for this purpose
List<ArbitraryTransactionDataHashWrapper> wrappedTransactions
= transactions.stream()
.map(transaction -> new ArbitraryTransactionDataHashWrapper(transaction))
.collect(Collectors.toList());
// create a set of wrappers and populate it first to last, so that all outdated transactions get rejected
Set<ArbitraryTransactionDataHashWrapper> transactionsToProcess = new HashSet<>(wrappedTransactions.size());
for(ArbitraryTransactionDataHashWrapper wrappedTransaction : wrappedTransactions) {
transactionsToProcess.add(wrappedTransaction);
}
// remove the matches for previously processed transactions,
// because these transactions have had updates that have already been processed
transactionsToProcess.removeAll(processedTransactions);
// add to processed transactions to compare and remove matches from future processing iterations
processedTransactions.addAll(transactionsToProcess);
List<byte[]> signatures
= transactionsToProcess.stream()
.map(transactionToProcess -> transactionToProcess.getData()
.getSignature())
.collect(Collectors.toList());
return signatures;
}
private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) {
try {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);

74
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@@ -155,31 +155,24 @@ public class ArbitraryDataStorageManager extends Thread {
* @param arbitraryTransactionData - the transaction
* @return boolean - whether to prefetch or not
*/
public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
public ArbitraryDataExamination shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
String name = arbitraryTransactionData.getName();
// Only fetch data associated with hashes, as we already have RAW_DATA
if (arbitraryTransactionData.getDataType() != ArbitraryTransactionData.DataType.DATA_HASH) {
return false;
return new ArbitraryDataExamination(false, "Only fetch data associated with hashes");
}
// Don't fetch anything more if we're (nearly) out of space
// Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
// avoid a fetch/delete loop
if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) {
return false;
}
// Don't fetch anything if we're (nearly) out of space for this name
// Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
// avoid a fetch/delete loop
if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
return false;
return new ArbitraryDataExamination(false,"Don't fetch anything more if we're (nearly) out of space");
}
// Don't store data unless it's an allowed type (public/private)
if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
return false;
return new ArbitraryDataExamination(false, "Don't store data unless it's an allowed type (public/private)");
}
// Handle transactions without names differently
@@ -189,21 +182,21 @@ public class ArbitraryDataStorageManager extends Thread {
// Never fetch data from blocked names, even if they are followed
if (ListUtils.isNameBlocked(name)) {
return false;
return new ArbitraryDataExamination(false, "blocked name");
}
switch (Settings.getInstance().getStoragePolicy()) {
case FOLLOWED:
case FOLLOWED_OR_VIEWED:
return ListUtils.isFollowingName(name);
return new ArbitraryDataExamination(ListUtils.isFollowingName(name), Settings.getInstance().getStoragePolicy().name());
case ALL:
return true;
return new ArbitraryDataExamination(true, Settings.getInstance().getStoragePolicy().name());
case NONE:
case VIEWED:
default:
return false;
return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
}
}
@@ -214,17 +207,17 @@ public class ArbitraryDataStorageManager extends Thread {
*
* @return boolean - whether the storage policy allows for unnamed data
*/
private boolean shouldPreFetchDataWithoutName() {
private ArbitraryDataExamination shouldPreFetchDataWithoutName() {
switch (Settings.getInstance().getStoragePolicy()) {
case ALL:
return true;
return new ArbitraryDataExamination(true, "Fetching all data");
case NONE:
case VIEWED:
case FOLLOWED:
case FOLLOWED_OR_VIEWED:
default:
return false;
return new ArbitraryDataExamination(false, Settings.getInstance().getStoragePolicy().name());
}
}
@@ -484,51 +477,6 @@ public class ArbitraryDataStorageManager extends Thread {
return true;
}
public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) {
if (!this.isStorageSpaceAvailable(threshold)) {
// No storage space available at all, so no need to check this name
return false;
}
if (Settings.getInstance().getStoragePolicy() == StoragePolicy.ALL) {
// Using storage policy ALL, so don't limit anything per name
return true;
}
if (name == null) {
// This transaction doesn't have a name, so fall back to total space limitations
return true;
}
int followedNamesCount = ListUtils.followedNamesCount();
if (followedNamesCount == 0) {
// Not following any names, so we have space
return true;
}
long totalSizeForName = 0;
long maxStoragePerName = this.storageCapacityPerName(threshold);
// Fetch all hosted transactions
List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository, null, null);
for (ArbitraryTransactionData transactionData : hostedTransactions) {
String transactionName = transactionData.getName();
if (!Objects.equals(name, transactionName)) {
// Transaction relates to a different name
continue;
}
totalSizeForName += transactionData.getSize();
}
// Have we reached the limit for this name?
if (totalSizeForName > maxStoragePerName) {
return false;
}
return true;
}
public long storageCapacityPerName(double threshold) {
int followedNamesCount = ListUtils.followedNamesCount();
if (followedNamesCount == 0) {

48
src/main/java/org/qortal/controller/arbitrary/ArbitraryTransactionDataHashWrapper.java Normal file
View File

@@ -0,0 +1,48 @@
package org.qortal.controller.arbitrary;
import org.qortal.arbitrary.misc.Service;
import org.qortal.data.transaction.ArbitraryTransactionData;
import java.util.Objects;
public class ArbitraryTransactionDataHashWrapper {
private ArbitraryTransactionData data;
private int service;
private String name;
private String identifier;
public ArbitraryTransactionDataHashWrapper(ArbitraryTransactionData data) {
this.data = data;
this.service = data.getService().value;
this.name = data.getName();
this.identifier = data.getIdentifier();
}
public ArbitraryTransactionDataHashWrapper(int service, String name, String identifier) {
this.service = service;
this.name = name;
this.identifier = identifier;
}
public ArbitraryTransactionData getData() {
return data;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ArbitraryTransactionDataHashWrapper that = (ArbitraryTransactionDataHashWrapper) o;
return service == that.service && name.equals(that.name) && Objects.equals(identifier, that.identifier);
}
@Override
public int hashCode() {
return Objects.hash(service, name, identifier);
}
}

33
src/main/java/org/qortal/controller/arbitrary/RebuildArbitraryResourceCacheTask.java Normal file
View File

@@ -0,0 +1,33 @@
package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import java.util.TimerTask;
public class RebuildArbitraryResourceCacheTask extends TimerTask {
private static final Logger LOGGER = LogManager.getLogger(RebuildArbitraryResourceCacheTask.class);
public static final long MILLIS_IN_HOUR = 60 * 60 * 1000;
public static final long MILLIS_IN_MINUTE = 60 * 1000;
private static final String REBUILD_ARBITRARY_RESOURCE_CACHE_TASK = "Rebuild Arbitrary Resource Cache Task";
@Override
public void run() {
Thread.currentThread().setName(REBUILD_ARBITRARY_RESOURCE_CACHE_TASK);
try (final Repository repository = RepositoryManager.getRepository()) {
ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, true);
}
catch( DataException e ) {
LOGGER.error(e.getMessage(), e);
}
}
}

1
src/main/java/org/qortal/crosschain/Bitcoiny.java
View File

@@ -83,6 +83,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
return this.bitcoinjContext;
}
@Override
public String getCurrencyCode() {
return this.currencyCode;
}

2
src/main/java/org/qortal/crosschain/ForeignBlockchain.java
View File

@@ -2,6 +2,8 @@ package org.qortal.crosschain;
public interface ForeignBlockchain {
public String getCurrencyCode();
public boolean isValidAddress(String address);
public boolean isValidWalletKey(String walletKey);

34
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndex.java Normal file
View File

@@ -0,0 +1,34 @@
package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndex {
public String t;
public String n;
public int c;
public String l;
public ArbitraryDataIndex() {}
public ArbitraryDataIndex(String t, String n, int c, String l) {
this.t = t;
this.n = n;
this.c = c;
this.l = l;
}
@Override
public String toString() {
return "ArbitraryDataIndex{" +
"t='" + t + '\'' +
", n='" + n + '\'' +
", c=" + c +
", l='" + l + '\'' +
'}';
}
}

41
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexDetail.java Normal file
View File

@@ -0,0 +1,41 @@
package org.qortal.data.arbitrary;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndexDetail {
public String issuer;
public int rank;
public String term;
public String name;
public int category;
public String link;
public String indexIdentifer;
public ArbitraryDataIndexDetail() {}
public ArbitraryDataIndexDetail(String issuer, int rank, ArbitraryDataIndex index, String indexIdentifer) {
this.issuer = issuer;
this.rank = rank;
this.term = index.t;
this.name = index.n;
this.category = index.c;
this.link = index.l;
this.indexIdentifer = indexIdentifer;
}
@Override
public String toString() {
return "ArbitraryDataIndexDetail{" +
"issuer='" + issuer + '\'' +
", rank=" + rank +
", term='" + term + '\'' +
", name='" + name + '\'' +
", category=" + category +
", link='" + link + '\'' +
", indexIdentifer='" + indexIdentifer + '\'' +
'}';
}
}

38
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScoreKey.java Normal file
View File

@@ -0,0 +1,38 @@
package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.Objects;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndexScoreKey {
public String name;
public int category;
public String link;
public ArbitraryDataIndexScoreKey() {}
public ArbitraryDataIndexScoreKey(String name, int category, String link) {
this.name = name;
this.category = category;
this.link = link;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ArbitraryDataIndexScoreKey that = (ArbitraryDataIndexScoreKey) o;
return category == that.category && Objects.equals(name, that.name) && Objects.equals(link, that.link);
}
@Override
public int hashCode() {
return Objects.hash(name, category, link);
}
}

38
src/main/java/org/qortal/data/arbitrary/ArbitraryDataIndexScorecard.java Normal file
View File

@@ -0,0 +1,38 @@
package org.qortal.data.arbitrary;
import org.qortal.arbitrary.misc.Service;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryDataIndexScorecard {
public double score;
public String name;
public int category;
public String link;
public ArbitraryDataIndexScorecard() {}
public ArbitraryDataIndexScorecard(double score, String name, int category, String link) {
this.score = score;
this.name = name;
this.category = category;
this.link = link;
}
public double getScore() {
return score;
}
@Override
public String toString() {
return "ArbitraryDataIndexScorecard{" +
"score=" + score +
", name='" + name + '\'' +
", category=" + category +
", link='" + link + '\'' +
'}';
}
}

57
src/main/java/org/qortal/data/arbitrary/DataMonitorInfo.java Normal file
View File

@@ -0,0 +1,57 @@
package org.qortal.data.arbitrary;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class DataMonitorInfo {
private long timestamp;
private String identifier;
private String name;
private String service;
private String description;
private long transactionTimestamp;
private long latestPutTimestamp;
public DataMonitorInfo() {
}
public DataMonitorInfo(long timestamp, String identifier, String name, String service, String description, long transactionTimestamp, long latestPutTimestamp) {
this.timestamp = timestamp;
this.identifier = identifier;
this.name = name;
this.service = service;
this.description = description;
this.transactionTimestamp = transactionTimestamp;
this.latestPutTimestamp = latestPutTimestamp;
}
public long getTimestamp() {
return timestamp;
}
public String getIdentifier() {
return identifier;
}
public String getName() {
return name;
}
public String getService() {
return service;
}
public String getDescription() {
return description;
}
public long getTransactionTimestamp() {
return transactionTimestamp;
}
public long getLatestPutTimestamp() {
return latestPutTimestamp;
}
}

23
src/main/java/org/qortal/data/arbitrary/IndexCache.java Normal file
View File

@@ -0,0 +1,23 @@
package org.qortal.data.arbitrary;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
public class IndexCache {
public static final IndexCache SINGLETON = new IndexCache();
private ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> indicesByTerm = new ConcurrentHashMap<>();
private ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> indicesByIssuer = new ConcurrentHashMap<>();
public static IndexCache getInstance() {
return SINGLETON;
}
public ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> getIndicesByTerm() {
return indicesByTerm;
}
public ConcurrentHashMap<String, List<ArbitraryDataIndexDetail>> getIndicesByIssuer() {
return indicesByIssuer;
}
}

22
src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java
View File

@@ -200,4 +200,26 @@ public class ArbitraryTransactionData extends TransactionData {
return this.payments;
}
@Override
public String toString() {
return "ArbitraryTransactionData{" +
"version=" + version +
", service=" + service +
", nonce=" + nonce +
", size=" + size +
", name='" + name + '\'' +
", identifier='" + identifier + '\'' +
", method=" + method +
", compression=" + compression +
", dataType=" + dataType +
", type=" + type +
", timestamp=" + timestamp +
", fee=" + fee +
", txGroupId=" + txGroupId +
", blockHeight=" + blockHeight +
", blockSequence=" + blockSequence +
", approvalStatus=" + approvalStatus +
", approvalHeight=" + approvalHeight +
'}';
}
}

57
src/main/java/org/qortal/event/DataMonitorEvent.java Normal file
View File

@@ -0,0 +1,57 @@
package org.qortal.event;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class DataMonitorEvent implements Event{
private long timestamp;
private String identifier;
private String name;
private String service;
private String description;
private long transactionTimestamp;
private long latestPutTimestamp;
public DataMonitorEvent() {
}
public DataMonitorEvent(long timestamp, String identifier, String name, String service, String description, long transactionTimestamp, long latestPutTimestamp) {
this.timestamp = timestamp;
this.identifier = identifier;
this.name = name;
this.service = service;
this.description = description;
this.transactionTimestamp = transactionTimestamp;
this.latestPutTimestamp = latestPutTimestamp;
}
public long getTimestamp() {
return timestamp;
}
public String getIdentifier() {
return identifier;
}
public String getName() {
return name;
}
public String getService() {
return service;
}
public String getDescription() {
return description;
}
public long getTransactionTimestamp() {
return transactionTimestamp;
}
public long getLatestPutTimestamp() {
return latestPutTimestamp;
}
}

4
src/main/java/org/qortal/group/Group.java
View File

@@ -674,8 +674,8 @@ public class Group {
public void uninvite(GroupInviteTransactionData groupInviteTransactionData) throws DataException {
String invitee = groupInviteTransactionData.getInvitee();
// If member exists then they were added when invite matched join request
if (this.memberExists(invitee)) {
// If member exists and the join request is present then they were added when invite matched join request
if (this.memberExists(invitee) && groupInviteTransactionData.getJoinReference() != null) {
// Rebuild join request using cached reference to transaction that created join request.
this.rebuildJoinRequest(invitee, groupInviteTransactionData.getJoinReference());

6
src/main/java/org/qortal/repository/ATRepository.java
View File

@@ -76,9 +76,9 @@ public interface ATRepository {
* Although <tt>expectedValue</tt>, if provided, is natively an unsigned long,
* the data segment comparison is done via unsigned hex string.
*/
public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, String buyerAddress, String sellerAddress, Boolean isFinished,
Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished,
Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
Integer limit, Integer offset, Boolean reverse) throws DataException;
/**
* Returns final ATStateData for ATs matching codeHash (required)

6
src/main/java/org/qortal/repository/ArbitraryRepository.java
View File

@@ -27,6 +27,10 @@ public interface ArbitraryRepository {
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException;
List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName(String name) throws DataException;
public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException;
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
@@ -42,7 +46,7 @@ public interface ArbitraryRepository {
public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException;
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, List<String> keywords, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException;
List<ArbitraryResourceData> searchArbitraryResourcesSimple(
Service service,

29
src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java
View File

@@ -5,6 +5,7 @@ import com.google.common.primitives.Longs;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.Controller;
import org.qortal.crypto.Crypto;
import org.qortal.data.at.ATData;
import org.qortal.data.at.ATStateData;
import org.qortal.repository.ATRepository;
@@ -403,9 +404,9 @@ public class HSQLDBATRepository implements ATRepository {
}
@Override
public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, String buyerAddress, String sellerAddress, Boolean isFinished,
Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
Integer limit, Integer offset, Boolean reverse) throws DataException {
public List<ATStateData> getMatchingFinalATStates(byte[] codeHash, byte[] buyerPublicKey, byte[] sellerPublicKey, Boolean isFinished,
Integer dataByteOffset, Long expectedValue, Integer minimumFinalHeight,
Integer limit, Integer offset, Boolean reverse) throws DataException {
StringBuilder sql = new StringBuilder(1024);
List<Object> bindParams = new ArrayList<>();
@@ -426,9 +427,9 @@ public class HSQLDBATRepository implements ATRepository {
// Both must be the same direction (DESC) also
sql.append("ORDER BY ATStates.height DESC LIMIT 1) AS FinalATStates ");
// Optional LEFT JOIN with ATTRANSACTIONS for buyerAddress
if (buyerAddress != null && !buyerAddress.isEmpty()) {
sql.append("LEFT JOIN ATTRANSACTIONS tx ON tx.at_address = ATs.AT_address ");
// Optional JOIN with ATTRANSACTIONS for buyerAddress
if (buyerPublicKey != null && buyerPublicKey.length > 0) {
sql.append("JOIN ATTRANSACTIONS tx ON tx.at_address = ATs.AT_address ");
}
sql.append("WHERE ATs.code_hash = ? ");
@@ -450,18 +451,18 @@ public class HSQLDBATRepository implements ATRepository {
bindParams.add(rawExpectedValue);
}
if (buyerAddress != null && !buyerAddress.isEmpty()) {
sql.append("AND tx.recipient = ? ");
bindParams.add(buyerAddress);
if (buyerPublicKey != null && buyerPublicKey.length > 0 ) {
// the buyer must be the recipient of the transaction and not the creator of the AT
sql.append("AND tx.recipient = ? AND ATs.creator != ? ");
bindParams.add(Crypto.toAddress(buyerPublicKey));
bindParams.add(buyerPublicKey);
}
if (sellerAddress != null && !sellerAddress.isEmpty()) {
// Convert sellerAddress to publicKey (method depends on your implementation)
AccountData accountData = this.repository.getAccountRepository().getAccount(sellerAddress);
byte[] publicKey = accountData.getPublicKey();
if (sellerPublicKey != null && sellerPublicKey.length > 0) {
sql.append("AND ATs.creator = ? ");
bindParams.add(publicKey);
bindParams.add(sellerPublicKey);
}
sql.append(" ORDER BY FinalATStates.height ");

165
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@@ -7,7 +7,6 @@ import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Category;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.data.arbitrary.ArbitraryResourceCache;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
@@ -29,6 +28,7 @@ import org.qortal.utils.ListUtils;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -227,6 +227,144 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
}
@Override
public List<ArbitraryTransactionData> getLatestArbitraryTransactions() throws DataException {
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE name IS NOT NULL " +
"ORDER BY created_when DESC";
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql)) {
if (resultSet == null)
return new ArrayList<>(0);
do {
byte[] reference = resultSet.getBytes(2);
byte[] signature = resultSet.getBytes(3);
byte[] creatorPublicKey = resultSet.getBytes(4);
long timestamp = resultSet.getLong(5);
Long fee = resultSet.getLong(6);
if (fee == 0 && resultSet.wasNull())
fee = null;
int txGroupId = resultSet.getInt(7);
Integer blockHeight = resultSet.getInt(8);
if (blockHeight == 0 && resultSet.wasNull())
blockHeight = null;
ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
Integer approvalHeight = resultSet.getInt(10);
if (approvalHeight == 0 && resultSet.wasNull())
approvalHeight = null;
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
int version = resultSet.getInt(11);
int nonce = resultSet.getInt(12);
int serviceInt = resultSet.getInt(13);
int size = resultSet.getInt(14);
boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
byte[] data = resultSet.getBytes(16);
byte[] metadataHash = resultSet.getBytes(17);
String nameResult = resultSet.getString(18);
String identifierResult = resultSet.getString(19);
Method method = Method.valueOf(resultSet.getInt(20));
byte[] secret = resultSet.getBytes(21);
Compression compression = Compression.valueOf(resultSet.getInt(22));
// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
compression, data, dataType, metadataHash, null);
arbitraryTransactionData.add(transactionData);
} while (resultSet.next());
return arbitraryTransactionData;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
return new ArrayList<>(0);
}
}
@Override
public List<ArbitraryTransactionData> getLatestArbitraryTransactionsByName( String name ) throws DataException {
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE name = ? " +
"ORDER BY created_when DESC";
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql, name)) {
if (resultSet == null)
return new ArrayList<>(0);
do {
byte[] reference = resultSet.getBytes(2);
byte[] signature = resultSet.getBytes(3);
byte[] creatorPublicKey = resultSet.getBytes(4);
long timestamp = resultSet.getLong(5);
Long fee = resultSet.getLong(6);
if (fee == 0 && resultSet.wasNull())
fee = null;
int txGroupId = resultSet.getInt(7);
Integer blockHeight = resultSet.getInt(8);
if (blockHeight == 0 && resultSet.wasNull())
blockHeight = null;
ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
Integer approvalHeight = resultSet.getInt(10);
if (approvalHeight == 0 && resultSet.wasNull())
approvalHeight = null;
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
int version = resultSet.getInt(11);
int nonce = resultSet.getInt(12);
int serviceInt = resultSet.getInt(13);
int size = resultSet.getInt(14);
boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
byte[] data = resultSet.getBytes(16);
byte[] metadataHash = resultSet.getBytes(17);
String nameResult = resultSet.getString(18);
String identifierResult = resultSet.getString(19);
Method method = Method.valueOf(resultSet.getInt(20));
byte[] secret = resultSet.getBytes(21);
Compression compression = Compression.valueOf(resultSet.getInt(22));
// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
compression, data, dataType, metadataHash, null);
arbitraryTransactionData.add(transactionData);
} while (resultSet.next());
return arbitraryTransactionData;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
} catch (Exception e) {
LOGGER.error(e.getMessage(), e);
return new ArrayList<>(0);
}
}
private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException {
if (name == null || service == null) {
// Required fields
@@ -724,12 +862,11 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
@Override
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, boolean prefixOnly,
public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, List<String> keywords, boolean prefixOnly,
List<String> exactMatchNames, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked,
Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException {
if(Settings.getInstance().isDbCacheEnabled()) {
List<ArbitraryResourceData> list
= HSQLDBCacheUtils.callCache(
ArbitraryResourceCache.getInstance(),
@@ -751,6 +888,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
Optional.ofNullable(description),
prefixOnly,
Optional.ofNullable(exactMatchNames),
Optional.ofNullable(keywords),
defaultResource,
Optional.ofNullable(minLevel),
Optional.ofNullable(() -> ListUtils.followedNames()),
@@ -771,6 +909,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
}
}
StringBuilder sql = new StringBuilder(512);
List<Object> bindParams = new ArrayList<>();
@@ -857,6 +996,26 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
bindParams.add(queryWildcard);
}
if (keywords != null && !keywords.isEmpty()) {
List<String> searchKeywords = new ArrayList<>(keywords);
List<String> conditions = new ArrayList<>();
List<String> bindValues = new ArrayList<>();
for (int i = 0; i < searchKeywords.size(); i++) {
conditions.add("LOWER(description) LIKE ?");
bindValues.add("%" + searchKeywords.get(i).trim().toLowerCase() + "%");
}
String finalCondition = String.join(" OR ", conditions);
sql.append(" AND (").append(finalCondition).append(")");
bindParams.addAll(bindValues);
}
// Handle name searches
if (names != null && !names.isEmpty()) {
sql.append(" AND (");

96
src/main/java/org/qortal/repository/hsqldb/HSQLDBCacheUtils.java
View File

@@ -167,6 +167,7 @@ public class HSQLDBCacheUtils {
Optional<String> description,
boolean prefixOnly,
Optional<List<String>> exactMatchNames,
Optional<List<String>> keywords,
boolean defaultResource,
Optional<Integer> minLevel,
Optional<Supplier<List<String>>> includeOnly,
@@ -182,6 +183,14 @@ public class HSQLDBCacheUtils {
// retain only candidates with names
Stream<ArbitraryResourceData> stream = candidates.stream().filter(candidate -> candidate.name != null );
if(after.isPresent()) {
stream = stream.filter( candidate -> candidate.created > after.get().longValue() );
}
if(before.isPresent()) {
stream = stream.filter( candidate -> candidate.created < before.get().longValue() );
}
if(exclude.isPresent())
stream = stream.filter( candidate -> !exclude.get().get().contains( candidate.name ));
@@ -207,6 +216,36 @@ public class HSQLDBCacheUtils {
stream = filterTerm(title, data -> data.metadata != null ? data.metadata.getTitle() : null, prefixOnly, stream);
stream = filterTerm(description, data -> data.metadata != null ? data.metadata.getDescription() : null, prefixOnly, stream);
// New: Filter by keywords if provided
if (keywords.isPresent() && !keywords.get().isEmpty()) {
List<String> searchKeywords = keywords.get().stream()
.map(String::toLowerCase)
.collect(Collectors.toList());
stream = stream.filter(candidate -> {
if (candidate.metadata != null && candidate.metadata.getDescription() != null) {
String descriptionLower = candidate.metadata.getDescription().toLowerCase();
return searchKeywords.stream().anyMatch(descriptionLower::contains);
}
return false;
});
}
if (keywords.isPresent() && !keywords.get().isEmpty()) {
List<String> searchKeywords = keywords.get().stream()
.map(String::toLowerCase)
.collect(Collectors.toList());
stream = stream.filter(candidate -> {
if (candidate.metadata != null && candidate.metadata.getDescription() != null) {
String descriptionLower = candidate.metadata.getDescription().toLowerCase();
return searchKeywords.stream().anyMatch(descriptionLower::contains);
}
return false;
});
}
// if exact names is set, retain resources with exact names
if( exactMatchNames.isPresent() && !exactMatchNames.get().isEmpty()) {
@@ -262,15 +301,58 @@ public class HSQLDBCacheUtils {
// truncate to limit
if( limit.isPresent() && limit.get() > 0 ) stream = stream.limit(limit.get());
// include metadata
if( includeMetadata.isEmpty() || !includeMetadata.get() )
stream = stream.peek( candidate -> candidate.metadata = null );
List<ArbitraryResourceData> listCopy1 = stream.collect(Collectors.toList());
// include status
if( includeStatus.isEmpty() || !includeStatus.get() )
stream = stream.peek( candidate -> candidate.status = null);
List<ArbitraryResourceData> listCopy2 = new ArrayList<>(listCopy1.size());
return stream.collect(Collectors.toList());
// remove metadata from the first copy
if( includeMetadata.isEmpty() || !includeMetadata.get() ) {
for( ArbitraryResourceData data : listCopy1 ) {
ArbitraryResourceData copy = new ArbitraryResourceData();
copy.name = data.name;
copy.service = data.service;
copy.identifier = data.identifier;
copy.status = data.status;
copy.metadata = null;
copy.size = data.size;
copy.created = data.created;
copy.updated = data.updated;
listCopy2.add(copy);
}
}
// put the list copy 1 into the second copy
else {
listCopy2.addAll(listCopy1);
}
// remove status from final copy
if( includeStatus.isEmpty() || !includeStatus.get() ) {
List<ArbitraryResourceData> finalCopy = new ArrayList<>(listCopy2.size());
for( ArbitraryResourceData data : listCopy2 ) {
ArbitraryResourceData copy = new ArbitraryResourceData();
copy.name = data.name;
copy.service = data.service;
copy.identifier = data.identifier;
copy.status = null;
copy.metadata = data.metadata;
copy.size = data.size;
copy.created = data.created;
copy.updated = data.updated;
finalCopy.add(copy);
}
return finalCopy;
}
// keep status included by returning the second copy
else {
return listCopy2;
}
}
/**

65
src/main/java/org/qortal/settings/Settings.java
View File

@@ -386,7 +386,7 @@ public class Settings {
/**
* DB Cache Enabled?
*/
private boolean dbCacheEnabled = false;
private boolean dbCacheEnabled = true;
/**
* DB Cache Thread Priority
@@ -508,7 +508,44 @@ public class Settings {
*/
private boolean connectionPoolMonitorEnabled = false;
// Domain mapping
/**
* Buiild Arbitrary Resources Batch Size
*
* The number resources to batch per iteration when rebuilding.
*/
private int buildArbitraryResourcesBatchSize = 200;
/**
* Arbitrary Indexing Priority
*
* The thread priority when indexing arbirary resources.
*/
private int arbitraryIndexingPriority = 5;
/**
* Arbitrary Indexing Frequency (In Minutes)
*
* The frequency at which the arbitrary indices are cached.
*/
private int arbitraryIndexingFrequency = 10;
private boolean rebuildArbitraryResourceCacheTaskEnabled = false;
/**
* Rebuild Arbitrary Resource Cache Task Delay (In Minutes)
*
* Waiting period before the first rebuild task is started.
*/
private int rebuildArbitraryResourceCacheTaskDelay = 300;
/**
* Rebuild Arbitrary Resource Cache Task Period (In Hours)
*
* The frequency the arbitrary resource cache is rebuilt.
*/
private int rebuildArbitraryResourceCacheTaskPeriod = 24;
// Domain mapping
public static class ThreadLimit {
private String messageType;
private Integer limit;
@@ -1333,4 +1370,28 @@ public class Settings {
public boolean isConnectionPoolMonitorEnabled() {
return connectionPoolMonitorEnabled;
}
public int getBuildArbitraryResourcesBatchSize() {
return buildArbitraryResourcesBatchSize;
}
public int getArbitraryIndexingPriority() {
return arbitraryIndexingPriority;
}
public int getArbitraryIndexingFrequency() {
return arbitraryIndexingFrequency;
}
public boolean isRebuildArbitraryResourceCacheTaskEnabled() {
return rebuildArbitraryResourceCacheTaskEnabled;
}
public int getRebuildArbitraryResourceCacheTaskDelay() {
return rebuildArbitraryResourceCacheTaskDelay;
}
public int getRebuildArbitraryResourceCacheTaskPeriod() {
return rebuildArbitraryResourceCacheTaskPeriod;
}
}

148
src/main/java/org/qortal/transaction/ArbitraryTransaction.java
View File

@@ -9,6 +9,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
import org.qortal.arbitrary.misc.Service;
import org.qortal.block.BlockChain;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.arbitrary.ArbitraryTransactionDataHashWrapper;
import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
import org.qortal.crypto.Crypto;
import org.qortal.crypto.MemoryPoW;
@@ -31,8 +32,12 @@ import org.qortal.utils.ArbitraryTransactionUtils;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
public class ArbitraryTransaction extends Transaction {
@@ -303,8 +308,13 @@ public class ArbitraryTransaction extends Transaction {
// Add/update arbitrary resource caches, but don't update the status as this involves time-consuming
// disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when
// accessing the resource.
this.updateArbitraryResourceCache(repository);
this.updateArbitraryMetadataCache(repository);
// Also, must add this transaction as a latest transaction, since the it has not been saved to the
// repository yet.
this.updateArbitraryResourceCacheIncludingMetadata(
repository,
Set.of(new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData)),
new HashMap<>(0)
);
repository.saveChanges();
@@ -360,7 +370,10 @@ public class ArbitraryTransaction extends Transaction {
*
* @throws DataException
*/
public void updateArbitraryResourceCache(Repository repository) throws DataException {
public void updateArbitraryResourceCacheIncludingMetadata(
Repository repository,
Set<ArbitraryTransactionDataHashWrapper> latestTransactionWrappers,
Map<ArbitraryTransactionDataHashWrapper, ArbitraryResourceData> resourceByWrapper) throws DataException {
// Don't cache resources without a name (such as auto updates)
if (arbitraryTransactionData.getName() == null) {
return;
@@ -385,29 +398,42 @@ public class ArbitraryTransaction extends Transaction {
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
// Get the latest transaction
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData);
// We don't have a latest transaction, so delete from cache
repository.getArbitraryRepository().delete(arbitraryResourceData);
return;
final ArbitraryTransactionDataHashWrapper wrapper = new ArbitraryTransactionDataHashWrapper(arbitraryTransactionData);
ArbitraryTransactionData latestTransactionData;
if( latestTransactionWrappers.contains(wrapper)) {
latestTransactionData
= latestTransactionWrappers.stream()
.filter( latestWrapper -> latestWrapper.equals(wrapper))
.findAny().get()
.getData();
}
else {
// Get the latest transaction
latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
LOGGER.info("We don't have a latest transaction, so delete from cache: arbitraryResourceData = " + arbitraryResourceData);
// We don't have a latest transaction, so delete from cache
repository.getArbitraryRepository().delete(arbitraryResourceData);
return;
}
}
ArbitraryResourceData existingArbitraryResourceData = resourceByWrapper.get(wrapper);
// Get existing cached entry if it exists
ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository()
.getArbitraryResource(service, name, identifier);
LOGGER.info("updating existing arbitraryResourceData" + existingArbitraryResourceData);
if( existingArbitraryResourceData == null ) {
// Get existing cached entry if it exists
existingArbitraryResourceData = repository.getArbitraryRepository()
.getArbitraryResource(service, name, identifier);
}
// Check for existing cached data
if (existingArbitraryResourceData == null) {
// Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed)
arbitraryResourceData.created = arbitraryTransactionData.getTimestamp();
arbitraryResourceData.updated = null;
LOGGER.info("updated = null, reason = existingArbitraryResourceData == null" );
}
else {
resourceByWrapper.put(wrapper, existingArbitraryResourceData);
// An entry already exists - update created time from current transaction if this is older
arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp());
@@ -415,22 +441,44 @@ public class ArbitraryTransaction extends Transaction {
if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) {
// Latest transaction matches created time, so it hasn't been updated
arbitraryResourceData.updated = null;
LOGGER.info(
"updated = null, reason: existingArbitraryResourceData.created == latestTransactionData.getTimestamp() == " +
existingArbitraryResourceData.created );
}
else {
arbitraryResourceData.updated = latestTransactionData.getTimestamp();
LOGGER.info("setting updated to a non-null value");
}
}
arbitraryResourceData.size = latestTransactionData.getSize();
LOGGER.info("saving updated arbitraryResourceData: updated = " + arbitraryResourceData.updated);
// Save
repository.getArbitraryRepository().save(arbitraryResourceData);
// Update metadata for latest transaction if it is local
if (latestTransactionData.getMetadataHash() != null) {
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
if (metadataFile.exists()) {
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
try {
transactionMetadata.read();
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
metadata.setTitle(transactionMetadata.getTitle());
metadata.setDescription(transactionMetadata.getDescription());
metadata.setCategory(transactionMetadata.getCategory());
metadata.setTags(transactionMetadata.getTags());
repository.getArbitraryRepository().save(metadata);
} catch (IOException e) {
// Ignore, as we can add it again later
}
} else {
// We don't have a local copy of this metadata file, so delete it from the cache
// It will be re-added if the file later arrives via the network
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
repository.getArbitraryRepository().delete(metadata);
}
}
}
public void updateArbitraryResourceStatus(Repository repository) throws DataException {
@@ -465,60 +513,4 @@ public class ArbitraryTransaction extends Transaction {
repository.getArbitraryRepository().setStatus(arbitraryResourceData, status);
}
public void updateArbitraryMetadataCache(Repository repository) throws DataException {
// Get the latest transaction
ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
if (latestTransactionData == null) {
// We don't have a latest transaction, so give up
return;
}
Service service = latestTransactionData.getService();
String name = latestTransactionData.getName();
String identifier = latestTransactionData.getIdentifier();
if (service == null) {
// Unsupported service - ignore this resource
return;
}
// In the cache we store null identifiers as "default", as it is part of the primary key
if (identifier == null) {
identifier = "default";
}
ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
arbitraryResourceData.service = service;
arbitraryResourceData.name = name;
arbitraryResourceData.identifier = identifier;
// Update metadata for latest transaction if it is local
if (latestTransactionData.getMetadataHash() != null) {
ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
if (metadataFile.exists()) {
ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
try {
transactionMetadata.read();
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
metadata.setTitle(transactionMetadata.getTitle());
metadata.setDescription(transactionMetadata.getDescription());
metadata.setCategory(transactionMetadata.getCategory());
metadata.setTags(transactionMetadata.getTags());
repository.getArbitraryRepository().save(metadata);
} catch (IOException e) {
// Ignore, as we can add it again later
}
} else {
// We don't have a local copy of this metadata file, so delete it from the cache
// It will be re-added if the file later arrives via the network
ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
metadata.setArbitraryResourceData(arbitraryResourceData);
repository.getArbitraryRepository().delete(metadata);
}
}
}
}

250
src/main/java/org/qortal/utils/ArbitraryIndexUtils.java Normal file
View File

@@ -0,0 +1,250 @@
package org.qortal.utils;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.exc.InvalidFormatException;
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.SearchMode;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
import org.qortal.data.arbitrary.ArbitraryDataIndex;
import org.qortal.data.arbitrary.ArbitraryDataIndexDetail;
import org.qortal.data.arbitrary.ArbitraryResourceData;
import org.qortal.data.arbitrary.IndexCache;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class ArbitraryIndexUtils {
public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final Logger LOGGER = LogManager.getLogger(ArbitraryIndexUtils.class);
public static final String INDEX_CACHE_TIMER = "Arbitrary Index Cache Timer";
public static final String INDEX_CACHE_TIMER_TASK = "Arbitrary Index Cache Timer Task";
public static void startCaching(int priorityRequested, int frequency) {
Timer timer = buildTimer(INDEX_CACHE_TIMER, priorityRequested);
TimerTask task = new TimerTask() {
@Override
public void run() {
Thread.currentThread().setName(INDEX_CACHE_TIMER_TASK);
try {
fillCache(IndexCache.getInstance());
} catch (IOException | DataException e) {
LOGGER.error(e.getMessage(), e);
}
}
};
// delay 1 second
timer.scheduleAtFixedRate(task, 1_000, frequency * 60_000);
}
private static void fillCache(IndexCache instance) throws DataException, IOException {
try (final Repository repository = RepositoryManager.getRepository()) {
List<ArbitraryResourceData> indexResources
= repository.getArbitraryRepository().searchArbitraryResources(
Service.JSON,
null,
"idx-",
null,
null,
null,
null,
true,
null,
false,
SearchMode.ALL,
0,
null,
null,
null,
null,
null,
null,
null,
null,
true);
List<ArbitraryDataIndexDetail> indexDetails = new ArrayList<>();
LOGGER.debug("processing index resource data: count = " + indexResources.size());
// process all index resources
for( ArbitraryResourceData indexResource : indexResources ) {
try {
LOGGER.debug("processing index resource: name = " + indexResource.name + ", identifier = " + indexResource.identifier);
String json = ArbitraryIndexUtils.getJson(indexResource.name, indexResource.identifier);
// map the JSON string to a list of Java objects
List<ArbitraryDataIndex> indices = OBJECT_MAPPER.readValue(json, new TypeReference<List<ArbitraryDataIndex>>() {});
LOGGER.debug("processed indices = " + indices);
// rank and create index detail for each index in this index resource
for( int rank = 1; rank <= indices.size(); rank++ ) {
indexDetails.add( new ArbitraryDataIndexDetail(indexResource.name, rank, indices.get(rank - 1), indexResource.identifier ));
}
} catch (InvalidFormatException e) {
LOGGER.debug("invalid format, skipping: " + indexResource);
} catch (UnrecognizedPropertyException e) {
LOGGER.debug("unrecognized property, skipping " + indexResource);
}
}
LOGGER.debug("processing indices by term ...");
Map<String, List<ArbitraryDataIndexDetail>> indicesByTerm
= indexDetails.stream().collect(
Collectors.toMap(
detail -> detail.term, // map by term
detail -> List.of(detail), // create list for term
(list1, list2) // merge lists for same term
-> Stream.of(list1, list2)
.flatMap(List::stream)
.collect(Collectors.toList())
)
);
LOGGER.info("processed indices by term: count = " + indicesByTerm.size());
// lock, clear old, load new
synchronized( IndexCache.getInstance().getIndicesByTerm() ) {
IndexCache.getInstance().getIndicesByTerm().clear();
IndexCache.getInstance().getIndicesByTerm().putAll(indicesByTerm);
}
LOGGER.info("loaded indices by term");
LOGGER.debug("processing indices by issuer ...");
Map<String, List<ArbitraryDataIndexDetail>> indicesByIssuer
= indexDetails.stream().collect(
Collectors.toMap(
detail -> detail.issuer, // map by issuer
detail -> List.of(detail), // create list for issuer
(list1, list2) // merge lists for same issuer
-> Stream.of(list1, list2)
.flatMap(List::stream)
.collect(Collectors.toList())
)
);
LOGGER.info("processed indices by issuer: count = " + indicesByIssuer.size());
// lock, clear old, load new
synchronized( IndexCache.getInstance().getIndicesByIssuer() ) {
IndexCache.getInstance().getIndicesByIssuer().clear();
IndexCache.getInstance().getIndicesByIssuer().putAll(indicesByIssuer);
}
LOGGER.info("loaded indices by issuer");
}
}
private static Timer buildTimer( final String name, int priorityRequested) {
// ensure priority is in between 1-10
final int priority = Math.max(0, Math.min(10, priorityRequested));
// Create a custom Timer with updated priority threads
Timer timer = new Timer(true) { // 'true' to make the Timer daemon
@Override
public void schedule(TimerTask task, long delay) {
Thread thread = new Thread(task, name) {
@Override
public void run() {
this.setPriority(priority);
super.run();
}
};
thread.setPriority(priority);
thread.start();
}
};
return timer;
}
public static String getJsonWithExceptionHandling( String name, String identifier ) {
try {
return getJson(name, identifier);
}
catch( Exception e ) {
LOGGER.error(e.getMessage(), e);
return e.getMessage();
}
}
public static String getJson(String name, String identifier) throws IOException {
try {
ArbitraryDataReader arbitraryDataReader
= new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, Service.JSON, identifier);
int attempts = 0;
Integer maxAttempts = 5;
while (!Controller.isStopping()) {
attempts++;
if (!arbitraryDataReader.isBuilding()) {
try {
arbitraryDataReader.loadSynchronously(false);
break;
} catch (MissingDataException e) {
if (attempts > maxAttempts) {
// Give up after 5 attempts
throw new IOException("Data unavailable. Please try again later.");
}
}
}
Thread.sleep(3000L);
}
java.nio.file.Path outputPath = arbitraryDataReader.getFilePath();
if (outputPath == null) {
// Assume the resource doesn't exist
throw new IOException( "File not found");
}
// No file path supplied - so check if this is a single file resource
String[] files = ArrayUtils.removeElement(outputPath.toFile().list(), ".qortal");
String filepath = files[0];
java.nio.file.Path path = Paths.get(outputPath.toString(), filepath);
if (!Files.exists(path)) {
String message = String.format("No file exists at filepath: %s", filepath);
throw new IOException( message );
}
String data = Files.readString(path);
return data;
} catch (Exception e) {
throw new IOException(String.format("Unable to load %s %s: %s", Service.JSON, name, e.getMessage()));
}
}
}

24
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@@ -24,6 +24,7 @@ import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
@@ -72,23 +73,23 @@ public class ArbitraryTransactionUtils {
return latestPut;
}
public static boolean hasMoreRecentPutTransaction(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
public static Optional<ArbitraryTransactionData> hasMoreRecentPutTransaction(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
byte[] signature = arbitraryTransactionData.getSignature();
if (signature == null) {
// We can't make a sensible decision without a signature
// so it's best to assume there is nothing newer
return false;
return Optional.empty();
}
ArbitraryTransactionData latestPut = ArbitraryTransactionUtils.fetchLatestPut(repository, arbitraryTransactionData);
if (latestPut == null) {
return false;
return Optional.empty();
}
// If the latest PUT transaction has a newer timestamp, it will override the existing transaction
// Any data relating to the older transaction is no longer needed
boolean hasNewerPut = (latestPut.getTimestamp() > arbitraryTransactionData.getTimestamp());
return hasNewerPut;
return hasNewerPut ? Optional.of(latestPut) : Optional.empty();
}
public static boolean completeFileExists(ArbitraryTransactionData transactionData) throws DataException {
@@ -208,7 +209,15 @@ public class ArbitraryTransactionUtils {
return ArbitraryTransactionUtils.isFileRecent(filePath, now, cleanupAfter);
}
public static void deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
/**
*
* @param arbitraryTransactionData
* @param now
* @param cleanupAfter
* @return true if file is deleted, otherwise return false
* @throws DataException
*/
public static boolean deleteCompleteFile(ArbitraryTransactionData arbitraryTransactionData, long now, long cleanupAfter) throws DataException {
byte[] completeHash = arbitraryTransactionData.getData();
byte[] signature = arbitraryTransactionData.getSignature();
@@ -219,6 +228,11 @@ public class ArbitraryTransactionUtils {
"if needed", Base58.encode(completeHash));
arbitraryDataFile.delete();
return true;
}
else {
return false;
}
}

8
src/main/java/org/qortal/utils/Blocks.java
View File

@@ -53,10 +53,10 @@ public class Blocks {
// all minting group member addresses
List<String> mintingGroupAddresses
= repository.getGroupRepository()
.getGroupMembers(BlockChain.getInstance().getMintingGroupId()).stream()
.map(GroupMemberData::getMember)
.collect(Collectors.toList());
= Groups.getAllMembers(
repository.getGroupRepository(),
Groups.getGroupIdsToMint(BlockChain.getInstance(), blockData.getHeight())
);
// all names, indexed by address
Map<String, String> nameByAddress

122
src/main/java/org/qortal/utils/Groups.java Normal file
View File

@@ -0,0 +1,122 @@
package org.qortal.utils;
import org.qortal.block.BlockChain;
import org.qortal.data.group.GroupAdminData;
import org.qortal.data.group.GroupMemberData;
import org.qortal.repository.DataException;
import org.qortal.repository.GroupRepository;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Class Groups
*
* A utility class for group related functionality.
*/
public class Groups {
/**
* Does the member exist in any of these groups?
*
* @param groupRepository the group data repository
* @param groupsIds the group Ids to look for the address
* @param address the address
*
* @return true if the address is in any of the groups listed otherwise false
* @throws DataException
*/
public static boolean memberExistsInAnyGroup(GroupRepository groupRepository, List<Integer> groupsIds, String address) throws DataException {
// if any of the listed groups have the address as a member, then return true
for( Integer groupIdToMint : groupsIds) {
if( groupRepository.memberExists(groupIdToMint, address) ) {
return true;
}
}
// if none of the listed groups have the address as a member, then return false
return false;
}
/**
* Get All Members
*
* Get all the group members from a list of groups.
*
* @param groupRepository the group data repository
* @param groupIds the list of group Ids to look at
*
* @return the list of all members belonging to any of the groups, no duplicates
* @throws DataException
*/
public static List<String> getAllMembers( GroupRepository groupRepository, List<Integer> groupIds ) throws DataException {
// collect all the members in a set, the set keeps out duplicates
Set<String> allMembers = new HashSet<>();
// add all members from each group to the all members set
for( int groupId : groupIds ) {
allMembers.addAll( groupRepository.getGroupMembers(groupId).stream().map(GroupMemberData::getMember).collect(Collectors.toList()));
}
return new ArrayList<>(allMembers);
}
/**
* Get All Admins
*
* Get all the admins from a list of groups.
*
* @param groupRepository the group data repository
* @param groupIds the list of group Ids to look at
*
* @return the list of all admins to any of the groups, no duplicates
* @throws DataException
*/
public static List<String> getAllAdmins( GroupRepository groupRepository, List<Integer> groupIds ) throws DataException {
// collect all the admins in a set, the set keeps out duplicates
Set<String> allAdmins = new HashSet<>();
// collect admins for each group
for( int groupId : groupIds ) {
allAdmins.addAll( groupRepository.getGroupAdmins(groupId).stream().map(GroupAdminData::getAdmin).collect(Collectors.toList()) );
}
return new ArrayList<>(allAdmins);
}
/**
* Get Group Ids To Mint
*
* @param blockchain the blockchain
* @param blockchainHeight the block height to mint
*
* @return the group Ids for the minting groups at the height given
*/
public static List<Integer> getGroupIdsToMint(BlockChain blockchain, int blockchainHeight) {
// sort heights lowest to highest
Comparator<BlockChain.IdsForHeight> compareByHeight = Comparator.comparingInt(entry -> entry.height);
// sort heights highest to lowest
Comparator<BlockChain.IdsForHeight> compareByHeightReversed = compareByHeight.reversed();
// get highest height that is less than the blockchain height
Optional<BlockChain.IdsForHeight> ids = blockchain.getMintingGroupIds().stream()
.filter(entry -> entry.height < blockchainHeight)
.sorted(compareByHeightReversed)
.findFirst();
if( ids.isPresent()) {
return ids.get().ids;
}
else {
return new ArrayList<>(0);
}
}
}

4
src/main/resources/blockchain.json
View File

@@ -38,7 +38,9 @@
"blockRewardBatchStartHeight": 1508000,
"blockRewardBatchSize": 1000,
"blockRewardBatchAccountsBlockCount": 25,
"mintingGroupId": 694,
"mintingGroupIds": [
{ "height": 0, "ids": [ 694 ]}
],
"rewardsByHeight": [
{ "height": 1, "reward": 5.00 },
{ "height": 259201, "reward": 4.75 },

14
src/main/resources/q-apps/q-apps.js
View File

@@ -84,6 +84,7 @@ isDOMContentLoaded: isDOMContentLoaded ? true : false
function handleQDNResourceDisplayed(pathurl, isDOMContentLoaded) {
// make sure that an empty string the root path
if(pathurl?.startsWith('/render/hash/')) return;
const path = pathurl || '/'
if (!isManualNavigation) {
isManualNavigation = true
@@ -284,11 +285,9 @@ window.addEventListener("message", async (event) => {
return;
}
console.log("Core received action: " + JSON.stringify(event.data.action));
let url;
let data = event.data;
let identifier;
switch (data.action) {
case "GET_ACCOUNT_DATA":
return httpGetAsyncWithEvent(event, "/addresses/" + data.address);
@@ -383,6 +382,7 @@ window.addEventListener("message", async (event) => {
if (data.identifier != null) url = url.concat("&identifier=" + data.identifier);
if (data.name != null) url = url.concat("&name=" + data.name);
if (data.names != null) data.names.forEach((x, i) => url = url.concat("&name=" + x));
if (data.keywords != null) data.keywords.forEach((x, i) => url = url.concat("&keywords=" + x));
if (data.title != null) url = url.concat("&title=" + data.title);
if (data.description != null) url = url.concat("&description=" + data.description);
if (data.prefix != null) url = url.concat("&prefix=" + new Boolean(data.prefix).toString());
@@ -419,7 +419,7 @@ window.addEventListener("message", async (event) => {
return httpGetAsyncWithEvent(event, url);
case "GET_QDN_RESOURCE_PROPERTIES":
let identifier = (data.identifier != null) ? data.identifier : "default";
identifier = (data.identifier != null) ? data.identifier : "default";
url = "/arbitrary/resource/properties/" + data.service + "/" + data.name + "/" + identifier;
return httpGetAsyncWithEvent(event, url);
@@ -456,7 +456,7 @@ window.addEventListener("message", async (event) => {
return httpGetAsyncWithEvent(event, url);
case "GET_AT":
url = "/at" + data.atAddress;
url = "/at/" + data.atAddress;
return httpGetAsyncWithEvent(event, url);
case "GET_AT_DATA":
@@ -473,7 +473,7 @@ window.addEventListener("message", async (event) => {
case "FETCH_BLOCK":
if (data.signature != null) {
url = "/blocks/" + data.signature;
url = "/blocks/signature/" + data.signature;
} else if (data.height != null) {
url = "/blocks/byheight/" + data.height;
}
@@ -694,6 +694,7 @@ const qortalRequestWithTimeout = (request, timeout) =>
* Send current page details to UI
*/
document.addEventListener('DOMContentLoaded', (event) => {
resetVariables()
qortalRequest({
action: "QDN_RESOURCE_DISPLAYED",
@@ -712,6 +713,7 @@ resetVariables()
* Handle app navigation
*/
navigation.addEventListener('navigate', (event) => {
const url = new URL(event.destination.url);
let fullpath = url.pathname + url.hash;

54
src/test/java/org/qortal/test/api/CrossChainUtilsTests.java
View File

@@ -3,10 +3,15 @@ package org.qortal.test.api;
import org.json.simple.JSONObject;
import org.junit.Assert;
import org.junit.Test;
import org.qortal.api.model.CrossChainTradeLedgerEntry;
import org.qortal.api.resource.CrossChainUtils;
import org.qortal.test.common.ApiCommon;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class CrossChainUtilsTests extends ApiCommon {
@@ -137,4 +142,53 @@ public class CrossChainUtilsTests extends ApiCommon {
Assert.assertEquals(5, versionDecimal, 0.001);
Assert.assertFalse(thrown);
}
@Test
public void testWriteToLedgerHeaderOnly() throws IOException {
CrossChainUtils.writeToLedger(new PrintWriter(System.out), new ArrayList<>());
}
@Test
public void testWriteToLedgerOneRow() throws IOException {
CrossChainUtils.writeToLedger(
new PrintWriter(System.out),
List.of(
new CrossChainTradeLedgerEntry(
"QORT",
"LTC",
1000,
0,
"LTC",
1,
System.currentTimeMillis())
)
);
}
@Test
public void testWriteToLedgerTwoRows() throws IOException {
CrossChainUtils.writeToLedger(
new PrintWriter(System.out),
List.of(
new CrossChainTradeLedgerEntry(
"QORT",
"LTC",
1000,
0,
"LTC",
1,
System.currentTimeMillis()
),
new CrossChainTradeLedgerEntry(
"LTC",
"QORT",
1,
0,
"LTC",
1000,
System.currentTimeMillis()
)
)
);
}
}

50
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java
View File

@@ -145,56 +145,6 @@ public class ArbitraryDataStorageCapacityTests extends Common {
}
}
@Test
public void testDeleteRandomFilesForName() throws DataException, IOException, InterruptedException, IllegalAccessException {
try (final Repository repository = RepositoryManager.getRepository()) {
String identifier = null; // Not used for this test
Service service = Service.ARBITRARY_DATA;
int chunkSize = 100;
int dataLength = 900; // Actual data length will be longer due to encryption
// Set originalCopyIndicatorFileEnabled to false, otherwise nothing will be deleted as it all originates from this node
FieldUtils.writeField(Settings.getInstance(), "originalCopyIndicatorFileEnabled", false, true);
// Alice hosts some data (with 10 chunks)
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String aliceName = "alice";
RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, alice);
Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength);
ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize);
// Bob hosts some data too (also with 10 chunks)
PrivateKeyAccount bob = Common.getTestAccount(repository, "bob");
String bobName = "bob";
transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, "");
transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
TransactionUtils.signAndMint(repository, transactionData, bob);
Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength);
ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize);
// All 20 chunks should exist
assertEquals(10, aliceArbitraryDataFile.chunkCount());
assertTrue(aliceArbitraryDataFile.allChunksExist());
assertEquals(10, bobArbitraryDataFile.chunkCount());
assertTrue(bobArbitraryDataFile.allChunksExist());
// Now pretend that Bob has reached his storage limit - this should delete random files
// Run it 10 times to remove the likelihood of the randomizer always picking Alice's files
for (int i=0; i<10; i++) {
ArbitraryDataCleanupManager.getInstance().storageLimitReachedForName(repository, bobName);
}
// Alice should still have all chunks
assertTrue(aliceArbitraryDataFile.allChunksExist());
// Bob should be missing some chunks
assertFalse(bobArbitraryDataFile.allChunksExist());
}
}
private void deleteListsDirectory() {
// Delete lists directory if exists
Path listsPath = Paths.get(Settings.getInstance().getListsPath());

22
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java
View File

@@ -73,14 +73,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
}
}
@@ -108,14 +108,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We shouldn't store or pre-fetch data for this transaction
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
}
}
@@ -143,14 +143,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store but not pre-fetch data for this transaction
assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
}
}
@@ -178,14 +178,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We should store and pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
}
}
@@ -213,14 +213,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We shouldn't store or pre-fetch data for this transaction
assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy());
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
// We shouldn't store or pre-fetch data for this transaction
assertFalse(storageManager.canStoreData(arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData).isPass());
}
}
@@ -236,7 +236,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData).isPass());
}
}

48
src/test/java/org/qortal/test/repository/HSQLDBCacheUtilsTests.java
View File

@@ -26,6 +26,7 @@ public class HSQLDBCacheUtilsTests {
private static final String DESCRIPTION = "description";
private static final String PREFIX_ONLY = "prefixOnly";
private static final String EXACT_MATCH_NAMES = "exactMatchNames";
private static final String KEYWORDS = "keywords";
private static final String DEFAULT_RESOURCE = "defaultResource";
private static final String MODE = "mode";
private static final String MIN_LEVEL = "minLevel";
@@ -299,6 +300,19 @@ public class HSQLDBCacheUtilsTests {
);
}
@Test
public void testAfterNegative() {
ArbitraryResourceData data = new ArbitraryResourceData();
data.created = 10L;
data.name = "Joe";
filterListByMap(
List.of(data),
NAME_LEVEL, new HashMap<>(Map.of(AFTER, 11L)),
0
);
}
@Test
public void testBeforePositive(){
ArbitraryResourceData data = new ArbitraryResourceData();
@@ -312,6 +326,19 @@ public class HSQLDBCacheUtilsTests {
);
}
@Test
public void testBeforeNegative(){
ArbitraryResourceData data = new ArbitraryResourceData();
data.created = 10L;
data.name = "Joe";
filterListByMap(
List.of(data),
NAME_LEVEL, new HashMap<>(Map.of(BEFORE, 9L)),
0
);
}
@Test
public void testTitlePositive() {
@@ -342,6 +369,25 @@ public class HSQLDBCacheUtilsTests {
);
}
@Test
public void testMetadataNullificationBugSolution(){
ArbitraryResourceData data = new ArbitraryResourceData();
data.metadata = new ArbitraryResourceMetadata();
data.metadata.setDescription("Once upon a time.");
data.name = "Joe";
List<ArbitraryResourceData> list = List.of(data);
filterListByMap(
List.of(data),
NAME_LEVEL, new HashMap<>(Map.of(DESCRIPTION, "Once upon a time.")),
1
);
Assert.assertNotNull(data.metadata);
}
@Test
public void testMinLevelPositive() {
@@ -615,6 +661,7 @@ public class HSQLDBCacheUtilsTests {
Optional<String> description = Optional.ofNullable((String) valueByKey.get(DESCRIPTION));
boolean prefixOnly = valueByKey.containsKey(PREFIX_ONLY);
Optional<List<String>> exactMatchNames = Optional.ofNullable((List<String>) valueByKey.get(EXACT_MATCH_NAMES));
Optional<List<String>> keywords = Optional.ofNullable((List<String>) valueByKey.get(KEYWORDS));
boolean defaultResource = valueByKey.containsKey(DEFAULT_RESOURCE);
Optional<SearchMode> mode = Optional.of((SearchMode) valueByKey.getOrDefault(MODE, SearchMode.ALL));
Optional<Integer> minLevel = Optional.ofNullable((Integer) valueByKey.get(MIN_LEVEL));
@@ -641,6 +688,7 @@ public class HSQLDBCacheUtilsTests {
description,
prefixOnly,
exactMatchNames,
keywords,
defaultResource,
minLevel,
followedOnly,

102
src/test/java/org/qortal/test/utils/GroupsTestUtils.java Normal file
View File

@@ -0,0 +1,102 @@
package org.qortal.test.utils;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.data.transaction.CreateGroupTransactionData;
import org.qortal.data.transaction.GroupInviteTransactionData;
import org.qortal.data.transaction.JoinGroupTransactionData;
import org.qortal.data.transaction.LeaveGroupTransactionData;
import org.qortal.group.Group;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.test.common.TransactionUtils;
import org.qortal.test.common.transaction.TestTransaction;
/**
* Class GroupsTestUtils
*
* Utility methods for testing the Groups class.
*/
public class GroupsTestUtils {
/**
* Create Group
*
* @param repository the data repository
* @param owner the group owner
* @param groupName the group name
* @param isOpen true if the group is public, false for private
*
* @return the group Id
* @throws DataException
*/
public static Integer createGroup(Repository repository, PrivateKeyAccount owner, String groupName, boolean isOpen) throws DataException {
String description = groupName + " (description)";
Group.ApprovalThreshold approvalThreshold = Group.ApprovalThreshold.ONE;
int minimumBlockDelay = 10;
int maximumBlockDelay = 1440;
CreateGroupTransactionData transactionData = new CreateGroupTransactionData(TestTransaction.generateBase(owner), groupName, description, isOpen, approvalThreshold, minimumBlockDelay, maximumBlockDelay);
TransactionUtils.signAndMint(repository, transactionData, owner);
return repository.getGroupRepository().fromGroupName(groupName).getGroupId();
}
/**
* Join Group
*
* @param repository the data repository
* @param joiner the address for the account joining the group
* @param groupId the Id for the group to join
*
* @throws DataException
*/
public static void joinGroup(Repository repository, PrivateKeyAccount joiner, int groupId) throws DataException {
JoinGroupTransactionData transactionData = new JoinGroupTransactionData(TestTransaction.generateBase(joiner), groupId);
TransactionUtils.signAndMint(repository, transactionData, joiner);
}
/**
* Group Invite
*
* @param repository the data repository
* @param admin the admin account to sign the invite
* @param groupId the Id of the group to invite to
* @param invitee the recipient address for the invite
* @param timeToLive the time length of the invite
*
* @throws DataException
*/
public static void groupInvite(Repository repository, PrivateKeyAccount admin, int groupId, String invitee, int timeToLive) throws DataException {
GroupInviteTransactionData transactionData = new GroupInviteTransactionData(TestTransaction.generateBase(admin), groupId, invitee, timeToLive);
TransactionUtils.signAndMint(repository, transactionData, admin);
}
/**
* Leave Group
*
* @param repository the data repository
* @param leaver the account leaving
* @param groupId the Id of the group being left
*
* @throws DataException
*/
public static void leaveGroup(Repository repository, PrivateKeyAccount leaver, int groupId) throws DataException {
LeaveGroupTransactionData transactionData = new LeaveGroupTransactionData(TestTransaction.generateBase(leaver), groupId);
TransactionUtils.signAndMint(repository, transactionData, leaver);
}
/**
* Is Member?
*
* @param repository the data repository
* @param address the account address
* @param groupId the group Id
*
* @return true if the account is a member of the group, otherwise false
* @throws DataException
*/
public static boolean isMember(Repository repository, String address, int groupId) throws DataException {
return repository.getGroupRepository().memberExists(groupId, address);
}
}

199
src/test/java/org/qortal/test/utils/GroupsTests.java Normal file
View File

@@ -0,0 +1,199 @@
package org.qortal.test.utils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.qortal.account.PrivateKeyAccount;
import org.qortal.block.Block;
import org.qortal.block.BlockChain;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.test.common.BlockUtils;
import org.qortal.test.common.Common;
import org.qortal.utils.Groups;
import java.util.List;
import static org.junit.Assert.*;
public class GroupsTests extends Common {
public static final String ALICE = "alice";
public static final String BOB = "bob";
public static final String CHLOE = "chloe";
public static final String DILBERT = "dilbert";
private static final int HEIGHT_1 = 5;
private static final int HEIGHT_2 = 8;
private static final int HEIGHT_3 = 12;
@Before
public void beforeTest() throws DataException {
Common.useDefaultSettings();
}
@After
public void afterTest() throws DataException {
Common.orphanCheck();
}
@Test
public void testGetGroupIdsToMintSimple() {
List<Integer> ids = Groups.getGroupIdsToMint(BlockChain.getInstance(), 0);
Assert.assertNotNull(ids);
Assert.assertEquals(0, ids.size());
}
@Test
public void testGetGroupIdsToMintComplex() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
Block block1 = BlockUtils.mintBlocks(repository, HEIGHT_1);
int height1 = block1.getBlockData().getHeight().intValue();
assertEquals(HEIGHT_1 + 1, height1);
List<Integer> ids1 = Groups.getGroupIdsToMint(BlockChain.getInstance(), height1);
Assert.assertEquals(1, ids1.size() );
Assert.assertTrue( ids1.contains( 694 ) );
Block block2 = BlockUtils.mintBlocks(repository, HEIGHT_2 - HEIGHT_1);
int height2 = block2.getBlockData().getHeight().intValue();
assertEquals( HEIGHT_2 + 1, height2);
List<Integer> ids2 = Groups.getGroupIdsToMint(BlockChain.getInstance(), height2);
Assert.assertEquals(2, ids2.size() );
Assert.assertTrue( ids2.contains( 694 ) );
Assert.assertTrue( ids2.contains( 800 ) );
Block block3 = BlockUtils.mintBlocks(repository, HEIGHT_3 - HEIGHT_2);
int height3 = block3.getBlockData().getHeight().intValue();
assertEquals( HEIGHT_3 + 1, height3);
List<Integer> ids3 = Groups.getGroupIdsToMint(BlockChain.getInstance(), height3);
Assert.assertEquals( 1, ids3.size() );
Assert.assertTrue( ids3.contains( 800 ) );
}
}
@Test
public void testMemberExistsInAnyGroupSimple() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
PrivateKeyAccount bob = Common.getTestAccount(repository, "bob");
// Create group
int groupId = GroupsTestUtils.createGroup(repository, alice, "closed-group", false);
// Confirm Bob is not a member
Assert.assertFalse( Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(groupId), bob.getAddress()) );
// Bob to join
GroupsTestUtils.joinGroup(repository, bob, groupId);
// Confirm Bob still not a member
assertFalse(GroupsTestUtils.isMember(repository, bob.getAddress(), groupId));
// Have Alice 'invite' Bob to confirm membership
GroupsTestUtils.groupInvite(repository, alice, groupId, bob.getAddress(), 0); // non-expiring invite
// Confirm Bob now a member
Assert.assertTrue( Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(groupId), bob.getAddress()) );
}
}
@Test
public void testGroupsListedFunctionality() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, ALICE);
PrivateKeyAccount bob = Common.getTestAccount(repository, BOB);
PrivateKeyAccount chloe = Common.getTestAccount(repository, CHLOE);
PrivateKeyAccount dilbert = Common.getTestAccount(repository, DILBERT);
// Create groups
int group1Id = GroupsTestUtils.createGroup(repository, alice, "group-1", false);
int group2Id = GroupsTestUtils.createGroup(repository, bob, "group-2", false);
// test memberExistsInAnyGroup
Assert.assertTrue(Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(group1Id, group2Id), alice.getAddress()));
Assert.assertFalse(Groups.memberExistsInAnyGroup(repository.getGroupRepository(), List.of(group1Id, group2Id), chloe.getAddress()));
// alice is a member
Assert.assertTrue(GroupsTestUtils.isMember(repository, alice.getAddress(), group1Id));
List<String> allMembersBeforeJoin = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id));
// assert one member
Assert.assertNotNull(allMembersBeforeJoin);
Assert.assertEquals(1, allMembersBeforeJoin.size());
List<String> allAdminsBeforeJoin = Groups.getAllAdmins(repository.getGroupRepository(), List.of(group1Id));
// assert one admin
Assert.assertNotNull(allAdminsBeforeJoin);
Assert.assertEquals( 1, allAdminsBeforeJoin.size());
// Bob to join
GroupsTestUtils.joinGroup(repository, bob, group1Id);
// Have Alice 'invite' Bob to confirm membership
GroupsTestUtils.groupInvite(repository, alice, group1Id, bob.getAddress(), 0); // non-expiring invite
List<String> allMembersAfterJoin = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id));
// alice and bob are members
Assert.assertNotNull(allMembersAfterJoin);
Assert.assertEquals(2, allMembersAfterJoin.size());
List<String> allAdminsAfterJoin = Groups.getAllAdmins(repository.getGroupRepository(), List.of(group1Id));
// assert still one admin
Assert.assertNotNull(allAdminsAfterJoin);
Assert.assertEquals(1, allAdminsAfterJoin.size());
List<String> allAdminsFor2Groups = Groups.getAllAdmins(repository.getGroupRepository(), List.of(group1Id, group2Id));
// assert 2 admins when including the second group
Assert.assertNotNull(allAdminsFor2Groups);
Assert.assertEquals(2, allAdminsFor2Groups.size());
List<String> allMembersFor2Groups = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id, group2Id));
// assert 2 members when including the seconds group
Assert.assertNotNull(allMembersFor2Groups);
Assert.assertEquals(2, allMembersFor2Groups.size());
GroupsTestUtils.leaveGroup(repository, bob, group1Id);
List<String> allMembersForAfterBobLeavesGroup1InAllGroups = Groups.getAllMembers(repository.getGroupRepository(), List.of(group1Id, group2Id));
// alice and bob are members of one group still
Assert.assertNotNull(allMembersForAfterBobLeavesGroup1InAllGroups);
Assert.assertEquals(2, allMembersForAfterBobLeavesGroup1InAllGroups.size());
GroupsTestUtils.groupInvite(repository, alice, group1Id, chloe.getAddress(), 3600);
GroupsTestUtils.groupInvite(repository, bob, group2Id, chloe.getAddress(), 3600);
GroupsTestUtils.joinGroup(repository, chloe, group1Id);
GroupsTestUtils.joinGroup(repository, chloe, group2Id);
List<String> allMembersAfterDilbert = Groups.getAllMembers((repository.getGroupRepository()), List.of(group1Id, group2Id));
// 3 accounts are now members of one group or another
Assert.assertNotNull(allMembersAfterDilbert);
Assert.assertEquals(3, allMembersAfterDilbert.size());
}
}
}

6
src/test/resources/test-chain-v2.json
View File

@@ -31,6 +31,12 @@
"blockRewardBatchStartHeight": 999999000,
"blockRewardBatchSize": 10,
"blockRewardBatchAccountsBlockCount": 3,
"mintingGroupIds": [
{ "height": 0, "ids": []},
{ "height": 5, "ids": [694]},
{ "height": 8, "ids": [694, 800]},
{ "height": 12, "ids": [800]}
],
"rewardsByHeight": [
{ "height": 1, "reward": 100 },
{ "height": 11, "reward": 10 },