AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Rework of file list requests and relays, allowing it to handle multiple chunk resources in a much more sensible way.

Browse Source 
This could create a lot of additional relay traffic as a result, so needs lots of testing and possibly optimizing.
This commit is contained in:
CalDescent 2022-01-16 20:39:37 +00:00
parent 8fb2d38cd1
commit 6f80a6c08a
2 changed files with 78 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java
View File

@ -291,7 +291,20 @@ public class ArbitraryDataFileListManager {
return true;
}
public void deleteFileListRequestsForSignature(byte[] signature) {
String signature58 = Base58.encode(signature);
for (Iterator<Map.Entry<Integer, Triple<String, Peer, Long>>> it = arbitraryDataFileListRequests.entrySet().iterator(); it.hasNext();) {
Map.Entry<Integer, Triple<String, Peer, Long>> entry = it.next();
if (entry == null || entry.getKey() == null || entry.getValue() != null) {
continue;
}
if (Objects.equals(entry.getValue().getA(), signature58)) {
// Update requests map to reflect that we've received all chunks
Triple<String, Peer, Long> newEntry = new Triple<>(null, null, entry.getValue().getC());
arbitraryDataFileListRequests.put(entry.getKey(), newEntry);
}
}
}
// Network handlers
@ -304,7 +317,7 @@ public class ArbitraryDataFileListManager {
ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
// Do we have a pending request for this data? // TODO: might we want to relay all of them anyway?
// Do we have a pending request for this data?
Triple<String, Peer, Long> request = arbitraryDataFileListRequests.get(message.getId());
if (request == null || request.getA() == null) {
return;
@ -350,10 +363,6 @@ public class ArbitraryDataFileListManager {
// }
// }
// Update requests map to reflect that we've received it
Triple<String, Peer, Long> newEntry = new Triple<>(null, null, request.getC());
arbitraryDataFileListRequests.put(message.getId(), newEntry);
if (!isRelayRequest || !Settings.getInstance().isRelayModeEnabled()) {
// Go and fetch the actual data, since this isn't a relay request
arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, hashes);
@ -412,6 +421,7 @@ public class ArbitraryDataFileListManager {
List<byte[]> hashes = new ArrayList<>();
ArbitraryTransactionData transactionData = null;
boolean allChunksExist = false;
try (final Repository repository = RepositoryManager.getRepository()) {
@ -435,6 +445,9 @@ public class ArbitraryDataFileListManager {
hashes.add(arbitraryDataFile.getMetadataHash());
}
// Check if we have all the chunks (and the metadata file)
allChunksExist = arbitraryDataFile.allChunksExist();
for (ArbitraryDataFileChunk chunk : arbitraryDataFile.getChunks()) {
if (chunk.exists()) {
hashes.add(chunk.getHash());
@ -459,20 +472,31 @@ public class ArbitraryDataFileListManager {
// We should only respond if we have at least one hash
if (hashes.size() > 0) {
// Update requests map to reflect that we've sent it
newEntry = new Triple<>(signature58, null, now);
// We have all the chunks, so update requests map to reflect that we've sent it
// There is no need to keep track of the request, as we can serve all the chunks
if (allChunksExist) {
newEntry = new Triple<>(null, null, now);
arbitraryDataFileListRequests.put(message.getId(), newEntry);
}
ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
arbitraryDataFileListMessage.setId(message.getId());
if (!peer.sendMessage(arbitraryDataFileListMessage)) {
LOGGER.debug("Couldn't send list of hashes");
peer.disconnect("failed to send list of hashes");
return;
}
LOGGER.debug("Sent list of hashes (count: {})", hashes.size());
if (allChunksExist) {
// Nothing left to do, so return to prevent any unnecessary forwarding from occurring
LOGGER.debug("No need for any forwarding because file list request is fully served");
return;
}
else {
}
// We may need to forward this request on
boolean isBlocked = (transactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName()));
if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
// In relay mode - so ask our other peers if they have it
@ -503,6 +527,5 @@ public class ArbitraryDataFileListManager {
}
}
}
}
}

24
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java
View File

@ -15,6 +15,7 @@ import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.utils.ArbitraryTransactionUtils;
import org.qortal.utils.Base58;
import org.qortal.utils.NTP;
import org.qortal.utils.Triple;
@ -187,6 +188,9 @@ public class ArbitraryDataFileManager {
arbitraryDataFileRequests.remove(hash58);
LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58));
// We may need to remove the file list request, if we have all the files for this transaction
this.handleFileListRequests(signature);
if (message == null || message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) {
return null;
}
@ -209,6 +213,26 @@ public class ArbitraryDataFileManager {
return arbitraryDataFileMessage;
}
private void handleFileListRequests(byte[] signature) {
try (final Repository repository = RepositoryManager.getRepository()) {
// Fetch the transaction data
ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
if (arbitraryTransactionData == null) {
return;
}
boolean allChunksExist = ArbitraryTransactionUtils.allChunksExist(arbitraryTransactionData);
if (allChunksExist) {
// Update requests map to reflect that we've received all chunks
ArbitraryDataFileListManager.getInstance().deleteFileListRequestsForSignature(signature);
}
} catch (DataException e) {
LOGGER.debug("Unable to handle file list requests: {}", e.getMessage());
}
}
public void handleArbitraryDataFileForwarding(Peer requestingPeer, Message message, Message originalMessage) {
// Return if there is no originally requesting peer to forward to