diff --git a/src/main/java/org/qortal/api/resource/PeersResource.java b/src/main/java/org/qortal/api/resource/PeersResource.java index 1c7947c6..d89f99c4 100644 --- a/src/main/java/org/qortal/api/resource/PeersResource.java +++ b/src/main/java/org/qortal/api/resource/PeersResource.java @@ -20,6 +20,11 @@ import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.LoggerContext; import org.qortal.api.*; import org.qortal.api.model.ConnectedPeer; import org.qortal.api.model.PeersSummary; @@ -127,9 +132,29 @@ public class PeersResource { } ) @SecurityRequirement(name = "apiKey") - public ExecuteProduceConsume.StatsSnapshot getEngineStats(@HeaderParam(Security.API_KEY_HEADER) String apiKey) { + public ExecuteProduceConsume.StatsSnapshot getEngineStats(@HeaderParam(Security.API_KEY_HEADER) String apiKey, @QueryParam("newLoggingLevel") Level newLoggingLevel) { Security.checkApiCallAllowed(request); + if (newLoggingLevel != null) { + final LoggerContext ctx = (LoggerContext) LogManager.getContext(false); + final Configuration config = ctx.getConfiguration(); + + String epcClassName = "org.qortal.network.Network.NetworkProcessor"; + LoggerConfig loggerConfig = config.getLoggerConfig(epcClassName); + LoggerConfig specificConfig = loggerConfig; + + // We need a specific configuration for this logger, + // otherwise we would change the level of all other loggers + // having the original configuration as parent as well + if (!loggerConfig.getName().equals(epcClassName)) { + specificConfig = new LoggerConfig(epcClassName, newLoggingLevel, true); + specificConfig.setParent(loggerConfig); + config.addLogger(epcClassName, specificConfig); + } + specificConfig.setLevel(newLoggingLevel); + ctx.updateLoggers(); + } + return Network.getInstance().getStatsSnapshot(); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 02dddcc9..7aae376a 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -62,6 +62,7 @@ import org.qortal.repository.hsqldb.HSQLDBRepositoryFactory; import org.qortal.settings.Settings; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; +import org.qortal.transform.TransformationException; import org.qortal.utils.*; public class Controller extends Thread { @@ -1323,7 +1324,7 @@ public class Controller extends Thread { this.stats.getBlockMessageStats.cacheHits.incrementAndGet(); // We need to duplicate it to prevent multiple threads setting ID on the same message - CachedBlockMessage clonedBlockMessage = cachedBlockMessage.cloneWithNewId(message.getId()); + CachedBlockMessage clonedBlockMessage = Message.cloneWithNewId(cachedBlockMessage, message.getId()); if (!peer.sendMessage(clonedBlockMessage)) peer.disconnect("failed to send block"); @@ -1382,7 +1383,6 @@ public class Controller extends Thread { CachedBlockMessage blockMessage = new CachedBlockMessage(block); blockMessage.setId(message.getId()); - // This call also causes the other needed data to be pulled in from repository if (!peer.sendMessage(blockMessage)) { peer.disconnect("failed to send block"); // Don't fall-through to caching because failure to send might be from failure to build message @@ -1396,7 +1396,9 @@ public class Controller extends Thread { this.blockMessageCache.put(ByteArray.wrap(blockData.getSignature()), blockMessage); } } catch (DataException e) { - LOGGER.error(String.format("Repository issue while send block %s to peer %s", Base58.encode(signature), peer), e); + LOGGER.error(String.format("Repository issue while sending block %s to peer %s", Base58.encode(signature), peer), e); + } catch (TransformationException e) { + LOGGER.error(String.format("Serialization issue while sending block %s to peer %s", Base58.encode(signature), peer), e); } } diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index e1dc2b89..21ffd3fc 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -33,7 +33,7 @@ import org.qortal.network.message.GetBlockSummariesMessage; import org.qortal.network.message.GetSignaturesV2Message; import org.qortal.network.message.Message; import org.qortal.network.message.SignaturesMessage; -import org.qortal.network.message.Message.MessageType; +import org.qortal.network.message.MessageType; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; diff --git a/src/main/java/org/qortal/controller/TransactionImporter.java b/src/main/java/org/qortal/controller/TransactionImporter.java index 233887f7..84198a7d 100644 --- a/src/main/java/org/qortal/controller/TransactionImporter.java +++ b/src/main/java/org/qortal/controller/TransactionImporter.java @@ -13,6 +13,7 @@ import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; import org.qortal.transaction.Transaction; +import org.qortal.transform.TransformationException; import org.qortal.utils.Base58; import org.qortal.utils.NTP; @@ -300,7 +301,9 @@ public class TransactionImporter extends Thread { if (!peer.sendMessage(transactionMessage)) peer.disconnect("failed to send transaction"); } catch (DataException e) { - LOGGER.error(String.format("Repository issue while send transaction %s to peer %s", Base58.encode(signature), peer), e); + LOGGER.error(String.format("Repository issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e); + } catch (TransformationException e) { + LOGGER.error(String.format("Serialization issue while sending transaction %s to peer %s", Base58.encode(signature), peer), e); } } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java index e855171d..05a45425 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java @@ -511,18 +511,23 @@ public class ArbitraryDataFileListManager { // Bump requestHops if it exists if (requestHops != null) { - arbitraryDataFileListMessage.setRequestHops(++requestHops); + requestHops++; } + ArbitraryDataFileListMessage forwardArbitraryDataFileListMessage; + // Remove optional parameters if the requesting peer doesn't support it yet // A message with less statistical data is better than no message at all if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { - arbitraryDataFileListMessage.removeOptionalStats(); + forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + } else { + forwardArbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, + arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible()); } // Forward to requesting peer LOGGER.debug("Forwarding file list with {} hashes to requesting peer: {}", hashes.size(), requestingPeer); - if (!requestingPeer.sendMessage(arbitraryDataFileListMessage)) { + if (!requestingPeer.sendMessage(forwardArbitraryDataFileListMessage)) { requestingPeer.disconnect("failed to forward arbitrary data file list"); } } @@ -639,16 +644,19 @@ public class ArbitraryDataFileListManager { } String ourAddress = Network.getInstance().getOurExternalIpAddressAndPort(); - ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, - hashes, NTP.getTime(), 0, ourAddress, true); - arbitraryDataFileListMessage.setId(message.getId()); + ArbitraryDataFileListMessage arbitraryDataFileListMessage; // Remove optional parameters if the requesting peer doesn't support it yet // A message with less statistical data is better than no message at all if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) { - arbitraryDataFileListMessage.removeOptionalStats(); + arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes); + } else { + arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, + hashes, NTP.getTime(), 0, ourAddress, true); } + arbitraryDataFileListMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryDataFileListMessage)) { LOGGER.debug("Couldn't send list of hashes"); peer.disconnect("failed to send list of hashes"); @@ -670,8 +678,7 @@ public class ArbitraryDataFileListManager { // In relay mode - so ask our other peers if they have it long requestTime = getArbitraryDataFileListMessage.getRequestTime(); - int requestHops = getArbitraryDataFileListMessage.getRequestHops(); - getArbitraryDataFileListMessage.setRequestHops(++requestHops); + int requestHops = getArbitraryDataFileListMessage.getRequestHops() + 1; long totalRequestTime = now - requestTime; if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { @@ -679,11 +686,13 @@ public class ArbitraryDataFileListManager { if (requestHops < RELAY_REQUEST_MAX_HOPS) { // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + Message relayGetArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature, hashes, requestTime, requestHops, requestingPeer); + LOGGER.debug("Rebroadcasting hash list request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); Network.getInstance().broadcast( broadcastPeer -> broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) - ? null : getArbitraryDataFileListMessage); + ? null : relayGetArbitraryDataFileListMessage); } else { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java index 809c15ea..11e15414 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java @@ -7,7 +7,6 @@ import org.qortal.controller.Controller; import org.qortal.data.arbitrary.ArbitraryDirectConnectionInfo; import org.qortal.data.arbitrary.ArbitraryFileListResponseInfo; import org.qortal.data.arbitrary.ArbitraryRelayInfo; -import org.qortal.data.network.ArbitraryPeerData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.network.Network; @@ -187,7 +186,7 @@ public class ArbitraryDataFileManager extends Thread { ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature); boolean fileAlreadyExists = existingFile.exists(); String hash58 = Base58.encode(hash); - Message message = null; + ArbitraryDataFileMessage arbitraryDataFileMessage; // Fetch the file if it doesn't exist locally if (!fileAlreadyExists) { @@ -195,10 +194,11 @@ public class ArbitraryDataFileManager extends Thread { arbitraryDataFileRequests.put(hash58, NTP.getTime()); Message getArbitraryDataFileMessage = new GetArbitraryDataFileMessage(signature, hash); + Message response = null; try { - message = peer.getResponseWithTimeout(getArbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT); + response = peer.getResponseWithTimeout(getArbitraryDataFileMessage, (int) ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT); } catch (InterruptedException e) { - // Will return below due to null message + // Will return below due to null response } arbitraryDataFileRequests.remove(hash58); LOGGER.trace(String.format("Removed hash %.8s from arbitraryDataFileRequests", hash58)); @@ -206,22 +206,24 @@ public class ArbitraryDataFileManager extends Thread { // We may need to remove the file list request, if we have all the files for this transaction this.handleFileListRequests(signature); - if (message == null) { - LOGGER.debug("Received null message from peer {}", peer); + if (response == null) { + LOGGER.debug("Received null response from peer {}", peer); return null; } - if (message.getType() != Message.MessageType.ARBITRARY_DATA_FILE) { - LOGGER.debug("Received message with invalid type: {} from peer {}", message.getType(), peer); + if (response.getType() != MessageType.ARBITRARY_DATA_FILE) { + LOGGER.debug("Received response with invalid type: {} from peer {}", response.getType(), peer); return null; } - } - else { + + ArbitraryDataFileMessage peersArbitraryDataFileMessage = (ArbitraryDataFileMessage) response; + arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, peersArbitraryDataFileMessage.getArbitraryDataFile()); + } else { LOGGER.debug(String.format("File hash %s already exists, so skipping the request", hash58)); + arbitraryDataFileMessage = new ArbitraryDataFileMessage(signature, existingFile); } - ArbitraryDataFileMessage arbitraryDataFileMessage = (ArbitraryDataFileMessage) message; // We might want to forward the request to the peer that originally requested it - this.handleArbitraryDataFileForwarding(requestingPeer, message, originalMessage); + this.handleArbitraryDataFileForwarding(requestingPeer, arbitraryDataFileMessage, originalMessage); boolean isRelayRequest = (requestingPeer != null); if (isRelayRequest) { diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java index acc97f35..0903de60 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java @@ -338,9 +338,11 @@ public class ArbitraryMetadataManager { Peer requestingPeer = request.getB(); if (requestingPeer != null) { + ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile()); + // Forward to requesting peer LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer); - if (!requestingPeer.sendMessage(arbitraryMetadataMessage)) { + if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) { requestingPeer.disconnect("failed to forward arbitrary metadata"); } } @@ -423,8 +425,7 @@ public class ArbitraryMetadataManager { // In relay mode - so ask our other peers if they have it long requestTime = getArbitraryMetadataMessage.getRequestTime(); - int requestHops = getArbitraryMetadataMessage.getRequestHops(); - getArbitraryMetadataMessage.setRequestHops(++requestHops); + int requestHops = getArbitraryMetadataMessage.getRequestHops() + 1; long totalRequestTime = now - requestTime; if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) { @@ -432,11 +433,13 @@ public class ArbitraryMetadataManager { if (requestHops < RELAY_REQUEST_MAX_HOPS) { // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast + Message relayGetArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, requestTime, requestHops); + LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops); Network.getInstance().broadcast( broadcastPeer -> broadcastPeer == peer || Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost()) - ? null : getArbitraryMetadataMessage); + ? null : relayGetArbitraryMetadataMessage); } else { diff --git a/src/main/java/org/qortal/network/Handshake.java b/src/main/java/org/qortal/network/Handshake.java index cdcff1d7..22354cc4 100644 --- a/src/main/java/org/qortal/network/Handshake.java +++ b/src/main/java/org/qortal/network/Handshake.java @@ -13,7 +13,7 @@ import org.qortal.crypto.MemoryPoW; import org.qortal.network.message.ChallengeMessage; import org.qortal.network.message.HelloMessage; import org.qortal.network.message.Message; -import org.qortal.network.message.Message.MessageType; +import org.qortal.network.message.MessageType; import org.qortal.settings.Settings; import org.qortal.network.message.ResponseMessage; import org.qortal.utils.DaemonThreadFactory; diff --git a/src/main/java/org/qortal/network/Network.java b/src/main/java/org/qortal/network/Network.java index a58db403..8ca59210 100644 --- a/src/main/java/org/qortal/network/Network.java +++ b/src/main/java/org/qortal/network/Network.java @@ -13,6 +13,7 @@ import org.qortal.data.block.BlockData; import org.qortal.data.network.PeerData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.message.*; +import org.qortal.network.task.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -32,6 +33,7 @@ import java.nio.channels.*; import java.security.SecureRandom; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; @@ -41,9 +43,8 @@ import java.util.stream.Collectors; // For managing peers public class Network { private static final Logger LOGGER = LogManager.getLogger(Network.class); - private static Network instance; - private static final int LISTEN_BACKLOG = 10; + private static final int LISTEN_BACKLOG = 5; /** * How long before retrying after a connection failure, in milliseconds. */ @@ -122,14 +123,8 @@ public class Network { private final ExecuteProduceConsume networkEPC; private Selector channelSelector; private ServerSocketChannel serverChannel; - private Iterator channelIterator = null; - - // volatile because value is updated inside any one of the EPC threads - private volatile long nextConnectTaskTimestamp = 0L; // ms - try first connect once NTP syncs - - private final ExecutorService broadcastExecutor = Executors.newCachedThreadPool(); - // volatile because value is updated inside any one of the EPC threads - private volatile long nextBroadcastTimestamp = 0L; // ms - try first broadcast once NTP syncs + private SelectionKey serverSelectionKey; + private final Set channelsPendingWrite = ConcurrentHashMap.newKeySet(); private final Lock mergePeersLock = new ReentrantLock(); @@ -137,6 +132,8 @@ public class Network { private String ourExternalIpAddress = null; private int ourExternalPort = Settings.getInstance().getListenPort(); + private volatile boolean isShuttingDown = false; + // Constructors private Network() { @@ -170,7 +167,7 @@ public class Network { serverChannel.configureBlocking(false); serverChannel.setOption(StandardSocketOptions.SO_REUSEADDR, true); serverChannel.bind(endpoint, LISTEN_BACKLOG); - serverChannel.register(channelSelector, SelectionKey.OP_ACCEPT); + serverSelectionKey = serverChannel.register(channelSelector, SelectionKey.OP_ACCEPT); } catch (UnknownHostException e) { LOGGER.error("Can't bind listen socket to address {}", Settings.getInstance().getBindAddress()); throw new IOException("Can't bind listen socket to address", e); @@ -180,7 +177,8 @@ public class Network { } // Load all known peers from repository - synchronized (this.allKnownPeers) { List fixedNetwork = Settings.getInstance().getFixedNetwork(); + synchronized (this.allKnownPeers) { + List fixedNetwork = Settings.getInstance().getFixedNetwork(); if (fixedNetwork != null && !fixedNetwork.isEmpty()) { Long addedWhen = NTP.getTime(); String addedBy = "fixedNetwork"; @@ -214,12 +212,16 @@ public class Network { // Getters / setters - public static synchronized Network getInstance() { - if (instance == null) { - instance = new Network(); - } + private static class SingletonContainer { + private static final Network INSTANCE = new Network(); + } - return instance; + public static Network getInstance() { + return SingletonContainer.INSTANCE; + } + + public int getMaxPeers() { + return this.maxPeers; } public byte[] getMessageMagic() { @@ -453,6 +455,11 @@ public class Network { class NetworkProcessor extends ExecuteProduceConsume { + private final AtomicLong nextConnectTaskTimestamp = new AtomicLong(0L); // ms - try first connect once NTP syncs + private final AtomicLong nextBroadcastTimestamp = new AtomicLong(0L); // ms - try first broadcast once NTP syncs + + private Iterator channelIterator = null; + NetworkProcessor(ExecutorService executor) { super(executor); } @@ -494,43 +501,23 @@ public class Network { } private Task maybeProducePeerMessageTask() { - for (Peer peer : getImmutableConnectedPeers()) { - Task peerTask = peer.getMessageTask(); - if (peerTask != null) { - return peerTask; - } - } - - return null; + return getImmutableConnectedPeers().stream() + .map(Peer::getMessageTask) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); } private Task maybeProducePeerPingTask(Long now) { - // Ask connected peers whether they need a ping - for (Peer peer : getImmutableHandshakedPeers()) { - Task peerTask = peer.getPingTask(now); - if (peerTask != null) { - return peerTask; - } - } - - return null; - } - - class PeerConnectTask implements ExecuteProduceConsume.Task { - private final Peer peer; - - PeerConnectTask(Peer peer) { - this.peer = peer; - } - - @Override - public void perform() throws InterruptedException { - connectPeer(peer); - } + return getImmutableHandshakedPeers().stream() + .map(peer -> peer.getPingTask(now)) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); } private Task maybeProduceConnectPeerTask(Long now) throws InterruptedException { - if (now == null || now < nextConnectTaskTimestamp) { + if (now == null || now < nextConnectTaskTimestamp.get()) { return null; } @@ -538,7 +525,7 @@ public class Network { return null; } - nextConnectTaskTimestamp = now + 1000L; + nextConnectTaskTimestamp.set(now + 1000L); Peer targetPeer = getConnectablePeer(now); if (targetPeer == null) { @@ -550,66 +537,15 @@ public class Network { } private Task maybeProduceBroadcastTask(Long now) { - if (now == null || now < nextBroadcastTimestamp) { + if (now == null || now < nextBroadcastTimestamp.get()) { return null; } - nextBroadcastTimestamp = now + BROADCAST_INTERVAL; - return () -> Controller.getInstance().doNetworkBroadcast(); - } - - class ChannelTask implements ExecuteProduceConsume.Task { - private final SelectionKey selectionKey; - - ChannelTask(SelectionKey selectionKey) { - this.selectionKey = selectionKey; - } - - @Override - public void perform() throws InterruptedException { - try { - LOGGER.trace("Thread {} has pending channel: {}, with ops {}", - Thread.currentThread().getId(), selectionKey.channel(), selectionKey.readyOps()); - - // process pending channel task - if (selectionKey.isReadable()) { - connectionRead((SocketChannel) selectionKey.channel()); - } else if (selectionKey.isAcceptable()) { - acceptConnection((ServerSocketChannel) selectionKey.channel()); - } - - LOGGER.trace("Thread {} processed channel: {}", - Thread.currentThread().getId(), selectionKey.channel()); - } catch (CancelledKeyException e) { - LOGGER.trace("Thread {} encountered cancelled channel: {}", - Thread.currentThread().getId(), selectionKey.channel()); - } - } - - private void connectionRead(SocketChannel socketChannel) { - Peer peer = getPeerFromChannel(socketChannel); - if (peer == null) { - return; - } - - try { - peer.readChannel(); - } catch (IOException e) { - if (e.getMessage() != null && e.getMessage().toLowerCase().contains("connection reset")) { - peer.disconnect("Connection reset"); - return; - } - - LOGGER.trace("[{}] Network thread {} encountered I/O error: {}", peer.getPeerConnectionId(), - Thread.currentThread().getId(), e.getMessage(), e); - peer.disconnect("I/O error"); - } - } + nextBroadcastTimestamp.set(now + BROADCAST_INTERVAL); + return new BroadcastTask(); } private Task maybeProduceChannelTask(boolean canBlock) throws InterruptedException { - final SelectionKey nextSelectionKey; - // Synchronization here to enforce thread-safety on channelIterator synchronized (channelSelector) { // anything to do? @@ -630,91 +566,73 @@ public class Network { } channelIterator = channelSelector.selectedKeys().iterator(); + LOGGER.trace("Thread {}, after {} select, channelIterator now {}", + Thread.currentThread().getId(), + canBlock ? "blocking": "non-blocking", + channelIterator); } - if (channelIterator.hasNext()) { - nextSelectionKey = channelIterator.next(); - channelIterator.remove(); - } else { - nextSelectionKey = null; + if (!channelIterator.hasNext()) { channelIterator = null; // Nothing to do so reset iterator to cause new select + + LOGGER.trace("Thread {}, channelIterator now null", Thread.currentThread().getId()); + return null; } - LOGGER.trace("Thread {}, nextSelectionKey {}, channelIterator now {}", - Thread.currentThread().getId(), nextSelectionKey, channelIterator); - } + final SelectionKey nextSelectionKey = channelIterator.next(); + channelIterator.remove(); - if (nextSelectionKey == null) { - return null; - } + // Just in case underlying socket channel already closed elsewhere, etc. + if (!nextSelectionKey.isValid()) + return null; - return new ChannelTask(nextSelectionKey); - } - } + LOGGER.trace("Thread {}, nextSelectionKey {}", Thread.currentThread().getId(), nextSelectionKey); - private void acceptConnection(ServerSocketChannel serverSocketChannel) throws InterruptedException { - SocketChannel socketChannel; + SelectableChannel socketChannel = nextSelectionKey.channel(); - try { - socketChannel = serverSocketChannel.accept(); - } catch (IOException e) { - return; - } - - // No connection actually accepted? - if (socketChannel == null) { - return; - } - PeerAddress address = PeerAddress.fromSocket(socketChannel.socket()); - List fixedNetwork = Settings.getInstance().getFixedNetwork(); - if (fixedNetwork != null && !fixedNetwork.isEmpty() && ipNotInFixedList(address, fixedNetwork)) { - try { - LOGGER.debug("Connection discarded from peer {} as not in the fixed network list", address); - socketChannel.close(); - } catch (IOException e) { - // IGNORE - } - return; - } - - final Long now = NTP.getTime(); - Peer newPeer; - - try { - if (now == null) { - LOGGER.debug("Connection discarded from peer {} due to lack of NTP sync", address); - socketChannel.close(); - return; - } - - if (getImmutableConnectedPeers().size() >= maxPeers) { - // We have enough peers - LOGGER.debug("Connection discarded from peer {} because the server is full", address); - socketChannel.close(); - return; - } - - LOGGER.debug("Connection accepted from peer {}", address); - - newPeer = new Peer(socketChannel, channelSelector); - this.addConnectedPeer(newPeer); - - } catch (IOException e) { - if (socketChannel.isOpen()) { try { - LOGGER.debug("Connection failed from peer {} while connecting/closing", address); - socketChannel.close(); - } catch (IOException ce) { - // Couldn't close? + if (nextSelectionKey.isReadable()) { + clearInterestOps(nextSelectionKey, SelectionKey.OP_READ); + Peer peer = getPeerFromChannel((SocketChannel) socketChannel); + if (peer == null) + return null; + + return new ChannelReadTask((SocketChannel) socketChannel, peer); + } + + if (nextSelectionKey.isWritable()) { + clearInterestOps(nextSelectionKey, SelectionKey.OP_WRITE); + Peer peer = getPeerFromChannel((SocketChannel) socketChannel); + if (peer == null) + return null; + + // Any thread that queues a message to send can set OP_WRITE, + // but we only allow one pending/active ChannelWriteTask per Peer + if (!channelsPendingWrite.add(socketChannel)) + return null; + + return new ChannelWriteTask((SocketChannel) socketChannel, peer); + } + + if (nextSelectionKey.isAcceptable()) { + clearInterestOps(nextSelectionKey, SelectionKey.OP_ACCEPT); + return new ChannelAcceptTask((ServerSocketChannel) socketChannel); + } + } catch (CancelledKeyException e) { + /* + * Sometimes nextSelectionKey is cancelled / becomes invalid between the isValid() test at line 586 + * and later calls to isReadable() / isWritable() / isAcceptable() which themselves call isValid()! + * Those isXXXable() calls could throw CancelledKeyException, so we catch it here and return null. + */ + return null; } } - return; - } - this.onPeerReady(newPeer); + return null; + } } - private boolean ipNotInFixedList(PeerAddress address, List fixedNetwork) { + public boolean ipNotInFixedList(PeerAddress address, List fixedNetwork) { for (String ipAddress : fixedNetwork) { String[] bits = ipAddress.split(":"); if (bits.length >= 1 && bits.length <= 2 && address.getHost().equals(bits[0])) { @@ -750,8 +668,9 @@ public class Network { peers.removeIf(isConnectedPeer); // Don't consider already connected peers (resolved address match) - // XXX This might be too slow if we end up waiting a long time for hostnames to resolve via DNS - peers.removeIf(isResolvedAsConnectedPeer); + // Disabled because this might be too slow if we end up waiting a long time for hostnames to resolve via DNS + // Which is ok because duplicate connections to the same peer are handled during handshaking + // peers.removeIf(isResolvedAsConnectedPeer); this.checkLongestConnection(now); @@ -781,8 +700,12 @@ public class Network { } } - private boolean connectPeer(Peer newPeer) throws InterruptedException { - SocketChannel socketChannel = newPeer.connect(this.channelSelector); + public boolean connectPeer(Peer newPeer) throws InterruptedException { + // NOT CORRECT: + if (getImmutableConnectedPeers().size() >= minOutboundPeers) + return false; + + SocketChannel socketChannel = newPeer.connect(); if (socketChannel == null) { return false; } @@ -797,7 +720,7 @@ public class Network { return true; } - private Peer getPeerFromChannel(SocketChannel socketChannel) { + public Peer getPeerFromChannel(SocketChannel socketChannel) { for (Peer peer : this.getImmutableConnectedPeers()) { if (peer.getSocketChannel() == socketChannel) { return peer; @@ -830,7 +753,74 @@ public class Network { nextDisconnectionCheck = now + DISCONNECTION_CHECK_INTERVAL; } - // Peer callbacks + // SocketChannel interest-ops manipulations + + private static final String[] OP_NAMES = new String[SelectionKey.OP_ACCEPT * 2]; + static { + for (int i = 0; i < OP_NAMES.length; i++) { + StringJoiner joiner = new StringJoiner(","); + + if ((i & SelectionKey.OP_READ) != 0) joiner.add("OP_READ"); + if ((i & SelectionKey.OP_WRITE) != 0) joiner.add("OP_WRITE"); + if ((i & SelectionKey.OP_CONNECT) != 0) joiner.add("OP_CONNECT"); + if ((i & SelectionKey.OP_ACCEPT) != 0) joiner.add("OP_ACCEPT"); + + OP_NAMES[i] = joiner.toString(); + } + } + + public void clearInterestOps(SelectableChannel socketChannel, int interestOps) { + SelectionKey selectionKey = socketChannel.keyFor(channelSelector); + if (selectionKey == null) + return; + + clearInterestOps(selectionKey, interestOps); + } + + private void clearInterestOps(SelectionKey selectionKey, int interestOps) { + if (!selectionKey.channel().isOpen()) + return; + + LOGGER.trace("Thread {} clearing {} interest-ops on channel: {}", + Thread.currentThread().getId(), + OP_NAMES[interestOps], + selectionKey.channel()); + + selectionKey.interestOpsAnd(~interestOps); + } + + public void setInterestOps(SelectableChannel socketChannel, int interestOps) { + SelectionKey selectionKey = socketChannel.keyFor(channelSelector); + if (selectionKey == null) { + try { + selectionKey = socketChannel.register(this.channelSelector, interestOps); + } catch (ClosedChannelException e) { + // Channel already closed so ignore + return; + } + // Fall-through to allow logging + } + + setInterestOps(selectionKey, interestOps); + } + + private void setInterestOps(SelectionKey selectionKey, int interestOps) { + if (!selectionKey.channel().isOpen()) + return; + + LOGGER.trace("Thread {} setting {} interest-ops on channel: {}", + Thread.currentThread().getId(), + OP_NAMES[interestOps], + selectionKey.channel()); + + selectionKey.interestOpsOr(interestOps); + } + + // Peer / Task callbacks + + public void notifyChannelNotWriting(SelectableChannel socketChannel) { + this.channelsPendingWrite.remove(socketChannel); + } protected void wakeupChannelSelector() { this.channelSelector.wakeup(); @@ -856,8 +846,6 @@ public class Network { } public void onDisconnect(Peer peer) { - // Notify Controller - Controller.getInstance().onPeerDisconnect(peer); if (peer.getConnectionEstablishedTime() > 0L) { LOGGER.debug("[{}] Disconnected from peer {}", peer.getPeerConnectionId(), peer); } else { @@ -865,6 +853,25 @@ public class Network { } this.removeConnectedPeer(peer); + this.channelsPendingWrite.remove(peer.getSocketChannel()); + + if (this.isShuttingDown) + // No need to do any further processing, like re-enabling listen socket or notifying Controller + return; + + if (getImmutableConnectedPeers().size() < maxPeers - 1 + && serverSelectionKey.isValid() + && (serverSelectionKey.interestOps() & SelectionKey.OP_ACCEPT) == 0) { + try { + LOGGER.debug("Re-enabling accepting incoming connections because the server is not longer full"); + setInterestOps(serverSelectionKey, SelectionKey.OP_ACCEPT); + } catch (CancelledKeyException e) { + LOGGER.error("Failed to re-enable accepting of incoming connections: {}", e.getMessage()); + } + } + + // Notify Controller + Controller.getInstance().onPeerDisconnect(peer); } public void peerMisbehaved(Peer peer) { @@ -1304,8 +1311,9 @@ public class Network { try { InetSocketAddress knownAddress = peerAddress.toSocketAddress(); - List peers = this.getImmutableConnectedPeers(); - peers.removeIf(peer -> !Peer.addressEquals(knownAddress, peer.getResolvedAddress())); + List peers = this.getImmutableConnectedPeers().stream() + .filter(peer -> Peer.addressEquals(knownAddress, peer.getResolvedAddress())) + .collect(Collectors.toList()); for (Peer peer : peers) { peer.disconnect("to be forgotten"); @@ -1463,54 +1471,27 @@ public class Network { } public void broadcast(Function peerMessageBuilder) { - class Broadcaster implements Runnable { - private final Random random = new Random(); + for (Peer peer : getImmutableHandshakedPeers()) { + if (this.isShuttingDown) + return; - private List targetPeers; - private Function peerMessageBuilder; + Message message = peerMessageBuilder.apply(peer); - Broadcaster(List targetPeers, Function peerMessageBuilder) { - this.targetPeers = targetPeers; - this.peerMessageBuilder = peerMessageBuilder; + if (message == null) { + continue; } - @Override - public void run() { - Thread.currentThread().setName("Network Broadcast"); - - for (Peer peer : targetPeers) { - // Very short sleep to reduce strain, improve multi-threading and catch interrupts - try { - Thread.sleep(random.nextInt(20) + 20L); - } catch (InterruptedException e) { - break; - } - - Message message = peerMessageBuilder.apply(peer); - - if (message == null) { - continue; - } - - if (!peer.sendMessage(message)) { - peer.disconnect("failed to broadcast message"); - } - } - - Thread.currentThread().setName("Network Broadcast (dormant)"); + if (!peer.sendMessage(message)) { + peer.disconnect("failed to broadcast message"); } } - - try { - broadcastExecutor.execute(new Broadcaster(this.getImmutableHandshakedPeers(), peerMessageBuilder)); - } catch (RejectedExecutionException e) { - // Can't execute - probably because we're shutting down, so ignore - } } // Shutdown public void shutdown() { + this.isShuttingDown = true; + // Close listen socket to prevent more incoming connections if (this.serverChannel.isOpen()) { try { @@ -1529,16 +1510,6 @@ public class Network { LOGGER.warn("Interrupted while waiting for networking threads to terminate"); } - // Stop broadcasts - this.broadcastExecutor.shutdownNow(); - try { - if (!this.broadcastExecutor.awaitTermination(1000, TimeUnit.MILLISECONDS)) { - LOGGER.warn("Broadcast threads failed to terminate"); - } - } catch (InterruptedException e) { - LOGGER.warn("Interrupted while waiting for broadcast threads failed to terminate"); - } - // Close all peer connections for (Peer peer : this.getImmutableConnectedPeers()) { peer.shutdown(); diff --git a/src/main/java/org/qortal/network/Peer.java b/src/main/java/org/qortal/network/Peer.java index da4a70a9..dbb03fda 100644 --- a/src/main/java/org/qortal/network/Peer.java +++ b/src/main/java/org/qortal/network/Peer.java @@ -11,25 +11,21 @@ import org.qortal.data.network.PeerChainTipData; import org.qortal.data.network.PeerData; import org.qortal.network.message.ChallengeMessage; import org.qortal.network.message.Message; -import org.qortal.network.message.Message.MessageException; -import org.qortal.network.message.Message.MessageType; -import org.qortal.network.message.PingMessage; +import org.qortal.network.message.MessageException; +import org.qortal.network.task.MessageTask; +import org.qortal.network.task.PingTask; import org.qortal.settings.Settings; -import org.qortal.utils.ExecuteProduceConsume; +import org.qortal.utils.ExecuteProduceConsume.Task; import org.qortal.utils.NTP; import java.io.IOException; import java.net.*; import java.nio.ByteBuffer; import java.nio.channels.SelectionKey; -import java.nio.channels.Selector; import java.nio.channels.SocketChannel; import java.security.SecureRandom; import java.util.*; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -48,9 +44,9 @@ public class Peer { private static final int RESPONSE_TIMEOUT = 3000; // ms /** - * Maximum time to wait for a peer to respond with blocks (ms) + * Maximum time to wait for a message to be added to sendQueue (ms) */ - public static final int FETCH_BLOCKS_TIMEOUT = 10000; + private static final int QUEUE_TIMEOUT = 1000; // ms /** * Interval between PING messages to a peer. (ms) @@ -71,10 +67,14 @@ public class Peer { private final UUID peerConnectionId = UUID.randomUUID(); private final Object byteBufferLock = new Object(); private ByteBuffer byteBuffer; - private Map> replyQueues; private LinkedBlockingQueue pendingMessages; + private TransferQueue sendQueue; + private ByteBuffer outputBuffer; + private String outputMessageType; + private int outputMessageId; + /** * True if we created connection to peer, false if we accepted incoming connection from peer. */ @@ -98,7 +98,7 @@ public class Peer { /** * When last PING message was sent, or null if pings not started yet. */ - private Long lastPingSent; + private Long lastPingSent = null; byte[] ourChallenge; @@ -160,10 +160,10 @@ public class Peer { /** * Construct Peer using existing, connected socket */ - public Peer(SocketChannel socketChannel, Selector channelSelector) throws IOException { + public Peer(SocketChannel socketChannel) throws IOException { this.isOutbound = false; this.socketChannel = socketChannel; - sharedSetup(channelSelector); + sharedSetup(); this.resolvedAddress = ((InetSocketAddress) socketChannel.socket().getRemoteSocketAddress()); this.isLocal = isAddressLocal(this.resolvedAddress.getAddress()); @@ -276,7 +276,7 @@ public class Peer { } } - protected void setLastPing(long lastPing) { + public void setLastPing(long lastPing) { synchronized (this.peerInfoLock) { this.lastPing = lastPing; } @@ -346,12 +346,6 @@ public class Peer { } } - protected void queueMessage(Message message) { - if (!this.pendingMessages.offer(message)) { - LOGGER.info("[{}] No room to queue message from peer {} - discarding", this.peerConnectionId, this); - } - } - public boolean isSyncInProgress() { return this.syncInProgress; } @@ -396,13 +390,14 @@ public class Peer { // Processing - private void sharedSetup(Selector channelSelector) throws IOException { + private void sharedSetup() throws IOException { this.connectionTimestamp = NTP.getTime(); this.socketChannel.setOption(StandardSocketOptions.TCP_NODELAY, true); this.socketChannel.configureBlocking(false); - this.socketChannel.register(channelSelector, SelectionKey.OP_READ); + Network.getInstance().setInterestOps(this.socketChannel, SelectionKey.OP_READ); this.byteBuffer = null; // Defer allocation to when we need it, to save memory. Sorry GC! - this.replyQueues = Collections.synchronizedMap(new HashMap>()); + this.sendQueue = new LinkedTransferQueue<>(); + this.replyQueues = new ConcurrentHashMap<>(); this.pendingMessages = new LinkedBlockingQueue<>(); Random random = new SecureRandom(); @@ -410,7 +405,7 @@ public class Peer { random.nextBytes(this.ourChallenge); } - public SocketChannel connect(Selector channelSelector) { + public SocketChannel connect() { LOGGER.trace("[{}] Connecting to peer {}", this.peerConnectionId, this); try { @@ -418,6 +413,8 @@ public class Peer { this.isLocal = isAddressLocal(this.resolvedAddress.getAddress()); this.socketChannel = SocketChannel.open(); + InetAddress bindAddr = InetAddress.getByName(Settings.getInstance().getBindAddress()); + this.socketChannel.socket().bind(new InetSocketAddress(bindAddr, 0)); this.socketChannel.socket().connect(resolvedAddress, CONNECT_TIMEOUT); } catch (SocketTimeoutException e) { LOGGER.trace("[{}] Connection timed out to peer {}", this.peerConnectionId, this); @@ -432,7 +429,7 @@ public class Peer { try { LOGGER.debug("[{}] Connected to peer {}", this.peerConnectionId, this); - sharedSetup(channelSelector); + sharedSetup(); return socketChannel; } catch (IOException e) { LOGGER.trace("[{}] Post-connection setup failed, peer {}", this.peerConnectionId, this); @@ -450,7 +447,7 @@ public class Peer { * * @throws IOException If this channel is not yet connected */ - protected void readChannel() throws IOException { + public void readChannel() throws IOException { synchronized (this.byteBufferLock) { while (true) { if (!this.socketChannel.isOpen() || this.socketChannel.socket().isClosed()) { @@ -556,7 +553,67 @@ public class Peer { } } - protected ExecuteProduceConsume.Task getMessageTask() { + /** Maybe send some pending outgoing messages. + * + * @return true if more data is pending to be sent + */ + public boolean writeChannel() throws IOException { + // It is the responsibility of ChannelWriteTask's producer to produce only one call to writeChannel() at a time + + while (true) { + // If output byte buffer is null, fetch next message from queue (if any) + while (this.outputBuffer == null) { + Message message; + + try { + // Allow other thread time to add message to queue having raised OP_WRITE. + // Timeout is overkill but not excessive enough to clog up networking / EPC. + // This is to avoid race condition in sendMessageWithTimeout() below. + message = this.sendQueue.poll(QUEUE_TIMEOUT, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + // Shutdown situation + return false; + } + + // No message? No further work to be done + if (message == null) + return false; + + try { + this.outputBuffer = ByteBuffer.wrap(message.toBytes()); + this.outputMessageType = message.getType().name(); + this.outputMessageId = message.getId(); + + LOGGER.trace("[{}] Sending {} message with ID {} to peer {}", + this.peerConnectionId, this.outputMessageType, this.outputMessageId, this); + } catch (MessageException e) { + // Something went wrong converting message to bytes, so discard but allow another round + LOGGER.warn("[{}] Failed to send {} message with ID {} to peer {}: {}", this.peerConnectionId, + message.getType().name(), message.getId(), this, e.getMessage()); + } + } + + // If output byte buffer is not null, send from that + int bytesWritten = this.socketChannel.write(outputBuffer); + + LOGGER.trace("[{}] Sent {} bytes of {} message with ID {} to peer {} ({} total)", this.peerConnectionId, + bytesWritten, this.outputMessageType, this.outputMessageId, this, outputBuffer.limit()); + + // If we've sent 0 bytes then socket buffer is full so we need to wait until it's empty again + if (bytesWritten == 0) { + return true; + } + + // If we then exhaust the byte buffer, set it to null (otherwise loop and try to send more) + if (!this.outputBuffer.hasRemaining()) { + this.outputMessageType = null; + this.outputMessageId = 0; + this.outputBuffer = null; + } + } + } + + protected Task getMessageTask() { /* * If we are still handshaking and there is a message yet to be processed then * don't produce another message task. This allows us to process handshake @@ -580,7 +637,7 @@ public class Peer { } // Return a task to process message in queue - return () -> Network.getInstance().onMessage(this, nextMessage); + return new MessageTask(this, nextMessage); } /** @@ -605,54 +662,25 @@ public class Peer { } try { - // Send message - LOGGER.trace("[{}] Sending {} message with ID {} to peer {}", this.peerConnectionId, + // Queue message, to be picked up by ChannelWriteTask and then peer.writeChannel() + LOGGER.trace("[{}] Queuing {} message with ID {} to peer {}", this.peerConnectionId, message.getType().name(), message.getId(), this); - ByteBuffer outputBuffer = ByteBuffer.wrap(message.toBytes()); + // Check message properly constructed + message.checkValidOutgoing(); - synchronized (this.socketChannel) { - final long sendStart = System.currentTimeMillis(); - long totalBytes = 0; - - while (outputBuffer.hasRemaining()) { - int bytesWritten = this.socketChannel.write(outputBuffer); - totalBytes += bytesWritten; - - LOGGER.trace("[{}] Sent {} bytes of {} message with ID {} to peer {} ({} total)", this.peerConnectionId, - bytesWritten, message.getType().name(), message.getId(), this, totalBytes); - - if (bytesWritten == 0) { - // Underlying socket's internal buffer probably full, - // so wait a short while for bytes to actually be transmitted over the wire - - /* - * NOSONAR squid:S2276 - we don't want to use this.socketChannel.wait() - * as this releases the lock held by synchronized() above - * and would allow another thread to send another message, - * potentially interleaving them on-the-wire, causing checksum failures - * and connection loss. - */ - Thread.sleep(1L); //NOSONAR squid:S2276 - - if (System.currentTimeMillis() - sendStart > timeout) { - // We've taken too long to send this message - return false; - } - } - } - } - } catch (MessageException e) { - LOGGER.warn("[{}] Failed to send {} message with ID {} to peer {}: {}", this.peerConnectionId, - message.getType().name(), message.getId(), this, e.getMessage()); - return false; - } catch (IOException | InterruptedException e) { + // Possible race condition: + // We set OP_WRITE, EPC creates ChannelWriteTask which calls Peer.writeChannel, writeChannel's poll() finds no message to send + // Avoided by poll-with-timeout in writeChannel() above. + Network.getInstance().setInterestOps(this.socketChannel, SelectionKey.OP_WRITE); + return this.sendQueue.tryTransfer(message, timeout, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { // Send failure return false; + } catch (MessageException e) { + LOGGER.error(e.getMessage(), e); + return false; } - - // Sent OK - return true; } /** @@ -720,7 +748,7 @@ public class Peer { this.lastPingSent = NTP.getTime(); } - protected ExecuteProduceConsume.Task getPingTask(Long now) { + protected Task getPingTask(Long now) { // Pings not enabled yet? if (now == null || this.lastPingSent == null) { return null; @@ -734,19 +762,7 @@ public class Peer { // Not strictly true, but prevents this peer from being immediately chosen again this.lastPingSent = now; - return () -> { - PingMessage pingMessage = new PingMessage(); - Message message = this.getResponse(pingMessage); - - if (message == null || message.getType() != MessageType.PING) { - LOGGER.debug("[{}] Didn't receive reply from {} for PING ID {}", this.peerConnectionId, this, - pingMessage.getId()); - this.disconnect("no ping received"); - return; - } - - this.setLastPing(NTP.getTime() - now); - }; + return new PingTask(this, now); } public void disconnect(String reason) { diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java index 32ba3fa7..ed3cae76 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java @@ -9,38 +9,59 @@ import org.qortal.utils.Serialization; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; public class ArbitraryDataFileListMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private static final int HASH_LENGTH = Transformer.SHA256_LENGTH; - private static final int MAX_PEER_ADDRESS_LENGTH = PeerData.MAX_PEER_ADDRESS_SIZE; - - private final byte[] signature; - private final List hashes; + private byte[] signature; + private List hashes; private Long requestTime; private Integer requestHops; private String peerAddress; private Boolean isRelayPossible; - public ArbitraryDataFileListMessage(byte[] signature, List hashes, Long requestTime, - Integer requestHops, String peerAddress, boolean isRelayPossible) { + Integer requestHops, String peerAddress, Boolean isRelayPossible) { super(MessageType.ARBITRARY_DATA_FILE_LIST); - this.signature = signature; - this.hashes = hashes; - this.requestTime = requestTime; - this.requestHops = requestHops; - this.peerAddress = peerAddress; - this.isRelayPossible = isRelayPossible; + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Ints.toByteArray(hashes.size())); + + for (byte[] hash : hashes) { + bytes.write(hash); + } + + if (requestTime != null) { + // The remaining fields are optional + + bytes.write(Longs.toByteArray(requestTime)); + + bytes.write(Ints.toByteArray(requestHops)); + + Serialization.serializeSizedStringV2(bytes, peerAddress); + + bytes.write(Ints.toByteArray(Boolean.TRUE.equals(isRelayPossible) ? 1 : 0)); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } - public ArbitraryDataFileListMessage(int id, byte[] signature, List hashes, Long requestTime, + /** Legacy version */ + public ArbitraryDataFileListMessage(byte[] signature, List hashes) { + this(signature, hashes, null, null, null, null); + } + + private ArbitraryDataFileListMessage(int id, byte[] signature, List hashes, Long requestTime, Integer requestHops, String peerAddress, boolean isRelayPossible) { super(id, MessageType.ARBITRARY_DATA_FILE_LIST); @@ -52,24 +73,39 @@ public class ArbitraryDataFileListMessage extends Message { this.isRelayPossible = isRelayPossible; } - public List getHashes() { - return this.hashes; - } - public byte[] getSignature() { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException, TransformationException { - byte[] signature = new byte[SIGNATURE_LENGTH]; + public List getHashes() { + return this.hashes; + } + + public Long getRequestTime() { + return this.requestTime; + } + + public Integer getRequestHops() { + return this.requestHops; + } + + public String getPeerAddress() { + return this.peerAddress; + } + + public Boolean isRelayPossible() { + return this.isRelayPossible; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); int count = bytes.getInt(); List hashes = new ArrayList<>(); for (int i = 0; i < count; ++i) { - - byte[] hash = new byte[HASH_LENGTH]; + byte[] hash = new byte[Transformer.SHA256_LENGTH]; bytes.get(hash); hashes.add(hash); } @@ -80,99 +116,21 @@ public class ArbitraryDataFileListMessage extends Message { boolean isRelayPossible = true; // Legacy versions only send this message when relaying is possible // The remaining fields are optional - if (bytes.hasRemaining()) { + try { + requestTime = bytes.getLong(); - requestTime = bytes.getLong(); + requestHops = bytes.getInt(); - requestHops = bytes.getInt(); - - peerAddress = Serialization.deserializeSizedStringV2(bytes, MAX_PEER_ADDRESS_LENGTH); - - isRelayPossible = bytes.getInt() > 0; + peerAddress = Serialization.deserializeSizedStringV2(bytes, PeerData.MAX_PEER_ADDRESS_SIZE); + isRelayPossible = bytes.getInt() > 0; + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); + } } return new ArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, peerAddress, isRelayPossible); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - bytes.write(Ints.toByteArray(this.hashes.size())); - - for (byte[] hash : this.hashes) { - bytes.write(hash); - } - - if (this.requestTime == null) { // To maintain backwards support - return bytes.toByteArray(); - } - - // The remaining fields are optional - - bytes.write(Longs.toByteArray(this.requestTime)); - - bytes.write(Ints.toByteArray(this.requestHops)); - - Serialization.serializeSizedStringV2(bytes, this.peerAddress); - - bytes.write(Ints.toByteArray(this.isRelayPossible ? 1 : 0)); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - - public ArbitraryDataFileListMessage cloneWithNewId(int newId) { - ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes, - this.requestTime, this.requestHops, this.peerAddress, this.isRelayPossible); - clone.setId(newId); - return clone; - } - - public void removeOptionalStats() { - this.requestTime = null; - this.requestHops = null; - this.peerAddress = null; - this.isRelayPossible = null; - } - - public Long getRequestTime() { - return this.requestTime; - } - - public void setRequestTime(Long requestTime) { - this.requestTime = requestTime; - } - - public Integer getRequestHops() { - return this.requestHops; - } - - public void setRequestHops(Integer requestHops) { - this.requestHops = requestHops; - } - - public String getPeerAddress() { - return this.peerAddress; - } - - public void setPeerAddress(String peerAddress) { - this.peerAddress = peerAddress; - } - - public Boolean isRelayPossible() { - return this.isRelayPossible; - } - - public void setIsRelayPossible(Boolean isRelayPossible) { - this.isRelayPossible = isRelayPossible; - } - } diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java index b9f24e29..50991be3 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java @@ -9,44 +9,60 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; public class ArbitraryDataFileMessage extends Message { private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileMessage.class); - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - - private final byte[] signature; - private final ArbitraryDataFile arbitraryDataFile; + private byte[] signature; + private ArbitraryDataFile arbitraryDataFile; public ArbitraryDataFileMessage(byte[] signature, ArbitraryDataFile arbitraryDataFile) { super(MessageType.ARBITRARY_DATA_FILE); - this.signature = signature; - this.arbitraryDataFile = arbitraryDataFile; + byte[] data = arbitraryDataFile.getBytes(); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Ints.toByteArray(data.length)); + + bytes.write(data); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } - public ArbitraryDataFileMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) { + private ArbitraryDataFileMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) { super(id, MessageType.ARBITRARY_DATA_FILE); this.signature = signature; this.arbitraryDataFile = arbitraryDataFile; } + public byte[] getSignature() { + return this.signature; + } + public ArbitraryDataFile getArbitraryDataFile() { return this.arbitraryDataFile; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; byteBuffer.get(signature); int dataLength = byteBuffer.getInt(); - if (byteBuffer.remaining() != dataLength) - return null; + if (byteBuffer.remaining() < dataLength) + throw new BufferUnderflowException(); byte[] data = new byte[dataLength]; byteBuffer.get(data); @@ -54,43 +70,10 @@ public class ArbitraryDataFileMessage extends Message { try { ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data, signature); return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile); - } - catch (DataException e) { + } catch (DataException e) { LOGGER.info("Unable to process received file: {}", e.getMessage()); - return null; + throw new MessageException("Unable to process received file: " + e.getMessage(), e); } } - @Override - protected byte[] toData() { - if (this.arbitraryDataFile == null) { - return null; - } - - byte[] data = this.arbitraryDataFile.getBytes(); - if (data == null) { - return null; - } - - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(signature); - - bytes.write(Ints.toByteArray(data.length)); - - bytes.write(data); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - - public ArbitraryDataFileMessage cloneWithNewId(int newId) { - ArbitraryDataFileMessage clone = new ArbitraryDataFileMessage(this.signature, this.arbitraryDataFile); - clone.setId(newId); - return clone; - } - } diff --git a/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java b/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java index 1ce149f7..142e35cc 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryDataMessage.java @@ -2,7 +2,7 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import org.qortal.transform.Transformer; @@ -11,13 +11,26 @@ import com.google.common.primitives.Ints; public class ArbitraryDataMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private byte[] signature; private byte[] data; public ArbitraryDataMessage(byte[] signature, byte[] data) { - this(-1, signature, data); + super(MessageType.ARBITRARY_DATA); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Ints.toByteArray(data.length)); + + bytes.write(data); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private ArbitraryDataMessage(int id, byte[] signature, byte[] data) { @@ -35,14 +48,14 @@ public class ArbitraryDataMessage extends Message { return this.data; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; byteBuffer.get(signature); int dataLength = byteBuffer.getInt(); - if (byteBuffer.remaining() != dataLength) - return null; + if (byteBuffer.remaining() < dataLength) + throw new BufferUnderflowException(); byte[] data = new byte[dataLength]; byteBuffer.get(data); @@ -50,24 +63,4 @@ public class ArbitraryDataMessage extends Message { return new ArbitraryDataMessage(id, signature, data); } - @Override - protected byte[] toData() { - if (this.data == null) - return null; - - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - bytes.write(Ints.toByteArray(this.data.length)); - - bytes.write(this.data); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java b/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java index 9228d458..26601d4b 100644 --- a/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java @@ -7,28 +7,40 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; public class ArbitraryMetadataMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + private byte[] signature; + private ArbitraryDataFile arbitraryMetadataFile; - private final byte[] signature; - private final ArbitraryDataFile arbitraryMetadataFile; - - public ArbitraryMetadataMessage(byte[] signature, ArbitraryDataFile arbitraryDataFile) { + public ArbitraryMetadataMessage(byte[] signature, ArbitraryDataFile arbitraryMetadataFile) { super(MessageType.ARBITRARY_METADATA); - this.signature = signature; - this.arbitraryMetadataFile = arbitraryDataFile; + byte[] data = arbitraryMetadataFile.getBytes(); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Ints.toByteArray(data.length)); + + bytes.write(data); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } - public ArbitraryMetadataMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) { + private ArbitraryMetadataMessage(int id, byte[] signature, ArbitraryDataFile arbitraryMetadataFile) { super(id, MessageType.ARBITRARY_METADATA); this.signature = signature; - this.arbitraryMetadataFile = arbitraryDataFile; + this.arbitraryMetadataFile = arbitraryMetadataFile; } public byte[] getSignature() { @@ -39,14 +51,14 @@ public class ArbitraryMetadataMessage extends Message { return this.arbitraryMetadataFile; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; byteBuffer.get(signature); int dataLength = byteBuffer.getInt(); - if (byteBuffer.remaining() != dataLength) - return null; + if (byteBuffer.remaining() < dataLength) + throw new BufferUnderflowException(); byte[] data = new byte[dataLength]; byteBuffer.get(data); @@ -54,42 +66,9 @@ public class ArbitraryMetadataMessage extends Message { try { ArbitraryDataFile arbitraryMetadataFile = new ArbitraryDataFile(data, signature); return new ArbitraryMetadataMessage(id, signature, arbitraryMetadataFile); + } catch (DataException e) { + throw new MessageException("Unable to process arbitrary metadata message: " + e.getMessage(), e); } - catch (DataException e) { - return null; - } - } - - @Override - protected byte[] toData() { - if (this.arbitraryMetadataFile == null) { - return null; - } - - byte[] data = this.arbitraryMetadataFile.getBytes(); - if (data == null) { - return null; - } - - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(signature); - - bytes.write(Ints.toByteArray(data.length)); - - bytes.write(data); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - - public ArbitraryMetadataMessage cloneWithNewId(int newId) { - ArbitraryMetadataMessage clone = new ArbitraryMetadataMessage(this.signature, this.arbitraryMetadataFile); - clone.setId(newId); - return clone; } } diff --git a/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java b/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java index 1f980b3c..aa75b2a1 100644 --- a/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java +++ b/src/main/java/org/qortal/network/message/ArbitrarySignaturesMessage.java @@ -8,21 +8,37 @@ import org.qortal.utils.Serialization; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; public class ArbitrarySignaturesMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private String peerAddress; private int requestHops; private List signatures; public ArbitrarySignaturesMessage(String peerAddress, int requestHops, List signatures) { - this(-1, peerAddress, requestHops, signatures); + super(MessageType.ARBITRARY_SIGNATURES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + Serialization.serializeSizedStringV2(bytes, peerAddress); + + bytes.write(Ints.toByteArray(requestHops)); + + bytes.write(Ints.toByteArray(signatures.size())); + + for (byte[] signature : signatures) + bytes.write(signature); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private ArbitrarySignaturesMessage(int id, String peerAddress, int requestHops, List signatures) { @@ -41,27 +57,24 @@ public class ArbitrarySignaturesMessage extends Message { return this.signatures; } - public int getRequestHops() { - return this.requestHops; - } - - public void setRequestHops(int requestHops) { - this.requestHops = requestHops; - } - - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException, TransformationException { - String peerAddress = Serialization.deserializeSizedStringV2(bytes, PeerData.MAX_PEER_ADDRESS_SIZE); + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + String peerAddress; + try { + peerAddress = Serialization.deserializeSizedStringV2(bytes, PeerData.MAX_PEER_ADDRESS_SIZE); + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); + } int requestHops = bytes.getInt(); int signatureCount = bytes.getInt(); - if (bytes.remaining() != signatureCount * SIGNATURE_LENGTH) - return null; + if (bytes.remaining() < signatureCount * Transformer.SIGNATURE_LENGTH) + throw new BufferUnderflowException(); List signatures = new ArrayList<>(); for (int i = 0; i < signatureCount; ++i) { - byte[] signature = new byte[SIGNATURE_LENGTH]; + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); signatures.add(signature); } @@ -69,24 +82,4 @@ public class ArbitrarySignaturesMessage extends Message { return new ArbitrarySignaturesMessage(id, peerAddress, requestHops, signatures); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - Serialization.serializeSizedStringV2(bytes, this.peerAddress); - - bytes.write(Ints.toByteArray(this.requestHops)); - - bytes.write(Ints.toByteArray(this.signatures.size())); - - for (byte[] signature : this.signatures) - bytes.write(signature); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/BlockMessage.java b/src/main/java/org/qortal/network/message/BlockMessage.java index b07dc8b1..2dd4db87 100644 --- a/src/main/java/org/qortal/network/message/BlockMessage.java +++ b/src/main/java/org/qortal/network/message/BlockMessage.java @@ -1,14 +1,10 @@ package org.qortal.network.message; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.qortal.block.Block; import org.qortal.data.at.ATStateData; import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; @@ -16,27 +12,15 @@ import org.qortal.transform.TransformationException; import org.qortal.transform.block.BlockTransformer; import org.qortal.utils.Triple; -import com.google.common.primitives.Ints; - public class BlockMessage extends Message { private static final Logger LOGGER = LogManager.getLogger(BlockMessage.class); - private Block block = null; + private final BlockData blockData; + private final List transactions; + private final List atStates; - private BlockData blockData = null; - private List transactions = null; - private List atStates = null; - - private int height; - - public BlockMessage(Block block) { - super(MessageType.BLOCK); - - this.block = block; - this.blockData = block.getBlockData(); - this.height = block.getBlockData().getHeight(); - } + // No public constructor as we're an incoming-only message type. private BlockMessage(int id, BlockData blockData, List transactions, List atStates) { super(id, MessageType.BLOCK); @@ -44,8 +28,6 @@ public class BlockMessage extends Message { this.blockData = blockData; this.transactions = transactions; this.atStates = atStates; - - this.height = blockData.getHeight(); } public BlockData getBlockData() { @@ -60,7 +42,7 @@ public class BlockMessage extends Message { return this.atStates; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { try { int height = byteBuffer.getInt(); @@ -72,32 +54,8 @@ public class BlockMessage extends Message { return new BlockMessage(id, blockData, blockInfo.getB(), blockInfo.getC()); } catch (TransformationException e) { LOGGER.info(String.format("Received garbled BLOCK message: %s", e.getMessage())); - return null; + throw new MessageException(e.getMessage(), e); } } - @Override - protected byte[] toData() { - if (this.block == null) - return null; - - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.height)); - - bytes.write(BlockTransformer.toBytes(this.block)); - - return bytes.toByteArray(); - } catch (TransformationException | IOException e) { - return null; - } - } - - public BlockMessage cloneWithNewId(int newId) { - BlockMessage clone = new BlockMessage(this.block); - clone.setId(newId); - return clone; - } - } diff --git a/src/main/java/org/qortal/network/message/BlockSummariesMessage.java b/src/main/java/org/qortal/network/message/BlockSummariesMessage.java index 6a30608b..513e30ae 100644 --- a/src/main/java/org/qortal/network/message/BlockSummariesMessage.java +++ b/src/main/java/org/qortal/network/message/BlockSummariesMessage.java @@ -2,7 +2,7 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -20,7 +20,25 @@ public class BlockSummariesMessage extends Message { private List blockSummaries; public BlockSummariesMessage(List blockSummaries) { - this(-1, blockSummaries); + super(MessageType.BLOCK_SUMMARIES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(blockSummaries.size())); + + for (BlockSummaryData blockSummary : blockSummaries) { + bytes.write(Ints.toByteArray(blockSummary.getHeight())); + bytes.write(blockSummary.getSignature()); + bytes.write(blockSummary.getMinterPublicKey()); + bytes.write(Ints.toByteArray(blockSummary.getOnlineAccountsCount())); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private BlockSummariesMessage(int id, List blockSummaries) { @@ -33,11 +51,11 @@ public class BlockSummariesMessage extends Message { return this.blockSummaries; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int count = bytes.getInt(); - if (bytes.remaining() != count * BLOCK_SUMMARY_LENGTH) - return null; + if (bytes.remaining() < count * BLOCK_SUMMARY_LENGTH) + throw new BufferUnderflowException(); List blockSummaries = new ArrayList<>(); for (int i = 0; i < count; ++i) { @@ -58,24 +76,4 @@ public class BlockSummariesMessage extends Message { return new BlockSummariesMessage(id, blockSummaries); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.blockSummaries.size())); - - for (BlockSummaryData blockSummary : this.blockSummaries) { - bytes.write(Ints.toByteArray(blockSummary.getHeight())); - bytes.write(blockSummary.getSignature()); - bytes.write(blockSummary.getMinterPublicKey()); - bytes.write(Ints.toByteArray(blockSummary.getOnlineAccountsCount())); - } - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/CachedBlockMessage.java b/src/main/java/org/qortal/network/message/CachedBlockMessage.java index e5029ab0..48e9ef36 100644 --- a/src/main/java/org/qortal/network/message/CachedBlockMessage.java +++ b/src/main/java/org/qortal/network/message/CachedBlockMessage.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import org.qortal.block.Block; @@ -12,59 +11,34 @@ import org.qortal.transform.block.BlockTransformer; import com.google.common.primitives.Ints; // This is an OUTGOING-only Message which more readily lends itself to being cached -public class CachedBlockMessage extends Message { +public class CachedBlockMessage extends Message implements Cloneable { - private Block block = null; - private byte[] cachedBytes = null; - - public CachedBlockMessage(Block block) { + public CachedBlockMessage(Block block) throws TransformationException { super(MessageType.BLOCK); - this.block = block; + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); + + bytes.write(BlockTransformer.toBytes(block)); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } public CachedBlockMessage(byte[] cachedBytes) { super(MessageType.BLOCK); - this.block = null; - this.cachedBytes = cachedBytes; + this.dataBytes = cachedBytes; + this.checksumBytes = Message.generateChecksum(this.dataBytes); } - - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) { throw new UnsupportedOperationException("CachedBlockMessage is for outgoing messages only"); } - @Override - protected byte[] toData() { - // Already serialized? - if (this.cachedBytes != null) - return cachedBytes; - - if (this.block == null) - return null; - - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.block.getBlockData().getHeight())); - - bytes.write(BlockTransformer.toBytes(this.block)); - - this.cachedBytes = bytes.toByteArray(); - // We no longer need source Block - // and Block contains repository handle which is highly likely to be invalid after this call - this.block = null; - - return this.cachedBytes; - } catch (TransformationException | IOException e) { - return null; - } - } - - public CachedBlockMessage cloneWithNewId(int newId) { - CachedBlockMessage clone = new CachedBlockMessage(this.cachedBytes); - clone.setId(newId); - return clone; - } - } diff --git a/src/main/java/org/qortal/network/message/ChallengeMessage.java b/src/main/java/org/qortal/network/message/ChallengeMessage.java index 425f9790..bb5b2ae9 100644 --- a/src/main/java/org/qortal/network/message/ChallengeMessage.java +++ b/src/main/java/org/qortal/network/message/ChallengeMessage.java @@ -10,8 +10,25 @@ public class ChallengeMessage extends Message { public static final int CHALLENGE_LENGTH = 32; - private final byte[] publicKey; - private final byte[] challenge; + private byte[] publicKey; + private byte[] challenge; + + public ChallengeMessage(byte[] publicKey, byte[] challenge) { + super(MessageType.CHALLENGE); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(publicKey.length + challenge.length); + + try { + bytes.write(publicKey); + + bytes.write(challenge); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } private ChallengeMessage(int id, byte[] publicKey, byte[] challenge) { super(id, MessageType.CHALLENGE); @@ -20,10 +37,6 @@ public class ChallengeMessage extends Message { this.challenge = challenge; } - public ChallengeMessage(byte[] publicKey, byte[] challenge) { - this(-1, publicKey, challenge); - } - public byte[] getPublicKey() { return this.publicKey; } @@ -42,15 +55,4 @@ public class ChallengeMessage extends Message { return new ChallengeMessage(id, publicKey, challenge); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.publicKey); - - bytes.write(this.challenge); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java index 542854a5..467a229f 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileListMessage.java @@ -5,33 +5,54 @@ import com.google.common.primitives.Longs; import org.qortal.data.network.PeerData; import org.qortal.transform.TransformationException; import org.qortal.transform.Transformer; -import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.utils.Serialization; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import static org.qortal.transform.Transformer.INT_LENGTH; -import static org.qortal.transform.Transformer.LONG_LENGTH; - public class GetArbitraryDataFileListMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; - private static final int MAX_PEER_ADDRESS_LENGTH = PeerData.MAX_PEER_ADDRESS_SIZE; - - private final byte[] signature; + private byte[] signature; private List hashes; - private final long requestTime; + private long requestTime; private int requestHops; private String requestingPeer; public GetArbitraryDataFileListMessage(byte[] signature, List hashes, long requestTime, int requestHops, String requestingPeer) { - this(-1, signature, hashes, requestTime, requestHops, requestingPeer); + super(MessageType.GET_ARBITRARY_DATA_FILE_LIST); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Longs.toByteArray(requestTime)); + + bytes.write(Ints.toByteArray(requestHops)); + + if (hashes != null) { + bytes.write(Ints.toByteArray(hashes.size())); + + for (byte[] hash : hashes) { + bytes.write(hash); + } + } + else { + bytes.write(Ints.toByteArray(0)); + } + + if (requestingPeer != null) { + Serialization.serializeSizedStringV2(bytes, requestingPeer); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetArbitraryDataFileListMessage(int id, byte[] signature, List hashes, long requestTime, int requestHops, String requestingPeer) { @@ -52,8 +73,20 @@ public class GetArbitraryDataFileListMessage extends Message { return this.hashes; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException, TransformationException { - byte[] signature = new byte[SIGNATURE_LENGTH]; + public long getRequestTime() { + return this.requestTime; + } + + public int getRequestHops() { + return this.requestHops; + } + + public String getRequestingPeer() { + return this.requestingPeer; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); @@ -67,7 +100,7 @@ public class GetArbitraryDataFileListMessage extends Message { hashes = new ArrayList<>(); for (int i = 0; i < hashCount; ++i) { - byte[] hash = new byte[HASH_LENGTH]; + byte[] hash = new byte[Transformer.SHA256_LENGTH]; bytes.get(hash); hashes.add(hash); } @@ -75,57 +108,14 @@ public class GetArbitraryDataFileListMessage extends Message { String requestingPeer = null; if (bytes.hasRemaining()) { - requestingPeer = Serialization.deserializeSizedStringV2(bytes, MAX_PEER_ADDRESS_LENGTH); + try { + requestingPeer = Serialization.deserializeSizedStringV2(bytes, PeerData.MAX_PEER_ADDRESS_SIZE); + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); + } } return new GetArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, requestingPeer); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - bytes.write(Longs.toByteArray(this.requestTime)); - - bytes.write(Ints.toByteArray(this.requestHops)); - - if (this.hashes != null) { - bytes.write(Ints.toByteArray(this.hashes.size())); - - for (byte[] hash : this.hashes) { - bytes.write(hash); - } - } - else { - bytes.write(Ints.toByteArray(0)); - } - - if (this.requestingPeer != null) { - Serialization.serializeSizedStringV2(bytes, this.requestingPeer); - } - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - - public long getRequestTime() { - return this.requestTime; - } - - public int getRequestHops() { - return this.requestHops; - } - public void setRequestHops(int requestHops) { - this.requestHops = requestHops; - } - - public String getRequestingPeer() { - return this.requestingPeer; - } - } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java index 809b983d..d97a4847 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataFileMessage.java @@ -1,23 +1,31 @@ package org.qortal.network.message; import org.qortal.transform.Transformer; -import org.qortal.transform.transaction.TransactionTransformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; public class GetArbitraryDataFileMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH; - - private final byte[] signature; - private final byte[] hash; + private byte[] signature; + private byte[] hash; public GetArbitraryDataFileMessage(byte[] signature, byte[] hash) { - this(-1, signature, hash); + super(MessageType.GET_ARBITRARY_DATA_FILE); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(signature.length + hash.length); + + try { + bytes.write(signature); + + bytes.write(hash); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetArbitraryDataFileMessage(int id, byte[] signature, byte[] hash) { @@ -35,32 +43,14 @@ public class GetArbitraryDataFileMessage extends Message { return this.hash; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != HASH_LENGTH + SIGNATURE_LENGTH) - return null; - - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); - byte[] hash = new byte[HASH_LENGTH]; + byte[] hash = new byte[Transformer.SHA256_LENGTH]; bytes.get(hash); return new GetArbitraryDataFileMessage(id, signature, hash); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - bytes.write(this.hash); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java index 689d704b..bf604fe7 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryDataMessage.java @@ -1,20 +1,19 @@ package org.qortal.network.message; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.util.Arrays; import org.qortal.transform.Transformer; public class GetArbitraryDataMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private byte[] signature; public GetArbitraryDataMessage(byte[] signature) { - this(-1, signature); + super(MessageType.GET_ARBITRARY_DATA); + + this.dataBytes = Arrays.copyOf(signature, signature.length); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetArbitraryDataMessage(int id, byte[] signature) { @@ -27,28 +26,12 @@ public class GetArbitraryDataMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != SIGNATURE_LENGTH) - return null; - - byte[] signature = new byte[SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); return new GetArbitraryDataMessage(id, signature); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java b/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java index 66c8f86c..2501d5c3 100644 --- a/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java +++ b/src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java @@ -6,22 +6,31 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; -import static org.qortal.transform.Transformer.INT_LENGTH; -import static org.qortal.transform.Transformer.LONG_LENGTH; - public class GetArbitraryMetadataMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - - private final byte[] signature; - private final long requestTime; + private byte[] signature; + private long requestTime; private int requestHops; public GetArbitraryMetadataMessage(byte[] signature, long requestTime, int requestHops) { - this(-1, signature, requestTime, requestHops); + super(MessageType.GET_ARBITRARY_METADATA); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(signature); + + bytes.write(Longs.toByteArray(requestTime)); + + bytes.write(Ints.toByteArray(requestHops)); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetArbitraryMetadataMessage(int id, byte[] signature, long requestTime, int requestHops) { @@ -36,12 +45,16 @@ public class GetArbitraryMetadataMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != SIGNATURE_LENGTH + LONG_LENGTH + INT_LENGTH) - return null; + public long getRequestTime() { + return this.requestTime; + } - byte[] signature = new byte[SIGNATURE_LENGTH]; + public int getRequestHops() { + return this.requestHops; + } + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); long requestTime = bytes.getLong(); @@ -51,33 +64,4 @@ public class GetArbitraryMetadataMessage extends Message { return new GetArbitraryMetadataMessage(id, signature, requestTime, requestHops); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - bytes.write(Longs.toByteArray(this.requestTime)); - - bytes.write(Ints.toByteArray(this.requestHops)); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - - public long getRequestTime() { - return this.requestTime; - } - - public int getRequestHops() { - return this.requestHops; - } - - public void setRequestHops(int requestHops) { - this.requestHops = requestHops; - } - } diff --git a/src/main/java/org/qortal/network/message/GetBlockMessage.java b/src/main/java/org/qortal/network/message/GetBlockMessage.java index 43484e69..d39dcca0 100644 --- a/src/main/java/org/qortal/network/message/GetBlockMessage.java +++ b/src/main/java/org/qortal/network/message/GetBlockMessage.java @@ -1,20 +1,19 @@ package org.qortal.network.message; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.util.Arrays; import org.qortal.transform.block.BlockTransformer; public class GetBlockMessage extends Message { - private static final int BLOCK_SIGNATURE_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH; - private byte[] signature; public GetBlockMessage(byte[] signature) { - this(-1, signature); + super(MessageType.GET_BLOCK); + + this.dataBytes = Arrays.copyOf(signature, signature.length); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetBlockMessage(int id, byte[] signature) { @@ -27,28 +26,11 @@ public class GetBlockMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != BLOCK_SIGNATURE_LENGTH) - return null; - - byte[] signature = new byte[BLOCK_SIGNATURE_LENGTH]; - + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] signature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; bytes.get(signature); return new GetBlockMessage(id, signature); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java b/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java index 148640fd..70f0d5c5 100644 --- a/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java +++ b/src/main/java/org/qortal/network/message/GetBlockSummariesMessage.java @@ -2,23 +2,32 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; -import org.qortal.transform.Transformer; import org.qortal.transform.block.BlockTransformer; import com.google.common.primitives.Ints; public class GetBlockSummariesMessage extends Message { - private static final int BLOCK_SIGNATURE_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH; - private byte[] parentSignature; private int numberRequested; public GetBlockSummariesMessage(byte[] parentSignature, int numberRequested) { - this(-1, parentSignature, numberRequested); + super(MessageType.GET_BLOCK_SUMMARIES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(parentSignature); + + bytes.write(Ints.toByteArray(numberRequested)); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetBlockSummariesMessage(int id, byte[] parentSignature, int numberRequested) { @@ -36,11 +45,8 @@ public class GetBlockSummariesMessage extends Message { return this.numberRequested; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != BLOCK_SIGNATURE_LENGTH + Transformer.INT_LENGTH) - return null; - - byte[] parentSignature = new byte[BLOCK_SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] parentSignature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; bytes.get(parentSignature); int numberRequested = bytes.getInt(); @@ -48,19 +54,4 @@ public class GetBlockSummariesMessage extends Message { return new GetBlockSummariesMessage(id, parentSignature, numberRequested); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.parentSignature); - - bytes.write(Ints.toByteArray(this.numberRequested)); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java index 23c21bc5..ae98cf40 100644 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -20,7 +19,24 @@ public class GetOnlineAccountsMessage extends Message { private List onlineAccounts; public GetOnlineAccountsMessage(List onlineAccounts) { - this(-1, onlineAccounts); + super(MessageType.GET_ONLINE_ACCOUNTS); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(onlineAccounts.size())); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); + + bytes.write(onlineAccountData.getPublicKey()); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetOnlineAccountsMessage(int id, List onlineAccounts) { @@ -33,7 +49,7 @@ public class GetOnlineAccountsMessage extends Message { return this.onlineAccounts; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { final int accountCount = bytes.getInt(); List onlineAccounts = new ArrayList<>(accountCount); @@ -50,24 +66,4 @@ public class GetOnlineAccountsMessage extends Message { return new GetOnlineAccountsMessage(id, onlineAccounts); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.onlineAccounts.size())); - - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); - bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - - bytes.write(onlineAccountData.getPublicKey()); - } - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java index 709f9782..fe6b5d72 100644 --- a/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java +++ b/src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java @@ -7,7 +7,6 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -24,11 +23,51 @@ import java.util.Map; * Also V2 only builds online accounts message once! */ public class GetOnlineAccountsV2Message extends Message { + private List onlineAccounts; - private byte[] cachedData; public GetOnlineAccountsV2Message(List onlineAccounts) { - this(-1, onlineAccounts); + super(MessageType.GET_ONLINE_ACCOUNTS_V2); + + // If we don't have ANY online accounts then it's an easier construction... + if (onlineAccounts.isEmpty()) { + // Always supply a number of accounts + this.dataBytes = Ints.toByteArray(0); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + return; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH; + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + try { + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + if (onlineAccountData.getTimestamp() == timestamp) + bytes.write(onlineAccountData.getPublicKey()); + } + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetOnlineAccountsV2Message(int id, List onlineAccounts) { @@ -41,7 +80,7 @@ public class GetOnlineAccountsV2Message extends Message { return this.onlineAccounts; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int accountCount = bytes.getInt(); List onlineAccounts = new ArrayList<>(accountCount); @@ -67,51 +106,4 @@ public class GetOnlineAccountsV2Message extends Message { return new GetOnlineAccountsV2Message(id, onlineAccounts); } - @Override - protected synchronized byte[] toData() { - if (this.cachedData != null) - return this.cachedData; - - // Shortcut in case we have no online accounts - if (this.onlineAccounts.isEmpty()) { - this.cachedData = Ints.toByteArray(0); - return this.cachedData; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); - Long timestamp = onlineAccountData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + this.onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH; - - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); - - if (onlineAccountData.getTimestamp() == timestamp) - bytes.write(onlineAccountData.getPublicKey()); - } - } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/GetPeersMessage.java b/src/main/java/org/qortal/network/message/GetPeersMessage.java index 21b06df5..b8f7e128 100644 --- a/src/main/java/org/qortal/network/message/GetPeersMessage.java +++ b/src/main/java/org/qortal/network/message/GetPeersMessage.java @@ -1,25 +1,21 @@ package org.qortal.network.message; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; public class GetPeersMessage extends Message { public GetPeersMessage() { - this(-1); + super(MessageType.GET_PEERS); + + this.dataBytes = EMPTY_DATA_BYTES; } private GetPeersMessage(int id) { super(id, MessageType.GET_PEERS); } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { return new GetPeersMessage(id); } - @Override - protected byte[] toData() { - return new byte[0]; - } - } diff --git a/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java b/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java index 2dc54365..0f88ba7d 100644 --- a/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java +++ b/src/main/java/org/qortal/network/message/GetSignaturesV2Message.java @@ -2,24 +2,32 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; -import org.qortal.transform.Transformer; import org.qortal.transform.block.BlockTransformer; import com.google.common.primitives.Ints; public class GetSignaturesV2Message extends Message { - private static final int BLOCK_SIGNATURE_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH; - private static final int NUMBER_REQUESTED_LENGTH = Transformer.INT_LENGTH; - private byte[] parentSignature; private int numberRequested; public GetSignaturesV2Message(byte[] parentSignature, int numberRequested) { - this(-1, parentSignature, numberRequested); + super(MessageType.GET_SIGNATURES_V2); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(parentSignature); + + bytes.write(Ints.toByteArray(numberRequested)); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetSignaturesV2Message(int id, byte[] parentSignature, int numberRequested) { @@ -37,11 +45,8 @@ public class GetSignaturesV2Message extends Message { return this.numberRequested; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != BLOCK_SIGNATURE_LENGTH + NUMBER_REQUESTED_LENGTH) - return null; - - byte[] parentSignature = new byte[BLOCK_SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] parentSignature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; bytes.get(parentSignature); int numberRequested = bytes.getInt(); @@ -49,19 +54,4 @@ public class GetSignaturesV2Message extends Message { return new GetSignaturesV2Message(id, parentSignature, numberRequested); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.parentSignature); - - bytes.write(Ints.toByteArray(this.numberRequested)); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java b/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java index d9be3c1b..7246c424 100644 --- a/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java +++ b/src/main/java/org/qortal/network/message/GetTradePresencesMessage.java @@ -7,7 +7,6 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -21,10 +20,48 @@ import java.util.Map; */ public class GetTradePresencesMessage extends Message { private List tradePresences; - private byte[] cachedData; public GetTradePresencesMessage(List tradePresences) { - this(-1, tradePresences); + super(MessageType.GET_TRADE_PRESENCES); + + // Shortcut in case we have no trade presences + if (tradePresences.isEmpty()) { + this.dataBytes = Ints.toByteArray(0); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + return; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (TradePresenceData tradePresenceData : tradePresences) { + Long timestamp = tradePresenceData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + tradePresences.size() * Transformer.PUBLIC_KEY_LENGTH; + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + try { + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (TradePresenceData tradePresenceData : tradePresences) { + if (tradePresenceData.getTimestamp() == timestamp) + bytes.write(tradePresenceData.getPublicKey()); + } + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetTradePresencesMessage(int id, List tradePresences) { @@ -37,7 +74,7 @@ public class GetTradePresencesMessage extends Message { return this.tradePresences; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int groupedEntriesCount = bytes.getInt(); List tradePresences = new ArrayList<>(groupedEntriesCount); @@ -63,48 +100,4 @@ public class GetTradePresencesMessage extends Message { return new GetTradePresencesMessage(id, tradePresences); } - @Override - protected synchronized byte[] toData() { - if (this.cachedData != null) - return this.cachedData; - - // Shortcut in case we have no trade presences - if (this.tradePresences.isEmpty()) { - this.cachedData = Ints.toByteArray(0); - return this.cachedData; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (TradePresenceData tradePresenceData : this.tradePresences) { - Long timestamp = tradePresenceData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + this.tradePresences.size() * Transformer.PUBLIC_KEY_LENGTH; - - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (TradePresenceData tradePresenceData : this.tradePresences) { - if (tradePresenceData.getTimestamp() == timestamp) - bytes.write(tradePresenceData.getPublicKey()); - } - } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/GetTransactionMessage.java b/src/main/java/org/qortal/network/message/GetTransactionMessage.java index 2ea06580..fe0c750f 100644 --- a/src/main/java/org/qortal/network/message/GetTransactionMessage.java +++ b/src/main/java/org/qortal/network/message/GetTransactionMessage.java @@ -1,20 +1,19 @@ package org.qortal.network.message; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; +import java.util.Arrays; import org.qortal.transform.Transformer; public class GetTransactionMessage extends Message { - private static final int TRANSACTION_SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private byte[] signature; public GetTransactionMessage(byte[] signature) { - this(-1, signature); + super(MessageType.GET_TRANSACTION); + + this.dataBytes = Arrays.copyOf(signature, signature.length); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private GetTransactionMessage(int id, byte[] signature) { @@ -27,28 +26,12 @@ public class GetTransactionMessage extends Message { return this.signature; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { - if (bytes.remaining() != TRANSACTION_SIGNATURE_LENGTH) - return null; - - byte[] signature = new byte[TRANSACTION_SIGNATURE_LENGTH]; + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); return new GetTransactionMessage(id, signature); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(this.signature); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java b/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java index 18260568..fccd4c74 100644 --- a/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java +++ b/src/main/java/org/qortal/network/message/GetUnconfirmedTransactionsMessage.java @@ -1,25 +1,21 @@ package org.qortal.network.message; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; public class GetUnconfirmedTransactionsMessage extends Message { public GetUnconfirmedTransactionsMessage() { - this(-1); + super(MessageType.GET_UNCONFIRMED_TRANSACTIONS); + + this.dataBytes = EMPTY_DATA_BYTES; } private GetUnconfirmedTransactionsMessage(int id) { super(id, MessageType.GET_UNCONFIRMED_TRANSACTIONS); } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { return new GetUnconfirmedTransactionsMessage(id); } - @Override - protected byte[] toData() { - return new byte[0]; - } - } diff --git a/src/main/java/org/qortal/network/message/GoodbyeMessage.java b/src/main/java/org/qortal/network/message/GoodbyeMessage.java index 75864060..74130be2 100644 --- a/src/main/java/org/qortal/network/message/GoodbyeMessage.java +++ b/src/main/java/org/qortal/network/message/GoodbyeMessage.java @@ -3,7 +3,6 @@ package org.qortal.network.message; import static java.util.Arrays.stream; import static java.util.stream.Collectors.toMap; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.Map; @@ -22,7 +21,7 @@ public class GoodbyeMessage extends Message { private static final Map map = stream(Reason.values()) .collect(toMap(reason -> reason.value, reason -> reason)); - private Reason(int value) { + Reason(int value) { this.value = value; } @@ -31,7 +30,14 @@ public class GoodbyeMessage extends Message { } } - private final Reason reason; + private Reason reason; + + public GoodbyeMessage(Reason reason) { + super(MessageType.GOODBYE); + + this.dataBytes = Ints.toByteArray(reason.value); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } private GoodbyeMessage(int id, Reason reason) { super(id, MessageType.GOODBYE); @@ -39,27 +45,18 @@ public class GoodbyeMessage extends Message { this.reason = reason; } - public GoodbyeMessage(Reason reason) { - this(-1, reason); - } - public Reason getReason() { return this.reason; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) { + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { int reasonValue = byteBuffer.getInt(); Reason reason = Reason.valueOf(reasonValue); if (reason == null) - return null; + throw new MessageException("Invalid reason " + reasonValue + " in GOODBYE message"); return new GoodbyeMessage(id, reason); } - @Override - protected byte[] toData() throws IOException { - return Ints.toByteArray(this.reason.value); - } - } diff --git a/src/main/java/org/qortal/network/message/HeightV2Message.java b/src/main/java/org/qortal/network/message/HeightV2Message.java index 4d6f3f21..0e775a84 100644 --- a/src/main/java/org/qortal/network/message/HeightV2Message.java +++ b/src/main/java/org/qortal/network/message/HeightV2Message.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import org.qortal.transform.Transformer; @@ -19,7 +18,24 @@ public class HeightV2Message extends Message { private byte[] minterPublicKey; public HeightV2Message(int height, byte[] signature, long timestamp, byte[] minterPublicKey) { - this(-1, height, signature, timestamp, minterPublicKey); + super(MessageType.HEIGHT_V2); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(height)); + + bytes.write(signature); + + bytes.write(Longs.toByteArray(timestamp)); + + bytes.write(minterPublicKey); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private HeightV2Message(int id, int height, byte[] signature, long timestamp, byte[] minterPublicKey) { @@ -47,7 +63,7 @@ public class HeightV2Message extends Message { return this.minterPublicKey; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int height = bytes.getInt(); byte[] signature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; @@ -61,23 +77,4 @@ public class HeightV2Message extends Message { return new HeightV2Message(id, height, signature, timestamp, minterPublicKey); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.height)); - - bytes.write(this.signature); - - bytes.write(Longs.toByteArray(this.timestamp)); - - bytes.write(this.minterPublicKey); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/HelloMessage.java b/src/main/java/org/qortal/network/message/HelloMessage.java index 1b6de17d..30b7d9be 100644 --- a/src/main/java/org/qortal/network/message/HelloMessage.java +++ b/src/main/java/org/qortal/network/message/HelloMessage.java @@ -11,9 +11,28 @@ import com.google.common.primitives.Longs; public class HelloMessage extends Message { - private final long timestamp; - private final String versionString; - private final String senderPeerAddress; + private long timestamp; + private String versionString; + private String senderPeerAddress; + + public HelloMessage(long timestamp, String versionString, String senderPeerAddress) { + super(MessageType.HELLO); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Longs.toByteArray(timestamp)); + + Serialization.serializeSizedString(bytes, versionString); + + Serialization.serializeSizedString(bytes, senderPeerAddress); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } private HelloMessage(int id, long timestamp, String versionString, String senderPeerAddress) { super(id, MessageType.HELLO); @@ -23,10 +42,6 @@ public class HelloMessage extends Message { this.senderPeerAddress = senderPeerAddress; } - public HelloMessage(long timestamp, String versionString, String senderPeerAddress) { - this(-1, timestamp, versionString, senderPeerAddress); - } - public long getTimestamp() { return this.timestamp; } @@ -39,31 +54,23 @@ public class HelloMessage extends Message { return this.senderPeerAddress; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws TransformationException { + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { long timestamp = byteBuffer.getLong(); - String versionString = Serialization.deserializeSizedString(byteBuffer, 255); - - // Sender peer address added in v3.0, so is an optional field. Older versions won't send it. + String versionString; String senderPeerAddress = null; - if (byteBuffer.hasRemaining()) { - senderPeerAddress = Serialization.deserializeSizedString(byteBuffer, 255); + try { + versionString = Serialization.deserializeSizedString(byteBuffer, 255); + + // Sender peer address added in v3.0, so is an optional field. Older versions won't send it. + if (byteBuffer.hasRemaining()) { + senderPeerAddress = Serialization.deserializeSizedString(byteBuffer, 255); + } + } catch (TransformationException e) { + throw new MessageException(e.getMessage(), e); } return new HelloMessage(id, timestamp, versionString, senderPeerAddress); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Longs.toByteArray(this.timestamp)); - - Serialization.serializeSizedString(bytes, this.versionString); - - Serialization.serializeSizedString(bytes, this.senderPeerAddress); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/Message.java b/src/main/java/org/qortal/network/message/Message.java index 747988be..e92aca89 100644 --- a/src/main/java/org/qortal/network/message/Message.java +++ b/src/main/java/org/qortal/network/message/Message.java @@ -1,175 +1,67 @@ package org.qortal.network.message; -import java.util.Map; - import org.qortal.crypto.Crypto; import org.qortal.network.Network; -import org.qortal.transform.TransformationException; import com.google.common.primitives.Ints; -import static java.util.Arrays.stream; -import static java.util.stream.Collectors.toMap; - import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.Arrays; +/** + * Network message for sending over network, or unpacked data received from network. + *

+ *

+ * For messages received from network, subclass's {@code fromByteBuffer()} method is used + * to construct a subclassed instance. Original bytes from network are not retained. + * Access to deserialized data should be via subclass's getters. Ideally there should be NO setters! + *

+ *

+ *

+ * Each subclass's public constructor is for building a message to send only. + * The constructor will serialize into byte form but not store the passed args. + * Serialized bytes are saved into superclass (Message) {@code dataBytes} and, if not empty, + * a checksum is created and saved into {@code checksumBytes}. + * Therefore: do not use subclass's getters after using constructor! + *

+ *

+ *

+ * For subclasses where outgoing versions might be usefully cached, they can implement Clonable + * as long if they are safe to use {@link Object#clone()}. + *

+ */ public abstract class Message { // MAGIC(4) + TYPE(4) + HAS-ID(1) + ID?(4) + DATA-SIZE(4) + CHECKSUM?(4) + DATA?(*) private static final int MAGIC_LENGTH = 4; + private static final int TYPE_LENGTH = 4; + private static final int HAS_ID_LENGTH = 1; + private static final int ID_LENGTH = 4; + private static final int DATA_SIZE_LENGTH = 4; private static final int CHECKSUM_LENGTH = 4; private static final int MAX_DATA_SIZE = 10 * 1024 * 1024; // 10MB - @SuppressWarnings("serial") - public static class MessageException extends Exception { - public MessageException() { - } + protected static final byte[] EMPTY_DATA_BYTES = new byte[0]; - public MessageException(String message) { - super(message); - } + protected int id; + protected final MessageType type; - public MessageException(String message, Throwable cause) { - super(message, cause); - } - - public MessageException(Throwable cause) { - super(cause); - } - } - - public enum MessageType { - // Handshaking - HELLO(0), - GOODBYE(1), - CHALLENGE(2), - RESPONSE(3), - - // Status / notifications - HEIGHT_V2(10), - PING(11), - PONG(12), - - // Requesting data - PEERS_V2(20), - GET_PEERS(21), - - TRANSACTION(30), - GET_TRANSACTION(31), - - TRANSACTION_SIGNATURES(40), - GET_UNCONFIRMED_TRANSACTIONS(41), - - BLOCK(50), - GET_BLOCK(51), - - SIGNATURES(60), - GET_SIGNATURES_V2(61), - - BLOCK_SUMMARIES(70), - GET_BLOCK_SUMMARIES(71), - - ONLINE_ACCOUNTS(80), - GET_ONLINE_ACCOUNTS(81), - ONLINE_ACCOUNTS_V2(82), - GET_ONLINE_ACCOUNTS_V2(83), - - ARBITRARY_DATA(90), - GET_ARBITRARY_DATA(91), - - BLOCKS(100), - GET_BLOCKS(101), - - ARBITRARY_DATA_FILE(110), - GET_ARBITRARY_DATA_FILE(111), - - ARBITRARY_DATA_FILE_LIST(120), - GET_ARBITRARY_DATA_FILE_LIST(121), - - ARBITRARY_SIGNATURES(130), - - TRADE_PRESENCES(140), - GET_TRADE_PRESENCES(141), - - ARBITRARY_METADATA(150), - GET_ARBITRARY_METADATA(151), - - // Lite node support - ACCOUNT(160), - GET_ACCOUNT(161), - - ACCOUNT_BALANCE(170), - GET_ACCOUNT_BALANCE(171), - - NAMES(180), - GET_ACCOUNT_NAMES(181), - GET_NAME(182), - - TRANSACTIONS(190), - GET_ACCOUNT_TRANSACTIONS(191); - - public final int value; - public final Method fromByteBufferMethod; - - private static final Map map = stream(MessageType.values()) - .collect(toMap(messageType -> messageType.value, messageType -> messageType)); - - private MessageType(int value) { - this.value = value; - - String[] classNameParts = this.name().toLowerCase().split("_"); - - for (int i = 0; i < classNameParts.length; ++i) - classNameParts[i] = classNameParts[i].substring(0, 1).toUpperCase().concat(classNameParts[i].substring(1)); - - String className = String.join("", classNameParts); - - Method method; - try { - Class subclass = Class.forName(String.join("", Message.class.getPackage().getName(), ".", className, "Message")); - - method = subclass.getDeclaredMethod("fromByteBuffer", int.class, ByteBuffer.class); - } catch (ClassNotFoundException | NoSuchMethodException | SecurityException e) { - method = null; - } - - this.fromByteBufferMethod = method; - } - - public static MessageType valueOf(int value) { - return map.get(value); - } - - public Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { - if (this.fromByteBufferMethod == null) - throw new MessageException("Unsupported message type [" + value + "] during conversion from bytes"); - - try { - return (Message) this.fromByteBufferMethod.invoke(null, id, byteBuffer); - } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { - if (e.getCause() instanceof BufferUnderflowException) - throw new MessageException("Byte data too short for " + name() + " message"); - - throw new MessageException("Internal error with " + name() + " message during conversion from bytes"); - } - } - } - - private int id; - private MessageType type; + /** Serialized outgoing message data. Expected to be written to by subclass. */ + protected byte[] dataBytes; + /** Serialized outgoing message checksum. Expected to be written to by subclass. */ + protected byte[] checksumBytes; + /** Typically called by subclass when constructing message from received network data. */ protected Message(int id, MessageType type) { this.id = id; this.type = type; } + /** Typically called by subclass when constructing outgoing message. */ protected Message(MessageType type) { this(-1, type); } @@ -193,9 +85,9 @@ public abstract class Message { /** * Attempt to read a message from byte buffer. * - * @param readOnlyBuffer + * @param readOnlyBuffer ByteBuffer containing bytes read from network * @return null if no complete message can be read - * @throws MessageException + * @throws MessageException if message could not be decoded or is invalid */ public static Message fromByteBuffer(ByteBuffer readOnlyBuffer) throws MessageException { try { @@ -270,9 +162,27 @@ public abstract class Message { return Arrays.copyOfRange(Crypto.digest(dataBuffer), 0, CHECKSUM_LENGTH); } + public void checkValidOutgoing() throws MessageException { + // We expect subclass to have initialized these + if (this.dataBytes == null) + throw new MessageException("Missing data payload"); + if (this.dataBytes.length > 0 && this.checksumBytes == null) + throw new MessageException("Missing data checksum"); + } + public byte[] toBytes() throws MessageException { + checkValidOutgoing(); + + // We can calculate exact length + int messageLength = MAGIC_LENGTH + TYPE_LENGTH + HAS_ID_LENGTH; + messageLength += this.hasId() ? ID_LENGTH : 0; + messageLength += DATA_SIZE_LENGTH + this.dataBytes.length > 0 ? CHECKSUM_LENGTH + this.dataBytes.length : 0; + + if (messageLength > MAX_DATA_SIZE) + throw new MessageException(String.format("About to send message with length %d larger than allowed %d", messageLength, MAX_DATA_SIZE)); + try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(256); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(messageLength); // Magic bytes.write(Network.getInstance().getMessageMagic()); @@ -287,26 +197,30 @@ public abstract class Message { bytes.write(0); } - byte[] data = this.toData(); - if (data == null) - throw new MessageException("Missing data payload"); + bytes.write(Ints.toByteArray(this.dataBytes.length)); - bytes.write(Ints.toByteArray(data.length)); - - if (data.length > 0) { - bytes.write(generateChecksum(data)); - bytes.write(data); + if (this.dataBytes.length > 0) { + bytes.write(this.checksumBytes); + bytes.write(this.dataBytes); } - if (bytes.size() > MAX_DATA_SIZE) - throw new MessageException(String.format("About to send message with length %d larger than allowed %d", bytes.size(), MAX_DATA_SIZE)); - return bytes.toByteArray(); - } catch (IOException | TransformationException e) { + } catch (IOException e) { throw new MessageException("Failed to serialize message", e); } } - protected abstract byte[] toData() throws IOException, TransformationException; + public static M cloneWithNewId(M message, int newId) { + M clone; + + try { + clone = (M) message.clone(); + } catch (CloneNotSupportedException e) { + throw new UnsupportedOperationException("Message sub-class not cloneable"); + } + + clone.setId(newId); + return clone; + } } diff --git a/src/main/java/org/qortal/network/message/MessageException.java b/src/main/java/org/qortal/network/message/MessageException.java new file mode 100644 index 00000000..97e8d0be --- /dev/null +++ b/src/main/java/org/qortal/network/message/MessageException.java @@ -0,0 +1,19 @@ +package org.qortal.network.message; + +@SuppressWarnings("serial") +public class MessageException extends Exception { + public MessageException() { + } + + public MessageException(String message) { + super(message); + } + + public MessageException(String message, Throwable cause) { + super(message, cause); + } + + public MessageException(Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/org/qortal/network/message/MessageProducer.java b/src/main/java/org/qortal/network/message/MessageProducer.java new file mode 100644 index 00000000..7f203788 --- /dev/null +++ b/src/main/java/org/qortal/network/message/MessageProducer.java @@ -0,0 +1,8 @@ +package org.qortal.network.message; + +import java.nio.ByteBuffer; + +@FunctionalInterface +public interface MessageProducer { + Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException; +} diff --git a/src/main/java/org/qortal/network/message/MessageType.java b/src/main/java/org/qortal/network/message/MessageType.java new file mode 100644 index 00000000..48039a4d --- /dev/null +++ b/src/main/java/org/qortal/network/message/MessageType.java @@ -0,0 +1,96 @@ +package org.qortal.network.message; + +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import java.util.Map; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toMap; + +public enum MessageType { + // Handshaking + HELLO(0, HelloMessage::fromByteBuffer), + GOODBYE(1, GoodbyeMessage::fromByteBuffer), + CHALLENGE(2, ChallengeMessage::fromByteBuffer), + RESPONSE(3, ResponseMessage::fromByteBuffer), + + // Status / notifications + HEIGHT_V2(10, HeightV2Message::fromByteBuffer), + PING(11, PingMessage::fromByteBuffer), + PONG(12, PongMessage::fromByteBuffer), + + // Requesting data + PEERS_V2(20, PeersV2Message::fromByteBuffer), + GET_PEERS(21, GetPeersMessage::fromByteBuffer), + + TRANSACTION(30, TransactionMessage::fromByteBuffer), + GET_TRANSACTION(31, GetTransactionMessage::fromByteBuffer), + + TRANSACTION_SIGNATURES(40, TransactionSignaturesMessage::fromByteBuffer), + GET_UNCONFIRMED_TRANSACTIONS(41, GetUnconfirmedTransactionsMessage::fromByteBuffer), + + BLOCK(50, BlockMessage::fromByteBuffer), + GET_BLOCK(51, GetBlockMessage::fromByteBuffer), + + SIGNATURES(60, SignaturesMessage::fromByteBuffer), + GET_SIGNATURES_V2(61, GetSignaturesV2Message::fromByteBuffer), + + BLOCK_SUMMARIES(70, BlockSummariesMessage::fromByteBuffer), + GET_BLOCK_SUMMARIES(71, GetBlockSummariesMessage::fromByteBuffer), + + ONLINE_ACCOUNTS(80, OnlineAccountsMessage::fromByteBuffer), + GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer), + ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer), + GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer), + + ARBITRARY_DATA(90, ArbitraryDataMessage::fromByteBuffer), + GET_ARBITRARY_DATA(91, GetArbitraryDataMessage::fromByteBuffer), + + BLOCKS(100, null), // unsupported + GET_BLOCKS(101, null), // unsupported + + ARBITRARY_DATA_FILE(110, ArbitraryDataFileMessage::fromByteBuffer), + GET_ARBITRARY_DATA_FILE(111, GetArbitraryDataFileMessage::fromByteBuffer), + + ARBITRARY_DATA_FILE_LIST(120, ArbitraryDataFileListMessage::fromByteBuffer), + GET_ARBITRARY_DATA_FILE_LIST(121, GetArbitraryDataFileListMessage::fromByteBuffer), + + ARBITRARY_SIGNATURES(130, ArbitrarySignaturesMessage::fromByteBuffer), + + TRADE_PRESENCES(140, TradePresencesMessage::fromByteBuffer), + GET_TRADE_PRESENCES(141, GetTradePresencesMessage::fromByteBuffer), + + ARBITRARY_METADATA(150, ArbitraryMetadataMessage::fromByteBuffer), + GET_ARBITRARY_METADATA(151, GetArbitraryMetadataMessage::fromByteBuffer); + + public final int value; + public final MessageProducer fromByteBufferMethod; + + private static final Map map = stream(MessageType.values()) + .collect(toMap(messageType -> messageType.value, messageType -> messageType)); + + MessageType(int value, MessageProducer fromByteBufferMethod) { + this.value = value; + this.fromByteBufferMethod = fromByteBufferMethod; + } + + public static MessageType valueOf(int value) { + return map.get(value); + } + + /** + * Attempt to read a message from byte buffer. + * + * @param id message ID or -1 + * @param byteBuffer ByteBuffer source for message + * @return null if no complete message can be read + * @throws MessageException if message could not be decoded or is invalid + * @throws BufferUnderflowException if not enough bytes in buffer to read message + */ + public Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { + if (this.fromByteBufferMethod == null) + throw new MessageException("Message type " + this.name() + " unsupported"); + + return this.fromByteBufferMethod.fromByteBuffer(id, byteBuffer); + } +} diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java b/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java index 02c46717..e7e4c32c 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsMessage.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -20,7 +19,26 @@ public class OnlineAccountsMessage extends Message { private List onlineAccounts; public OnlineAccountsMessage(List onlineAccounts) { - this(-1, onlineAccounts); + super(MessageType.ONLINE_ACCOUNTS); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(onlineAccounts.size())); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); + + bytes.write(onlineAccountData.getSignature()); + + bytes.write(onlineAccountData.getPublicKey()); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private OnlineAccountsMessage(int id, List onlineAccounts) { @@ -33,7 +51,7 @@ public class OnlineAccountsMessage extends Message { return this.onlineAccounts; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { final int accountCount = bytes.getInt(); List onlineAccounts = new ArrayList<>(accountCount); @@ -54,27 +72,4 @@ public class OnlineAccountsMessage extends Message { return new OnlineAccountsMessage(id, onlineAccounts); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.onlineAccounts.size())); - - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); - - bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp())); - - bytes.write(onlineAccountData.getSignature()); - - bytes.write(onlineAccountData.getPublicKey()); - } - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java index f0fce81e..6803e3bf 100644 --- a/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java +++ b/src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java @@ -7,13 +7,11 @@ import org.qortal.transform.Transformer; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; /** * For sending online accounts info to remote peer. @@ -25,11 +23,52 @@ import java.util.stream.Collectors; * Also V2 only builds online accounts message once! */ public class OnlineAccountsV2Message extends Message { + private List onlineAccounts; - private byte[] cachedData; public OnlineAccountsV2Message(List onlineAccounts) { - this(-1, onlineAccounts); + super(MessageType.ONLINE_ACCOUNTS_V2); + + // Shortcut in case we have no online accounts + if (onlineAccounts.isEmpty()) { + this.dataBytes = Ints.toByteArray(0); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + return; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + Long timestamp = onlineAccountData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + try { + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (OnlineAccountData onlineAccountData : onlineAccounts) { + if (onlineAccountData.getTimestamp() == timestamp) { + bytes.write(onlineAccountData.getSignature()); + bytes.write(onlineAccountData.getPublicKey()); + } + } + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private OnlineAccountsV2Message(int id, List onlineAccounts) { @@ -42,7 +81,7 @@ public class OnlineAccountsV2Message extends Message { return this.onlineAccounts; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException { int accountCount = bytes.getInt(); List onlineAccounts = new ArrayList<>(accountCount); @@ -71,54 +110,4 @@ public class OnlineAccountsV2Message extends Message { return new OnlineAccountsV2Message(id, onlineAccounts); } - @Override - protected synchronized byte[] toData() { - if (this.cachedData != null) - return this.cachedData; - - // Shortcut in case we have no online accounts - if (this.onlineAccounts.isEmpty()) { - this.cachedData = Ints.toByteArray(0); - return this.cachedData; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); - Long timestamp = onlineAccountData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + this.onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH); - - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (int i = 0; i < this.onlineAccounts.size(); ++i) { - OnlineAccountData onlineAccountData = this.onlineAccounts.get(i); - - if (onlineAccountData.getTimestamp() == timestamp) { - bytes.write(onlineAccountData.getSignature()); - - bytes.write(onlineAccountData.getPublicKey()); - } - } - } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/PeersV2Message.java b/src/main/java/org/qortal/network/message/PeersV2Message.java index bfea87c7..e844246f 100644 --- a/src/main/java/org/qortal/network/message/PeersV2Message.java +++ b/src/main/java/org/qortal/network/message/PeersV2Message.java @@ -2,7 +2,6 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -19,7 +18,35 @@ public class PeersV2Message extends Message { private List peerAddresses; public PeersV2Message(List peerAddresses) { - this(-1, peerAddresses); + super(MessageType.PEERS_V2); + + List addresses = new ArrayList<>(); + + // First entry represents sending node but contains only port number with empty address. + addresses.add(("0.0.0.0:" + Settings.getInstance().getListenPort()).getBytes(StandardCharsets.UTF_8)); + + for (PeerAddress peerAddress : peerAddresses) + addresses.add(peerAddress.toString().getBytes(StandardCharsets.UTF_8)); + + // We can't send addresses that are longer than 255 bytes as length itself is encoded in one byte. + addresses.removeIf(addressString -> addressString.length > 255); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + // Number of entries + bytes.write(Ints.toByteArray(addresses.size())); + + for (byte[] address : addresses) { + bytes.write(address.length); + bytes.write(address); + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private PeersV2Message(int id, List peerAddresses) { @@ -32,7 +59,7 @@ public class PeersV2Message extends Message { return this.peerAddresses; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { // Read entry count int count = byteBuffer.getInt(); @@ -49,43 +76,11 @@ public class PeersV2Message extends Message { PeerAddress peerAddress = PeerAddress.fromString(addressString); peerAddresses.add(peerAddress); } catch (IllegalArgumentException e) { - // Not valid - ignore + throw new MessageException("Invalid peer address in received PEERS_V2 message"); } } return new PeersV2Message(id, peerAddresses); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - List addresses = new ArrayList<>(); - - // First entry represents sending node but contains only port number with empty address. - addresses.add(("0.0.0.0:" + Settings.getInstance().getListenPort()).getBytes(StandardCharsets.UTF_8)); - - for (PeerAddress peerAddress : this.peerAddresses) - addresses.add(peerAddress.toString().getBytes(StandardCharsets.UTF_8)); - - // We can't send addresses that are longer than 255 bytes as length itself is encoded in one byte. - addresses.removeIf(addressString -> addressString.length > 255); - - // Serialize - - // Number of entries - bytes.write(Ints.toByteArray(addresses.size())); - - for (byte[] address : addresses) { - bytes.write(address.length); - bytes.write(address); - } - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/PingMessage.java b/src/main/java/org/qortal/network/message/PingMessage.java index ddec0fd7..0b66d507 100644 --- a/src/main/java/org/qortal/network/message/PingMessage.java +++ b/src/main/java/org/qortal/network/message/PingMessage.java @@ -1,25 +1,21 @@ package org.qortal.network.message; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; public class PingMessage extends Message { public PingMessage() { - this(-1); + super(MessageType.PING); + + this.dataBytes = EMPTY_DATA_BYTES; } private PingMessage(int id) { super(id, MessageType.PING); } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { return new PingMessage(id); } - @Override - protected byte[] toData() { - return new byte[0]; - } - } diff --git a/src/main/java/org/qortal/network/message/PongMessage.java b/src/main/java/org/qortal/network/message/PongMessage.java new file mode 100644 index 00000000..4e73c07c --- /dev/null +++ b/src/main/java/org/qortal/network/message/PongMessage.java @@ -0,0 +1,21 @@ +package org.qortal.network.message; + +import java.nio.ByteBuffer; + +public class PongMessage extends Message { + + public PongMessage() { + super(MessageType.PONG); + + this.dataBytes = EMPTY_DATA_BYTES; + } + + private PongMessage(int id) { + super(id, MessageType.PONG); + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) { + return new PongMessage(id); + } + +} diff --git a/src/main/java/org/qortal/network/message/ResponseMessage.java b/src/main/java/org/qortal/network/message/ResponseMessage.java index 6fed6d6a..292fe697 100644 --- a/src/main/java/org/qortal/network/message/ResponseMessage.java +++ b/src/main/java/org/qortal/network/message/ResponseMessage.java @@ -10,8 +10,25 @@ public class ResponseMessage extends Message { public static final int DATA_LENGTH = 32; - private final int nonce; - private final byte[] data; + private int nonce; + private byte[] data; + + public ResponseMessage(int nonce, byte[] data) { + super(MessageType.RESPONSE); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(4 + DATA_LENGTH); + + try { + bytes.write(Ints.toByteArray(nonce)); + + bytes.write(data); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } private ResponseMessage(int id, int nonce, byte[] data) { super(id, MessageType.RESPONSE); @@ -20,10 +37,6 @@ public class ResponseMessage extends Message { this.data = data; } - public ResponseMessage(int nonce, byte[] data) { - this(-1, nonce, data); - } - public int getNonce() { return this.nonce; } @@ -41,15 +54,4 @@ public class ResponseMessage extends Message { return new ResponseMessage(id, nonce, data); } - @Override - protected byte[] toData() throws IOException { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(4 + DATA_LENGTH); - - bytes.write(Ints.toByteArray(this.nonce)); - - bytes.write(data); - - return bytes.toByteArray(); - } - } diff --git a/src/main/java/org/qortal/network/message/SignaturesMessage.java b/src/main/java/org/qortal/network/message/SignaturesMessage.java index 008f4c1a..c0b44fcd 100644 --- a/src/main/java/org/qortal/network/message/SignaturesMessage.java +++ b/src/main/java/org/qortal/network/message/SignaturesMessage.java @@ -2,7 +2,7 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -13,12 +13,24 @@ import com.google.common.primitives.Ints; public class SignaturesMessage extends Message { - private static final int BLOCK_SIGNATURE_LENGTH = BlockTransformer.BLOCK_SIGNATURE_LENGTH; - private List signatures; public SignaturesMessage(List signatures) { - this(-1, signatures); + super(MessageType.SIGNATURES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(signatures.size())); + + for (byte[] signature : signatures) + bytes.write(signature); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private SignaturesMessage(int id, List signatures) { @@ -31,15 +43,15 @@ public class SignaturesMessage extends Message { return this.signatures; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int count = bytes.getInt(); - if (bytes.remaining() != count * BLOCK_SIGNATURE_LENGTH) - return null; + if (bytes.remaining() < count * BlockTransformer.BLOCK_SIGNATURE_LENGTH) + throw new BufferUnderflowException(); List signatures = new ArrayList<>(); for (int i = 0; i < count; ++i) { - byte[] signature = new byte[BLOCK_SIGNATURE_LENGTH]; + byte[] signature = new byte[BlockTransformer.BLOCK_SIGNATURE_LENGTH]; bytes.get(signature); signatures.add(signature); } @@ -47,20 +59,4 @@ public class SignaturesMessage extends Message { return new SignaturesMessage(id, signatures); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.signatures.size())); - - for (byte[] signature : this.signatures) - bytes.write(signature); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/TradePresencesMessage.java b/src/main/java/org/qortal/network/message/TradePresencesMessage.java index 9d846722..8d7da156 100644 --- a/src/main/java/org/qortal/network/message/TradePresencesMessage.java +++ b/src/main/java/org/qortal/network/message/TradePresencesMessage.java @@ -8,7 +8,6 @@ import org.qortal.utils.Base58; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashMap; @@ -21,11 +20,55 @@ import java.util.Map; * Groups of: number of entries, timestamp, then pubkey + sig + AT address for each entry. */ public class TradePresencesMessage extends Message { + private List tradePresences; - private byte[] cachedData; public TradePresencesMessage(List tradePresences) { - this(-1, tradePresences); + super(MessageType.TRADE_PRESENCES); + + // Shortcut in case we have no trade presences + if (tradePresences.isEmpty()) { + this.dataBytes = Ints.toByteArray(0); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + return; + } + + // How many of each timestamp + Map countByTimestamp = new HashMap<>(); + + for (TradePresenceData tradePresenceData : tradePresences) { + Long timestamp = tradePresenceData.getTimestamp(); + countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); + } + + // We should know exactly how many bytes to allocate now + int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) + + tradePresences.size() * (Transformer.PUBLIC_KEY_LENGTH + Transformer.SIGNATURE_LENGTH + Transformer.ADDRESS_LENGTH); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); + + try { + for (long timestamp : countByTimestamp.keySet()) { + bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); + + bytes.write(Longs.toByteArray(timestamp)); + + for (TradePresenceData tradePresenceData : tradePresences) { + if (tradePresenceData.getTimestamp() == timestamp) { + bytes.write(tradePresenceData.getPublicKey()); + + bytes.write(tradePresenceData.getSignature()); + + bytes.write(Base58.decode(tradePresenceData.getAtAddress())); + } + } + } + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private TradePresencesMessage(int id, List tradePresences) { @@ -38,7 +81,7 @@ public class TradePresencesMessage extends Message { return this.tradePresences; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int groupedEntriesCount = bytes.getInt(); List tradePresences = new ArrayList<>(groupedEntriesCount); @@ -71,53 +114,4 @@ public class TradePresencesMessage extends Message { return new TradePresencesMessage(id, tradePresences); } - @Override - protected synchronized byte[] toData() { - if (this.cachedData != null) - return this.cachedData; - - // Shortcut in case we have no trade presences - if (this.tradePresences.isEmpty()) { - this.cachedData = Ints.toByteArray(0); - return this.cachedData; - } - - // How many of each timestamp - Map countByTimestamp = new HashMap<>(); - - for (TradePresenceData tradePresenceData : this.tradePresences) { - Long timestamp = tradePresenceData.getTimestamp(); - countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v); - } - - // We should know exactly how many bytes to allocate now - int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH) - + this.tradePresences.size() * (Transformer.PUBLIC_KEY_LENGTH + Transformer.SIGNATURE_LENGTH + Transformer.ADDRESS_LENGTH); - - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize); - - for (long timestamp : countByTimestamp.keySet()) { - bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp))); - - bytes.write(Longs.toByteArray(timestamp)); - - for (TradePresenceData tradePresenceData : this.tradePresences) { - if (tradePresenceData.getTimestamp() == timestamp) { - bytes.write(tradePresenceData.getPublicKey()); - - bytes.write(tradePresenceData.getSignature()); - - bytes.write(Base58.decode(tradePresenceData.getAtAddress())); - } - } - } - - this.cachedData = bytes.toByteArray(); - return this.cachedData; - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/message/TransactionMessage.java b/src/main/java/org/qortal/network/message/TransactionMessage.java index 92cce086..51db6cf9 100644 --- a/src/main/java/org/qortal/network/message/TransactionMessage.java +++ b/src/main/java/org/qortal/network/message/TransactionMessage.java @@ -1,6 +1,5 @@ package org.qortal.network.message; -import java.io.UnsupportedEncodingException; import java.nio.ByteBuffer; import org.qortal.data.transaction.TransactionData; @@ -11,8 +10,11 @@ public class TransactionMessage extends Message { private TransactionData transactionData; - public TransactionMessage(TransactionData transactionData) { - this(-1, transactionData); + public TransactionMessage(TransactionData transactionData) throws TransformationException { + super(MessageType.TRANSACTION); + + this.dataBytes = TransactionTransformer.toBytes(transactionData); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private TransactionMessage(int id, TransactionData transactionData) { @@ -25,26 +27,16 @@ public class TransactionMessage extends Message { return this.transactionData; } - public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { - try { - TransactionData transactionData = TransactionTransformer.fromByteBuffer(byteBuffer); - - return new TransactionMessage(id, transactionData); - } catch (TransformationException e) { - return null; - } - } - - @Override - protected byte[] toData() { - if (this.transactionData == null) - return null; + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { + TransactionData transactionData; try { - return TransactionTransformer.toBytes(this.transactionData); + transactionData = TransactionTransformer.fromByteBuffer(byteBuffer); } catch (TransformationException e) { - return null; + throw new MessageException(e.getMessage(), e); } + + return new TransactionMessage(id, transactionData); } } diff --git a/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java b/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java index 082a7187..395d3f00 100644 --- a/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java +++ b/src/main/java/org/qortal/network/message/TransactionSignaturesMessage.java @@ -2,7 +2,7 @@ package org.qortal.network.message; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.UnsupportedEncodingException; +import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -13,12 +13,24 @@ import com.google.common.primitives.Ints; public class TransactionSignaturesMessage extends Message { - private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; - private List signatures; public TransactionSignaturesMessage(List signatures) { - this(-1, signatures); + super(MessageType.TRANSACTION_SIGNATURES); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(signatures.size())); + + for (byte[] signature : signatures) + bytes.write(signature); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); } private TransactionSignaturesMessage(int id, List signatures) { @@ -31,15 +43,15 @@ public class TransactionSignaturesMessage extends Message { return this.signatures; } - public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + public static Message fromByteBuffer(int id, ByteBuffer bytes) { int count = bytes.getInt(); - if (bytes.remaining() != count * SIGNATURE_LENGTH) - return null; + if (bytes.remaining() < count * Transformer.SIGNATURE_LENGTH) + throw new BufferUnderflowException(); List signatures = new ArrayList<>(); for (int i = 0; i < count; ++i) { - byte[] signature = new byte[SIGNATURE_LENGTH]; + byte[] signature = new byte[Transformer.SIGNATURE_LENGTH]; bytes.get(signature); signatures.add(signature); } @@ -47,20 +59,4 @@ public class TransactionSignaturesMessage extends Message { return new TransactionSignaturesMessage(id, signatures); } - @Override - protected byte[] toData() { - try { - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - - bytes.write(Ints.toByteArray(this.signatures.size())); - - for (byte[] signature : this.signatures) - bytes.write(signature); - - return bytes.toByteArray(); - } catch (IOException e) { - return null; - } - } - } diff --git a/src/main/java/org/qortal/network/task/BroadcastTask.java b/src/main/java/org/qortal/network/task/BroadcastTask.java new file mode 100644 index 00000000..5714ebf6 --- /dev/null +++ b/src/main/java/org/qortal/network/task/BroadcastTask.java @@ -0,0 +1,22 @@ +package org.qortal.network.task; + +import org.qortal.controller.Controller; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.Message; +import org.qortal.utils.ExecuteProduceConsume.Task; + +public class BroadcastTask implements Task { + public BroadcastTask() { + } + + @Override + public String getName() { + return "BroadcastTask"; + } + + @Override + public void perform() throws InterruptedException { + Controller.getInstance().doNetworkBroadcast(); + } +} diff --git a/src/main/java/org/qortal/network/task/ChannelAcceptTask.java b/src/main/java/org/qortal/network/task/ChannelAcceptTask.java new file mode 100644 index 00000000..3e2a3033 --- /dev/null +++ b/src/main/java/org/qortal/network/task/ChannelAcceptTask.java @@ -0,0 +1,97 @@ +package org.qortal.network.task; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.PeerAddress; +import org.qortal.settings.Settings; +import org.qortal.utils.ExecuteProduceConsume.Task; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.nio.channels.SelectionKey; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.util.List; + +public class ChannelAcceptTask implements Task { + private static final Logger LOGGER = LogManager.getLogger(ChannelAcceptTask.class); + + private final ServerSocketChannel serverSocketChannel; + + public ChannelAcceptTask(ServerSocketChannel serverSocketChannel) { + this.serverSocketChannel = serverSocketChannel; + } + + @Override + public String getName() { + return "ChannelAcceptTask"; + } + + @Override + public void perform() throws InterruptedException { + Network network = Network.getInstance(); + SocketChannel socketChannel; + + try { + if (network.getImmutableConnectedPeers().size() >= network.getMaxPeers()) { + // We have enough peers + LOGGER.debug("Ignoring pending incoming connections because the server is full"); + return; + } + + socketChannel = serverSocketChannel.accept(); + + network.setInterestOps(serverSocketChannel, SelectionKey.OP_ACCEPT); + } catch (IOException e) { + return; + } + + // No connection actually accepted? + if (socketChannel == null) { + return; + } + + PeerAddress address = PeerAddress.fromSocket(socketChannel.socket()); + List fixedNetwork = Settings.getInstance().getFixedNetwork(); + if (fixedNetwork != null && !fixedNetwork.isEmpty() && network.ipNotInFixedList(address, fixedNetwork)) { + try { + LOGGER.debug("Connection discarded from peer {} as not in the fixed network list", address); + socketChannel.close(); + } catch (IOException e) { + // IGNORE + } + return; + } + + final Long now = NTP.getTime(); + Peer newPeer; + + try { + if (now == null) { + LOGGER.debug("Connection discarded from peer {} due to lack of NTP sync", address); + socketChannel.close(); + return; + } + + LOGGER.debug("Connection accepted from peer {}", address); + + newPeer = new Peer(socketChannel); + network.addConnectedPeer(newPeer); + + } catch (IOException e) { + if (socketChannel.isOpen()) { + try { + LOGGER.debug("Connection failed from peer {} while connecting/closing", address); + socketChannel.close(); + } catch (IOException ce) { + // Couldn't close? + } + } + return; + } + + network.onPeerReady(newPeer); + } +} diff --git a/src/main/java/org/qortal/network/task/ChannelReadTask.java b/src/main/java/org/qortal/network/task/ChannelReadTask.java new file mode 100644 index 00000000..edd4e8c0 --- /dev/null +++ b/src/main/java/org/qortal/network/task/ChannelReadTask.java @@ -0,0 +1,49 @@ +package org.qortal.network.task; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.utils.ExecuteProduceConsume.Task; + +import java.io.IOException; +import java.nio.channels.SelectionKey; +import java.nio.channels.SocketChannel; + +public class ChannelReadTask implements Task { + private static final Logger LOGGER = LogManager.getLogger(ChannelReadTask.class); + + private final SocketChannel socketChannel; + private final Peer peer; + private final String name; + + public ChannelReadTask(SocketChannel socketChannel, Peer peer) { + this.socketChannel = socketChannel; + this.peer = peer; + this.name = "ChannelReadTask::" + peer; + } + + @Override + public String getName() { + return name; + } + + @Override + public void perform() throws InterruptedException { + try { + peer.readChannel(); + + Network.getInstance().setInterestOps(socketChannel, SelectionKey.OP_READ); + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().toLowerCase().contains("connection reset")) { + peer.disconnect("Connection reset"); + return; + } + + LOGGER.trace("[{}] Network thread {} encountered I/O error: {}", peer.getPeerConnectionId(), + Thread.currentThread().getId(), e.getMessage(), e); + peer.disconnect("I/O error"); + } + } +} diff --git a/src/main/java/org/qortal/network/task/ChannelWriteTask.java b/src/main/java/org/qortal/network/task/ChannelWriteTask.java new file mode 100644 index 00000000..59bc557e --- /dev/null +++ b/src/main/java/org/qortal/network/task/ChannelWriteTask.java @@ -0,0 +1,52 @@ +package org.qortal.network.task; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.utils.ExecuteProduceConsume.Task; + +import java.io.IOException; +import java.nio.channels.SelectionKey; +import java.nio.channels.SocketChannel; + +public class ChannelWriteTask implements Task { + private static final Logger LOGGER = LogManager.getLogger(ChannelWriteTask.class); + + private final SocketChannel socketChannel; + private final Peer peer; + private final String name; + + public ChannelWriteTask(SocketChannel socketChannel, Peer peer) { + this.socketChannel = socketChannel; + this.peer = peer; + this.name = "ChannelWriteTask::" + peer; + } + + @Override + public String getName() { + return name; + } + + @Override + public void perform() throws InterruptedException { + try { + boolean isSocketClogged = peer.writeChannel(); + + // Tell Network that we've finished + Network.getInstance().notifyChannelNotWriting(socketChannel); + + if (isSocketClogged) + Network.getInstance().setInterestOps(this.socketChannel, SelectionKey.OP_WRITE); + } catch (IOException e) { + if (e.getMessage() != null && e.getMessage().toLowerCase().contains("connection reset")) { + peer.disconnect("Connection reset"); + return; + } + + LOGGER.trace("[{}] Network thread {} encountered I/O error: {}", peer.getPeerConnectionId(), + Thread.currentThread().getId(), e.getMessage(), e); + peer.disconnect("I/O error"); + } + } +} diff --git a/src/main/java/org/qortal/network/task/MessageTask.java b/src/main/java/org/qortal/network/task/MessageTask.java new file mode 100644 index 00000000..c1907b62 --- /dev/null +++ b/src/main/java/org/qortal/network/task/MessageTask.java @@ -0,0 +1,28 @@ +package org.qortal.network.task; + +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.Message; +import org.qortal.utils.ExecuteProduceConsume.Task; + +public class MessageTask implements Task { + private final Peer peer; + private final Message nextMessage; + private final String name; + + public MessageTask(Peer peer, Message nextMessage) { + this.peer = peer; + this.nextMessage = nextMessage; + this.name = "MessageTask::" + peer + "::" + nextMessage.getType(); + } + + @Override + public String getName() { + return name; + } + + @Override + public void perform() throws InterruptedException { + Network.getInstance().onMessage(peer, nextMessage); + } +} diff --git a/src/main/java/org/qortal/network/task/PeerConnectTask.java b/src/main/java/org/qortal/network/task/PeerConnectTask.java new file mode 100644 index 00000000..759cabce --- /dev/null +++ b/src/main/java/org/qortal/network/task/PeerConnectTask.java @@ -0,0 +1,33 @@ +package org.qortal.network.task; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.network.Network; +import org.qortal.network.Peer; +import org.qortal.network.message.Message; +import org.qortal.network.message.MessageType; +import org.qortal.network.message.PingMessage; +import org.qortal.utils.ExecuteProduceConsume.Task; +import org.qortal.utils.NTP; + +public class PeerConnectTask implements Task { + private static final Logger LOGGER = LogManager.getLogger(PeerConnectTask.class); + + private final Peer peer; + private final String name; + + public PeerConnectTask(Peer peer) { + this.peer = peer; + this.name = "PeerConnectTask::" + peer; + } + + @Override + public String getName() { + return name; + } + + @Override + public void perform() throws InterruptedException { + Network.getInstance().connectPeer(peer); + } +} diff --git a/src/main/java/org/qortal/network/task/PingTask.java b/src/main/java/org/qortal/network/task/PingTask.java new file mode 100644 index 00000000..f47ecd32 --- /dev/null +++ b/src/main/java/org/qortal/network/task/PingTask.java @@ -0,0 +1,44 @@ +package org.qortal.network.task; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.network.Peer; +import org.qortal.network.message.Message; +import org.qortal.network.message.MessageType; +import org.qortal.network.message.PingMessage; +import org.qortal.utils.ExecuteProduceConsume.Task; +import org.qortal.utils.NTP; + +public class PingTask implements Task { + private static final Logger LOGGER = LogManager.getLogger(PingTask.class); + + private final Peer peer; + private final Long now; + private final String name; + + public PingTask(Peer peer, Long now) { + this.peer = peer; + this.now = now; + this.name = "PingTask::" + peer; + } + + @Override + public String getName() { + return name; + } + + @Override + public void perform() throws InterruptedException { + PingMessage pingMessage = new PingMessage(); + Message message = peer.getResponse(pingMessage); + + if (message == null || message.getType() != MessageType.PING) { + LOGGER.debug("[{}] Didn't receive reply from {} for PING ID {}", + peer.getPeerConnectionId(), peer, pingMessage.getId()); + peer.disconnect("no ping received"); + return; + } + + peer.setLastPing(NTP.getTime() - now); + } +} diff --git a/src/main/java/org/qortal/utils/ExecuteProduceConsume.java b/src/main/java/org/qortal/utils/ExecuteProduceConsume.java index 57caab9c..223d0e93 100644 --- a/src/main/java/org/qortal/utils/ExecuteProduceConsume.java +++ b/src/main/java/org/qortal/utils/ExecuteProduceConsume.java @@ -28,7 +28,6 @@ public abstract class ExecuteProduceConsume implements Runnable { private final String className; private final Logger logger; - private final boolean isLoggerTraceEnabled; protected ExecutorService executor; @@ -43,12 +42,12 @@ public abstract class ExecuteProduceConsume implements Runnable { private volatile int tasksConsumed = 0; private volatile int spawnFailures = 0; + /** Whether a new thread has already been spawned and is waiting to start. Used to prevent spawning multiple new threads. */ private volatile boolean hasThreadPending = false; public ExecuteProduceConsume(ExecutorService executor) { this.className = this.getClass().getSimpleName(); this.logger = LogManager.getLogger(this.getClass()); - this.isLoggerTraceEnabled = this.logger.isTraceEnabled(); this.executor = executor; } @@ -98,15 +97,14 @@ public abstract class ExecuteProduceConsume implements Runnable { */ protected abstract Task produceTask(boolean canBlock) throws InterruptedException; - @FunctionalInterface public interface Task { - public abstract void perform() throws InterruptedException; + String getName(); + void perform() throws InterruptedException; } @Override public void run() { - if (this.isLoggerTraceEnabled) - Thread.currentThread().setName(this.className + "-" + Thread.currentThread().getId()); + Thread.currentThread().setName(this.className + "-" + Thread.currentThread().getId()); boolean wasThreadPending; synchronized (this) { @@ -114,25 +112,19 @@ public abstract class ExecuteProduceConsume implements Runnable { if (this.activeThreadCount > this.greatestActiveThreadCount) this.greatestActiveThreadCount = this.activeThreadCount; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] started, hasThreadPending was: %b, activeThreadCount now: %d", - Thread.currentThread().getId(), this.hasThreadPending, this.activeThreadCount)); - } + this.logger.trace(() -> String.format("[%d] started, hasThreadPending was: %b, activeThreadCount now: %d", + Thread.currentThread().getId(), this.hasThreadPending, this.activeThreadCount)); // Defer clearing hasThreadPending to prevent unnecessary threads waiting to produce... wasThreadPending = this.hasThreadPending; } try { - // It's possible this might need to become a class instance private volatile - boolean canBlock = false; - while (!Thread.currentThread().isInterrupted()) { Task task = null; + String taskType; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] waiting to produce...", Thread.currentThread().getId())); - } + this.logger.trace(() -> String.format("[%d] waiting to produce...", Thread.currentThread().getId())); synchronized (this) { if (wasThreadPending) { @@ -141,13 +133,13 @@ public abstract class ExecuteProduceConsume implements Runnable { wasThreadPending = false; } - final boolean lambdaCanIdle = canBlock; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] producing, activeThreadCount: %d, consumerCount: %d, canBlock is %b...", - Thread.currentThread().getId(), this.activeThreadCount, this.consumerCount, lambdaCanIdle)); - } + // If we're the only non-consuming thread - producer can afford to block this round + boolean canBlock = this.activeThreadCount - this.consumerCount <= 1; - final long beforeProduce = isLoggerTraceEnabled ? System.currentTimeMillis() : 0; + this.logger.trace(() -> String.format("[%d] producing... [activeThreadCount: %d, consumerCount: %d, canBlock: %b]", + Thread.currentThread().getId(), this.activeThreadCount, this.consumerCount, canBlock)); + + final long beforeProduce = this.logger.isDebugEnabled() ? System.currentTimeMillis() : 0; try { task = produceTask(canBlock); @@ -158,31 +150,36 @@ public abstract class ExecuteProduceConsume implements Runnable { this.logger.warn(() -> String.format("[%d] exception while trying to produce task", Thread.currentThread().getId()), e); } - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] producing took %dms", Thread.currentThread().getId(), System.currentTimeMillis() - beforeProduce)); + if (this.logger.isDebugEnabled()) { + final long productionPeriod = System.currentTimeMillis() - beforeProduce; + taskType = task == null ? "no task" : task.getName(); + + this.logger.debug(() -> String.format("[%d] produced [%s] in %dms [canBlock: %b]", + Thread.currentThread().getId(), + taskType, + productionPeriod, + canBlock + )); + } else { + taskType = null; } } if (task == null) synchronized (this) { - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] no task, activeThreadCount: %d, consumerCount: %d", - Thread.currentThread().getId(), this.activeThreadCount, this.consumerCount)); - } + this.logger.trace(() -> String.format("[%d] no task, activeThreadCount: %d, consumerCount: %d", + Thread.currentThread().getId(), this.activeThreadCount, this.consumerCount)); - if (this.activeThreadCount > this.consumerCount + 1) { + // If we have an excess of non-consuming threads then we can exit + if (this.activeThreadCount - this.consumerCount > 1) { --this.activeThreadCount; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] ending, activeThreadCount now: %d", - Thread.currentThread().getId(), this.activeThreadCount)); - } + + this.logger.trace(() -> String.format("[%d] ending, activeThreadCount now: %d", + Thread.currentThread().getId(), this.activeThreadCount)); return; } - // We're the last surviving thread - producer can afford to block next round - canBlock = true; - continue; } @@ -192,16 +189,13 @@ public abstract class ExecuteProduceConsume implements Runnable { ++this.tasksProduced; ++this.consumerCount; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] hasThreadPending: %b, activeThreadCount: %d, consumerCount now: %d", - Thread.currentThread().getId(), this.hasThreadPending, this.activeThreadCount, this.consumerCount)); - } + this.logger.trace(() -> String.format("[%d] hasThreadPending: %b, activeThreadCount: %d, consumerCount now: %d", + Thread.currentThread().getId(), this.hasThreadPending, this.activeThreadCount, this.consumerCount)); // If we have no thread pending and no excess of threads then we should spawn a fresh thread - if (!this.hasThreadPending && this.activeThreadCount <= this.consumerCount + 1) { - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] spawning another thread", Thread.currentThread().getId())); - } + if (!this.hasThreadPending && this.activeThreadCount == this.consumerCount) { + this.logger.trace(() -> String.format("[%d] spawning another thread", Thread.currentThread().getId())); + this.hasThreadPending = true; try { @@ -209,21 +203,19 @@ public abstract class ExecuteProduceConsume implements Runnable { } catch (RejectedExecutionException e) { ++this.spawnFailures; this.hasThreadPending = false; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] failed to spawn another thread", Thread.currentThread().getId())); - } + + this.logger.trace(() -> String.format("[%d] failed to spawn another thread", Thread.currentThread().getId())); + this.onSpawnFailure(); } } else { - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] NOT spawning another thread", Thread.currentThread().getId())); - } + this.logger.trace(() -> String.format("[%d] NOT spawning another thread", Thread.currentThread().getId())); } } - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] performing task...", Thread.currentThread().getId())); - } + this.logger.trace(() -> String.format("[%d] consuming [%s] task...", Thread.currentThread().getId(), taskType)); + + final long beforePerform = this.logger.isDebugEnabled() ? System.currentTimeMillis() : 0; try { task.perform(); // This can block for a while @@ -231,29 +223,25 @@ public abstract class ExecuteProduceConsume implements Runnable { // We're in shutdown situation so exit Thread.currentThread().interrupt(); } catch (Exception e) { - this.logger.warn(() -> String.format("[%d] exception while performing task", Thread.currentThread().getId()), e); + this.logger.warn(() -> String.format("[%d] exception while consuming task", Thread.currentThread().getId()), e); } - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] finished task", Thread.currentThread().getId())); + if (this.logger.isDebugEnabled()) { + final long productionPeriod = System.currentTimeMillis() - beforePerform; + + this.logger.debug(() -> String.format("[%d] consumed [%s] task in %dms", Thread.currentThread().getId(), taskType, productionPeriod)); } synchronized (this) { ++this.tasksConsumed; --this.consumerCount; - if (this.isLoggerTraceEnabled) { - this.logger.trace(() -> String.format("[%d] consumerCount now: %d", - Thread.currentThread().getId(), this.consumerCount)); - } - - // Quicker, non-blocking produce next round - canBlock = false; + this.logger.trace(() -> String.format("[%d] consumerCount now: %d", + Thread.currentThread().getId(), this.consumerCount)); } } } finally { - if (this.isLoggerTraceEnabled) - Thread.currentThread().setName(this.className); + Thread.currentThread().setName(this.className); } } diff --git a/src/test/java/org/qortal/test/EPCTests.java b/src/test/java/org/qortal/test/EPCTests.java index fe48af24..1a41b75d 100644 --- a/src/test/java/org/qortal/test/EPCTests.java +++ b/src/test/java/org/qortal/test/EPCTests.java @@ -13,9 +13,25 @@ import org.junit.Test; import org.qortal.utils.ExecuteProduceConsume; import org.qortal.utils.ExecuteProduceConsume.StatsSnapshot; +import static org.junit.Assert.fail; + public class EPCTests { - class RandomEPC extends ExecuteProduceConsume { + static class SleepTask implements ExecuteProduceConsume.Task { + private static final Random RANDOM = new Random(); + + @Override + public String getName() { + return "SleepTask"; + } + + @Override + public void perform() throws InterruptedException { + Thread.sleep(RANDOM.nextInt(500) + 100); + } + } + + static class RandomEPC extends ExecuteProduceConsume { private final int TASK_PERCENT; private final int PAUSE_PERCENT; @@ -37,9 +53,7 @@ public class EPCTests { // Sometimes produce a task if (percent < TASK_PERCENT) { - return () -> { - Thread.sleep(random.nextInt(500) + 100); - }; + return new SleepTask(); } else { // If we don't produce a task, then maybe simulate a pause until work arrives if (canIdle && percent < PAUSE_PERCENT) @@ -50,45 +64,6 @@ public class EPCTests { } } - private void testEPC(ExecuteProduceConsume testEPC) throws InterruptedException { - final int runTime = 60; // seconds - System.out.println(String.format("Testing EPC for %s seconds:", runTime)); - - final long start = System.currentTimeMillis(); - testEPC.start(); - - // Status reports every second (bar waiting for synchronization) - ScheduledExecutorService statusExecutor = Executors.newSingleThreadScheduledExecutor(); - - statusExecutor.scheduleAtFixedRate(() -> { - final StatsSnapshot snapshot = testEPC.getStatsSnapshot(); - final long seconds = (System.currentTimeMillis() - start) / 1000L; - System.out.print(String.format("After %d second%s, ", seconds, (seconds != 1 ? "s" : ""))); - printSnapshot(snapshot); - }, 1L, 1L, TimeUnit.SECONDS); - - // Let it run for a minute - Thread.sleep(runTime * 1000L); - statusExecutor.shutdownNow(); - - final long before = System.currentTimeMillis(); - testEPC.shutdown(30 * 1000); - final long after = System.currentTimeMillis(); - - System.out.println(String.format("Shutdown took %d milliseconds", after - before)); - - final StatsSnapshot snapshot = testEPC.getStatsSnapshot(); - System.out.print("After shutdown, "); - printSnapshot(snapshot); - } - - private void printSnapshot(final StatsSnapshot snapshot) { - System.out.println(String.format("threads: %d active (%d max, %d exhaustion%s), tasks: %d produced / %d consumed", - snapshot.activeThreadCount, snapshot.greatestActiveThreadCount, - snapshot.spawnFailures, (snapshot.spawnFailures != 1 ? "s": ""), - snapshot.tasksProduced, snapshot.tasksConsumed)); - } - @Test public void testRandomEPC() throws InterruptedException { final int TASK_PERCENT = 25; // Produce a task this % of the time @@ -131,18 +106,39 @@ public class EPCTests { final int MAX_PEERS = 20; - final List lastPings = new ArrayList<>(Collections.nCopies(MAX_PEERS, System.currentTimeMillis())); + final List lastPingProduced = new ArrayList<>(Collections.nCopies(MAX_PEERS, System.currentTimeMillis())); class PingTask implements ExecuteProduceConsume.Task { private final int peerIndex; + private final long lastPing; + private final long productionTimestamp; + private final String name; - public PingTask(int peerIndex) { + public PingTask(int peerIndex, long lastPing, long productionTimestamp) { this.peerIndex = peerIndex; + this.lastPing = lastPing; + this.productionTimestamp = productionTimestamp; + this.name = "PingTask::[" + this.peerIndex + "]"; + } + + @Override + public String getName() { + return name; } @Override public void perform() throws InterruptedException { - System.out.println("Pinging peer " + peerIndex); + long now = System.currentTimeMillis(); + + System.out.println(String.format("Pinging peer %d after post-production delay of %dms and ping interval of %dms", + peerIndex, + now - productionTimestamp, + now - lastPing + )); + + long threshold = now - PING_INTERVAL - PRODUCER_SLEEP_TIME; + if (lastPing < threshold) + fail("excessive peer ping interval for peer " + peerIndex); // At least half the worst case ping round-trip Random random = new Random(); @@ -155,32 +151,73 @@ public class EPCTests { class PingEPC extends ExecuteProduceConsume { @Override protected Task produceTask(boolean canIdle) throws InterruptedException { - // If we can idle, then we do, to simulate worst case - if (canIdle) - Thread.sleep(PRODUCER_SLEEP_TIME); - // Is there a peer that needs a ping? final long now = System.currentTimeMillis(); - synchronized (lastPings) { - for (int peerIndex = 0; peerIndex < lastPings.size(); ++peerIndex) { - long lastPing = lastPings.get(peerIndex); - - if (lastPing < now - PING_INTERVAL - PING_ROUND_TRIP_TIME - PRODUCER_SLEEP_TIME) - throw new RuntimeException("excessive peer ping interval for peer " + peerIndex); + synchronized (lastPingProduced) { + for (int peerIndex = 0; peerIndex < lastPingProduced.size(); ++peerIndex) { + long lastPing = lastPingProduced.get(peerIndex); if (lastPing < now - PING_INTERVAL) { - lastPings.set(peerIndex, System.currentTimeMillis()); - return new PingTask(peerIndex); + lastPingProduced.set(peerIndex, System.currentTimeMillis()); + return new PingTask(peerIndex, lastPing, now); } } } + // If we can idle, then we do, to simulate worst case + if (canIdle) + Thread.sleep(PRODUCER_SLEEP_TIME); + // No work to do return null; } } + System.out.println(String.format("Pings should start after %s seconds", PING_INTERVAL)); + testEPC(new PingEPC()); } + private void testEPC(ExecuteProduceConsume testEPC) throws InterruptedException { + final int runTime = 60; // seconds + System.out.println(String.format("Testing EPC for %s seconds:", runTime)); + + final long start = System.currentTimeMillis(); + + // Status reports every second (bar waiting for synchronization) + ScheduledExecutorService statusExecutor = Executors.newSingleThreadScheduledExecutor(); + + statusExecutor.scheduleAtFixedRate( + () -> { + final StatsSnapshot snapshot = testEPC.getStatsSnapshot(); + final long seconds = (System.currentTimeMillis() - start) / 1000L; + System.out.println(String.format("After %d second%s, %s", seconds, seconds != 1 ? "s" : "", formatSnapshot(snapshot))); + }, + 0L, 1L, TimeUnit.SECONDS + ); + + testEPC.start(); + + // Let it run for a minute + Thread.sleep(runTime * 1000L); + statusExecutor.shutdownNow(); + + final long before = System.currentTimeMillis(); + testEPC.shutdown(30 * 1000); + final long after = System.currentTimeMillis(); + + System.out.println(String.format("Shutdown took %d milliseconds", after - before)); + + final StatsSnapshot snapshot = testEPC.getStatsSnapshot(); + System.out.println("After shutdown, " + formatSnapshot(snapshot)); + } + + private String formatSnapshot(StatsSnapshot snapshot) { + return String.format("threads: %d active (%d max, %d exhaustion%s), tasks: %d produced / %d consumed", + snapshot.activeThreadCount, snapshot.greatestActiveThreadCount, + snapshot.spawnFailures, (snapshot.spawnFailures != 1 ? "s": ""), + snapshot.tasksProduced, snapshot.tasksConsumed + ); + } + } diff --git a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java index b1c5ec4f..4154121c 100644 --- a/src/test/java/org/qortal/test/network/OnlineAccountsTests.java +++ b/src/test/java/org/qortal/test/network/OnlineAccountsTests.java @@ -29,7 +29,7 @@ public class OnlineAccountsTests { @Test - public void testGetOnlineAccountsV2() throws Message.MessageException { + public void testGetOnlineAccountsV2() throws MessageException { List onlineAccountsOut = generateOnlineAccounts(false); Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut); @@ -58,7 +58,7 @@ public class OnlineAccountsTests { } @Test - public void testOnlineAccountsV2() throws Message.MessageException { + public void testOnlineAccountsV2() throws MessageException { List onlineAccountsOut = generateOnlineAccounts(true); Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut);