diff --git a/core/src/main/java/com/google/bitcoin/core/PeerGroup.java b/core/src/main/java/com/google/bitcoin/core/PeerGroup.java
index bfccffec..bf308152 100644
--- a/core/src/main/java/com/google/bitcoin/core/PeerGroup.java
+++ b/core/src/main/java/com/google/bitcoin/core/PeerGroup.java
@@ -82,7 +82,7 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
     // The peer that has been selected for the purposes of downloading announced data.
     @GuardedBy("lock") private Peer downloadPeer;
     // Callback for events related to chain download
-    @GuardedBy("lock") private PeerEventListener downloadListener;
+    @Nullable @GuardedBy("lock") private PeerEventListener downloadListener;
     // Callbacks for events related to peer connection/disconnection
     private final CopyOnWriteArrayList<ListenerRegistration<PeerEventListener>> peerEventListeners;
     // Peer discovery sources, will be polled occasionally if there aren't enough inactives.
@@ -734,12 +734,14 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
     public void startBlockChainDownload(PeerEventListener listener) {
         lock.lock();
         try {
+            if (downloadPeer != null && this.downloadListener != null)
+                downloadPeer.removeEventListener(this.downloadListener);
             this.downloadListener = listener;
             // TODO: be more nuanced about which peer to download from.  We can also try
             // downloading from multiple peers and handle the case when a new peer comes along
             // with a longer chain after we thought we were done.
             if (!peers.isEmpty()) {
-                startBlockChainDownloadFromPeer(peers.iterator().next());
+                startBlockChainDownloadFromPeer(peers.iterator().next()); // Will add the new download listener
             }
         } finally {
             lock.unlock();
@@ -877,11 +879,15 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
             }
             if (downloadPeer != null) {
                 log.info("Unsetting download peer: {}", downloadPeer);
+                if (downloadListener != null)
+                    downloadPeer.removeEventListener(downloadListener);
                 downloadPeer.setDownloadData(false);
             }
             downloadPeer = peer;
             if (downloadPeer != null) {
                 log.info("Setting download peer: {}", downloadPeer);
+                if (downloadListener != null)
+                    peer.addEventListener(downloadListener, Threading.SAME_THREAD);
                 downloadPeer.setDownloadData(true);
                 downloadPeer.setDownloadParameters(fastCatchupTimeSecs, bloomFilter != null);
             }
@@ -990,7 +996,6 @@ public class PeerGroup extends AbstractIdleService implements TransactionBroadca
     private void startBlockChainDownloadFromPeer(Peer peer) {
         lock.lock();
         try {
-            peer.addEventListener(downloadListener, Threading.SAME_THREAD);
             setDownloadPeer(peer);
             // startBlockChainDownload will setDownloadData(true) on itself automatically.
             peer.startBlockChainDownload();