remove unused conversion fn

mev_inspect/aave_liquidations.py

@@ -42,6 +42,7 @@ def get_aave_liquidations(
             trace.classification == Classification.liquidate
             and isinstance(trace, DecodedCallTrace)
             and not is_child_of_any_address(trace, parent_liquidations)
+            and trace.protocol == Protocol.aave
         ):
 
             parent_liquidations.append(trace.trace_address)

mev_inspect/abi.py

@@ -12,15 +12,32 @@ THIS_FILE_DIRECTORY = Path(__file__).parents[0]
 ABI_DIRECTORY_PATH = THIS_FILE_DIRECTORY / "abis"
 
 
-def get_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[ABI]:
+def get_abi_path(abi_name: str, protocol: Optional[Protocol]) -> Optional[Path]:
     abi_filename = f"{abi_name}.json"
     abi_path = (
         ABI_DIRECTORY_PATH / abi_filename
         if protocol is None
         else ABI_DIRECTORY_PATH / protocol.value / abi_filename
     )
-
     if abi_path.is_file():
+        return abi_path
+
+    return None
+
+
+# raw abi, for instantiating contract for queries (as opposed to classification, see below)
+def get_raw_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[str]:
+    abi_path = get_abi_path(abi_name, protocol)
+    if abi_path is not None:
+        with abi_path.open() as abi_file:
+            return abi_file.read()
+
+    return None
+
+
+def get_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[ABI]:
+    abi_path = get_abi_path(abi_name, protocol)
+    if abi_path is not None:
         with abi_path.open() as abi_file:
             abi_json = json.load(abi_file)
             return parse_obj_as(ABI, abi_json)

mev_inspect/block.py

@@ -82,4 +82,4 @@ def cache_block(cache_path: Path, block: Block):
 
 def _get_cache_path(block_number: int) -> Path:
     cache_directory_path = Path(cache_directory)
-    return cache_directory_path / f"{block_number}-new.json"
+    return cache_directory_path / f"{block_number}.json"

mev_inspect/classifiers/specs/__init__.py

@@ -7,10 +7,10 @@ from .aave import AAVE_CLASSIFIER_SPECS
 from .curve import CURVE_CLASSIFIER_SPECS
 from .erc20 import ERC20_CLASSIFIER_SPECS
 from .uniswap import UNISWAP_CLASSIFIER_SPECS
-from .weth import WETH_CLASSIFIER_SPECS
+from .weth import WETH_CLASSIFIER_SPECS, WETH_ADDRESS
 from .zero_ex import ZEROX_CLASSIFIER_SPECS
 from .balancer import BALANCER_CLASSIFIER_SPECS
-
+from .compound import COMPOUND_CLASSIFIER_SPECS
 
 ALL_CLASSIFIER_SPECS = (
     ERC20_CLASSIFIER_SPECS
@@ -20,6 +20,7 @@ ALL_CLASSIFIER_SPECS = (
     + AAVE_CLASSIFIER_SPECS
     + ZEROX_CLASSIFIER_SPECS
     + BALANCER_CLASSIFIER_SPECS
+    + COMPOUND_CLASSIFIER_SPECS
 )
 
 _SPECS_BY_ABI_NAME_AND_PROTOCOL: Dict[

mev_inspect/classifiers/specs/compound.py

@@ -0,0 +1,28 @@
+from mev_inspect.schemas.classified_traces import (
+    Protocol,
+)
+from mev_inspect.schemas.classifiers import (
+    ClassifierSpec,
+    LiquidationClassifier,
+    SeizeClassifier,
+)
+
+COMPOUND_V2_CETH_SPEC = ClassifierSpec(
+    abi_name="CEther",
+    protocol=Protocol.compound_v2,
+    classifiers={
+        "liquidateBorrow(address,address)": LiquidationClassifier,
+        "seize(address,address,uint256)": SeizeClassifier,
+    },
+)
+
+COMPOUND_V2_CTOKEN_SPEC = ClassifierSpec(
+    abi_name="CToken",
+    protocol=Protocol.compound_v2,
+    classifiers={
+        "liquidateBorrow(address,uint256,address)": LiquidationClassifier,
+        "seize(address,address,uint256)": SeizeClassifier,
+    },
+)
+
+COMPOUND_CLASSIFIER_SPECS = [COMPOUND_V2_CETH_SPEC, COMPOUND_V2_CTOKEN_SPEC]

mev_inspect/classifiers/specs/weth.py

@@ -23,10 +23,12 @@ class WethTransferClassifier(TransferClassifier):
         )
 
 
+WETH_ADDRESS = "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"
+
 WETH_SPEC = ClassifierSpec(
     abi_name="WETH9",
     protocol=Protocol.weth,
-    valid_contract_addresses=["0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"],
+    valid_contract_addresses=[WETH_ADDRESS],
     classifiers={
         "transferFrom(address,address,uint256)": WethTransferClassifier,
         "transfer(address,uint256)": WethTransferClassifier,

mev_inspect/compound_liquidations.py

@@ -0,0 +1,102 @@
+from typing import Dict, List, Optional
+from web3 import Web3
+
+from mev_inspect.traces import get_child_traces
+from mev_inspect.schemas.classified_traces import (
+    ClassifiedTrace,
+    Classification,
+    Protocol,
+)
+
+from mev_inspect.schemas.liquidations import Liquidation
+from mev_inspect.classifiers.specs import WETH_ADDRESS
+from mev_inspect.abi import get_raw_abi
+
+V2_COMPTROLLER_ADDRESS = "0x3d9819210A31b4961b30EF54bE2aeD79B9c9Cd3B"
+V2_C_ETHER = "0x4Ddc2D193948926D02f9B1fE9e1daa0718270ED5"
+
+# helper, only queried once in the beginning (inspect_block)
+def fetch_all_comp_markets(w3: Web3) -> Dict[str, str]:
+    c_token_mapping = {}
+    comp_v2_comptroller_abi = get_raw_abi("Comptroller", Protocol.compound_v2)
+    comptroller_instance = w3.eth.contract(
+        address=V2_COMPTROLLER_ADDRESS, abi=comp_v2_comptroller_abi
+    )
+    markets = comptroller_instance.functions.getAllMarkets().call()
+    comp_v2_ctoken_abi = get_raw_abi("CToken", Protocol.compound_v2)
+    for c_token in markets:
+        # make an exception for cETH (as it has no .underlying())
+        if c_token != V2_C_ETHER:
+            ctoken_instance = w3.eth.contract(address=c_token, abi=comp_v2_ctoken_abi)
+            underlying_token = ctoken_instance.functions.underlying().call()
+            c_token_mapping[
+                c_token.lower()
+            ] = underlying_token.lower()  # make k:v lowercase for consistancy
+    return c_token_mapping
+
+
+def get_compound_liquidations(
+    traces: List[ClassifiedTrace], collateral_by_c_token_address: Dict[str, str]
+) -> List[Liquidation]:
+
+    """Inspect list of classified traces and identify liquidation"""
+    liquidations: List[Liquidation] = []
+
+    for trace in traces:
+        if (
+            trace.classification == Classification.liquidate
+            and trace.protocol == Protocol.compound_v2
+            and trace.inputs is not None
+            and trace.to_address is not None
+        ):
+            # First, we look for cEther liquidations (position paid back via tx.value)
+            child_traces = get_child_traces(
+                trace.transaction_hash, trace.trace_address, traces
+            )
+            seize_trace = _get_seize_call(child_traces)
+            if seize_trace is not None and seize_trace.inputs is not None:
+                c_token_collateral = trace.inputs["cTokenCollateral"]
+                if trace.abi_name == "CEther":
+                    liquidations.append(
+                        Liquidation(
+                            liquidated_user=trace.inputs["borrower"],
+                            collateral_token_address=WETH_ADDRESS,  # WETH since all cEther liquidations provide Ether
+                            debt_token_address=c_token_collateral,
+                            liquidator_user=seize_trace.inputs["liquidator"],
+                            debt_purchase_amount=trace.value,
+                            protocol=Protocol.compound_v2,
+                            received_amount=seize_trace.inputs["seizeTokens"],
+                            transaction_hash=trace.transaction_hash,
+                            trace_address=trace.trace_address,
+                            block_number=trace.block_number,
+                        )
+                    )
+                elif (
+                    trace.abi_name == "CToken"
+                ):  # cToken liquidations where liquidator pays back via token transfer
+                    c_token_address = trace.to_address
+                    liquidations.append(
+                        Liquidation(
+                            liquidated_user=trace.inputs["borrower"],
+                            collateral_token_address=collateral_by_c_token_address[
+                                c_token_address
+                            ],
+                            debt_token_address=c_token_collateral,
+                            liquidator_user=seize_trace.inputs["liquidator"],
+                            debt_purchase_amount=trace.inputs["repayAmount"],
+                            protocol=Protocol.compound_v2,
+                            received_amount=seize_trace.inputs["seizeTokens"],
+                            transaction_hash=trace.transaction_hash,
+                            trace_address=trace.trace_address,
+                            block_number=trace.block_number,
+                        )
+                    )
+    return liquidations
+
+
+def _get_seize_call(traces: List[ClassifiedTrace]) -> Optional[ClassifiedTrace]:
+    """Find the call to `seize` in the child traces (successful liquidation)"""
+    for trace in traces:
+        if trace.classification == Classification.seize:
+            return trace
+    return None

mev_inspect/inspect_block.py

@@ -27,8 +27,7 @@ from mev_inspect.crud.liquidations import (
 from mev_inspect.miner_payments import get_miner_payments
 from mev_inspect.swaps import get_swaps
 from mev_inspect.transfers import get_transfers
-from mev_inspect.aave_liquidations import get_aave_liquidations
-
+from mev_inspect.liquidations import get_liquidations
 
 logger = logging.getLogger(__name__)
 
@@ -64,8 +63,6 @@ def inspect_block(
         write_classified_traces(db_session, classified_traces)
 
     transfers = get_transfers(classified_traces)
-    logger.info(f"Found {len(transfers)} transfers")
-
     if should_write_transfers:
         delete_transfers_for_block(db_session, block_number)
         write_transfers(db_session, transfers)
@@ -84,7 +81,7 @@ def inspect_block(
         delete_arbitrages_for_block(db_session, block_number)
         write_arbitrages(db_session, arbitrages)
 
-    liquidations = get_aave_liquidations(classified_traces)
+    liquidations = get_liquidations(classified_traces, w3)
     logger.info(f"Found {len(liquidations)} liquidations")
 
     if should_write_liquidations:

mev_inspect/liquidations.py

@@ -0,0 +1,19 @@
+from typing import List
+
+from web3 import Web3
+from mev_inspect.aave_liquidations import get_aave_liquidations
+from mev_inspect.compound_liquidations import (
+    get_compound_liquidations,
+    fetch_all_comp_markets,
+)
+from mev_inspect.schemas.classified_traces import ClassifiedTrace
+from mev_inspect.schemas.liquidations import Liquidation
+
+
+def get_liquidations(
+    classified_traces: List[ClassifiedTrace], w3: Web3
+) -> List[Liquidation]:
+    aave_liquidations = get_aave_liquidations(classified_traces)
+    comp_markets = fetch_all_comp_markets(w3)
+    compound_liquidations = get_compound_liquidations(classified_traces, comp_markets)
+    return aave_liquidations + compound_liquidations

mev_inspect/schemas/classified_traces.py

@@ -9,6 +9,7 @@ class Classification(Enum):
     swap = "swap"
     transfer = "transfer"
     liquidate = "liquidate"
+    seize = "seize"
 
 
 class Protocol(Enum):
@@ -20,6 +21,7 @@ class Protocol(Enum):
     curve = "curve"
     zero_ex = "0x"
     balancer_v1 = "balancer_v1"
+    compound_v2 = "compound_v2"
 
 
 class ClassifiedTrace(Trace):

mev_inspect/schemas/classifiers.py

@@ -42,6 +42,12 @@ class LiquidationClassifier(Classifier):
         return Classification.liquidate
 
 
+class SeizeClassifier(Classifier):
+    @staticmethod
+    def get_classification() -> Classification:
+        return Classification.seize
+
+
 class ClassifierSpec(BaseModel):
     abi_name: str
     protocol: Optional[Protocol] = None

mev_inspect/schemas/utils.py

@@ -1,8 +1,8 @@
 import json
 
 from hexbytes import HexBytes
-from pydantic import BaseModel
 from web3.datastructures import AttributeDict
+from pydantic import BaseModel
 
 
 def to_camel(string: str) -> str:

tests/comp_markets.json

@@ -0,0 +1 @@
+{"0x6c8c6b02e7b2be14d4fa6022dfd6d75921d90e4e": "0x0d8775f648430679a709e98d2b0cb6250d2887ef", "0x5d3a536e4d6dbd6114cc1ead35777bab948e3643": "0x6b175474e89094c44da98b954eedeac495271d0f", "0x158079ee67fce2f58472a96584a73c7ab9ac95c1": "0x1985365e9f78359a9b6ad760e32412f4a445e862", "0x39aa39c021dfbae8fac545936693ac917d5e7563": "0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48", "0xf650c3d88d12db855b8bf7d11be6c55a4e07dcc9": "0xdac17f958d2ee523a2206206994597c13d831ec7", "0xc11b1268c1a384e55c48c2391d8d480264a3a7f4": "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599", "0xb3319f5d18bc0d84dd1b4825dcde5d5f7266d407": "0xe41d2489571d322189246dafa5ebde1f4699f498", "0xf5dce57282a584d2746faf1593d3121fcac444dc": "0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359", "0x35a18000230da775cac24873d00ff85bccded550": "0x1f9840a85d5af5bf1d1762f925bdaddc4201f984", "0x70e36f6bf80a52b3b46b3af8e106cc0ed743e8e4": "0xc00e94cb662c3520282e6f5717214004a7f26888", "0xccf4429db6322d5c611ee964527d42e5d685dd6a": "0x2260fac5e5542a773aa44fbcfedf7c193bc2c599", "0x12392f67bdf24fae0af363c24ac620a2f67dad86": "0x0000000000085d4780b73119b644ae5ecd22b376", "0xface851a4921ce59e912d19329929ce6da6eb0c7": "0x514910771af9ca656af840dff83e8264ecf986ca", "0x95b4ef2869ebd94beb4eee400a99824bf5dc325b": "0x9f8f72aa9304c8b593d555f12ef6589cc3a579a2", "0x4b0181102a0112a2ef11abee5563bb4a3176c9d7": "0x6b3595068778dd592e39a122f4f5a5cf09c90fe2", "0xe65cdb6479bac1e22340e4e755fae7e509ecd06c": "0x7fc66500c84a76ad7e9c93437bfc5ac33e2ddae9", "0x80a2ae356fc9ef4305676f7a3e2ed04e12c33946": "0x0bc529c00c6401aef6d220be8c6ea1667f6ad93e"}

tests/test_compound.py

@@ -0,0 +1,112 @@
+from mev_inspect.compound_liquidations import get_compound_liquidations
+from mev_inspect.schemas.liquidations import Liquidation
+from mev_inspect.schemas.classified_traces import Protocol
+from mev_inspect.classifiers.trace import TraceClassifier
+from tests.utils import load_test_block, load_comp_markets
+
+comp_markets = load_comp_markets()
+
+
+def test_c_ether_liquidations():
+    block_number = 13234998
+    transaction_hash = (
+        "0x78f7e67391c2bacde45e5057241f8b9e21a59330bce4332eecfff8fac279d090"
+    )
+
+    liquidations = [
+        Liquidation(
+            liquidated_user="0xb5535a3681cf8d5431b8acfd779e2f79677ecce9",
+            liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
+            collateral_token_address="0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2",
+            debt_token_address="0x39aa39c021dfbae8fac545936693ac917d5e7563",
+            debt_purchase_amount=268066492249420078,
+            received_amount=4747650169097,
+            protocol=Protocol.compound_v2,
+            transaction_hash=transaction_hash,
+            trace_address=[1],
+            block_number=block_number,
+        )
+    ]
+    block = load_test_block(block_number)
+    trace_classifier = TraceClassifier()
+    classified_traces = trace_classifier.classify(block.traces)
+    result = get_compound_liquidations(classified_traces, comp_markets)
+    assert result == liquidations
+
+    block_number = 13207907
+    transaction_hash = (
+        "0x42a575e3f41d24f3bb00ae96f220a8bd1e24e6a6282c2e0059bb7820c61e91b1"
+    )
+
+    liquidations = [
+        Liquidation(
+            liquidated_user="0x45df6f00166c3fb77dc16b9e47ff57bc6694e898",
+            liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
+            collateral_token_address="0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2",
+            debt_token_address="0x35a18000230da775cac24873d00ff85bccded550",
+            debt_purchase_amount=414547860568297082,
+            received_amount=321973320649,
+            protocol=Protocol.compound_v2,
+            transaction_hash=transaction_hash,
+            trace_address=[1],
+            block_number=block_number,
+        )
+    ]
+
+    block = load_test_block(block_number)
+    trace_classifier = TraceClassifier()
+    classified_traces = trace_classifier.classify(block.traces)
+    result = get_compound_liquidations(classified_traces, comp_markets)
+    assert result == liquidations
+
+    block_number = 13298725
+    transaction_hash = (
+        "0x22a98b27a1d2c4f3cba9d65257d18ee961d6c98f21c7eade37da0543847eb654"
+    )
+
+    liquidations = [
+        Liquidation(
+            liquidated_user="0xacbcf5d2970eef25f02a27e9d9cd31027b058b9b",
+            liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
+            collateral_token_address="0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2",
+            debt_token_address="0x35a18000230da775cac24873d00ff85bccded550",
+            debt_purchase_amount=1106497772527562662,
+            received_amount=910895850496,
+            protocol=Protocol.compound_v2,
+            transaction_hash=transaction_hash,
+            trace_address=[1],
+            block_number=block_number,
+        )
+    ]
+    block = load_test_block(block_number)
+    trace_classifier = TraceClassifier()
+    classified_traces = trace_classifier.classify(block.traces)
+    result = get_compound_liquidations(classified_traces, comp_markets)
+    assert result == liquidations
+
+
+def test_c_token_liquidation():
+    block_number = 13326607
+    transaction_hash = (
+        "0x012215bedd00147c58e1f59807664914b2abbfc13c260190dc9cfc490be3e343"
+    )
+
+    liquidations = [
+        Liquidation(
+            liquidated_user="0xacdd5528c1c92b57045041b5278efa06cdade4d8",
+            liquidator_user="0xe0090ec6895c087a393f0e45f1f85098a6c33bef",
+            collateral_token_address="0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48",
+            debt_token_address="0x70e36f6bf80a52b3b46b3af8e106cc0ed743e8e4",
+            debt_purchase_amount=1207055531,
+            received_amount=21459623305,
+            protocol=Protocol.compound_v2,
+            transaction_hash=transaction_hash,
+            trace_address=[1],
+            block_number=block_number,
+        )
+    ]
+    block = load_test_block(block_number)
+    trace_classifier = TraceClassifier()
+    classified_traces = trace_classifier.classify(block.traces)
+    result = get_compound_liquidations(classified_traces, comp_markets)
+    assert result == liquidations

tests/utils.py

@@ -1,5 +1,6 @@
 import json
 import os
+from typing import Dict
 
 from mev_inspect.schemas.blocks import Block
 
@@ -14,3 +15,10 @@ def load_test_block(block_number: int) -> Block:
     with open(block_path, "r") as block_file:
         block_json = json.load(block_file)
         return Block(**block_json)
+
+
+def load_comp_markets() -> Dict[str, str]:
+    comp_markets_path = f"{THIS_FILE_DIRECTORY}/comp_markets.json"
+    with open(comp_markets_path, "r") as markets_file:
+        markets = json.load(markets_file)
+        return markets