fix cream test

Fix whitespace for isort

.dockerignore

@@ -0,0 +1 @@
+cache

.env

@@ -1,8 +0,0 @@
-# Postgres
-POSTGRES_SERVER=db
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=password
-POSTGRES_DB=mev_inspect
-
-# SQLAlchemy
-SQLALCHEMY_DATABASE_URI=postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_SERVER/$POSTGRES_DB

.github/workflows/github-actions.yml

@@ -51,8 +51,8 @@ jobs:
 
       - name: Run precommit
         run: |
-          poetry run pre-commit
+          poetry run pre-commit run --all-files
 
       - name: Test with pytest
         shell: bash
-        run: poetry run test
+        run: poetry run pytest --cov=mev_inspect tests

.gitignore

@@ -19,3 +19,6 @@ cache
 
 # k8s
 .helm
+
+# env
+.envrc

.pre-commit-config.yaml

@@ -2,8 +2,15 @@ repos:
 -   repo: https://github.com/ambv/black
     rev: 20.8b1
     hooks:
-    - id: black
-      language_version: python3.9
+      - id: black
+        language_version: python3.9
+-   repo: local
+    hooks:
+      - id: isort
+        name: isort
+        entry: poetry run isort .
+        language: system
+        types: [python]
 -   repo: local
     hooks:
     - id: pylint
@@ -18,3 +25,4 @@ repos:
       - id: 'mypy'
         additional_dependencies:
           - 'pydantic'
+          - 'types-requests'

.pylintrc

@@ -433,7 +433,7 @@ int-import-graph=
 known-standard-library=
 
 # Force import order to recognize a module as part of a third party library.
-known-third-party=enchant
+known-third-party=alembic
 
 # Couples of modules and preferred modules, separated by a comma.
 preferred-modules=

CONTRIBUTING.md

@@ -0,0 +1,36 @@
+# Contributing guide
+
+Welcome to the Flashbots collective! We just ask you to be nice when you play with us.
+
+## Pre-commit
+
+We use pre-commit to maintain a consistent style, prevent errors, and ensure test coverage.
+
+To set up, install dependencies through `poetry`:
+
+```
+poetry install
+```
+
+Then install pre-commit hooks with:
+
+```
+poetry run pre-commit install
+```
+
+## Tests
+
+Run tests with:
+
+```
+kubectl exec deploy/mev-inspect-deployment -- poetry run pytest --cov=mev_inspect tests
+```
+
+## Send a pull request
+
+- Your proposed changes should be first described and discussed in an issue.
+- Open the branch in a personal fork, not in the team repository.
+- Every pull request should be small and represent a single change. If the problem is complicated, split it in multiple issues and pull requests.
+- Every pull request should be covered by unit tests.
+
+We appreciate you, friend <3.

Dockerfile

@@ -18,4 +18,5 @@ COPY . /app
 # easter eggs 😝
 RUN echo "PS1='🕵️:\[\033[1;36m\]\h \[\033[1;34m\]\W\[\033[0;35m\]\[\033[1;36m\]$ \[\033[0m\]'" >> ~/.bashrc
 
-CMD ["/bin/bash"]
+ENTRYPOINT [ "poetry" ]
+CMD [ "run", "python", "loop.py" ]

README.md

@@ -1,104 +1,239 @@
-# mev-inspect
-A [WIP] Ethereum MEV Inspector in Python managed by Poetry
+# mev-inspect-py
 
-## Containers
-mev-inspect's local setup is built on [Docker Compose](https://docs.docker.com/compose/)
+[![standard-readme compliant](https://img.shields.io/badge/readme%20style-standard-brightgreen.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme)
+[![Discord](https://img.shields.io/discord/755466764501909692)](https://discord.gg/7hvTycdNcK)
 
-By default it starts up:
-- `mev-inspect` - a container with the code in this repo used for running scripts
-- `db` - a postgres database instance
-- `pgadmin` - a postgres DB UI for querying and more (avaiable at localhost:5050)
+[Maximal extractable value](https://ethereum.org/en/developers/docs/mev/) inspector for Ethereum, to illuminate the [dark forest](https://www.paradigm.xyz/2020/08/ethereum-is-a-dark-forest/) 🌲💡
 
-## Running locally
-Setup [Docker](https://www.docker.com/products/docker-desktop)
-Setup [Poetry](https://python-poetry.org/docs/#osx--linux--bashonwindows-install-instructions)
+Given a block, mev-inspect finds:
+- miner payments (gas + coinbase)
+- tokens transfers and profit
+- swaps and [arbitrages](https://twitter.com/bertcmiller/status/1427632028263059462)
+- ...and more
+
+Data is stored in Postgres for analysis.
+
+## Install
+
+mev-inspect-py is built to run on kubernetes locally and in production.
+
+### Dependencies
+
+- [docker](https://www.docker.com/products/docker-desktop)
+- [kind](https://kind.sigs.k8s.io/docs/user/quick-start), or a similar tool for running local Kubernetes clusters
+- [kubectl](https://kubernetes.io/docs/tasks/tools/)
+- [helm](https://helm.sh/docs/intro/install/)
+- [tilt](https://docs.tilt.dev/install.html)
+
+### Set up
+
+Create a new cluster with:
 
-Install dependencies through poetry
 ```
-poetry install
+kind create cluster
 ```
 
-Start the services (optionally as daemon)
-```
-poetry run start [-d]
-```
+Set an environment variable `RPC_URL` to an RPC for fetching blocks.
 
-Apply the latest migrations against the local DB:
-```
-poetry run exec alembic upgrade head
-``` 
+mev-inspect-py currently requires a node with support for Erigon traces and receipts (not geth yet 😔).
 
-Run inspect on a block
-```
-poetry run inspect -b/--block-number 11931270 -r/--rpc 'http://111.11.11.111:8545/'
-``` 
+[pokt.network](pokt.network)'s "Ethereum Mainnet Archival with trace calls" is a good hosted option.
 
-To stop the services (if running in the background, otherwise just ctrl+c)
-```
-poetry run stop
-```
+Example:
 
-MEV container can be attached via
 ```
-poetry run attach
-```
-
-Running additional compose commands are possible through standard `docker
-compose ...` calls.  Check `docker compose help` for more tools available
-
-## Executing scripts
-Any script can be run from the mev-inspect container like
-```
-poetry run exec <your command here>
-```
-
-For example
-```
-poetry run exec python examples/uniswap_inspect.py -block_number=123 -rpc='111.111.111'
-```
-
-### Poetry Scripts
-```bash
-# code check
-poetry run lint # linting via Pylint
-poetry run test # testing and code coverage with Pytest
-poetry run isort # fixing imports 
-poetry run mypy # type checking 
-poetry run black # style guide 
-poetry run pre-commit run --all-files # runs Black, PyLint and MyPy
-# docker management
-poetry run start [-d] # starts all services, optionally as a daemon
-poetry run stop # shutsdown all services or just ctrl + c if foreground
-poetry run build # rebuilds containers
-poetry run attach # enters the mev-inspect container in interactive mode
-# launches inspection script
-poetry run inspect -b/--block-number 11931270 -r/--rpc 'http://111.11.11.111:8545/'
+export RPC_URL="http://111.111.111.111:8546"
 ```
 
 
-## Rebuilding containers
-After changes to the app's Dockerfile, rebuild with
+Next, start all services with:
+
 ```
-poetry run build
+tilt up
 ```
 
-## Using PGAdmin
+Press "space" to see a browser of the services starting up.
 
-1. Go to [localhost:5050](localhost:5050)
+On first startup, you'll need to apply database migrations with:
 
-2. Login with the PGAdmin username and password in `.env`
+```
+./mev exec alembic upgrade head
+```
 
-3. Add a new engine for mev_inspect with
-    - host: db
-    - user / password: see `.env`
+## Usage
+
+### Inspect a single block
+
+Inspecting block [12914944](https://twitter.com/mevalphaleak/status/1420416437575901185):
+
+```
+./mev inspect 12914944
+```
+
+### Inspect many blocks
+
+Inspecting blocks 12914944 to 12914954:
+
+```
+./mev inspect-many 12914944 12914954
+```
+
+### Inspect all incoming blocks
+
+Start a block listener with:
+
+```
+./mev listener start
+```
+
+By default, it will pick up wherever you left off.
+If running for the first time, listener starts at the latest block.
+
+Tail logs for the listener with:
+
+```
+./mev listener tail
+```
+
+And stop the listener with:
+
+```
+./mev listener stop
+```
+
+### Backfilling
+
+For larger backfills, you can inspect many blocks in parallel using kubernetes
+
+To inspect blocks 12914944 to 12915044 divided across 10 worker pods:
+```
+./mev backfill 12914944 12915044 10
+```
+
+You can see worker pods spin up then complete by watching the status of all pods
+```
+watch kubectl get pods
+```
+
+To watch the logs for a given pod, take its pod name using the above, then run:
+```
+kubectl logs -f pod/mev-inspect-backfill-abcdefg
+```
+
+(where `mev-inspect-backfill-abcdefg` is your actual pod name)
+
+
+### Exploring
+
+All inspect output data is stored in Postgres.
+
+To connect to the local Postgres database for querying, launch a client container with:
+
+```
+./mev db
+```
+
+When you see the prompt:
+
+```
+mev_inspect=#
+```
+
+You're ready to query!
+
+Try finding the total number of swaps decoded with UniswapV3Pool:
+
+```
+SELECT COUNT(*) FROM swaps WHERE abi_name='UniswapV3Pool';
+```
+
+or top 10 arbs by gross profit that took profit in WETH:
+
+```
+SELECT *
+FROM arbitrages
+WHERE profit_token_address = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2'
+ORDER BY profit_amount DESC
+LIMIT 10;
+```
+
+Postgres tip: Enter `\x` to enter "Explanded display" mode which looks nicer for results with many columns.
+
+## FAQ
+
+### How do I delete / reset my local postgres data?
+
+Stop the system if running:
+
+```
+tilt down
+```
+
+Delete it with:
+
+```
+kubectl delete pvc data-postgresql-postgresql-0
+```
+
+Start back up again:
+
+```
+tilt up
+```
+
+And rerun migrations to create the tables again:
+
+```
+./mev exec alembic upgrade head
+```
+
+### I was using the docker-compose setup and want to switch to kube, now what?
+
+Re-add the old `docker-compose.yml` file to your mev-inspect-py directory.
+
+A copy can be found [here](https://github.com/flashbots/mev-inspect-py/blob/ef60c097719629a7d2dc56c6e6c9a100fb706f76/docker-compose.yml)
+
+Tear down docker-compose resources:
+
+```
+docker compose down
+```
+
+Then go through the steps in the current README for kube setup.
+
+### Error from server (AlreadyExists): pods "postgres-client" already exists
+
+This means the postgres client container didn't shut down correctly.
+
+Delete this one with:
+
+```
+kubectl delete pod/postgres-client
+```
+
+Then start it back up again.
+
+## Maintainers
+
+- [@lukevs](https://github.com/lukevs)
+- [@gheise](https://github.com/gheise)
+- [@bertmiller](https://github.com/bertmiller)
 
 ## Contributing
 
-Pre-commit is used to maintain a consistent style, prevent errors and ensure test coverage. 
+[Flashbots](https://flashbots.net) is a research and development collective working on mitigating the negative externalities of decentralized economies. We contribute with the larger free software community to illuminate the dark forest.
 
-Install pre-commit with:
-```
-poetry run pre-commit install
-```
+You are welcome here <3.
 
-Update README if needed
+- If you want to join us, come and say hi in our [Discord chat](https://discord.gg/7hvTycdNcK).
+- If you have a question, feedback or a bug report for this project, please [open a new Issue](https://github.com/flashbots/mev-inspect-py/issues).
+- If you would like to contribute with code, check the [CONTRIBUTING file](CONTRIBUTING.md).
+- We just ask you to be nice.
+
+## Security
+
+If you find a security vulnerability on this project or any other initiative related to Flashbots, please let us know sending an email to security@flashbots.net.
+
+---
+
+Made with ☀️  by the ⚡🤖 collective.

Tiltfile

@@ -1,22 +1,50 @@
-load('ext://helm_remote', 'helm_remote')
+load("ext://helm_remote", "helm_remote")
+load("ext://secret", "secret_from_dict")
+load("ext://configmap", "configmap_from_dict")
+
 helm_remote("postgresql",
-            repo_name='bitnami',
-            repo_url='https://charts.bitnami.com/bitnami',
+            repo_name="bitnami",
+            repo_url="https://charts.bitnami.com/bitnami",
             set=["postgresqlPassword=password", "postgresqlDatabase=mev_inspect"],
 )
 
-load('ext://secret', 'secret_from_dict')
+k8s_yaml(configmap_from_dict("mev-inspect-rpc", inputs = {
+    "url" : os.environ["RPC_URL"],
+}))
+
+k8s_yaml(configmap_from_dict("mev-inspect-listener-healthcheck", inputs = {
+    "url" : os.getenv("LISTENER_HEALTHCHECK_URL", default=""),
+}))
+
 k8s_yaml(secret_from_dict("mev-inspect-db-credentials", inputs = {
     "username" : "postgres",
     "password": "password",
+    "host": "postgresql",
 }))
 
-docker_build('mev-inspect', '.',
+# if using https://github.com/taarushv/trace-db
+# k8s_yaml(secret_from_dict("trace-db-credentials", inputs = {
+#     "username" : "username",
+#     "password": "password",
+#     "host": "trace-db-postgresql",
+# }))
+
+docker_build("mev-inspect-py", ".",
     live_update=[
-        sync('.', '/app'),
-        run('cd /app && poetry install',
-            trigger='./pyproject.toml'),
+        sync(".", "/app"),
+        run("cd /app && poetry install",
+            trigger="./pyproject.toml"),
     ],
 )
+k8s_yaml(helm('./k8s/mev-inspect', name='mev-inspect'))
+k8s_resource(workload="mev-inspect", resource_deps=["postgresql-postgresql"])
 
-k8s_yaml("k8s/app.yaml")
+# uncomment to enable price monitor
+# k8s_yaml(helm('./k8s/mev-inspect-prices', name='mev-inspect-prices'))
+# k8s_resource(workload="mev-inspect-prices", resource_deps=["postgresql-postgresql"])
+
+local_resource(
+    'pg-port-forward',
+    serve_cmd='kubectl port-forward --namespace default svc/postgresql 5432:5432',
+    resource_deps=["postgresql-postgresql"]
+)

alembic/env.py

@@ -1,16 +1,14 @@
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
-
 from alembic import context
+from sqlalchemy import engine_from_config, pool
 
-from mev_inspect.db import get_sqlalchemy_database_uri
+from mev_inspect.db import get_inspect_database_uri
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
-config.set_main_option("sqlalchemy.url", get_sqlalchemy_database_uri())
+config.set_main_option("sqlalchemy.url", get_inspect_database_uri())
 
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.

alembic/versions/04a3bb3740c3_change_miner_payments_and_transfers_.py

@@ -0,0 +1,54 @@
+"""Change miner payments and transfers primary keys to include block number
+
+Revision ID: 04a3bb3740c3
+Revises: a10d68643476
+Create Date: 2021-11-02 22:42:01.702538
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "04a3bb3740c3"
+down_revision = "a10d68643476"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # transfers
+    op.execute("ALTER TABLE transfers DROP CONSTRAINT transfers_pkey")
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+    op.drop_index("ix_transfers_block_number")
+
+    # miner_payments
+    op.execute("ALTER TABLE miner_payments DROP CONSTRAINT miner_payments_pkey")
+    op.create_primary_key(
+        "miner_payments_pkey",
+        "miner_payments",
+        ["block_number", "transaction_hash"],
+    )
+    op.drop_index("ix_block_number")
+
+
+def downgrade():
+    # transfers
+    op.execute("ALTER TABLE transfers DROP CONSTRAINT transfers_pkey")
+    op.create_index("ix_transfers_block_number", "transfers", ["block_number"])
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["transaction_hash", "trace_address"],
+    )
+
+    # miner_payments
+    op.execute("ALTER TABLE miner_payments DROP CONSTRAINT miner_payments_pkey")
+    op.create_index("ix_block_number", "miner_payments", ["block_number"])
+    op.create_primary_key(
+        "miner_payments_pkey",
+        "miner_payments",
+        ["transaction_hash"],
+    )

alembic/versions/04b76ab1d2af_change_blocks_timestamp_to_timestamp.py

@@ -0,0 +1,35 @@
+"""Change blocks.timestamp to timestamp
+
+Revision ID: 04b76ab1d2af
+Revises: 2c90b2b8a80b
+Create Date: 2021-11-26 15:31:21.111693
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "04b76ab1d2af"
+down_revision = "0cef835f7b36"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column(
+        "blocks",
+        "block_timestamp",
+        type_=sa.TIMESTAMP,
+        nullable=False,
+        postgresql_using="TO_TIMESTAMP(block_timestamp)",
+    )
+
+
+def downgrade():
+    op.alter_column(
+        "blocks",
+        "block_timestamp",
+        type_=sa.Numeric,
+        nullable=False,
+        postgresql_using="extract(epoch FROM block_timestamp)",
+    )

alembic/versions/070819d86587_create_punk_snipe.py

@@ -0,0 +1,34 @@
+"""empty message
+
+Revision ID: 070819d86587
+Revises: d498bdb0a641
+Create Date: 2021-11-26 18:25:13.402822
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d498bdb0a641"
+down_revision = "b9fa1ecc9929"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "punk_snipes",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("from_address", sa.String(256), nullable=False),
+        sa.Column("punk_index", sa.Numeric, nullable=False),
+        sa.Column("min_acceptance_price", sa.Numeric, nullable=False),
+        sa.Column("acceptance_price", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("punk_snipes")

alembic/versions/083978d6e455_create_miner_payments_table.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-30 17:42:25.548130
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "083978d6e455"
 down_revision = "92f28a2b4f52"

alembic/versions/0cef835f7b36_rename_pool_address_to_contract_address.py

@@ -0,0 +1,26 @@
+"""Rename pool_address to contract_address
+
+Revision ID: 0cef835f7b36
+Revises: 5427d62a2cc0
+Create Date: 2021-11-19 15:36:15.152622
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "0cef835f7b36"
+down_revision = "5427d62a2cc0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column(
+        "swaps", "pool_address", nullable=False, new_column_name="contract_address"
+    )
+
+
+def downgrade():
+    op.alter_column(
+        "swaps", "contract_address", nullable=False, new_column_name="pool_address"
+    )

alembic/versions/15ba9c27ee8a_add_nullable_transaction_position_field_.py

@@ -0,0 +1,28 @@
+"""Add nullable transaction_position field to swaps and traces
+
+Revision ID: 15ba9c27ee8a
+Revises: 04b76ab1d2af
+Create Date: 2021-12-02 18:24:18.218880
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "15ba9c27ee8a"
+down_revision = "ead7eb8283b9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "classified_traces",
+        sa.Column("transaction_position", sa.Numeric, nullable=True),
+    )
+    op.add_column("swaps", sa.Column("transaction_position", sa.Numeric, nullable=True))
+
+
+def downgrade():
+    op.drop_column("classified_traces", "transaction_position")
+    op.drop_column("swaps", "transaction_position")

alembic/versions/205ce02374b3_add_received_collateral_address_to_liquidations.py

@@ -0,0 +1,26 @@
+"""Add received_collateral_address to liquidations
+
+Revision ID: 205ce02374b3
+Revises: c8363617aa07
+Create Date: 2021-10-04 19:52:40.017084
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "205ce02374b3"
+down_revision = "c8363617aa07"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "liquidations",
+        sa.Column("received_token_address", sa.String(256), nullable=True),
+    )
+
+
+def downgrade():
+    op.drop_column("liquidations", "received_token_address")

alembic/versions/2c90b2b8a80b_add_blocks_table.py

@@ -0,0 +1,28 @@
+"""Add blocks table
+
+Revision ID: 2c90b2b8a80b
+Revises: 04a3bb3740c3
+Create Date: 2021-11-17 18:29:13.065944
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "2c90b2b8a80b"
+down_revision = "04a3bb3740c3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "blocks",
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("block_timestamp", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("block_number"),
+    )
+
+
+def downgrade():
+    op.drop_table("blocks")

alembic/versions/320e56b0a99f_add_index_on_block_number_for_miner_.py

@@ -0,0 +1,22 @@
+"""Add index on block_number for miner_payments
+
+Revision ID: 320e56b0a99f
+Revises: a02f3f2c469f
+Create Date: 2021-09-14 11:11:41.559137
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "320e56b0a99f"
+down_revision = "a02f3f2c469f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_index("ix_block_number", "miner_payments", ["block_number"])
+
+
+def downgrade():
+    op.drop_index("ix_block_number", "miner_payments")

alembic/versions/3417f49d97b3_cahnge_swap_primary_key_to_include_.py

@@ -0,0 +1,45 @@
+"""Cahnge swap primary key to include block number
+
+Revision ID: 3417f49d97b3
+Revises: 205ce02374b3
+Create Date: 2021-11-02 20:50:32.854996
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "3417f49d97b3"
+down_revision = "205ce02374b3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TABLE swaps DROP CONSTRAINT swaps_pkey CASCADE")
+    op.create_primary_key(
+        "swaps_pkey",
+        "swaps",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+    op.create_index(
+        "arbitrage_swaps_swaps_idx",
+        "arbitrage_swaps",
+        ["swap_transaction_hash", "swap_trace_address"],
+    )
+
+
+def downgrade():
+    op.drop_index("arbitrage_swaps_swaps_idx")
+    op.execute("ALTER TABLE swaps DROP CONSTRAINT swaps_pkey CASCADE")
+    op.create_primary_key(
+        "swaps_pkey",
+        "swaps",
+        ["transaction_hash", "trace_address"],
+    )
+    op.create_foreign_key(
+        "arbitrage_swaps_swaps_fkey",
+        "arbitrage_swaps",
+        "swaps",
+        ["swap_transaction_hash", "swap_trace_address"],
+        ["transaction_hash", "trace_address"],
+    )

alembic/versions/52d75a7e0533_add_punk_bid_acceptances.py

@@ -0,0 +1,33 @@
+"""empty message
+
+Revision ID: 52d75a7e0533
+Revises: 7cf0eeb41da0
+Create Date: 2021-11-26 20:35:58.954138
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "52d75a7e0533"
+down_revision = "7cf0eeb41da0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "punk_bid_acceptances",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("from_address", sa.String(256), nullable=False),
+        sa.Column("punk_index", sa.Numeric, nullable=False),
+        sa.Column("min_price", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("punk_bid_acceptances")

alembic/versions/5427d62a2cc0_change_transfers_trace_address_to_array.py

@@ -0,0 +1,46 @@
+"""Change transfers trace address to ARRAY
+
+Revision ID: 5427d62a2cc0
+Revises: d540242ae368
+Create Date: 2021-11-19 13:25:11.252774
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "5427d62a2cc0"
+down_revision = "d540242ae368"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_constraint("transfers_pkey", "transfers")
+    op.alter_column(
+        "transfers",
+        "trace_address",
+        type_=sa.ARRAY(sa.Integer),
+        nullable=False,
+        postgresql_using="trace_address::int[]",
+    )
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+
+
+def downgrade():
+    op.drop_constraint("transfers_pkey", "transfers")
+    op.alter_column(
+        "transfers",
+        "trace_address",
+        type_=sa.String(256),
+        nullable=False,
+    )
+    op.create_primary_key(
+        "transfers_pkey",
+        "transfers",
+        ["block_number", "transaction_hash", "trace_address"],
+    )

alembic/versions/7cf0eeb41da0_add_punk_bids.py

@@ -0,0 +1,33 @@
+"""empty message
+
+Revision ID: 7cf0eeb41da0
+Revises: d498bdb0a641
+Create Date: 2021-11-26 20:27:28.936516
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "7cf0eeb41da0"
+down_revision = "d498bdb0a641"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "punk_bids",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("from_address", sa.String(256), nullable=False),
+        sa.Column("punk_index", sa.Numeric, nullable=False),
+        sa.Column("price", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("block_number", "transaction_hash", "trace_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("punk_bids")

alembic/versions/7eec417a4f3e_change_trace_addresses_to_array_types.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-06 15:58:04.556762
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "7eec417a4f3e"
 down_revision = "9d8c69b3dccb"

alembic/versions/92f28a2b4f52_add_error_column_to_swaps.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-17 03:46:21.498821
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "92f28a2b4f52"
 down_revision = "9b8ae51c5d56"

alembic/versions/9b8ae51c5d56_add_swap_arbitrage_join_table.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-06 17:06:55.364516
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "9b8ae51c5d56"
 down_revision = "7eec417a4f3e"

alembic/versions/9d8c69b3dccb_add_arbitrages_and_swap_join_table.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-05 21:46:35.209199
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "9d8c69b3dccb"
 down_revision = "2116e2f36a19"

alembic/versions/a02f3f2c469f_create_latest_block_table.py

@@ -0,0 +1,27 @@
+"""Create latest block table
+
+Revision ID: a02f3f2c469f
+Revises: d70c08b4db6f
+Create Date: 2021-09-13 21:32:27.181344
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a02f3f2c469f"
+down_revision = "d70c08b4db6f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "latest_block_update",
+        sa.Column("block_number", sa.Numeric, primary_key=True),
+        sa.Column("updated_at", sa.TIMESTAMP, server_default=sa.func.now()),
+    )
+
+
+def downgrade():
+    op.drop_table("latest_block_update")

alembic/versions/a10d68643476_change_classified_traces_primary_key_to_.py

@@ -0,0 +1,34 @@
+"""Change classified traces primary key to include block number
+
+Revision ID: a10d68643476
+Revises: 3417f49d97b3
+Create Date: 2021-11-02 22:03:26.312317
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "a10d68643476"
+down_revision = "3417f49d97b3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TABLE classified_traces DROP CONSTRAINT classified_traces_pkey")
+    op.create_primary_key(
+        "classified_traces_pkey",
+        "classified_traces",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+    op.drop_index("i_block_number")
+
+
+def downgrade():
+    op.execute("ALTER TABLE classified_traces DROP CONSTRAINT classified_traces_pkey")
+    op.create_index("i_block_number", "classified_traces", ["block_number"])
+    op.create_primary_key(
+        "classified_traces_pkey",
+        "classified_traces",
+        ["transaction_hash", "trace_address"],
+    )

alembic/versions/b9fa1ecc9929_remove_liq_column.py

@@ -0,0 +1,26 @@
+"""Remove collateral_token_address column
+
+Revision ID: b9fa1ecc9929
+Revises: 04b76ab1d2af
+Create Date: 2021-12-01 23:32:40.574108
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b9fa1ecc9929"
+down_revision = "04b76ab1d2af"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.drop_column("liquidations", "collateral_token_address")
+
+
+def downgrade():
+    op.add_column(
+        "liquidations",
+        sa.Column("collateral_token_address", sa.String(256), nullable=False),
+    )

alembic/versions/c5da44eb072c_add_index_for_classified_traces_block_.py

@@ -7,7 +7,6 @@ Create Date: 2021-07-30 17:37:27.335475
 """
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "c5da44eb072c"
 down_revision = "0660432b9840"

alembic/versions/c8363617aa07_create_liquidations_table.py

@@ -0,0 +1,37 @@
+"""Create liquidations table
+
+Revision ID: c8363617aa07
+Revises: cd96af55108e
+Create Date: 2021-09-29 14:00:06.857103
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c8363617aa07"
+down_revision = "cd96af55108e"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "liquidations",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("liquidated_user", sa.String(256), nullable=False),
+        sa.Column("liquidator_user", sa.String(256), nullable=False),
+        sa.Column("collateral_token_address", sa.String(256), nullable=False),
+        sa.Column("debt_token_address", sa.String(256), nullable=False),
+        sa.Column("debt_purchase_amount", sa.Numeric, nullable=False),
+        sa.Column("received_amount", sa.Numeric, nullable=False),
+        sa.Column("protocol", sa.String(256), nullable=True),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.PrimaryKeyConstraint("transaction_hash", "trace_address"),
+    )
+
+
+def downgrade():
+    op.drop_table("liquidations")

alembic/versions/cd96af55108e_add_transfers_table.py

@@ -0,0 +1,38 @@
+"""Add transfers table
+
+Revision ID: cd96af55108e
+Revises: 5437dc68f4df
+Create Date: 2021-09-17 12:44:45.245137
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "cd96af55108e"
+down_revision = "320e56b0a99f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "transfers",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("transaction_hash", sa.String(66), nullable=False),
+        sa.Column("trace_address", sa.String(256), nullable=False),
+        sa.Column("protocol", sa.String(256), nullable=True),
+        sa.Column("from_address", sa.String(256), nullable=False),
+        sa.Column("to_address", sa.String(256), nullable=False),
+        sa.Column("token_address", sa.String(256), nullable=False),
+        sa.Column("amount", sa.Numeric, nullable=False),
+        sa.Column("error", sa.String(256), nullable=True),
+        sa.PrimaryKeyConstraint("transaction_hash", "trace_address"),
+    )
+    op.create_index("ix_transfers_block_number", "transfers", ["block_number"])
+
+
+def downgrade():
+    op.drop_index("ix_transfers_block_number", "transfers")
+    op.drop_table("transfers")

alembic/versions/d540242ae368_create_usd_prices_table.py

@@ -0,0 +1,29 @@
+"""Create usd_prices table
+
+Revision ID: d540242ae368
+Revises: 2c90b2b8a80b
+Create Date: 2021-11-18 04:30:06.802857
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "d540242ae368"
+down_revision = "2c90b2b8a80b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "prices",
+        sa.Column("timestamp", sa.TIMESTAMP),
+        sa.Column("usd_price", sa.Numeric, nullable=False),
+        sa.Column("token_address", sa.String(256), nullable=False),
+        sa.PrimaryKeyConstraint("token_address", "timestamp"),
+    )
+
+
+def downgrade():
+    op.drop_table("prices")

alembic/versions/d70c08b4db6f_add_to_address_and_from_address_to_.py

@@ -8,7 +8,6 @@ Create Date: 2021-08-30 22:10:04.186251
 import sqlalchemy as sa
 from alembic import op
 
-
 # revision identifiers, used by Alembic.
 revision = "d70c08b4db6f"
 down_revision = "083978d6e455"

alembic/versions/ead7eb8283b9_create_sandwiches_and_sandwiched_swaps_.py

@@ -0,0 +1,69 @@
+"""Create sandwiches and sandwiched swaps tables
+
+Revision ID: ead7eb8283b9
+Revises: a5d80460f0e6
+Create Date: 2021-12-03 16:37:28.077158
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "ead7eb8283b9"
+down_revision = "52d75a7e0533"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "sandwiches",
+        sa.Column("id", sa.String(256), primary_key=True),
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("block_number", sa.Numeric, nullable=False),
+        sa.Column("sandwicher_address", sa.String(256), nullable=False),
+        sa.Column("frontrun_swap_transaction_hash", sa.String(256), nullable=False),
+        sa.Column("frontrun_swap_trace_address", sa.ARRAY(sa.Integer), nullable=False),
+        sa.Column("backrun_swap_transaction_hash", sa.String(256), nullable=False),
+        sa.Column("backrun_swap_trace_address", sa.ARRAY(sa.Integer), nullable=False),
+    )
+
+    op.create_index(
+        "ik_sandwiches_frontrun",
+        "sandwiches",
+        [
+            "block_number",
+            "frontrun_swap_transaction_hash",
+            "frontrun_swap_trace_address",
+        ],
+    )
+
+    op.create_index(
+        "ik_sandwiches_backrun",
+        "sandwiches",
+        ["block_number", "backrun_swap_transaction_hash", "backrun_swap_trace_address"],
+    )
+
+    op.create_table(
+        "sandwiched_swaps",
+        sa.Column("created_at", sa.TIMESTAMP, server_default=sa.func.now()),
+        sa.Column("sandwich_id", sa.String(1024), primary_key=True),
+        sa.Column("block_number", sa.Numeric, primary_key=True),
+        sa.Column("transaction_hash", sa.String(66), primary_key=True),
+        sa.Column("trace_address", sa.ARRAY(sa.Integer), primary_key=True),
+        sa.ForeignKeyConstraint(["sandwich_id"], ["sandwiches.id"], ondelete="CASCADE"),
+    )
+
+    op.create_index(
+        "ik_sandwiched_swaps_secondary",
+        "sandwiched_swaps",
+        ["block_number", "transaction_hash", "trace_address"],
+    )
+
+
+def downgrade():
+    op.drop_index("ik_sandwiched_swaps_secondary")
+    op.drop_table("sandwiched_swaps")
+    op.drop_index("ik_sandwiches_frontrun")
+    op.drop_index("ik_sandwiches_backrun")
+    op.drop_table("sandwiches")

backfill.py

@@ -0,0 +1,57 @@
+import subprocess
+import sys
+from typing import Iterator, Tuple
+
+
+def get_block_after_before_chunks(
+    after_block: int,
+    before_block: int,
+    n_workers: int,
+) -> Iterator[Tuple[int, int]]:
+    n_blocks = before_block - after_block
+    remainder = n_blocks % n_workers
+    floor_chunk_size = n_blocks // n_workers
+
+    last_before_block = None
+
+    for worker_index in range(n_workers):
+        chunk_size = floor_chunk_size
+
+        if worker_index < remainder:
+            chunk_size += 1
+
+        batch_after_block = (
+            last_before_block if last_before_block is not None else after_block
+        )
+
+        batch_before_block = batch_after_block + chunk_size
+        yield batch_after_block, batch_before_block
+        last_before_block = batch_before_block
+
+
+def backfill(after_block: int, before_block: int, n_workers: int):
+    if n_workers <= 0:
+        raise ValueError("Need at least one worker")
+
+    for batch_after_block, batch_before_block in get_block_after_before_chunks(
+        after_block,
+        before_block,
+        n_workers,
+    ):
+        print(f"Backfilling {batch_after_block} to {batch_before_block}")
+        backfill_command = f"sh backfill.sh {batch_after_block} {batch_before_block}"
+        process = subprocess.Popen(backfill_command.split(), stdout=subprocess.PIPE)
+        output, _ = process.communicate()
+        print(output)
+
+
+def main():
+    after_block = int(sys.argv[1])
+    before_block = int(sys.argv[2])
+    n_workers = int(sys.argv[3])
+
+    backfill(after_block, before_block, n_workers)
+
+
+if __name__ == "__main__":
+    main()

backfill.sh

@@ -0,0 +1,6 @@
+current_image=$(kubectl get deployment mev-inspect -o=jsonpath='{$.spec.template.spec.containers[:1].image}')
+
+helm template mev-inspect-backfill ./k8s/mev-inspect-backfill \
+    --set image.repository=$current_image \
+    --set command.startBlockNumber=$1 \
+    --set command.endBlockNumber=$2 | kubectl apply -f -

cli.py

@@ -0,0 +1,102 @@
+import logging
+import os
+import sys
+
+import click
+
+from mev_inspect.concurrency import coro
+from mev_inspect.crud.prices import write_prices
+from mev_inspect.db import get_inspect_session, get_trace_session
+from mev_inspect.inspector import MEVInspector
+from mev_inspect.prices import fetch_all_supported_prices
+
+RPC_URL_ENV = "RPC_URL"
+
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+@click.group()
+def cli():
+    pass
+
+
+@cli.command()
+@click.argument("block_number", type=int)
+@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
+@coro
+async def inspect_block_command(block_number: int, rpc: str):
+    inspect_db_session = get_inspect_session()
+    trace_db_session = get_trace_session()
+
+    inspector = MEVInspector(rpc, inspect_db_session, trace_db_session)
+    await inspector.inspect_single_block(block=block_number)
+
+
+@cli.command()
+@click.argument("block_number", type=int)
+@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
+@coro
+async def fetch_block_command(block_number: int, rpc: str):
+    inspect_db_session = get_inspect_session()
+    trace_db_session = get_trace_session()
+
+    inspector = MEVInspector(rpc, inspect_db_session, trace_db_session)
+    block = await inspector.create_from_block(block_number=block_number)
+    print(block.json())
+
+
+@cli.command()
+@click.argument("after_block", type=int)
+@click.argument("before_block", type=int)
+@click.option("--rpc", default=lambda: os.environ.get(RPC_URL_ENV, ""))
+@click.option(
+    "--max-concurrency",
+    type=int,
+    help="maximum number of concurrent connections",
+    default=5,
+)
+@click.option(
+    "--request-timeout", type=int, help="timeout for requests to nodes", default=500
+)
+@coro
+async def inspect_many_blocks_command(
+    after_block: int,
+    before_block: int,
+    rpc: str,
+    max_concurrency: int,
+    request_timeout: int,
+):
+    inspect_db_session = get_inspect_session()
+    trace_db_session = get_trace_session()
+
+    inspector = MEVInspector(
+        rpc,
+        inspect_db_session,
+        trace_db_session,
+        max_concurrency=max_concurrency,
+        request_timeout=request_timeout,
+    )
+    await inspector.inspect_many_blocks(
+        after_block=after_block, before_block=before_block
+    )
+
+
+@cli.command()
+@coro
+async def fetch_all_prices():
+    inspect_db_session = get_inspect_session()
+
+    logger.info("Fetching prices")
+    prices = await fetch_all_supported_prices()
+
+    logger.info("Writing prices")
+    write_prices(inspect_db_session, prices)
+
+
+def get_rpc_url() -> str:
+    return os.environ["RPC_URL"]
+
+
+if __name__ == "__main__":
+    cli()

docker-compose.yml

@@ -1,24 +0,0 @@
-services:
-  mev-inspect:
-    build: .
-    depends_on:
-      - db
-    env_file:
-      - .env
-    volumes:
-      - .:/app
-    tty: true
-
-  db:
-    image: postgres:12
-    volumes:
-      - mev-inspect-db-data:/var/lib/postgresql/data/pgdata
-    env_file:
-      - .env
-    environment:
-      - PGDATA=/var/lib/postgresql/data/pgdata
-    ports:
-      - 5432:5432
-
-volumes:
-  mev-inspect-db-data:

get_helm.sh

@@ -1,326 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright The Helm Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# The install script is based off of the MIT-licensed script from glide,
-# the package manager for Go: https://github.com/Masterminds/glide.sh/blob/master/get
-
-: ${BINARY_NAME:="helm"}
-: ${USE_SUDO:="true"}
-: ${DEBUG:="false"}
-: ${VERIFY_CHECKSUM:="true"}
-: ${VERIFY_SIGNATURES:="false"}
-: ${HELM_INSTALL_DIR:="/usr/local/bin"}
-: ${GPG_PUBRING:="pubring.kbx"}
-
-HAS_CURL="$(type "curl" &> /dev/null && echo true || echo false)"
-HAS_WGET="$(type "wget" &> /dev/null && echo true || echo false)"
-HAS_OPENSSL="$(type "openssl" &> /dev/null && echo true || echo false)"
-HAS_GPG="$(type "gpg" &> /dev/null && echo true || echo false)"
-
-# initArch discovers the architecture for this system.
-initArch() {
-  ARCH=$(uname -m)
-  case $ARCH in
-    armv5*) ARCH="armv5";;
-    armv6*) ARCH="armv6";;
-    armv7*) ARCH="arm";;
-    aarch64) ARCH="arm64";;
-    x86) ARCH="386";;
-    x86_64) ARCH="amd64";;
-    i686) ARCH="386";;
-    i386) ARCH="386";;
-  esac
-}
-
-# initOS discovers the operating system for this system.
-initOS() {
-  OS=$(echo `uname`|tr '[:upper:]' '[:lower:]')
-
-  case "$OS" in
-    # Minimalist GNU for Windows
-    mingw*) OS='windows';;
-  esac
-}
-
-# runs the given command as root (detects if we are root already)
-runAsRoot() {
-  if [ $EUID -ne 0 -a "$USE_SUDO" = "true" ]; then
-    sudo "${@}"
-  else
-    "${@}"
-  fi
-}
-
-# verifySupported checks that the os/arch combination is supported for
-# binary builds, as well whether or not necessary tools are present.
-verifySupported() {
-  local supported="darwin-amd64\ndarwin-arm64\nlinux-386\nlinux-amd64\nlinux-arm\nlinux-arm64\nlinux-ppc64le\nlinux-s390x\nwindows-amd64"
-  if ! echo "${supported}" | grep -q "${OS}-${ARCH}"; then
-    echo "No prebuilt binary for ${OS}-${ARCH}."
-    echo "To build from source, go to https://github.com/helm/helm"
-    exit 1
-  fi
-
-  if [ "${HAS_CURL}" != "true" ] && [ "${HAS_WGET}" != "true" ]; then
-    echo "Either curl or wget is required"
-    exit 1
-  fi
-
-  if [ "${VERIFY_CHECKSUM}" == "true" ] && [ "${HAS_OPENSSL}" != "true" ]; then
-    echo "In order to verify checksum, openssl must first be installed."
-    echo "Please install openssl or set VERIFY_CHECKSUM=false in your environment."
-    exit 1
-  fi
-
-  if [ "${VERIFY_SIGNATURES}" == "true" ]; then
-    if [ "${HAS_GPG}" != "true" ]; then
-      echo "In order to verify signatures, gpg must first be installed."
-      echo "Please install gpg or set VERIFY_SIGNATURES=false in your environment."
-      exit 1
-    fi
-    if [ "${OS}" != "linux" ]; then
-      echo "Signature verification is currently only supported on Linux."
-      echo "Please set VERIFY_SIGNATURES=false or verify the signatures manually."
-      exit 1
-    fi
-  fi
-}
-
-# checkDesiredVersion checks if the desired version is available.
-checkDesiredVersion() {
-  if [ "x$DESIRED_VERSION" == "x" ]; then
-    # Get tag from release URL
-    local latest_release_url="https://github.com/helm/helm/releases"
-    if [ "${HAS_CURL}" == "true" ]; then
-      TAG=$(curl -Ls $latest_release_url | grep 'href="/helm/helm/releases/tag/v3.[0-9]*.[0-9]*\"' | grep -v no-underline | head -n 1 | cut -d '"' -f 2 | awk '{n=split($NF,a,"/");print a[n]}' | awk 'a !~ $0{print}; {a=$0}')
-    elif [ "${HAS_WGET}" == "true" ]; then
-      TAG=$(wget $latest_release_url -O - 2>&1 | grep 'href="/helm/helm/releases/tag/v3.[0-9]*.[0-9]*\"' | grep -v no-underline | head -n 1 | cut -d '"' -f 2 | awk '{n=split($NF,a,"/");print a[n]}' | awk 'a !~ $0{print}; {a=$0}')
-    fi
-  else
-    TAG=$DESIRED_VERSION
-  fi
-}
-
-# checkHelmInstalledVersion checks which version of helm is installed and
-# if it needs to be changed.
-checkHelmInstalledVersion() {
-  if [[ -f "${HELM_INSTALL_DIR}/${BINARY_NAME}" ]]; then
-    local version=$("${HELM_INSTALL_DIR}/${BINARY_NAME}" version --template="{{ .Version }}")
-    if [[ "$version" == "$TAG" ]]; then
-      echo "Helm ${version} is already ${DESIRED_VERSION:-latest}"
-      return 0
-    else
-      echo "Helm ${TAG} is available. Changing from version ${version}."
-      return 1
-    fi
-  else
-    return 1
-  fi
-}
-
-# downloadFile downloads the latest binary package and also the checksum
-# for that binary.
-downloadFile() {
-  HELM_DIST="helm-$TAG-$OS-$ARCH.tar.gz"
-  DOWNLOAD_URL="https://get.helm.sh/$HELM_DIST"
-  CHECKSUM_URL="$DOWNLOAD_URL.sha256"
-  HELM_TMP_ROOT="$(mktemp -dt helm-installer-XXXXXX)"
-  HELM_TMP_FILE="$HELM_TMP_ROOT/$HELM_DIST"
-  HELM_SUM_FILE="$HELM_TMP_ROOT/$HELM_DIST.sha256"
-  echo "Downloading $DOWNLOAD_URL"
-  if [ "${HAS_CURL}" == "true" ]; then
-    curl -SsL "$CHECKSUM_URL" -o "$HELM_SUM_FILE"
-    curl -SsL "$DOWNLOAD_URL" -o "$HELM_TMP_FILE"
-  elif [ "${HAS_WGET}" == "true" ]; then
-    wget -q -O "$HELM_SUM_FILE" "$CHECKSUM_URL"
-    wget -q -O "$HELM_TMP_FILE" "$DOWNLOAD_URL"
-  fi
-}
-
-# verifyFile verifies the SHA256 checksum of the binary package
-# and the GPG signatures for both the package and checksum file
-# (depending on settings in environment).
-verifyFile() {
-  if [ "${VERIFY_CHECKSUM}" == "true" ]; then
-    verifyChecksum
-  fi
-  if [ "${VERIFY_SIGNATURES}" == "true" ]; then
-    verifySignatures
-  fi
-}
-
-# installFile installs the Helm binary.
-installFile() {
-  HELM_TMP="$HELM_TMP_ROOT/$BINARY_NAME"
-  mkdir -p "$HELM_TMP"
-  tar xf "$HELM_TMP_FILE" -C "$HELM_TMP"
-  HELM_TMP_BIN="$HELM_TMP/$OS-$ARCH/helm"
-  echo "Preparing to install $BINARY_NAME into ${HELM_INSTALL_DIR}"
-  runAsRoot cp "$HELM_TMP_BIN" "$HELM_INSTALL_DIR/$BINARY_NAME"
-  echo "$BINARY_NAME installed into $HELM_INSTALL_DIR/$BINARY_NAME"
-}
-
-# verifyChecksum verifies the SHA256 checksum of the binary package.
-verifyChecksum() {
-  printf "Verifying checksum... "
-  local sum=$(openssl sha1 -sha256 ${HELM_TMP_FILE} | awk '{print $2}')
-  local expected_sum=$(cat ${HELM_SUM_FILE})
-  if [ "$sum" != "$expected_sum" ]; then
-    echo "SHA sum of ${HELM_TMP_FILE} does not match. Aborting."
-    exit 1
-  fi
-  echo "Done."
-}
-
-# verifySignatures obtains the latest KEYS file from GitHub main branch
-# as well as the signature .asc files from the specific GitHub release,
-# then verifies that the release artifacts were signed by a maintainer's key.
-verifySignatures() {
-  printf "Verifying signatures... "
-  local keys_filename="KEYS"
-  local github_keys_url="https://raw.githubusercontent.com/helm/helm/main/${keys_filename}"
-  if [ "${HAS_CURL}" == "true" ]; then
-    curl -SsL "${github_keys_url}" -o "${HELM_TMP_ROOT}/${keys_filename}"
-  elif [ "${HAS_WGET}" == "true" ]; then
-    wget -q -O "${HELM_TMP_ROOT}/${keys_filename}" "${github_keys_url}"
-  fi
-  local gpg_keyring="${HELM_TMP_ROOT}/keyring.gpg"
-  local gpg_homedir="${HELM_TMP_ROOT}/gnupg"
-  mkdir -p -m 0700 "${gpg_homedir}"
-  local gpg_stderr_device="/dev/null"
-  if [ "${DEBUG}" == "true" ]; then
-    gpg_stderr_device="/dev/stderr"
-  fi
-  gpg --batch --quiet --homedir="${gpg_homedir}" --import "${HELM_TMP_ROOT}/${keys_filename}" 2> "${gpg_stderr_device}"
-  gpg --batch --no-default-keyring --keyring "${gpg_homedir}/${GPG_PUBRING}" --export > "${gpg_keyring}"
-  local github_release_url="https://github.com/helm/helm/releases/download/${TAG}"
-  if [ "${HAS_CURL}" == "true" ]; then
-    curl -SsL "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" -o "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc"
-    curl -SsL "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" -o "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc"
-  elif [ "${HAS_WGET}" == "true" ]; then
-    wget -q -O "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc"
-    wget -q -O "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" "${github_release_url}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc"
-  fi
-  local error_text="If you think this might be a potential security issue,"
-  error_text="${error_text}\nplease see here: https://github.com/helm/community/blob/master/SECURITY.md"
-  local num_goodlines_sha=$(gpg --verify --keyring="${gpg_keyring}" --status-fd=1 "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256.asc" 2> "${gpg_stderr_device}" | grep -c -E '^\[GNUPG:\] (GOODSIG|VALIDSIG)')
-  if [[ ${num_goodlines_sha} -lt 2 ]]; then
-    echo "Unable to verify the signature of helm-${TAG}-${OS}-${ARCH}.tar.gz.sha256!"
-    echo -e "${error_text}"
-    exit 1
-  fi
-  local num_goodlines_tar=$(gpg --verify --keyring="${gpg_keyring}" --status-fd=1 "${HELM_TMP_ROOT}/helm-${TAG}-${OS}-${ARCH}.tar.gz.asc" 2> "${gpg_stderr_device}" | grep -c -E '^\[GNUPG:\] (GOODSIG|VALIDSIG)')
-  if [[ ${num_goodlines_tar} -lt 2 ]]; then
-    echo "Unable to verify the signature of helm-${TAG}-${OS}-${ARCH}.tar.gz!"
-    echo -e "${error_text}"
-    exit 1
-  fi
-  echo "Done."
-}
-
-# fail_trap is executed if an error occurs.
-fail_trap() {
-  result=$?
-  if [ "$result" != "0" ]; then
-    if [[ -n "$INPUT_ARGUMENTS" ]]; then
-      echo "Failed to install $BINARY_NAME with the arguments provided: $INPUT_ARGUMENTS"
-      help
-    else
-      echo "Failed to install $BINARY_NAME"
-    fi
-    echo -e "\tFor support, go to https://github.com/helm/helm."
-  fi
-  cleanup
-  exit $result
-}
-
-# testVersion tests the installed client to make sure it is working.
-testVersion() {
-  set +e
-  HELM="$(command -v $BINARY_NAME)"
-  if [ "$?" = "1" ]; then
-    echo "$BINARY_NAME not found. Is $HELM_INSTALL_DIR on your "'$PATH?'
-    exit 1
-  fi
-  set -e
-}
-
-# help provides possible cli installation arguments
-help () {
-  echo "Accepted cli arguments are:"
-  echo -e "\t[--help|-h ] ->> prints this help"
-  echo -e "\t[--version|-v <desired_version>] . When not defined it fetches the latest release from GitHub"
-  echo -e "\te.g. --version v3.0.0 or -v canary"
-  echo -e "\t[--no-sudo]  ->> install without sudo"
-}
-
-# cleanup temporary files to avoid https://github.com/helm/helm/issues/2977
-cleanup() {
-  if [[ -d "${HELM_TMP_ROOT:-}" ]]; then
-    rm -rf "$HELM_TMP_ROOT"
-  fi
-}
-
-# Execution
-
-#Stop execution on any error
-trap "fail_trap" EXIT
-set -e
-
-# Set debug if desired
-if [ "${DEBUG}" == "true" ]; then
-  set -x
-fi
-
-# Parsing input arguments (if any)
-export INPUT_ARGUMENTS="${@}"
-set -u
-while [[ $# -gt 0 ]]; do
-  case $1 in
-    '--version'|-v)
-       shift
-       if [[ $# -ne 0 ]]; then
-           export DESIRED_VERSION="${1}"
-       else
-           echo -e "Please provide the desired version. e.g. --version v3.0.0 or -v canary"
-           exit 0
-       fi
-       ;;
-    '--no-sudo')
-       USE_SUDO="false"
-       ;;
-    '--help'|-h)
-       help
-       exit 0
-       ;;
-    *) exit 1
-       ;;
-  esac
-  shift
-done
-set +u
-
-initArch
-initOS
-verifySupported
-checkDesiredVersion
-if ! checkHelmInstalledVersion; then
-  downloadFile
-  verifyFile
-  installFile
-fi
-testVersion
-cleanup

k8s/app.yaml

@@ -1,39 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: mev-inspect-deployment
-  labels:
-    app: mev-inspect
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: mev-inspect
-  template:
-    metadata:
-      labels:
-        app: mev-inspect
-    spec:
-      containers:
-      - name: mev-inspect
-        image: mev-inspect:latest
-        command: [ "/bin/bash", "-c", "--" ]
-        args: [ "while true; do sleep 30; done;" ]
-        env:
-        - name: POSTGRES_USER
-          valueFrom:
-            secretKeyRef:
-              name: mev-inspect-db-credentials
-              key: username
-        - name: POSTGRES_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: mev-inspect-db-credentials
-              key: password
-    livenessProbe:
-      exec:
-        command:
-        - ls
-        - /
-      initialDelaySeconds: 20
-      periodSeconds: 5

k8s/mev-inspect-backfill/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8s/mev-inspect-backfill/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: mev-inspect-backfill
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

k8s/mev-inspect-backfill/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mev-inspect-backfill.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mev-inspect-backfill.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mev-inspect-backfill.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "mev-inspect-backfill.labels" -}}
+helm.sh/chart: {{ include "mev-inspect-backfill.chart" . }}
+{{ include "mev-inspect-backfill.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "mev-inspect-backfill.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "mev-inspect-backfill.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mev-inspect-backfill.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "mev-inspect-backfill.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

k8s/mev-inspect-backfill/templates/job.yaml

@@ -0,0 +1,68 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "mev-inspect-backfill.fullname" . }}-{{ randAlphaNum 5 | lower }}
+  labels:
+    {{- include "mev-inspect-backfill.labels" . | nindent 4 }}
+spec:
+  completions: 1
+  parallelism: 1
+  ttlSecondsAfterFinished: 5
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    spec:
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            - run
+            - inspect-many-blocks
+            - {{ .Values.command.startBlockNumber | quote }}
+            - {{ .Values.command.endBlockNumber | quote }}
+          env:
+          - name: POSTGRES_HOST
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: host
+          - name: POSTGRES_USER
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: username
+          - name: POSTGRES_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: password
+          - name: TRACE_DB_HOST
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: host
+                optional: true
+          - name: TRACE_DB_USER
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: username
+                optional: true
+          - name: TRACE_DB_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: password
+                optional: true
+          - name: RPC_URL
+            valueFrom:
+              configMapKeyRef:
+                name: mev-inspect-rpc
+                key: url
+      restartPolicy: OnFailure

k8s/mev-inspect-backfill/values.yaml

@@ -0,0 +1,42 @@
+# Default values for mev-inspect.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+image:
+  repository: mev-inspect-py
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

k8s/mev-inspect-prices/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8s/mev-inspect-prices/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: mev-inspect-prices
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

k8s/mev-inspect-prices/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mev-inspect-prices.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mev-inspect-prices.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mev-inspect-prices.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "mev-inspect-prices.labels" -}}
+helm.sh/chart: {{ include "mev-inspect-prices.chart" . }}
+{{ include "mev-inspect-prices.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "mev-inspect-prices.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "mev-inspect-prices.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mev-inspect-prices.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "mev-inspect-prices.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

k8s/mev-inspect-prices/templates/cronjob.yaml

@@ -0,0 +1,35 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: {{ include "mev-inspect-prices.fullname" . }}
+spec:
+  schedule: "0 */1 * * *"
+  successfulJobsHistoryLimit: 0
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+            - name: {{ .Chart.Name }}
+              image: "{{ .Values.image.repository }}"
+              imagePullPolicy: {{ .Values.image.pullPolicy }}
+              args:
+              - run
+              - fetch-all-prices
+              env:
+              - name: POSTGRES_HOST
+                valueFrom:
+                  secretKeyRef:
+                    name: mev-inspect-db-credentials
+                    key: host
+              - name: POSTGRES_USER
+                valueFrom:
+                  secretKeyRef:
+                    name: mev-inspect-db-credentials
+                    key: username
+              - name: POSTGRES_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: mev-inspect-db-credentials
+                    key: password
+          restartPolicy: Never

k8s/mev-inspect-prices/values.yaml

@@ -0,0 +1,7 @@
+image:
+  repository: mev-inspect-py
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""

k8s/mev-inspect/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

k8s/mev-inspect/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: mev-inspect
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

k8s/mev-inspect/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mev-inspect.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mev-inspect.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mev-inspect.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "mev-inspect.labels" -}}
+helm.sh/chart: {{ include "mev-inspect.chart" . }}
+{{ include "mev-inspect.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "mev-inspect.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "mev-inspect.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mev-inspect.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "mev-inspect.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

k8s/mev-inspect/templates/deployment.yaml

@@ -0,0 +1,99 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "mev-inspect.fullname" . }}
+  labels:
+    {{- include "mev-inspect.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "mev-inspect.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "mev-inspect.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args: ["run", "python", "loop.py"]
+          livenessProbe:
+            exec:
+              command:
+              - ls
+              - /
+            initialDelaySeconds: 20
+            periodSeconds: 5
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          env:
+          - name: POSTGRES_HOST
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: host
+          - name: POSTGRES_USER
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: username
+          - name: POSTGRES_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: mev-inspect-db-credentials
+                key: password
+          - name: TRACE_DB_HOST
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: host
+                optional: true
+          - name: TRACE_DB_USER
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: username
+                optional: true
+          - name: TRACE_DB_PASSWORD
+            valueFrom:
+              secretKeyRef:
+                name: trace-db-credentials
+                key: password
+                optional: true
+          - name: RPC_URL
+            valueFrom:
+              configMapKeyRef:
+                name: mev-inspect-rpc
+                key: url
+          - name: LISTENER_HEALTHCHECK_URL
+            valueFrom:
+              configMapKeyRef:
+                name: mev-inspect-listener-healthcheck
+                key: url
+                optional: true
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

k8s/mev-inspect/values.yaml

@@ -0,0 +1,44 @@
+# Default values for mev-inspect.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: mev-inspect-py:latest
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

listener

@@ -0,0 +1,50 @@
+#!/bin/sh
+
+set -e
+
+NAME=listener
+PIDFILE=/var/run/$NAME.pid
+DAEMON=/root/.poetry/bin/poetry
+DAEMON_OPTS="run python listener.py"
+
+case "$1" in
+  start)
+        echo -n "Starting daemon: "$NAME
+	start-stop-daemon \
+        --background \
+        --chdir /app \
+        --start \
+        --quiet \
+        --pidfile $PIDFILE \
+        --make-pidfile \
+        --startas $DAEMON -- $DAEMON_OPTS
+        echo "."
+	;;
+  stop)
+        echo -n "Stopping daemon: "$NAME
+	start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE
+        echo "."
+	;;
+  tail)
+        tail -f listener.log
+	;;
+  restart)
+        echo -n "Restarting daemon: "$NAME
+	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile $PIDFILE
+	start-stop-daemon \
+        --background \
+        --chdir /app \
+        --start \
+        --quiet \
+        --pidfile $PIDFILE \
+        --make-pidfile \
+        --startas $DAEMON -- $DAEMON_OPTS
+        echo "."
+    ;;
+
+  *)
+	echo "Usage: "$1" {start|stop|restart|tail}"
+	exit 1
+esac
+
+exit 0

listener.py

@@ -0,0 +1,98 @@
+import asyncio
+import logging
+import os
+
+import aiohttp
+
+from mev_inspect.block import get_latest_block_number
+from mev_inspect.concurrency import coro
+from mev_inspect.crud.latest_block_update import (
+    find_latest_block_update,
+    update_latest_block,
+)
+from mev_inspect.db import get_inspect_session, get_trace_session
+from mev_inspect.inspector import MEVInspector
+from mev_inspect.provider import get_base_provider
+from mev_inspect.signal_handler import GracefulKiller
+
+logging.basicConfig(filename="listener.log", filemode="a", level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# lag to make sure the blocks we see are settled
+BLOCK_NUMBER_LAG = 5
+
+
+@coro
+async def run():
+    rpc = os.getenv("RPC_URL")
+    if rpc is None:
+        raise RuntimeError("Missing environment variable RPC_URL")
+
+    healthcheck_url = os.getenv("LISTENER_HEALTHCHECK_URL")
+
+    logger.info("Starting...")
+
+    killer = GracefulKiller()
+
+    inspect_db_session = get_inspect_session()
+    trace_db_session = get_trace_session()
+
+    inspector = MEVInspector(rpc, inspect_db_session, trace_db_session)
+    base_provider = get_base_provider(rpc)
+
+    while not killer.kill_now:
+        await inspect_next_block(
+            inspector,
+            inspect_db_session,
+            base_provider,
+            healthcheck_url,
+        )
+
+    logger.info("Stopping...")
+
+
+async def inspect_next_block(
+    inspector: MEVInspector,
+    inspect_db_session,
+    base_provider,
+    healthcheck_url,
+):
+    latest_block_number = await get_latest_block_number(base_provider)
+    last_written_block = find_latest_block_update(inspect_db_session)
+
+    logger.info(f"Latest block: {latest_block_number}")
+    logger.info(f"Last written block: {last_written_block}")
+
+    if last_written_block is None:
+        # maintain lag if no blocks written yet
+        last_written_block = latest_block_number - 1
+
+    if last_written_block < (latest_block_number - BLOCK_NUMBER_LAG):
+        block_number = (
+            latest_block_number
+            if last_written_block is None
+            else last_written_block + 1
+        )
+
+        logger.info(f"Writing block: {block_number}")
+
+        await inspector.inspect_single_block(block=block_number)
+        update_latest_block(inspect_db_session, block_number)
+
+        if healthcheck_url:
+            await ping_healthcheck_url(healthcheck_url)
+    else:
+        await asyncio.sleep(5)
+
+
+async def ping_healthcheck_url(url):
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url):
+            pass
+
+
+if __name__ == "__main__":
+    try:
+        run()
+    except Exception as e:
+        logger.error(e)

loop.py

@@ -0,0 +1,21 @@
+import logging
+import time
+
+from mev_inspect.signal_handler import GracefulKiller
+
+logging.basicConfig(filename="loop.log", level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def run():
+    logger.info("Starting...")
+
+    killer = GracefulKiller()
+    while not killer.kill_now:
+        time.sleep(1)
+
+    logger.info("Stopping...")
+
+
+if __name__ == "__main__":
+    run()

mev

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+set -e
+
+DB_NAME=mev_inspect
+
+function get_kube_db_secret(){
+    kubectl get secrets mev-inspect-db-credentials -o jsonpath="{.data.$1}" | base64 --decode
+}
+
+function db(){
+    host=$(get_kube_db_secret "host")
+    username=$(get_kube_db_secret "username")
+    password=$(get_kube_db_secret "password")
+
+    kubectl run -i --rm --tty postgres-client \
+        --env="PGPASSWORD=$password"  \
+        --image=jbergknoff/postgresql-client \
+        -- $DB_NAME --host=$host --user=$username
+}
+
+case "$1" in
+  db)
+        echo "Connecting to $DB_NAME"
+        db
+	;;
+  listener)
+        kubectl exec -ti deploy/mev-inspect -- ./listener $2
+	;;
+  backfill)
+        start_block_number=$2
+        end_block_number=$3
+        n_workers=$4
+
+        echo "Backfilling from $start_block_number to $end_block_number with $n_workers workers"
+        python backfill.py $start_block_number $end_block_number $n_workers
+	;;
+  inspect)
+        block_number=$2
+        echo "Inspecting block $block_number"
+        kubectl exec -ti deploy/mev-inspect -- poetry run inspect-block $block_number
+	;;
+  inspect-many)
+        start_block_number=$2
+        end_block_number=$3
+        echo "Inspecting from block $start_block_number to $end_block_number"
+        kubectl exec -ti deploy/mev-inspect -- \
+            poetry run inspect-many-blocks $start_block_number $end_block_number
+	;;
+  test)
+        shift
+        echo "Running tests"
+        kubectl exec -ti deploy/mev-inspect -- poetry run pytest tests $@
+	;;
+  fetch)
+      block_number=$2
+      echo "Fetching block $block_number"
+      kubectl exec -ti deploy/mev-inspect -- poetry run fetch-block $block_number
+  ;;
+  prices)
+        shift
+        case "$1" in
+          fetch-all)
+                echo "Running price fetch-all"
+                kubectl exec -ti deploy/mev-inspect -- \
+                    poetry run fetch-all-prices
+            ;;
+          *)
+            echo "prices usage: "$1" {fetch-all}"
+            exit 1
+        esac
+  ;;
+  exec)
+        shift
+        kubectl exec -ti deploy/mev-inspect -- $@
+  ;;
+  *)
+	echo "Usage: "$1" {db|backfill|inspect|test}"
+	exit 1
+esac
+
+exit 0

mev_inspect/aave_liquidations.py

@@ -0,0 +1,100 @@
+from typing import List, Optional, Tuple
+
+from mev_inspect.schemas.liquidations import Liquidation
+from mev_inspect.schemas.traces import (
+    CallTrace,
+    Classification,
+    ClassifiedTrace,
+    DecodedCallTrace,
+    Protocol,
+)
+from mev_inspect.schemas.transfers import Transfer
+from mev_inspect.traces import get_child_traces, is_child_of_any_address
+from mev_inspect.transfers import get_transfer
+
+AAVE_CONTRACT_ADDRESSES: List[str] = [
+    # AAVE Proxy
+    "0x398ec7346dcd622edc5ae82352f02be94c62d119",
+    # AAVE V2
+    "0x7d2768de32b0b80b7a3454c06bdac94a69ddc7a9",
+    # AAVE V1
+    "0x3dfd23a6c5e8bbcfc9581d2e864a68feb6a076d3",
+    # AAVE V2 WETH
+    "0x030ba81f1c18d280636f32af80b9aad02cf0854e",
+    # AAVE AMM Market DAI
+    "0x79be75ffc64dd58e66787e4eae470c8a1fd08ba4",
+    # AAVE i
+    "0x030ba81f1c18d280636f32af80b9aad02cf0854e",
+    "0xbcca60bb61934080951369a648fb03df4f96263c",
+]
+
+
+def get_aave_liquidations(
+    traces: List[ClassifiedTrace],
+) -> List[Liquidation]:
+
+    """Inspect list of classified traces and identify liquidation"""
+    liquidations: List[Liquidation] = []
+    parent_liquidations: List[List[int]] = []
+
+    for trace in traces:
+
+        if (
+            trace.classification == Classification.liquidate
+            and isinstance(trace, DecodedCallTrace)
+            and not is_child_of_any_address(trace, parent_liquidations)
+            and trace.protocol == Protocol.aave
+        ):
+
+            parent_liquidations.append(trace.trace_address)
+            liquidator = trace.from_address
+
+            child_traces = get_child_traces(
+                trace.transaction_hash, trace.trace_address, traces
+            )
+
+            (
+                received_token_address,
+                received_amount,
+            ) = _get_payback_token_and_amount(trace, child_traces, liquidator)
+
+            liquidations.append(
+                Liquidation(
+                    liquidated_user=trace.inputs["_user"],
+                    debt_token_address=trace.inputs["_reserve"],
+                    liquidator_user=liquidator,
+                    debt_purchase_amount=trace.inputs["_purchaseAmount"],
+                    protocol=Protocol.aave,
+                    received_amount=received_amount,
+                    received_token_address=received_token_address,
+                    transaction_hash=trace.transaction_hash,
+                    trace_address=trace.trace_address,
+                    block_number=trace.block_number,
+                )
+            )
+
+    return liquidations
+
+
+def _get_payback_token_and_amount(
+    liquidation: DecodedCallTrace, child_traces: List[ClassifiedTrace], liquidator: str
+) -> Tuple[str, int]:
+
+    """Look for and return liquidator payback from liquidation"""
+
+    for child in child_traces:
+
+        if isinstance(child, CallTrace):
+
+            child_transfer: Optional[Transfer] = get_transfer(child)
+
+            if child_transfer is not None:
+
+                if (
+                    child_transfer.to_address == liquidator
+                    and child.from_address in AAVE_CONTRACT_ADDRESSES
+                ):
+
+                    return child_transfer.token_address, child_transfer.amount
+
+    return liquidation.inputs["_collateral"], 0

mev_inspect/abi.py

@@ -4,23 +4,39 @@ from typing import Optional
 
 from pydantic import parse_obj_as
 
-from mev_inspect.schemas import ABI
-from mev_inspect.schemas.classified_traces import Protocol
-
+from mev_inspect.schemas.abi import ABI
+from mev_inspect.schemas.traces import Protocol
 
 THIS_FILE_DIRECTORY = Path(__file__).parents[0]
 ABI_DIRECTORY_PATH = THIS_FILE_DIRECTORY / "abis"
 
 
-def get_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[ABI]:
+def get_abi_path(abi_name: str, protocol: Optional[Protocol]) -> Optional[Path]:
     abi_filename = f"{abi_name}.json"
     abi_path = (
         ABI_DIRECTORY_PATH / abi_filename
         if protocol is None
         else ABI_DIRECTORY_PATH / protocol.value / abi_filename
     )
-
     if abi_path.is_file():
+        return abi_path
+
+    return None
+
+
+# raw abi, for instantiating contract for queries (as opposed to classification, see below)
+def get_raw_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[str]:
+    abi_path = get_abi_path(abi_name, protocol)
+    if abi_path is not None:
+        with abi_path.open() as abi_file:
+            return abi_file.read()
+
+    return None
+
+
+def get_abi(abi_name: str, protocol: Optional[Protocol]) -> Optional[ABI]:
+    abi_path = get_abi_path(abi_name, protocol)
+    if abi_path is not None:
         with abi_path.open() as abi_file:
             abi_json = json.load(abi_file)
             return parse_obj_as(ABI, abi_json)

mev_inspect/abis/aave/aTokens.json

@@ -0,0 +1,615 @@
+[
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "owner",
+        "type": "address"
+      },
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "spender",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      }
+    ],
+    "name": "Approval",
+    "type": "event"
+  },
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "from",
+        "type": "address"
+      },
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "to",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "BalanceTransfer",
+    "type": "event"
+  },
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "from",
+        "type": "address"
+      },
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "target",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "Burn",
+    "type": "event"
+  },
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "underlyingAsset",
+        "type": "address"
+      },
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "pool",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "address",
+        "name": "treasury",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "address",
+        "name": "incentivesController",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint8",
+        "name": "aTokenDecimals",
+        "type": "uint8"
+      },
+      {
+        "indexed": false,
+        "internalType": "string",
+        "name": "aTokenName",
+        "type": "string"
+      },
+      {
+        "indexed": false,
+        "internalType": "string",
+        "name": "aTokenSymbol",
+        "type": "string"
+      },
+      {
+        "indexed": false,
+        "internalType": "bytes",
+        "name": "params",
+        "type": "bytes"
+      }
+    ],
+    "name": "Initialized",
+    "type": "event"
+  },
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "from",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "Mint",
+    "type": "event"
+  },
+  {
+    "anonymous": false,
+    "inputs": [
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "from",
+        "type": "address"
+      },
+      {
+        "indexed": true,
+        "internalType": "address",
+        "name": "to",
+        "type": "address"
+      },
+      {
+        "indexed": false,
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      }
+    ],
+    "name": "Transfer",
+    "type": "event"
+  },
+  {
+    "inputs": [
+      
+    ],
+    "name": "UNDERLYING_ASSET_ADDRESS",
+    "outputs": [
+      {
+        "internalType": "address",
+        "name": "",
+        "type": "address"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "owner",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "spender",
+        "type": "address"
+      }
+    ],
+    "name": "allowance",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "spender",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      }
+    ],
+    "name": "approve",
+    "outputs": [
+      {
+        "internalType": "bool",
+        "name": "",
+        "type": "bool"
+      }
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "account",
+        "type": "address"
+      }
+    ],
+    "name": "balanceOf",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "receiverOfUnderlying",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      },
+      {
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "burn",
+    "outputs": [
+      
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      
+    ],
+    "name": "getIncentivesController",
+    "outputs": [
+      {
+        "internalType": "contract IAaveIncentivesController",
+        "name": "",
+        "type": "address"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      }
+    ],
+    "name": "getScaledUserBalanceAndSupply",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      },
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      }
+    ],
+    "name": "handleRepayment",
+    "outputs": [
+      
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "contract ILendingPool",
+        "name": "pool",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "treasury",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "underlyingAsset",
+        "type": "address"
+      },
+      {
+        "internalType": "contract IAaveIncentivesController",
+        "name": "incentivesController",
+        "type": "address"
+      },
+      {
+        "internalType": "uint8",
+        "name": "aTokenDecimals",
+        "type": "uint8"
+      },
+      {
+        "internalType": "string",
+        "name": "aTokenName",
+        "type": "string"
+      },
+      {
+        "internalType": "string",
+        "name": "aTokenSymbol",
+        "type": "string"
+      },
+      {
+        "internalType": "bytes",
+        "name": "params",
+        "type": "bytes"
+      }
+    ],
+    "name": "initialize",
+    "outputs": [
+      
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      },
+      {
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "mint",
+    "outputs": [
+      {
+        "internalType": "bool",
+        "name": "",
+        "type": "bool"
+      }
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      },
+      {
+        "internalType": "uint256",
+        "name": "index",
+        "type": "uint256"
+      }
+    ],
+    "name": "mintToTreasury",
+    "outputs": [
+      
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      }
+    ],
+    "name": "scaledBalanceOf",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      
+    ],
+    "name": "scaledTotalSupply",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      
+    ],
+    "name": "totalSupply",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "view",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "recipient",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      }
+    ],
+    "name": "transfer",
+    "outputs": [
+      {
+        "internalType": "bool",
+        "name": "",
+        "type": "bool"
+      }
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "sender",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "recipient",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      }
+    ],
+    "name": "transferFrom",
+    "outputs": [
+      {
+        "internalType": "bool",
+        "name": "",
+        "type": "bool"
+      }
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "from",
+        "type": "address"
+      },
+      {
+        "internalType": "address",
+        "name": "to",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "value",
+        "type": "uint256"
+      }
+    ],
+    "name": "transferOnLiquidation",
+    "outputs": [
+      
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  },
+  {
+    "inputs": [
+      {
+        "internalType": "address",
+        "name": "user",
+        "type": "address"
+      },
+      {
+        "internalType": "uint256",
+        "name": "amount",
+        "type": "uint256"
+      }
+    ],
+    "name": "transferUnderlyingTo",
+    "outputs": [
+      {
+        "internalType": "uint256",
+        "name": "",
+        "type": "uint256"
+      }
+    ],
+    "stateMutability": "nonpayable",
+    "type": "function"
+  }
+]

mev_inspect/arbitrages.py

@@ -1,5 +1,5 @@
 from itertools import groupby
-from typing import List, Optional
+from typing import List, Tuple
 
 from mev_inspect.schemas.arbitrages import Arbitrage
 from mev_inspect.schemas.swaps import Swap
@@ -23,70 +23,112 @@ def get_arbitrages(swaps: List[Swap]) -> List[Arbitrage]:
 
 
 def _get_arbitrages_from_swaps(swaps: List[Swap]) -> List[Arbitrage]:
-    pool_addresses = {swap.pool_address for swap in swaps}
+    """
+    An arbitrage is defined as multiple swaps in a series that result in the initial token being returned
+    to the initial sender address.
+
+    There are 2 types of swaps that are most common (99%+).
+    Case I (fully routed):
+    BOT -> A/B -> B/C -> C/A -> BOT
+
+    Case II (always return to bot):
+    BOT -> A/B -> BOT -> B/C -> BOT -> A/C -> BOT
+
+    There is only 1 correct way to route Case I, but for Case II the following valid routes could be found:
+    A->B->C->A / B->C->A->B / C->A->B->C. Thus when multiple valid routes are found we filter to the set that
+    happen in valid order.
+    """
 
     all_arbitrages = []
 
-    for index, first_swap in enumerate(swaps):
-        other_swaps = swaps[:index] + swaps[index + 1 :]
+    start_ends = _get_all_start_end_swaps(swaps)
+    if len(start_ends) == 0:
+        return []
 
-        if first_swap.from_address not in pool_addresses:
-            arbitrage = _get_arbitrage_starting_with_swap(first_swap, other_swaps)
+    # for (start, end) in filtered_start_ends:
+    for (start, end) in start_ends:
+        potential_intermediate_swaps = [
+            swap for swap in swaps if swap is not start and swap is not end
+        ]
+        routes = _get_all_routes(start, end, potential_intermediate_swaps)
 
-            if arbitrage is not None:
-                all_arbitrages.append(arbitrage)
-
-    return all_arbitrages
-
-
-def _get_arbitrage_starting_with_swap(
-    start_swap: Swap,
-    other_swaps: List[Swap],
-) -> Optional[Arbitrage]:
-    swap_path = [start_swap]
-    current_swap: Swap = start_swap
-
-    while True:
-        next_swap = _get_swap_from_address(
-            current_swap.to_address,
-            current_swap.token_out_address,
-            other_swaps,
-        )
-
-        if next_swap is None:
-            return None
-
-        swap_path.append(next_swap)
-        current_swap = next_swap
-
-        if (
-            current_swap.to_address == start_swap.from_address
-            and current_swap.token_out_address == start_swap.token_in_address
-        ):
-
-            start_amount = start_swap.token_in_amount
-            end_amount = current_swap.token_out_amount
+        for route in routes:
+            start_amount = route[0].token_in_amount
+            end_amount = route[-1].token_out_amount
             profit_amount = end_amount - start_amount
 
-            return Arbitrage(
-                swaps=swap_path,
-                block_number=start_swap.block_number,
-                transaction_hash=start_swap.transaction_hash,
-                account_address=start_swap.from_address,
-                profit_token_address=start_swap.token_in_address,
+            arb = Arbitrage(
+                swaps=route,
+                block_number=route[0].block_number,
+                transaction_hash=route[0].transaction_hash,
+                account_address=route[0].from_address,
+                profit_token_address=route[0].token_in_address,
                 start_amount=start_amount,
                 end_amount=end_amount,
                 profit_amount=profit_amount,
             )
-
-    return None
+            all_arbitrages.append(arb)
+    if len(all_arbitrages) == 1:
+        return all_arbitrages
+    else:
+        return [
+            arb
+            for arb in all_arbitrages
+            if (arb.swaps[0].trace_address < arb.swaps[-1].trace_address)
+        ]
 
 
-def _get_swap_from_address(
-    address: str, token_address: str, swaps: List[Swap]
-) -> Optional[Swap]:
-    for swap in swaps:
-        if swap.pool_address == address and swap.token_in_address == token_address:
-            return swap
+def _get_all_start_end_swaps(swaps: List[Swap]) -> List[Tuple[Swap, Swap]]:
+    """
+    Gets the set of all possible opening and closing swap pairs in an arbitrage via
+    - swap[start].token_in == swap[end].token_out
+    - swap[start].from_address == swap[end].to_address
+    - not swap[start].from_address in all_pool_addresses
+    - not swap[end].to_address in all_pool_addresses
+    """
+    pool_addrs = [swap.contract_address for swap in swaps]
+    valid_start_ends: List[Tuple[Swap, Swap]] = []
+    for index, potential_start_swap in enumerate(swaps):
+        remaining_swaps = swaps[:index] + swaps[index + 1 :]
+        for potential_end_swap in remaining_swaps:
+            if (
+                potential_start_swap.token_in_address
+                == potential_end_swap.token_out_address
+                and potential_start_swap.from_address == potential_end_swap.to_address
+                and not potential_start_swap.from_address in pool_addrs
+            ):
+                valid_start_ends.append((potential_start_swap, potential_end_swap))
+    return valid_start_ends
 
-    return None
+
+def _get_all_routes(
+    start_swap: Swap, end_swap: Swap, other_swaps: List[Swap]
+) -> List[List[Swap]]:
+    """
+    Returns all routes (List[Swap]) from start to finish between a start_swap and an end_swap only accounting for token_address_in and token_address_out.
+    """
+    # If the path is complete, return
+    if start_swap.token_out_address == end_swap.token_in_address:
+        return [[start_swap, end_swap]]
+    elif len(other_swaps) == 0:
+        return []
+
+    # Collect all potential next steps, check if valid, recursively find routes from next_step to end_swap
+    routes: List[List[Swap]] = []
+    for potential_next_swap in other_swaps:
+        if start_swap.token_out_address == potential_next_swap.token_in_address and (
+            start_swap.contract_address == potential_next_swap.from_address
+            or start_swap.to_address == potential_next_swap.contract_address
+            or start_swap.to_address == potential_next_swap.from_address
+        ):
+            remaining_swaps = [
+                swap for swap in other_swaps if swap != potential_next_swap
+            ]
+            next_swap_routes = _get_all_routes(
+                potential_next_swap, end_swap, remaining_swaps
+            )
+            if len(next_swap_routes) > 0:
+                for next_swap_route in next_swap_routes:
+                    next_swap_route.insert(0, start_swap)
+                    routes.append(next_swap_route)
+    return routes

mev_inspect/block.py

@@ -1,51 +1,72 @@
-from pathlib import Path
-from typing import List
+import asyncio
+import logging
+from typing import List, Optional
 
+from sqlalchemy import orm
 from web3 import Web3
 
 from mev_inspect.fees import fetch_base_fee_per_gas
-from mev_inspect.schemas import Block, Trace, TraceType
+from mev_inspect.schemas.blocks import Block
 from mev_inspect.schemas.receipts import Receipt
+from mev_inspect.schemas.traces import Trace, TraceType
+from mev_inspect.utils import hex_to_int
+
+logger = logging.getLogger(__name__)
 
 
-cache_directory = "./cache"
+async def get_latest_block_number(base_provider) -> int:
+    latest_block = await base_provider.make_request(
+        "eth_getBlockByNumber",
+        ["latest", False],
+    )
+
+    return hex_to_int(latest_block["result"]["number"])
 
 
-def create_from_block_number(
-    base_provider, w3: Web3, block_number: int, should_cache: bool
+async def create_from_block_number(
+    base_provider,
+    w3: Web3,
+    block_number: int,
+    trace_db_session: Optional[orm.Session],
 ) -> Block:
-    if not should_cache:
-        return fetch_block(w3, base_provider, block_number)
+    block: Optional[Block] = None
 
-    cache_path = _get_cache_path(block_number)
+    if trace_db_session is not None:
+        block = _find_block(trace_db_session, block_number)
 
-    if cache_path.is_file():
-        print(f"Cache for block {block_number} exists, " "loading data from cache")
-
-        return Block.parse_file(cache_path)
+    if block is None:
+        block = await _fetch_block(w3, base_provider, block_number)
+        return block
     else:
-        print(f"Cache for block {block_number} did not exist, getting data")
-
-        block = fetch_block(w3, base_provider, block_number)
-
-        cache_block(cache_path, block)
-
         return block
 
 
-def fetch_block(w3, base_provider, block_number: int) -> Block:
-    block_json = w3.eth.get_block(block_number)
-    receipts_json = base_provider.make_request("eth_getBlockReceipts", [block_number])
-    traces_json = w3.parity.trace_block(block_number)
+async def _fetch_block(w3, base_provider, block_number: int, retries: int = 0) -> Block:
+    block_json, receipts_json, traces_json, base_fee_per_gas = await asyncio.gather(
+        w3.eth.get_block(block_number),
+        base_provider.make_request("eth_getBlockReceipts", [block_number]),
+        base_provider.make_request("trace_block", [block_number]),
+        fetch_base_fee_per_gas(w3, block_number),
+    )
 
-    receipts: List[Receipt] = [
-        Receipt(**receipt) for receipt in receipts_json["result"]
-    ]
-    traces = [Trace(**trace_json) for trace_json in traces_json]
-    base_fee_per_gas = fetch_base_fee_per_gas(w3, block_number)
+    try:
+        receipts: List[Receipt] = [
+            Receipt(**receipt) for receipt in receipts_json["result"]
+        ]
+        traces = [Trace(**trace_json) for trace_json in traces_json["result"]]
+    except KeyError as e:
+        logger.warning(
+            f"Failed to create objects from block: {block_number}: {e}, retrying: {retries + 1} / 3"
+        )
+        if retries < 3:
+            await asyncio.sleep(5)
+            return await _fetch_block(w3, base_provider, block_number, retries)
+        else:
+            raise
 
     return Block(
         block_number=block_number,
+        block_timestamp=block_json["timestamp"],
         miner=block_json["miner"],
         base_fee_per_gas=base_fee_per_gas,
         traces=traces,
@@ -53,6 +74,110 @@ def fetch_block(w3, base_provider, block_number: int) -> Block:
     )
 
 
+def _find_block(
+    trace_db_session: orm.Session,
+    block_number: int,
+) -> Optional[Block]:
+    block_timestamp = _find_block_timestamp(trace_db_session, block_number)
+    traces = _find_traces(trace_db_session, block_number)
+    receipts = _find_receipts(trace_db_session, block_number)
+    base_fee_per_gas = _find_base_fee(trace_db_session, block_number)
+
+    if (
+        block_timestamp is None
+        or traces is None
+        or receipts is None
+        or base_fee_per_gas is None
+    ):
+        return None
+
+    miner_address = _get_miner_address_from_traces(traces)
+
+    if miner_address is None:
+        return None
+
+    return Block(
+        block_number=block_number,
+        block_timestamp=block_timestamp,
+        miner=miner_address,
+        base_fee_per_gas=base_fee_per_gas,
+        traces=traces,
+        receipts=receipts,
+    )
+
+
+def _find_block_timestamp(
+    trace_db_session: orm.Session,
+    block_number: int,
+) -> Optional[int]:
+    result = trace_db_session.execute(
+        "SELECT block_timestamp FROM block_timestamps WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    ).one_or_none()
+
+    if result is None:
+        return None
+    else:
+        (block_timestamp,) = result
+        return block_timestamp
+
+
+def _find_traces(
+    trace_db_session: orm.Session,
+    block_number: int,
+) -> Optional[List[Trace]]:
+    result = trace_db_session.execute(
+        "SELECT raw_traces FROM block_traces WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    ).one_or_none()
+
+    if result is None:
+        return None
+    else:
+        (traces_json,) = result
+        return [Trace(**trace_json) for trace_json in traces_json]
+
+
+def _find_receipts(
+    trace_db_session: orm.Session,
+    block_number: int,
+) -> Optional[List[Receipt]]:
+    result = trace_db_session.execute(
+        "SELECT raw_receipts FROM block_receipts WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    ).one_or_none()
+
+    if result is None:
+        return None
+    else:
+        (receipts_json,) = result
+        return [Receipt(**receipt) for receipt in receipts_json]
+
+
+def _find_base_fee(
+    trace_db_session: orm.Session,
+    block_number: int,
+) -> Optional[int]:
+    result = trace_db_session.execute(
+        "SELECT base_fee_in_wei FROM base_fee WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    ).one_or_none()
+
+    if result is None:
+        return None
+    else:
+        (base_fee,) = result
+        return base_fee
+
+
+def _get_miner_address_from_traces(traces: List[Trace]) -> Optional[str]:
+    for trace in traces:
+        if trace.type == TraceType.reward:
+            return trace.action["author"]
+
+    return None
+
+
 def get_transaction_hashes(calls: List[Trace]) -> List[str]:
     result = []
 
@@ -65,15 +190,3 @@ def get_transaction_hashes(calls: List[Trace]) -> List[str]:
                 result.append(call.transaction_hash)
 
     return result
-
-
-def cache_block(cache_path: Path, block: Block):
-    write_mode = "w" if cache_path.is_file() else "x"
-
-    with open(cache_path, mode=write_mode) as cache_file:
-        cache_file.write(block.json())
-
-
-def _get_cache_path(block_number: int) -> Path:
-    cache_directory_path = Path(cache_directory)
-    return cache_directory_path / f"{block_number}-new.json"

mev_inspect/classifiers/helpers.py

@@ -0,0 +1,124 @@
+from typing import List, Optional, Sequence
+
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import ClassifiedTrace, DecodedCallTrace
+from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS, Transfer
+
+
+def create_swap_from_pool_transfers(
+    trace: DecodedCallTrace,
+    recipient_address: str,
+    prior_transfers: List[Transfer],
+    child_transfers: List[Transfer],
+) -> Optional[Swap]:
+    pool_address = trace.to_address
+
+    transfers_to_pool = []
+
+    if trace.value is not None and trace.value > 0:
+        transfers_to_pool = [_build_eth_transfer(trace)]
+
+    if len(transfers_to_pool) == 0:
+        transfers_to_pool = _filter_transfers(prior_transfers, to_address=pool_address)
+
+    if len(transfers_to_pool) == 0:
+        transfers_to_pool = _filter_transfers(child_transfers, to_address=pool_address)
+
+    if len(transfers_to_pool) == 0:
+        return None
+
+    transfers_from_pool_to_recipient = _filter_transfers(
+        child_transfers, to_address=recipient_address, from_address=pool_address
+    )
+
+    if len(transfers_from_pool_to_recipient) != 1:
+        return None
+
+    transfer_in = transfers_to_pool[-1]
+    transfer_out = transfers_from_pool_to_recipient[0]
+
+    return Swap(
+        abi_name=trace.abi_name,
+        transaction_hash=trace.transaction_hash,
+        transaction_position=trace.transaction_position,
+        block_number=trace.block_number,
+        trace_address=trace.trace_address,
+        contract_address=pool_address,
+        protocol=trace.protocol,
+        from_address=transfer_in.from_address,
+        to_address=transfer_out.to_address,
+        token_in_address=transfer_in.token_address,
+        token_in_amount=transfer_in.amount,
+        token_out_address=transfer_out.token_address,
+        token_out_amount=transfer_out.amount,
+        error=trace.error,
+    )
+
+
+def create_swap_from_recipient_transfers(
+    trace: DecodedCallTrace,
+    pool_address: str,
+    recipient_address: str,
+    prior_transfers: List[Transfer],
+    child_transfers: List[Transfer],
+) -> Optional[Swap]:
+    transfers_from_recipient = _filter_transfers(
+        [*prior_transfers, *child_transfers], from_address=recipient_address
+    )
+    transfers_to_recipient = _filter_transfers(
+        child_transfers, to_address=recipient_address
+    )
+
+    if len(transfers_from_recipient) != 1 or len(transfers_to_recipient) != 1:
+        return None
+
+    transfer_in = transfers_from_recipient[0]
+    transfer_out = transfers_to_recipient[0]
+
+    return Swap(
+        abi_name=trace.abi_name,
+        transaction_hash=trace.transaction_hash,
+        transaction_position=trace.transaction_position,
+        block_number=trace.block_number,
+        trace_address=trace.trace_address,
+        contract_address=pool_address,
+        protocol=trace.protocol,
+        from_address=transfer_in.from_address,
+        to_address=transfer_out.to_address,
+        token_in_address=transfer_in.token_address,
+        token_in_amount=transfer_in.amount,
+        token_out_address=transfer_out.token_address,
+        token_out_amount=transfer_out.amount,
+        error=trace.error,
+    )
+
+
+def _build_eth_transfer(trace: ClassifiedTrace) -> Transfer:
+    return Transfer(
+        block_number=trace.block_number,
+        transaction_hash=trace.transaction_hash,
+        trace_address=trace.trace_address,
+        amount=trace.value,
+        to_address=trace.to_address,
+        from_address=trace.from_address,
+        token_address=ETH_TOKEN_ADDRESS,
+    )
+
+
+def _filter_transfers(
+    transfers: Sequence[Transfer],
+    to_address: Optional[str] = None,
+    from_address: Optional[str] = None,
+) -> List[Transfer]:
+    filtered_transfers = []
+
+    for transfer in transfers:
+        if to_address is not None and transfer.to_address != to_address:
+            continue
+
+        if from_address is not None and transfer.from_address != from_address:
+            continue
+
+        filtered_transfers.append(transfer)
+
+    return filtered_transfers

mev_inspect/classifiers/specs/__init__.py

@@ -1,10 +1,18 @@
+from typing import Dict, Optional, Tuple, Type
+
+from mev_inspect.schemas.classifiers import Classifier, ClassifierSpec
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+
 from .aave import AAVE_CLASSIFIER_SPECS
+from .balancer import BALANCER_CLASSIFIER_SPECS
+from .bancor import BANCOR_CLASSIFIER_SPECS
+from .compound import COMPOUND_CLASSIFIER_SPECS
+from .cryptopunks import CRYPTOPUNKS_CLASSIFIER_SPECS
 from .curve import CURVE_CLASSIFIER_SPECS
 from .erc20 import ERC20_CLASSIFIER_SPECS
 from .uniswap import UNISWAP_CLASSIFIER_SPECS
-from .weth import WETH_CLASSIFIER_SPECS
-from .zerox import ZEROX_CLASSIFIER_SPECS
-
+from .weth import WETH_ADDRESS, WETH_CLASSIFIER_SPECS
+from .zero_ex import ZEROX_CLASSIFIER_SPECS
 
 ALL_CLASSIFIER_SPECS = (
     ERC20_CLASSIFIER_SPECS
@@ -13,4 +21,24 @@ ALL_CLASSIFIER_SPECS = (
     + UNISWAP_CLASSIFIER_SPECS
     + AAVE_CLASSIFIER_SPECS
     + ZEROX_CLASSIFIER_SPECS
+    + BALANCER_CLASSIFIER_SPECS
+    + COMPOUND_CLASSIFIER_SPECS
+    + CRYPTOPUNKS_CLASSIFIER_SPECS
+    + BANCOR_CLASSIFIER_SPECS
 )
+
+_SPECS_BY_ABI_NAME_AND_PROTOCOL: Dict[
+    Tuple[str, Optional[Protocol]], ClassifierSpec
+] = {(spec.abi_name, spec.protocol): spec for spec in ALL_CLASSIFIER_SPECS}
+
+
+def get_classifier(
+    trace: DecodedCallTrace,
+) -> Optional[Type[Classifier]]:
+    abi_name_and_protocol = (trace.abi_name, trace.protocol)
+    spec = _SPECS_BY_ABI_NAME_AND_PROTOCOL.get(abi_name_and_protocol)
+
+    if spec is not None:
+        return spec.classifiers.get(trace.function_signature)
+
+    return None

mev_inspect/classifiers/specs/aave.py

@@ -1,15 +1,42 @@
-from mev_inspect.schemas.classified_traces import (
-    Classification,
+from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
-    Protocol,
+    DecodedCallTrace,
+    LiquidationClassifier,
+    TransferClassifier,
 )
+from mev_inspect.schemas.traces import Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+
+class AaveTransferClassifier(TransferClassifier):
+    @staticmethod
+    def get_transfer(trace: DecodedCallTrace) -> Transfer:
+        return Transfer(
+            block_number=trace.block_number,
+            transaction_hash=trace.transaction_hash,
+            trace_address=trace.trace_address,
+            amount=trace.inputs["value"],
+            to_address=trace.inputs["to"],
+            from_address=trace.inputs["from"],
+            token_address=trace.to_address,
+        )
+
 
 AAVE_SPEC = ClassifierSpec(
     abi_name="AaveLendingPool",
     protocol=Protocol.aave,
-    classifications={
-        "liquidationCall(address,address,address,uint256,bool)": Classification.liquidate,
+    classifiers={
+        "liquidationCall(address,address,address,uint256,bool)": LiquidationClassifier,
     },
 )
 
-AAVE_CLASSIFIER_SPECS = [AAVE_SPEC]
+ATOKENS_SPEC = ClassifierSpec(
+    abi_name="aTokens",
+    protocol=Protocol.aave,
+    classifiers={
+        "transferOnLiquidation(address,address,uint256)": AaveTransferClassifier,
+        "transferFrom(address,address,uint256)": AaveTransferClassifier,
+    },
+)
+
+AAVE_CLASSIFIER_SPECS = [AAVE_SPEC, ATOKENS_SPEC]

mev_inspect/classifiers/specs/balancer.py

@@ -0,0 +1,41 @@
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
+from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+BALANCER_V1_POOL_ABI_NAME = "BPool"
+
+
+class BalancerSwapClassifier(SwapClassifier):
+    @staticmethod
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+
+        recipient_address = trace.from_address
+
+        swap = create_swap_from_pool_transfers(
+            trace, recipient_address, prior_transfers, child_transfers
+        )
+        return swap
+
+
+BALANCER_V1_SPECS = [
+    ClassifierSpec(
+        abi_name=BALANCER_V1_POOL_ABI_NAME,
+        protocol=Protocol.balancer_v1,
+        classifiers={
+            "swapExactAmountIn(address,uint256,address,uint256,uint256)": BalancerSwapClassifier,
+            "swapExactAmountOut(address,uint256,address,uint256,uint256)": BalancerSwapClassifier,
+        },
+    ),
+]
+
+BALANCER_CLASSIFIER_SPECS = [
+    *BALANCER_V1_SPECS,
+]

mev_inspect/classifiers/specs/bancor.py

@@ -0,0 +1,41 @@
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import create_swap_from_recipient_transfers
+from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+BANCOR_NETWORK_ABI_NAME = "BancorNetwork"
+BANCOR_NETWORK_CONTRACT_ADDRESS = "0x2F9EC37d6CcFFf1caB21733BdaDEdE11c823cCB0"
+
+
+class BancorSwapClassifier(SwapClassifier):
+    @staticmethod
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+        recipient_address = trace.from_address
+
+        swap = create_swap_from_recipient_transfers(
+            trace,
+            BANCOR_NETWORK_CONTRACT_ADDRESS,
+            recipient_address,
+            prior_transfers,
+            child_transfers,
+        )
+        return swap
+
+
+BANCOR_NETWORK_SPEC = ClassifierSpec(
+    abi_name=BANCOR_NETWORK_ABI_NAME,
+    protocol=Protocol.bancor,
+    classifiers={
+        "convertByPath(address[],uint256,uint256,address,address,uint256)": BancorSwapClassifier,
+    },
+    valid_contract_addresses=[BANCOR_NETWORK_CONTRACT_ADDRESS],
+)
+
+BANCOR_CLASSIFIER_SPECS = [BANCOR_NETWORK_SPEC]

mev_inspect/classifiers/specs/compound.py

@@ -0,0 +1,163 @@
+from mev_inspect.schemas.classifiers import (
+    ClassifierSpec,
+    LiquidationClassifier,
+    SeizeClassifier,
+)
+from mev_inspect.schemas.traces import Protocol
+
+COMPOUND_V2_CETH_SPEC = ClassifierSpec(
+    abi_name="CEther",
+    protocol=Protocol.compound_v2,
+    valid_contract_addresses=["0x4ddc2d193948926d02f9b1fe9e1daa0718270ed5"],
+    classifiers={
+        "liquidateBorrow(address,address)": LiquidationClassifier,
+        "seize(address,address,uint256)": SeizeClassifier,
+    },
+)
+
+CREAM_CETH_SPEC = ClassifierSpec(
+    abi_name="CEther",
+    protocol=Protocol.cream,
+    valid_contract_addresses=["0xD06527D5e56A3495252A528C4987003b712860eE"],
+    classifiers={
+        "liquidateBorrow(address,address)": LiquidationClassifier,
+        "seize(address,address,uint256)": SeizeClassifier,
+    },
+)
+
+COMPOUND_V2_CTOKEN_SPEC = ClassifierSpec(
+    abi_name="CToken",
+    protocol=Protocol.compound_v2,
+    valid_contract_addresses=[
+        "0x6c8c6b02e7b2be14d4fa6022dfd6d75921d90e4e",
+        "0x5d3a536e4d6dbd6114cc1ead35777bab948e3643",
+        "0x158079ee67fce2f58472a96584a73c7ab9ac95c1",
+        "0x39aa39c021dfbae8fac545936693ac917d5e7563",
+        "0xf650c3d88d12db855b8bf7d11be6c55a4e07dcc9",
+        "0xc11b1268c1a384e55c48c2391d8d480264a3a7f4",
+        "0xb3319f5d18bc0d84dd1b4825dcde5d5f7266d407",
+        "0xf5dce57282a584d2746faf1593d3121fcac444dc",
+        "0x35a18000230da775cac24873d00ff85bccded550",
+        "0x70e36f6bf80a52b3b46b3af8e106cc0ed743e8e4",
+        "0xccf4429db6322d5c611ee964527d42e5d685dd6a",
+        "0x12392f67bdf24fae0af363c24ac620a2f67dad86",
+        "0xface851a4921ce59e912d19329929ce6da6eb0c7",
+        "0x95b4ef2869ebd94beb4eee400a99824bf5dc325b",
+        "0x4b0181102a0112a2ef11abee5563bb4a3176c9d7",
+        "0xe65cdb6479bac1e22340e4e755fae7e509ecd06c",
+        "0x80a2ae356fc9ef4305676f7a3e2ed04e12c33946",
+    ],
+    classifiers={
+        "liquidateBorrow(address,uint256,address)": LiquidationClassifier,
+        "seize(address,address,uint256)": SeizeClassifier,
+    },
+)
+
+CREAM_CTOKEN_SPEC = ClassifierSpec(
+    abi_name="CToken",
+    protocol=Protocol.cream,
+    valid_contract_addresses=[
+        "0xd06527d5e56a3495252a528c4987003b712860ee",
+        "0x51f48b638f82e8765f7a26373a2cb4ccb10c07af",
+        "0x44fbebd2f576670a6c33f6fc0b00aa8c5753b322",
+        "0xcbae0a83f4f9926997c8339545fb8ee32edc6b76",
+        "0xce4fe9b4b8ff61949dcfeb7e03bc9faca59d2eb3",
+        "0x19d1666f543d42ef17f66e376944a22aea1a8e46",
+        "0x9baf8a5236d44ac410c0186fe39178d5aad0bb87",
+        "0x797aab1ce7c01eb727ab980762ba88e7133d2157",
+        "0x892b14321a4fcba80669ae30bd0cd99a7ecf6ac0",
+        "0x697256caa3ccafd62bb6d3aa1c7c5671786a5fd9",
+        "0x8b86e0598616a8d4f1fdae8b59e55fb5bc33d0d6",
+        "0xc7fd8dcee4697ceef5a2fd4608a7bd6a94c77480",
+        "0x17107f40d70f4470d20cb3f138a052cae8ebd4be",
+        "0x1ff8cdb51219a8838b52e9cac09b71e591bc998e",
+        "0x3623387773010d9214b10c551d6e7fc375d31f58",
+        "0x4ee15f44c6f0d8d1136c83efd2e8e4ac768954c6",
+        "0x338286c0bc081891a4bda39c7667ae150bf5d206",
+        "0x10fdbd1e48ee2fd9336a482d746138ae19e649db",
+        "0x01da76dea59703578040012357b81ffe62015c2d",
+        "0xef58b2d5a1b8d3cde67b8ab054dc5c831e9bc025",
+        "0xe89a6d0509faf730bd707bf868d9a2a744a363c7",
+        "0xeff039c3c1d668f408d09dd7b63008622a77532c",
+        "0x22b243b96495c547598d9042b6f94b01c22b2e9e",
+        "0x8b3ff1ed4f36c2c2be675afb13cc3aa5d73685a5",
+        "0x2a537fa9ffaea8c1a41d3c2b68a9cb791529366d",
+        "0x7ea9c63e216d5565c3940a2b3d150e59c2907db3",
+        "0x3225e3c669b39c7c8b3e204a8614bb218c5e31bc",
+        "0xf55bbe0255f7f4e70f63837ff72a577fbddbe924",
+        "0x903560b1cce601794c584f58898da8a8b789fc5d",
+        "0x054b7ed3f45714d3091e82aad64a1588dc4096ed",
+        "0xd5103afcd0b3fa865997ef2984c66742c51b2a8b",
+        "0xfd609a03b393f1a1cfcacedabf068cad09a924e2",
+        "0xd692ac3245bb82319a31068d6b8412796ee85d2c",
+        "0x92b767185fb3b04f881e3ac8e5b0662a027a1d9f",
+        "0x10a3da2bb0fae4d591476fd97d6636fd172923a8",
+        "0x3c6c553a95910f9fc81c98784736bd628636d296",
+        "0x21011bc93d9e515b9511a817a1ed1d6d468f49fc",
+        "0x85759961b116f1d36fd697855c57a6ae40793d9b",
+        "0x7c3297cfb4c4bbd5f44b450c0872e0ada5203112",
+        "0x7aaa323d7e398be4128c7042d197a2545f0f1fea",
+        "0x011a014d5e8eb4771e575bb1000318d509230afa",
+        "0xe6c3120f38f56deb38b69b65cc7dcaf916373963",
+        "0x4fe11bc316b6d7a345493127fbe298b95adaad85",
+        "0xcd22c4110c12ac41acefa0091c432ef44efaafa0",
+        "0x228619cca194fbe3ebeb2f835ec1ea5080dafbb2",
+        "0x73f6cba38922960b7092175c0add22ab8d0e81fc",
+        "0x38f27c03d6609a86ff7716ad03038881320be4ad",
+        "0x5ecad8a75216cea7dff978525b2d523a251eea92",
+        "0x5c291bc83d15f71fb37805878161718ea4b6aee9",
+        "0x6ba0c66c48641e220cf78177c144323b3838d375",
+        "0xd532944df6dfd5dd629e8772f03d4fc861873abf",
+        "0x197070723ce0d3810a0e47f06e935c30a480d4fc",
+        "0xc25eae724f189ba9030b2556a1533e7c8a732e14",
+        "0x25555933a8246ab67cbf907ce3d1949884e82b55",
+        "0xc68251421edda00a10815e273fa4b1191fac651b",
+        "0x65883978ada0e707c3b2be2a6825b1c4bdf76a90",
+        "0x8b950f43fcac4931d408f1fcda55c6cb6cbf3096",
+        "0x59089279987dd76fc65bf94cb40e186b96e03cb3",
+        "0x2db6c82ce72c8d7d770ba1b5f5ed0b6e075066d6",
+        "0xb092b4601850e23903a42eacbc9d8a0eec26a4d5",
+        "0x081fe64df6dc6fc70043aedf3713a3ce6f190a21",
+        "0x1d0986fb43985c88ffa9ad959cc24e6a087c7e35",
+        "0xc36080892c64821fa8e396bc1bd8678fa3b82b17",
+        "0x8379baa817c5c5ab929b03ee8e3c48e45018ae41",
+        "0x299e254a8a165bbeb76d9d69305013329eea3a3b",
+        "0xf8445c529d363ce114148662387eba5e62016e20",
+        "0x28526bb33d7230e65e735db64296413731c5402e",
+        "0x45406ba53bb84cd32a58e7098a2d4d1b11b107f6",
+        "0x6d1b9e01af17dd08d6dec08e210dfd5984ff1c20",
+        "0x1f9b4756b008106c806c7e64322d7ed3b72cb284",
+        "0xab10586c918612ba440482db77549d26b7abf8f7",
+        "0xdfff11dfe6436e42a17b86e7f419ac8292990393",
+        "0xdbb5e3081def4b6cdd8864ac2aeda4cbf778fecf",
+        "0x71cefcd324b732d4e058afacba040d908c441847",
+        "0x1a122348b73b58ea39f822a89e6ec67950c2bbd0",
+        "0x523effc8bfefc2948211a05a905f761cba5e8e9e",
+        "0x4202d97e00b9189936edf37f8d01cff88bdd81d4",
+        "0x4baa77013ccd6705ab0522853cb0e9d453579dd4",
+        "0x98e329eb5aae2125af273102f3440de19094b77c",
+        "0x8c3b7a4320ba70f8239f83770c4015b5bc4e6f91",
+        "0xe585c76573d7593abf21537b607091f76c996e73",
+        "0x81e346729723c4d15d0fb1c5679b9f2926ff13c6",
+        "0x766175eac1a99c969ddd1ebdbe7e270d508d8fff",
+        "0xd7394428536f63d5659cc869ef69d10f9e66314b",
+        "0x1241b10e7ea55b22f5b2d007e8fecdf73dcff999",
+        "0x2a867fd776b83e1bd4e13c6611afd2f6af07ea6d",
+        "0x250fb308199fe8c5220509c1bf83d21d60b7f74a",
+        "0x4112a717edd051f77d834a6703a1ef5e3d73387f",
+        "0xf04ce2e71d32d789a259428ddcd02d3c9f97fb4e",
+        "0x89e42987c39f72e2ead95a8a5bc92114323d5828",
+        "0x58da9c9fc3eb30abbcbbab5ddabb1e6e2ef3d2ef",
+    ],
+    classifiers={
+        "liquidateBorrow(address,uint256,address)": LiquidationClassifier,
+        "seize(address,address,uint256)": SeizeClassifier,
+    },
+)
+
+COMPOUND_CLASSIFIER_SPECS = [
+    COMPOUND_V2_CETH_SPEC,
+    COMPOUND_V2_CTOKEN_SPEC,
+    CREAM_CETH_SPEC,
+    CREAM_CTOKEN_SPEC,
+]

mev_inspect/classifiers/specs/cryptopunks.py

@@ -0,0 +1,27 @@
+from mev_inspect.schemas.classifiers import Classifier, ClassifierSpec
+from mev_inspect.schemas.traces import Classification, Protocol
+
+
+class PunkBidAcceptanceClassifier(Classifier):
+    @staticmethod
+    def get_classification() -> Classification:
+        return Classification.punk_accept_bid
+
+
+class PunkBidClassifier(Classifier):
+    @staticmethod
+    def get_classification() -> Classification:
+        return Classification.punk_bid
+
+
+CRYPTO_PUNKS_SPEC = ClassifierSpec(
+    abi_name="cryptopunks",
+    protocol=Protocol.cryptopunks,
+    valid_contract_addresses=["0xb47e3cd837dDF8e4c57F05d70Ab865de6e193BBB"],
+    classifiers={
+        "enterBidForPunk(uint256)": PunkBidClassifier,
+        "acceptBidForPunk(uint256,uint256)": PunkBidAcceptanceClassifier,
+    },
+)
+
+CRYPTOPUNKS_CLASSIFIER_SPECS = [CRYPTO_PUNKS_SPEC]

mev_inspect/classifiers/specs/curve.py

@@ -1,29 +1,28 @@
-from mev_inspect.schemas.classified_traces import (
-    ClassifierSpec,
-    Protocol,
-)
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
+from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+
+class CurveSwapClassifier(SwapClassifier):
+    @staticmethod
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+
+        recipient_address = trace.from_address
+
+        swap = create_swap_from_pool_transfers(
+            trace, recipient_address, prior_transfers, child_transfers
+        )
+        return swap
 
-"""
-Deployment addresses found here
-https://curve.readthedocs.io/ref-addresses.html
 
-organized into 3 groups
-1. Base Pools: 2 or more tokens implementing stable swap
-  - StableSwap<pool>
-  - Deposit<pool>
-  - CurveContract<version>
-  - CurveTokenV1/V2
-2. Meta Pools: 1 token trading with an LP from above
-  - StableSwap<pool>
-  - Deposit<pool>
-  - CurveTokenV1/V2
-3. Liquidity Gauges: stake LP get curve governance token?
-  - LiquidityGauge
-  - LiquidityGaugeV1/V2
-  - LiquidityGaugeReward
-4. DAO stuff
-5..? Other stuff, haven't decided if important
-"""
 CURVE_BASE_POOLS = [
     ClassifierSpec(
         abi_name="CurveTokenV1",
@@ -72,101 +71,171 @@ CURVE_BASE_POOLS = [
         abi_name="StableSwap3Pool",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xbEbc44782C7dB0a1A60Cb6fe97d0b483032FF1C7"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapAAVE",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xDeBF20617708857ebe4F679508E7b7863a8A8EeE"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapAETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xA96A65c051bF88B4095Ee1f2451C2A9d43F53Ae2"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapBUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x79a8C46DeA5aDa233ABaFFD40F3A0A2B1e5A4F27"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapCompound",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xA2B47E3D5c44877cca798226B7B8118F9BFb7A56"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapEURS",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x0Ce6a5fF5217e38315f87032CF90686C96627CAA"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwaphBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x4CA9b3063Ec5866A4B82E437059D2C43d1be596F"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapIronBank",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x2dded6Da1BF5DBdF597C45fcFaa3194e53EcfeAF"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapLink",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xf178c0b5bb7e7abf4e12a4838c7b7c5ba2c623c0"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapPAX",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x06364f10B501e868329afBc005b3492902d6C763"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwaprenBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x93054188d876f558f4a66B2EF1d97d16eDf0895B"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwaprETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xF9440930043eb3997fc70e1339dBb11F341de7A8"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsAAVE",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xEB16Ae0052ed37f479f7fe63849198Df1765a733"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x7fC77b5c7614E1533320Ea6DDc2Eb61fa00A9714"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xc5424B857f758E906013F3555Dad202e4bdB4567"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapstETH",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xDC24316b9AE028F1497c275EB9192a3Ea0f67022"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapsUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xA5407eAE9Ba41422680e2e00537571bcC53efBfD"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUSDT",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x52EA46506B9CC5Ef470C5bf89f17Dc28bB35D85C"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapY",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x45F783CCE6B7FF23B2ab2D70e416cdb7D6055f51"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapYv2",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x8925D9d9B4569D737a48499DeF3f67BaA5a144b9"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="DepositBUSD",
@@ -300,51 +369,91 @@ CURVE_META_POOLS = [
         abi_name="StableSwapbBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x071c661B4DeefB59E2a3DdB20Db036821eeE8F4b"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapDUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x8038C01A0390a8c547446a0b2c18fc9aEFEcc10c"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapGUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x4f062658EaAF2C1ccf8C8e36D6824CDf41167956"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapHUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x3eF6A01A0f81D6046290f3e2A8c5b843e738E604"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapLinkUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xE7a24EF0C5e95Ffb0f6684b813A78F2a3AD7D171"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapMUSD",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x8474DdbE98F5aA3179B3B3F5942D724aFcdec9f6"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapoBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xd81dA8D904b52208541Bade1bD6595D8a251F8dd"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwappBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x7F55DDe206dbAD629C080068923b36fe9D6bDBeF"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapRSV",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xC18cC39da8b11dA8c3541C598eE022258F9744da"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwaptBTC",
         protocol=Protocol.curve,
         valid_contract_addresses=["0xC25099792E9349C7DD09759744ea681C7de2cb66"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUSD",
@@ -353,82 +462,29 @@ CURVE_META_POOLS = [
             "0x3E01dD8a5E1fb3481F0F589056b428Fc308AF0Fb",
             "0x0f9cb53Ebe405d49A0bbdBD291A65Ff571bC83e1",
         ],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUSDP",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x42d7025938bEc20B69cBae5A77421082407f053A"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
     ClassifierSpec(
         abi_name="StableSwapUST",
         protocol=Protocol.curve,
         valid_contract_addresses=["0x890f4e345B1dAED0367A877a1612f86A1f86985f"],
+        classifiers={
+            "exchange(int128,int128,uint256,uint256)": CurveSwapClassifier,
+            "exchange_underlying(int128,int128,uint256,uint256)": CurveSwapClassifier,
+        },
     ),
 ]
 
-"""
-CURVE_LIQUIDITY_GAUGES = [
-    ClassifierSpec(
-        abi_name="LiquidityGauge",
-        protocol=Protocol.curve,
-        valid_contract_addresses=[
-            "0xbFcF63294aD7105dEa65aA58F8AE5BE2D9d0952A",  # 3Pool
-            "0x69Fb7c45726cfE2baDeE8317005d3F94bE838840",  # BUSD
-            "0x7ca5b0a2910B33e9759DC7dDB0413949071D7575",  # Compound
-            "0xC5cfaDA84E902aD92DD40194f0883ad49639b023",  # GUSD
-            "0x4c18E409Dc8619bFb6a1cB56D114C3f592E0aE79",  # hBTC
-            "0x2db0E83599a91b508Ac268a6197b8B14F5e72840",  # HUSD
-            "0x64E3C23bfc40722d3B649844055F1D51c1ac041d",  # PAX
-            "0xB1F2cdeC61db658F091671F5f199635aEF202CAC",  # renBTC
-            "0xC2b1DF84112619D190193E48148000e3990Bf627",  # USDK
-            "0xF98450B5602fa59CC66e1379DFfB6FDDc724CfC4",  # USDN
-            "0xBC89cd85491d81C6AD2954E6d0362Ee29fCa8F53",  # USDT
-            "0xFA712EE4788C042e2B7BB55E6cb8ec569C4530c1",  # Y
-        ],
-    ),
-    ClassifierSpec(
-        abi_name="LiquidityGaugeV2",
-        protocol=Protocol.curve,
-        valid_contract_addresses=[
-            "0xd662908ADA2Ea1916B3318327A97eB18aD588b5d",  # AAVE
-            "0x6d10ed2cF043E6fcf51A0e7b4C2Af3Fa06695707",  # ankrETH
-            "0xdFc7AdFa664b08767b735dE28f9E84cd30492aeE",  # bBTC
-            "0x90Bb609649E0451E5aD952683D64BD2d1f245840",  # EURS
-            "0x72e158d38dbd50a483501c24f792bdaaa3e7d55c",  # FRAX
-            "0x11137B10C210b579405c21A07489e28F3c040AB1",  # oBTC
-            "0xF5194c3325202F456c95c1Cf0cA36f8475C1949F",  # IronBank
-            "0xFD4D8a17df4C27c1dD245d153ccf4499e806C87D",  # Link
-            "0xd7d147c6Bb90A718c3De8C0568F9B560C79fa416",  # pBTC
-            "0x462253b8F74B72304c145DB0e4Eebd326B22ca39",  # sAAVE
-            "0x3C0FFFF15EA30C35d7A85B85c0782D6c94e1d238",  # sETH
-            "0x182B723a58739a9c974cFDB385ceaDb237453c28",  # stETH
-            "0x055be5DDB7A925BfEF3417FC157f53CA77cA7222",  # USDP
-            "0x3B7020743Bc2A4ca9EaF9D0722d42E20d6935855",  # UST
-            "0x8101E6760130be2C8Ace79643AB73500571b7162",  # Yv2
-        ],
-    ),
-    ClassifierSpec(
-        abi_name="LiquidityGaugeV3",
-        protocol=Protocol.curve,
-        valid_contract_addresses=[
-            "0x9582C4ADACB3BCE56Fea3e590F05c3ca2fb9C477",  # alUSD
-            "0x824F13f1a2F29cFEEa81154b46C0fc820677A637",  # rETH
-            "0x6955a55416a06839309018A8B0cB72c4DDC11f15",  # TriCrypto
-        ],
-    ),
-    ClassifierSpec(
-        abi_name="LiquidityGaugeReward",
-        protocol=Protocol.curve,
-        valid_contract_addresses=[
-            "0xAEA6c312f4b3E04D752946d329693F7293bC2e6D",  # DUSD
-            "0x5f626c30EC1215f4EdCc9982265E8b1F411D1352",  # MUSD
-            "0x4dC4A289a8E33600D8bD4cf5F6313E43a37adec7",  # RSV
-            "0x705350c4BcD35c9441419DdD5d2f097d7a55410F",  # sBTC
-            "0xA90996896660DEcC6E997655E065b23788857849",  # sUSDv2
-            "0x6828bcF74279eE32f2723eC536c22c51Eed383C6",  # tBTC
-        ],
-    ),
-]
-"""
-
 CURVE_CLASSIFIER_SPECS = [*CURVE_BASE_POOLS, *CURVE_META_POOLS]

mev_inspect/classifiers/specs/erc20.py

@@ -1,15 +1,27 @@
-from mev_inspect.schemas.classified_traces import (
-    Classification,
-    ClassifierSpec,
-)
+from mev_inspect.schemas.classifiers import ClassifierSpec, TransferClassifier
+from mev_inspect.schemas.traces import DecodedCallTrace
+from mev_inspect.schemas.transfers import Transfer
+
+
+class ERC20TransferClassifier(TransferClassifier):
+    @staticmethod
+    def get_transfer(trace: DecodedCallTrace) -> Transfer:
+        return Transfer(
+            block_number=trace.block_number,
+            transaction_hash=trace.transaction_hash,
+            trace_address=trace.trace_address,
+            amount=trace.inputs["amount"],
+            to_address=trace.inputs["recipient"],
+            from_address=trace.inputs.get("sender", trace.from_address),
+            token_address=trace.to_address,
+        )
 
 
 ERC20_SPEC = ClassifierSpec(
     abi_name="ERC20",
-    classifications={
-        "transferFrom(address,address,uint256)": Classification.transfer,
-        "transfer(address,uint256)": Classification.transfer,
-        "burn(address)": Classification.burn,
+    classifiers={
+        "transferFrom(address,address,uint256)": ERC20TransferClassifier,
+        "transfer(address,uint256)": ERC20TransferClassifier,
     },
 )
 

mev_inspect/classifiers/specs/uniswap.py

@@ -1,8 +1,45 @@
-from mev_inspect.schemas.classified_traces import (
-    Classification,
-    ClassifierSpec,
-    Protocol,
-)
+from typing import List, Optional
+
+from mev_inspect.classifiers.helpers import create_swap_from_pool_transfers
+from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+UNISWAP_V2_PAIR_ABI_NAME = "UniswapV2Pair"
+UNISWAP_V3_POOL_ABI_NAME = "UniswapV3Pool"
+
+
+class UniswapV3SwapClassifier(SwapClassifier):
+    @staticmethod
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+
+        recipient_address = trace.inputs.get("recipient", trace.from_address)
+
+        swap = create_swap_from_pool_transfers(
+            trace, recipient_address, prior_transfers, child_transfers
+        )
+        return swap
+
+
+class UniswapV2SwapClassifier(SwapClassifier):
+    @staticmethod
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+
+        recipient_address = trace.inputs.get("to", trace.from_address)
+
+        swap = create_swap_from_pool_transfers(
+            trace, recipient_address, prior_transfers, child_transfers
+        )
+        return swap
 
 
 UNISWAP_V3_CONTRACT_SPECS = [
@@ -65,9 +102,9 @@ UNISWAP_V3_CONTRACT_SPECS = [
 
 UNISWAP_V3_GENERAL_SPECS = [
     ClassifierSpec(
-        abi_name="UniswapV3Pool",
-        classifications={
-            "swap(address,bool,int256,uint160,bytes)": Classification.swap,
+        abi_name=UNISWAP_V3_POOL_ABI_NAME,
+        classifiers={
+            "swap(address,bool,int256,uint160,bytes)": UniswapV3SwapClassifier,
         },
     ),
     ClassifierSpec(
@@ -96,13 +133,13 @@ UNISWAPPY_V2_CONTRACT_SPECS = [
 ]
 
 UNISWAPPY_V2_PAIR_SPEC = ClassifierSpec(
-    abi_name="UniswapV2Pair",
-    classifications={
-        "swap(uint256,uint256,address,bytes)": Classification.swap,
+    abi_name=UNISWAP_V2_PAIR_ABI_NAME,
+    classifiers={
+        "swap(uint256,uint256,address,bytes)": UniswapV2SwapClassifier,
     },
 )
 
-UNISWAP_CLASSIFIER_SPECS = [
+UNISWAP_CLASSIFIER_SPECS: List = [
     *UNISWAP_V3_CONTRACT_SPECS,
     *UNISWAPPY_V2_CONTRACT_SPECS,
     *UNISWAP_V3_GENERAL_SPECS,

mev_inspect/classifiers/specs/weth.py

@@ -1,16 +1,35 @@
-from mev_inspect.schemas.classified_traces import (
-    Classification,
+from mev_inspect.schemas.classifiers import (
     ClassifierSpec,
-    Protocol,
+    DecodedCallTrace,
+    TransferClassifier,
 )
+from mev_inspect.schemas.traces import Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+
+class WethTransferClassifier(TransferClassifier):
+    @staticmethod
+    def get_transfer(trace: DecodedCallTrace) -> Transfer:
+        return Transfer(
+            block_number=trace.block_number,
+            transaction_hash=trace.transaction_hash,
+            trace_address=trace.trace_address,
+            amount=trace.inputs["wad"],
+            to_address=trace.inputs["dst"],
+            from_address=trace.from_address,
+            token_address=trace.to_address,
+        )
+
+
+WETH_ADDRESS = "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"
 
 WETH_SPEC = ClassifierSpec(
     abi_name="WETH9",
     protocol=Protocol.weth,
-    valid_contract_addresses=["0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"],
-    classifications={
-        "transferFrom(address,address,uint256)": Classification.transfer,
-        "transfer(address,uint256)": Classification.transfer,
+    valid_contract_addresses=[WETH_ADDRESS],
+    classifiers={
+        "transferFrom(address,address,uint256)": WethTransferClassifier,
+        "transfer(address,uint256)": WethTransferClassifier,
     },
 )
 

mev_inspect/classifiers/specs/zero_ex.py

@@ -0,0 +1,274 @@
+from typing import List, Optional, Tuple
+
+from mev_inspect.schemas.classifiers import ClassifierSpec, SwapClassifier
+from mev_inspect.schemas.swaps import Swap
+from mev_inspect.schemas.traces import DecodedCallTrace, Protocol
+from mev_inspect.schemas.transfers import Transfer
+
+ANY_TAKER_ADDRESS = "0x0000000000000000000000000000000000000000"
+
+RFQ_SIGNATURES = [
+    "fillRfqOrder((address,address,uint128,uint128,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)",
+    "_fillRfqOrder((address,address,uint128,uint128,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128,address,bool,address)",
+]
+LIMIT_SIGNATURES = [
+    "fillOrKillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)",
+    "fillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)",
+    "_fillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128,address,address)",
+]
+
+
+class ZeroExSwapClassifier(SwapClassifier):
+    @staticmethod
+    def parse_swap(
+        trace: DecodedCallTrace,
+        prior_transfers: List[Transfer],
+        child_transfers: List[Transfer],
+    ) -> Optional[Swap]:
+
+        token_in_address, token_in_amount = _get_0x_token_in_data(
+            trace, child_transfers
+        )
+
+        token_out_address, token_out_amount = _get_0x_token_out_data(trace)
+
+        return Swap(
+            abi_name=trace.abi_name,
+            transaction_hash=trace.transaction_hash,
+            transaction_position=trace.transaction_position,
+            block_number=trace.block_number,
+            trace_address=trace.trace_address,
+            contract_address=trace.to_address,
+            protocol=Protocol.zero_ex,
+            from_address=trace.from_address,
+            to_address=trace.to_address,
+            token_in_address=token_in_address,
+            token_in_amount=token_in_amount,
+            token_out_address=token_out_address,
+            token_out_amount=token_out_amount,
+            error=trace.error,
+        )
+
+
+ZEROX_CONTRACT_SPECS = [
+    ClassifierSpec(
+        abi_name="exchangeProxy",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0xDef1C0ded9bec7F1a1670819833240f027b25EfF"],
+    ),
+    ClassifierSpec(
+        abi_name="exchangeProxyAllowanceTarget",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0xf740b67da229f2f10bcbd38a7979992fcc71b8eb"],
+    ),
+    ClassifierSpec(
+        abi_name="exchangeProxyFlashWallet",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0x22f9dcf4647084d6c31b2765f6910cd85c178c18"],
+    ),
+    ClassifierSpec(
+        abi_name="exchangeProxyGovernor",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0x618f9c67ce7bf1a50afa1e7e0238422601b0ff6e"],
+    ),
+    ClassifierSpec(
+        abi_name="exchangeProxyLiquidityProviderSandbox",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0x407b4128e9ecad8769b2332312a9f655cb9f5f3a"],
+    ),
+    ClassifierSpec(
+        abi_name="exchangeProxyTransformerDeployer",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0x39dce47a67ad34344eab877eae3ef1fa2a1d50bb"],
+    ),
+    ClassifierSpec(
+        abi_name="wethTransformer",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0xb2bc06a4efb20fc6553a69dbfa49b7be938034a7"],
+    ),
+    ClassifierSpec(
+        abi_name="payTakerTransformer",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0x4638a7ebe75b911b995d0ec73a81e4f85f41f24e"],
+    ),
+    ClassifierSpec(
+        abi_name="fillQuoteTransformer",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0x5ce5174d7442061135ea849970ffc7763920e0fd"],
+    ),
+    ClassifierSpec(
+        abi_name="affiliateFeeTransformer",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0xda6d9fc5998f550a094585cf9171f0e8ee3ac59f"],
+    ),
+    ClassifierSpec(
+        abi_name="staking",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0x2a17c35ff147b32f13f19f2e311446eeb02503f3"],
+    ),
+    ClassifierSpec(
+        abi_name="stakingProxy",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0xa26e80e7dea86279c6d778d702cc413e6cffa777"],
+    ),
+    ClassifierSpec(
+        abi_name="zrxToken",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0xe41d2489571d322189246dafa5ebde1f4699f498"],
+    ),
+    ClassifierSpec(
+        abi_name="zrxVault",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0xba7f8b5fb1b19c1211c5d49550fcd149177a5eaf"],
+    ),
+    ClassifierSpec(
+        abi_name="devUtils",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0x74134cf88b21383713e096a5ecf59e297dc7f547"],
+    ),
+    ClassifierSpec(
+        abi_name="etherToken",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2"],
+    ),
+    ClassifierSpec(
+        abi_name="erc20BridgeSampler",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0xd8c38704c9937ea3312de29f824b4ad3450a5e61"],
+    ),
+]
+
+ZEROX_GENERIC_SPECS = [
+    ClassifierSpec(
+        abi_name="IBatchFillNativeOrdersFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="IFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="IFundRecoveryFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="ILiquidityProviderFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="IMetaTransactionsFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="IMultiplexFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="INativeOrdersFeature",
+        protocol=Protocol.zero_ex,
+        valid_contract_addresses=["0xdef1c0ded9bec7f1a1670819833240f027b25eff"],
+        classifiers={
+            "fillOrKillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)": ZeroExSwapClassifier,
+            "fillRfqOrder((address,address,uint128,uint128,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)": ZeroExSwapClassifier,
+            "fillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128)": ZeroExSwapClassifier,
+            "_fillRfqOrder((address,address,uint128,uint128,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128,address,bool,address)": ZeroExSwapClassifier,
+            "_fillLimitOrder((address,address,uint128,uint128,uint128,address,address,address,address,bytes32,uint64,uint256),(uint8,uint8,bytes32,bytes32),uint128,address,address)": ZeroExSwapClassifier,
+        },
+    ),
+    ClassifierSpec(
+        abi_name="IOtcOrdersFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="IOwnableFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="IPancakeSwapFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="ISimpleFunctionRegistryFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="ITestSimpleFunctionRegistryFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="ITokenSpenderFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="ITransformERC20Feature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="IUniswapFeature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="IUniswapV3Feature",
+        protocol=Protocol.zero_ex,
+    ),
+    ClassifierSpec(
+        abi_name="IBootstrapFeature",
+        protocol=Protocol.zero_ex,
+    ),
+]
+
+ZEROX_CLASSIFIER_SPECS = ZEROX_CONTRACT_SPECS + ZEROX_GENERIC_SPECS
+
+
+def _get_taker_token_in_amount(
+    taker_address: str, token_in_address: str, child_transfers: List[Transfer]
+) -> int:
+
+    if len(child_transfers) != 2:
+        raise ValueError(
+            f"A settled order should consist of 2 child transfers, not {len(child_transfers)}."
+        )
+
+    if taker_address == ANY_TAKER_ADDRESS:
+        for transfer in child_transfers:
+            if transfer.token_address == token_in_address:
+                return transfer.amount
+    else:
+        for transfer in child_transfers:
+            if transfer.to_address == taker_address:
+                return transfer.amount
+    return 0
+
+
+def _get_0x_token_in_data(
+    trace: DecodedCallTrace, child_transfers: List[Transfer]
+) -> Tuple[str, int]:
+
+    order: List = trace.inputs["order"]
+    token_in_address = order[0]
+
+    if trace.function_signature in RFQ_SIGNATURES:
+        taker_address = order[5]
+
+    elif trace.function_signature in LIMIT_SIGNATURES:
+        taker_address = order[6]
+
+    else:
+        raise RuntimeError(
+            f"0x orderbook function {trace.function_signature} is not supported"
+        )
+
+    token_in_amount = _get_taker_token_in_amount(
+        taker_address, token_in_address, child_transfers
+    )
+
+    return token_in_address, token_in_amount
+
+
+def _get_0x_token_out_data(trace: DecodedCallTrace) -> Tuple[str, int]:
+
+    order: List = trace.inputs["order"]
+    token_out_address = order[1]
+    token_out_amount = trace.inputs["takerTokenFillAmount"]
+
+    return token_out_address, token_out_amount

mev_inspect/classifiers/specs/zerox.py

@@ -1,166 +0,0 @@
-from mev_inspect.schemas.classified_traces import (
-    ClassifierSpec,
-    Protocol,
-)
-
-
-ZEROX_CONTRACT_SPECS = [
-    ClassifierSpec(
-        abi_name="exchangeProxy",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0xDef1C0ded9bec7F1a1670819833240f027b25EfF"],
-    ),
-    ClassifierSpec(
-        abi_name="exchangeProxyAllowanceTarget",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0xf740b67da229f2f10bcbd38a7979992fcc71b8eb"],
-    ),
-    ClassifierSpec(
-        abi_name="exchangeProxyFlashWallet",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0x22f9dcf4647084d6c31b2765f6910cd85c178c18"],
-    ),
-    ClassifierSpec(
-        abi_name="exchangeProxyGovernor",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0x618f9c67ce7bf1a50afa1e7e0238422601b0ff6e"],
-    ),
-    ClassifierSpec(
-        abi_name="exchangeProxyLiquidityProviderSandbox",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0x407b4128e9ecad8769b2332312a9f655cb9f5f3a"],
-    ),
-    ClassifierSpec(
-        abi_name="exchangeProxyTransformerDeployer",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0x39dce47a67ad34344eab877eae3ef1fa2a1d50bb"],
-    ),
-    ClassifierSpec(
-        abi_name="wethTransformer",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0xb2bc06a4efb20fc6553a69dbfa49b7be938034a7"],
-    ),
-    ClassifierSpec(
-        abi_name="payTakerTransformer",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0x4638a7ebe75b911b995d0ec73a81e4f85f41f24e"],
-    ),
-    ClassifierSpec(
-        abi_name="fillQuoteTransformer",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0x5ce5174d7442061135ea849970ffc7763920e0fd"],
-    ),
-    ClassifierSpec(
-        abi_name="affiliateFeeTransformer",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0xda6d9fc5998f550a094585cf9171f0e8ee3ac59f"],
-    ),
-    ClassifierSpec(
-        abi_name="staking",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0x2a17c35ff147b32f13f19f2e311446eeb02503f3"],
-    ),
-    ClassifierSpec(
-        abi_name="stakingProxy",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0xa26e80e7dea86279c6d778d702cc413e6cffa777"],
-    ),
-    ClassifierSpec(
-        abi_name="zrxToken",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0xe41d2489571d322189246dafa5ebde1f4699f498"],
-    ),
-    ClassifierSpec(
-        abi_name="zrxVault",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0xba7f8b5fb1b19c1211c5d49550fcd149177a5eaf"],
-    ),
-    ClassifierSpec(
-        abi_name="devUtils",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0x74134cf88b21383713e096a5ecf59e297dc7f547"],
-    ),
-    ClassifierSpec(
-        abi_name="etherToken",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2"],
-    ),
-    ClassifierSpec(
-        abi_name="erc20BridgeSampler",
-        protocol=Protocol.zerox,
-        valid_contract_addresses=["0xd8c38704c9937ea3312de29f824b4ad3450a5e61"],
-    ),
-]
-
-ZEROX_GENERIC_SPECS = [
-    ClassifierSpec(
-        abi_name="IBatchFillNativeOrdersFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="IFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="IFundRecoveryFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="ILiquidityProviderFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="IMetaTransactionsFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="IMultiplexFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="INativeOrdersFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="IOtcOrdersFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="IOwnableFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="IPancakeSwapFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="ISimpleFunctionRegistryFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="ITestSimpleFunctionRegistryFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="ITokenSpenderFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="ITransformERC20Feature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="IUniswapFeature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="IUniswapV3Feature",
-        protocol=Protocol.zerox,
-    ),
-    ClassifierSpec(
-        abi_name="IBootstrapFeature",
-        protocol=Protocol.zerox,
-    ),
-]
-
-ZEROX_CLASSIFIER_SPECS = ZEROX_CONTRACT_SPECS + ZEROX_GENERIC_SPECS

mev_inspect/classifiers/trace.py

@@ -2,12 +2,14 @@ from typing import Dict, List, Optional
 
 from mev_inspect.abi import get_abi
 from mev_inspect.decode import ABIDecoder
-from mev_inspect.schemas.blocks import CallAction, CallResult, Trace, TraceType
-from mev_inspect.schemas.classified_traces import (
+from mev_inspect.schemas.blocks import CallAction, CallResult
+from mev_inspect.schemas.traces import (
+    CallTrace,
     Classification,
     ClassifiedTrace,
-    CallTrace,
     DecodedCallTrace,
+    Trace,
+    TraceType,
 )
 
 from .specs import ALL_CLASSIFIER_SPECS
@@ -67,8 +69,11 @@ class TraceClassifier:
 
             if call_data is not None:
                 signature = call_data.function_signature
-                classification = spec.classifications.get(
-                    signature, Classification.unknown
+                classifier = spec.classifiers.get(signature)
+                classification = (
+                    Classification.unknown
+                    if classifier is None
+                    else classifier.get_classification()
                 )
 
                 return DecodedCallTrace(

mev_inspect/coinbase.py

@@ -0,0 +1,40 @@
+import aiohttp
+
+from mev_inspect.classifiers.specs.weth import WETH_ADDRESS
+from mev_inspect.schemas.coinbase import CoinbasePrices, CoinbasePricesResponse
+from mev_inspect.schemas.prices import (
+    AAVE_TOKEN_ADDRESS,
+    LINK_TOKEN_ADDRESS,
+    REN_TOKEN_ADDRESS,
+    UNI_TOKEN_ADDRESS,
+    USDC_TOKEN_ADDRESS_ADDRESS,
+    WBTC_TOKEN_ADDRESS,
+    YEARN_TOKEN_ADDRESS,
+)
+from mev_inspect.schemas.transfers import ETH_TOKEN_ADDRESS
+
+COINBASE_API_BASE = "https://www.coinbase.com/api/v2"
+COINBASE_TOKEN_NAME_BY_ADDRESS = {
+    WETH_ADDRESS: "weth",
+    ETH_TOKEN_ADDRESS: "ethereum",
+    WBTC_TOKEN_ADDRESS: "wrapped-bitcoin",
+    LINK_TOKEN_ADDRESS: "link",
+    YEARN_TOKEN_ADDRESS: "yearn-finance",
+    AAVE_TOKEN_ADDRESS: "aave",
+    UNI_TOKEN_ADDRESS: "uniswap",
+    USDC_TOKEN_ADDRESS_ADDRESS: "usdc",
+    REN_TOKEN_ADDRESS: "ren",
+}
+
+
+async def fetch_coinbase_prices(token_address: str) -> CoinbasePrices:
+    if token_address not in COINBASE_TOKEN_NAME_BY_ADDRESS:
+        raise ValueError(f"Unsupported token_address {token_address}")
+
+    coinbase_token_name = COINBASE_TOKEN_NAME_BY_ADDRESS[token_address]
+    url = f"{COINBASE_API_BASE}/assets/prices/{coinbase_token_name}"
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, params={"base": "USD"}) as response:
+            json_data = await response.json()
+            return CoinbasePricesResponse(**json_data).data.prices

mev_inspect/compound_liquidations.py

@@ -0,0 +1,78 @@
+from typing import List, Optional
+
+from mev_inspect.schemas.liquidations import Liquidation
+from mev_inspect.schemas.traces import Classification, ClassifiedTrace, Protocol
+from mev_inspect.traces import get_child_traces
+
+V2_COMPTROLLER_ADDRESS = "0x3d9819210A31b4961b30EF54bE2aeD79B9c9Cd3B"
+V2_C_ETHER = "0x4Ddc2D193948926D02f9B1fE9e1daa0718270ED5"
+CREAM_COMPTROLLER_ADDRESS = "0x3d5BC3c8d13dcB8bF317092d84783c2697AE9258"
+CREAM_CR_ETHER = "0xD06527D5e56A3495252A528C4987003b712860eE"
+
+
+def get_compound_liquidations(
+    traces: List[ClassifiedTrace],
+) -> List[Liquidation]:
+
+    """Inspect list of classified traces and identify liquidation"""
+    liquidations: List[Liquidation] = []
+
+    for trace in traces:
+        if (
+            trace.classification == Classification.liquidate
+            and (
+                trace.protocol == Protocol.compound_v2
+                or trace.protocol == Protocol.cream
+            )
+            and trace.inputs is not None
+            and trace.to_address is not None
+        ):
+            # First, we look for cEther liquidations (position paid back via tx.value)
+            child_traces = get_child_traces(
+                trace.transaction_hash, trace.trace_address, traces
+            )
+            seize_trace = _get_seize_call(child_traces)
+
+            if seize_trace is not None and seize_trace.inputs is not None:
+                c_token_collateral = trace.inputs["cTokenCollateral"]
+                if trace.abi_name == "CEther":
+                    liquidations.append(
+                        Liquidation(
+                            liquidated_user=trace.inputs["borrower"],
+                            debt_token_address=c_token_collateral,
+                            liquidator_user=seize_trace.inputs["liquidator"],
+                            debt_purchase_amount=trace.value,
+                            protocol=trace.protocol,
+                            received_amount=seize_trace.inputs["seizeTokens"],
+                            received_token_address=trace.to_address,
+                            transaction_hash=trace.transaction_hash,
+                            trace_address=trace.trace_address,
+                            block_number=trace.block_number,
+                        )
+                    )
+                elif (
+                    trace.abi_name == "CToken"
+                ):  # cToken liquidations where liquidator pays back via token transfer
+                    liquidations.append(
+                        Liquidation(
+                            liquidated_user=trace.inputs["borrower"],
+                            debt_token_address=c_token_collateral,
+                            liquidator_user=seize_trace.inputs["liquidator"],
+                            debt_purchase_amount=trace.inputs["repayAmount"],
+                            protocol=trace.protocol,
+                            received_amount=seize_trace.inputs["seizeTokens"],
+                            received_token_address=trace.to_address,
+                            transaction_hash=trace.transaction_hash,
+                            trace_address=trace.trace_address,
+                            block_number=trace.block_number,
+                        )
+                    )
+    return liquidations
+
+
+def _get_seize_call(traces: List[ClassifiedTrace]) -> Optional[ClassifiedTrace]:
+    """Find the call to `seize` in the child traces (successful liquidation)"""
+    for trace in traces:
+        if trace.classification == Classification.seize:
+            return trace
+    return None

mev_inspect/concurrency.py

@@ -0,0 +1,22 @@
+import asyncio
+import signal
+from functools import wraps
+
+
+def coro(f):
+    @wraps(f)
+    def wrapper(*args, **kwargs):
+        loop = asyncio.get_event_loop()
+
+        def cancel_task_callback():
+            for task in asyncio.all_tasks():
+                task.cancel()
+
+        for sig in (signal.SIGINT, signal.SIGTERM):
+            loop.add_signal_handler(sig, cancel_task_callback)
+        try:
+            loop.run_until_complete(f(*args, **kwargs))
+        finally:
+            loop.run_until_complete(loop.shutdown_asyncgens())
+
+    return wrapper

mev_inspect/config.ini

@@ -1,7 +0,0 @@
-[RPC]
-Endpoint = http://localhost:8545/
-
-[ADDRESSES]
-UniswapV2Router = 0x7a250d5630B4cF539739dF2C5dAcb4c659F2488D
-SushiswapV2Router = 0xd9e1cE17f2641f24aE83637ab66a2cca9C378B9F
-WETH = 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2

mev_inspect/config.py

@@ -1,13 +0,0 @@
-import os
-import configparser
-
-
-THIS_FILE_DIRECTORY = os.path.dirname(__file__)
-CONFIG_PATH = os.path.join(THIS_FILE_DIRECTORY, "config.ini")
-
-
-def load_config():
-    config = configparser.ConfigParser()
-    config.read(CONFIG_PATH)
-
-    return config

mev_inspect/crud/blocks.py

@@ -0,0 +1,28 @@
+from datetime import datetime
+
+from mev_inspect.schemas.blocks import Block
+
+
+def delete_block(
+    db_session,
+    block_number: int,
+) -> None:
+    db_session.execute(
+        "DELETE FROM blocks WHERE block_number = :block_number",
+        params={"block_number": block_number},
+    )
+    db_session.commit()
+
+
+def write_block(
+    db_session,
+    block: Block,
+) -> None:
+    db_session.execute(
+        "INSERT INTO blocks (block_number, block_timestamp) VALUES (:block_number, :block_timestamp)",
+        params={
+            "block_number": block.block_number,
+            "block_timestamp": datetime.fromtimestamp(block.block_timestamp),
+        },
+    )
+    db_session.commit()

mev_inspect/crud/classified_traces.py

@@ -1,30 +0,0 @@
-import json
-from typing import List
-
-from mev_inspect.models.classified_traces import ClassifiedTraceModel
-from mev_inspect.schemas.classified_traces import ClassifiedTrace
-
-
-def delete_classified_traces_for_block(
-    db_session,
-    block_number: int,
-) -> None:
-    (
-        db_session.query(ClassifiedTraceModel)
-        .filter(ClassifiedTraceModel.block_number == block_number)
-        .delete()
-    )
-
-    db_session.commit()
-
-
-def write_classified_traces(
-    db_session,
-    classified_traces: List[ClassifiedTrace],
-) -> None:
-    models = [
-        ClassifiedTraceModel(**json.loads(trace.json())) for trace in classified_traces
-    ]
-
-    db_session.bulk_save_objects(models)
-    db_session.commit()

mev_inspect/crud/latest_block_update.py

@@ -0,0 +1,25 @@
+from typing import Optional
+
+
+def find_latest_block_update(db_session) -> Optional[int]:
+    result = db_session.execute(
+        "SELECT block_number FROM latest_block_update LIMIT 1"
+    ).one_or_none()
+    if result is None:
+        return None
+    else:
+        return int(result[0])
+
+
+def update_latest_block(db_session, block_number) -> None:
+    db_session.execute(
+        """
+            UPDATE latest_block_update
+                SET block_number = :block_number, updated_at = current_timestamp;
+            INSERT INTO latest_block_update
+                (block_number, updated_at)
+            SELECT :block_number, current_timestamp
+                WHERE NOT EXISTS (SELECT 1 FROM latest_block_update);
+            """,
+        params={"block_number": block_number},
+    )

mev_inspect/crud/liquidations.py

@@ -0,0 +1,31 @@
+import json
+from typing import List
+
+from mev_inspect.models.liquidations import LiquidationModel
+from mev_inspect.schemas.liquidations import Liquidation
+
+
+def delete_liquidations_for_block(
+    db_session,
+    block_number: int,
+) -> None:
+    (
+        db_session.query(LiquidationModel)
+        .filter(LiquidationModel.block_number == block_number)
+        .delete()
+    )
+
+    db_session.commit()
+
+
+def write_liquidations(
+    db_session,
+    liquidations: List[Liquidation],
+) -> None:
+    models = [
+        LiquidationModel(**json.loads(liquidation.json()))
+        for liquidation in liquidations
+    ]
+
+    db_session.bulk_save_objects(models)
+    db_session.commit()