Add support for decoding asset data

packages/pipeline/package.json

@@ -40,7 +40,9 @@
     },
     "dependencies": {
         "@0xproject/contract-wrappers": "^1.0.1",
+        "@0xproject/order-utils": "^1.0.2",
         "@0xproject/subproviders": "^2.0.2",
+        "@0xproject/types": "^1.0.1",
         "@0xproject/utils": "^1.0.8",
         "@types/ramda": "^0.25.38",
         "axios": "^0.18.0",

packages/pipeline/src/data-sources/etherscan/events.ts

@@ -1,8 +1,15 @@
-import { ExchangeEventArgs } from '@0xproject/contract-wrappers';
-import { AbiDecoder } from '@0xproject/utils';
+import { ExchangeEventArgs, ExchangeFillEventArgs } from '@0xproject/contract-wrappers';
+import { assetDataUtils } from '@0xproject/order-utils';
+import { AssetProxyId, ERC721AssetData } from '@0xproject/types';
+import { AbiDecoder, BigNumber } from '@0xproject/utils';
 import { AbiDefinition, LogEntry, LogWithDecodedArgs } from 'ethereum-types';
 import * as R from 'ramda';
 
+import { ExchangeFillEvent } from '../../entities/ExchangeFillEvent';
+
+// TODO(albrow): Union with other exchange event entity types
+export type ExchangeEventEntity = ExchangeFillEvent;
+
 // Raw events response from etherescan.io
 export interface EventsResponse {
     status: string;
@@ -56,17 +63,74 @@ export const _decodeLogEntry = R.curry((contractAbi: AbiDefinition[], log: LogEn
     return logWithDecodedArgs as LogWithDecodedArgs<ExchangeEventArgs>;
 });
 
+export function _convertToEntity(eventLog: LogWithDecodedArgs<ExchangeEventArgs>): ExchangeEventEntity {
+    switch (eventLog.event) {
+        case 'Fill':
+            return _convertToExchangeFillEvent(eventLog as LogWithDecodedArgs<ExchangeFillEventArgs>);
+        default:
+            throw new Error('unexpected eventLog.event type: ' + eventLog.event);
+    }
+}
+
+export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<ExchangeFillEventArgs>): ExchangeFillEvent {
+    const makerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.makerAssetData);
+    const makerAssetType = makerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
+    const takerAssetData = assetDataUtils.decodeAssetDataOrThrow(eventLog.args.takerAssetData);
+    const takerAssetType = takerAssetData.assetProxyId === AssetProxyId.ERC20 ? 'erc20' : 'erc721';
+    const exchangeFillEvent = new ExchangeFillEvent();
+    exchangeFillEvent.logIndex = eventLog.logIndex as number;
+    exchangeFillEvent.address = eventLog.address as string;
+    exchangeFillEvent.rawData = eventLog.data as string;
+    exchangeFillEvent.blockNumber = eventLog.blockNumber as number;
+    exchangeFillEvent.makerAddress = eventLog.args.makerAddress.toString();
+    exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString();
+    exchangeFillEvent.feeRecepientAddress = eventLog.args.feeRecipientAddress;
+    exchangeFillEvent.senderAddress = eventLog.args.senderAddress;
+    exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount.toString();
+    exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount.toString();
+    exchangeFillEvent.makerFeePaid = eventLog.args.makerFeePaid.toString();
+    exchangeFillEvent.takerFeePaid = eventLog.args.takerFeePaid.toString();
+    exchangeFillEvent.orderHash = eventLog.args.orderHash;
+    exchangeFillEvent.rawMakerAssetData = eventLog.args.makerAssetData;
+    exchangeFillEvent.makerAssetType = makerAssetType;
+    exchangeFillEvent.makerAssetProxyId = makerAssetData.assetProxyId;
+    exchangeFillEvent.makerTokenAddress = makerAssetData.tokenAddress;
+    exchangeFillEvent.makerTokenId = bigNumbertoStringOrNull((makerAssetData as ERC721AssetData).tokenId);
+    exchangeFillEvent.rawTakerAssetData = eventLog.args.takerAssetData;
+    exchangeFillEvent.takerAssetType = takerAssetType;
+    exchangeFillEvent.takerAssetProxyId = takerAssetData.assetProxyId;
+    exchangeFillEvent.takerTokenAddress = takerAssetData.tokenAddress;
+    exchangeFillEvent.takerTokenId = bigNumbertoStringOrNull((takerAssetData as ERC721AssetData).tokenId);
+    return exchangeFillEvent;
+}
+
+function bigNumbertoStringOrNull(n: BigNumber): string | null {
+    if (n == null) {
+        return null;
+    }
+    return n.toString();
+}
+
+function filterEventLogs(
+    eventLogs: Array<LogWithDecodedArgs<ExchangeEventArgs>>,
+): Array<LogWithDecodedArgs<ExchangeEventArgs>> {
+    return R.filter(eventLog => eventLog.event === 'Fill', eventLogs);
+}
+
 /**
  * Parses and abi-decodes the raw events response from etherscan.io.
  * @param contractAbi The ABI for the contract that the events where emited from.
  * @param rawEventsResponse The raw events response from etherescan.io.
- * @returns Parsed and decoded events.
+ * @returns Parsed and decoded event entities, ready to be saved to database.
  */
 export function parseRawEventsResponse(
     contractAbi: AbiDefinition[],
     rawEventsResponse: EventsResponse,
-): Array<LogWithDecodedArgs<ExchangeEventArgs>> {
-    return R.pipe(R.map(_convertResponseToLogEntry), R.map(_decodeLogEntry(contractAbi)))(rawEventsResponse.result);
+): ExchangeEventEntity[] {
+    return R.pipe(
+        R.map(_convertResponseToLogEntry),
+        R.map(_decodeLogEntry(contractAbi)),
+        filterEventLogs,
+        R.map(_convertToEntity),
+    )(rawEventsResponse.result);
 }
-
-// export const parseRawEventsResponse = R.pipe(R.map(_convertResponseToLogEntry), R.map(_decodeLogEntry));

packages/pipeline/src/data-sources/etherscan/index.ts

@@ -1,7 +1,7 @@
 import { default as axios } from 'axios';
-import { AbiDefinition, BlockParam, BlockParamLiteral, DecodedLogArgs, LogWithDecodedArgs } from 'ethereum-types';
+import { AbiDefinition, BlockParam, BlockParamLiteral } from 'ethereum-types';
 
-import { EventsResponse, parseRawEventsResponse } from './events';
+import { EventsResponse, ExchangeEventEntity, parseRawEventsResponse } from './events';
 
 const ETHERSCAN_URL = 'https://api.etherscan.io/api';
 
@@ -24,7 +24,7 @@ export class Etherscan {
         contractAbi: AbiDefinition[],
         fromBlock: BlockParam = BlockParamLiteral.Earliest,
         toBlock: BlockParam = BlockParamLiteral.Latest,
-    ): Promise<Array<LogWithDecodedArgs<DecodedLogArgs>>> {
+    ): Promise<ExchangeEventEntity[]> {
         const fullURL = `${ETHERSCAN_URL}?module=logs&action=getLogs&address=${contractAddress}&fromBlock=${fromBlock}&toBlock=${toBlock}&apikey=${
             this._apiKey
         }`;

packages/pipeline/src/entities/ExchangeFillEvent.ts

@@ -1,5 +1,7 @@
 import { Column, Entity, PrimaryColumn } from 'typeorm';
 
+export type ExchangeFillEventAssetType = 'erc20' | 'erc721';
+
 @Entity()
 export class ExchangeFillEvent {
     @PrimaryColumn() public logIndex!: number;
@@ -17,8 +19,17 @@ export class ExchangeFillEvent {
     @Column() public makerFeePaid!: string;
     @Column() public takerFeePaid!: string;
     @Column() public orderHash!: string;
-    // TODO(albrow): Decode asset data.
-    @Column() public makerAssetData!: string;
-    @Column() public takerAssetData!: string;
+    @Column() public rawMakerAssetData!: string;
+    @Column() public makerAssetType!: ExchangeFillEventAssetType;
+    @Column() public makerAssetProxyId!: string;
+    @Column() public makerTokenAddress!: string;
+    @Column({ nullable: true, type: String })
+    public makerTokenId!: string | null;
+    @Column() public rawTakerAssetData!: string;
+    @Column() public takerAssetType!: ExchangeFillEventAssetType;
+    @Column() public takerAssetProxyId!: string;
+    @Column() public takerTokenAddress!: string;
+    @Column({ nullable: true, type: String })
+    public takerTokenId!: string | null;
     // TODO(albrow): Include topics?
 }

packages/pipeline/src/index.ts

@@ -1,4 +1,5 @@
 import { ExchangeFillEventArgs } from '@0xproject/contract-wrappers';
+import { assetDataUtils } from '@0xproject/order-utils';
 import { LogWithDecodedArgs } from 'ethereum-types';
 import 'reflect-metadata';
 import { createConnection } from 'typeorm';
@@ -14,32 +15,12 @@ const etherscan = new Etherscan(process.env.ETHERSCAN_API_KEY as string);
     const connection = await createConnection(config);
     const repository = connection.getRepository(ExchangeFillEvent);
     console.log(`found ${await repository.count()} existing fill events`);
-    const eventLogs = await etherscan.getContractEventsAsync(
+    const events = await etherscan.getContractEventsAsync(
         '0x4f833a24e1f95d70f028921e27040ca56e09ab0b',
         artifacts.Exchange.compilerOutput.abi,
     );
-    for (const eventLog of eventLogs) {
-        if (eventLog.event !== 'Fill') {
-            continue;
-        }
-        const fillEventLog = eventLog as LogWithDecodedArgs<ExchangeFillEventArgs>;
-        const exchangeFillEvent = new ExchangeFillEvent();
-        exchangeFillEvent.logIndex = fillEventLog.logIndex as number;
-        exchangeFillEvent.address = fillEventLog.address as string;
-        exchangeFillEvent.rawData = fillEventLog.data as string;
-        exchangeFillEvent.blockNumber = fillEventLog.blockNumber as number;
-        exchangeFillEvent.makerAddress = fillEventLog.args.makerAddress.toString();
-        exchangeFillEvent.takerAddress = fillEventLog.args.takerAddress.toString();
-        exchangeFillEvent.feeRecepientAddress = fillEventLog.args.feeRecipientAddress;
-        exchangeFillEvent.senderAddress = fillEventLog.args.senderAddress;
-        exchangeFillEvent.makerAssetFilledAmount = fillEventLog.args.makerAssetFilledAmount.toString();
-        exchangeFillEvent.takerAssetFilledAmount = fillEventLog.args.takerAssetFilledAmount.toString();
-        exchangeFillEvent.makerFeePaid = fillEventLog.args.makerFeePaid.toString();
-        exchangeFillEvent.takerFeePaid = fillEventLog.args.takerFeePaid.toString();
-        exchangeFillEvent.orderHash = fillEventLog.args.orderHash;
-        exchangeFillEvent.makerAssetData = fillEventLog.args.makerAssetData;
-        exchangeFillEvent.takerAssetData = fillEventLog.args.takerAssetData;
-        await repository.save(exchangeFillEvent);
+    for (const event of events) {
+        await repository.save(event);
     }
     console.log(`now ${await repository.count()} total fill events`);
 })();

yarn.lock

@@ -6225,9 +6225,9 @@ ethereumjs-wallet@0.6.0:
     utf8 "^2.1.1"
     uuid "^2.0.1"
 
-ethers@3.0.22:
-  version "3.0.22"
-  resolved "https://registry.yarnpkg.com/ethers/-/ethers-3.0.22.tgz#7fab1ea16521705837aa43c15831877b2716b436"
+ethers@0xproject/ethers.js#eip-838-reasons, ethers@3.0.22:
+  version "3.0.18"
+  resolved "https://codeload.github.com/0xproject/ethers.js/tar.gz/b91342bd200d142af0165d6befddf783c8ae8447"
   dependencies:
     aes-js "3.0.0"
     bn.js "^4.4.0"