crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for pulling Cancel and CancelUpTo events

Browse Source
This commit is contained in:
Alex Browne
2018-11-16 12:55:54 -08:00
committed by Fred Carlsen
parent 4a715c30fd
commit d6dff5f86a
6 changed files with 171 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
packages/pipeline/migrations/1542401122477-MakeTakerAddressNullable.ts Normal file
View File

@@ -0,0 +1,17 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class MakeTakerAddressNullable1542401122477 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
await queryRunner.query(
`ALTER TABLE raw.exchange_cancel_events
ALTER COLUMN taker_address DROP NOT NULL;`,
);
}
public async down(queryRunner: QueryRunner): Promise<any> {
await queryRunner.query(
`ALTER TABLE raw.exchange_cancel_events
ALTER COLUMN taker_address SET NOT NULL;`,
);
}
}

50
packages/pipeline/src/data_sources/contract-wrappers/exchange_events.ts
View File

@@ -1,4 +1,12 @@
import { ContractWrappers, ExchangeEvents, ExchangeFillEventArgs, ExchangeWrapper } from '@0x/contract-wrappers'; import {
ContractWrappers,
ExchangeCancelEventArgs,
ExchangeCancelUpToEventArgs,
ExchangeEventArgs,
ExchangeEvents,
ExchangeFillEventArgs,
ExchangeWrapper,
} from '@0x/contract-wrappers';
import { Web3ProviderEngine } from '@0x/subproviders'; import { Web3ProviderEngine } from '@0x/subproviders';
import { Web3Wrapper } from '@0x/web3-wrapper'; import { Web3Wrapper } from '@0x/web3-wrapper';
import { LogWithDecodedArgs } from 'ethereum-types'; import { LogWithDecodedArgs } from 'ethereum-types';
@@ -16,20 +24,41 @@ export class ExchangeEventsSource {
this._exchangeWrapper = contractWrappers.exchange; this._exchangeWrapper = contractWrappers.exchange;
} }
// TODO(albrow): Get Cancel and CancelUpTo events.
public async getFillEventsAsync( public async getFillEventsAsync(
fromBlock: number = EXCHANGE_START_BLOCK, fromBlock?: number,
toBlock?: number, toBlock?: number,
): Promise<Array<LogWithDecodedArgs<ExchangeFillEventArgs>>> { ): Promise<Array<LogWithDecodedArgs<ExchangeFillEventArgs>>> {
return this._getEventsAsync<ExchangeFillEventArgs>(ExchangeEvents.Fill, fromBlock, toBlock);
}
public async getCancelEventsAsync(
fromBlock?: number,
toBlock?: number,
): Promise<Array<LogWithDecodedArgs<ExchangeCancelEventArgs>>> {
return this._getEventsAsync<ExchangeCancelEventArgs>(ExchangeEvents.Cancel, fromBlock, toBlock);
}
public async getCancelUpToEventsAsync(
fromBlock?: number,
toBlock?: number,
): Promise<Array<LogWithDecodedArgs<ExchangeCancelUpToEventArgs>>> {
return this._getEventsAsync<ExchangeCancelUpToEventArgs>(ExchangeEvents.CancelUpTo, fromBlock, toBlock);
}
private async _getEventsAsync<ArgsType extends ExchangeEventArgs>(
eventName: ExchangeEvents,
fromBlock: number = EXCHANGE_START_BLOCK,
toBlock?: number,
): Promise<Array<LogWithDecodedArgs<ArgsType>>> {
const calculatedToBlock = const calculatedToBlock =
toBlock === undefined toBlock === undefined
? (await this._web3Wrapper.getBlockNumberAsync()) - BLOCK_FINALITY_THRESHOLD ? (await this._web3Wrapper.getBlockNumberAsync()) - BLOCK_FINALITY_THRESHOLD
: toBlock; : toBlock;
let events: Array<LogWithDecodedArgs<ExchangeFillEventArgs>> = []; let events: Array<LogWithDecodedArgs<ArgsType>> = [];
for (let currFromBlock = fromBlock; currFromBlock <= calculatedToBlock; currFromBlock += NUM_BLOCKS_PER_QUERY) { for (let currFromBlock = fromBlock; currFromBlock <= calculatedToBlock; currFromBlock += NUM_BLOCKS_PER_QUERY) {
events = events.concat( events = events.concat(
await this._getFillEventsForRangeAsync( await this._getEventsForRangeAsync<ArgsType>(
eventName,
currFromBlock, currFromBlock,
Math.min(currFromBlock + NUM_BLOCKS_PER_QUERY - 1, calculatedToBlock), Math.min(currFromBlock + NUM_BLOCKS_PER_QUERY - 1, calculatedToBlock),
), ),
@@ -38,12 +67,13 @@ export class ExchangeEventsSource {
return events; return events;
} }
private async _getFillEventsForRangeAsync( private async _getEventsForRangeAsync<ArgsType extends ExchangeEventArgs>(
eventName: ExchangeEvents,
fromBlock: number, fromBlock: number,
toBlock: number, toBlock: number,
): Promise<Array<LogWithDecodedArgs<ExchangeFillEventArgs>>> { ): Promise<Array<LogWithDecodedArgs<ArgsType>>> {
return this._exchangeWrapper.getLogsAsync<ExchangeFillEventArgs>( return this._exchangeWrapper.getLogsAsync<ArgsType>(
ExchangeEvents.Fill, eventName,
{ {
fromBlock, fromBlock,
toBlock, toBlock,

6
packages/pipeline/src/entities/index.ts
View File

@@ -1,3 +1,7 @@
import { ExchangeCancelEvent } from './exchange_cancel_event';
import { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event';
import { ExchangeFillEvent } from './exchange_fill_event';
export { Block } from './block'; export { Block } from './block';
export { ExchangeCancelEvent } from './exchange_cancel_event'; export { ExchangeCancelEvent } from './exchange_cancel_event';
export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event'; export { ExchangeCancelUpToEvent } from './exchange_cancel_up_to_event';
@@ -7,3 +11,5 @@ export { SraOrder } from './sra_order';
export { Transaction } from './transaction'; export { Transaction } from './transaction';
export { TokenOnChainMetadata } from './token_on_chain_metadata'; export { TokenOnChainMetadata } from './token_on_chain_metadata';
export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp'; export { SraOrdersObservedTimeStamp, createObservedTimestampForOrder } from './sra_order_observed_timestamp';
export type ExchangeEvent = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent;

60
packages/pipeline/src/parsers/events/index.ts
View File

@@ -1,9 +1,4 @@
import { import { ExchangeCancelEventArgs, ExchangeCancelUpToEventArgs, ExchangeFillEventArgs } from '@0x/contract-wrappers';
ExchangeCancelEventArgs,
ExchangeCancelUpToEventArgs,
ExchangeEventArgs,
ExchangeFillEventArgs,
} from '@0x/contract-wrappers';
import { assetDataUtils } from '@0x/order-utils'; import { assetDataUtils } from '@0x/order-utils';
import { AssetProxyId, ERC721AssetData } from '@0x/types'; import { AssetProxyId, ERC721AssetData } from '@0x/types';
import { LogWithDecodedArgs } from 'ethereum-types'; import { LogWithDecodedArgs } from 'ethereum-types';
@@ -12,39 +7,32 @@ import * as R from 'ramda';
import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent } from '../../entities'; import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeFillEvent } from '../../entities';
import { bigNumbertoStringOrNull } from '../../utils'; import { bigNumbertoStringOrNull } from '../../utils';
export type ExchangeEventEntity = ExchangeFillEvent | ExchangeCancelEvent | ExchangeCancelUpToEvent; /**
* Parses raw event logs for a fill event and returns an array of
export const parseExchangeEvents: ( * ExchangeFillEvent entities.
eventLogs: Array<LogWithDecodedArgs<ExchangeEventArgs>>, * @param eventLogs Raw event logs (e.g. returned from contract-wrappers).
) => ExchangeEventEntity[] = R.map(_convertToEntity); */
export const parseExchangeFillEvents: (
eventLogs: Array<LogWithDecodedArgs<ExchangeFillEventArgs>>,
) => ExchangeFillEvent[] = R.map(_convertToExchangeFillEvent);
/** /**
* Converts a raw event log to an Entity. Automatically detects the type of * Parses raw event logs for a cancel event and returns an array of
* event and returns the appropriate entity type. Throws for unknown event * ExchangeCancelEvent entities.
* types. * @param eventLogs Raw event logs (e.g. returned from contract-wrappers).
* @param eventLog Raw event log (e.g. returned from contract-wrappers).
*/ */
export function _convertToEntity(eventLog: LogWithDecodedArgs<ExchangeEventArgs>): ExchangeEventEntity { export const parseExchangeCancelEvents: (
switch (eventLog.event) { eventLogs: Array<LogWithDecodedArgs<ExchangeCancelEventArgs>>,
case 'Fill': ) => ExchangeCancelEvent[] = R.map(_convertToExchangeCancelEvent);
// tslint has a false positive here. We need to type assert in order
// to change the type argument to the more specific /**
// ExchangeFillEventArgs. * Parses raw event logs for a CancelUpTo event and returns an array of
// tslint:disable-next-line:no-unnecessary-type-assertion * ExchangeCancelUpToEvent entities.
return _convertToExchangeFillEvent(eventLog as LogWithDecodedArgs<ExchangeFillEventArgs>); * @param eventLogs Raw event logs (e.g. returned from contract-wrappers).
case 'Cancel': */
// tslint:disable-next-line:no-unnecessary-type-assertion export const parseExchangeCancelUpToEvents: (
return _convertToExchangeCancelEvent(eventLog as LogWithDecodedArgs<ExchangeCancelEventArgs>); eventLogs: Array<LogWithDecodedArgs<ExchangeCancelUpToEventArgs>>,
case 'CancelUpTo': ) => ExchangeCancelUpToEvent[] = R.map(_convertToExchangeCancelUpToEvent);
// tslint:disable-next-line:no-unnecessary-type-assertion
return _convertToExchangeCancelUpToEvent(eventLog as LogWithDecodedArgs<ExchangeCancelUpToEventArgs>);
default:
// Another false positive here. We are adding two strings, but
// tslint seems confused about the types.
// tslint:disable-next-line:restrict-plus-operands
throw new Error('unexpected eventLog.event type: ' + eventLog.event);
}
}
/** /**
* Converts a raw event log for a fill event into an ExchangeFillEvent entity. * Converts a raw event log for a fill event into an ExchangeFillEvent entity.

123
packages/pipeline/src/scripts/pull_missing_events.ts
View File

@@ -1,14 +1,13 @@
// tslint:disable:no-console // tslint:disable:no-console
import { web3Factory } from '@0x/dev-utils'; import { web3Factory } from '@0x/dev-utils';
import { Web3ProviderEngine } from '@0x/subproviders';
import R = require('ramda'); import R = require('ramda');
import 'reflect-metadata'; import 'reflect-metadata';
import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm'; import { Connection, ConnectionOptions, createConnection, Repository } from 'typeorm';
import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange_events'; import { ExchangeEventsSource } from '../data_sources/contract-wrappers/exchange_events';
import { ExchangeFillEvent } from '../entities'; import { ExchangeCancelEvent, ExchangeCancelUpToEvent, ExchangeEvent, ExchangeFillEvent } from '../entities';
import * as ormConfig from '../ormconfig'; import * as ormConfig from '../ormconfig';
import { ExchangeEventEntity, parseExchangeEvents } from '../parsers/events'; import { parseExchangeCancelEvents, parseExchangeCancelUpToEvents, parseExchangeFillEvents } from '../parsers/events';
import { handleError } from '../utils'; import { handleError } from '../utils';
const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet. const EXCHANGE_START_BLOCK = 6271590; // Block number when the Exchange contract was deployed to mainnet.
@@ -22,40 +21,88 @@ let connection: Connection;
const provider = web3Factory.getRpcProvider({ const provider = web3Factory.getRpcProvider({
rpcUrl: 'https://mainnet.infura.io', rpcUrl: 'https://mainnet.infura.io',
}); });
await getExchangeEventsAsync(provider); const eventsSource = new ExchangeEventsSource(provider, 1);
await getFillEventsAsync(eventsSource);
await getCancelEventsAsync(eventsSource);
await getCancelUpToEventsAsync(eventsSource);
process.exit(0); process.exit(0);
})().catch(handleError); })().catch(handleError);
async function getExchangeEventsAsync(provider: Web3ProviderEngine): Promise<void> { async function getFillEventsAsync(eventsSource: ExchangeEventsSource): Promise<void> {
console.log('Checking existing event logs...'); console.log('Checking existing fill events...');
const eventsRepository = connection.getRepository(ExchangeFillEvent); const repository = connection.getRepository(ExchangeFillEvent);
const manager = connection.createEntityManager(); const startBlock = await getStartBlockAsync(repository);
const startBlock = await getStartBlockAsync(eventsRepository); console.log(`Getting fill events starting at ${startBlock}...`);
console.log(`Getting event logs starting at ${startBlock}...`); const eventLogs = await eventsSource.getFillEventsAsync(startBlock);
const exchangeEvents = new ExchangeEventsSource(provider, 1); console.log('Parsing fill events...');
const eventLogs = await exchangeEvents.getFillEventsAsync(startBlock); const events = parseExchangeFillEvents(eventLogs);
console.log('Parsing events...'); console.log(`Retrieved and parsed ${events.length} total fill events.`);
const events = parseExchangeEvents(eventLogs); await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events);
console.log(`Retrieved and parsed ${events.length} total events.`); }
console.log('Saving events...');
if (startBlock === EXCHANGE_START_BLOCK) { async function getCancelEventsAsync(eventsSource: ExchangeEventsSource): Promise<void> {
console.log('Checking existing cancel events...');
const repository = connection.getRepository(ExchangeCancelEvent);
const startBlock = await getStartBlockAsync(repository);
console.log(`Getting cancel events starting at ${startBlock}...`);
const eventLogs = await eventsSource.getCancelEventsAsync(startBlock);
console.log('Parsing cancel events...');
const events = parseExchangeCancelEvents(eventLogs);
console.log(`Retrieved and parsed ${events.length} total cancel events.`);
await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events);
}
async function getCancelUpToEventsAsync(eventsSource: ExchangeEventsSource): Promise<void> {
console.log('Checking existing CancelUpTo events...');
const repository = connection.getRepository(ExchangeCancelUpToEvent);
const startBlock = await getStartBlockAsync(repository);
console.log(`Getting CancelUpTo events starting at ${startBlock}...`);
const eventLogs = await eventsSource.getCancelUpToEventsAsync(startBlock);
console.log('Parsing CancelUpTo events...');
const events = parseExchangeCancelUpToEvents(eventLogs);
console.log(`Retrieved and parsed ${events.length} total CancelUpTo events.`);
await saveEventsAsync(startBlock === EXCHANGE_START_BLOCK, repository, events);
}
async function getStartBlockAsync<T extends ExchangeEvent>(repository: Repository<T>): Promise<number> {
const fillEventCount = await repository.count();
if (fillEventCount === 0) {
console.log(`No existing ${repository.metadata.name}s found.`);
return EXCHANGE_START_BLOCK;
}
const queryResult = await connection.query(
// TODO(albrow): Would prefer to use a prepared statement here to reduce
// surface area for SQL injections, but it doesn't appear to be working.
`SELECT block_number FROM raw.${repository.metadata.tableName} ORDER BY block_number DESC LIMIT 1`,
);
const lastKnownBlock = queryResult[0].block_number;
return lastKnownBlock - START_BLOCK_OFFSET;
}
async function saveEventsAsync<T extends ExchangeEvent>(
isInitialPull: boolean,
repository: Repository<T>,
events: T[],
): Promise<void> {
console.log(`Saving ${repository.metadata.name}s...`);
if (isInitialPull) {
// Split data into numChunks pieces of maximum size BATCH_SAVE_SIZE // Split data into numChunks pieces of maximum size BATCH_SAVE_SIZE
// each. // each.
for (const eventsBatch of R.splitEvery(BATCH_SAVE_SIZE, events)) { for (const eventsBatch of R.splitEvery(BATCH_SAVE_SIZE, events)) {
await eventsRepository.insert(eventsBatch); await repository.insert(eventsBatch);
} }
} else { } else {
// If we possibly have some overlap where we need to update some // If we possibly have some overlap where we need to update some
// existing events, we need to use our workaround/fallback. // existing events, we need to use our workaround/fallback.
await saveIndividuallyWithFallbackAsync(eventsRepository, events); await saveIndividuallyWithFallbackAsync(repository, events);
} }
const totalEvents = await eventsRepository.count(); const totalEvents = await repository.count();
console.log(`Done saving events. There are now ${totalEvents} total events.`); console.log(`Done saving events. There are now ${totalEvents} total ${repository.metadata.name}s.`);
} }
async function saveIndividuallyWithFallbackAsync( async function saveIndividuallyWithFallbackAsync<T extends ExchangeEvent>(
eventsRepository: Repository<ExchangeFillEvent>, repository: Repository<T>,
events: ExchangeEventEntity[], events: T[],
): Promise<void> { ): Promise<void> {
// Note(albrow): This is a temporary hack because `save` is not working as // Note(albrow): This is a temporary hack because `save` is not working as
// documented and is causing a foreign key constraint violation. Hopefully // documented and is causing a foreign key constraint violation. Hopefully
@@ -63,32 +110,24 @@ async function saveIndividuallyWithFallbackAsync(
// on one event at a time and is therefore much slower. // on one event at a time and is therefore much slower.
for (const event of events) { for (const event of events) {
try { try {
// First try and insert. // First try an insert.
await eventsRepository.insert(event); await repository.insert(event);
} catch { } catch {
// If it fails, assume it was a foreign key constraint error and try // If it fails, assume it was a foreign key constraint error and try
// doing an update instead. // doing an update instead.
await eventsRepository.update( // Note(albrow): Unfortunately the `as any` hack here seems
// required. I can't figure out how to convince the type-checker
// that the criteria and the entity itself are the correct type for
// the given repository. If we can remove the `save` hack then this
// will probably no longer be necessary.
await repository.update(
{ {
contractAddress: event.contractAddress, contractAddress: event.contractAddress,
blockNumber: event.blockNumber, blockNumber: event.blockNumber,
logIndex: event.logIndex, logIndex: event.logIndex,
}, } as any,
event, event as any,
); );
} }
} }
} }
async function getStartBlockAsync(eventsRepository: Repository<ExchangeFillEvent>): Promise<number> {
const fillEventCount = await eventsRepository.count();
if (fillEventCount === 0) {
console.log('No existing fill events found.');
return EXCHANGE_START_BLOCK;
}
const queryResult = await connection.query(
'SELECT block_number FROM raw.exchange_fill_events ORDER BY block_number DESC LIMIT 1',
);
const lastKnownBlock = queryResult[0].block_number;
return lastKnownBlock - START_BLOCK_OFFSET;
}

6
packages/pipeline/test/parsers/events/index_test.ts
View File

@@ -5,7 +5,7 @@ import { LogWithDecodedArgs } from 'ethereum-types';
import 'mocha'; import 'mocha';
import { ExchangeFillEvent } from '../../../src/entities'; import { ExchangeFillEvent } from '../../../src/entities';
import { _convertToEntity } from '../../../src/parsers/events'; import { _convertToExchangeFillEvent } from '../../../src/parsers/events';
import { chaiSetup } from '../../utils/chai_setup'; import { chaiSetup } from '../../utils/chai_setup';
chaiSetup.configure(); chaiSetup.configure();
@@ -13,7 +13,7 @@ const expect = chai.expect;
// tslint:disable:custom-no-magic-numbers // tslint:disable:custom-no-magic-numbers
describe('exchange_events', () => { describe('exchange_events', () => {
describe('_convertToEntity', () => { describe('_convertToExchangeFillEvent', () => {
it('converts LogWithDecodedArgs to ExchangeFillEvent entity', () => { it('converts LogWithDecodedArgs to ExchangeFillEvent entity', () => {
const input: LogWithDecodedArgs<ExchangeFillEventArgs> = { const input: LogWithDecodedArgs<ExchangeFillEventArgs> = {
logIndex: 102, logIndex: 102,
@@ -71,7 +71,7 @@ describe('exchange_events', () => {
expected.takerAssetProxyId = '0xf47261b0'; expected.takerAssetProxyId = '0xf47261b0';
expected.takerTokenAddress = '0xe41d2489571d322189246dafa5ebde1f4699f498'; expected.takerTokenAddress = '0xe41d2489571d322189246dafa5ebde1f4699f498';
expected.takerTokenId = null; expected.takerTokenId = null;
const actual = _convertToEntity(input); const actual = _convertToExchangeFillEvent(input);
expect(actual).deep.equal(expected); expect(actual).deep.equal(expected);
}); });
}); });