crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update script to work with existing v1 pipeline data

Browse Source
This commit is contained in:
Alex Browne
2018-10-18 17:44:48 -07:00
committed by Fred Carlsen
parent 4912affbe8
commit 1544e5ed9f
7 changed files with 68 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
packages/pipeline/src/entities/ExchangeCancelEvent.ts
View File

@@ -1,15 +1,16 @@
import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm';
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { AssetType } from '../types';
@Entity()
export class ExchangeCancelEvent extends BaseEntity {
export class ExchangeCancelEvent {
@PrimaryColumn() public contractAddress!: string;
@PrimaryColumn() public logIndex!: number;
@PrimaryColumn() public blockNumber!: number;
@Column() public rawData!: string;
// TODO(albrow): Include transaction hash
@Column() public makerAddress!: string;
@Column({ nullable: true, type: String })
public takerAddress!: string;

5
packages/pipeline/src/entities/ExchangeCancelUpToEvent.ts
View File

@@ -1,11 +1,12 @@
import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm';
import { Column, Entity, PrimaryColumn } from 'typeorm';
@Entity()
export class ExchangeCancelUpToEvent extends BaseEntity {
export class ExchangeCancelUpToEvent {
@PrimaryColumn() public contractAddress!: string;
@PrimaryColumn() public logIndex!: number;
@PrimaryColumn() public blockNumber!: number;
// TODO(albrow): Include transaction hash
@Column() public rawData!: string;
@Column() public makerAddress!: string;

7
packages/pipeline/src/entities/ExchangeFillEvent.ts
View File

@@ -1,18 +1,19 @@
import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm';
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { AssetType } from '../types';
@Entity()
export class ExchangeFillEvent extends BaseEntity {
export class ExchangeFillEvent {
@PrimaryColumn() public contractAddress!: string;
@PrimaryColumn() public logIndex!: number;
@PrimaryColumn() public blockNumber!: number;
@Column() public rawData!: string;
@Column() public transactionHash!: string;
@Column() public makerAddress!: string;
@Column() public takerAddress!: string;
@Column() public feeRecepientAddress!: string;
@Column() public feeRecipientAddress!: string;
@Column() public senderAddress!: string;
@Column() public makerAssetFilledAmount!: string;
@Column() public takerAssetFilledAmount!: string;

4
packages/pipeline/src/entities/SraOrder.ts
View File

@@ -1,9 +1,9 @@
import { BaseEntity, Column, Entity, PrimaryColumn } from 'typeorm';
import { Column, Entity, PrimaryColumn } from 'typeorm';
import { AssetType } from '../types';
@Entity()
export class SraOrder extends BaseEntity {
export class SraOrder {
@PrimaryColumn() public exchangeAddress!: string;
@PrimaryColumn() public orderHashHex!: string;

76
packages/pipeline/src/index.ts
View File

@@ -1,50 +1,78 @@
import { HttpClient } from '@0x/connect';
import { web3Factory } from '@0x/dev-utils';
import 'reflect-metadata';
import { Connection, createConnection } from 'typeorm';
import { ExchangeEventsSource } from './data_sources/contract-wrappers/exchange_events';
import { SraOrder } from './entities/SraOrder';
import { config } from './ormconfig';
import { deployConfig } from './ormconfig';
import { parseExchangeEvents } from './parsers/events';
import { parseSraOrders } from './parsers/sra_orders';
import { ExchangeFillEvent } from './entities/ExchangeFillEvent';
let connection: Connection;
(async () => {
connection = await createConnection(config);
connection = await createConnection(deployConfig);
await getExchangeEventsAsync();
// await getSraOrdersAsync();
await mergeExchangeEventsAsync();
console.log('Exiting process');
process.exit(0);
})();
// TODO(albrow): Separately: Errors do not appear to be handled correctly. If you use the
// wrong rpcUrl it just returns early with no error.
async function getExchangeEventsAsync(): Promise<void> {
console.log('Getting event logs...');
const provider = web3Factory.getRpcProvider({
rpcUrl: 'https://mainnet.infura.io',
});
const eventsRepository = connection.getRepository(ExchangeFillEvent);
const exchangeEvents = new ExchangeEventsSource(provider, 1);
const eventLogs = await exchangeEvents.getFillEventsAsync();
console.log('Parsing events...');
const events = parseExchangeEvents(eventLogs);
console.log('Got events: ' + events.length);
for (const event of events) {
await event.save();
}
console.log(`Retrieved and parsed ${events.length} total events.`);
console.log('Saving events...');
eventsRepository.save(events);
console.log('Saved events.');
console.log('Exiting process');
process.exit(0);
}
async function getSraOrdersAsync(): Promise<void> {
const orderRepository = connection.getRepository(SraOrder);
console.log(`found ${await orderRepository.count()} existing orders`);
const sraUrl = 'https://api.radarrelay.com/0x/v2';
const connect = new HttpClient(sraUrl);
const rawOrders = await connect.getOrdersAsync();
const orders = parseSraOrders(rawOrders);
for (const order of orders) {
order.sourceUrl = sraUrl;
await order.save();
}
console.log(`now there are ${await orderRepository.count()} total orders`);
const insertEventsRawQuery = `INSERT INTO events_raw (
event_type,
error_id,
order_hash,
maker,
maker_amount,
maker_fee,
maker_token,
taker,
taker_amount,
taker_fee,
taker_token,
txn_hash,
fee_recipient,
block_number,
log_index
)
(
SELECT
'LogFill',
null,
"orderHash",
"makerAddress",
"makerAssetFilledAmount"::numeric(78),
"makerFeePaid"::numeric(78),
"makerTokenAddress",
"takerAddress",
"takerAssetFilledAmount"::numeric(78),
"takerFeePaid"::numeric(78),
"takerTokenAddress",
"transactionHash",
"feeRecipientAddress",
"blockNumber",
"logIndex"
FROM exchange_fill_event
) ON CONFLICT (order_hash, txn_hash, log_index) DO NOTHING`;
async function mergeExchangeEventsAsync(): Promise<void> {
console.log('Merging results into events_raw...');
await connection.query(insertEventsRawQuery);
}

3
packages/pipeline/src/parsers/events/index.ts
View File

@@ -43,9 +43,10 @@ export function _convertToExchangeFillEvent(eventLog: LogWithDecodedArgs<Exchang
exchangeFillEvent.blockNumber = eventLog.blockNumber as number;
exchangeFillEvent.logIndex = eventLog.logIndex as number;
exchangeFillEvent.rawData = eventLog.data as string;
exchangeFillEvent.transactionHash = eventLog.transactionHash;
exchangeFillEvent.makerAddress = eventLog.args.makerAddress.toString();
exchangeFillEvent.takerAddress = eventLog.args.takerAddress.toString();
exchangeFillEvent.feeRecepientAddress = eventLog.args.feeRecipientAddress;
exchangeFillEvent.feeRecipientAddress = eventLog.args.feeRecipientAddress;
exchangeFillEvent.senderAddress = eventLog.args.senderAddress;
exchangeFillEvent.makerAssetFilledAmount = eventLog.args.makerAssetFilledAmount.toString();
exchangeFillEvent.takerAssetFilledAmount = eventLog.args.takerAssetFilledAmount.toString();

3
packages/pipeline/test/parsers/events/index_test.ts
View File

@@ -51,9 +51,10 @@ describe('exchange_events', () => {
expected.logIndex = 102;
expected.rawData =
'0x000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000f6da68519f78b0d0bc93c701e86affcb75c92428000000000000000000000000000000000000000000000000002386f26fc10000000000000000000000000000000000000000000000000000016345785d8a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000024f47261b0000000000000000000000000e41d2489571d322189246dafa5ebde1f4699f49800000000000000000000000000000000000000000000000000000000';
expected.transactionHash = '0x6dd106d002873746072fc5e496dd0fb2541b68c77bcf9184ae19a42fd33657fe';
expected.makerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
expected.takerAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
expected.feeRecepientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd';
expected.feeRecipientAddress = '0xc370d2a5920344aa6b7d8d11250e3e861434cbdd';
expected.senderAddress = '0xf6da68519f78b0d0bc93c701e86affcb75c92428';
expected.makerAssetFilledAmount = '10000000000000000';
expected.takerAssetFilledAmount = '100000000000000000';