crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1612 from 0xProject/feature/pipeline/add-radar

Browse Source 
[pipeline] Add Radar orders and maker_address column to token_orderbook_snapshots
This commit is contained in:
Francesco Agosti
2019-02-20 14:17:42 -08:00
committed by GitHub
parent 154e7a6a8a 4952cd5afb
commit 39390865da
22 changed files with 499 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
packages/pipeline/migrations/1550163069315-TokenOrderBookSnapshotsAddMakerAddress.ts Normal file
View File

@@ -0,0 +1,40 @@
import { MigrationInterface, QueryRunner, TableColumn } from 'typeorm';
const TOKEN_ORDERBOOK_SNAPSHOT_TABLE = 'raw.token_orderbook_snapshots';
const NEW_COLUMN_NAME = 'maker_address';
export class TokenOrderBookSnapshotsAddMakerAddress1550163069315 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE);
if (snapshotTable) {
const newColumn = new TableColumn({
name: NEW_COLUMN_NAME,
type: 'varchar',
isNullable: true,
});
await queryRunner.addColumn(TOKEN_ORDERBOOK_SNAPSHOT_TABLE, newColumn);
// backfill null values
await queryRunner.query(`
UPDATE ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE}
SET ${NEW_COLUMN_NAME}='unknown'
WHERE ${NEW_COLUMN_NAME} is NULL;
`);
await queryRunner.query(`
ALTER TABLE ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE}
DROP CONSTRAINT "token_orderbook_snapshots_pkey1",
ADD PRIMARY KEY (observed_timestamp, source, order_type, price, base_asset_symbol, quote_asset_symbol, maker_address);
`);
} else {
throw new Error(`Could not find table with name ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE}`);
}
}
public async down(queryRunner: QueryRunner): Promise<any> {
const snapshotTable = await queryRunner.getTable(TOKEN_ORDERBOOK_SNAPSHOT_TABLE);
if (snapshotTable) {
await queryRunner.dropColumn(snapshotTable, NEW_COLUMN_NAME);
} else {
throw new Error(`Could not find table with name ${TOKEN_ORDERBOOK_SNAPSHOT_TABLE}`);
}
}
}

1
packages/pipeline/package.json
View File

@@ -49,6 +49,7 @@
"@0x/types": "^2.0.2",
"@0x/utils": "^4.1.0",
"@0x/web3-wrapper": "^5.0.0",
"@radarrelay/types": "^1.2.1",
"@types/dockerode": "^2.5.9",
"@types/p-limit": "^2.0.0",
"async-parallel": "^1.2.3",

53
packages/pipeline/src/data_sources/radar/index.ts Normal file
View File

@@ -0,0 +1,53 @@
import { orderParsingUtils } from '@0x/order-utils';
import { fetchAsync, logUtils } from '@0x/utils';
import { RadarBook, RadarMarket, RadarSignedOrder } from '@radarrelay/types';
const RADAR_BASE_URL = 'https://api.radarrelay.com/v2/';
const ACTIVE_MARKETS_URL = `${RADAR_BASE_URL}/markets`;
const MAX_PER_PAGE = 10000;
export const RADAR_SOURCE = 'radar';
// tslint:disable:prefer-function-over-method
// ^ Keep consistency with other sources and help logical organization
export class RadarSource {
public static parseRadarOrderResponse(radarOrderResponse: any): RadarSignedOrder {
return {
...radarOrderResponse,
...orderParsingUtils.convertStringsFieldsToBigNumbers(radarOrderResponse, [
'remainingBaseTokenAmount',
'remainingQuoteTokenAmount',
'price',
]),
signedOrder: orderParsingUtils.convertOrderStringFieldsToBigNumber(radarOrderResponse.signedOrder),
};
}
/**
* Call Radar API to find out which markets they are maintaining orderbooks for.
*/
public async getActiveMarketsAsync(): Promise<RadarMarket[]> {
logUtils.log('Getting all active Radar markets');
const resp = await fetchAsync(`${ACTIVE_MARKETS_URL}?perPage=${MAX_PER_PAGE}`);
const markets: RadarMarket[] = await resp.json();
logUtils.log(`Got ${markets.length} markets.`);
return markets;
}
/**
* Retrieve orderbook from Radar API for a given market.
* @param marketId String identifying the market we want data for. Eg. 'REP/AUG'
*/
public async getMarketOrderbookAsync(marketId: string): Promise<RadarBook> {
logUtils.log(`${marketId}: Retrieving orderbook.`);
const marketOrderbookUrl = `${ACTIVE_MARKETS_URL}/${marketId}/book?perPage=${MAX_PER_PAGE}`;
const resp = await fetchAsync(marketOrderbookUrl);
const jsonResp = await resp.json();
return {
...jsonResp,
// tslint:disable-next-line:no-unbound-method
bids: jsonResp.bids.map(RadarSource.parseRadarOrderResponse),
// tslint:disable-next-line:no-unbound-method
asks: jsonResp.asks.map(RadarSource.parseRadarOrderResponse),
};
}
}

6
packages/pipeline/src/entities/token_order.ts
View File

@@ -15,12 +15,14 @@ export class TokenOrderbookSnapshot {
public price!: BigNumber;
@PrimaryColumn({ name: 'base_asset_symbol' })
public baseAssetSymbol!: string;
@PrimaryColumn({ name: 'quote_asset_symbol' })
public quoteAssetSymbol!: string;
@PrimaryColumn({ type: String, name: 'maker_address', default: 'unknown' })
public makerAddress!: string;
@Column({ nullable: true, type: String, name: 'base_asset_address' })
public baseAssetAddress!: string | null;
@Column({ name: 'base_volume', type: 'numeric', transformer: bigNumberTransformer })
public baseVolume!: BigNumber;
@PrimaryColumn({ name: 'quote_asset_symbol' })
public quoteAssetSymbol!: string;
@Column({ nullable: true, type: String, name: 'quote_asset_address' })
public quoteAssetAddress!: string | null;
@Column({ name: 'quote_volume', type: 'numeric', transformer: bigNumberTransformer })

19
packages/pipeline/src/parsers/ddex_orders/index.ts
View File

@@ -2,33 +2,27 @@ import { BigNumber } from '@0x/utils';
import { aggregateOrders } from '../utils';
import { DdexMarket, DdexOrderbook } from '../../data_sources/ddex';
import { DDEX_SOURCE, DdexMarket, DdexOrderbook } from '../../data_sources/ddex';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
/**
* Marque function of this file.
* Marquee function of this file.
* 1) Takes in orders from an orderbook,
* other information attached.
* @param ddexOrderbook A raw orderbook that we pull from the Ddex API.
* @param ddexMarket An object containing market data also directly from the API.
* @param observedTimestamp Time at which the orders for the market were pulled.
* @param source The exchange where these orders are placed. In this case 'ddex'.
*/
export function parseDdexOrders(
ddexOrderbook: DdexOrderbook,
ddexMarket: DdexMarket,
observedTimestamp: number,
source: string,
): TokenOrder[] {
const aggregatedBids = aggregateOrders(ddexOrderbook.bids);
const aggregatedAsks = aggregateOrders(ddexOrderbook.asks);
const parsedBids = aggregatedBids.map(order =>
parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, source, order),
);
const parsedAsks = aggregatedAsks.map(order =>
parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, source, order),
);
const parsedBids = aggregatedBids.map(order => parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Bid, order));
const parsedAsks = aggregatedAsks.map(order => parseDdexOrder(ddexMarket, observedTimestamp, OrderType.Ask, order));
return parsedBids.concat(parsedAsks);
}
@@ -46,14 +40,13 @@ export function parseDdexOrder(
ddexMarket: DdexMarket,
observedTimestamp: number,
orderType: OrderType,
source: string,
ddexOrder: [string, BigNumber],
): TokenOrder {
const tokenOrder = new TokenOrder();
const price = new BigNumber(ddexOrder[0]);
const amount = ddexOrder[1];
tokenOrder.source = source;
tokenOrder.source = DDEX_SOURCE;
tokenOrder.observedTimestamp = observedTimestamp;
tokenOrder.orderType = orderType;
tokenOrder.price = price;
@@ -65,5 +58,7 @@ export function parseDdexOrder(
tokenOrder.quoteAssetSymbol = ddexMarket.quoteToken;
tokenOrder.quoteAssetAddress = ddexMarket.quoteTokenAddress;
tokenOrder.quoteVolume = price.times(amount);
tokenOrder.makerAddress = 'unknown';
return tokenOrder;
}

20
packages/pipeline/src/parsers/idex_orders/index.ts
View File

@@ -2,28 +2,25 @@ import { BigNumber } from '@0x/utils';
import { aggregateOrders } from '../utils';
import { IdexOrderbook, IdexOrderParam } from '../../data_sources/idex';
import { IDEX_SOURCE, IdexOrderbook, IdexOrderParam } from '../../data_sources/idex';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
/**
* Marque function of this file.
* Marquee function of this file.
* 1) Takes in orders from an orderbook,
* 2) Aggregates them by price point,
* 3) Parses them into entities which are then saved into the database.
* @param idexOrderbook raw orderbook that we pull from the Idex API.
* @param observedTimestamp Time at which the orders for the market were pulled.
* @param source The exchange where these orders are placed. In this case 'idex'.
*/
export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: number, source: string): TokenOrder[] {
export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp: number): TokenOrder[] {
const aggregatedBids = aggregateOrders(idexOrderbook.bids);
// Any of the bid orders' params will work
const idexBidOrder = idexOrderbook.bids[0];
const parsedBids =
aggregatedBids.length > 0
? aggregatedBids.map(order =>
parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, source, order),
)
? aggregatedBids.map(order => parseIdexOrder(idexBidOrder.params, observedTimestamp, OrderType.Bid, order))
: [];
const aggregatedAsks = aggregateOrders(idexOrderbook.asks);
@@ -31,9 +28,7 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp:
const idexAskOrder = idexOrderbook.asks[0];
const parsedAsks =
aggregatedAsks.length > 0
? aggregatedAsks.map(order =>
parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, source, order),
)
? aggregatedAsks.map(order => parseIdexOrder(idexAskOrder.params, observedTimestamp, OrderType.Ask, order))
: [];
return parsedBids.concat(parsedAsks);
}
@@ -45,26 +40,25 @@ export function parseIdexOrders(idexOrderbook: IdexOrderbook, observedTimestamp:
* trades have been placed.
* @param observedTimestamp The time when the API response returned back to us.
* @param orderType 'bid' or 'ask' enum.
* @param source Exchange where these orders were placed.
* @param idexOrder A <price, amount> tuple which we will convert to volume-basis.
*/
export function parseIdexOrder(
idexOrderParam: IdexOrderParam,
observedTimestamp: number,
orderType: OrderType,
source: string,
idexOrder: [string, BigNumber],
): TokenOrder {
const tokenOrder = new TokenOrder();
const price = new BigNumber(idexOrder[0]);
const amount = idexOrder[1];
tokenOrder.source = source;
tokenOrder.source = IDEX_SOURCE;
tokenOrder.observedTimestamp = observedTimestamp;
tokenOrder.orderType = orderType;
tokenOrder.price = price;
tokenOrder.baseVolume = amount;
tokenOrder.quoteVolume = price.times(amount);
tokenOrder.makerAddress = 'unknown';
if (orderType === OrderType.Bid) {
tokenOrder.baseAssetSymbol = idexOrderParam.buySymbol;

14
packages/pipeline/src/parsers/oasis_orders/index.ts
View File

@@ -3,33 +3,31 @@ import * as R from 'ramda';
import { aggregateOrders } from '../utils';
import { OasisMarket, OasisOrder } from '../../data_sources/oasis';
import { OASIS_SOURCE, OasisMarket, OasisOrder } from '../../data_sources/oasis';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
/**
* Marque function of this file.
* Marquee function of this file.
* 1) Takes in orders from an orderbook,
* 2) Aggregates them according to price point,
* 3) Builds TokenOrder entity with other information attached.
* @param oasisOrderbook A raw orderbook that we pull from the Oasis API.
* @param oasisMarket An object containing market data also directly from the API.
* @param observedTimestamp Time at which the orders for the market were pulled.
* @param source The exchange where these orders are placed. In this case 'oasis'.
*/
export function parseOasisOrders(
oasisOrderbook: OasisOrder[],
oasisMarket: OasisMarket,
observedTimestamp: number,
source: string,
): TokenOrder[] {
const aggregatedBids = aggregateOrders(R.filter(R.propEq('act', OrderType.Bid), oasisOrderbook));
const aggregatedAsks = aggregateOrders(R.filter(R.propEq('act', OrderType.Ask), oasisOrderbook));
const parsedBids = aggregatedBids.map(order =>
parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, source, order),
parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Bid, order),
);
const parsedAsks = aggregatedAsks.map(order =>
parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, source, order),
parseOasisOrder(oasisMarket, observedTimestamp, OrderType.Ask, order),
);
return parsedBids.concat(parsedAsks);
}
@@ -48,14 +46,13 @@ export function parseOasisOrder(
oasisMarket: OasisMarket,
observedTimestamp: number,
orderType: OrderType,
source: string,
oasisOrder: [string, BigNumber],
): TokenOrder {
const tokenOrder = new TokenOrder();
const price = new BigNumber(oasisOrder[0]);
const amount = oasisOrder[1];
tokenOrder.source = source;
tokenOrder.source = OASIS_SOURCE;
tokenOrder.observedTimestamp = observedTimestamp;
tokenOrder.orderType = orderType;
tokenOrder.price = price;
@@ -67,5 +64,6 @@ export function parseOasisOrder(
tokenOrder.quoteAssetSymbol = oasisMarket.quote;
tokenOrder.quoteAssetAddress = null; // Oasis doesn't provide address information
tokenOrder.quoteVolume = price.times(amount);
tokenOrder.makerAddress = 'unknown';
return tokenOrder;
}

15
packages/pipeline/src/parsers/paradex_orders/index.ts
View File

@@ -1,30 +1,28 @@
import { BigNumber } from '@0x/utils';
import { ParadexMarket, ParadexOrder, ParadexOrderbookResponse } from '../../data_sources/paradex';
import { PARADEX_SOURCE, ParadexMarket, ParadexOrder, ParadexOrderbookResponse } from '../../data_sources/paradex';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
/**
* Marque function of this file.
* Marquee function of this file.
* 1) Takes in orders from an orderbook (orders are already aggregated by price point),
* 2) For each aggregated order, forms a TokenOrder entity with market data and
* other information attached.
* @param paradexOrderbookResponse An orderbook response from the Paradex API.
* @param paradexMarket An object containing market data also directly from the API.
* @param observedTimestamp Time at which the orders for the market were pulled.
* @param source The exchange where these orders are placed. In this case 'paradex'.
*/
export function parseParadexOrders(
paradexOrderbookResponse: ParadexOrderbookResponse,
paradexMarket: ParadexMarket,
observedTimestamp: number,
source: string,
): TokenOrder[] {
const parsedBids = paradexOrderbookResponse.bids.map(order =>
parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, source, order),
parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Bid, order),
);
const parsedAsks = paradexOrderbookResponse.asks.map(order =>
parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, source, order),
parseParadexOrder(paradexMarket, observedTimestamp, OrderType.Ask, order),
);
return parsedBids.concat(parsedAsks);
}
@@ -36,21 +34,19 @@ export function parseParadexOrders(
* orders have been placed.
* @param observedTimestamp The time when the API response returned back to us.
* @param orderType 'bid' or 'ask' enum.
* @param source Exchange where these orders were placed.
* @param paradexOrder A ParadexOrder object; basically price, amount tuple.
*/
export function parseParadexOrder(
paradexMarket: ParadexMarket,
observedTimestamp: number,
orderType: OrderType,
source: string,
paradexOrder: ParadexOrder,
): TokenOrder {
const tokenOrder = new TokenOrder();
const price = new BigNumber(paradexOrder.price);
const amount = new BigNumber(paradexOrder.amount);
tokenOrder.source = source;
tokenOrder.source = PARADEX_SOURCE;
tokenOrder.observedTimestamp = observedTimestamp;
tokenOrder.orderType = orderType;
tokenOrder.price = price;
@@ -62,5 +58,6 @@ export function parseParadexOrder(
tokenOrder.quoteAssetSymbol = paradexMarket.quoteToken;
tokenOrder.quoteAssetAddress = paradexMarket.quoteTokenAddress as string;
tokenOrder.quoteVolume = price.times(amount);
tokenOrder.makerAddress = 'unknown';
return tokenOrder;
}

118
packages/pipeline/src/parsers/radar_orders/index.ts Normal file
View File

@@ -0,0 +1,118 @@
import { ObjectMap } from '@0x/types';
import { BigNumber } from '@0x/utils';
import { RadarBook, RadarMarket, RadarSignedOrder } from '@radarrelay/types';
import * as R from 'ramda';
import { aggregateOrders, GenericRawOrder } from '../utils';
import { RADAR_SOURCE } from '../../data_sources/radar';
import { TokenOrderbookSnapshot as TokenOrder } from '../../entities';
import { OrderType } from '../../types';
export interface AggregateOrdersByMaker {
makerAddress: string;
price: string;
amount: BigNumber;
}
/**
* Marquee function of this file.
* 1) Takes in orders from an orderbook,
* other information attached.
* @param radarOrderbook A raw orderbook that we pull from the radar API.
* @param radarMarket An object containing market data also directly from the API.
* @param observedTimestamp Time at which the orders for the market were pulled.
*/
export function parseRadarOrders(
radarOrderbook: RadarBook,
radarMarket: RadarMarket,
observedTimestamp: number,
): TokenOrder[] {
const aggregatedBids = _aggregateOrdersByMaker(radarMarket, radarOrderbook.bids);
const aggregatedAsks = _aggregateOrdersByMaker(radarMarket, radarOrderbook.asks);
const parsedBids = aggregatedBids.map(order =>
parseRadarOrder(radarMarket, observedTimestamp, OrderType.Bid, order),
);
const parsedAsks = aggregatedAsks.map(order =>
parseRadarOrder(radarMarket, observedTimestamp, OrderType.Ask, order),
);
return parsedBids.concat(parsedAsks);
}
/**
* Parse a single aggregated radar order in order to form a tokenOrder entity
* which can be saved into the database.
* @param radarMarket An object containing information about the market where these
* trades have been placed.
* @param observedTimestamp The time when the API response returned back to us.
* @param orderType 'bid' or 'ask' enum.
* @param aggregateOrder An AggregateOrdersByMaker instance which we will convert to volume-basis.
*/
export function parseRadarOrder(
radarMarket: RadarMarket,
observedTimestamp: number,
orderType: OrderType,
aggregateOrder: AggregateOrdersByMaker,
): TokenOrder {
const tokenOrder = new TokenOrder();
const price = new BigNumber(aggregateOrder.price);
const amount = aggregateOrder.amount;
const splitId = radarMarket.id.split('-');
tokenOrder.source = RADAR_SOURCE;
tokenOrder.observedTimestamp = observedTimestamp;
tokenOrder.orderType = orderType;
tokenOrder.price = price;
tokenOrder.baseAssetSymbol = splitId[0];
tokenOrder.baseAssetAddress = radarMarket.baseTokenAddress || null;
tokenOrder.baseVolume = amount;
tokenOrder.quoteAssetSymbol = splitId[1];
tokenOrder.quoteAssetAddress = radarMarket.quoteTokenAddress || null;
tokenOrder.quoteVolume = price.times(amount);
tokenOrder.makerAddress = aggregateOrder.makerAddress;
return tokenOrder;
}
function _toGeneric(radarMarket: RadarMarket, radarOrder: RadarSignedOrder): GenericRawOrder | undefined {
if (radarMarket.baseTokenDecimals === undefined) {
return undefined;
}
return {
price: radarOrder.price.toString(),
// Use the remaining fillable amount
amount: radarOrder.remainingBaseTokenAmount.toString(),
};
}
function _aggregateOrdersByMaker(radarMarket: RadarMarket, radarOrders: RadarSignedOrder[]): AggregateOrdersByMaker[] {
// group all orders by their maker
const ordersByMaker: ObjectMap<RadarSignedOrder[]> = radarOrders.reduce(
(acc: ObjectMap<RadarSignedOrder[]>, val: RadarSignedOrder) => {
const makerAddress = val.signedOrder.makerAddress;
if (acc[makerAddress]) {
acc[makerAddress].push(val);
} else {
acc[makerAddress] = [];
}
return acc;
},
{},
);
const transformToGeneric = (radarOrder: RadarSignedOrder) => _toGeneric(radarMarket, radarOrder);
const aggregationTuples: AggregateOrdersByMaker[][] = (R.keys(ordersByMaker) as string[]).map((maker: string) => {
const generalizedOrders = _removeUndefined(R.map(transformToGeneric, ordersByMaker[maker]));
const aggregatedOrders = aggregateOrders(generalizedOrders);
return aggregatedOrders.map((order: [string, BigNumber]) => ({
makerAddress: maker,
price: order[0],
amount: order[1],
}));
});
return R.unnest(aggregationTuples);
}
// tslint:disable-next-line:no-unbound-method
const _removeUndefined = R.reject(R.isNil);

4
packages/pipeline/src/scripts/pull_ddex_orderbook_snapshots.ts
View File

@@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils';
import * as R from 'ramda';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { DDEX_SOURCE, DdexMarket, DdexSource } from '../data_sources/ddex';
import { DdexMarket, DdexSource } from '../data_sources/ddex';
import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseDdexOrders } from '../parsers/ddex_orders';
@@ -43,7 +43,7 @@ async function getAndSaveMarketOrderbookAsync(ddexSource: DdexSource, market: Dd
const observedTimestamp = Date.now();
logUtils.log(`${market.id}: Parsing orders.`);
const orders = parseDdexOrders(orderBook, market, observedTimestamp, DDEX_SOURCE);
const orders = parseDdexOrders(orderBook, market, observedTimestamp);
if (orders.length > 0) {
logUtils.log(`${market.id}: Saving ${orders.length} orders.`);

4
packages/pipeline/src/scripts/pull_idex_orderbook_snapshots.ts
View File

@@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils';
import * as R from 'ramda';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { IDEX_SOURCE, IdexSource } from '../data_sources/idex';
import { IdexSource } from '../data_sources/idex';
import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseIdexOrders } from '../parsers/idex_orders';
@@ -51,7 +51,7 @@ async function getAndSaveMarketOrderbookAsync(idexSource: IdexSource, marketId:
}
logUtils.log(`${marketId}: Parsing orders.`);
const orders = parseIdexOrders(orderBook, observedTimestamp, IDEX_SOURCE);
const orders = parseIdexOrders(orderBook, observedTimestamp);
if (orders.length > 0) {
logUtils.log(`${marketId}: Saving ${orders.length} orders.`);

4
packages/pipeline/src/scripts/pull_oasis_orderbook_snapshots.ts
View File

@@ -2,7 +2,7 @@ import { logUtils } from '@0x/utils';
import * as R from 'ramda';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { OASIS_SOURCE, OasisMarket, OasisSource } from '../data_sources/oasis';
import { OasisMarket, OasisSource } from '../data_sources/oasis';
import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseOasisOrders } from '../parsers/oasis_orders';
@@ -46,7 +46,7 @@ async function getAndSaveMarketOrderbookAsync(oasisSource: OasisSource, market:
const observedTimestamp = Date.now();
logUtils.log(`${market.id}: Parsing orders.`);
const orders = parseOasisOrders(orderBook, market, observedTimestamp, OASIS_SOURCE);
const orders = parseOasisOrders(orderBook, market, observedTimestamp);
if (orders.length > 0) {
logUtils.log(`${market.id}: Saving ${orders.length} orders.`);

3
packages/pipeline/src/scripts/pull_paradex_orderbook_snapshots.ts
View File

@@ -2,7 +2,6 @@ import { logUtils } from '@0x/utils';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import {
PARADEX_SOURCE,
ParadexActiveMarketsResponse,
ParadexMarket,
ParadexSource,
@@ -75,7 +74,7 @@ async function getAndSaveMarketOrderbookAsync(paradexSource: ParadexSource, mark
const observedTimestamp = Date.now();
logUtils.log(`${market.symbol}: Parsing orders.`);
const orders = parseParadexOrders(paradexOrderbookResponse, market, observedTimestamp, PARADEX_SOURCE);
const orders = parseParadexOrders(paradexOrderbookResponse, market, observedTimestamp);
if (orders.length > 0) {
logUtils.log(`${market.symbol}: Saving ${orders.length} orders.`);

56
packages/pipeline/src/scripts/pull_radar_orderbook_snapshots.ts Normal file
View File

@@ -0,0 +1,56 @@
import { logUtils } from '@0x/utils';
import { RadarMarket } from '@radarrelay/types';
import * as R from 'ramda';
import { Connection, ConnectionOptions, createConnection } from 'typeorm';
import { RadarSource } from '../data_sources/radar';
import { TokenOrderbookSnapshot as TokenOrder } from '../entities';
import * as ormConfig from '../ormconfig';
import { parseRadarOrders } from '../parsers/radar_orders';
import { handleError } from '../utils';
// Number of orders to save at once.
const BATCH_SAVE_SIZE = 1000;
// Number of markets to retrieve orderbooks for at once.
const MARKET_ORDERBOOK_REQUEST_BATCH_SIZE = 50;
// Delay between market orderbook requests.
const MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY = 5000;
let connection: Connection;
(async () => {
connection = await createConnection(ormConfig as ConnectionOptions);
const radarSource = new RadarSource();
const markets = await radarSource.getActiveMarketsAsync();
for (const marketsChunk of R.splitEvery(MARKET_ORDERBOOK_REQUEST_BATCH_SIZE, markets)) {
await Promise.all(
marketsChunk.map(async (market: RadarMarket) => getAndSaveMarketOrderbookAsync(radarSource, market)),
);
await new Promise<void>(resolve => setTimeout(resolve, MILLISEC_MARKET_ORDERBOOK_REQUEST_DELAY));
}
process.exit(0);
})().catch(handleError);
/**
* Retrieve orderbook from radar API for a given market. Parse orders and insert
* them into our database.
* @param radarSource Data source which can query radar API.
* @param market Object from radar API containing market data.
*/
async function getAndSaveMarketOrderbookAsync(radarSource: RadarSource, market: RadarMarket): Promise<void> {
const orderBook = await radarSource.getMarketOrderbookAsync(market.id);
const observedTimestamp = Date.now();
logUtils.log(`${market.id}: Parsing orders.`);
const orders = parseRadarOrders(orderBook, market, observedTimestamp);
if (orders.length > 0) {
logUtils.log(`${market.id}: Saving ${orders.length} orders.`);
const TokenOrderRepository = connection.getRepository(TokenOrder);
await TokenOrderRepository.save(orders, { chunk: Math.ceil(orders.length / BATCH_SAVE_SIZE) });
} else {
logUtils.log(`${market.id}: 0 orders to save.`);
}
}

76
packages/pipeline/test/data_sources/radar/index_test.ts Normal file
View File

@@ -0,0 +1,76 @@
import { BigNumber } from '@0x/utils';
import { RadarOrderState, RadarOrderType } from '@radarrelay/types';
import * as chai from 'chai';
import 'mocha';
import { RadarSource } from '../../../src/data_sources/radar';
import { chaiSetup } from '../../utils/chai_setup';
chaiSetup.configure();
const expect = chai.expect;
const rawResponse = {
orderHash: '0x60bc235f7887a50801c8fc1fc18fb0625ac5f3962cdc1bd59567a6929db8b2ec',
type: 'BID',
state: 'OPEN',
baseTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
quoteTokenAddress: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359',
remainingBaseTokenAmount: '9.079731811797989766',
remainingQuoteTokenAmount: '1099.999999999999999889',
price: '121.14895272244560081697',
createdDate: '2019-02-13 21:35:53',
signedOrder: {
exchangeAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b',
senderAddress: '0x0000000000000000000000000000000000000000',
makerAddress: '0x56178a0d5f301baf6cf3e1cd53d9863437345bf9',
takerAddress: '0x0000000000000000000000000000000000000000',
makerAssetData: '0xf47261b000000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359',
takerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
feeRecipientAddress: '0xa258b39954cef5cb142fd567a46cddb31a670124',
makerAssetAmount: '1099999999999999999889',
takerAssetAmount: '9079731811797989766',
makerFee: '0',
takerFee: '0',
expirationTimeSeconds: '1550094353',
signature:
'0x1ce161d02ad63fe7308e9cd5e97583a8873331d1b72d90e9f3863d9fcba2518cb91ab2fe7de94e4afb39742acdc820abbff2dc0622c8d3865917fade62f16322ae03',
salt: '1550093753237',
},
};
const parsedResponse = {
orderHash: '0x60bc235f7887a50801c8fc1fc18fb0625ac5f3962cdc1bd59567a6929db8b2ec',
type: 'BID' as RadarOrderType,
state: 'OPEN' as RadarOrderState,
baseTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
quoteTokenAddress: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359',
remainingBaseTokenAmount: new BigNumber('9.079731811797989766'),
remainingQuoteTokenAmount: new BigNumber('1099.999999999999999889'),
price: new BigNumber('121.14895272244560081697'),
createdDate: '2019-02-13 21:35:53',
signedOrder: {
exchangeAddress: '0x4f833a24e1f95d70f028921e27040ca56e09ab0b',
senderAddress: '0x0000000000000000000000000000000000000000',
makerAddress: '0x56178a0d5f301baf6cf3e1cd53d9863437345bf9',
takerAddress: '0x0000000000000000000000000000000000000000',
makerAssetData: '0xf47261b000000000000000000000000089d24a6b4ccb1b6faa2625fe562bdd9a23260359',
takerAssetData: '0xf47261b0000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
feeRecipientAddress: '0xa258b39954cef5cb142fd567a46cddb31a670124',
makerAssetAmount: new BigNumber('1099999999999999999889'),
takerAssetAmount: new BigNumber('9079731811797989766'),
makerFee: new BigNumber('0'),
takerFee: new BigNumber('0'),
expirationTimeSeconds: new BigNumber('1550094353'),
signature:
'0x1ce161d02ad63fe7308e9cd5e97583a8873331d1b72d90e9f3863d9fcba2518cb91ab2fe7de94e4afb39742acdc820abbff2dc0622c8d3865917fade62f16322ae03',
salt: new BigNumber('1550093753237'),
},
};
describe('RadarSource', () => {
describe('parseRadarOrderResponse', () => {
it('Correctly parses a Radar orderbook response to a RadarBook', () => {
expect(RadarSource.parseRadarOrderResponse(rawResponse)).deep.equal(parsedResponse);
});
});
});

1
packages/pipeline/test/entities/token_order_test.ts
View File

@@ -20,6 +20,7 @@ const tokenOrderbookSnapshot: TokenOrderbookSnapshot = {
quoteAssetSymbol: 'ABC',
quoteAssetAddress: '0x00923b9a074762b93650716333b3e1473a15048e',
quoteVolume: new BigNumber(12.3234234),
makerAddress: 'unknown',
};
describe('TokenOrderbookSnapshot entity', () => {

5
packages/pipeline/test/parsers/ddex_orders/index_test.ts
View File

@@ -31,7 +31,6 @@ describe('ddex_orders', () => {
};
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Bid;
const source: string = 'ddex';
const expected = new TokenOrder();
expected.source = 'ddex';
@@ -44,8 +43,8 @@ describe('ddex_orders', () => {
expected.baseAssetSymbol = 'DEF';
expected.baseAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
expected.baseVolume = new BigNumber(10);
const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, source, ddexOrder);
expected.makerAddress = 'unknown';
const actual = parseDdexOrder(ddexMarket, observedTimestamp, orderType, ddexOrder);
expect(actual).deep.equal(expected);
});
});

10
packages/pipeline/test/parsers/idex_orders/index_test.ts
View File

@@ -32,7 +32,6 @@ describe('idex_orders', () => {
};
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Bid;
const source: string = 'idex';
const expected = new TokenOrder();
expected.source = 'idex';
@@ -45,8 +44,8 @@ describe('idex_orders', () => {
expected.quoteAssetSymbol = 'DEF';
expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
expected.quoteVolume = new BigNumber(5);
const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder);
expected.makerAddress = 'unknown';
const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, idexOrder);
expect(actual).deep.equal(expected);
});
it('correctly converts ask type idexOrder to TokenOrder entity', () => {
@@ -66,7 +65,6 @@ describe('idex_orders', () => {
};
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Ask;
const source: string = 'idex';
const expected = new TokenOrder();
expected.source = 'idex';
@@ -79,8 +77,8 @@ describe('idex_orders', () => {
expected.quoteAssetSymbol = 'DEF';
expected.quoteAssetAddress = '0xb45df06e38540a675fdb5b598abf2c0dbe9d6b81';
expected.quoteVolume = new BigNumber(5);
const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, source, idexOrder);
expected.makerAddress = 'unknown';
const actual = parseIdexOrder(idexOrderParam, observedTimestamp, orderType, idexOrder);
expect(actual).deep.equal(expected);
});
});

5
packages/pipeline/test/parsers/oasis_orders/index_test.ts
View File

@@ -28,7 +28,6 @@ describe('oasis_orders', () => {
};
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Bid;
const source: string = 'oasis';
const expected = new TokenOrder();
expected.source = 'oasis';
@@ -41,8 +40,8 @@ describe('oasis_orders', () => {
expected.quoteAssetSymbol = 'ABC';
expected.quoteAssetAddress = null;
expected.quoteVolume = new BigNumber(5);
const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, source, oasisOrder);
expected.makerAddress = 'unknown';
const actual = parseOasisOrder(oasisMarket, observedTimestamp, orderType, oasisOrder);
expect(actual).deep.equal(expected);
});
});

5
packages/pipeline/test/parsers/paradex_orders/index_test.ts
View File

@@ -33,7 +33,6 @@ describe('paradex_orders', () => {
};
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Bid;
const source: string = 'paradex';
const expected = new TokenOrder();
expected.source = 'paradex';
@@ -46,8 +45,8 @@ describe('paradex_orders', () => {
expected.quoteAssetSymbol = 'ABC';
expected.quoteAssetAddress = '0x0000000000000000000000000000000000000000';
expected.quoteVolume = new BigNumber(412 * 0.1245);
const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, source, paradexOrder);
expected.makerAddress = 'unknown';
const actual = parseParadexOrder(paradexMarket, observedTimestamp, orderType, paradexOrder);
expect(actual).deep.equal(expected);
});
});

55
packages/pipeline/test/parsers/radar_orders/index_test.ts Normal file
View File

@@ -0,0 +1,55 @@
import { BigNumber } from '@0x/utils';
import { RadarMarket } from '@radarrelay/types';
import * as chai from 'chai';
import 'mocha';
import { TokenOrderbookSnapshot as TokenOrder } from '../../../src/entities';
import { AggregateOrdersByMaker, parseRadarOrder } from '../../../src/parsers/radar_orders';
import { OrderType } from '../../../src/types';
import { chaiSetup } from '../../utils/chai_setup';
chaiSetup.configure();
const expect = chai.expect;
// tslint:disable:custom-no-magic-numbers
describe('radar_orders', () => {
describe('parseRadarOrder', () => {
it('converts radarOrder to TokenOrder entity', () => {
const radarOrder: AggregateOrdersByMaker = {
makerAddress: '0x6eC92694ea172ebC430C30fa31De87620967A082',
price: '0.01',
amount: new BigNumber(10000000000),
};
const radarMarket = ({
id: 'WETH-DAI',
displayName: 'WETH/DAI',
baseTokenAddress: '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2',
quoteTokenAddress: '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359',
baseTokenDecimals: 18,
quoteTokenDecimals: 18,
quoteIncrement: 8,
minOrderSize: new BigNumber('0.00692535'),
maxOrderSize: new BigNumber('1000000000'),
score: 99.66,
// Radar types are defined using an older version of BigNumber, so need to be force cast.
} as any) as RadarMarket;
const observedTimestamp: number = Date.now();
const orderType: OrderType = OrderType.Bid;
const expected = new TokenOrder();
expected.source = 'radar';
expected.observedTimestamp = observedTimestamp;
expected.orderType = OrderType.Bid;
expected.price = new BigNumber(0.01);
expected.quoteAssetSymbol = 'DAI';
expected.quoteAssetAddress = '0x89d24a6b4ccb1b6faa2625fe562bdd9a23260359';
expected.quoteVolume = new BigNumber(100000000);
expected.baseAssetSymbol = 'WETH';
expected.baseAssetAddress = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2';
expected.baseVolume = new BigNumber(10000000000);
expected.makerAddress = '0x6eC92694ea172ebC430C30fa31De87620967A082';
const actual = parseRadarOrder(radarMarket, observedTimestamp, orderType, radarOrder);
expect(actual).deep.equal(expected);
});
});
});

53
yarn.lock
View File

@@ -606,6 +606,14 @@
dependencies:
npm-registry-client "7.0.9"
"@0xproject/types@^1.0.1-rc.3":
version "1.1.4"
resolved "https://registry.npmjs.org/@0xproject/types/-/types-1.1.4.tgz#3ffd65e670d6a21dab19ee0ffd5fad0056291b8e"
dependencies:
"@types/node" "*"
bignumber.js "~4.1.0"
ethereum-types "^1.0.11"
"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.0.0-beta.35":
version "7.0.0"
resolved "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz#06e2ab19bdb535385559aabb5ba59729482800f8"
@@ -1280,6 +1288,13 @@
call-me-maybe "^1.0.1"
glob-to-regexp "^0.3.0"
"@radarrelay/types@^1.2.1":
version "1.2.1"
resolved "https://registry.npmjs.org/@radarrelay/types/-/types-1.2.1.tgz#d16edb43d0735a31c887b9e79ff6e53924ac8cc5"
dependencies:
"@0xproject/types" "^1.0.1-rc.3"
bignumber.js "^5.0.0"
"@reach/component-component@^0.1.1":
version "0.1.1"
resolved "https://registry.yarnpkg.com/@reach/component-component/-/component-component-0.1.1.tgz#62ea2ec290da32f5e3a9872fb51f9a3ae4370cc4"
@@ -3445,6 +3460,10 @@ bignumber.js@7.2.1:
version "7.2.1"
resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz#80c048759d826800807c4bfd521e50edbba57a5f"
bignumber.js@^5.0.0:
version "5.0.0"
resolved "https://registry.npmjs.org/bignumber.js/-/bignumber.js-5.0.0.tgz#fbce63f09776b3000a83185badcde525daf34833"
"bignumber.js@git+https://github.com/debris/bignumber.js.git#94d7146671b9719e00a09c29b01a691bc85048c2":
version "2.0.7"
resolved "git+https://github.com/debris/bignumber.js.git#94d7146671b9719e00a09c29b01a691bc85048c2"
@@ -6417,6 +6436,13 @@ ethereum-common@^0.0.18:
version "0.0.18"
resolved "https://registry.yarnpkg.com/ethereum-common/-/ethereum-common-0.0.18.tgz#2fdc3576f232903358976eb39da783213ff9523f"
ethereum-types@^1.0.11:
version "1.1.6"
resolved "https://registry.npmjs.org/ethereum-types/-/ethereum-types-1.1.6.tgz#14437dbf401de361e70dac6358e5f2915ad3c35d"
dependencies:
"@types/node" "*"
bignumber.js "~4.1.0"
ethereumjs-abi@0.6.5:
version "0.6.5"
resolved "https://registry.yarnpkg.com/ethereumjs-abi/-/ethereumjs-abi-0.6.5.tgz#5a637ef16ab43473fa72a29ad90871405b3f5241"
@@ -7925,7 +7951,7 @@ got@^6.7.1:
graceful-fs@4.1.15, graceful-fs@^3.0.0, graceful-fs@^4.0.0, graceful-fs@^4.1.10, graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@~1.2.0:
version "4.1.15"
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00"
resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00"
"graceful-readlink@>= 1.0.0":
version "1.0.1"
@@ -13478,6 +13504,15 @@ react-dom@^16.3.2:
object-assign "^4.1.1"
prop-types "^15.6.0"
react-dom@^16.4.2:
version "16.8.1"
resolved "https://registry.npmjs.org/react-dom/-/react-dom-16.8.1.tgz#ec860f98853d09d39bafd3a6f1e12389d283dbb4"
dependencies:
loose-envify "^1.1.0"
object-assign "^4.1.1"
prop-types "^15.6.2"
scheduler "^0.13.1"
react-dom@^16.5.2:
version "16.5.2"
resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-16.5.2.tgz#b69ee47aa20bab5327b2b9d7c1fe2a30f2cfa9d7"
@@ -13781,6 +13816,15 @@ react@^16.3.2:
object-assign "^4.1.1"
prop-types "^15.6.0"
react@^16.4.2:
version "16.8.1"
resolved "https://registry.npmjs.org/react/-/react-16.8.1.tgz#ae11831f6cb2a05d58603a976afc8a558e852c4a"
dependencies:
loose-envify "^1.1.0"
object-assign "^4.1.1"
prop-types "^15.6.2"
scheduler "^0.13.1"
react@^16.5.2:
version "16.5.2"
resolved "https://registry.yarnpkg.com/react/-/react-16.5.2.tgz#19f6b444ed139baa45609eee6dc3d318b3895d42"
@@ -14652,6 +14696,13 @@ schedule@^0.5.0:
dependencies:
object-assign "^4.1.1"
scheduler@^0.13.1:
version "0.13.1"
resolved "https://registry.npmjs.org/scheduler/-/scheduler-0.13.1.tgz#1a217df1bfaabaf4f1b92a9127d5d732d85a9591"
dependencies:
loose-envify "^1.1.0"
object-assign "^4.1.1"
schema-utils@^0.4.4:
version "0.4.7"
resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187"