crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add special case to scrape OHLCV for eth/usd (#1428)

Browse Source
This commit is contained in:
Xianny
2018-12-13 13:13:28 -08:00
committed by GitHub
parent 5744a7f61b
commit 8906c30f37
1 changed files with 34 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
packages/pipeline/src/utils/get_ohlcv_trading_pairs.ts
View File

@@ -23,6 +23,18 @@ interface CryptoCompareCoin {
const TO_CURRENCIES = ['USD', 'EUR', 'ETH', 'USDT']; const TO_CURRENCIES = ['USD', 'EUR', 'ETH', 'USDT'];
const ETHEREUM_IDENTIFIER = '7605'; const ETHEREUM_IDENTIFIER = '7605';
const HTTP_OK_STATUS = 200; const HTTP_OK_STATUS = 200;
interface StaticPair {
fromSymbol: string;
toSymbol: string;
}
const SPECIAL_CASES: StaticPair[] = [
{
fromSymbol: 'ETH',
toSymbol: 'USD',
},
];
/** /**
* Get trading pairs with latest scraped time for OHLCV records * Get trading pairs with latest scraped time for OHLCV records
* @param conn a typeorm Connection to postgres * @param conn a typeorm Connection to postgres
@@ -44,6 +56,7 @@ export async function fetchOHLCVTradingPairsAsync(
FROM raw.ohlcv_external FROM raw.ohlcv_external
GROUP BY from_symbol, to_symbol;`); GROUP BY from_symbol, to_symbol;`);
// build addressable index: { fromsym: { tosym: time }}
const latestTradingPairsIndex: { [fromSym: string]: { [toSym: string]: number } } = {}; const latestTradingPairsIndex: { [fromSym: string]: { [toSym: string]: number } } = {};
latestTradingPairs.forEach(pair => { latestTradingPairs.forEach(pair => {
const latestIndex: { [toSym: string]: number } = latestTradingPairsIndex[pair.from_symbol] || {}; const latestIndex: { [toSym: string]: number } = latestTradingPairsIndex[pair.from_symbol] || {};
@@ -51,6 +64,13 @@ export async function fetchOHLCVTradingPairsAsync(
latestTradingPairsIndex[pair.from_symbol] = latestIndex; latestTradingPairsIndex[pair.from_symbol] = latestIndex;
}); });
// match time to special cases
const specialCases: TradingPair[] = SPECIAL_CASES.map(pair => {
const latestSavedTime =
R.path<number>([pair.fromSymbol, pair.toSymbol], latestTradingPairsIndex) || earliestBackfillTime;
return R.assoc('latestSavedTime', latestSavedTime, pair);
});
// get token symbols used by Crypto Compare // get token symbols used by Crypto Compare
const allCoinsResp = await fetchAsync(COINLIST_API); const allCoinsResp = await fetchAsync(COINLIST_API);
if (allCoinsResp.status !== HTTP_OK_STATUS) { if (allCoinsResp.status !== HTTP_OK_STATUS) {
@@ -66,27 +86,31 @@ export async function fetchOHLCVTradingPairsAsync(
}); });
// fetch all tokens that are traded on 0x // fetch all tokens that are traded on 0x
const rawTokenAddresses: Array<{ tokenaddress: string }> = await conn.query( const rawEventTokenAddresses: Array<{ tokenaddress: string }> = await conn.query(
`SELECT DISTINCT(maker_token_address) as tokenaddress FROM raw.exchange_fill_events UNION `SELECT DISTINCT(maker_token_address) as tokenaddress FROM raw.exchange_fill_events UNION
SELECT DISTINCT(taker_token_address) as tokenaddress FROM raw.exchange_fill_events`, SELECT DISTINCT(taker_token_address) as tokenaddress FROM raw.exchange_fill_events`,
); );
const tokenAddresses = R.pluck('tokenaddress', rawTokenAddresses);
// tslint:disable-next-line:no-unbound-method
const eventTokenAddresses = R.pluck('tokenaddress', rawEventTokenAddresses).map(R.toLower);
// join token addresses with CC symbols // join token addresses with CC symbols
const allTokenSymbols: string[] = tokenAddresses const eventTokenSymbols: string[] = eventTokenAddresses
.map(tokenAddress => erc20CoinsIndex.get(tokenAddress.toLowerCase()) || '') .filter(tokenAddress => erc20CoinsIndex.has(tokenAddress))
.filter(x => x); .map(tokenAddress => erc20CoinsIndex.get(tokenAddress) as string);
// generate list of all tokens with time of latest existing record OR default earliest time // join traded tokens with fiat and latest backfill time
const allTradingPairCombinations: TradingPair[] = R.chain(sym => { const eventTradingPairs: TradingPair[] = R.chain(sym => {
return TO_CURRENCIES.map(fiat => { return TO_CURRENCIES.map(fiat => {
return { const pair = {
fromSymbol: sym, fromSymbol: sym,
toSymbol: fiat, toSymbol: fiat,
latestSavedTime: R.path<number>([sym, fiat], latestTradingPairsIndex) || earliestBackfillTime, latestSavedTime: R.path<number>([sym, fiat], latestTradingPairsIndex) || earliestBackfillTime,
}; };
return pair;
}); });
}, allTokenSymbols); }, eventTokenSymbols);
return allTradingPairCombinations; // join with special cases
return R.concat(eventTradingPairs, specialCases);
} }