crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

upgrade throttling code

Browse Source
This commit is contained in:
xianny
2018-12-06 12:07:55 -08:00
committed by Fred Carlsen
parent 05217bb1a2
commit 56b82bbc8b
2 changed files with 12 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
packages/pipeline/package.json
View File

@@ -58,9 +58,9 @@
"@types/p-limit": "^2.0.0",
"async-parallel": "^1.2.3",
"axios": "^0.18.0",
"bottleneck": "^2.13.2",
"dockerode": "^2.5.7",
"ethereum-types": "^1.0.6",
"p-limit": "^2.0.0",
"pg": "^7.5.0",
"prettier": "^1.15.3",
"ramda": "^0.25.0",

24
packages/pipeline/src/data_sources/ohlcv_external/crypto_compare.ts
View File

@@ -1,6 +1,6 @@
// tslint:disable:no-duplicate-imports
import { fetchAsync } from '@0x/utils';
import promiseLimit = require('p-limit');
import Bottleneck from 'bottleneck';
import { stringify } from 'querystring';
import * as R from 'ramda';
@@ -35,6 +35,7 @@ export interface CryptoCompareOHLCVParams {
const ONE_WEEK = 7 * 24 * 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
const ONE_HOUR = 60 * 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
const ONE_MINUTE = 60 * 1000; // tslint:disable-line:custom-no-magic-numbers
const ONE_SECOND = 1000;
const ONE_HOUR_AGO = new Date().getTime() - ONE_HOUR;
const HTTP_OK_STATUS = 200;
@@ -47,9 +48,14 @@ export class CryptoCompareOHLCVSource {
private readonly _url: string = 'https://min-api.cryptocompare.com/data/histohour?';
// rate-limit for all API calls through this class instance
private readonly _promiseLimit: (fetchFn: () => Promise<Response>) => Promise<Response>;
constructor(maxConcurrentRequests: number = 50) {
this._promiseLimit = promiseLimit(maxConcurrentRequests);
private readonly _limiter: Bottleneck;
constructor(maxReqsPerSecond: number = 40) {
this._limiter = new Bottleneck({
minTime: Math.ceil(ONE_SECOND / maxReqsPerSecond),
reservoir: 2000,
reservoirRefreshAmount: 2000,
reservoirRefreshInterval: ONE_MINUTE,
});
}
// gets OHLCV records starting from pair.latest
@@ -61,15 +67,7 @@ export class CryptoCompareOHLCVSource {
toTs: Math.floor((pair.latestSavedTime + this.interval) / ONE_SECOND), // CryptoCompare uses timestamp in seconds. not ms
};
const url = this._url + stringify(params);
// go through the instance-wide rate-limit
const fetchPromise: Promise<Response> = this._promiseLimit(() => {
// tslint:disable-next-line:no-console
console.log(`Scraping Crypto Compare at ${url}`);
return fetchAsync(url);
});
const response = await Promise.resolve(fetchPromise);
const response = await this._limiter.schedule(() => fetchAsync(url));
if (response.status !== HTTP_OK_STATUS) {
throw new Error(`HTTP error while scraping Crypto Compare: [${response}]`);
}