crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #691 from 0xProject/fix/sol-cov-memory

Browse Source 
Refactor sol-cov to avoid keeping traceInfo in memory
This commit is contained in:
Alex Browne
2018-06-12 15:38:28 -07:00
committed by GitHub
parent 787015f537 627ea6c860
commit f50d3088dc
6 changed files with 76 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
packages/migrations/artifacts/2.0.0/TestLibAssetProxyDecoder.json vendored
View File

File diff suppressed because one or more lines are too long

69
packages/sol-cov/src/coverage_manager.ts
View File

@@ -36,7 +36,8 @@ const mkdirpAsync = promisify<undefined>(mkdirp);
export class CoverageManager {
private _artifactAdapter: AbstractArtifactAdapter;
private _logger: Logger;
private _traceInfos: TraceInfo[] = [];
private _contractsData!: ContractData[];
private _collector = new Collector();
/**
* Computed partial coverage for a single file & subtrace
* @param contractData Contract metadata (source, srcMap, bytecode)
@@ -130,49 +131,39 @@ export class CoverageManager {
this._logger = getLogger('sol-cov');
this._logger.setLevel(isVerbose ? levels.TRACE : levels.ERROR);
}
public appendTraceInfo(traceInfo: TraceInfo): void {
this._traceInfos.push(traceInfo);
}
public async writeCoverageAsync(): Promise<void> {
const finalCoverage = await this._computeCoverageAsync();
const finalCoverage = this._collector.getFinalCoverage();
const stringifiedCoverage = JSON.stringify(finalCoverage, null, '\t');
await mkdirpAsync('coverage');
fs.writeFileSync('coverage/coverage.json', stringifiedCoverage);
}
private async _computeCoverageAsync(): Promise<Coverage> {
const contractsData = await this._artifactAdapter.collectContractsDataAsync();
const collector = new Collector();
for (const traceInfo of this._traceInfos) {
const isContractCreation = traceInfo.address === constants.NEW_CONTRACT;
const bytecode = isContractCreation
? (traceInfo as TraceInfoNewContract).bytecode
: (traceInfo as TraceInfoExistingContract).runtimeBytecode;
const contractData = utils.getContractDataIfExists(contractsData, bytecode);
if (_.isUndefined(contractData)) {
const errMsg = isContractCreation
? `Unknown contract creation transaction`
: `Transaction to an unknown address: ${traceInfo.address}`;
this._logger.warn(errMsg);
continue;
}
const bytecodeHex = stripHexPrefix(bytecode);
const sourceMap = isContractCreation ? contractData.sourceMap : contractData.sourceMapRuntime;
const pcToSourceRange = parseSourceMap(
contractData.sourceCodes,
sourceMap,
bytecodeHex,
contractData.sources,
);
for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) {
const singleFileCoverageForTrace = CoverageManager._getSingleFileCoverageForSubtrace(
contractData,
traceInfo.subtrace,
pcToSourceRange,
fileIndex,
);
collector.add(singleFileCoverageForTrace);
}
public async computeSingleTraceCoverageAsync(traceInfo: TraceInfo): Promise<void> {
if (_.isUndefined(this._contractsData)) {
this._contractsData = await this._artifactAdapter.collectContractsDataAsync();
}
const isContractCreation = traceInfo.address === constants.NEW_CONTRACT;
const bytecode = isContractCreation
? (traceInfo as TraceInfoNewContract).bytecode
: (traceInfo as TraceInfoExistingContract).runtimeBytecode;
const contractData = utils.getContractDataIfExists(this._contractsData, bytecode);
if (_.isUndefined(contractData)) {
const errMsg = isContractCreation
? `Unknown contract creation transaction`
: `Transaction to an unknown address: ${traceInfo.address}`;
this._logger.warn(errMsg);
return;
}
const bytecodeHex = stripHexPrefix(bytecode);
const sourceMap = isContractCreation ? contractData.sourceMap : contractData.sourceMapRuntime;
const pcToSourceRange = parseSourceMap(contractData.sourceCodes, sourceMap, bytecodeHex, contractData.sources);
for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) {
const singleFileCoverageForTrace = CoverageManager._getSingleFileCoverageForSubtrace(
contractData,
traceInfo.subtrace,
pcToSourceRange,
fileIndex,
);
this._collector.add(singleFileCoverageForTrace);
}
return collector.getFinalCoverage();
}
}

8
packages/sol-cov/src/coverage_subprovider.ts
View File

@@ -1,8 +1,7 @@
import * as _ from 'lodash';
import { AbstractArtifactAdapter } from './artifact_adapters/abstract_artifact_adapter';
import { CoverageManager } from './coverage_manager';
import { TraceCollectionSubprovider } from './trace_collection_subprovider';
import { TraceInfo } from './types';
/**
* This class implements the [web3-provider-engine](https://github.com/MetaMask/provider-engine) subprovider interface.
@@ -25,12 +24,13 @@ export class CoverageSubprovider extends TraceCollectionSubprovider {
super(defaultFromAddress, traceCollectionSubproviderConfig);
this._coverageManager = new CoverageManager(artifactAdapter, isVerbose);
}
public async handleTraceInfoAsync(traceInfo: TraceInfo): Promise<void> {
await this._coverageManager.computeSingleTraceCoverageAsync(traceInfo);
}
/**
* Write the test coverage results to a file in Istanbul format.
*/
public async writeCoverageAsync(): Promise<void> {
const traceInfos = this.getCollectedTraceInfos();
_.forEach(traceInfos, traceInfo => this._coverageManager.appendTraceInfo(traceInfo));
await this._coverageManager.writeCoverageAsync();
}
}

69
packages/sol-cov/src/profiler_manager.ts
View File

@@ -31,7 +31,8 @@ const mkdirpAsync = promisify<undefined>(mkdirp);
export class ProfilerManager {
private _artifactAdapter: AbstractArtifactAdapter;
private _logger: Logger;
private _traceInfos: TraceInfo[] = [];
private _contractsData!: ContractData[];
private _collector = new Collector();
/**
* Computed partial coverage for a single file & subtrace
* @param contractData Contract metadata (source, srcMap, bytecode)
@@ -86,49 +87,39 @@ export class ProfilerManager {
this._logger = getLogger('sol-cov');
this._logger.setLevel(isVerbose ? levels.TRACE : levels.ERROR);
}
public appendTraceInfo(traceInfo: TraceInfo): void {
this._traceInfos.push(traceInfo);
}
public async writeProfilerOutputAsync(): Promise<void> {
const finalCoverage = await this._computeCoverageAsync();
const finalCoverage = this._collector.getFinalCoverage();
const stringifiedCoverage = JSON.stringify(finalCoverage, null, '\t');
await mkdirpAsync('coverage');
fs.writeFileSync('coverage/coverage.json', stringifiedCoverage);
}
private async _computeCoverageAsync(): Promise<Coverage> {
const contractsData = await this._artifactAdapter.collectContractsDataAsync();
const collector = new Collector();
for (const traceInfo of this._traceInfos) {
const isContractCreation = traceInfo.address === constants.NEW_CONTRACT;
const bytecode = isContractCreation
? (traceInfo as TraceInfoNewContract).bytecode
: (traceInfo as TraceInfoExistingContract).runtimeBytecode;
const contractData = utils.getContractDataIfExists(contractsData, bytecode);
if (_.isUndefined(contractData)) {
const errMsg = isContractCreation
? `Unknown contract creation transaction`
: `Transaction to an unknown address: ${traceInfo.address}`;
this._logger.warn(errMsg);
continue;
}
const bytecodeHex = stripHexPrefix(bytecode);
const sourceMap = isContractCreation ? contractData.sourceMap : contractData.sourceMapRuntime;
const pcToSourceRange = parseSourceMap(
contractData.sourceCodes,
sourceMap,
bytecodeHex,
contractData.sources,
);
for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) {
const singleFileCoverageForTrace = ProfilerManager._getSingleFileCoverageForSubtrace(
contractData,
traceInfo.subtrace,
pcToSourceRange,
fileIndex,
);
collector.add(singleFileCoverageForTrace);
}
public async computeSingleTraceCoverageAsync(traceInfo: TraceInfo): Promise<void> {
if (_.isUndefined(this._contractsData)) {
this._contractsData = await this._artifactAdapter.collectContractsDataAsync();
}
const isContractCreation = traceInfo.address === constants.NEW_CONTRACT;
const bytecode = isContractCreation
? (traceInfo as TraceInfoNewContract).bytecode
: (traceInfo as TraceInfoExistingContract).runtimeBytecode;
const contractData = utils.getContractDataIfExists(this._contractsData, bytecode);
if (_.isUndefined(contractData)) {
const errMsg = isContractCreation
? `Unknown contract creation transaction`
: `Transaction to an unknown address: ${traceInfo.address}`;
this._logger.warn(errMsg);
return;
}
const bytecodeHex = stripHexPrefix(bytecode);
const sourceMap = isContractCreation ? contractData.sourceMap : contractData.sourceMapRuntime;
const pcToSourceRange = parseSourceMap(contractData.sourceCodes, sourceMap, bytecodeHex, contractData.sources);
for (let fileIndex = 0; fileIndex < contractData.sources.length; fileIndex++) {
const singleFileCoverageForTrace = ProfilerManager._getSingleFileCoverageForSubtrace(
contractData,
traceInfo.subtrace,
pcToSourceRange,
fileIndex,
);
this._collector.add(singleFileCoverageForTrace);
}
return collector.getFinalCoverage();
}
}

8
packages/sol-cov/src/profiler_subprovider.ts
View File

@@ -1,8 +1,7 @@
import * as _ from 'lodash';
import { AbstractArtifactAdapter } from './artifact_adapters/abstract_artifact_adapter';
import { ProfilerManager } from './profiler_manager';
import { TraceCollectionSubprovider } from './trace_collection_subprovider';
import { TraceInfo } from './types';
/**
* This class implements the [web3-provider-engine](https://github.com/MetaMask/provider-engine) subprovider interface.
@@ -25,12 +24,13 @@ export class ProfilerSubprovider extends TraceCollectionSubprovider {
super(defaultFromAddress, traceCollectionSubproviderConfig);
this._profilerManager = new ProfilerManager(artifactAdapter, isVerbose);
}
public async handleTraceInfoAsync(traceInfo: TraceInfo): Promise<void> {
await this._profilerManager.computeSingleTraceCoverageAsync(traceInfo);
}
/**
* Write the test profiler results to a file in Istanbul format.
*/
public async writeProfilerOutputAsync(): Promise<void> {
const traceInfos = this.getCollectedTraceInfos();
_.forEach(traceInfos, traceInfo => this._profilerManager.appendTraceInfo(traceInfo));
await this._profilerManager.writeProfilerOutputAsync();
}
}

18
packages/sol-cov/src/trace_collection_subprovider.ts
View File

@@ -29,12 +29,11 @@ export interface TraceCollectionSubproviderConfig {
* This class implements the [web3-provider-engine](https://github.com/MetaMask/provider-engine) subprovider interface.
* It collects traces of all transactions that were sent and all calls that were executed through JSON RPC.
*/
export class TraceCollectionSubprovider extends Subprovider {
export abstract class TraceCollectionSubprovider extends Subprovider {
// Lock is used to not accept normal transactions while doing call/snapshot magic because they'll be reverted later otherwise
private _lock = new Lock();
private _defaultFromAddress: string;
private _web3Wrapper!: Web3Wrapper;
private _traceInfos: TraceInfo[] = [];
private _isEnabled = true;
private _config: TraceCollectionSubproviderConfig;
/**
@@ -46,12 +45,6 @@ export class TraceCollectionSubprovider extends Subprovider {
this._defaultFromAddress = defaultFromAddress;
this._config = config;
}
/**
* Returns all trace infos collected by the subprovider so far
*/
public getCollectedTraceInfos(): TraceInfo[] {
return this._traceInfos;
}
/**
* Starts trace collection
*/
@@ -64,6 +57,11 @@ export class TraceCollectionSubprovider extends Subprovider {
public stop(): void {
this._isEnabled = false;
}
/**
* Called for each subtrace.
* @param traceInfo Trace info for this subtrace.
*/
public abstract handleTraceInfoAsync(traceInfo: TraceInfo): Promise<void>;
/**
* This method conforms to the web3-provider-engine interface.
* It is called internally by the ProviderEngine when it is this subproviders
@@ -192,7 +190,7 @@ export class TraceCollectionSubprovider extends Subprovider {
runtimeBytecode,
};
}
this._traceInfos.push(traceInfo);
await this.handleTraceInfoAsync(traceInfo);
}
} else {
for (const subcallAddress of subcallAddresses) {
@@ -204,7 +202,7 @@ export class TraceCollectionSubprovider extends Subprovider {
address: subcallAddress,
runtimeBytecode,
};
this._traceInfos.push(traceInfo);
await this.handleTraceInfoAsync(traceInfo);
}
}
}