crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #965 from feuGeneA/sol-compile-lot

Browse Source 
[sol-compiler] Compile in batches rather than one at a time
This commit is contained in:
Fabio Berger
2018-08-16 16:42:37 -07:00
committed by GitHub
parent d0d6a6a500 c1ec849554
commit 2604d40751
8 changed files with 278 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
packages/sol-compiler/CHANGELOG.json
View File

@@ -1,4 +1,14 @@
[
{
"version": "1.1.0",
"changes": [
{
"note":
"Quicken compilation by sending multiple contracts to the same solcjs invocation, batching them together based on compiler version requirements.",
"pr": 965
}
]
},
{
"timestamp": 1534210131,
"version": "1.0.5",

1
packages/sol-compiler/package.json
View File

@@ -61,6 +61,7 @@
"@types/semver": "^5.5.0",
"chai": "^4.0.1",
"chai-as-promised": "^7.1.0",
"chai-bignumber": "^2.0.2",
"copyfiles": "^2.0.0",
"dirty-chai": "^2.0.1",
"make-promises-safe": "^1.1.0",

239
packages/sol-compiler/src/compiler.ts
View File

@@ -53,6 +53,23 @@ const DEFAULT_COMPILER_SETTINGS: solc.CompilerSettings = {
};
const CONFIG_FILE = 'compiler.json';
interface VersionToInputs {
[solcVersion: string]: {
standardInput: solc.StandardInput;
contractsToCompile: string[];
};
}
interface ContractPathToData {
[contractPath: string]: ContractData;
}
interface ContractData {
currentArtifactIfExists: ContractArtifact | void;
sourceTreeHashHex: string;
contractName: string;
}
/**
* The Compiler facilitates compiling Solidity smart contracts and saves the results
* to artifact files.
@@ -65,6 +82,34 @@ export class Compiler {
private readonly _artifactsDir: string;
private readonly _solcVersionIfExists: string | undefined;
private readonly _specifiedContracts: string[] | TYPE_ALL_FILES_IDENTIFIER;
private static async _getSolcAsync(
solcVersion: string,
): Promise<{ solcInstance: solc.SolcInstance; fullSolcVersion: string }> {
const fullSolcVersion = binPaths[solcVersion];
if (_.isUndefined(fullSolcVersion)) {
throw new Error(`${solcVersion} is not a known compiler version`);
}
const compilerBinFilename = path.join(SOLC_BIN_DIR, fullSolcVersion);
let solcjs: string;
if (await fsWrapper.doesFileExistAsync(compilerBinFilename)) {
solcjs = (await fsWrapper.readFileAsync(compilerBinFilename)).toString();
} else {
logUtils.log(`Downloading ${fullSolcVersion}...`);
const url = `${constants.BASE_COMPILER_URL}${fullSolcVersion}`;
const response = await fetchAsync(url);
const SUCCESS_STATUS = 200;
if (response.status !== SUCCESS_STATUS) {
throw new Error(`Failed to load ${fullSolcVersion}`);
}
solcjs = await response.text();
await fsWrapper.writeFileAsync(compilerBinFilename, solcjs);
}
if (solcjs.length === 0) {
throw new Error('No compiler available');
}
const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename));
return { solcInstance, fullSolcVersion };
}
/**
* Instantiates a new instance of the Compiler class.
* @return An instance of the Compiler class.
@@ -107,97 +152,101 @@ export class Compiler {
} else {
contractNamesToCompile = this._specifiedContracts;
}
for (const contractNameToCompile of contractNamesToCompile) {
await this._compileContractAsync(contractNameToCompile);
}
await this._compileContractsAsync(contractNamesToCompile);
}
/**
* Compiles contract and saves artifact to artifactsDir.
* @param fileName Name of contract with '.sol' extension.
*/
private async _compileContractAsync(contractName: string): Promise<void> {
const contractSource = this._resolver.resolve(contractName);
const absoluteContractPath = path.join(this._contractsDir, contractSource.path);
const currentArtifactIfExists = await getContractArtifactIfExistsAsync(this._artifactsDir, contractName);
const sourceTreeHashHex = `0x${this._getSourceTreeHash(absoluteContractPath).toString('hex')}`;
let shouldCompile = false;
if (_.isUndefined(currentArtifactIfExists)) {
shouldCompile = true;
private async _compileContractsAsync(contractNames: string[]): Promise<void> {
// batch input contracts together based on the version of the compiler that they require.
const versionToInputs: VersionToInputs = {};
// map contract paths to data about them for later verification and persistence
const contractPathToData: ContractPathToData = {};
for (const contractName of contractNames) {
const contractSource = this._resolver.resolve(contractName);
const contractData = {
contractName,
currentArtifactIfExists: await getContractArtifactIfExistsAsync(this._artifactsDir, contractName),
sourceTreeHashHex: `0x${this._getSourceTreeHash(
path.join(this._contractsDir, contractSource.path),
).toString('hex')}`,
};
if (!this._shouldCompile(contractData)) {
continue;
}
contractPathToData[contractSource.path] = contractData;
const solcVersion = _.isUndefined(this._solcVersionIfExists)
? semver.maxSatisfying(_.keys(binPaths), parseSolidityVersionRange(contractSource.source))
: this._solcVersionIfExists;
const isFirstContractWithThisVersion = _.isUndefined(versionToInputs[solcVersion]);
if (isFirstContractWithThisVersion) {
versionToInputs[solcVersion] = {
standardInput: {
language: 'Solidity',
sources: {},
settings: this._compilerSettings,
},
contractsToCompile: [],
};
}
// add input to the right version batch
versionToInputs[solcVersion].standardInput.sources[contractSource.path] = {
content: contractSource.source,
};
versionToInputs[solcVersion].contractsToCompile.push(contractSource.path);
}
const solcVersions = _.keys(versionToInputs);
for (const solcVersion of solcVersions) {
const input = versionToInputs[solcVersion];
logUtils.log(
`Compiling ${input.contractsToCompile.length} contracts (${
input.contractsToCompile
}) with Solidity v${solcVersion}...`,
);
const { solcInstance, fullSolcVersion } = await Compiler._getSolcAsync(solcVersion);
const compilerOutput = this._compile(solcInstance, input.standardInput);
for (const contractPath of input.contractsToCompile) {
await this._verifyAndPersistCompiledContractAsync(
contractPath,
contractPathToData[contractPath].currentArtifactIfExists,
contractPathToData[contractPath].sourceTreeHashHex,
contractPathToData[contractPath].contractName,
fullSolcVersion,
compilerOutput,
);
}
}
}
private _shouldCompile(contractData: ContractData): boolean {
if (_.isUndefined(contractData.currentArtifactIfExists)) {
return true;
} else {
const currentArtifact = currentArtifactIfExists as ContractArtifact;
const currentArtifact = contractData.currentArtifactIfExists as ContractArtifact;
const isUserOnLatestVersion = currentArtifact.schemaVersion === constants.LATEST_ARTIFACT_VERSION;
const didCompilerSettingsChange = !_.isEqual(currentArtifact.compiler.settings, this._compilerSettings);
const didSourceChange = currentArtifact.sourceTreeHashHex !== sourceTreeHashHex;
shouldCompile = !isUserOnLatestVersion || didCompilerSettingsChange || didSourceChange;
const didSourceChange = currentArtifact.sourceTreeHashHex !== contractData.sourceTreeHashHex;
return !isUserOnLatestVersion || didCompilerSettingsChange || didSourceChange;
}
if (!shouldCompile) {
return;
}
let solcVersion = this._solcVersionIfExists;
if (_.isUndefined(solcVersion)) {
const solcVersionRange = parseSolidityVersionRange(contractSource.source);
const availableCompilerVersions = _.keys(binPaths);
solcVersion = semver.maxSatisfying(availableCompilerVersions, solcVersionRange);
}
const fullSolcVersion = binPaths[solcVersion];
const compilerBinFilename = path.join(SOLC_BIN_DIR, fullSolcVersion);
let solcjs: string;
const isCompilerAvailableLocally = fs.existsSync(compilerBinFilename);
if (isCompilerAvailableLocally) {
solcjs = fs.readFileSync(compilerBinFilename).toString();
} else {
logUtils.log(`Downloading ${fullSolcVersion}...`);
const url = `${constants.BASE_COMPILER_URL}${fullSolcVersion}`;
const response = await fetchAsync(url);
const SUCCESS_STATUS = 200;
if (response.status !== SUCCESS_STATUS) {
throw new Error(`Failed to load ${fullSolcVersion}`);
}
solcjs = await response.text();
fs.writeFileSync(compilerBinFilename, solcjs);
}
const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename));
logUtils.log(`Compiling ${contractName} with Solidity v${solcVersion}...`);
const standardInput: solc.StandardInput = {
language: 'Solidity',
sources: {
[contractSource.path]: {
content: contractSource.source,
},
},
settings: this._compilerSettings,
};
const compiled: solc.StandardOutput = JSON.parse(
solcInstance.compileStandardWrapper(JSON.stringify(standardInput), importPath => {
const sourceCodeIfExists = this._resolver.resolve(importPath);
return { contents: sourceCodeIfExists.source };
}),
);
if (!_.isUndefined(compiled.errors)) {
const SOLIDITY_WARNING = 'warning';
const errors = _.filter(compiled.errors, entry => entry.severity !== SOLIDITY_WARNING);
const warnings = _.filter(compiled.errors, entry => entry.severity === SOLIDITY_WARNING);
if (!_.isEmpty(errors)) {
errors.forEach(error => {
const normalizedErrMsg = getNormalizedErrMsg(error.formattedMessage || error.message);
logUtils.log(chalk.red(normalizedErrMsg));
});
process.exit(1);
} else {
warnings.forEach(warning => {
const normalizedWarningMsg = getNormalizedErrMsg(warning.formattedMessage || warning.message);
logUtils.log(chalk.yellow(normalizedWarningMsg));
});
}
}
const compiledData = compiled.contracts[contractSource.path][contractName];
}
private async _verifyAndPersistCompiledContractAsync(
contractPath: string,
currentArtifactIfExists: ContractArtifact | void,
sourceTreeHashHex: string,
contractName: string,
fullSolcVersion: string,
compilerOutput: solc.StandardOutput,
): Promise<void> {
const compiledData = compilerOutput.contracts[contractPath][contractName];
if (_.isUndefined(compiledData)) {
throw new Error(
`Contract ${contractName} not found in ${
contractSource.path
}. Please make sure your contract has the same name as it's file name`,
`Contract ${contractName} not found in ${contractPath}. Please make sure your contract has the same name as it's file name`,
);
}
if (!_.isUndefined(compiledData.evm)) {
@@ -215,12 +264,12 @@ export class Compiler {
}
const sourceCodes = _.mapValues(
compiled.sources,
compilerOutput.sources,
(_1, sourceFilePath) => this._resolver.resolve(sourceFilePath).source,
);
const contractVersion: ContractVersionData = {
compilerOutput: compiledData,
sources: compiled.sources,
sources: compilerOutput.sources,
sourceCodes,
sourceTreeHashHex,
compiler: {
@@ -251,6 +300,32 @@ export class Compiler {
await fsWrapper.writeFileAsync(currentArtifactPath, artifactString);
logUtils.log(`${contractName} artifact saved!`);
}
private _compile(solcInstance: solc.SolcInstance, standardInput: solc.StandardInput): solc.StandardOutput {
const compiled: solc.StandardOutput = JSON.parse(
solcInstance.compileStandardWrapper(JSON.stringify(standardInput), importPath => {
const sourceCodeIfExists = this._resolver.resolve(importPath);
return { contents: sourceCodeIfExists.source };
}),
);
if (!_.isUndefined(compiled.errors)) {
const SOLIDITY_WARNING = 'warning';
const errors = _.filter(compiled.errors, entry => entry.severity !== SOLIDITY_WARNING);
const warnings = _.filter(compiled.errors, entry => entry.severity === SOLIDITY_WARNING);
if (!_.isEmpty(errors)) {
errors.forEach(error => {
const normalizedErrMsg = getNormalizedErrMsg(error.formattedMessage || error.message);
logUtils.log(chalk.red(normalizedErrMsg));
});
throw new Error('Compilation errors encountered');
} else {
warnings.forEach(warning => {
const normalizedWarningMsg = getNormalizedErrMsg(warning.formattedMessage || warning.message);
logUtils.log(chalk.yellow(normalizedWarningMsg));
});
}
}
return compiled;
}
/**
* Gets the source tree hash for a file and its dependencies.
* @param fileName Name of contract file.

15
packages/sol-compiler/src/utils/fs_wrapper.ts
View File

@@ -10,4 +10,19 @@ export const fsWrapper = {
doesPathExistSync: fs.existsSync,
rmdirSync: fs.rmdirSync,
removeFileAsync: promisify<undefined>(fs.unlink),
statAsync: promisify<fs.Stats>(fs.stat),
appendFileAsync: promisify<undefined>(fs.appendFile),
accessAsync: promisify<boolean>(fs.access),
doesFileExistAsync: async (filePath: string): Promise<boolean> => {
try {
await fsWrapper.accessAsync(
filePath,
// node says we need to use bitwise, but tslint says no:
fs.constants.F_OK | fs.constants.R_OK, // tslint:disable-line:no-bitwise
);
} catch (err) {
return false;
}
return true;
},
};

88
packages/sol-compiler/test/compiler_test.ts
View File

@@ -1,4 +1,5 @@
import { DoneCallback } from '@0xproject/types';
import { join } from 'path';
import * as chai from 'chai';
import 'mocha';
@@ -7,31 +8,31 @@ import { fsWrapper } from '../src/utils/fs_wrapper';
import { CompilerOptions, ContractArtifact } from '../src/utils/types';
import { exchange_binary } from './fixtures/exchange_bin';
import { chaiSetup } from './util/chai_setup';
import { constants } from './util/constants';
chaiSetup.configure();
const expect = chai.expect;
describe('#Compiler', function(): void {
this.timeout(constants.timeoutMs); // tslint:disable-line:no-invalid-this
const artifactsDir = `${__dirname}/fixtures/artifacts`;
const contractsDir = `${__dirname}/fixtures/contracts`;
const exchangeArtifactPath = `${artifactsDir}/Exchange.json`;
const compilerOpts: CompilerOptions = {
artifactsDir,
contractsDir,
contracts: constants.contracts,
};
const compiler = new Compiler(compilerOpts);
beforeEach((done: DoneCallback) => {
(async () => {
if (fsWrapper.doesPathExistSync(exchangeArtifactPath)) {
await fsWrapper.removeFileAsync(exchangeArtifactPath);
}
await compiler.compileAsync();
done();
})().catch(done);
});
it('should create an Exchange artifact with the correct unlinked binary', async () => {
compilerOpts.contracts = ['Exchange'];
const exchangeArtifactPath = `${artifactsDir}/Exchange.json`;
if (fsWrapper.doesPathExistSync(exchangeArtifactPath)) {
await fsWrapper.removeFileAsync(exchangeArtifactPath);
}
await new Compiler(compilerOpts).compileAsync();
const opts = {
encoding: 'utf8',
};
@@ -47,4 +48,67 @@ describe('#Compiler', function(): void {
const exchangeBinaryWithoutMetadata = exchange_binary.slice(0, -metadataHexLength);
expect(unlinkedBinaryWithoutMetadata).to.equal(exchangeBinaryWithoutMetadata);
});
it("should throw when Whatever.sol doesn't contain a Whatever contract", async () => {
const contract = 'BadContractName';
const exchangeArtifactPath = `${artifactsDir}/${contract}.json`;
if (fsWrapper.doesPathExistSync(exchangeArtifactPath)) {
await fsWrapper.removeFileAsync(exchangeArtifactPath);
}
compilerOpts.contracts = [contract];
const compiler = new Compiler(compilerOpts);
expect(compiler.compileAsync()).to.be.rejected();
});
describe('after a successful compilation', () => {
const contract = 'Exchange';
let artifactPath: string;
let artifactCreatedAtMs: number;
beforeEach(async () => {
compilerOpts.contracts = [contract];
artifactPath = `${artifactsDir}/${contract}.json`;
if (fsWrapper.doesPathExistSync(artifactPath)) {
await fsWrapper.removeFileAsync(artifactPath);
}
await new Compiler(compilerOpts).compileAsync();
artifactCreatedAtMs = (await fsWrapper.statAsync(artifactPath)).mtimeMs;
});
it('recompilation should update artifact when source has changed', async () => {
// append some meaningless data to the contract, so that its hash
// will change, so that the compiler will decide to recompile it.
fsWrapper.appendFileAsync(join(contractsDir, `${contract}.sol`), ' ');
await new Compiler(compilerOpts).compileAsync();
const artifactModifiedAtMs = (await fsWrapper.statAsync(artifactPath)).mtimeMs;
expect(artifactModifiedAtMs).to.be.greaterThan(artifactCreatedAtMs);
});
it("recompilation should NOT update artifact when source hasn't changed", async () => {
await new Compiler(compilerOpts).compileAsync();
const artifactModifiedAtMs = (await fsWrapper.statAsync(artifactPath)).mtimeMs;
expect(artifactModifiedAtMs).to.equal(artifactCreatedAtMs);
});
});
it('should only compile what was requested', async () => {
// remove all artifacts
for (const artifact of await fsWrapper.readdirAsync(artifactsDir)) {
await fsWrapper.removeFileAsync(join(artifactsDir, artifact));
}
// compile EmptyContract
compilerOpts.contracts = ['EmptyContract'];
await new Compiler(compilerOpts).compileAsync();
// make sure the artifacts dir only contains EmptyContract.json
for (const artifact of await fsWrapper.readdirAsync(artifactsDir)) {
expect(artifact).to.equal('EmptyContract.json');
}
});
});

3
packages/sol-compiler/test/fixtures/contracts/BadContractName.sol vendored Normal file
View File

@@ -0,0 +1,3 @@
pragma solidity ^0.4.14;
contract ContractNameThatDoesntMatchFilename { }

3
packages/sol-compiler/test/fixtures/contracts/EmptyContract.sol vendored Normal file
View File

@@ -0,0 +1,3 @@
pragma solidity ^0.4.14;
contract EmptyContract { }

13
packages/sol-compiler/test/util/chai_setup.ts Normal file
View File

@@ -0,0 +1,13 @@
import * as chai from 'chai';
import chaiAsPromised = require('chai-as-promised');
import ChaiBigNumber = require('chai-bignumber');
import * as dirtyChai from 'dirty-chai';
export const chaiSetup = {
configure(): void {
chai.config.includeStack = true;
chai.use(ChaiBigNumber());
chai.use(dirtyChai);
chai.use(chaiAsPromised);
},
};