Merge pull request #965 from feuGeneA/sol-compile-lot

[sol-compiler] Compile in batches rather than one at a time

packages/sol-compiler/CHANGELOG.json

@@ -1,4 +1,14 @@
 [
+    {
+        "version": "1.1.0",
+        "changes": [
+            {
+                "note":
+                    "Quicken compilation by sending multiple contracts to the same solcjs invocation, batching them together based on compiler version requirements.",
+                "pr": 965
+            }
+        ]
+    },
     {
         "timestamp": 1534210131,
         "version": "1.0.5",

packages/sol-compiler/package.json

@@ -61,6 +61,7 @@
         "@types/semver": "^5.5.0",
         "chai": "^4.0.1",
         "chai-as-promised": "^7.1.0",
+        "chai-bignumber": "^2.0.2",
         "copyfiles": "^2.0.0",
         "dirty-chai": "^2.0.1",
         "make-promises-safe": "^1.1.0",

packages/sol-compiler/src/compiler.ts

@@ -53,6 +53,23 @@ const DEFAULT_COMPILER_SETTINGS: solc.CompilerSettings = {
 };
 const CONFIG_FILE = 'compiler.json';
 
+interface VersionToInputs {
+    [solcVersion: string]: {
+        standardInput: solc.StandardInput;
+        contractsToCompile: string[];
+    };
+}
+
+interface ContractPathToData {
+    [contractPath: string]: ContractData;
+}
+
+interface ContractData {
+    currentArtifactIfExists: ContractArtifact | void;
+    sourceTreeHashHex: string;
+    contractName: string;
+}
+
 /**
  * The Compiler facilitates compiling Solidity smart contracts and saves the results
  * to artifact files.
@@ -65,6 +82,34 @@ export class Compiler {
     private readonly _artifactsDir: string;
     private readonly _solcVersionIfExists: string | undefined;
     private readonly _specifiedContracts: string[] | TYPE_ALL_FILES_IDENTIFIER;
+    private static async _getSolcAsync(
+        solcVersion: string,
+    ): Promise<{ solcInstance: solc.SolcInstance; fullSolcVersion: string }> {
+        const fullSolcVersion = binPaths[solcVersion];
+        if (_.isUndefined(fullSolcVersion)) {
+            throw new Error(`${solcVersion} is not a known compiler version`);
+        }
+        const compilerBinFilename = path.join(SOLC_BIN_DIR, fullSolcVersion);
+        let solcjs: string;
+        if (await fsWrapper.doesFileExistAsync(compilerBinFilename)) {
+            solcjs = (await fsWrapper.readFileAsync(compilerBinFilename)).toString();
+        } else {
+            logUtils.log(`Downloading ${fullSolcVersion}...`);
+            const url = `${constants.BASE_COMPILER_URL}${fullSolcVersion}`;
+            const response = await fetchAsync(url);
+            const SUCCESS_STATUS = 200;
+            if (response.status !== SUCCESS_STATUS) {
+                throw new Error(`Failed to load ${fullSolcVersion}`);
+            }
+            solcjs = await response.text();
+            await fsWrapper.writeFileAsync(compilerBinFilename, solcjs);
+        }
+        if (solcjs.length === 0) {
+            throw new Error('No compiler available');
+        }
+        const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename));
+        return { solcInstance, fullSolcVersion };
+    }
     /**
      * Instantiates a new instance of the Compiler class.
      * @return An instance of the Compiler class.
@@ -107,97 +152,101 @@ export class Compiler {
         } else {
             contractNamesToCompile = this._specifiedContracts;
         }
-        for (const contractNameToCompile of contractNamesToCompile) {
+        await this._compileContractsAsync(contractNamesToCompile);
-            await this._compileContractAsync(contractNameToCompile);
-        }
     }
     /**
      * Compiles contract and saves artifact to artifactsDir.
      * @param fileName Name of contract with '.sol' extension.
      */
-    private async _compileContractAsync(contractName: string): Promise<void> {
+    private async _compileContractsAsync(contractNames: string[]): Promise<void> {
-        const contractSource = this._resolver.resolve(contractName);
+        // batch input contracts together based on the version of the compiler that they require.
-        const absoluteContractPath = path.join(this._contractsDir, contractSource.path);
+        const versionToInputs: VersionToInputs = {};
-        const currentArtifactIfExists = await getContractArtifactIfExistsAsync(this._artifactsDir, contractName);
-        const sourceTreeHashHex = `0x${this._getSourceTreeHash(absoluteContractPath).toString('hex')}`;
-        let shouldCompile = false;
-        if (_.isUndefined(currentArtifactIfExists)) {
-            shouldCompile = true;
-        } else {
-            const currentArtifact = currentArtifactIfExists as ContractArtifact;
-            const isUserOnLatestVersion = currentArtifact.schemaVersion === constants.LATEST_ARTIFACT_VERSION;
-            const didCompilerSettingsChange = !_.isEqual(currentArtifact.compiler.settings, this._compilerSettings);
-            const didSourceChange = currentArtifact.sourceTreeHashHex !== sourceTreeHashHex;
-            shouldCompile = !isUserOnLatestVersion || didCompilerSettingsChange || didSourceChange;
-        }
-        if (!shouldCompile) {
-            return;
-        }
-        let solcVersion = this._solcVersionIfExists;
-        if (_.isUndefined(solcVersion)) {
-            const solcVersionRange = parseSolidityVersionRange(contractSource.source);
-            const availableCompilerVersions = _.keys(binPaths);
-            solcVersion = semver.maxSatisfying(availableCompilerVersions, solcVersionRange);
-        }
-        const fullSolcVersion = binPaths[solcVersion];
-        const compilerBinFilename = path.join(SOLC_BIN_DIR, fullSolcVersion);
-        let solcjs: string;
-        const isCompilerAvailableLocally = fs.existsSync(compilerBinFilename);
-        if (isCompilerAvailableLocally) {
-            solcjs = fs.readFileSync(compilerBinFilename).toString();
-        } else {
-            logUtils.log(`Downloading ${fullSolcVersion}...`);
-            const url = `${constants.BASE_COMPILER_URL}${fullSolcVersion}`;
-            const response = await fetchAsync(url);
-            const SUCCESS_STATUS = 200;
-            if (response.status !== SUCCESS_STATUS) {
-                throw new Error(`Failed to load ${fullSolcVersion}`);
-            }
-            solcjs = await response.text();
-            fs.writeFileSync(compilerBinFilename, solcjs);
-        }
-        const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename));
 
-        logUtils.log(`Compiling ${contractName} with Solidity v${solcVersion}...`);
+        // map contract paths to data about them for later verification and persistence
-        const standardInput: solc.StandardInput = {
+        const contractPathToData: ContractPathToData = {};
-            language: 'Solidity',
+
-            sources: {
+        for (const contractName of contractNames) {
-                [contractSource.path]: {
+            const contractSource = this._resolver.resolve(contractName);
-                    content: contractSource.source,
+            const contractData = {
-                },
+                contractName,
-            },
+                currentArtifactIfExists: await getContractArtifactIfExistsAsync(this._artifactsDir, contractName),
-            settings: this._compilerSettings,
+                sourceTreeHashHex: `0x${this._getSourceTreeHash(
+                    path.join(this._contractsDir, contractSource.path),
+                ).toString('hex')}`,
             };
-        const compiled: solc.StandardOutput = JSON.parse(
+            if (!this._shouldCompile(contractData)) {
-            solcInstance.compileStandardWrapper(JSON.stringify(standardInput), importPath => {
+                continue;
-                const sourceCodeIfExists = this._resolver.resolve(importPath);
+            }
-                return { contents: sourceCodeIfExists.source };
+            contractPathToData[contractSource.path] = contractData;
-            }),
+            const solcVersion = _.isUndefined(this._solcVersionIfExists)
+                ? semver.maxSatisfying(_.keys(binPaths), parseSolidityVersionRange(contractSource.source))
+                : this._solcVersionIfExists;
+            const isFirstContractWithThisVersion = _.isUndefined(versionToInputs[solcVersion]);
+            if (isFirstContractWithThisVersion) {
+                versionToInputs[solcVersion] = {
+                    standardInput: {
+                        language: 'Solidity',
+                        sources: {},
+                        settings: this._compilerSettings,
+                    },
+                    contractsToCompile: [],
+                };
+            }
+            // add input to the right version batch
+            versionToInputs[solcVersion].standardInput.sources[contractSource.path] = {
+                content: contractSource.source,
+            };
+            versionToInputs[solcVersion].contractsToCompile.push(contractSource.path);
+        }
+
+        const solcVersions = _.keys(versionToInputs);
+        for (const solcVersion of solcVersions) {
+            const input = versionToInputs[solcVersion];
+            logUtils.log(
+                `Compiling ${input.contractsToCompile.length} contracts (${
+                    input.contractsToCompile
+                }) with Solidity v${solcVersion}...`,
             );
 
-        if (!_.isUndefined(compiled.errors)) {
+            const { solcInstance, fullSolcVersion } = await Compiler._getSolcAsync(solcVersion);
-            const SOLIDITY_WARNING = 'warning';
+
-            const errors = _.filter(compiled.errors, entry => entry.severity !== SOLIDITY_WARNING);
+            const compilerOutput = this._compile(solcInstance, input.standardInput);
-            const warnings = _.filter(compiled.errors, entry => entry.severity === SOLIDITY_WARNING);
+
-            if (!_.isEmpty(errors)) {
+            for (const contractPath of input.contractsToCompile) {
-                errors.forEach(error => {
+                await this._verifyAndPersistCompiledContractAsync(
-                    const normalizedErrMsg = getNormalizedErrMsg(error.formattedMessage || error.message);
+                    contractPath,
-                    logUtils.log(chalk.red(normalizedErrMsg));
+                    contractPathToData[contractPath].currentArtifactIfExists,
-                });
+                    contractPathToData[contractPath].sourceTreeHashHex,
-                process.exit(1);
+                    contractPathToData[contractPath].contractName,
+                    fullSolcVersion,
+                    compilerOutput,
+                );
+            }
+        }
+    }
+    private _shouldCompile(contractData: ContractData): boolean {
+        if (_.isUndefined(contractData.currentArtifactIfExists)) {
+            return true;
         } else {
-                warnings.forEach(warning => {
+            const currentArtifact = contractData.currentArtifactIfExists as ContractArtifact;
-                    const normalizedWarningMsg = getNormalizedErrMsg(warning.formattedMessage || warning.message);
+            const isUserOnLatestVersion = currentArtifact.schemaVersion === constants.LATEST_ARTIFACT_VERSION;
-                    logUtils.log(chalk.yellow(normalizedWarningMsg));
+            const didCompilerSettingsChange = !_.isEqual(currentArtifact.compiler.settings, this._compilerSettings);
-                });
+            const didSourceChange = currentArtifact.sourceTreeHashHex !== contractData.sourceTreeHashHex;
+            return !isUserOnLatestVersion || didCompilerSettingsChange || didSourceChange;
         }
     }
-        const compiledData = compiled.contracts[contractSource.path][contractName];
+    private async _verifyAndPersistCompiledContractAsync(
+        contractPath: string,
+        currentArtifactIfExists: ContractArtifact | void,
+        sourceTreeHashHex: string,
+        contractName: string,
+        fullSolcVersion: string,
+        compilerOutput: solc.StandardOutput,
+    ): Promise<void> {
+        const compiledData = compilerOutput.contracts[contractPath][contractName];
         if (_.isUndefined(compiledData)) {
             throw new Error(
-                `Contract ${contractName} not found in ${
+                `Contract ${contractName} not found in ${contractPath}. Please make sure your contract has the same name as it's file name`,
-                    contractSource.path
-                }. Please make sure your contract has the same name as it's file name`,
             );
         }
         if (!_.isUndefined(compiledData.evm)) {
@@ -215,12 +264,12 @@ export class Compiler {
         }
 
         const sourceCodes = _.mapValues(
-            compiled.sources,
+            compilerOutput.sources,
             (_1, sourceFilePath) => this._resolver.resolve(sourceFilePath).source,
         );
         const contractVersion: ContractVersionData = {
             compilerOutput: compiledData,
-            sources: compiled.sources,
+            sources: compilerOutput.sources,
             sourceCodes,
             sourceTreeHashHex,
             compiler: {
@@ -251,6 +300,32 @@ export class Compiler {
         await fsWrapper.writeFileAsync(currentArtifactPath, artifactString);
         logUtils.log(`${contractName} artifact saved!`);
     }
+    private _compile(solcInstance: solc.SolcInstance, standardInput: solc.StandardInput): solc.StandardOutput {
+        const compiled: solc.StandardOutput = JSON.parse(
+            solcInstance.compileStandardWrapper(JSON.stringify(standardInput), importPath => {
+                const sourceCodeIfExists = this._resolver.resolve(importPath);
+                return { contents: sourceCodeIfExists.source };
+            }),
+        );
+        if (!_.isUndefined(compiled.errors)) {
+            const SOLIDITY_WARNING = 'warning';
+            const errors = _.filter(compiled.errors, entry => entry.severity !== SOLIDITY_WARNING);
+            const warnings = _.filter(compiled.errors, entry => entry.severity === SOLIDITY_WARNING);
+            if (!_.isEmpty(errors)) {
+                errors.forEach(error => {
+                    const normalizedErrMsg = getNormalizedErrMsg(error.formattedMessage || error.message);
+                    logUtils.log(chalk.red(normalizedErrMsg));
+                });
+                throw new Error('Compilation errors encountered');
+            } else {
+                warnings.forEach(warning => {
+                    const normalizedWarningMsg = getNormalizedErrMsg(warning.formattedMessage || warning.message);
+                    logUtils.log(chalk.yellow(normalizedWarningMsg));
+                });
+            }
+        }
+        return compiled;
+    }
     /**
      * Gets the source tree hash for a file and its dependencies.
      * @param fileName Name of contract file.

packages/sol-compiler/src/utils/fs_wrapper.ts

@@ -10,4 +10,19 @@ export const fsWrapper = {
     doesPathExistSync: fs.existsSync,
     rmdirSync: fs.rmdirSync,
     removeFileAsync: promisify<undefined>(fs.unlink),
+    statAsync: promisify<fs.Stats>(fs.stat),
+    appendFileAsync: promisify<undefined>(fs.appendFile),
+    accessAsync: promisify<boolean>(fs.access),
+    doesFileExistAsync: async (filePath: string): Promise<boolean> => {
+        try {
+            await fsWrapper.accessAsync(
+                filePath,
+                // node says we need to use bitwise, but tslint says no:
+                fs.constants.F_OK | fs.constants.R_OK, // tslint:disable-line:no-bitwise
+            );
+        } catch (err) {
+            return false;
+        }
+        return true;
+    },
 };

packages/sol-compiler/test/compiler_test.ts

@@ -1,4 +1,5 @@
-import { DoneCallback } from '@0xproject/types';
+import { join } from 'path';
+
 import * as chai from 'chai';
 import 'mocha';
 
@@ -7,31 +8,31 @@ import { fsWrapper } from '../src/utils/fs_wrapper';
 import { CompilerOptions, ContractArtifact } from '../src/utils/types';
 
 import { exchange_binary } from './fixtures/exchange_bin';
+import { chaiSetup } from './util/chai_setup';
 import { constants } from './util/constants';
 
+chaiSetup.configure();
 const expect = chai.expect;
 
 describe('#Compiler', function(): void {
     this.timeout(constants.timeoutMs); // tslint:disable-line:no-invalid-this
     const artifactsDir = `${__dirname}/fixtures/artifacts`;
     const contractsDir = `${__dirname}/fixtures/contracts`;
-    const exchangeArtifactPath = `${artifactsDir}/Exchange.json`;
     const compilerOpts: CompilerOptions = {
         artifactsDir,
         contractsDir,
         contracts: constants.contracts,
     };
-    const compiler = new Compiler(compilerOpts);
+    it('should create an Exchange artifact with the correct unlinked binary', async () => {
-    beforeEach((done: DoneCallback) => {
+        compilerOpts.contracts = ['Exchange'];
-        (async () => {
+
+        const exchangeArtifactPath = `${artifactsDir}/Exchange.json`;
         if (fsWrapper.doesPathExistSync(exchangeArtifactPath)) {
             await fsWrapper.removeFileAsync(exchangeArtifactPath);
         }
-            await compiler.compileAsync();
+
-            done();
+        await new Compiler(compilerOpts).compileAsync();
-        })().catch(done);
+
-    });
-    it('should create an Exchange artifact with the correct unlinked binary', async () => {
         const opts = {
             encoding: 'utf8',
         };
@@ -47,4 +48,67 @@ describe('#Compiler', function(): void {
         const exchangeBinaryWithoutMetadata = exchange_binary.slice(0, -metadataHexLength);
         expect(unlinkedBinaryWithoutMetadata).to.equal(exchangeBinaryWithoutMetadata);
     });
+    it("should throw when Whatever.sol doesn't contain a Whatever contract", async () => {
+        const contract = 'BadContractName';
+
+        const exchangeArtifactPath = `${artifactsDir}/${contract}.json`;
+        if (fsWrapper.doesPathExistSync(exchangeArtifactPath)) {
+            await fsWrapper.removeFileAsync(exchangeArtifactPath);
+        }
+
+        compilerOpts.contracts = [contract];
+        const compiler = new Compiler(compilerOpts);
+
+        expect(compiler.compileAsync()).to.be.rejected();
+    });
+    describe('after a successful compilation', () => {
+        const contract = 'Exchange';
+        let artifactPath: string;
+        let artifactCreatedAtMs: number;
+        beforeEach(async () => {
+            compilerOpts.contracts = [contract];
+
+            artifactPath = `${artifactsDir}/${contract}.json`;
+            if (fsWrapper.doesPathExistSync(artifactPath)) {
+                await fsWrapper.removeFileAsync(artifactPath);
+            }
+
+            await new Compiler(compilerOpts).compileAsync();
+
+            artifactCreatedAtMs = (await fsWrapper.statAsync(artifactPath)).mtimeMs;
+        });
+        it('recompilation should update artifact when source has changed', async () => {
+            // append some meaningless data to the contract, so that its hash
+            // will change, so that the compiler will decide to recompile it.
+            fsWrapper.appendFileAsync(join(contractsDir, `${contract}.sol`), ' ');
+
+            await new Compiler(compilerOpts).compileAsync();
+
+            const artifactModifiedAtMs = (await fsWrapper.statAsync(artifactPath)).mtimeMs;
+
+            expect(artifactModifiedAtMs).to.be.greaterThan(artifactCreatedAtMs);
+        });
+        it("recompilation should NOT update artifact when source hasn't changed", async () => {
+            await new Compiler(compilerOpts).compileAsync();
+
+            const artifactModifiedAtMs = (await fsWrapper.statAsync(artifactPath)).mtimeMs;
+
+            expect(artifactModifiedAtMs).to.equal(artifactCreatedAtMs);
+        });
+    });
+    it('should only compile what was requested', async () => {
+        // remove all artifacts
+        for (const artifact of await fsWrapper.readdirAsync(artifactsDir)) {
+            await fsWrapper.removeFileAsync(join(artifactsDir, artifact));
+        }
+
+        // compile EmptyContract
+        compilerOpts.contracts = ['EmptyContract'];
+        await new Compiler(compilerOpts).compileAsync();
+
+        // make sure the artifacts dir only contains EmptyContract.json
+        for (const artifact of await fsWrapper.readdirAsync(artifactsDir)) {
+            expect(artifact).to.equal('EmptyContract.json');
+        }
+    });
 });

packages/sol-compiler/test/fixtures/contracts/BadContractName.sol

@@ -0,0 +1,3 @@
+pragma solidity ^0.4.14;
+
+contract ContractNameThatDoesntMatchFilename { }

packages/sol-compiler/test/fixtures/contracts/EmptyContract.sol

@@ -0,0 +1,3 @@
+pragma solidity ^0.4.14;
+
+contract EmptyContract { }

packages/sol-compiler/test/util/chai_setup.ts

@@ -0,0 +1,13 @@
+import * as chai from 'chai';
+import chaiAsPromised = require('chai-as-promised');
+import ChaiBigNumber = require('chai-bignumber');
+import * as dirtyChai from 'dirty-chai';
+
+export const chaiSetup = {
+    configure(): void {
+        chai.config.includeStack = true;
+        chai.use(ChaiBigNumber());
+        chai.use(dirtyChai);
+        chai.use(chaiAsPromised);
+    },
+};