Implement docker as another backend for sol-compiler

contracts/examples/compiler.json

@@ -1,6 +1,8 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "workspaceDir": "../..",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,

contracts/extensions/compiler.json

@@ -1,6 +1,8 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "workspaceDir": "../..",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,

contracts/interfaces/compiler.json

@@ -1,6 +1,8 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "workspaceDir": "../..",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,

contracts/libs/compiler.json

@@ -1,6 +1,8 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "workspaceDir": "../..",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,

contracts/multisig/compiler.json

@@ -1,6 +1,8 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "workspaceDir": "../..",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,

contracts/protocol/compiler.json

@@ -1,6 +1,8 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "workspaceDir": "../..",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,

contracts/tokens/compiler.json

@@ -1,6 +1,8 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "workspaceDir": "../..",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,

contracts/utils/compiler.json

@@ -1,6 +1,8 @@
 {
     "artifactsDir": "./generated-artifacts",
     "contractsDir": "./contracts",
+    "workspaceDir": "../..",
+    "useDockerisedSolc": true,
     "compilerSettings": {
         "optimizer": {
             "enabled": true,

packages/ethereum-types/src/index.ts

@@ -503,19 +503,23 @@ export interface Source {
 
 /**
  * Options you can specify (as flags or in a compiler.json file) when invoking sol-compiler
- * contractsDir: Directory containing your project's Solidity contracts. Can contain nested directories.
+ * contractsDir: Directory containing your package's Solidity contracts. Can contain nested directories.
+ * workspaceDir: Directory containing your project's Solidity contracts. All the contracts used in compilation must be withing it. Similar to --allow-paths in Solidity.
  * artifactsDir: Directory where you want the generated artifacts.json written to
  * compilerSettings: Desired settings to pass to the Solidity compiler during compilation.
  * (http://solidity.readthedocs.io/en/v0.4.24/using-the-compiler.html#compiler-input-and-output-json-description)
  * contracts: List of contract names you wish to compile, or alternatively ['*'] to compile all contracts in the
  * specified directory.
+ * useDockerisedSolc: If set to true - sol-compiler will try using docker to achieve faster compilation times. Otherwise and by default - solcjs will be used.
  * solcVersion: If you don't want to compile each contract with the Solidity version specified in-file, you can force all
  * contracts to compile with the the version specified here.
  */
 export interface CompilerOptions {
     contractsDir?: string;
+    workspaceDir?: string;
     artifactsDir?: string;
     compilerSettings?: CompilerSettings;
     contracts?: string[] | '*';
+    useDockerisedSolc?: boolean;
     solcVersion?: string;
 } // tslint:disable-line:max-file-line-count

packages/sol-compiler/src/compiler.ts

@@ -10,6 +10,7 @@ import {
     URLResolver,
 } from '@0x/sol-resolver';
 import { logUtils } from '@0x/utils';
+import { execSync } from 'child_process';
 import * as chokidar from 'chokidar';
 import { CompilerOptions, ContractArtifact, ContractVersionData, StandardOutput } from 'ethereum-types';
 import * as fs from 'fs';
@@ -23,10 +24,11 @@ import { compilerOptionsSchema } from './schemas/compiler_options_schema';
 import { binPaths } from './solc/bin_paths';
 import {
     addHexPrefixToContractBytecode,
-    compile,
+    compileDocker,
+    compileSolcJS,
     createDirIfDoesNotExistAsync,
     getContractArtifactIfExistsAsync,
-    getSolcAsync,
+    getSolcJSAsync,
     getSourcesWithDependencies,
     getSourceTreeHash,
     parseSolidityVersionRange,
@@ -40,6 +42,7 @@ const ALL_CONTRACTS_IDENTIFIER = '*';
 const ALL_FILES_IDENTIFIER = '*';
 const DEFAULT_CONTRACTS_DIR = path.resolve('contracts');
 const DEFAULT_ARTIFACTS_DIR = path.resolve('artifacts');
+const DEFAULT_USE_DOCKERISED_SOLC = false;
 // Solc compiler settings cannot be configured from the commandline.
 // If you need this configured, please create a `compiler.json` config file
 // with your desired configurations.
@@ -80,10 +83,12 @@ export class Compiler {
     private readonly _resolver: Resolver;
     private readonly _nameResolver: NameResolver;
     private readonly _contractsDir: string;
+    private readonly _workspaceDir: string;
     private readonly _compilerSettings: solc.CompilerSettings;
     private readonly _artifactsDir: string;
     private readonly _solcVersionIfExists: string | undefined;
     private readonly _specifiedContracts: string[] | TYPE_ALL_FILES_IDENTIFIER;
+    private readonly _useDockerisedSolc: boolean;
     /**
      * Instantiates a new instance of the Compiler class.
      * @param opts Optional compiler options
@@ -97,16 +102,23 @@ export class Compiler {
             : {};
         const passedOpts = opts || {};
         assert.doesConformToSchema('compiler.json', config, compilerOptionsSchema);
-        this._contractsDir = passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR;
+        this._contractsDir = path.resolve(passedOpts.contractsDir || config.contractsDir || DEFAULT_CONTRACTS_DIR);
+        this._workspaceDir = path.resolve(passedOpts.workspaceDir || config.workspaceDir || this._contractsDir);
+        if (!this._contractsDir.includes(this._workspaceDir)) {
+            throw new Error(
+                `Contracts dir ${this._contractsDir} is outside of the workspace dir ${this._workspaceDir}`,
+            );
+        }
         this._solcVersionIfExists = passedOpts.solcVersion || config.solcVersion;
         this._compilerSettings = passedOpts.compilerSettings || config.compilerSettings || DEFAULT_COMPILER_SETTINGS;
         this._artifactsDir = passedOpts.artifactsDir || config.artifactsDir || DEFAULT_ARTIFACTS_DIR;
         this._specifiedContracts = passedOpts.contracts || config.contracts || ALL_CONTRACTS_IDENTIFIER;
-        this._nameResolver = new NameResolver(path.resolve(this._contractsDir));
+        this._useDockerisedSolc =
+            passedOpts.useDockerisedSolc || config.useDockerisedSolc || DEFAULT_USE_DOCKERISED_SOLC;
+        this._nameResolver = new NameResolver(this._contractsDir);
         const resolver = new FallthroughResolver();
         resolver.appendResolver(new URLResolver());
-        const packagePath = path.resolve('');
+        resolver.appendResolver(new NPMResolver(this._contractsDir, this._workspaceDir));
-        resolver.appendResolver(new NPMResolver(packagePath));
         resolver.appendResolver(new RelativeFSResolver(this._contractsDir));
         resolver.appendResolver(new FSResolver());
         resolver.appendResolver(this._nameResolver);
@@ -205,11 +217,12 @@ export class Compiler {
 
         // map contract paths to data about them for later verification and persistence
         const contractPathToData: ContractPathToData = {};
+        const spyResolver = new SpyResolver(this._resolver);
 
         for (const contractName of contractNames) {
-            const contractSource = this._resolver.resolve(contractName);
+            const contractSource = spyResolver.resolve(contractName);
             const sourceTreeHashHex = getSourceTreeHash(
-                this._resolver,
+                spyResolver,
                 path.join(this._contractsDir, contractSource.path),
             ).toString('hex');
             const contractData = {
@@ -242,6 +255,30 @@ export class Compiler {
             versionToInputs[solcVersion].contractsToCompile.push(contractSource.path);
         }
 
+        const allTouchedFiles = spyResolver.resolvedContractSources.map(
+            contractSource => `${contractSource.absolutePath}`,
+        );
+        const NODE_MODULES = 'node_modules';
+        const allTouchedDependencies = _.filter(allTouchedFiles, filePath => filePath.includes(NODE_MODULES));
+        const dependencyNameToPackagePath: { [dependencyName: string]: string } = {};
+        _.map(allTouchedDependencies, dependencyFilePath => {
+            const lastNodeModulesStart = dependencyFilePath.lastIndexOf(NODE_MODULES);
+            const lastNodeModulesEnd = lastNodeModulesStart + NODE_MODULES.length;
+            const importPath = dependencyFilePath.substr(lastNodeModulesEnd + 1);
+            let packageName;
+            let packageScopeIfExists;
+            let dependencyName;
+            if (_.startsWith(importPath, '@')) {
+                [packageScopeIfExists, packageName] = importPath.split('/');
+                dependencyName = `${packageScopeIfExists}/${packageName}`;
+            } else {
+                [packageName] = importPath.split('/');
+                dependencyName = `${packageName}`;
+            }
+            const dependencyPackagePath = path.join(dependencyFilePath.substr(0, lastNodeModulesEnd), dependencyName);
+            dependencyNameToPackagePath[dependencyName] = dependencyPackagePath;
+        });
+
         const compilerOutputs: StandardOutput[] = [];
 
         const solcVersions = _.keys(versionToInputs);
@@ -252,12 +289,31 @@ export class Compiler {
                     input.contractsToCompile
                 }) with Solidity v${solcVersion}...`,
             );
+            let compilerOutput;
+            let fullSolcVersion;
+            if (this._useDockerisedSolc) {
+                const dockerCommand = `docker run ethereum/solc:${solcVersion} --version`;
+                const versionCommandOutput = execSync(dockerCommand).toString();
+                const versionCommandOutputParts = versionCommandOutput.split(' ');
+                fullSolcVersion = versionCommandOutputParts[versionCommandOutputParts.length - 1].trim();
+                compilerOutput = compileDocker(
+                    this._resolver,
+                    this._contractsDir,
+                    this._workspaceDir,
+                    solcVersion,
+                    dependencyNameToPackagePath,
+                    input.standardInput,
+                );
+            } else {
+                fullSolcVersion = binPaths[solcVersion];
+                const solcInstance = await getSolcJSAsync(solcVersion);
+                compilerOutput = compileSolcJS(this._resolver, solcInstance, input.standardInput);
+            }
 
-            const { solcInstance, fullSolcVersion } = await getSolcAsync(solcVersion);
-            const compilerOutput = compile(this._resolver, solcInstance, input.standardInput);
             compilerOutputs.push(compilerOutput);
 
             for (const contractPath of input.contractsToCompile) {
+                // console.log('contractsPath', contractPath);
                 const contractName = contractPathToData[contractPath].contractName;
 
                 const compiledContract = compilerOutput.contracts[contractPath][contractName];

packages/sol-compiler/src/schemas/compiler_options_schema.ts

@@ -2,6 +2,7 @@ export const compilerOptionsSchema = {
     id: '/CompilerOptions',
     properties: {
         contractsDir: { type: 'string' },
+        workspaceDir: { type: 'string' },
         artifactsDir: { type: 'string' },
         solcVersion: { type: 'string', pattern: '^\\d+.\\d+.\\d+$' },
         compilerSettings: { type: 'object' },
@@ -19,6 +20,7 @@ export const compilerOptionsSchema = {
                 },
             ],
         },
+        useDockerisedSolc: { type: 'boolean' },
     },
     type: 'object',
     required: [],

packages/sol-compiler/src/utils/compiler.ts

@@ -1,6 +1,7 @@
 import { ContractSource, Resolver } from '@0x/sol-resolver';
 import { fetchAsync, logUtils } from '@0x/utils';
 import chalk from 'chalk';
+import { execSync } from 'child_process';
 import { ContractArtifact } from 'ethereum-types';
 import * as ethUtil from 'ethereumjs-util';
 import * as _ from 'lodash';
@@ -121,7 +122,7 @@ export function parseDependencies(contractSource: ContractSource): string[] {
  * @param solcInstance Instance of a solc compiler
  * @param standardInput Solidity standard JSON input
  */
-export function compile(
+export function compileSolcJS(
     resolver: Resolver,
     solcInstance: solc.SolcInstance,
     standardInput: solc.StandardInput,
@@ -137,6 +138,80 @@ export function compile(
     }
     return compiled;
 }
+
+/**
+ * Compiles the contracts and prints errors/warnings
+ * @param resolver Resolver
+ * @param contractsDir Contracts directory
+ * @param workspaceDir Workspace directory
+ * @param solcVersion Version of a solc compiler
+ * @param dependencyNameToPackagePath Mapping of dependency name to it's package path
+ * @param standardInput Solidity standard JSON input
+ */
+export function compileDocker(
+    resolver: Resolver,
+    contractsDir: string,
+    workspaceDir: string,
+    solcVersion: string,
+    dependencyNameToPackagePath: { [dependencyName: string]: string },
+    standardInput: solc.StandardInput,
+): solc.StandardOutput {
+    const standardInputDocker = _.cloneDeep(standardInput);
+    standardInputDocker.settings.remappings = _.map(
+        dependencyNameToPackagePath,
+        (dependencyPackagePath: string, dependencyName: string) => `${dependencyName}=${dependencyPackagePath}`,
+    );
+    standardInputDocker.sources = _.mapKeys(
+        standardInputDocker.sources,
+        (_source: solc.Source, sourcePath: string) => resolver.resolve(sourcePath).absolutePath,
+    );
+
+    const standardInputStrDocker = JSON.stringify(standardInputDocker, null, 2);
+    const dockerCommand =
+        `docker run -i -a stdin -a stdout -a stderr -v ${workspaceDir}:${workspaceDir} ethereum/solc:${solcVersion} ` +
+        `solc --standard-json --allow-paths ${workspaceDir}`;
+    const standardOutputStrDocker = execSync(dockerCommand, { input: standardInputStrDocker }).toString();
+    const compiledDocker: solc.StandardOutput = JSON.parse(standardOutputStrDocker);
+
+    if (!_.isUndefined(compiledDocker.errors)) {
+        printCompilationErrorsAndWarnings(compiledDocker.errors);
+    }
+
+    compiledDocker.sources = makeContractPathsRelative(
+        compiledDocker.sources,
+        contractsDir,
+        dependencyNameToPackagePath,
+    );
+    compiledDocker.contracts = makeContractPathsRelative(
+        compiledDocker.contracts,
+        contractsDir,
+        dependencyNameToPackagePath,
+    );
+    return compiledDocker;
+}
+
+function makeContractPathRelative(
+    absolutePath: string,
+    contractsDir: string,
+    dependencyNameToPackagePath: { [dependencyName: string]: string },
+): string {
+    let contractPath = absolutePath.replace(`${contractsDir}/`, '');
+    _.map(dependencyNameToPackagePath, (packagePath: string, dependencyName: string) => {
+        contractPath = contractPath.replace(packagePath, dependencyName);
+    });
+    return contractPath;
+}
+
+function makeContractPathsRelative(
+    absolutePathToSmth: { [absoluteContractPath: string]: any },
+    contractsDir: string,
+    dependencyNameToPackagePath: { [dependencyName: string]: string },
+): { [contractPath: string]: any } {
+    return _.mapKeys(absolutePathToSmth, (_val: any, absoluteContractPath: string) =>
+        makeContractPathRelative(absoluteContractPath, contractsDir, dependencyNameToPackagePath),
+    );
+}
+
 /**
  * Separates errors from warnings, formats the messages and prints them. Throws if there is any compilation error (not warning).
  * @param solcErrors The errors field of standard JSON output that contains errors and warnings.
@@ -267,13 +342,11 @@ function recursivelyGatherDependencySources(
 }
 
 /**
- * Gets the solidity compiler instance and full version name. If the compiler is already cached - gets it from FS,
+ * Gets the solidity compiler instance. If the compiler is already cached - gets it from FS,
  * otherwise - fetches it and caches it.
  * @param solcVersion The compiler version. e.g. 0.5.0
  */
-export async function getSolcAsync(
+export async function getSolcJSAsync(solcVersion: string): Promise<solc.SolcInstance> {
-    solcVersion: string,
-): Promise<{ solcInstance: solc.SolcInstance; fullSolcVersion: string }> {
     const fullSolcVersion = binPaths[solcVersion];
     if (_.isUndefined(fullSolcVersion)) {
         throw new Error(`${solcVersion} is not a known compiler version`);
@@ -297,7 +370,7 @@ export async function getSolcAsync(
         throw new Error('No compiler available');
     }
     const solcInstance = solc.setupMethods(requireFromString(solcjs, compilerBinFilename));
-    return { solcInstance, fullSolcVersion };
+    return solcInstance;
 }
 
 /**

packages/sol-resolver/src/resolvers/fs_resolver.ts

@@ -1,4 +1,5 @@
 import * as fs from 'fs';
+import * as path from 'path';
 
 import { ContractSource } from '../types';
 
@@ -9,7 +10,8 @@ export class FSResolver extends Resolver {
     public resolveIfExists(importPath: string): ContractSource | undefined {
         if (fs.existsSync(importPath) && fs.lstatSync(importPath).isFile()) {
             const fileContent = fs.readFileSync(importPath).toString();
-            return { source: fileContent, path: importPath, absolutePath: importPath };
+            const absolutePath = path.resolve(importPath);
+            return { source: fileContent, path: importPath, absolutePath } as any;
         }
         return undefined;
     }

packages/sol-resolver/src/resolvers/npm_resolver.ts

@@ -8,9 +8,11 @@ import { Resolver } from './resolver';
 
 export class NPMResolver extends Resolver {
     private readonly _packagePath: string;
-    constructor(packagePath: string) {
+    private readonly _workspacePath: string;
+    constructor(packagePath: string, workspacePath: string) {
         super();
         this._packagePath = packagePath;
+        this._workspacePath = workspacePath;
     }
     public resolveIfExists(importPath: string): ContractSource | undefined {
         if (!importPath.startsWith('/')) {
@@ -23,9 +25,11 @@ export class NPMResolver extends Resolver {
                 [packageName, ...other] = importPath.split('/');
             }
             const pathWithinPackage = path.join(...other);
-            let currentPath = this._packagePath;
+            for (
-            const ROOT_PATH = '/';
+                let currentPath = this._packagePath;
-            while (currentPath !== ROOT_PATH) {
+                currentPath.includes(this._workspacePath);
+                currentPath = path.dirname(currentPath)
+            ) {
                 const packagePath = _.isUndefined(packageScopeIfExists)
                     ? packageName
                     : path.join(packageScopeIfExists, packageName);
@@ -34,7 +38,6 @@ export class NPMResolver extends Resolver {
                     const fileContent = fs.readFileSync(lookupPath).toString();
                     return { source: fileContent, path: importPath, absolutePath: lookupPath };
                 }
-                currentPath = path.dirname(currentPath);
             }
         }
         return undefined;