crowetic/protocol
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Check dependencies when determining if should be recompiled

Browse Source
This commit is contained in:
Amir Bandeali
2018-02-17 15:07:05 -07:00
parent 097fc477a2
commit 7fb66bf71a
6 changed files with 163 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
packages/deployer/src/cli.ts
View File

@@ -6,6 +6,7 @@ import * as Web3 from 'web3';
import * as yargs from 'yargs';
import { commands } from './commands';
import { constants } from './utils/constants';
import { CliOptions, CompilerOptions, DeployerOptions } from './utils/types';
const DEFAULT_OPTIMIZER_ENABLED = false;
@@ -15,7 +16,6 @@ const DEFAULT_NETWORK_ID = 50;
const DEFAULT_JSONRPC_PORT = 8545;
const DEFAULT_GAS_PRICE = (10 ** 9 * 2).toString();
const DEFAULT_CONTRACTS_LIST = '*';
/**
* Compiles all contracts with options passed in through CLI.
* @param argv Instance of process.argv provided by yargs.
@@ -101,7 +101,8 @@ function getContractsSetFromList(contracts: string): Set<string> {
const specifiedContracts = new Set();
const contractsArray = contracts.split(',');
_.forEach(contractsArray, contractName => {
specifiedContracts.add(contractName);
const fileName = `${contractName}${constants.SOLIDITY_FILE_EXTENSION}`;
specifiedContracts.add(fileName);
});
return specifiedContracts;
}

206
packages/deployer/src/compiler.ts
View File

@@ -5,19 +5,25 @@ import solc = require('solc');
import * as Web3 from 'web3';
import { binPaths } from './solc/bin_paths';
import { constants } from './utils/constants';
import { fsWrapper } from './utils/fs_wrapper';
import {
CompilerOptions,
ContractArtifact,
ContractData,
ContractNetworkData,
ContractNetworks,
ContractSourceData,
ContractSources,
ContractSpecificSourceData,
ImportContents,
} from './utils/types';
import { utils } from './utils/utils';
const SOLIDITY_FILE_EXTENSION = '.sol';
const ALL_CONTRACTS_IDENTIFIER = '*';
const SOLIDITY_VERSION_REGEX = /(?:solidity\s\^?)([0-9]{1,2}[.][0-9]{1,2}[.][0-9]{1,2})/;
const SOLIDITY_FILE_EXTENSION_REGEX = /(.*\.sol)/;
const IMPORT_REGEX = /(import\s)/;
const DEPENDENCY_PATH_REGEX = /"([^"]+)"/;
export class Compiler {
private _contractsDir: string;
@@ -25,12 +31,13 @@ export class Compiler {
private _optimizerEnabled: number;
private _artifactsDir: string;
private _contractSourcesIfExists?: ContractSources;
private _solcErrors: Set<string>;
private _specifiedContracts: Set<string>;
private _solcErrors: Set<string> = new Set();
private _specifiedContracts: Set<string> = new Set();
private _contractSourceData: ContractSourceData = {};
/**
* Recursively retrieves Solidity source code from directory.
* @param dirPath Directory to search.
* @return Mapping of contract name to contract source.
* @return Mapping of contract fileName to contract source.
*/
private static async _getContractSourcesAsync(dirPath: string): Promise<ContractSources> {
let dirContents: string[] = [];
@@ -40,15 +47,16 @@ export class Compiler {
throw new Error(`No directory found at ${dirPath}`);
}
let sources: ContractSources = {};
for (const name of dirContents) {
const contentPath = `${dirPath}/${name}`;
if (path.extname(name) === SOLIDITY_FILE_EXTENSION) {
for (const fileName of dirContents) {
const contentPath = `${dirPath}/${fileName}`;
if (path.extname(fileName) === constants.SOLIDITY_FILE_EXTENSION) {
try {
const opts = {
encoding: 'utf8',
};
sources[name] = await fsWrapper.readFileAsync(contentPath, opts);
utils.consoleLog(`Reading ${name} source...`);
const source = await fsWrapper.readFileAsync(contentPath, opts);
sources[fileName] = source;
utils.consoleLog(`Reading ${fileName} source...`);
} catch (err) {
utils.consoleLog(`Could not find file at ${contentPath}`);
}
@@ -60,19 +68,58 @@ export class Compiler {
...nestedSources,
};
} catch (err) {
utils.consoleLog(`${contentPath} is not a directory or ${SOLIDITY_FILE_EXTENSION} file`);
utils.consoleLog(`${contentPath} is not a directory or ${constants.SOLIDITY_FILE_EXTENSION} file`);
}
}
}
return sources;
}
/**
* Gets contract dependendencies and keccak256 hash from source.
* @param source Source code of contract.
* @return Object with contract dependencies and keccak256 hash of source.
*/
private static _getContractSpecificSourceData(source: string): ContractSpecificSourceData {
const dependencies: string[] = [];
const sourceHash = `0x${ethUtil.sha3(source).toString('hex')}`;
const solc_version = Compiler._parseSolidityVersion(source);
const contractSpecificSourceData: ContractSpecificSourceData = {
dependencies,
solc_version,
keccak256: sourceHash,
};
const lines = source.split('\n');
_.forEach(lines, line => {
if (!_.isNull(line.match(IMPORT_REGEX))) {
const dependencyMatch = line.match(DEPENDENCY_PATH_REGEX);
if (!_.isNull(dependencyMatch)) {
const dependencyPath = dependencyMatch[1];
const fileName = path.basename(dependencyPath);
contractSpecificSourceData.dependencies.push(fileName);
}
}
});
return contractSpecificSourceData;
}
/**
* Finds dependencies, keccak256 hashes, and compile flag for each contract.
* @param sources Mapping of contract file name to source code.
* @return Dependencies, keccak256 hash, and compile flag for each contract.
*/
private static _getContractSourceData(sources: ContractSources): ContractSourceData {
const contractSourceData: ContractSourceData = {};
_.forIn(sources, (source, fileName) => {
contractSourceData[fileName] = Compiler._getContractSpecificSourceData(source);
});
return contractSourceData;
}
/**
* Searches Solidity source code for compiler version.
* @param source Source code of contract.
* @return Solc compiler version.
*/
private static _parseSolidityVersion(source: string): string {
const solcVersionMatch = source.match(/(?:solidity\s\^?)([0-9]{1,2}[.][0-9]{1,2}[.][0-9]{1,2})/);
const solcVersionMatch = source.match(SOLIDITY_VERSION_REGEX);
if (_.isNull(solcVersionMatch)) {
throw new Error('Could not find Solidity version in source');
}
@@ -88,7 +135,7 @@ export class Compiler {
* @return The error message with directories truncated from the contract path.
*/
private static _getNormalizedErrMsg(errMsg: string): string {
const errPathMatch = errMsg.match(/(.*\.sol)/);
const errPathMatch = errMsg.match(SOLIDITY_FILE_EXTENSION_REGEX);
if (_.isNull(errPathMatch)) {
throw new Error('Could not find a path in error message');
}
@@ -107,7 +154,6 @@ export class Compiler {
this._networkId = opts.networkId;
this._optimizerEnabled = opts.optimizerEnabled;
this._artifactsDir = opts.artifactsDir;
this._solcErrors = new Set();
this._specifiedContracts = opts.specifiedContracts;
}
/**
@@ -116,11 +162,12 @@ export class Compiler {
public async compileAllAsync(): Promise<void> {
await this._createArtifactsDirIfDoesNotExistAsync();
this._contractSourcesIfExists = await Compiler._getContractSourcesAsync(this._contractsDir);
const contractBaseNames = _.keys(this._contractSourcesIfExists);
const compiledContractPromises = _.map(contractBaseNames, async (contractBaseName: string): Promise<void> => {
return this._compileContractAsync(contractBaseName);
});
await Promise.all(compiledContractPromises);
this._contractSourceData = Compiler._getContractSourceData(this._contractSourcesIfExists);
const fileNames = this._specifiedContracts.has(ALL_CONTRACTS_IDENTIFIER)
? _.keys(this._contractSourcesIfExists)
: Array.from(this._specifiedContracts.values());
await Promise.all(_.map(fileNames, async fileName => this._setCompileActionAsync(fileName)));
await Promise.all(_.map(fileNames, async fileName => this._compileContractAsync(fileName)));
this._solcErrors.forEach(errMsg => {
utils.consoleLog(errMsg);
@@ -128,55 +175,27 @@ export class Compiler {
}
/**
* Compiles contract and saves artifact to artifactsDir.
* @param contractBaseName Name of contract with '.sol' extension.
* @param fileName Name of contract with '.sol' extension.
*/
private async _compileContractAsync(contractBaseName: string): Promise<void> {
private async _compileContractAsync(fileName: string): Promise<void> {
if (_.isUndefined(this._contractSourcesIfExists)) {
throw new Error('Contract sources not yet initialized');
}
const source = this._contractSourcesIfExists[contractBaseName];
const contractName = path.basename(contractBaseName, SOLIDITY_FILE_EXTENSION);
const currentArtifactPath = `${this._artifactsDir}/${contractName}.json`;
const sourceHash = `0x${ethUtil.sha3(source).toString('hex')}`;
const isContractSpecified =
this._specifiedContracts.has(ALL_CONTRACTS_IDENTIFIER) || this._specifiedContracts.has(contractName);
let currentArtifactString: string;
let currentArtifact: ContractArtifact;
let oldNetworks: ContractNetworks;
let shouldCompile: boolean;
try {
const opts = {
encoding: 'utf8',
};
currentArtifactString = await fsWrapper.readFileAsync(currentArtifactPath, opts);
currentArtifact = JSON.parse(currentArtifactString);
oldNetworks = currentArtifact.networks;
const oldNetwork: ContractData = oldNetworks[this._networkId];
shouldCompile =
(_.isUndefined(oldNetwork) ||
oldNetwork.keccak256 !== sourceHash ||
oldNetwork.optimizer_enabled !== this._optimizerEnabled) &&
isContractSpecified;
} catch (err) {
shouldCompile = isContractSpecified;
}
if (!shouldCompile) {
const contractSpecificSourceData = this._contractSourceData[fileName];
if (!contractSpecificSourceData.shouldCompile) {
return;
}
const source = this._contractSourcesIfExists[fileName];
const input = {
[contractBaseName]: source,
[fileName]: source,
};
const solcVersion = Compiler._parseSolidityVersion(source);
const fullSolcVersion = binPaths[solcVersion];
const fullSolcVersion = binPaths[contractSpecificSourceData.solc_version];
const solcBinPath = `./solc/solc_bin/${fullSolcVersion}`;
const solcBin = require(solcBinPath);
const solcInstance = solc.setupMethods(solcBin);
utils.consoleLog(`Compiling ${contractBaseName}...`);
utils.consoleLog(`Compiling ${fileName}...`);
const sourcesToCompile = {
sources: input,
};
@@ -187,19 +206,20 @@ export class Compiler {
);
if (!_.isUndefined(compiled.errors)) {
_.each(compiled.errors, errMsg => {
_.forEach(compiled.errors, errMsg => {
const normalizedErrMsg = Compiler._getNormalizedErrMsg(errMsg);
this._solcErrors.add(normalizedErrMsg);
});
}
const contractIdentifier = `${contractBaseName}:${contractName}`;
const contractName = path.basename(fileName, constants.SOLIDITY_FILE_EXTENSION);
const contractIdentifier = `${fileName}:${contractName}`;
const abi: Web3.ContractAbi = JSON.parse(compiled.contracts[contractIdentifier].interface);
const unlinked_binary = `0x${compiled.contracts[contractIdentifier].bytecode}`;
const updated_at = Date.now();
const contractData: ContractData = {
solc_version: solcVersion,
keccak256: sourceHash,
const contractNetworkData: ContractNetworkData = {
solc_version: contractSpecificSourceData.solc_version,
keccak256: contractSpecificSourceData.keccak256,
optimizer_enabled: this._optimizerEnabled,
abi,
unlinked_binary,
@@ -207,26 +227,55 @@ export class Compiler {
};
let newArtifact: ContractArtifact;
if (!_.isUndefined(currentArtifactString)) {
const currentArtifact = (await this._getContractArtifactOrReturnAsync(fileName)) as ContractArtifact;
if (!_.isUndefined(currentArtifact)) {
newArtifact = {
...currentArtifact,
networks: {
...oldNetworks,
[this._networkId]: contractData,
...currentArtifact.networks,
[this._networkId]: contractNetworkData,
},
};
} else {
newArtifact = {
contract_name: contractName,
networks: {
[this._networkId]: contractData,
[this._networkId]: contractNetworkData,
},
};
}
const artifactString = utils.stringifyWithFormatting(newArtifact);
const currentArtifactPath = `${this._artifactsDir}/${contractName}.json`;
await fsWrapper.writeFileAsync(currentArtifactPath, artifactString);
utils.consoleLog(`${contractBaseName} artifact saved!`);
utils.consoleLog(`${fileName} artifact saved!`);
}
/**
* Recursively sets the compile action for a specific contract and dependencies.
* @param fileName Name of contracts file.
*/
private async _setCompileActionAsync(fileName: string): Promise<void> {
const contractSpecificSourceData = this._contractSourceData[fileName];
if (_.isUndefined(contractSpecificSourceData)) {
throw new Error(`Contract data for ${fileName} not yet set`);
}
if (_.isUndefined(contractSpecificSourceData.shouldCompile)) {
const contractArtifact = (await this._getContractArtifactOrReturnAsync(fileName)) as ContractArtifact;
if (_.isUndefined(contractArtifact)) {
contractSpecificSourceData.shouldCompile = true;
} else {
const contractNetworkData = contractArtifact.networks[this._networkId];
contractSpecificSourceData.shouldCompile =
contractNetworkData.keccak256 !== contractSpecificSourceData.keccak256 ||
this._optimizerEnabled !== contractNetworkData.optimizer_enabled ||
contractNetworkData.solc_version !== contractSpecificSourceData.solc_version;
}
}
_.forEach(contractSpecificSourceData.dependencies, async dependency => {
await this._setCompileActionAsync(dependency);
contractSpecificSourceData.shouldCompile =
contractSpecificSourceData.shouldCompile || this._contractSourceData[dependency].shouldCompile;
});
}
/**
* Callback to resolve dependencies with `solc.compile`.
@@ -238,8 +287,8 @@ export class Compiler {
if (_.isUndefined(this._contractSourcesIfExists)) {
throw new Error('Contract sources not yet initialized');
}
const contractBaseName = path.basename(importPath);
const source = this._contractSourcesIfExists[contractBaseName];
const fileName = path.basename(importPath);
const source = this._contractSourcesIfExists[fileName];
const importContents: ImportContents = {
contents: source,
};
@@ -254,4 +303,25 @@ export class Compiler {
await fsWrapper.mkdirAsync(this._artifactsDir);
}
}
/**
* Gets contract data on network or returns if an artifact does not exist.
* @param fileName Name of contracts file.
* @return Contract data on network or undefined.
*/
private async _getContractArtifactOrReturnAsync(fileName: string): Promise<ContractArtifact | void> {
let contractArtifact;
const contractName = path.basename(fileName, constants.SOLIDITY_FILE_EXTENSION);
const currentArtifactPath = `${this._artifactsDir}/${contractName}.json`;
try {
const opts = {
encoding: 'utf8',
};
const contractArtifactString = await fsWrapper.readFileAsync(currentArtifactPath, opts);
contractArtifact = JSON.parse(contractArtifactString);
return contractArtifact;
} catch (err) {
utils.consoleLog(`Artifact for ${fileName} does not exist`);
return contractArtifact;
}
}
}

8
packages/deployer/src/deployer.ts
View File

@@ -6,7 +6,7 @@ import * as Web3 from 'web3';
import { Contract } from './utils/contract';
import { encoder } from './utils/encoder';
import { fsWrapper } from './utils/fs_wrapper';
import { ContractArtifact, ContractData, DeployerOptions } from './utils/types';
import { ContractArtifact, ContractNetworkData, DeployerOptions } from './utils/types';
import { utils } from './utils/utils';
// Gas added to gas estimate to make sure there is sufficient gas for deployment.
@@ -36,7 +36,7 @@ export class Deployer {
*/
public async deployAsync(contractName: string, args: any[] = []): Promise<Web3.ContractInstance> {
const contractArtifact: ContractArtifact = this._loadContractArtifactIfExists(contractName);
const contractData: ContractData = this._getContractDataFromArtifactIfExists(contractArtifact);
const contractData: ContractNetworkData = this._getContractDataFromArtifactIfExists(contractArtifact);
const data = contractData.unlinked_binary;
const from = await this._getFromAddressAsync();
const gas = await this._getAllowableGasEstimateAsync(data);
@@ -101,7 +101,7 @@ export class Deployer {
args: any[],
): Promise<void> {
const contractArtifact: ContractArtifact = this._loadContractArtifactIfExists(contractName);
const contractData: ContractData = this._getContractDataFromArtifactIfExists(contractArtifact);
const contractData: ContractNetworkData = this._getContractDataFromArtifactIfExists(contractArtifact);
const abi = contractData.abi;
const encodedConstructorArgs = encoder.encodeConstructorArgsFromAbi(args, abi);
const newContractData = {
@@ -139,7 +139,7 @@ export class Deployer {
* @param contractArtifact The contract artifact.
* @return Network specific contract data.
*/
private _getContractDataFromArtifactIfExists(contractArtifact: ContractArtifact): ContractData {
private _getContractDataFromArtifactIfExists(contractArtifact: ContractArtifact): ContractNetworkData {
const contractData = contractArtifact.networks[this._networkId];
if (_.isUndefined(contractData)) {
throw new Error(`Data not found in artifact for contract: ${contractArtifact.contract_name}`);

1
packages/deployer/src/utils/constants.ts
View File

@@ -1,3 +1,4 @@
export const constants = {
NULL_BYTES: '0x',
SOLIDITY_FILE_EXTENSION: '.sol',
};

15
packages/deployer/src/utils/types.ts
View File

@@ -15,10 +15,10 @@ export interface ContractArtifact {
}
export interface ContractNetworks {
[key: number]: ContractData;
[key: number]: ContractNetworkData;
}
export interface ContractData {
export interface ContractNetworkData {
solc_version: string;
optimizer_enabled: number;
keccak256: string;
@@ -64,6 +64,17 @@ export interface ContractSources {
[key: string]: string;
}
export interface ContractSourceData {
[key: string]: ContractSpecificSourceData;
}
export interface ContractSpecificSourceData {
dependencies: string[];
solc_version: string;
keccak256: string;
shouldCompile?: boolean;
}
export interface ImportContents {
contents: string;
}

8
packages/deployer/test/deploy_test.ts
View File

@@ -4,7 +4,7 @@ import 'mocha';
import { Compiler } from '../src/compiler';
import { Deployer } from '../src/deployer';
import { fsWrapper } from '../src/utils/fs_wrapper';
import { CompilerOptions, ContractArtifact, ContractData, DoneCallback } from '../src/utils/types';
import { CompilerOptions, ContractArtifact, ContractNetworkData, DoneCallback } from '../src/utils/types';
import { constructor_args, exchange_binary } from './fixtures/exchange_bin';
import { constants } from './util/constants';
@@ -51,7 +51,7 @@ describe('#Compiler', () => {
};
const exchangeArtifactString = await fsWrapper.readFileAsync(exchangeArtifactPath, opts);
const exchangeArtifact: ContractArtifact = JSON.parse(exchangeArtifactString);
const exchangeContractData: ContractData = exchangeArtifact.networks[constants.networkId];
const exchangeContractData: ContractNetworkData = exchangeArtifact.networks[constants.networkId];
// The last 43 bytes of the binaries are metadata which may not be equivalent
const unlinkedBinaryWithoutMetadata = exchangeContractData.unlinked_binary.slice(0, -86);
const exchangeBinaryWithoutMetadata = exchange_binary.slice(0, -86);
@@ -68,7 +68,7 @@ describe('#Deployer', () => {
};
const exchangeArtifactString = await fsWrapper.readFileAsync(exchangeArtifactPath, opts);
const exchangeArtifact: ContractArtifact = JSON.parse(exchangeArtifactString);
const exchangeContractData: ContractData = exchangeArtifact.networks[constants.networkId];
const exchangeContractData: ContractNetworkData = exchangeArtifact.networks[constants.networkId];
const exchangeAddress = exchangeContractInstance.address;
expect(exchangeAddress).to.not.equal(undefined);
expect(exchangeContractData.address).to.equal(undefined);
@@ -84,7 +84,7 @@ describe('#Deployer', () => {
};
const exchangeArtifactString = await fsWrapper.readFileAsync(exchangeArtifactPath, opts);
const exchangeArtifact: ContractArtifact = JSON.parse(exchangeArtifactString);
const exchangeContractData: ContractData = exchangeArtifact.networks[constants.networkId];
const exchangeContractData: ContractNetworkData = exchangeArtifact.networks[constants.networkId];
const exchangeAddress = exchangeContractInstance.address;
expect(exchangeAddress).to.be.equal(exchangeContractData.address);
expect(constructor_args).to.be.equal(exchangeContractData.constructor_args);