chore: improve basic benchmark

This commit is contained in:
Roman Hotsiy 2018-01-29 13:09:38 +02:00
parent e9f23f76ae
commit a35541ff19
No known key found for this signature in database
GPG Key ID: 5CB7B3ACABA57CB0
10 changed files with 311 additions and 108 deletions

View File

@ -71,6 +71,8 @@ There are some other scripts available in the `scripts` section of the `package.
## Project Structure ## Project Structure
- **`benchmark`**: contains basic perf benchmark. Not fully ready yet
- **`demo`**: contains project demo with demo specs and HMR playground used in development - **`demo`**: contains project demo with demo specs and HMR playground used in development
- `demo/playground`: HMR Playground used in development - `demo/playground`: HMR Playground used in development
@ -79,7 +81,6 @@ There are some other scripts available in the `scripts` section of the `package.
- **`e2e`**: contains e2e tests. The e2e tests are written and run with [Cypress](https://www.cypress.io/). - **`e2e`**: contains e2e tests. The e2e tests are written and run with [Cypress](https://www.cypress.io/).
- **`perf`**: contains basic perf benchmark. Not ready yet
- **`src`**: contains the source code. The codebase is written in Typescript. CSS styles are managed with [Styled components](https://www.styled-components.com/). State is managed by [MobX](https://github.com/mobxjs/mobx) - **`src`**: contains the source code. The codebase is written in Typescript. CSS styles are managed with [Styled components](https://www.styled-components.com/). State is managed by [MobX](https://github.com/mobxjs/mobx)

2
.gitignore vendored
View File

@ -26,6 +26,8 @@ e2e/.build/
cypress/ cypress/
bundles bundles
/benchmark/revisions
/coverage /coverage
.ghpages-tmp .ghpages-tmp
stats.json stats.json

121
benchmark/benchmark.js Normal file
View File

@ -0,0 +1,121 @@
const beautifyBenchmark = require('beautify-benchmark');
const sh = require('shelljs');
const fs = require('fs');
const pathJoin = require('path').join;
const spawn = require('child_process').spawn;
const puppeteer = require('puppeteer');
const args = process.argv.slice(2);
args[0] = args[0] || 'HEAD';
args[1] = args[1] || 'local';
console.log('Benchmarking revisions: ' + args.join(', '));
const localDistDir = './benchmark/revisions/local/bundles';
sh.rm('-rf', localDistDir);
console.log(`Building local dist: ${localDistDir}`);
sh.mkdir('-p', localDistDir);
exec(`yarn bundle:lib --output-path ${localDistDir}`);
const revisions = [];
for (const arg of args) {
revisions.push({ name: arg, path: buildRevisionDist(arg) });
}
const configFile = `
export const revisions = [ ${revisions.map(rev => JSON.stringify(rev)).join(', ')} ];
`;
const configDir = './benchmark/revisions/config.js';
console.log(`Writing config "${configDir}"`);
fs.writeFileSync(configDir, configFile);
console.log('Starging benchmark server');
const proc = spawn('npm', ['run', 'start:benchmark']);
proc.stdout.on('data', data => {
if (data.toString().indexOf('Project is running at') > -1) {
console.log('Server started');
startBenchmark();
}
});
proc.stderr.on('data', data => {
console.error(data.toString());
});
proc.on('close', code => {
console.log(`Benchmark server stopped with code ${code}`);
});
async function runPuppeteer() {
return await puppeteer
.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] })
.then(async browser => {
const page = await browser.newPage();
let resolve;
const prom = new Promise(_resolve => {
resolve = _resolve;
});
page.on('console', obj => {
if (!obj) return;
if (obj.done) {
beautifyBenchmark.log();
// resolve(obj);
} else if (obj.cycle) {
beautifyBenchmark.add(obj.cycle);
} else if (obj.allDone) {
resolve();
} else {
console.log(obj);
}
});
await page.goto('http://localhost:9090', {
waitUntil: 'networkidle',
});
const res = await prom;
await browser.close();
return res;
});
}
async function startBenchmark() {
console.log('Starting benchmarks');
await runPuppeteer();
console.log('Killing benchmark server');
proc.kill('SIGINT');
}
function exec(command) {
const { code, stdout, stderr } = sh.exec(command, { silent: true });
if (code !== 0) {
console.error(stdout);
console.error(stderr);
sh.exit(code);
}
return stdout.trim();
}
function buildRevisionDist(revision) {
if (revision === 'local') {
return localDistDir;
}
const hash = exec(`git log -1 --format=%h "${revision}"`);
const buildDir = './benchmark/revisions/' + hash;
const distDir = buildDir + '/bundles';
if (sh.test('-d', distDir)) {
console.log(`Using prebuilt "${revision}"(${hash}) revision: ${buildDir}`);
return distDir;
}
console.log(`Building "${revision}"(${hash}) revision: ${buildDir}`);
sh.mkdir('-p', buildDir);
exec(`git archive "${hash}" | tar -xC "${buildDir}"`);
const pwd = sh.pwd();
sh.cd(buildDir);
exec('yarn remove cypress puppeteer && yarn && yarn bundle:lib');
sh.cd(pwd);
return distDir;
}

28
benchmark/index.html Normal file
View File

@ -0,0 +1,28 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8" />
<title>ReDoc</title>
<style>
body {
margin: 0;
padding: 0;
}
redoc {
display: block;
}
</style>
<link href="https://fonts.googleapis.com/css?family=Montserrat:300,400,700|Roboto:300,400,700" rel="stylesheet">
</head>
<body>
<redoc id="example"></redoc>
<!-- <redoc spec-url="./openapi.yaml"></redoc> -->
<script src="https://unpkg.com/lodash@4.17.4/lodash.js"></script>
<script src="https://unpkg.com/benchmark@2.1.4/benchmark.js"></script>
<!-- <script src="../bundles/redoc.standalone.js"></script> -->
</body>
</html>

126
benchmark/index.tsx Normal file
View File

@ -0,0 +1,126 @@
import * as React from 'react';
import { render, unmountComponentAtNode } from 'react-dom';
import { Redoc, RedocProps } from '../src/components';
import { loadAndBundleSpec } from '../src/utils';
import { revisions } from './revisions/config';
import { extras } from 'mobx';
declare var Benchmark;
extras.isolateGlobalState();
const node = document.getElementById('example');
const renderRoot = (Component: typeof Redoc, props: RedocProps) =>
render(<Component {...props} />, node!);
async function importRedocs() {
return Promise.all(
revisions.map(rev => {
return import('./' + rev.path.substring(12) + '/redoc.lib.js');
}),
);
}
function startFullTime(redocs, resolvedSpec) {
return new Promise(async resolve => {
const suite = new Benchmark.Suite('Full time', {
maxTime: 20,
initCount: 2,
onStart(event) {
console.log(' ⏱️ ' + event.currentTarget.name);
},
onCycle(event) {
console.log({ cycle: event.target });
},
onComplete() {
console.log({ done: true });
setTimeout(() => resolve(), 10);
},
});
revisions.forEach((rev, idx) => {
const redoc = redocs[idx];
suite.add(rev.name, () => {
const store = new redoc.AppStore(resolvedSpec, 'openapi.yaml');
renderRoot(redoc.Redoc, { store });
unmountComponentAtNode(node!);
});
});
suite.run({ async: true });
});
}
function startInitStore(redocs, resolvedSpec) {
return new Promise(async resolve => {
const suite = new Benchmark.Suite('Create Store Time', {
maxTime: 20,
initCount: 2,
onStart(event) {
console.log(' ⏱️ ' + event.currentTarget.name);
},
onCycle(event) {
console.log({ cycle: event.target });
},
onComplete() {
console.log({ done: true });
setTimeout(() => resolve(), 10);
},
});
revisions.forEach((rev, idx) => {
const redoc = redocs[idx];
suite.add(rev.name, () => {
const store = new redoc.AppStore(resolvedSpec, 'openapi.yaml');
store.dispose();
});
});
suite.run({ async: true });
});
}
function startRenderTime(redocs, resolvedSpec) {
return new Promise(async resolve => {
const suite = new Benchmark.Suite('Render time', {
maxTime: 20,
initCount: 2,
onStart(event) {
console.log(' ⏱️ ' + event.currentTarget.name);
},
onCycle(event) {
console.log({ cycle: event.target });
unmountComponentAtNode(node!);
},
onComplete() {
console.log({ done: true });
setTimeout(() => resolve(), 10);
},
});
revisions.forEach((rev, idx) => {
const redoc = redocs[idx];
const store = new redoc.AppStore(resolvedSpec, 'openapi.yaml');
suite.add(rev.name, () => {
renderRoot(redoc.Redoc, { store });
});
});
suite.run({ async: true });
});
}
async function runBenchmarks() {
const redocs: any[] = await importRedocs();
const resolvedSpec = await loadAndBundleSpec('openapi.yaml');
await startInitStore(redocs, resolvedSpec);
await startRenderTime(redocs, resolvedSpec);
await startFullTime(redocs, resolvedSpec);
console.log({ allDone: true });
}
runBenchmarks();

View File

@ -5,7 +5,7 @@
"main": "bundles/redoc.lib.js", "main": "bundles/redoc.lib.js",
"scripts": { "scripts": {
"start": "webpack-dev-server --hot", "start": "webpack-dev-server --hot",
"start:perf": "webpack-dev-server --env.prod --env.perf", "start:benchmark": "webpack-dev-server --env.prod --env.perf",
"start:prod": "webpack-dev-server --env.prod", "start:prod": "webpack-dev-server --env.prod",
"test": "npm run lint && npm run unit && npm run e2e", "test": "npm run lint && npm run unit && npm run e2e",
"unit": "jest", "unit": "jest",
@ -20,7 +20,8 @@
"stats": "webpack -p --env.lib --env.standalone --env.prod --json --profile > stats.json", "stats": "webpack -p --env.lib --env.standalone --env.prod --json --profile > stats.json",
"prettier": "prettier --write \"src/**/*.{ts,tsx}\"", "prettier": "prettier --write \"src/**/*.{ts,tsx}\"",
"changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 1", "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 1",
"lint": "tslint --project tsconfig.json" "lint": "tslint --project tsconfig.json",
"benchmark": "node ./benchmark/benchmark.js"
}, },
"author": "", "author": "",
"license": "MIT", "license": "MIT",
@ -43,6 +44,7 @@
"@types/webpack": "^3.0.5", "@types/webpack": "^3.0.5",
"@types/webpack-env": "^1.13.0", "@types/webpack-env": "^1.13.0",
"awesome-typescript-loader": "^3.2.2", "awesome-typescript-loader": "^3.2.2",
"beautify-benchmark": "^0.2.4",
"conventional-changelog-cli": "^1.3.5", "conventional-changelog-cli": "^1.3.5",
"core-js": "^2.5.1", "core-js": "^2.5.1",
"css-loader": "^0.28.7", "css-loader": "^0.28.7",
@ -61,6 +63,7 @@
"react-dev-utils": "^4.1.0", "react-dev-utils": "^4.1.0",
"react-dom": "^16.2.0", "react-dom": "^16.2.0",
"rimraf": "^2.6.2", "rimraf": "^2.6.2",
"shelljs": "^0.8.1",
"source-map-loader": "^0.2.1", "source-map-loader": "^0.2.1",
"style-loader": "^0.18.2", "style-loader": "^0.18.2",
"ts-jest": "^21.0.1", "ts-jest": "^21.0.1",
@ -105,6 +108,10 @@
"^.+\\.tsx?$": "<rootDir>/node_modules/ts-jest/preprocessor.js" "^.+\\.tsx?$": "<rootDir>/node_modules/ts-jest/preprocessor.js"
}, },
"setupTestFrameworkScriptFile": "<rootDir>/src/setupTests.ts", "setupTestFrameworkScriptFile": "<rootDir>/src/setupTests.ts",
"testPathIgnorePatterns": [
"/node_modules/",
"/benchmark/"
],
"testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$", "testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$",
"moduleFileExtensions": [ "moduleFileExtensions": [
"ts", "ts",

View File

@ -1,27 +0,0 @@
import * as React from 'react';
import { render } from 'react-dom';
import { Redoc, RedocProps } from '../src/components';
import { AppStore } from '../src/services/AppStore';
import { loadAndBundleSpec } from '../src/utils';
const renderRoot = (Component: typeof Redoc, props: RedocProps) =>
render(<Component {...props} />, document.getElementById('example'));
async function start() {
const resolvedSpec = await loadAndBundleSpec('big-openapi.json');
const t0 = performance.now();
const store = new AppStore(resolvedSpec, 'big-openapi.json');
var t1 = performance.now();
renderRoot(Redoc, { store });
var t2 = performance.now();
console.log({
timings: true,
'Total Time': t2 - t0,
'Store Init Time': t1 - t0,
'Render Time': t2 - t1,
});
}
start();

View File

@ -1,73 +0,0 @@
const puppeteer = require('puppeteer');
const crypto = require('crypto');
async function run() {
return await puppeteer
.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] })
.then(async browser => {
const page = await browser.newPage();
let resolve;
const prom = new Promise(_resolve => {
resolve = _resolve;
});
page.on('console', obj => {
if (obj && obj.timings) {
resolve(obj);
}
});
await page.goto('http://localhost:9090', {
waitUntil: 'networkidle',
});
const res = await prom;
await browser.close();
return res;
});
}
function clearLine() {
process.stdout.clearLine();
process.stdout.cursorTo(0);
}
const metrics = ['Total Time', 'Store Init Time', 'Render Time'];
const forEachMetric = fn => metrics.forEach(metric => fn(metric));
async function benchmark() {
const N = 5;
let sum = {};
let max = {};
let min = {};
forEachMetric(metric => {
sum[metric] = 0;
max[metric] = 0;
min[metric] = Number.MAX_SAFE_INTEGER;
});
for (let i = 0; i < N; i++) {
const res = await run();
forEachMetric(metric => {
if (res[metric] > max[metric]) max[metric] = res[metric];
if (res[metric] < min[metric]) min[metric] = res[metric];
sum[metric] += res[metric];
});
clearLine();
process.stdout.write(`Running: ${i + 1} of ${N}`);
}
clearLine();
const average = {};
forEachMetric(metric => {
average[metric] = sum[metric] / N;
});
console.log('Completed ', N, 'runs');
console.log('=======================');
forEachMetric(metric => {
console.log(`Average ${metric}: `, average[metric]);
console.log(`Minimum ${metric}: `, min[metric]);
console.log(`Maximum ${metric}: `, max[metric]);
console.log();
});
}
benchmark();

View File

@ -24,7 +24,7 @@ export default env => {
} else { } else {
// playground or performance test // playground or performance test
entry = env.perf entry = env.perf
? ['./perf/index.tsx'] // perf test ? ['./benchmark/index.tsx'] // perf test
: [ : [
// playground // playground
'./src/polyfills.ts', './src/polyfills.ts',
@ -73,7 +73,7 @@ export default env => {
{ {
loader: 'awesome-typescript-loader', loader: 'awesome-typescript-loader',
options: { options: {
module: 'es2015', module: env.perf ? 'esnext' : 'es2015',
}, },
}, },
], ],
@ -132,7 +132,7 @@ export default env => {
} else { } else {
config.plugins!.push( config.plugins!.push(
new HtmlWebpackPlugin({ new HtmlWebpackPlugin({
template: './demo/playground/index.html', template: env.perf ? './benchmark/index.html' : './demo/playground/index.html',
}), }),
); );
} }

View File

@ -685,6 +685,10 @@ bcrypt-pbkdf@^1.0.0:
dependencies: dependencies:
tweetnacl "^0.14.3" tweetnacl "^0.14.3"
beautify-benchmark@^0.2.4:
version "0.2.4"
resolved "https://registry.yarnpkg.com/beautify-benchmark/-/beautify-benchmark-0.2.4.tgz#3151def14c1a2e0d07ff2e476861c7ed0e1ae39b"
big.js@^3.1.3: big.js@^3.1.3:
version "3.2.0" version "3.2.0"
resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e" resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e"
@ -2937,7 +2941,7 @@ glob2base@^0.0.12:
dependencies: dependencies:
find-index "^0.1.1" find-index "^0.1.1"
glob@7.1.2, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2: glob@7.1.2, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2:
version "7.1.2" version "7.1.2"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
dependencies: dependencies:
@ -5930,6 +5934,12 @@ readdirp@^2.0.0:
readable-stream "^2.0.2" readable-stream "^2.0.2"
set-immediate-shim "^1.0.1" set-immediate-shim "^1.0.1"
rechoir@^0.6.2:
version "0.6.2"
resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384"
dependencies:
resolve "^1.1.6"
recursive-readdir@2.2.1: recursive-readdir@2.2.1:
version "2.2.1" version "2.2.1"
resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.1.tgz#90ef231d0778c5ce093c9a48d74e5c5422d13a99" resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.1.tgz#90ef231d0778c5ce093c9a48d74e5c5422d13a99"
@ -6159,7 +6169,7 @@ resolve@1.1.7:
version "1.1.7" version "1.1.7"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b"
resolve@^1.1.7, resolve@^1.3.2: resolve@^1.1.6, resolve@^1.1.7, resolve@^1.3.2:
version "1.5.0" version "1.5.0"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36"
dependencies: dependencies:
@ -6390,6 +6400,14 @@ shell-quote@1.6.1, shell-quote@^1.6.1:
array-reduce "~0.0.0" array-reduce "~0.0.0"
jsonify "~0.0.0" jsonify "~0.0.0"
shelljs@^0.8.1:
version "0.8.1"
resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.1.tgz#729e038c413a2254c4078b95ed46e0397154a9f1"
dependencies:
glob "^7.0.0"
interpret "^1.0.0"
rechoir "^0.6.2"
shellwords@^0.1.1: shellwords@^0.1.1:
version "0.1.1" version "0.1.1"
resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b" resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b"