diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 33543e7c..b6d4c738 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -71,6 +71,8 @@ There are some other scripts available in the `scripts` section of the `package. ## Project Structure +- **`benchmark`**: contains basic perf benchmark. Not fully ready yet + - **`demo`**: contains project demo with demo specs and HMR playground used in development - `demo/playground`: HMR Playground used in development @@ -79,7 +81,6 @@ There are some other scripts available in the `scripts` section of the `package. - **`e2e`**: contains e2e tests. The e2e tests are written and run with [Cypress](https://www.cypress.io/). -- **`perf`**: contains basic perf benchmark. Not ready yet - **`src`**: contains the source code. The codebase is written in Typescript. CSS styles are managed with [Styled components](https://www.styled-components.com/). State is managed by [MobX](https://github.com/mobxjs/mobx) diff --git a/.gitignore b/.gitignore index 9c9d4fd1..d35477d0 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,8 @@ e2e/.build/ cypress/ bundles +/benchmark/revisions + /coverage .ghpages-tmp stats.json diff --git a/benchmark/benchmark.js b/benchmark/benchmark.js new file mode 100644 index 00000000..b817d9a3 --- /dev/null +++ b/benchmark/benchmark.js @@ -0,0 +1,121 @@ +const beautifyBenchmark = require('beautify-benchmark'); +const sh = require('shelljs'); +const fs = require('fs'); +const pathJoin = require('path').join; +const spawn = require('child_process').spawn; +const puppeteer = require('puppeteer'); + +const args = process.argv.slice(2); +args[0] = args[0] || 'HEAD'; +args[1] = args[1] || 'local'; + +console.log('Benchmarking revisions: ' + args.join(', ')); + +const localDistDir = './benchmark/revisions/local/bundles'; +sh.rm('-rf', localDistDir); +console.log(`Building local dist: ${localDistDir}`); +sh.mkdir('-p', localDistDir); +exec(`yarn bundle:lib --output-path ${localDistDir}`); + +const revisions = []; +for (const arg of args) { + revisions.push({ name: arg, path: buildRevisionDist(arg) }); +} + +const configFile = ` + export const revisions = [ ${revisions.map(rev => JSON.stringify(rev)).join(', ')} ]; +`; + +const configDir = './benchmark/revisions/config.js'; +console.log(`Writing config "${configDir}"`); +fs.writeFileSync(configDir, configFile); + +console.log('Starging benchmark server'); +const proc = spawn('npm', ['run', 'start:benchmark']); + +proc.stdout.on('data', data => { + if (data.toString().indexOf('Project is running at') > -1) { + console.log('Server started'); + startBenchmark(); + } +}); + +proc.stderr.on('data', data => { + console.error(data.toString()); +}); + +proc.on('close', code => { + console.log(`Benchmark server stopped with code ${code}`); +}); + +async function runPuppeteer() { + return await puppeteer + .launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] }) + .then(async browser => { + const page = await browser.newPage(); + let resolve; + const prom = new Promise(_resolve => { + resolve = _resolve; + }); + page.on('console', obj => { + if (!obj) return; + + if (obj.done) { + beautifyBenchmark.log(); + // resolve(obj); + } else if (obj.cycle) { + beautifyBenchmark.add(obj.cycle); + } else if (obj.allDone) { + resolve(); + } else { + console.log(obj); + } + }); + await page.goto('http://localhost:9090', { + waitUntil: 'networkidle', + }); + const res = await prom; + await browser.close(); + return res; + }); +} + +async function startBenchmark() { + console.log('Starting benchmarks'); + await runPuppeteer(); + + console.log('Killing benchmark server'); + proc.kill('SIGINT'); +} + +function exec(command) { + const { code, stdout, stderr } = sh.exec(command, { silent: true }); + if (code !== 0) { + console.error(stdout); + console.error(stderr); + sh.exit(code); + } + return stdout.trim(); +} + +function buildRevisionDist(revision) { + if (revision === 'local') { + return localDistDir; + } + const hash = exec(`git log -1 --format=%h "${revision}"`); + const buildDir = './benchmark/revisions/' + hash; + const distDir = buildDir + '/bundles'; + if (sh.test('-d', distDir)) { + console.log(`Using prebuilt "${revision}"(${hash}) revision: ${buildDir}`); + return distDir; + } + console.log(`Building "${revision}"(${hash}) revision: ${buildDir}`); + sh.mkdir('-p', buildDir); + exec(`git archive "${hash}" | tar -xC "${buildDir}"`); + + const pwd = sh.pwd(); + sh.cd(buildDir); + exec('yarn remove cypress puppeteer && yarn && yarn bundle:lib'); + sh.cd(pwd); + return distDir; +} diff --git a/benchmark/index.html b/benchmark/index.html new file mode 100644 index 00000000..247406a0 --- /dev/null +++ b/benchmark/index.html @@ -0,0 +1,28 @@ + + + + + + ReDoc + + + + + + + + + + + + + \ No newline at end of file diff --git a/benchmark/index.tsx b/benchmark/index.tsx new file mode 100644 index 00000000..40158d5c --- /dev/null +++ b/benchmark/index.tsx @@ -0,0 +1,126 @@ +import * as React from 'react'; +import { render, unmountComponentAtNode } from 'react-dom'; + +import { Redoc, RedocProps } from '../src/components'; + +import { loadAndBundleSpec } from '../src/utils'; + +import { revisions } from './revisions/config'; +import { extras } from 'mobx'; + +declare var Benchmark; + +extras.isolateGlobalState(); + +const node = document.getElementById('example'); + +const renderRoot = (Component: typeof Redoc, props: RedocProps) => + render(, node!); + +async function importRedocs() { + return Promise.all( + revisions.map(rev => { + return import('./' + rev.path.substring(12) + '/redoc.lib.js'); + }), + ); +} + +function startFullTime(redocs, resolvedSpec) { + return new Promise(async resolve => { + const suite = new Benchmark.Suite('Full time', { + maxTime: 20, + initCount: 2, + onStart(event) { + console.log(' ⏱️ ' + event.currentTarget.name); + }, + onCycle(event) { + console.log({ cycle: event.target }); + }, + onComplete() { + console.log({ done: true }); + setTimeout(() => resolve(), 10); + }, + }); + + revisions.forEach((rev, idx) => { + const redoc = redocs[idx]; + suite.add(rev.name, () => { + const store = new redoc.AppStore(resolvedSpec, 'openapi.yaml'); + renderRoot(redoc.Redoc, { store }); + unmountComponentAtNode(node!); + }); + }); + + suite.run({ async: true }); + }); +} + +function startInitStore(redocs, resolvedSpec) { + return new Promise(async resolve => { + const suite = new Benchmark.Suite('Create Store Time', { + maxTime: 20, + initCount: 2, + onStart(event) { + console.log(' ⏱️ ' + event.currentTarget.name); + }, + onCycle(event) { + console.log({ cycle: event.target }); + }, + onComplete() { + console.log({ done: true }); + setTimeout(() => resolve(), 10); + }, + }); + + revisions.forEach((rev, idx) => { + const redoc = redocs[idx]; + suite.add(rev.name, () => { + const store = new redoc.AppStore(resolvedSpec, 'openapi.yaml'); + store.dispose(); + }); + }); + + suite.run({ async: true }); + }); +} + +function startRenderTime(redocs, resolvedSpec) { + return new Promise(async resolve => { + const suite = new Benchmark.Suite('Render time', { + maxTime: 20, + initCount: 2, + onStart(event) { + console.log(' ⏱️ ' + event.currentTarget.name); + }, + onCycle(event) { + console.log({ cycle: event.target }); + unmountComponentAtNode(node!); + }, + onComplete() { + console.log({ done: true }); + setTimeout(() => resolve(), 10); + }, + }); + + revisions.forEach((rev, idx) => { + const redoc = redocs[idx]; + const store = new redoc.AppStore(resolvedSpec, 'openapi.yaml'); + suite.add(rev.name, () => { + renderRoot(redoc.Redoc, { store }); + }); + }); + + suite.run({ async: true }); + }); +} + +async function runBenchmarks() { + const redocs: any[] = await importRedocs(); + const resolvedSpec = await loadAndBundleSpec('openapi.yaml'); + await startInitStore(redocs, resolvedSpec); + await startRenderTime(redocs, resolvedSpec); + await startFullTime(redocs, resolvedSpec); + console.log({ allDone: true }); +} + +runBenchmarks(); diff --git a/package.json b/package.json index 6f3645c4..aa40346d 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "bundles/redoc.lib.js", "scripts": { "start": "webpack-dev-server --hot", - "start:perf": "webpack-dev-server --env.prod --env.perf", + "start:benchmark": "webpack-dev-server --env.prod --env.perf", "start:prod": "webpack-dev-server --env.prod", "test": "npm run lint && npm run unit && npm run e2e", "unit": "jest", @@ -20,7 +20,8 @@ "stats": "webpack -p --env.lib --env.standalone --env.prod --json --profile > stats.json", "prettier": "prettier --write \"src/**/*.{ts,tsx}\"", "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 1", - "lint": "tslint --project tsconfig.json" + "lint": "tslint --project tsconfig.json", + "benchmark": "node ./benchmark/benchmark.js" }, "author": "", "license": "MIT", @@ -43,6 +44,7 @@ "@types/webpack": "^3.0.5", "@types/webpack-env": "^1.13.0", "awesome-typescript-loader": "^3.2.2", + "beautify-benchmark": "^0.2.4", "conventional-changelog-cli": "^1.3.5", "core-js": "^2.5.1", "css-loader": "^0.28.7", @@ -61,6 +63,7 @@ "react-dev-utils": "^4.1.0", "react-dom": "^16.2.0", "rimraf": "^2.6.2", + "shelljs": "^0.8.1", "source-map-loader": "^0.2.1", "style-loader": "^0.18.2", "ts-jest": "^21.0.1", @@ -105,6 +108,10 @@ "^.+\\.tsx?$": "/node_modules/ts-jest/preprocessor.js" }, "setupTestFrameworkScriptFile": "/src/setupTests.ts", + "testPathIgnorePatterns": [ + "/node_modules/", + "/benchmark/" + ], "testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$", "moduleFileExtensions": [ "ts", diff --git a/perf/index.tsx b/perf/index.tsx deleted file mode 100644 index 15f14fda..00000000 --- a/perf/index.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import * as React from 'react'; -import { render } from 'react-dom'; - -import { Redoc, RedocProps } from '../src/components'; -import { AppStore } from '../src/services/AppStore'; -import { loadAndBundleSpec } from '../src/utils'; - -const renderRoot = (Component: typeof Redoc, props: RedocProps) => - render(, document.getElementById('example')); - -async function start() { - const resolvedSpec = await loadAndBundleSpec('big-openapi.json'); - const t0 = performance.now(); - const store = new AppStore(resolvedSpec, 'big-openapi.json'); - var t1 = performance.now(); - renderRoot(Redoc, { store }); - var t2 = performance.now(); - - console.log({ - timings: true, - 'Total Time': t2 - t0, - 'Store Init Time': t1 - t0, - 'Render Time': t2 - t1, - }); -} - -start(); diff --git a/perf/mount-time.js b/perf/mount-time.js deleted file mode 100644 index 444dbd3f..00000000 --- a/perf/mount-time.js +++ /dev/null @@ -1,73 +0,0 @@ -const puppeteer = require('puppeteer'); -const crypto = require('crypto'); - -async function run() { - return await puppeteer - .launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] }) - .then(async browser => { - const page = await browser.newPage(); - let resolve; - const prom = new Promise(_resolve => { - resolve = _resolve; - }); - page.on('console', obj => { - if (obj && obj.timings) { - resolve(obj); - } - }); - await page.goto('http://localhost:9090', { - waitUntil: 'networkidle', - }); - const res = await prom; - await browser.close(); - return res; - }); -} - -function clearLine() { - process.stdout.clearLine(); - process.stdout.cursorTo(0); -} - -const metrics = ['Total Time', 'Store Init Time', 'Render Time']; -const forEachMetric = fn => metrics.forEach(metric => fn(metric)); - -async function benchmark() { - const N = 5; - - let sum = {}; - let max = {}; - let min = {}; - - forEachMetric(metric => { - sum[metric] = 0; - max[metric] = 0; - min[metric] = Number.MAX_SAFE_INTEGER; - }); - - for (let i = 0; i < N; i++) { - const res = await run(); - forEachMetric(metric => { - if (res[metric] > max[metric]) max[metric] = res[metric]; - if (res[metric] < min[metric]) min[metric] = res[metric]; - sum[metric] += res[metric]; - }); - clearLine(); - process.stdout.write(`Running: ${i + 1} of ${N}`); - } - clearLine(); - const average = {}; - forEachMetric(metric => { - average[metric] = sum[metric] / N; - }); - console.log('Completed ', N, 'runs'); - console.log('======================='); - forEachMetric(metric => { - console.log(`Average ${metric}: `, average[metric]); - console.log(`Minimum ${metric}: `, min[metric]); - console.log(`Maximum ${metric}: `, max[metric]); - console.log(); - }); -} - -benchmark(); diff --git a/webpack.config.ts b/webpack.config.ts index dbc0e9a3..c1924f21 100644 --- a/webpack.config.ts +++ b/webpack.config.ts @@ -24,7 +24,7 @@ export default env => { } else { // playground or performance test entry = env.perf - ? ['./perf/index.tsx'] // perf test + ? ['./benchmark/index.tsx'] // perf test : [ // playground './src/polyfills.ts', @@ -73,7 +73,7 @@ export default env => { { loader: 'awesome-typescript-loader', options: { - module: 'es2015', + module: env.perf ? 'esnext' : 'es2015', }, }, ], @@ -132,7 +132,7 @@ export default env => { } else { config.plugins!.push( new HtmlWebpackPlugin({ - template: './demo/playground/index.html', + template: env.perf ? './benchmark/index.html' : './demo/playground/index.html', }), ); } diff --git a/yarn.lock b/yarn.lock index 0cd291e8..46ea2b84 100644 --- a/yarn.lock +++ b/yarn.lock @@ -685,6 +685,10 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" +beautify-benchmark@^0.2.4: + version "0.2.4" + resolved "https://registry.yarnpkg.com/beautify-benchmark/-/beautify-benchmark-0.2.4.tgz#3151def14c1a2e0d07ff2e476861c7ed0e1ae39b" + big.js@^3.1.3: version "3.2.0" resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e" @@ -2937,7 +2941,7 @@ glob2base@^0.0.12: dependencies: find-index "^0.1.1" -glob@7.1.2, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2: +glob@7.1.2, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" dependencies: @@ -5930,6 +5934,12 @@ readdirp@^2.0.0: readable-stream "^2.0.2" set-immediate-shim "^1.0.1" +rechoir@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" + dependencies: + resolve "^1.1.6" + recursive-readdir@2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.1.tgz#90ef231d0778c5ce093c9a48d74e5c5422d13a99" @@ -6159,7 +6169,7 @@ resolve@1.1.7: version "1.1.7" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b" -resolve@^1.1.7, resolve@^1.3.2: +resolve@^1.1.6, resolve@^1.1.7, resolve@^1.3.2: version "1.5.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36" dependencies: @@ -6390,6 +6400,14 @@ shell-quote@1.6.1, shell-quote@^1.6.1: array-reduce "~0.0.0" jsonify "~0.0.0" +shelljs@^0.8.1: + version "0.8.1" + resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.1.tgz#729e038c413a2254c4078b95ed46e0397154a9f1" + dependencies: + glob "^7.0.0" + interpret "^1.0.0" + rechoir "^0.6.2" + shellwords@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b"