chore: improve basic benchmark

.github/CONTRIBUTING.md

@@ -71,6 +71,8 @@ There are some other scripts available in the `scripts` section of the `package.
 
 ## Project Structure
 
+- **`benchmark`**: contains basic perf benchmark. Not fully ready yet
+
 - **`demo`**: contains project demo with demo specs and HMR playground used in development
 
   - `demo/playground`: HMR Playground used in development
@@ -79,7 +81,6 @@ There are some other scripts available in the `scripts` section of the `package.
 
 - **`e2e`**: contains e2e tests. The e2e tests are written and run with [Cypress](https://www.cypress.io/).
 
-- **`perf`**: contains basic perf benchmark. Not ready yet
 
 - **`src`**: contains the source code. The codebase is written in Typescript. CSS styles are managed with [Styled components](https://www.styled-components.com/). State is managed by [MobX](https://github.com/mobxjs/mobx)
 

.gitignore

@@ -26,6 +26,8 @@ e2e/.build/
 cypress/
 bundles
 
+/benchmark/revisions
+
 /coverage
 .ghpages-tmp
 stats.json

benchmark/benchmark.js

@@ -0,0 +1,121 @@
+const beautifyBenchmark = require('beautify-benchmark');
+const sh = require('shelljs');
+const fs = require('fs');
+const pathJoin = require('path').join;
+const spawn = require('child_process').spawn;
+const puppeteer = require('puppeteer');
+
+const args = process.argv.slice(2);
+args[0] = args[0] || 'HEAD';
+args[1] = args[1] || 'local';
+
+console.log('Benchmarking revisions: ' + args.join(', '));
+
+const localDistDir = './benchmark/revisions/local/bundles';
+sh.rm('-rf', localDistDir);
+console.log(`Building local dist: ${localDistDir}`);
+sh.mkdir('-p', localDistDir);
+exec(`yarn bundle:lib --output-path ${localDistDir}`);
+
+const revisions = [];
+for (const arg of args) {
+  revisions.push({ name: arg, path: buildRevisionDist(arg) });
+}
+
+const configFile = `
+  export const revisions = [ ${revisions.map(rev => JSON.stringify(rev)).join(', ')} ];
+`;
+
+const configDir = './benchmark/revisions/config.js';
+console.log(`Writing config "${configDir}"`);
+fs.writeFileSync(configDir, configFile);
+
+console.log('Starging benchmark server');
+const proc = spawn('npm', ['run', 'start:benchmark']);
+
+proc.stdout.on('data', data => {
+  if (data.toString().indexOf('Project is running at') > -1) {
+    console.log('Server started');
+    startBenchmark();
+  }
+});
+
+proc.stderr.on('data', data => {
+  console.error(data.toString());
+});
+
+proc.on('close', code => {
+  console.log(`Benchmark server stopped with code ${code}`);
+});
+
+async function runPuppeteer() {
+  return await puppeteer
+    .launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] })
+    .then(async browser => {
+      const page = await browser.newPage();
+      let resolve;
+      const prom = new Promise(_resolve => {
+        resolve = _resolve;
+      });
+      page.on('console', obj => {
+        if (!obj) return;
+
+        if (obj.done) {
+          beautifyBenchmark.log();
+          // resolve(obj);
+        } else if (obj.cycle) {
+          beautifyBenchmark.add(obj.cycle);
+        } else if (obj.allDone) {
+          resolve();
+        } else {
+          console.log(obj);
+        }
+      });
+      await page.goto('http://localhost:9090', {
+        waitUntil: 'networkidle',
+      });
+      const res = await prom;
+      await browser.close();
+      return res;
+    });
+}
+
+async function startBenchmark() {
+  console.log('Starting benchmarks');
+  await runPuppeteer();
+
+  console.log('Killing benchmark server');
+  proc.kill('SIGINT');
+}
+
+function exec(command) {
+  const { code, stdout, stderr } = sh.exec(command, { silent: true });
+  if (code !== 0) {
+    console.error(stdout);
+    console.error(stderr);
+    sh.exit(code);
+  }
+  return stdout.trim();
+}
+
+function buildRevisionDist(revision) {
+  if (revision === 'local') {
+    return localDistDir;
+  }
+  const hash = exec(`git log -1 --format=%h "${revision}"`);
+  const buildDir = './benchmark/revisions/' + hash;
+  const distDir = buildDir + '/bundles';
+  if (sh.test('-d', distDir)) {
+    console.log(`Using prebuilt "${revision}"(${hash}) revision: ${buildDir}`);
+    return distDir;
+  }
+  console.log(`Building "${revision}"(${hash}) revision: ${buildDir}`);
+  sh.mkdir('-p', buildDir);
+  exec(`git archive "${hash}" | tar -xC "${buildDir}"`);
+
+  const pwd = sh.pwd();
+  sh.cd(buildDir);
+  exec('yarn remove cypress puppeteer && yarn && yarn bundle:lib');
+  sh.cd(pwd);
+  return distDir;
+}

benchmark/index.html

@@ -0,0 +1,28 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="UTF-8" />
+  <title>ReDoc</title>
+  <style>
+    body {
+      margin: 0;
+      padding: 0;
+    }
+
+    redoc {
+      display: block;
+    }
+  </style>
+  <link href="https://fonts.googleapis.com/css?family=Montserrat:300,400,700|Roboto:300,400,700" rel="stylesheet">
+</head>
+
+<body>
+  <redoc id="example"></redoc>
+  <!-- <redoc spec-url="./openapi.yaml"></redoc> -->
+  <script src="https://unpkg.com/lodash@4.17.4/lodash.js"></script>
+  <script src="https://unpkg.com/benchmark@2.1.4/benchmark.js"></script>
+  <!-- <script src="../bundles/redoc.standalone.js"></script> -->
+</body>
+
+</html>

benchmark/index.tsx

@@ -0,0 +1,126 @@
+import * as React from 'react';
+import { render, unmountComponentAtNode } from 'react-dom';
+
+import { Redoc, RedocProps } from '../src/components';
+
+import { loadAndBundleSpec } from '../src/utils';
+
+import { revisions } from './revisions/config';
+import { extras } from 'mobx';
+
+declare var Benchmark;
+
+extras.isolateGlobalState();
+
+const node = document.getElementById('example');
+
+const renderRoot = (Component: typeof Redoc, props: RedocProps) =>
+  render(<Component {...props} />, node!);
+
+async function importRedocs() {
+  return Promise.all(
+    revisions.map(rev => {
+      return import('./' + rev.path.substring(12) + '/redoc.lib.js');
+    }),
+  );
+}
+
+function startFullTime(redocs, resolvedSpec) {
+  return new Promise(async resolve => {
+    const suite = new Benchmark.Suite('Full time', {
+      maxTime: 20,
+      initCount: 2,
+      onStart(event) {
+        console.log('  ⏱️  ' + event.currentTarget.name);
+      },
+      onCycle(event) {
+        console.log({ cycle: event.target });
+      },
+      onComplete() {
+        console.log({ done: true });
+        setTimeout(() => resolve(), 10);
+      },
+    });
+
+    revisions.forEach((rev, idx) => {
+      const redoc = redocs[idx];
+      suite.add(rev.name, () => {
+        const store = new redoc.AppStore(resolvedSpec, 'openapi.yaml');
+        renderRoot(redoc.Redoc, { store });
+        unmountComponentAtNode(node!);
+      });
+    });
+
+    suite.run({ async: true });
+  });
+}
+
+function startInitStore(redocs, resolvedSpec) {
+  return new Promise(async resolve => {
+    const suite = new Benchmark.Suite('Create Store Time', {
+      maxTime: 20,
+      initCount: 2,
+      onStart(event) {
+        console.log('  ⏱️  ' + event.currentTarget.name);
+      },
+      onCycle(event) {
+        console.log({ cycle: event.target });
+      },
+      onComplete() {
+        console.log({ done: true });
+        setTimeout(() => resolve(), 10);
+      },
+    });
+
+    revisions.forEach((rev, idx) => {
+      const redoc = redocs[idx];
+      suite.add(rev.name, () => {
+        const store = new redoc.AppStore(resolvedSpec, 'openapi.yaml');
+        store.dispose();
+      });
+    });
+
+    suite.run({ async: true });
+  });
+}
+
+function startRenderTime(redocs, resolvedSpec) {
+  return new Promise(async resolve => {
+    const suite = new Benchmark.Suite('Render time', {
+      maxTime: 20,
+      initCount: 2,
+      onStart(event) {
+        console.log('  ⏱️  ' + event.currentTarget.name);
+      },
+      onCycle(event) {
+        console.log({ cycle: event.target });
+        unmountComponentAtNode(node!);
+      },
+      onComplete() {
+        console.log({ done: true });
+        setTimeout(() => resolve(), 10);
+      },
+    });
+
+    revisions.forEach((rev, idx) => {
+      const redoc = redocs[idx];
+      const store = new redoc.AppStore(resolvedSpec, 'openapi.yaml');
+      suite.add(rev.name, () => {
+        renderRoot(redoc.Redoc, { store });
+      });
+    });
+
+    suite.run({ async: true });
+  });
+}
+
+async function runBenchmarks() {
+  const redocs: any[] = await importRedocs();
+  const resolvedSpec = await loadAndBundleSpec('openapi.yaml');
+  await startInitStore(redocs, resolvedSpec);
+  await startRenderTime(redocs, resolvedSpec);
+  await startFullTime(redocs, resolvedSpec);
+  console.log({ allDone: true });
+}
+
+runBenchmarks();

package.json

@@ -5,7 +5,7 @@
   "main": "bundles/redoc.lib.js",
   "scripts": {
     "start": "webpack-dev-server --hot",
-    "start:perf": "webpack-dev-server --env.prod --env.perf",
+    "start:benchmark": "webpack-dev-server --env.prod --env.perf",
     "start:prod": "webpack-dev-server --env.prod",
     "test": "npm run lint && npm run unit && npm run e2e",
     "unit": "jest",
@@ -20,7 +20,8 @@
     "stats": "webpack -p --env.lib --env.standalone --env.prod --json --profile > stats.json",
     "prettier": "prettier --write \"src/**/*.{ts,tsx}\"",
     "changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 1",
-    "lint": "tslint --project tsconfig.json"
+    "lint": "tslint --project tsconfig.json",
+    "benchmark": "node ./benchmark/benchmark.js"
   },
   "author": "",
   "license": "MIT",
@@ -43,6 +44,7 @@
     "@types/webpack": "^3.0.5",
     "@types/webpack-env": "^1.13.0",
     "awesome-typescript-loader": "^3.2.2",
+    "beautify-benchmark": "^0.2.4",
     "conventional-changelog-cli": "^1.3.5",
     "core-js": "^2.5.1",
     "css-loader": "^0.28.7",
@@ -61,6 +63,7 @@
     "react-dev-utils": "^4.1.0",
     "react-dom": "^16.2.0",
     "rimraf": "^2.6.2",
+    "shelljs": "^0.8.1",
     "source-map-loader": "^0.2.1",
     "style-loader": "^0.18.2",
     "ts-jest": "^21.0.1",
@@ -105,6 +108,10 @@
       "^.+\\.tsx?$": "<rootDir>/node_modules/ts-jest/preprocessor.js"
     },
     "setupTestFrameworkScriptFile": "<rootDir>/src/setupTests.ts",
+    "testPathIgnorePatterns": [
+      "/node_modules/",
+      "/benchmark/"
+    ],
     "testRegex": "(/__tests__/.*|(\\.|/)(test|spec))\\.(jsx?|tsx?)$",
     "moduleFileExtensions": [
       "ts",

perf/index.tsx

@@ -1,27 +0,0 @@
-import * as React from 'react';
-import { render } from 'react-dom';
-
-import { Redoc, RedocProps } from '../src/components';
-import { AppStore } from '../src/services/AppStore';
-import { loadAndBundleSpec } from '../src/utils';
-
-const renderRoot = (Component: typeof Redoc, props: RedocProps) =>
-  render(<Component {...props} />, document.getElementById('example'));
-
-async function start() {
-  const resolvedSpec = await loadAndBundleSpec('big-openapi.json');
-  const t0 = performance.now();
-  const store = new AppStore(resolvedSpec, 'big-openapi.json');
-  var t1 = performance.now();
-  renderRoot(Redoc, { store });
-  var t2 = performance.now();
-
-  console.log({
-    timings: true,
-    'Total Time': t2 - t0,
-    'Store Init Time': t1 - t0,
-    'Render Time': t2 - t1,
-  });
-}
-
-start();

perf/mount-time.js

@@ -1,73 +0,0 @@
-const puppeteer = require('puppeteer');
-const crypto = require('crypto');
-
-async function run() {
-  return await puppeteer
-    .launch({ args: ['--no-sandbox', '--disable-setuid-sandbox'] })
-    .then(async browser => {
-      const page = await browser.newPage();
-      let resolve;
-      const prom = new Promise(_resolve => {
-        resolve = _resolve;
-      });
-      page.on('console', obj => {
-        if (obj && obj.timings) {
-          resolve(obj);
-        }
-      });
-      await page.goto('http://localhost:9090', {
-        waitUntil: 'networkidle',
-      });
-      const res = await prom;
-      await browser.close();
-      return res;
-    });
-}
-
-function clearLine() {
-  process.stdout.clearLine();
-  process.stdout.cursorTo(0);
-}
-
-const metrics = ['Total Time', 'Store Init Time', 'Render Time'];
-const forEachMetric = fn => metrics.forEach(metric => fn(metric));
-
-async function benchmark() {
-  const N = 5;
-
-  let sum = {};
-  let max = {};
-  let min = {};
-
-  forEachMetric(metric => {
-    sum[metric] = 0;
-    max[metric] = 0;
-    min[metric] = Number.MAX_SAFE_INTEGER;
-  });
-
-  for (let i = 0; i < N; i++) {
-    const res = await run();
-    forEachMetric(metric => {
-      if (res[metric] > max[metric]) max[metric] = res[metric];
-      if (res[metric] < min[metric]) min[metric] = res[metric];
-      sum[metric] += res[metric];
-    });
-    clearLine();
-    process.stdout.write(`Running: ${i + 1} of ${N}`);
-  }
-  clearLine();
-  const average = {};
-  forEachMetric(metric => {
-    average[metric] = sum[metric] / N;
-  });
-  console.log('Completed ', N, 'runs');
-  console.log('=======================');
-  forEachMetric(metric => {
-    console.log(`Average ${metric}: `, average[metric]);
-    console.log(`Minimum ${metric}: `, min[metric]);
-    console.log(`Maximum ${metric}: `, max[metric]);
-    console.log();
-  });
-}
-
-benchmark();

webpack.config.ts

@@ -24,7 +24,7 @@ export default env => {
   } else {
     // playground or performance test
     entry = env.perf
-      ? ['./perf/index.tsx'] // perf test
+      ? ['./benchmark/index.tsx'] // perf test
       : [
           // playground
           './src/polyfills.ts',
@@ -73,7 +73,7 @@ export default env => {
             {
               loader: 'awesome-typescript-loader',
               options: {
-                module: 'es2015',
+                module: env.perf ? 'esnext' : 'es2015',
               },
             },
           ],
@@ -132,7 +132,7 @@ export default env => {
   } else {
     config.plugins!.push(
       new HtmlWebpackPlugin({
-        template: './demo/playground/index.html',
+        template: env.perf ? './benchmark/index.html' : './demo/playground/index.html',
       }),
     );
   }

yarn.lock

@@ -685,6 +685,10 @@ bcrypt-pbkdf@^1.0.0:
   dependencies:
     tweetnacl "^0.14.3"
 
+beautify-benchmark@^0.2.4:
+  version "0.2.4"
+  resolved "https://registry.yarnpkg.com/beautify-benchmark/-/beautify-benchmark-0.2.4.tgz#3151def14c1a2e0d07ff2e476861c7ed0e1ae39b"
+
 big.js@^3.1.3:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/big.js/-/big.js-3.2.0.tgz#a5fc298b81b9e0dca2e458824784b65c52ba588e"
@@ -2937,7 +2941,7 @@ glob2base@^0.0.12:
   dependencies:
     find-index "^0.1.1"
 
-glob@7.1.2, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2:
+glob@7.1.2, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.1.1, glob@^7.1.2:
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
   dependencies:
@@ -5930,6 +5934,12 @@ readdirp@^2.0.0:
     readable-stream "^2.0.2"
     set-immediate-shim "^1.0.1"
 
+rechoir@^0.6.2:
+  version "0.6.2"
+  resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384"
+  dependencies:
+    resolve "^1.1.6"
+
 recursive-readdir@2.2.1:
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/recursive-readdir/-/recursive-readdir-2.2.1.tgz#90ef231d0778c5ce093c9a48d74e5c5422d13a99"
@@ -6159,7 +6169,7 @@ resolve@1.1.7:
   version "1.1.7"
   resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.1.7.tgz#203114d82ad2c5ed9e8e0411b3932875e889e97b"
 
-resolve@^1.1.7, resolve@^1.3.2:
+resolve@^1.1.6, resolve@^1.1.7, resolve@^1.3.2:
   version "1.5.0"
   resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.5.0.tgz#1f09acce796c9a762579f31b2c1cc4c3cddf9f36"
   dependencies:
@@ -6390,6 +6400,14 @@ shell-quote@1.6.1, shell-quote@^1.6.1:
     array-reduce "~0.0.0"
     jsonify "~0.0.0"
 
+shelljs@^0.8.1:
+  version "0.8.1"
+  resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.1.tgz#729e038c413a2254c4078b95ed46e0397154a9f1"
+  dependencies:
+    glob "^7.0.0"
+    interpret "^1.0.0"
+    rechoir "^0.6.2"
+
 shellwords@^0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b"