From ae2ad5becc6807c88276ce52c6f52a227cfc656a Mon Sep 17 00:00:00 2001 From: ines Date: Sun, 29 Oct 2017 03:58:19 +0100 Subject: [PATCH] Remove charts from model direcory and add speed benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With speed benchmarks, charts ended up taking up too much space – and they were mostly data porn and not particularly useful anyways. Instead, we might add a "Compare" page that fetches all models and lets the user compare two or more models in terms of accuracy, speed etc. --- website/_includes/_functions.jade | 2 +- website/_includes/_page_models.jade | 34 +++++++++--------- website/_includes/_scripts.jade | 5 +-- website/assets/js/main.js | 56 +++++++++++++---------------- website/models/_data.json | 24 +++++++------ website/styleguide.jade | 4 +-- 6 files changed, 57 insertions(+), 68 deletions(-) diff --git a/website/_includes/_functions.jade b/website/_includes/_functions.jade index 5209dbbec..eb16d9659 100644 --- a/website/_includes/_functions.jade +++ b/website/_includes/_functions.jade @@ -14,7 +14,7 @@ - MODEL_META = public.models._data.MODEL_META - MODEL_LICENSES = public.models._data.MODEL_LICENSES -- MODEL_ACCURACY = public.models._data.MODEL_ACCURACY +- MODEL_BENCHMARKS = public.models._data.MODEL_BENCHMARKS - EXAMPLE_SENTENCES = public.models._data.EXAMPLE_SENTENCES - IS_PAGE = (SECTION != "index") && !landing diff --git a/website/_includes/_page_models.jade b/website/_includes/_page_models.jade index d4ce55f43..10e7e1746 100644 --- a/website/_includes/_page_models.jade +++ b/website/_includes/_page_models.jade @@ -26,7 +26,7 @@ for id in CURRENT_MODELS | about this model, see the overview of the | #[+a(gh("spacy-models") + "/releases") latest model releases]. - +table(data-tpl=id data-tpl-key="table") + +table.o-block-small(data-tpl=id data-tpl-key="table") +row +cell #[+label Language] +cell #[+tag=comps.lang] #{LANGUAGES[comps.lang]} @@ -56,22 +56,20 @@ for id in CURRENT_MODELS select.o-field__select.u-text-small(data-tpl=id data-tpl-key="compat") .o-empty(data-tpl=id data-tpl-key="compat-versions")   - section(data-tpl=id data-tpl-key="accuracy-wrapper" style="display: none") - +grid.o-no-block - +grid-col("third") - +h(4) Accuracy - +table.o-block-small - for label, field in MODEL_ACCURACY - +row(style="display: none") - +cell.u-nowrap - +label=label - if MODEL_META[field] - | #[+help(MODEL_META[field]).u-color-subtle] - +cell.u-text-right(data-tpl=id data-tpl-key=field) - | n/a - - +grid-col("two-thirds") - +h(4) Comparison - +chart(id).u-padding-small + section(data-tpl=id data-tpl-key="benchmarks" style="display: none") + +grid.o-block-small + for keys, label in MODEL_BENCHMARKS + .u-flex-full.u-padding-small(data-tpl=id data-tpl-key=label.toLowerCase() style="display: none") + +table.o-block-small + +row("head") + +head-cell(colspan="2")=(MODEL_META["benchmark_" + label] || label) + for label, field in keys + +row(style="display: none") + +cell.u-nowrap + +label=label + if MODEL_META[field] + | #[+help(MODEL_META[field]).u-color-subtle] + +cell.u-text-right(data-tpl=id data-tpl-key=field) + | n/a p.u-text-small.u-color-dark(data-tpl=id data-tpl-key="notes") diff --git a/website/_includes/_scripts.jade b/website/_includes/_scripts.jade index 4bb4d87ef..5ecdd0711 100644 --- a/website/_includes/_scripts.jade +++ b/website/_includes/_scripts.jade @@ -6,9 +6,6 @@ if quickstart if IS_PAGE script(src="/assets/js/in-view.min.js") -if HAS_MODELS - script(src="/assets/js/chart.min.js") - if environment == "deploy" script(async src="https://www.google-analytics.com/analytics.js") @@ -35,7 +32,7 @@ script | }; if HAS_MODELS - | new ModelLoader('!{MODELS_REPO}', !{JSON.stringify(CURRENT_MODELS)}, !{JSON.stringify(MODEL_LICENSES)}, !{JSON.stringify(MODEL_ACCURACY)}); + | new ModelLoader('!{MODELS_REPO}', !{JSON.stringify(CURRENT_MODELS)}, !{JSON.stringify(MODEL_LICENSES)}, !{JSON.stringify(MODEL_BENCHMARKS)}); if environment == "deploy" | window.ga=window.ga||function(){ diff --git a/website/assets/js/main.js b/website/assets/js/main.js index 5cbd4d807..d9465bb67 100644 --- a/website/assets/js/main.js +++ b/website/assets/js/main.js @@ -108,22 +108,12 @@ class ModelLoader { * @param {Object} licenses - License IDs mapped to URLs. * @param {Object} accKeys - Available accuracy keys mapped to display labels. */ - constructor(repo, models = [], licenses = {}, accKeys = {}) { + constructor(repo, models = [], licenses = {}, benchmarkKeys = {}) { this.url = `https://raw.githubusercontent.com/${repo}/master`; this.repo = `https://github.com/${repo}`; this.modelIds = models; this.licenses = licenses; - this.accKeys = accKeys; - this.chartColor = '#09a3d5'; - this.chartOptions = { - type: 'bar', - options: { responsive: true, scales: { - yAxes: [{ label: 'Accuracy', ticks: { suggestedMin: 70 }}], - xAxes: [{ barPercentage: 0.425 }] - }} - } - Chart.defaults.global.legend.position = 'bottom'; - Chart.defaults.global.defaultFontFamily = "-apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'"; + this.benchKeys = benchmarkKeys; this.init(); } @@ -171,7 +161,7 @@ class ModelLoader { /** * Update model details in tables. Currently quite hacky :( */ - render({ lang, name, version, sources, pipeline, vectors, url, author, license, accuracy, size, description, notes }) { + render({ lang, name, version, sources, pipeline, vectors, url, author, license, accuracy, speed, size, description, notes }) { const modelId = `${lang}_${name}`; const model = `${modelId}-${version}`; const template = new Templater(modelId); @@ -194,11 +184,31 @@ class ModelLoader { if (license) template.fill('license', this.licenses[license] ? getLink(license, this.licenses[license]) : license, true); template.get('download').setAttribute('href', `${this.repo}/releases/tag/${model}`); - if (accuracy) this.renderAccuracy(template, accuracy, modelId); + + this.renderBenchmarks(template, accuracy, speed); this.renderCompat(template, modelId); template.get('table').removeAttribute('data-loading'); } + renderBenchmarks(template, accuracy = {}, speed = {}) { + if (!accuracy && !speed) return; + template.get('benchmarks').style.display = 'block'; + this.renderTable(template, 'parser', accuracy, val => val.toFixed(2)); + this.renderTable(template, 'ner', accuracy, val => val.toFixed(2)); + this.renderTable(template, 'speed', speed, Math.round); + } + + renderTable(template, id, benchmarks, convertVal = val => val) { + if (!this.benchKeys[id] || !Object.keys(this.benchKeys[id]).some(key => benchmarks[key])) return; + const keys = Object.keys(this.benchKeys[id]).map(k => benchmarks[k] ? k : false).filter(k => k); + template.get(id).style.display = 'block'; + for (let key of keys) { + template + .fill(key, this.convertNumber(convertVal(benchmarks[key]))) + .parentElement.style.display = 'table-row'; + } + } + renderCompat(template, modelId) { template.get('compat-wrapper').style.display = 'table-row'; const options = Object.keys(this.compat).map(v => ``).join(''); @@ -211,24 +221,6 @@ class ModelLoader { }); } - renderAccuracy(template, accuracy, modelId, compare=false) { - template.get('accuracy-wrapper').style.display = 'block'; - const metaKeys = Object.keys(this.accKeys).map(k => accuracy[k] ? k : false).filter(k => k); - for (let key of metaKeys) { - template.fill(key, accuracy[key].toFixed(2)).parentElement.style.display = 'table-row'; - } - - this.chartOptions.options.legend = { display: compare } - new Chart(`chart_${modelId}`, Object.assign({}, this.chartOptions, { data: { - datasets: [{ - label: modelId, - data: metaKeys.map(key => accuracy[key].toFixed(2)), - backgroundColor: this.chartColor - }], - labels: metaKeys.map(key => this.accKeys[key]) - }})) - } - getLatestVersion(model, compat = {}) { for (let spacy_v of Object.keys(compat)) { const models = compat[spacy_v]; diff --git a/website/models/_data.json b/website/models/_data.json index 293477b9a..d41d45e8e 100644 --- a/website/models/_data.json +++ b/website/models/_data.json @@ -50,25 +50,27 @@ "ents_f": "Entities (F-score)", "ents_p": "Entities (precision)", "ents_r": "Entities (recall)", + "cpu": "words per second on CPU", + "gpu": "words per second on GPU", "pipeline": "Processing pipeline components in order", - "sources": "Sources of training data" + "sources": "Sources of training data", + "benchmark_parser": "Parser accuracy", + "benchmark_ner": "NER accuracy", + "benchmark_speed": "Speed" }, "MODEL_LICENSES": { - "CC BY-SA": "https://creativecommons.org/licenses/by-sa/3.0/", + "CC BY-SA": "https://creativecommons.org/licenses/by-sa/3.0/", "CC BY-SA 3.0": "https://creativecommons.org/licenses/by-sa/3.0/", - "CC BY-NC": "https://creativecommons.org/licenses/by-nc/3.0/", + "CC BY-NC": "https://creativecommons.org/licenses/by-nc/3.0/", "CC BY-NC 3.0": "https://creativecommons.org/licenses/by-nc/3.0/", - "GPL": "http://www.gnu.de/documents/gpl.en.html" + "GPL": "http://www.gnu.de/documents/gpl.en.html" }, - "MODEL_ACCURACY": { - "uas": "UAS", - "las": "LAS", - "tags_acc": "POS", - "ents_f": "NER F", - "ents_p": "NER P", - "ents_r": "NER R" + "MODEL_BENCHMARKS": { + "parser": { "uas": "UAS", "las": "LAS", "tags_acc": "POS" }, + "ner": { "ents_f": "NER F", "ents_p": "NER P", "ents_r": "NER R" }, + "speed": { "nwords": "Words", "cpu": "w/s CPU", "gpu": "w/s GPU" } }, "LANGUAGES": { diff --git a/website/styleguide.jade b/website/styleguide.jade index 88c88435f..b503569b7 100644 --- a/website/styleguide.jade +++ b/website/styleguide.jade @@ -609,8 +609,8 @@ include _includes/_mixins +code(false, "json").o-no-block "CC BY-SA 3.0": "http://..." +row - +cell #[code MODEL_ACCURACY] - +cell Display labels for accuracy keys. + +cell #[code MODEL_BENCHMARKS] + +cell Display labels for accuracy and speed. +cell +code(false, "json").o-no-block "ents_f": "NER F"