diff --git a/.github/workflows/check-variants.yml b/.github/workflows/check-variants.yml new file mode 100644 index 0000000000..23ba21ae67 --- /dev/null +++ b/.github/workflows/check-variants.yml @@ -0,0 +1,17 @@ +name: Check variants + +on: + pull_request: + paths: + - 'frameworks/*/meta.json' + - 'scripts/check_variants.py' + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - run: python3 scripts/check_variants.py frameworks diff --git a/frameworks/hono-bun/meta.json b/frameworks/hono-bun/meta.json index f2f0e75d55..5034f00da7 100644 --- a/frameworks/hono-bun/meta.json +++ b/frameworks/hono-bun/meta.json @@ -1,5 +1,8 @@ { "display_name": "hono-bun", + "framework": "hono", + "variant": "bun", + "default_visible": true, "language": "TS", "type": "flagship", "mode": "standard", diff --git a/frameworks/hono-node/meta.json b/frameworks/hono-node/meta.json index 57e26e7507..58e66fc1cf 100644 --- a/frameworks/hono-node/meta.json +++ b/frameworks/hono-node/meta.json @@ -1,5 +1,8 @@ { "display_name": "hono-node", + "framework": "hono", + "variant": "node", + "default_visible": true, "language": "TS", "type": "flagship", "mode": "standard", diff --git a/scripts/check_badge_parity.js b/scripts/check_badge_parity.js index 1084d7ee86..6c507e00e8 100644 --- a/scripts/check_badge_parity.js +++ b/scripts/check_badge_parity.js @@ -76,9 +76,10 @@ const stubs = ` Object.keys(D.results).forEach(function(k){ D.results[k].forEach(function(r){ if(r.lang) FWLANG[r.fw]=r.lang; }); }); function typeOf(fw){ return (D.meta[fw] && D.meta[fw].type) || 'emerging'; } function modeOf(fw){ return (D.meta[fw] && D.meta[fw].mode) || 'standard'; } + function isHiddenVariant(fw){ return !!(D.meta[fw] && D.meta[fw].hidden); } function langOf(fw){ return FWLANG[fw] || ''; } function matchQ(fw){ return true; } // state.q is '' — no search filter - var state = { useMem:false, rescale:false, showTuned:true, q:'', lang:'', scope:'h1', types:[] }; + var state = { useMem:false, rescale:false, showTuned:true, showVariants:false, q:'', lang:'', scope:'h1', types:[] }; `; const run = new Function('D', ` ${stubs} diff --git a/scripts/check_variants.py b/scripts/check_variants.py new file mode 100644 index 0000000000..04eca0c690 --- /dev/null +++ b/scripts/check_variants.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Checks the variant fields of every meta.json. See the Variants section of the framework docs. + + python3 scripts/check_variants.py frameworks +""" +import json +import sys +from pathlib import Path + +MAX_VARIANTS = 2 + + +def problems(root): + found, seen = [], {} + for path in sorted(Path(root).glob("*/meta.json")): + name = path.parent.name + try: + meta = json.loads(path.read_text(encoding="utf-8")) + except ValueError as e: + found.append(f"{name}: {e}") + continue + framework, variant = meta.get("framework"), meta.get("variant") + if bool(framework) != bool(variant): + found.append(f"{name}: framework and variant go together") + if "default_visible" in meta and not (framework and variant): + found.append(f"{name}: default_visible is only for a variant") + if not isinstance(meta.get("default_visible", True), bool): + found.append(f"{name}: default_visible is true or false") + if framework and variant: + seen.setdefault(framework, []).append((name, variant)) + for framework, entries in seen.items(): + names = ", ".join(name for name, _ in entries) + if len(entries) > MAX_VARIANTS: + found.append(f"{framework}: at most {MAX_VARIANTS} variants, found {names}") + if len({variant for _, variant in entries}) != len(entries): + found.append(f"{framework}: two entries name the same variant: {names}") + return found + + +if __name__ == "__main__": + wrong = problems(sys.argv[1] if len(sys.argv) > 1 else "frameworks") + for line in wrong: + print("FAIL: " + line) + sys.exit(1 if wrong else 0) diff --git a/scripts/gen_leaderboard_data.py b/scripts/gen_leaderboard_data.py index c7a11011b1..48edf66354 100644 --- a/scripts/gen_leaderboard_data.py +++ b/scripts/gen_leaderboard_data.py @@ -1485,6 +1485,8 @@ def badge_composite(agg, profiles, meta, scope, types, show_tuned=True, lang=Non def shown(fw): """hidden() in index.html — display only, never touches the maxima.""" + if meta.get(fw, {}).get("hidden"): + return False if not show_tuned and meta.get(fw, {}).get("mode", "standard") == "tuned": return False if lang and fw_lang.get(fw) != lang: @@ -1770,7 +1772,8 @@ def profile_medals(agg, profiles, meta): if not p["scoredConns"]: continue field = [(fw, eff(pid, fw)) for fw in agg["avg"].get(pid, {}) - if in_league(fw) and _scored_for(prof, meta, pid, fw)] + if in_league(fw) and _scored_for(prof, meta, pid, fw) + and not meta.get(fw, {}).get("hidden")] field = [(fw, v) for fw, v in field if v > 0] for with_tuned in (False, True): rows = sorted((r for r in field if with_tuned or not is_tuned(r[0])), @@ -3623,6 +3626,10 @@ def build_meta(frameworks, tls_check=None): "repo": m.get("repo", ""), # optional and rare, so only on the entries that set it **({"website": m["website"]} if m.get("website") else {}), + **({"framework": m["framework"], "variant": m["variant"]} + if m.get("variant") else {}), + **({"hidden": True} + if m.get("variant") and m.get("default_visible") is not True else {}), "dir": m.get("dir", ""), "engine": m.get("engine", ""), "cmp": m.get("completeness"), diff --git a/scripts/rebuild_site_data.py b/scripts/rebuild_site_data.py index a3ccd5590f..8949ed6a88 100755 --- a/scripts/rebuild_site_data.py +++ b/scripts/rebuild_site_data.py @@ -76,6 +76,9 @@ def rebuild_frameworks_json(root: Path, site_data: Path) -> None: } if "mode" in m: entry["mode"] = m["mode"] + for key in ("framework", "variant", "default_visible"): + if key in m: + entry[key] = m[key] if m.get("website"): entry["website"] = m["website"] if "completeness" in m: diff --git a/site/content/docs/add-framework/implementation-rules/frameworks/_index.md b/site/content/docs/add-framework/implementation-rules/frameworks/_index.md index 44a8bddf56..1471a3b102 100644 --- a/site/content/docs/add-framework/implementation-rules/frameworks/_index.md +++ b/site/content/docs/add-framework/implementation-rules/frameworks/_index.md @@ -35,3 +35,17 @@ Every framework entry also declares a **mode** in `meta.json.mode` - how strictl {{< card link="standard" title="Standard" subtitle="Default, production-style usage: documented framework APIs, production settings, and standard libraries." icon="shield-check" >}} {{< card link="tuned" title="Tuned" subtitle="Non-default configs, experimental flags, and custom optimizations allowed." icon="adjustments" >}} {{< /cards >}} + +## Variants + +A **variant** is the same framework on another runtime or with another documented API, for example Hono on Node and Hono on Bun. It is a different concept from mode: each variant follows all the rules of its own mode, and no variant is the real one, they are all legitimate. + +```json +"framework": "hono", +"variant": "bun" +``` + +- A framework has **at most two** variants, because every entry runs on the same hardware. Subscribe a variant only to the tests where it is different. +- A variant has its own row, with a pill that says which variant it is, and it is scored like any other entry. +- A variant is listed only when the Variants filter is on, and it has no badges or medals, unless it sets `"default_visible": true`. Like a tuned entry that is filtered out, it still counts for the leader of each profile. +- Each variant has its own badge URL, under its own entry name. diff --git a/site/content/docs/add-framework/meta-json.md b/site/content/docs/add-framework/meta-json.md index 8f73ca0bda..c450c96625 100644 --- a/site/content/docs/add-framework/meta-json.md +++ b/site/content/docs/add-framework/meta-json.md @@ -31,6 +31,9 @@ Create a `meta.json` file in your framework directory: | `engine` | HTTP server engine (e.g., `Kestrel`, `Tomcat`, `hyper`) | | `type` | `flagship`, `emerging` or `experimental` for frameworks - servers you write application code against, tiered by how production-proven they are; `engine` for implementations applications are not written against (raw sockets, WSGI/ASGI hosts); `infrastructure` for reverse proxies / static-file servers. See [Implementation Rules](implementation-rules/) | | `mode` | Frameworks only: `standard` (default - idiomatic, production-style usage) or `tuned` (non-default config / optimizations) | +| `framework` | With `variant`: the framework this entry is a variant of. See [Variants](/docs/add-framework/implementation-rules/frameworks/#variants) | +| `variant` | With `framework`: what is different in this entry, a runtime or an API, e.g. `bun` | +| `default_visible` | Variants only: `true` shows the entry in the default view of the board and gives it badges and medals. Default is `false` | | `completeness` | Frameworks only: which of `routing`, `middleware`, `request` and `response` the framework does for you. Each one set to `false` takes 2.5% off the composite; an axis you leave out reads as done, and omitting the field scores a full ×1.00. Omit it entirely on an entry that runs only WebSocket or gRPC profiles - the factor never applies on those boards. See [Completeness](/docs/scoring/completeness/) | | `description` | Shown in the framework detail popup on the leaderboard | | `repo` | Link to the framework's source repository | diff --git a/site/data/frameworks.json b/site/data/frameworks.json index 57f98c2eab..9d470e6717 100644 --- a/site/data/frameworks.json +++ b/site/data/frameworks.json @@ -1067,6 +1067,9 @@ "engine": "helidon-nima" }, "hono-bun": { + "framework": "hono", + "variant": "bun", + "default_visible": true, "dir": "hono-bun", "description": "Ultrafast Web Standards framework running natively on Bun \u2014 no adapter overhead.", "repo": "https://github.com/honojs/hono", @@ -1081,6 +1084,9 @@ } }, "hono-node": { + "framework": "hono", + "variant": "node", + "default_visible": true, "dir": "hono-node", "description": "Hono on node:http through @hono/node-server, with the cluster module for multi-core scaling.", "repo": "https://github.com/honojs/node-server", diff --git a/site/leaderboard/index.html b/site/leaderboard/index.html index 945155a24d..43d078e59d 100644 --- a/site/leaderboard/index.html +++ b/site/leaderboard/index.html @@ -330,6 +330,8 @@ sits snug against the square; empty on standard rows. */ .tndslot{width:2.75rem; flex:none; display:flex; justify-content:flex-end; align-items:center} .b-tuned{background:rgba(200,154,60,.16); color:#8a6414} + .b-variant{background:rgba(79,120,200,.16); color:#2f5597; margin-left:.35rem} + html[data-theme="dark"] .b-variant{background:rgba(79,120,200,.26); color:#9dbcf0} html[data-theme="dark"] .b-tuned{background:rgba(200,154,60,.2); color:#e0b45e} /* medals - how many golds/silvers/bronzes the entry holds. Dropped below 820px, where the name column halves and the count is not worth what it would cost. */ @@ -636,6 +638,7 @@ +
Frameworks @@ -757,7 +760,7 @@

// useless as an addressable filter - q=C matches 73 of 78 entries, q=V matches // 33. The language rank badges link here, so the field they claim has to be // the field the page shows. - var state = { view:'composite', profile:null, conns:null, sortKey:'rps', sortDir:-1, types:['flagship','emerging'], q:'', lang:'', useMem:false, scope:'h1', showTuned:true, rescale:false, compactFw:false, heat:false }; + var state = { view:'composite', profile:null, conns:null, sortKey:'rps', sortDir:-1, types:['flagship','emerging'], q:'', lang:'', useMem:false, scope:'h1', showTuned:true, showVariants:false, rescale:false, compactFw:false, heat:false }; // ── parsing helpers ────────────────────────────────────── function lat(s){ if(!s) return Infinity; var m=String(s).match(/([\d.]+)\s*(us|ms|s)/); if(!m) return Infinity; var v=parseFloat(m[1]); return m[2]==='us'?v:m[2]==='ms'?v*1e3:v*1e6; } @@ -777,6 +780,16 @@

return modeOf(fw)==='tuned' ? 'tuned' : ''; } + // Same framework, another runtime or API: every variant is a legitimate way to run it, so this + // only names which one a row is and which others exist. + function variantPill(fw, cls){ + var m=D.meta[fw]; + if(!m || !m.variant) return ''; + var others=Object.keys(D.meta).filter(function(o){ return o!==fw && D.meta[o].framework===m.framework; }); + return ''+esc(m.variant)+''; + } + function isHiddenVariant(fw){ return !!(D.meta[fw] && D.meta[fw].hidden); } // A shield beside the name for an entry whose TLS was probed and came back // clean: the mounted certificate, TLS 1.3, an AEAD cipher, and no obsolete // protocol or weak cipher accepted. Absent means unverified, not failed -- @@ -987,7 +1000,7 @@

var impl='https://github.com/MDA2AV/HttpArena/tree/main/frameworks/'+encodeURIComponent(dir); var h='
' + '' - + '

'+esc(fw)+'

'+tlsCheckBadge(fw)+''+typeAbbr(t)+''+tunedPill(fw)+'
' + + '

'+esc(fw)+'

'+tlsCheckBadge(fw)+variantPill(fw)+''+typeAbbr(t)+''+tunedPill(fw)+'
' + '
'+esc([lang, m.engine, typeAbbr(t), modeOf(fw)==='tuned'?'tuned':'', cmpDecl(fw)&&CMP_TYPES.indexOf(t)>=0 ? 'completeness '+(4-cmpMissing(fw).length)+'/4' : ''].filter(Boolean).join(' · '))+'
'; if(m.desc) h+='

'+esc(m.desc)+'

'; @@ -1474,9 +1487,11 @@

var tf=document.getElementById('typeFilter'); Array.prototype.forEach.call(tf.querySelectorAll('button[data-type]'), function(b){ b.classList.toggle('on', state.types.indexOf(b.getAttribute('data-type'))>=0); }); var tb=document.getElementById('tunedToggle'); if(tb) tb.classList.toggle('on', state.showTuned); + var vb=document.getElementById('variantsToggle'); if(vb) vb.classList.toggle('on', state.showVariants); } function saveTypes(){ try{ localStorage.setItem('lb-types', JSON.stringify(state.types)); }catch(e){} } function saveTuned(){ try{ localStorage.setItem('lb-showtuned', state.showTuned?'1':'0'); }catch(e){} } + function saveVariants(){ try{ localStorage.setItem('lb-showvariants', state.showVariants?'1':'0'); }catch(e){} } function saveCompactFw(){ try{ localStorage.setItem('lb-fwcompact', state.compactFw?'1':'0'); }catch(e){} } function toggleCompactFw(){ state.compactFw=!state.compactFw; saveCompactFw(); render(); scheduleHash(); } // flagship/emerging/experimental combine (one framework pool); engine and @@ -1614,7 +1629,7 @@

// once filtered to Ruby). The `rescale` toggle restores that behaviour for // when comparing a subset against itself is the point. function outOfLeague(fw){ return state.types.indexOf(typeOf(fw))<0; } - function hidden(fw){ if(!state.showTuned && modeOf(fw)==='tuned') return true; if(state.lang && langOf(fw)!==state.lang) return true; return !matchQ(fw); } + function hidden(fw){ if(!state.showVariants && isHiddenVariant(fw)) return true; if(!state.showTuned && modeOf(fw)==='tuned') return true; if(state.lang && langOf(fw)!==state.lang) return true; return !matchQ(fw); } function excl(fw){ return outOfLeague(fw) || hidden(fw); } var exclFromScale = state.rescale ? excl : outOfLeague; var minBpr=null; @@ -1784,6 +1799,7 @@

if(!defSort) p.push('sort='+state.sortKey+':'+state.sortDir); var ts=state.types.slice().sort().join(','); if(ts!=='emerging,flagship') p.push('type='+ts); if(!state.showTuned) p.push('tuned=0'); + if(state.showVariants) p.push('variants=1'); if(state.compactFw) p.push('fw=compact'); if(state.lang) p.push('lang='+encodeURIComponent(state.lang)); if(state.q) p.push('q='+encodeURIComponent(state.q)); @@ -1797,6 +1813,7 @@

if(p.doc){ location.replace(docUrl(p.doc)); return; } // legacy #doc=x bookmark -> /docs/x/ if(p.type){ var ts=p.type.split(',').filter(Boolean); if(ts.length) state.types=ts; } if(p.hasOwnProperty('tuned')) state.showTuned = p.tuned!=='0'; + if(p.hasOwnProperty('variants')) state.showVariants = p.variants==='1'; if(p.hasOwnProperty('fw')) state.compactFw = p.fw==='compact'; state.lang = p.lang || ''; state.q = p.q || ''; @@ -1903,7 +1920,7 @@

? '' + ''+nm+'' - : starHtml(r.fw)+''+tunedPill(r.fw,'tnd')+''+esc(r.lang)+''+nm+''+tlsCheckBadge(r.fw)+medalChips(r.fw)) + : starHtml(r.fw)+''+tunedPill(r.fw,'tnd')+''+esc(r.lang)+''+nm+''+variantPill(r.fw,'tnd')+tlsCheckBadge(r.fw)+medalChips(r.fw)) + '' + ''+r.score.toFixed(0)+cmpDelta(r)+''; pids.forEach(function(pid){ @@ -2051,7 +2068,7 @@

} function profileCell(k, r, x){ if(k==='rank') return '
'+x.rank+'
'; - if(k==='fw'){ var nm=esc(r.fw); return '
'+starHtml(r.fw)+''+tunedPill(r.fw,'tnd')+''+esc(r.lang||'')+''+nm+''+tlsCheckBadge(r.fw)+'
'; } + if(k==='fw'){ var nm=esc(r.fw); return '
'+starHtml(r.fw)+''+tunedPill(r.fw,'tnd')+''+esc(r.lang||'')+''+nm+''+variantPill(r.fw,'tnd')+tlsCheckBadge(r.fw)+'
'; } if(k==='rps'){ var pct=x.maxRps>0?Math.max((r.rps||0)/x.maxRps*100,(r.rps>0?1.5:0)):0; return '
'+fmt(r.rps||0)+''+statusHtml(r)+'
'; } if(k==='bw') return '
'+esc(r.bandwidth||'-')+'
'; if(k==='bwreq'){ var b=bw(r.bandwidth), q=r.rps||0; return '
'+(b>0&&q>0?fmtBytes(b/q):'-')+'
'; } @@ -2078,6 +2095,7 @@

renderTypes(); var rows=all.filter(function(r){ if(state.types.indexOf(typeOf(r.fw))<0) return false; + if(!state.showVariants && isHiddenVariant(r.fw)) return false; if(!state.showTuned && modeOf(r.fw)==='tuned') return false; if(!matchQ(r.fw, r.lang)) return false; return true; @@ -2130,8 +2148,10 @@

try{ var _vt=['flagship','emerging','experimental','engine','infrastructure']; var _t=JSON.parse(localStorage.getItem('lb-types')||'null'); if(_t&&_t.length){ _t=_t.filter(function(x){return _vt.indexOf(x)>=0;}); if(_t.length) state.types=_t; } }catch(e){} document.getElementById('typeFilter').addEventListener('click', function(e){ var b=e.target.closest('button[data-type]'); if(!b) return; toggleType(b.getAttribute('data-type')); }); try{ var _st=localStorage.getItem('lb-showtuned'); if(_st!==null) state.showTuned=_st==='1'; }catch(e){} + try{ var _sv=localStorage.getItem('lb-showvariants'); if(_sv!==null) state.showVariants=_sv==='1'; }catch(e){} try{ var _fc=localStorage.getItem('lb-fwcompact'); if(_fc!==null) state.compactFw=_fc==='1'; }catch(e){} document.getElementById('tunedToggle').addEventListener('click', function(){ state.showTuned=!state.showTuned; saveTuned(); render(); }); + document.getElementById('variantsToggle').addEventListener('click', function(){ state.showVariants=!state.showVariants; saveVariants(); render(); }); restoreFromHash(); initTips(); initHeat(); initModal(); initDocLinks(); initSearchHelp(); initRounds(); initTheme(); initNavSearch(); hw(); buildNav(); renderHead(); render(); (function(){ var b=document.getElementById('brandHome'); if(b) b.onclick=function(e){ e.preventDefault(); pickScope('h1'); }; })();