<?xml version="1.0"?>
<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en">
	<id>https://wiki.kunix.org/index.php?action=history&amp;feed=atom&amp;title=Module%3AModel</id>
	<title>Module:Model - Revision history</title>
	<link rel="self" type="application/atom+xml" href="https://wiki.kunix.org/index.php?action=history&amp;feed=atom&amp;title=Module%3AModel"/>
	<link rel="alternate" type="text/html" href="https://wiki.kunix.org/index.php?title=Module:Model&amp;action=history"/>
	<updated>2026-11-22T13:45:24Z</updated>
	<subtitle>Revision history for this page on the wiki</subtitle>
	<generator>MediaWiki 1.43.9</generator>
	<entry>
		<id>https://wiki.kunix.org/index.php?title=Module:Model&amp;diff=110&amp;oldid=prev</id>
		<title>Jdj: Model helpers for Infobox model: benchmark table, tokens per parameter</title>
		<link rel="alternate" type="text/html" href="https://wiki.kunix.org/index.php?title=Module:Model&amp;diff=110&amp;oldid=prev"/>
		<updated>2026-11-16T00:01:53Z</updated>

		<summary type="html">&lt;p&gt;Model helpers for Infobox model: benchmark table, tokens per parameter&lt;/p&gt;
&lt;p&gt;&lt;b&gt;New page&lt;/b&gt;&lt;/p&gt;&lt;div&gt;-- Helpers for {{Infobox model}}: the benchmark table and tokens-per-parameter.&lt;br /&gt;
-- Both read the wrapper&amp;#039;s own arguments (frame:getParent()), like Module:Formation.&lt;br /&gt;
local p = {}&lt;br /&gt;
&lt;br /&gt;
local lang = mw.language.getContentLanguage()&lt;br /&gt;
&lt;br /&gt;
local function clean(v)&lt;br /&gt;
	if v then v = mw.text.trim(v) end&lt;br /&gt;
	return (v ~= nil and v ~= &amp;#039;&amp;#039;) and v or nil&lt;br /&gt;
end&lt;br /&gt;
&lt;br /&gt;
-- Common benchmarks with their own parameters, in display order.&lt;br /&gt;
-- Each takes NAME, NAME_shots, NAME_ref and NAME_mean (= no to leave it out of the mean).&lt;br /&gt;
local NAMED = {&lt;br /&gt;
	{ &amp;#039;hellaswag&amp;#039;, &amp;#039;HellaSwag&amp;#039; },&lt;br /&gt;
	{ &amp;#039;arc_easy&amp;#039;, &amp;#039;ARC-Easy&amp;#039; },&lt;br /&gt;
	{ &amp;#039;arc_challenge&amp;#039;, &amp;#039;ARC-Challenge&amp;#039; },&lt;br /&gt;
	{ &amp;#039;piqa&amp;#039;, &amp;#039;PIQA&amp;#039; },&lt;br /&gt;
	{ &amp;#039;winogrande&amp;#039;, &amp;#039;WinoGrande&amp;#039; },&lt;br /&gt;
	{ &amp;#039;openbookqa&amp;#039;, &amp;#039;OpenBookQA&amp;#039; },&lt;br /&gt;
	{ &amp;#039;boolq&amp;#039;, &amp;#039;BoolQ&amp;#039; },&lt;br /&gt;
	{ &amp;#039;sciq&amp;#039;, &amp;#039;SciQ&amp;#039; },&lt;br /&gt;
	{ &amp;#039;lambada&amp;#039;, &amp;#039;LAMBADA&amp;#039; },&lt;br /&gt;
	{ &amp;#039;mmlu&amp;#039;, &amp;#039;MMLU&amp;#039; },&lt;br /&gt;
	{ &amp;#039;gsm8k&amp;#039;, &amp;#039;GSM8K&amp;#039; },&lt;br /&gt;
	{ &amp;#039;humaneval&amp;#039;, &amp;#039;HumanEval&amp;#039; },&lt;br /&gt;
}&lt;br /&gt;
&lt;br /&gt;
-- &amp;quot;61.2&amp;quot;, &amp;quot;61.2%&amp;quot;, &amp;quot;61.2 %&amp;quot; -&amp;gt; 61.2. Anything else (e.g. &amp;quot;24.1 ppl&amp;quot;) is not a percentage.&lt;br /&gt;
local function percent(s)&lt;br /&gt;
	local n = s:match(&amp;#039;^%s*(%-?%d+%.?%d*)%s*%%?%s*$&amp;#039;)&lt;br /&gt;
	n = tonumber(n)&lt;br /&gt;
	if n and n &amp;gt;= 0 and n &amp;lt;= 100 then return n end&lt;br /&gt;
	return nil&lt;br /&gt;
end&lt;br /&gt;
&lt;br /&gt;
local function fmt(n)&lt;br /&gt;
	return lang:formatNum(math.floor(n * 10 + 0.5) / 10)&lt;br /&gt;
end&lt;br /&gt;
&lt;br /&gt;
-- Benchmark table. Rows come from the named parameters above, then bench1 ... bench30&lt;br /&gt;
-- (benchN = name, benchN_score, benchN_shots, benchN_ref, benchN_mean).&lt;br /&gt;
function p.benchmarks(frame)&lt;br /&gt;
	local args = frame:getParent().args&lt;br /&gt;
	local function arg(k) return clean(args[k]) end&lt;br /&gt;
&lt;br /&gt;
	local rows = {}&lt;br /&gt;
	local function add(name, score, shots, ref, inMean)&lt;br /&gt;
		if not score then return end&lt;br /&gt;
		table.insert(rows, { name = name, score = score, shots = shots, ref = ref,&lt;br /&gt;
			pct = percent(score), inMean = inMean ~= &amp;#039;no&amp;#039; })&lt;br /&gt;
	end&lt;br /&gt;
	for _, b in ipairs(NAMED) do&lt;br /&gt;
		local k = b[1]&lt;br /&gt;
		add(b[2], arg(k), arg(k .. &amp;#039;_shots&amp;#039;), arg(k .. &amp;#039;_ref&amp;#039;), arg(k .. &amp;#039;_mean&amp;#039;))&lt;br /&gt;
	end&lt;br /&gt;
	for i = 1, 30 do&lt;br /&gt;
		local k = &amp;#039;bench&amp;#039; .. i&lt;br /&gt;
		if arg(k) then&lt;br /&gt;
			add(arg(k), arg(k .. &amp;#039;_score&amp;#039;), arg(k .. &amp;#039;_shots&amp;#039;), arg(k .. &amp;#039;_ref&amp;#039;), arg(k .. &amp;#039;_mean&amp;#039;))&lt;br /&gt;
		end&lt;br /&gt;
	end&lt;br /&gt;
	if #rows == 0 then return &amp;#039;&amp;#039; end&lt;br /&gt;
&lt;br /&gt;
	local hasRef = false&lt;br /&gt;
	for _, r in ipairs(rows) do if r.ref then hasRef = true end end&lt;br /&gt;
	local refLabel = arg(&amp;#039;bench_ref_label&amp;#039;) or &amp;#039;Ref.&amp;#039;&lt;br /&gt;
&lt;br /&gt;
	local out = { &amp;#039;&amp;lt;div class=&amp;quot;dm-bench&amp;quot;&amp;gt;&amp;#039; }&lt;br /&gt;
	local function line(class, name, shots, score, pct, ref)&lt;br /&gt;
		local bar = &amp;#039;&amp;#039;&lt;br /&gt;
		if pct then&lt;br /&gt;
			bar = &amp;#039;&amp;lt;div class=&amp;quot;dm-bench-bar&amp;quot;&amp;gt;&amp;lt;div class=&amp;quot;dm-bench-fill&amp;quot; style=&amp;quot;width:&amp;#039;&lt;br /&gt;
				.. string.format(&amp;#039;%.1f&amp;#039;, pct) .. &amp;#039;%&amp;quot;&amp;gt;&amp;lt;/div&amp;gt;&amp;lt;/div&amp;gt;&amp;#039;&lt;br /&gt;
		end&lt;br /&gt;
		table.insert(out, &amp;#039;&amp;lt;div class=&amp;quot;dm-bench-row &amp;#039; .. class .. (hasRef and &amp;#039; dm-bench-hasref&amp;#039; or &amp;#039;&amp;#039;) .. &amp;#039;&amp;quot;&amp;gt;&amp;#039;&lt;br /&gt;
			.. &amp;#039;&amp;lt;div class=&amp;quot;dm-bench-name&amp;quot;&amp;gt;&amp;#039; .. name&lt;br /&gt;
			.. (shots and &amp;#039; &amp;lt;span class=&amp;quot;dm-bench-shots&amp;quot;&amp;gt;&amp;#039; .. shots .. &amp;#039;-shot&amp;lt;/span&amp;gt;&amp;#039; or &amp;#039;&amp;#039;) .. &amp;#039;&amp;lt;/div&amp;gt;&amp;#039;&lt;br /&gt;
			.. &amp;#039;&amp;lt;div class=&amp;quot;dm-bench-score&amp;quot;&amp;gt;&amp;#039; .. score .. &amp;#039;&amp;lt;/div&amp;gt;&amp;#039;&lt;br /&gt;
			.. (hasRef and &amp;#039;&amp;lt;div class=&amp;quot;dm-bench-ref&amp;quot;&amp;gt;&amp;#039; .. (ref or &amp;#039;–&amp;#039;) .. &amp;#039;&amp;lt;/div&amp;gt;&amp;#039; or &amp;#039;&amp;#039;)&lt;br /&gt;
			.. bar .. &amp;#039;&amp;lt;/div&amp;gt;&amp;#039;)&lt;br /&gt;
	end&lt;br /&gt;
	if hasRef then&lt;br /&gt;
		table.insert(out, &amp;#039;&amp;lt;div class=&amp;quot;dm-bench-row dm-bench-head dm-bench-hasref&amp;quot;&amp;gt;&amp;lt;div class=&amp;quot;dm-bench-name&amp;quot;&amp;gt;Benchmark&amp;lt;/div&amp;gt;&amp;#039;&lt;br /&gt;
			.. &amp;#039;&amp;lt;div class=&amp;quot;dm-bench-score&amp;quot;&amp;gt;Score&amp;lt;/div&amp;gt;&amp;lt;div class=&amp;quot;dm-bench-ref&amp;quot;&amp;gt;&amp;#039; .. refLabel .. &amp;#039;&amp;lt;/div&amp;gt;&amp;lt;/div&amp;gt;&amp;#039;)&lt;br /&gt;
	end&lt;br /&gt;
	for _, r in ipairs(rows) do&lt;br /&gt;
		-- &amp;quot;0&amp;quot; shots reads better as zero-shot.&lt;br /&gt;
		local shots = r.shots&lt;br /&gt;
		if shots == &amp;#039;0&amp;#039; then shots = &amp;#039;zero&amp;#039; end&lt;br /&gt;
		line(&amp;#039;&amp;#039;, r.name, shots, r.score, r.pct, r.ref)&lt;br /&gt;
	end&lt;br /&gt;
&lt;br /&gt;
	-- Mean over the percentage scores, unless turned off.&lt;br /&gt;
	if (arg(&amp;#039;bench_mean&amp;#039;) or &amp;#039;yes&amp;#039;) ~= &amp;#039;no&amp;#039; then&lt;br /&gt;
		local sum, n, refSum, refN = 0, 0, 0, 0&lt;br /&gt;
		for _, r in ipairs(rows) do&lt;br /&gt;
			if r.pct and r.inMean then&lt;br /&gt;
				sum, n = sum + r.pct, n + 1&lt;br /&gt;
				local rp = r.ref and percent(r.ref)&lt;br /&gt;
				if rp then refSum, refN = refSum + rp, refN + 1 end&lt;br /&gt;
			end&lt;br /&gt;
		end&lt;br /&gt;
		if n &amp;gt;= 2 then&lt;br /&gt;
			local mean = sum / n&lt;br /&gt;
			-- Only compare means when the reference covers the same tasks.&lt;br /&gt;
			local refMean = (refN == n) and fmt(refSum / refN) or nil&lt;br /&gt;
			line(&amp;#039;dm-bench-mean&amp;#039;, &amp;#039;Mean &amp;lt;span class=&amp;quot;dm-bench-shots&amp;quot;&amp;gt;&amp;#039; .. n .. &amp;#039; tasks&amp;lt;/span&amp;gt;&amp;#039;,&lt;br /&gt;
				nil, fmt(mean), mean, refMean)&lt;br /&gt;
		end&lt;br /&gt;
	end&lt;br /&gt;
&lt;br /&gt;
	local note, source = arg(&amp;#039;bench_note&amp;#039;), arg(&amp;#039;bench_source&amp;#039;)&lt;br /&gt;
	if note or source then&lt;br /&gt;
		table.insert(out, &amp;#039;&amp;lt;div class=&amp;quot;dm-bench-note&amp;quot;&amp;gt;&amp;#039; .. (note or &amp;#039;&amp;#039;)&lt;br /&gt;
			.. ((note and source) and &amp;#039; · &amp;#039; or &amp;#039;&amp;#039;) .. (source and (&amp;#039;Source: &amp;#039; .. source) or &amp;#039;&amp;#039;) .. &amp;#039;&amp;lt;/div&amp;gt;&amp;#039;)&lt;br /&gt;
	end&lt;br /&gt;
	table.insert(out, &amp;#039;&amp;lt;/div&amp;gt;&amp;#039;)&lt;br /&gt;
	return table.concat(out)&lt;br /&gt;
end&lt;br /&gt;
&lt;br /&gt;
-- &amp;quot;125M&amp;quot;, &amp;quot;1.5B&amp;quot;, &amp;quot;174 billion&amp;quot;, &amp;quot;2,000,000&amp;quot; -&amp;gt; number.&lt;br /&gt;
local MULT = { k = 1e3, m = 1e6, b = 1e9, t = 1e12,&lt;br /&gt;
	thousand = 1e3, million = 1e6, billion = 1e9, trillion = 1e12 }&lt;br /&gt;
local function count(s)&lt;br /&gt;
	if not s then return nil end&lt;br /&gt;
	s = s:gsub(&amp;#039;&amp;lt;[^&amp;gt;]*&amp;gt;&amp;#039;, &amp;#039;&amp;#039;):gsub(&amp;#039;%[%[[^%]|]*|&amp;#039;, &amp;#039;&amp;#039;):gsub(&amp;#039;[%[%]]&amp;#039;, &amp;#039;&amp;#039;):gsub(&amp;#039;,&amp;#039;, &amp;#039;&amp;#039;):lower()&lt;br /&gt;
	local n, unit = s:match(&amp;#039;(%d+%.?%d*)%s*(%a*)&amp;#039;)&lt;br /&gt;
	n = tonumber(n)&lt;br /&gt;
	if not n then return nil end&lt;br /&gt;
	if unit ~= &amp;#039;&amp;#039; then&lt;br /&gt;
		local m = MULT[unit]&lt;br /&gt;
		if not m then&lt;br /&gt;
			-- &amp;quot;tokens&amp;quot;, &amp;quot;params&amp;quot; etc. after a bare number mean no multiplier&lt;br /&gt;
			m = (unit:match(&amp;#039;^tok&amp;#039;) or unit:match(&amp;#039;^par&amp;#039;)) and 1 or nil&lt;br /&gt;
		end&lt;br /&gt;
		if not m then return nil end&lt;br /&gt;
		n = n * m&lt;br /&gt;
	end&lt;br /&gt;
	return n&lt;br /&gt;
end&lt;br /&gt;
&lt;br /&gt;
-- Training tokens per parameter, with the Chinchilla-optimal ~20 for comparison.&lt;br /&gt;
function p.ratio(frame)&lt;br /&gt;
	local args = frame:getParent().args&lt;br /&gt;
	local given = clean(args.tokens_per_param)&lt;br /&gt;
	if given then return given end&lt;br /&gt;
	local params = count(clean(args.parameters))&lt;br /&gt;
	local tokens = count(clean(args.training_tokens))&lt;br /&gt;
	if not params or not tokens or params == 0 then return &amp;#039;&amp;#039; end&lt;br /&gt;
	local r = tokens / params&lt;br /&gt;
	local text = r &amp;gt;= 100 and lang:formatNum(math.floor(r + 0.5)) or fmt(r)&lt;br /&gt;
	return text .. &amp;#039; &amp;lt;span class=&amp;quot;dm-bench-shots&amp;quot;&amp;gt;Chinchilla-optimal ≈ 20&amp;lt;/span&amp;gt;&amp;#039;&lt;br /&gt;
end&lt;br /&gt;
&lt;br /&gt;
return p&lt;/div&gt;</summary>
		<author><name>Jdj</name></author>
	</entry>
</feed>