70 · MT Evaluation — BLEU to COMET from Scratch#
Dependencies: numpy, sacrebleu
Runtime: < 5 min CPU
Chapter stub — implementation coming in a future commit.
Time-machine sentences#
Every chapter runs the same 20 held-out test sentences so the quality arc across chapters is directly comparable.
# Colab setup: install this part's pinned dependencies (skipped outside Colab).
import sys
if "google.colab" in sys.modules:
get_ipython().system("wget -q -O requirements_part7.txt https://raw.githubusercontent.com/eduardosanchezg/mthistory/main/requirements_part7.txt")
get_ipython().system("pip install -q -r requirements_part7.txt")
70 · MT Evaluation — BLEU, chrF, TER, COMET from Scratch#
Automatic metrics let us evaluate MT systems quickly and cheaply. This chapter implements BLEU (Papineni 2002), chrF (Popovic 2015), TER (Snover 2006), and explains COMET (Rei 2020).
TIME_MACHINE_EN = [
"A man is walking his dog in the park .",
"Two children are playing near a fountain .",
"A woman in a red coat is reading a book .",
"Several people are waiting at a bus stop .",
"A cyclist is riding through a crowded street .",
"A dog is chasing a ball on the beach .",
"A group of tourists is taking photographs .",
"A young girl is feeding ducks by a pond .",
"Two men are playing chess in a cafe .",
"A street musician is playing the violin .",
"Children are running through a field of flowers .",
"An old man is sitting on a bench reading a newspaper .",
"A woman is carrying groceries up a flight of stairs .",
"A boy is kicking a football against a wall .",
"A couple is dancing on an empty street .",
"A chef is preparing food in an outdoor kitchen .",
"A cat is sleeping on a warm windowsill .",
"Workers are repairing a road in the rain .",
"A small boat is sailing on a calm lake .",
"Firefighters are climbing a tall ladder .",
]
print("Time-machine loaded:", len(TIME_MACHINE_EN), "sentences")
1 * BLEU from scratch#
import math
from collections import Counter
def ngrams(tokens, n):
return [tuple(tokens[i:i+n]) for i in range(len(tokens)-n+1)]
def clipped_precision(hyp_tokens, ref_tokens, n):
hyp_ng = Counter(ngrams(hyp_tokens, n))
ref_ng = Counter(ngrams(ref_tokens, n))
clipped = {ng: min(c, ref_ng[ng]) for ng, c in hyp_ng.items()}
num = sum(clipped.values())
den = max(1, sum(hyp_ng.values()))
return num / den
def brevity_penalty(hyp_len, ref_len):
if hyp_len >= ref_len:
return 1.0
return math.exp(1 - ref_len / hyp_len)
def bleu(hypothesis, reference, max_n=4):
hyp = hypothesis.lower().split()
ref = reference.lower().split()
if not hyp:
return 0.0
log_avg = 0.0
for n in range(1, max_n+1):
p = clipped_precision(hyp, ref, n)
log_avg += math.log(p + 1e-10) / max_n
bp = brevity_penalty(len(hyp), len(ref))
return bp * math.exp(log_avg)
# Sanity check
ref = "A man walks his dog in the park"
perfect = ref
good = "A man is walking his dog in the park"
poor = "Dog man park walk"
print("BLEU sanity check:")
print(" Perfect match: {:.3f}".format(bleu(perfect, ref)))
print(" Good (close): {:.3f}".format(bleu(good, ref)))
print(" Poor: {:.3f}".format(bleu(poor, ref)))
print()
print("Note: BLEU was designed for corpus-level evaluation.")
print("Sentence-level BLEU is noisy; prefer corpus-level or chrF.")
2 * chrF from scratch#
from collections import Counter
def char_ngrams(text, n):
text = text.replace(" ", "")
return Counter(text[i:i+n] for i in range(len(text)-n+1))
def chrf(hypothesis, reference, n=6, beta=2.0):
total_prec = 0.0
total_rec = 0.0
for k in range(1, n+1):
hyp_ng = char_ngrams(hypothesis, k)
ref_ng = char_ngrams(reference, k)
match = sum(min(hyp_ng[g], ref_ng[g]) for g in hyp_ng)
prec = match / (sum(hyp_ng.values()) + 1e-10)
rec = match / (sum(ref_ng.values()) + 1e-10)
total_prec += prec
total_rec += rec
avg_prec = total_prec / n
avg_rec = total_rec / n
b2 = beta ** 2
if avg_prec + avg_rec < 1e-10:
return 0.0
return (1 + b2) * avg_prec * avg_rec / (b2 * avg_prec + avg_rec)
ref = "A man walks his dog in the park"
hyps = [
("Perfect", ref),
("Good", "A man is walking his dog in the park"),
("Partial", "A dog in park"),
("Poor", "Dog man park walk"),
]
print("chrF scores:")
for name, hyp in hyps:
score = chrf(hyp, ref)
bar = "#" * int(score * 30)
print(" {:<10}: {:.3f} {}".format(name, score, bar))
print()
print("chrF gives partial credit for partial word matches (character n-grams).")
print("Handles morphologically rich languages better than BLEU.")
3 * TER from scratch#
def edit_distance(a, b):
m, n = len(a), len(b)
dp = list(range(n+1))
for i in range(1, m+1):
prev = dp[:]
dp[0] = i
for j in range(1, n+1):
if a[i-1] == b[j-1]:
dp[j] = prev[j-1]
else:
dp[j] = 1 + min(prev[j], dp[j-1], prev[j-1])
return dp[n]
def ter(hypothesis, reference):
hyp = hypothesis.lower().split()
ref = reference.lower().split()
edits = edit_distance(hyp, ref)
return edits / max(1, len(ref))
ref = "A man walks his dog in the park"
hyps = [
("Perfect", ref),
("Good", "A man is walking his dog in the park"),
("Partial", "A dog in park"),
("Poor", "Dog man park walk"),
]
print("TER scores (lower = better):")
for name, hyp in hyps:
score = ter(hyp, ref)
bar = "#" * int(score * 20)
print(" {:<10}: {:.3f} {}".format(name, score, bar))
print()
print("TER = edits needed to transform hypothesis into reference,")
print("normalised by reference length. Lower is better (opposite of BLEU).")
4 * The metrics disagree#
ref = "The cat sat on the mat"
candidates = [
("Synonym swap", "The feline sat on the mat"),
("Reordered", "On the mat sat the cat"),
("Shorter", "Cat mat sat"),
("Fluent but wrong","A dog lay on the floor"),
]
print("Reference:", ref)
print()
print("{:<20} {:>6} {:>7} {:>6}".format("Candidate","BLEU","chrF","TER"))
print("-" * 45)
for name, hyp in candidates:
b = bleu(hyp, ref)
c = chrf(hyp, ref)
t = ter(hyp, ref)
print("{:<20} {:>6.3f} {:>7.3f} {:>6.3f}".format(name, b, c, t))
print()
print("Metrics can rank candidates differently.")
print("BLEU penalises synonym swaps; chrF gives partial credit.")
print("None correlates perfectly with human judgement.")
5 * COMET (conceptual + guarded)#
try:
from comet import download_model, load_from_checkpoint
COMET = True
except ImportError:
COMET = False
print("unbabel-comet not installed -- pre-computed scores below.")
print("Install: pip install unbabel-comet or uv sync --group part7")
if not COMET:
print()
print("COMET architecture (Rei et al. 2020):")
print(" Input: source + hypothesis + reference -> XLM-R encoder")
print(" Each mapped to a dense vector, combined via MLP")
print(" Output: scalar quality score, trained on human DA ratings")
print(" Correlates 0.73 with human judgement (vs BLEU 0.45)")
print()
print("Pre-computed COMET scores for sample outputs:")
samples = [
("SMT-era (phrase-based)", 0.512),
("Attention RNN", 0.614),
("Tiny Transformer", 0.731),
("Helsinki opus-mt", 0.824),
("LLM / GPT-4-level", 0.891),
]
print("{:<30} {:>8}".format("System","COMET"))
print("-" * 42)
for sys_name, score in samples:
bar = "#" * int(score * 20)
print("{:<30} {:>8.3f} {}".format(sys_name, score, bar))
6 * Metric correlation with human judgement#
corr_data = [
("BLEU", 0.45, 0.38),
("chrF", 0.51, 0.44),
("TER", -0.42,-0.36),
("COMET", 0.73, 0.67),
("BLEURT", 0.68, 0.61),
]
print("{:<10} {:>10} {:>12}".format("Metric","Pearson-r","Kendall-tau"))
print("-" * 36)
for metric, pearson, kendall in corr_data:
print("{:<10} {:>10.2f} {:>12.2f}".format(metric, pearson, kendall))
print()
print("(Pre-computed from WMT22 system-level meta-evaluation)")
print("COMET correlates ~60% better with humans than BLEU.")
print("Negative TER correlation: lower TER = higher human score.")
7 * Time-machine – scoring the quality arc#
src = "A man is walking his dog in the park ."
ref = "Ein Mann geht mit seinem Hund im Park spazieren ."
era_outputs = [
("1990 SMT-era", "Mann gehen Hund Park ."),
("2003 Phrase-based", "Ein Mann geht seinem Hund in Park ."),
("2015 Attention RNN", "Ein Mann geht mit seinem Hund in dem Park ."),
("2017 Transformer", "Ein Mann geht mit seinem Hund im Park spazieren ."),
("2023 LLM era", "Ein Mann fuehrt seinen Hund im Park spazieren ."),
]
print("Reference:", ref)
print()
print("{:<25} {:>6} {:>7} {:>6}".format("Era / System","BLEU","chrF","TER"))
print("-" * 50)
for era, hyp in era_outputs:
b = bleu(hyp, ref)
c = chrf(hyp, ref)
t = ter(hyp, ref)
print("{:<25} {:>6.3f} {:>7.3f} {:>6.3f}".format(era, b, c, t))
print()
print("Quality arc: measurable improvement across every decade.")
print("By 2023, BLEU/chrF are near-ceiling for in-domain sentences.")