71 · Human Evaluation — MQM, DA, Ranking

71 · Human Evaluation — MQM, DA, Ranking#

Dependencies: stdlib only
Runtime: reference only

Chapter stub — implementation coming in a future commit.

Time-machine sentences#

Every chapter runs the same 20 held-out test sentences so the quality arc across chapters is directly comparable.

Open In Colab

# Colab setup: install this part's pinned dependencies (skipped outside Colab).
import sys
if "google.colab" in sys.modules:
    get_ipython().system("wget -q -O requirements_part7.txt https://raw.githubusercontent.com/eduardosanchezg/mthistory/main/requirements_part7.txt")
    get_ipython().system("pip install -q -r requirements_part7.txt")

71 · Human Evaluation — MQM, DA, Ranking Protocols#

Automatic metrics are proxies. This chapter covers the protocols used when accuracy really matters: Direct Assessment, relative ranking, and the Multidimensional Quality Metrics framework.

1 * Direct Assessment (DA)#

import math

# Simulate 3 annotators scoring 4 systems on 5 sentences (0-100)
raw_scores = {
    "A1": {"SystemA": [72,65,80,70,68], "SystemB": [55,60,52,58,50],
           "SystemC": [88,82,90,85,87], "SystemD": [40,45,38,42,44]},
    "A2": {"SystemA": [60,55,70,62,59], "SystemB": [48,50,45,52,46],
           "SystemC": [80,75,82,78,81], "SystemD": [32,38,30,35,37]},
    "A3": {"SystemA": [78,72,85,75,74], "SystemB": [62,65,58,63,57],
           "SystemC": [92,88,94,90,91], "SystemD": [48,52,44,49,51]},
}

def z_normalize(scores):
    mean = sum(scores) / len(scores)
    var = sum((x - mean)**2 for x in scores) / len(scores)
    std = math.sqrt(var) if var > 0 else 1.0
    return [(x - mean) / std for x in scores]

systems = ["SystemA", "SystemB", "SystemC", "SystemD"]
normalized = {sys: [] for sys in systems}

for ann, ann_scores in raw_scores.items():
    all_ann = [s for scores in ann_scores.values() for s in scores]
    ann_mean = sum(all_ann) / len(all_ann)
    ann_var = sum((x - ann_mean)**2 for x in all_ann) / len(all_ann)
    ann_std = math.sqrt(ann_var) if ann_var > 0 else 1.0
    for sys in systems:
        z = [(x - ann_mean) / ann_std for x in ann_scores[sys]]
        normalized[sys].extend(z)

print("Direct Assessment -- z-normalised scores:")
print()
ranked = sorted(systems, key=lambda s: -sum(normalized[s])/len(normalized[s]))
for sys in ranked:
    z = sum(normalized[sys]) / len(normalized[sys])
    bar = "#" * int((z + 2) * 8)
    print("  {:<10}: {:.3f}  {}".format(sys, z, bar))
print()
print("Z-normalisation removes per-annotator scale bias.")

2 * Relative ranking#

# Pairwise preferences from 10 comparison tasks
preferences = [
    ("SystemC", "SystemA"), ("SystemA", "SystemB"), ("SystemC", "SystemD"),
    ("SystemA", "SystemD"), ("SystemC", "SystemB"), ("SystemC", "SystemA"),
    ("SystemB", "SystemD"), ("SystemA", "SystemB"), ("SystemC", "SystemD"),
    ("SystemA", "SystemD"),
]

wins = {s: 0 for s in systems}
total = {s: 0 for s in systems}
for winner, loser in preferences:
    wins[winner] += 1
    total[winner] += 1
    total[loser] += 1

print("Pairwise ranking (win rate):")
print()
ranked = sorted(systems, key=lambda s: -(wins[s]/(total[s]+1e-9)))
for sys in ranked:
    rate = wins[sys] / (total[sys] + 1e-9)
    bar = "#" * int(rate * 20)
    print("  {:<10}: {}/{} = {:.1%}  {}".format(
          sys, wins[sys], total[sys], rate, bar))

3 * MQM – Multidimensional Quality Metrics#

ERROR_TAXONOMY = {
    "mistranslation":   {"category": "accuracy",  "minor": 1, "major": 5, "critical": 10},
    "omission":         {"category": "accuracy",  "minor": 1, "major": 5, "critical": 10},
    "addition":         {"category": "accuracy",  "minor": 1, "major": 5, "critical": 10},
    "grammar":          {"category": "fluency",   "minor": 1, "major": 5, "critical": 10},
    "spelling":         {"category": "fluency",   "minor": 1, "major": 5, "critical": 10},
    "punctuation":      {"category": "fluency",   "minor": 1, "major": 5, "critical": 10},
    "terminology":      {"category": "terminology","minor":1, "major": 5, "critical": 10},
}

def mqm_score(errors, start=25.0):
    total_penalty = sum(ERROR_TAXONOMY[etype][severity]
                        for etype, severity in errors)
    return max(0.0, start - total_penalty)

src = "The quick brown fox jumps over the lazy dog."
hyp = "Le rapide renard saute au-dessus du chien."
errors = [
    ("omission",     "minor"),
    ("mistranslation","minor"),
    ("grammar",      "major"),
]

score = mqm_score(errors)
print("MQM scoring:")
print("  Source:     ", src)
print("  Hypothesis: ", hyp)
print()
print("  Errors found:")
for etype, severity in errors:
    penalty = ERROR_TAXONOMY[etype][severity]
    print("    {:<20} {:<8} -> -{} points".format(etype, severity, penalty))
print()
print("  MQM score: {:.1f} / 25.0".format(score))
print()
print("MQM is now standard at WMT human evaluation.")

4 * Inter-annotator agreement#

def cohens_kappa(ann1, ann2):
    assert len(ann1) == len(ann2)
    n = len(ann1)
    labels = list(set(ann1) | set(ann2))
    agree = sum(a == b for a,b in zip(ann1, ann2)) / n
    p_e = sum(
        (ann1.count(l)/n) * (ann2.count(l)/n)
        for l in labels
    )
    if p_e >= 1.0:
        return 1.0
    return (agree - p_e) / (1 - p_e)

# Toy: two annotators rate 10 sentences as Good/OK/Bad
ann1 = ["Good","Good","OK","Bad","Good","OK","Bad","Good","OK","Bad"]
ann2 = ["Good","OK",  "OK","Bad","Good","Bad","Bad","Good","Good","Bad"]

kappa = cohens_kappa(ann1, ann2)
agree = sum(a==b for a,b in zip(ann1,ann2)) / len(ann1)
print("Inter-annotator agreement:")
print("  Raw agreement:  {:.1%}".format(agree))
print("  Cohen kappa:    {:.3f}".format(kappa))
print()
print("Kappa interpretation: <0.2 poor, 0.2-0.4 fair, 0.4-0.6 moderate,")
print("                      0.6-0.8 substantial, >0.8 almost perfect")
print()
print("MT human evaluation typically achieves kappa 0.3-0.5.")
print("FLORES annotators achieve 0.7+ via MQM guidelines + training.")

5 * Pitfalls#

  • Reference bias: annotators unconsciously prefer output that matches the reference.

  • Annotator fatigue: quality degrades after 30+ minutes per session.

  • Domain expertise: fluency judges miss content errors in specialised text.

  • Near-human MT: when output is very good, humans struggle to distinguish errors.

6 * MQM at WMT#

Since WMT21, MQM replaced Direct Assessment as the primary human evaluation. Annotators mark specific error spans with type and severity. This produces richer signal and higher inter-annotator agreement than holistic scoring.

Contents