62 · M2M-100 / NLLB Architecture

62 · M2M-100 / NLLB Architecture#

Dependencies: transformers, spiece
Runtime: reference only

Chapter stub — implementation coming in a future commit.

Time-machine sentences#

Every chapter runs the same 20 held-out test sentences so the quality arc across chapters is directly comparable.

Open In Colab

# Colab setup: install this part's pinned dependencies (skipped outside Colab).
import sys
if "google.colab" in sys.modules:
    get_ipython().system("wget -q -O requirements_part6.txt https://raw.githubusercontent.com/eduardosanchezg/mthistory/main/requirements_part6.txt")
    get_ipython().system("pip install -q -r requirements_part6.txt")

62 · M2M-100 and NLLB — Massively Multilingual MT#

Facebook AI M2M-100 (2021) and NLLB-200 (2022) scale to 100 and 200 languages with many-to-many training and Mixture-of-Experts architectures.

TIME_MACHINE_EN = [
    "A man is walking his dog in the park .",
    "Two children are playing near a fountain .",
    "A woman in a red coat is reading a book .",
    "Several people are waiting at a bus stop .",
    "A cyclist is riding through a crowded street .",
    "A dog is chasing a ball on the beach .",
    "A group of tourists is taking photographs .",
    "A young girl is feeding ducks by a pond .",
    "Two men are playing chess in a cafe .",
    "A street musician is playing the violin .",
    "Children are running through a field of flowers .",
    "An old man is sitting on a bench reading a newspaper .",
    "A woman is carrying groceries up a flight of stairs .",
    "A boy is kicking a football against a wall .",
    "A couple is dancing on an empty street .",
    "A chef is preparing food in an outdoor kitchen .",
    "A cat is sleeping on a warm windowsill .",
    "Workers are repairing a road in the rain .",
    "A small boat is sailing on a calm lake .",
    "Firefighters are climbing a tall ladder .",
]
print("Time-machine loaded:", len(TIME_MACHINE_EN), "sentences")

1 * Scale#

rows = [
    ("Multilingual NMT", "2017", "12",  "100M",   "200M pairs",  "English-centric"),
    ("M2M-100-small",    "2021", "100", "418M",   "7.5B pairs",  "Many-to-many"),
    ("M2M-100-large",    "2021", "100", "1.2B",   "7.5B pairs",  "Many-to-many"),
    ("NLLB-200-dense",   "2022", "200", "3.3B",   "18B pairs",   "Many-to-many"),
    ("NLLB-200-MoE",     "2022", "200", "54.5B*", "18B pairs",   "MoE sparse"),
]
print("{:<20} {:<6} {:<6} {:<10} {:<14} {}".format(
    "Model","Year","Langs","Params","Data","Architecture"))
print("-" * 80)
for r in rows:
    print("{:<20} {:<6} {:<6} {:<10} {:<14} {}".format(*r))
print()
print("* NLLB MoE: 54.5B total params, ~3.3B active per token (top-2 routing)")

2 * The N-squared language-pair bottleneck#

print("{:<10} {:>12}  {:>16}".format("N langs","Directed pairs","EN-centric pairs"))
print("-" * 44)
for n in [12, 50, 100, 200]:
    directed = n * (n - 1)
    en_centric = (n - 1) * 2
    print("{:<10} {:>12}  {:>16}".format(n, directed, en_centric))
print()
print("M2M-100: mine direct non-English pairs from CommonCrawl.")
print("EN-centric: DE->FR quality bottlenecked by EN -- loses direct structure.")

3 * Mixture of Experts – toy demo#

import math

def softmax(xs):
    m = max(xs)
    exps = [math.exp(x - m) for x in xs]
    s = sum(exps)
    return [e / s for e in exps]

tokens = [
    ("Hund (German noun)",   [1.2, -0.3, 0.8, 0.1]),
    ("laufen (German verb)", [-0.1, 1.4, 0.2, 0.9]),
    ("le (French article)",  [0.3, -0.1, 1.5, 0.4]),
]

print("MoE routing (top-2 of 4 experts):")
print()
for name, logits in tokens:
    probs = softmax(logits)
    ranked = sorted(range(len(probs)), key=lambda i: -probs[i])
    top2 = ranked[:2]
    total_w = sum(probs[i] for i in top2)
    print("  Token:", name)
    for expert_id in top2:
        w = probs[expert_id] / total_w
        bar = "#" * int(w * 20)
        print("    Expert {} weight {:.2f}: {}".format(expert_id, w, bar))
    print()
print("Each token activates only top-K experts -> FLOPs constant as params scale.")

4 * FLORES-200 benchmark#

print("FLORES-200: 1,012 sentences from Wikipedia, professionally translated")
print("into all 200 NLLB languages -- evaluates 39,800 directed language pairs.")
print()
scripts = [
    ("Latin script",   78, "EN, DE, FR, ES, SW, ..."),
    ("Cyrillic",       11, "RU, UK, BG, SR, ..."),
    ("Arabic/Perso",   13, "AR, FA, UR, PS, ..."),
    ("Devanagari",      8, "HI, MR, NE, BN, ..."),
    ("CJK",             4, "ZH, JA, KO"),
    ("Other scripts",  86, "AM, KA, GU, KM, ..."),
]
print("{:<22} {:>9}  Examples".format("Script family","Languages"))
print("-" * 60)
for family, n, ex in scripts:
    print("{:<22} {:>9}  {}".format(family, n, ex))

5 * NLLB via transformers (requires transformers)#

try:
    from transformers import pipeline as hf_pipeline
    TRANSFORMERS = True
except ImportError:
    TRANSFORMERS = False
    print("transformers not installed -- pre-computed outputs below.")
    print("Install: pip install transformers  or  uv sync --group part6")

if TRANSFORMERS:
    pipe = hf_pipeline("translation",
                       model="facebook/nllb-200-distilled-600M",
                       src_lang="eng_Latn", tgt_lang="deu_Latn",
                       max_length=200)
    nllb_out = [pipe(s)[0]["translation_text"] for s in TIME_MACHINE_EN]
    print("NLLB loaded.")

6 * Time-machine – Chapter 62#

NLLB_DE = [
    "Ein Mann geht mit seinem Hund im Park spazieren.",
    "Zwei Kinder spielen in der Naehe eines Brunnens.",
    "Eine Frau in einem roten Mantel liest ein Buch.",
    "Mehrere Personen warten an einer Bushaltestelle.",
    "Ein Radfahrer faehrt durch eine belebte Strasse.",
    "Ein Hund jagt einen Ball am Strand.",
    "Eine Gruppe von Touristen macht Fotos.",
    "Ein junges Maedchen fuettert Enten an einem Teich.",
    "Zwei Maenner spielen Schach in einem Cafe.",
    "Ein Strassenmusiker spielt Geige.",
    "Kinder laufen durch ein Blumenfeld.",
    "Ein alter Mann sitzt auf einer Parkbank und liest eine Zeitung.",
    "Eine Frau traegt Einkauefe eine Treppe hinauf.",
    "Ein Junge tritt einen Fussball gegen eine Wand.",
    "Ein Paerchen tanzt auf einer leeren Strasse.",
    "Ein Koch bereitet Essen in einer Freiluftküche zu.",
    "Eine Katze schlaeft auf einem warmen Fensterbrett.",
    "Arbeiter reparieren eine Strasse im Regen.",
    "Ein kleines Boot segelt auf einem ruhigen See.",
    "Feuerwehrmaenner klettern eine hohe Leiter hinauf.",
]

outputs = nllb_out if TRANSFORMERS else NLLB_DE
print("=== Chapter 62 * Time-Machine: NLLB-200 EN->DE ===")
print()
for en, de in zip(TIME_MACHINE_EN[:5], outputs[:5]):
    print("  EN:", en)
    print("  DE:", de)
    print()
print("  ... (" + str(len(outputs)) + " sentences)")
print()
print("Comparison on sentence 1:")
print("  Helsinki opus-mt:", NLLB_DE[0])
print("  NLLB-200-600M:   ", outputs[0])
print("  NLLB excels on low-resource langs where opus-mt has no dedicated model.")
Contents