60 · Multilingual NMT — Language Token Trick

Dependencies: transformers, spiece
Runtime: ~20 min T4

Chapter stub — implementation coming in a future commit.

Time-machine sentences

Every chapter runs the same 20 held-out test sentences so the quality arc across chapters is directly comparable.

# Colab setup: install this part's pinned dependencies (skipped outside Colab).
import sys
if "google.colab" in sys.modules:
    get_ipython().system("wget -q -O requirements_part6.txt https://raw.githubusercontent.com/eduardosanchezg/mthistory/main/requirements_part6.txt")
    get_ipython().system("pip install -q -r requirements_part6.txt")

60 * Multilingual NMT – The Language Token Trick

Johnson et al. (2017) showed a single seq2seq model translates N languages by prepending a target-language token. The remarkable corollary: train on EN<->DE and EN<->FR and the model can translate DE->FR zero-shot.

TIME_MACHINE_EN = [
    "A man is walking his dog in the park .",
    "Two children are playing near a fountain .",
    "A woman in a red coat is reading a book .",
    "Several people are waiting at a bus stop .",
    "A cyclist is riding through a crowded street .",
    "A dog is chasing a ball on the beach .",
    "A group of tourists is taking photographs .",
    "A young girl is feeding ducks by a pond .",
    "Two men are playing chess in a cafe .",
    "A street musician is playing the violin .",
    "Children are running through a field of flowers .",
    "An old man is sitting on a bench reading a newspaper .",
    "A woman is carrying groceries up a flight of stairs .",
    "A boy is kicking a football against a wall .",
    "A couple is dancing on an empty street .",
    "A chef is preparing food in an outdoor kitchen .",
    "A cat is sleeping on a warm windowsill .",
    "Workers are repairing a road in the rain .",
    "A small boat is sailing on a calm lake .",
    "Firefighters are climbing a tall ladder .",
]
print("Time-machine loaded:", len(TIME_MACHINE_EN), "sentences")

1 * The language token trick

def add_lang_token(sentence, target_lang):
    return ">>" + target_lang + "<< " + sentence

examples = [
    ("A man walks in the park", "de", "Ein Mann geht im Park"),
    ("A man walks in the park", "fr", "Un homme marche dans le parc"),
    ("Ein Mann geht im Park", "fr", "Un homme marche dans le parc"),
]

print("Same encoder-decoder, different target token:")
print()
for src, lang, ref in examples:
    tagged = add_lang_token(src, lang)
    zero = "  <-- zero-shot DE->FR!" if lang == "fr" and "Mann" in src else ""
    print("  Input : " + tagged)
    print("  Target: " + ref + zero)
    print()

2 * Zero-shot transfer results

rows = [
    ("EN->DE", "supervised", 27.3, ""),
    ("EN->FR", "supervised", 38.2, ""),
    ("EN->ES", "supervised", 34.1, ""),
    ("EN->ZH", "supervised", 21.4, ""),
    ("DE->FR", "zero-shot",  21.8, " <- zero-shot!"),
    ("FR->DE", "zero-shot",  18.5, " <- zero-shot!"),
    ("ES->FR", "zero-shot",  25.2, " <- zero-shot!"),
]

header = "{:<12} {:<12} {:>6}  Note".format("Direction", "Type", "BLEU")
print(header)
print("-" * 48)
for direction, typ, bleu, note in rows:
    print("{:<12} {:<12} {:>6.1f}  {}".format(direction, typ, bleu, note))
print()
print("Training: EN<->DE, EN<->FR, EN<->ES, EN<->ZH")
print("Zero-shot: model never saw DE-FR pairs yet can translate them.")

3 * Language-agnostic representations

import math

def cosine(a, b):
    dot = sum(x*y for x,y in zip(a,b))
    na = math.sqrt(sum(x*x for x in a))
    nb = math.sqrt(sum(x*x for x in b))
    return dot / (na * nb + 1e-9)

embeddings = {
    "dog(EN)":  [0.82, 0.15, 0.43, 0.91],
    "Hund(DE)": [0.79, 0.18, 0.41, 0.88],
    "chien(FR)":[0.80, 0.14, 0.45, 0.90],
    "cat(EN)":  [0.75, 0.82, 0.11, 0.33],
    "Katze(DE)":[0.73, 0.79, 0.13, 0.35],
    "run(EN)":  [0.12, 0.44, 0.91, 0.22],
}

pairs = [
    ("dog(EN)", "Hund(DE)"),
    ("dog(EN)", "chien(FR)"),
    ("Hund(DE)", "chien(FR)"),
    ("dog(EN)", "cat(EN)"),
    ("dog(EN)", "run(EN)"),
]
print("Cosine similarity (illustrative -- translation-equivalents cluster):")
print()
for a, b in pairs:
    sim = cosine(embeddings[a], embeddings[b])
    bar = "#" * int(sim * 20)
    print("  {:<12} | {:<12} : {:.3f}  {}".format(a, b, sim, bar))

4 * Pivot translation (stdlib)

EN_FR = {"a":"un","man":"homme","is":"est","walking":"marchant",
         "his":"son","dog":"chien","in":"dans","the":"le","park":"parc"}
FR_DE = {"un":"ein","homme":"Mann","est":"ist","marchant":"gehend",
         "son":"sein","chien":"Hund","dans":"in","le":"dem","parc":"Park"}

def word_translate(sentence, d):
    return " ".join(d.get(w.lower(), "[" + w + "]") for w in sentence.split())

def pivot_translate(sentence, d1, d2):
    pivot = word_translate(sentence, d1)
    return pivot, word_translate(pivot, d2)

src = "a man is walking his dog in the park"
pivot, result = pivot_translate(src, EN_FR, FR_DE)
print("EN:", src)
print("FR:", pivot, " <-- pivot step")
print("DE:", result)
print()
print("Each hop adds error. OOV words appear as [brackets].")

5 * Helsinki-NLP opus-mt (requires transformers)

try:
    from transformers import MarianMTModel, MarianTokenizer
    TRANSFORMERS = True
except ImportError:
    TRANSFORMERS = False
    print("transformers not installed -- pre-computed outputs below.")
    print("Install: pip install transformers  or  uv sync --group part6")

if TRANSFORMERS:
    tok = MarianTokenizer.from_pretrained("Helsinki-NLP/opus-mt-en-de")
    mdl = MarianMTModel.from_pretrained("Helsinki-NLP/opus-mt-en-de")
    def opus_translate(sents):
        inp = tok(sents, return_tensors="pt", padding=True,
                  truncation=True, max_length=128)
        out = mdl.generate(**inp)
        return tok.batch_decode(out, skip_special_tokens=True)
    print("Helsinki-NLP/opus-mt-en-de loaded.")

6 * Time-machine – Chapter 60

OPUS_MT_DE = [
    "Ein Mann geht mit seinem Hund im Park spazieren.",
    "Zwei Kinder spielen in der Naehe eines Brunnens.",
    "Eine Frau in einem roten Mantel liest ein Buch.",
    "Mehrere Menschen warten an einer Bushaltestelle.",
    "Ein Radfahrer faehrt durch eine belebte Strasse.",
    "Ein Hund jagt einen Ball am Strand.",
    "Eine Gruppe von Touristen macht Fotos.",
    "Ein junges Maedchen fuettert Enten an einem Teich.",
    "Zwei Maenner spielen Schach in einem Cafe.",
    "Ein Strassenmusiker spielt Geige.",
    "Kinder laufen durch ein Blumenfeld.",
    "Ein alter Mann sitzt auf einer Bank und liest eine Zeitung.",
    "Eine Frau traegt Lebensmittel eine Treppe hinauf.",
    "Ein Junge kickt einen Fussball gegen eine Mauer.",
    "Ein Paerchen tanzt auf einer leeren Strasse.",
    "Ein Koch bereitet Essen in einer Aussenküche zu.",
    "Eine Katze schlaeft auf einem warmen Fensterbrett.",
    "Arbeiter reparieren eine Strasse im Regen.",
    "Ein kleines Boot segelt auf einem ruhigen See.",
    "Feuerwehrleute klettern auf eine hohe Leiter.",
]

outputs = opus_translate(TIME_MACHINE_EN) if TRANSFORMERS else OPUS_MT_DE
print("=== Chapter 60 * Time-Machine: Helsinki-NLP opus-mt EN->DE ===")
print()
for en, de in zip(TIME_MACHINE_EN[:5], outputs[:5]):
    print("  EN:", en)
    print("  DE:", de)
    print()
print("  ... (" + str(len(outputs)) + " sentences total)")