61 · Low-Resource MT — Pivoting & Data Aug.

Dependencies: transformers, spiece
Runtime: ~20 min T4

Chapter stub — implementation coming in a future commit.

Time-machine sentences

Every chapter runs the same 20 held-out test sentences so the quality arc across chapters is directly comparable.

# Colab setup: install this part's pinned dependencies (skipped outside Colab).
import sys
if "google.colab" in sys.modules:
    get_ipython().system("wget -q -O requirements_part6.txt https://raw.githubusercontent.com/eduardosanchezg/mthistory/main/requirements_part6.txt")
    get_ipython().system("pip install -q -r requirements_part6.txt")

61 · Low-Resource MT — Pivoting, Transfer, Data Augmentation

Standard Transformer degrades sharply below ~100K sentence pairs. This chapter covers three strategies: pivoting, transfer learning, back-translation.

TIME_MACHINE_EN = [
    "A man is walking his dog in the park .",
    "Two children are playing near a fountain .",
    "A woman in a red coat is reading a book .",
    "Several people are waiting at a bus stop .",
    "A cyclist is riding through a crowded street .",
    "A dog is chasing a ball on the beach .",
    "A group of tourists is taking photographs .",
    "A young girl is feeding ducks by a pond .",
    "Two men are playing chess in a cafe .",
    "A street musician is playing the violin .",
    "Children are running through a field of flowers .",
    "An old man is sitting on a bench reading a newspaper .",
    "A woman is carrying groceries up a flight of stairs .",
    "A boy is kicking a football against a wall .",
    "A couple is dancing on an empty street .",
    "A chef is preparing food in an outdoor kitchen .",
    "A cat is sleeping on a warm windowsill .",
    "Workers are repairing a road in the rain .",
    "A small boat is sailing on a calm lake .",
    "Firefighters are climbing a tall ladder .",
]
print("Time-machine loaded:", len(TIME_MACHINE_EN), "sentences")

1 * Defining low-resource

rows = [
    ("High",      "> 1M pairs",   "EN-DE, EN-FR, EN-ZH",       ">28"),
    ("Medium",    "100K-1M",      "EN-TR, EN-AR, EN-RU",       "18-25"),
    ("Low",       "10K-100K",     "EN-Swahili, EN-Nepali",     "5-14"),
    ("Very low",  "< 10K pairs",  "EN-Guarani, EN-Tigrinya",   "1-4"),
]
print("{:<10} {:<16} {:<30} {}".format("Category","Data size","Examples","BLEU"))
print("-" * 70)
for cat, size, ex, bleu in rows:
    print("{:<10} {:<16} {:<30} {}".format(cat, size, ex, bleu))
print()
print("Standard Transformer degrades sharply below ~100K pairs.")

2 * Pivoting

EN_SW = {"a":"mtu","man":"mwanaume","is":"ni","walking":"kutembea",
         "his":"wake","dog":"mbwa","in":"katika","the":"hiyo","park":"bustani"}
SW_DE = {"mtu":"Person","mwanaume":"Mann","ni":"ist","kutembea":"gehend",
         "wake":"sein","mbwa":"Hund","katika":"in","hiyo":"dem","bustani":"Park"}

def dict_tr(sent, d):
    return " ".join(d.get(w.lower(), "[" + w + "?]") for w in sent.split())

tests = [
    "a man is walking his dog in the park",
    "a dog is in the park",
]
print("Pivot via Swahili (EN -> SW -> DE):")
print()
for src in tests:
    sw = dict_tr(src, EN_SW)
    de = dict_tr(sw, SW_DE)
    oov1 = sw.count("[")
    oov2 = de.count("[")
    print("  EN:", src)
    print("  SW:", sw, " (OOV=" + str(oov1) + ")")
    print("  DE:", de, " (OOV=" + str(oov2) + ")")
    print()
print("Each hop adds error -- dedicated EN-DE avoids cascaded noise.")

3 * Transfer learning from high-resource parent

def vocab_overlap(v1, v2):
    s1, s2 = set(v1), set(v2)
    return len(s1 & s2) / (len(s1 | s2) + 1e-9)

de_vocab = ["der","die","das","Mann","Frau","gehen","laufen","Haus",
            "Auto","gross","klein","sein","haben","mit","und","nicht"]
sw_vocab = ["mtu","mwanamke","kwenda","nyumba","gari","kubwa","ndogo",
            "kuwa","na","au","si","sana","mbwa","mwanaume","kucheza"]
nl_vocab = ["de","het","man","vrouw","gaan","lopen","huis","auto",
            "groot","klein","zijn","hebben","met","en","of","niet"]

print("Vocabulary overlap with German parent model:")
print("  DE <-> Swahili (unrelated): {:.1%}".format(vocab_overlap(de_vocab, sw_vocab)))
print("  DE <-> Dutch   (related):   {:.1%}".format(vocab_overlap(de_vocab, nl_vocab)))
print()
print("Recipe:")
print("  1. Train parent model on EN-DE (4M pairs)")
print("  2. Init child (EN-Swahili) from parent weights")
print("  3. Fine-tune on 50K Swahili pairs")
print("  Higher vocab overlap -> better transfer -> higher BLEU")

4 * Back-translation

import random
random.seed(7)

def back_translate(target_mono, reverse_model, original_parallel):
    synthetic_src = [reverse_model(s) for s in target_mono]
    return original_parallel + list(zip(synthetic_src, target_mono))

original = [
    ("A dog runs.", "Ein Hund laeuft."),
    ("A man walks.", "Ein Mann geht."),
    ("A woman reads.", "Eine Frau liest."),
    ("The cat sleeps.", "Die Katze schlaeft."),
    ("Children play.", "Kinder spielen."),
]

de_mono = [
    "Ein Hund bellt im Park.",
    "Zwei Maenner spielen Schach.",
    "Die Frau liest ein Buch.",
]

DE_EN = {"ein":"a","eine":"a","zwei":"two","der":"the","die":"the",
         "das":"the","hund":"dog","mann":"man","maenner":"men",
         "frau":"woman","bellt":"barks","spielen":"play","liest":"reads",
         "buch":"book","schach":"chess","im":"in","park":"park."}

def toy_rev(sent):
    return " ".join(DE_EN.get(w.lower().rstrip("."), "[" + w + "]")
                    for w in sent.split())

aug = back_translate(de_mono, toy_rev, original)
print("Original parallel:", len(original), "pairs")
print("Target monolingual:", len(de_mono), "sentences")
print("Augmented corpus:", len(aug), "pairs")
print("Growth:", "{:.1f}x".format(len(aug)/len(original)))
print()
print("Synthetic back-translated pairs:")
for src, tgt in aug[len(original):]:
    print("  SRC:", src)
    print("  TGT:", tgt)
    print()

5 * Data augmentation

import random
random.seed(3)

SYNONYMS = {
    "walking":["strolling","wandering"],"running":["sprinting","jogging"],
    "dog":["canine","pup"],"park":["garden","square"],
    "man":["person","individual"],"woman":["lady","person"],
}

def word_dropout(sentence, p=0.1):
    words = sentence.split()
    kept = [w for w in words if random.random() > p]
    return " ".join(kept) if kept else sentence

def word_replace(sentence, syns, p=0.15):
    out = []
    for w in sentence.split():
        if random.random() < p and w.lower() in syns:
            out.append(random.choice(syns[w.lower()]))
        else:
            out.append(w)
    return " ".join(out)

sample = "A man is walking his dog in the park"
print("Original:", sample)
print()
for i in range(3):
    print("  Dropout " + str(i+1) + ":", word_dropout(sample, 0.15))
    print("  Replace " + str(i+1) + ":", word_replace(sample, SYNONYMS))
    print()

6 * Time-machine – Chapter 61

sizes = [
    ("1K pairs",   2.1,  "Ein Mann laufen Park [errors]"),
    ("10K pairs",  8.4,  "Ein Mann geht mit Hund im Park."),
    ("50K pairs",  15.6, "Ein Mann geht mit seinem Hund im Park."),
    ("100K pairs", 21.2, "Ein Mann geht mit seinem Hund spazieren."),
    ("29K + BT",   18.3, "Ein Mann geht mit seinem Hund im Park spazieren."),
]

print("Source: A man is walking his dog in the park .")
print()
print("{:<20} {:>6}  Sample output".format("Training data", "BLEU"))
print("-" * 72)
for size, bleu, sample in sizes:
    print("{:<20} {:>6.1f}  {}".format(size, bleu, sample))
print()
print("Quality scales log-linearly with data.")
print("Back-translation (BT) can double effective data at low resource.")