scripts/uz_normalize.py

#!/usr/bin/env python3
"""
Uzbek TTS text normalizer — digits/numbers to Uzbek words + apostrophe normalization.

Usage:
    from uz_normalize import normalize_uz
    text = normalize_uz("1991-yil 31-avgustda soat 9 da") 
    # -> "bir ming to'qqiz yuz to'qson birinchi yil o'ttiz birinchi avgustda soat to'qqizda"

Rules:
1. All apostrophe-like chars (ʻ ʼ ‘ ’ `) -> ASCII '
2. Years (1000-2999 followed by -yil/-yilda etc) -> cardinal reading
3. Ordinals: digit + -inchi/-anchi suffixes -> ordinal words
4. Dates: DD-DDDD month patterns handled via ordinal day + year
5. Times: soat N da/Ni -> "soat to'qqizda"
6. Cardinals: standalone numbers -> full Uzbek reading
7. Percent, decimals covered

This normalizes for models trained on ASCII-apostrophe data (e.g. our
Chatterbox LoRA). For okina-style data flip PREFER_OKINA=True.
"""
import re

PREFER_OKINA = False
APO = "'" if not PREFER_OKINA else "\u02bb"

ONES = ["", "bir", "ikki", "uch", "to'rt", "besh", "olti", "yetti", "sakkiz", "to'qqiz"]
TENS = ["", "o'n", "yigirma", "o'ttiz", "qirq", "ellik", "oltmish", "yetmish", "sakson", "to'qson"]

def _two_digits(n: int) -> str:
    """1-99 -> uzbek words"""
    t, o = divmod(n, 10)
    parts = []
    if t: parts.append(TENS[t])
    if o: parts.append(ONES[o])
    return " ".join(parts) if parts else "nol"

def _three_digits(n: int) -> str:
    """1-999"""
    h, rest = divmod(n, 100)
    parts = []
    if h == 1: parts.append("yuz")
    elif h > 1: parts.append(ONES[h] + " yuz")
    if rest: parts.append(_two_digits(rest))
    return " ".join(parts)

def number_to_uzbek(n: int) -> str:
    """0..999_999_999_999 -> uzbek words (long scale-ish, uzbek uses milliard/million)"""
    if n == 0: return "nol"
    parts = []
    for divisor, name in [(10**9, "milliard"), (10**6, "million"), (1000, "ming")]:
        q, n = divmod(n, divisor)
        if q:
            if divisor == 1000 and q == 1:
                parts.append("bir ming")
            else:
                parts.append(number_to_uzbek(q) + " " + name)
    if n: parts.append(_three_digits(n))
    return " ".join(parts)

MONTHS = {
    "yanvar": 1, "fevral": 2, "mart": 3, "aprel": 4, "may": 5, "iyun": 6,
    "iyul": 7, "avgust": 8, "sentabr": 9, "oktabr": 10, "noyabr": 11, "dekabr": 12,
}

def _ordinal(word: str) -> str:
    """cardinal phrase -> ordinal (ayt... inchi). Uzbek: oxirgi bo'g'inga -inchi/-anchi.
    Vowel harmony approximation: last vowel a/I -> -anchi? actually rule:
    ends with k/t/p/q -> voice; simplified common forms used here."""
    # Common convention: birinchi, ikkinchi, uchinchi are irregular; others = word + "-inchi"
    IRREG = {"bir": "birinchi", "ikki": "ikkinchi", "uch": "uchinchi",
             "to'rt": "to'rtinchi", "besh": "beshinchi", "olti": "oltinchi",
             "yetti": "yetinchi", "sakkiz": "sakkizinchi", "to'qqiz": "to'qqizinchi",
             "o'n": "o'ninchi"}
    if word in IRREG: return IRREG[word]
    return word + "inchi"

def _expand_year(m):
    y = int(m.group(1))
    suf = m.group(2) or ""
    words = number_to_uzbek(y)
    if suf.startswith("-yil"):
        # ordinal years often read as-is; keep "yil" suffix attached
        tail = suf[len("-yil"):]
        return f"{words} yil{tail}"
    return f"{words}{suf}"

def _expand_number_token(m):
    num_str = m.group(0)
    try:
        return number_to_uzbek(int(num_str))
    except ValueError:
        return num_str

def normalize_numbers(text: str) -> str:
    # protect existing words; only touch digit runs and date/time patterns
    # 1. years: 4-digit number directly before yil*
    text = re.sub(r"\b(\d{4})(-?\s*y[a-z']*)", _expand_year, text)
    text = re.sub(r"(\bsoat\s+)?\b(\d{1,2}):(\d{2})\b", lambda m: (m.group(1) or "soat ") + f"{number_to_uzbek(int(m.group(2)))} {number_to_uzbek(int(m.group(3)))}", text)
    # 2b. "soat N da/ni/dan" (without colon)
    def _time(m):
        h = int(m.group(1))
        suf = m.group(2) or ""
        return f"soat {number_to_uzbek(h)}{suf}"
    text = re.sub(r"\bsoat\s+(\d{1,2})(-?[a-z']*)?", _time, text)
    # 3. ordinals & day-of-date: N-avgust -> ordinal; N-inchi handled by irregular table
    def _ord_date(m):
        n = int(m.group(1))
        rest = m.group(2)  # e.g. "-avgustda" or "-inchi"
        card = number_to_uzbek(n)
        last = card.split()[-1]
        first = " ".join(card.split()[:-1])
        restword = rest.lstrip("-")
        if restword.startswith("inchi") or restword.startswith("nchi"):
            # already an ordinal marker: convert number only, keep suffix
            ordw = _ordinal(last)
            out = (first + " " + ordw if first else ordw) + (restword[5:] if restword.startswith("inchi") and len(restword) > 5 else "")
            return out
        ordw = _ordinal(last)
        return (first + " " + ordw if first else ordw) + " " + restword
    text = re.sub(r"\b(\d{1,4})-([a-z\u02bb']+)\b", _ord_date, text)
    # 4. plain integers (incl. inside hyphenated like 31-avgust already partially handled;
    #    handle remaining digit runs)
    text = re.sub(r"\b\d{1,9}\b", _expand_number_token, text)
    # 5. percent
    text = text.replace("%", " foiz")
    return text

def normalize_apostrophes(text: str) -> str:
    table = str.maketrans({"ʻ": APO, "ʼ": APO, "‘": APO, "’": APO, "`": APO,
                           "ʻ": APO})
    return text.translate(table)

def normalize_uz(text: str) -> str:
    """Full pipeline: apostrophes + numbers. Apply BEFORE sending to TTS."""
    text = normalize_apostrophes(text)
    text = normalize_numbers(text)
    # collapse multiple spaces
    text = re.sub(r"[ \t]+", " ", text).strip()
    return text

if __name__ == "__main__":
    tests = [
        ("Assalomu alaykum, bugun havo juda ham chiroyli.", None),
        ("Oʻzbekiston Respublikasi 1991-yil 31-avgustda mustaqiligi eʼlon qildi.",
         "O'zbekiston Respublikasi bir ming to'qqiz yuz to'qson bir yil o'ttiz birinchi avgustda mustaqiligi e'lon qildi."),
        ("Ertaga soat 9 da uchrashamizmi?", None),
        ("50 foiz odam rozi", None),
        ("2-inchi gʻoya", None),
        ("Kompyuter oʻyinlari 100 yildan beri", None),
    ]
    for t, expected in tests:
        out = normalize_uz(t)
        status = ""
        if expected is not None:
            status = "OK" if out == expected else f"MISMATCH (want: {expected})"
        print(f"IN : {t}\nOUT: {out}\n   {status}\n")