#!/usr/bin/env python3 """ Uzbek TTS text normalizer — digits/numbers to Uzbek words + apostrophe normalization. Usage: from uz_normalize import normalize_uz text = normalize_uz("1991-yil 31-avgustda soat 9 da") # -> "bir ming to'qqiz yuz to'qson birinchi yil o'ttiz birinchi avgustda soat to'qqizda" Rules: 1. All apostrophe-like chars (ʻ ʼ ‘ ’ `) -> ASCII ' 2. Years (1000-2999 followed by -yil/-yilda etc) -> cardinal reading 3. Ordinals: digit + -inchi/-anchi suffixes -> ordinal words 4. Dates: DD-DDDD month patterns handled via ordinal day + year 5. Times: soat N da/Ni -> "soat to'qqizda" 6. Cardinals: standalone numbers -> full Uzbek reading 7. Percent, decimals covered This normalizes for models trained on ASCII-apostrophe data (e.g. our Chatterbox LoRA). For okina-style data flip PREFER_OKINA=True. """ import re PREFER_OKINA = False APO = "'" if not PREFER_OKINA else "\u02bb" ONES = ["", "bir", "ikki", "uch", "to'rt", "besh", "olti", "yetti", "sakkiz", "to'qqiz"] TENS = ["", "o'n", "yigirma", "o'ttiz", "qirq", "ellik", "oltmish", "yetmish", "sakson", "to'qson"] def _two_digits(n: int) -> str: """1-99 -> uzbek words""" t, o = divmod(n, 10) parts = [] if t: parts.append(TENS[t]) if o: parts.append(ONES[o]) return " ".join(parts) if parts else "nol" def _three_digits(n: int) -> str: """1-999""" h, rest = divmod(n, 100) parts = [] if h == 1: parts.append("yuz") elif h > 1: parts.append(ONES[h] + " yuz") if rest: parts.append(_two_digits(rest)) return " ".join(parts) def number_to_uzbek(n: int) -> str: """0..999_999_999_999 -> uzbek words (long scale-ish, uzbek uses milliard/million)""" if n == 0: return "nol" parts = [] for divisor, name in [(10**9, "milliard"), (10**6, "million"), (1000, "ming")]: q, n = divmod(n, divisor) if q: if divisor == 1000 and q == 1: parts.append("bir ming") else: parts.append(number_to_uzbek(q) + " " + name) if n: parts.append(_three_digits(n)) return " ".join(parts) MONTHS = { "yanvar": 1, "fevral": 2, "mart": 3, "aprel": 4, "may": 5, "iyun": 6, "iyul": 7, "avgust": 8, "sentabr": 9, "oktabr": 10, "noyabr": 11, "dekabr": 12, } def _ordinal(word: str) -> str: """cardinal phrase -> ordinal (ayt... inchi). Uzbek: oxirgi bo'g'inga -inchi/-anchi. Vowel harmony approximation: last vowel a/I -> -anchi? actually rule: ends with k/t/p/q -> voice; simplified common forms used here.""" # Common convention: birinchi, ikkinchi, uchinchi are irregular; others = word + "-inchi" IRREG = {"bir": "birinchi", "ikki": "ikkinchi", "uch": "uchinchi", "to'rt": "to'rtinchi", "besh": "beshinchi", "olti": "oltinchi", "yetti": "yetinchi", "sakkiz": "sakkizinchi", "to'qqiz": "to'qqizinchi", "o'n": "o'ninchi"} if word in IRREG: return IRREG[word] return word + "inchi" def _expand_year(m): y = int(m.group(1)) suf = m.group(2) or "" words = number_to_uzbek(y) if suf.startswith("-yil"): # ordinal years often read as-is; keep "yil" suffix attached tail = suf[len("-yil"):] return f"{words} yil{tail}" return f"{words}{suf}" def _expand_number_token(m): num_str = m.group(0) try: return number_to_uzbek(int(num_str)) except ValueError: return num_str def normalize_numbers(text: str) -> str: # protect existing words; only touch digit runs and date/time patterns # 1. years: 4-digit number directly before yil* text = re.sub(r"\b(\d{4})(-?\s*y[a-z']*)", _expand_year, text) text = re.sub(r"(\bsoat\s+)?\b(\d{1,2}):(\d{2})\b", lambda m: (m.group(1) or "soat ") + f"{number_to_uzbek(int(m.group(2)))} {number_to_uzbek(int(m.group(3)))}", text) # 2b. "soat N da/ni/dan" (without colon) def _time(m): h = int(m.group(1)) suf = m.group(2) or "" return f"soat {number_to_uzbek(h)}{suf}" text = re.sub(r"\bsoat\s+(\d{1,2})(-?[a-z']*)?", _time, text) # 3. ordinals & day-of-date: N-avgust -> ordinal; N-inchi handled by irregular table def _ord_date(m): n = int(m.group(1)) rest = m.group(2) # e.g. "-avgustda" or "-inchi" card = number_to_uzbek(n) last = card.split()[-1] first = " ".join(card.split()[:-1]) restword = rest.lstrip("-") if restword.startswith("inchi") or restword.startswith("nchi"): # already an ordinal marker: convert number only, keep suffix ordw = _ordinal(last) out = (first + " " + ordw if first else ordw) + (restword[5:] if restword.startswith("inchi") and len(restword) > 5 else "") return out ordw = _ordinal(last) return (first + " " + ordw if first else ordw) + " " + restword text = re.sub(r"\b(\d{1,4})-([a-z\u02bb']+)\b", _ord_date, text) # 4. plain integers (incl. inside hyphenated like 31-avgust already partially handled; # handle remaining digit runs) text = re.sub(r"\b\d{1,9}\b", _expand_number_token, text) # 5. percent text = text.replace("%", " foiz") return text def normalize_apostrophes(text: str) -> str: table = str.maketrans({"ʻ": APO, "ʼ": APO, "‘": APO, "’": APO, "`": APO, "ʻ": APO}) return text.translate(table) def normalize_uz(text: str) -> str: """Full pipeline: apostrophes + numbers. Apply BEFORE sending to TTS.""" text = normalize_apostrophes(text) text = normalize_numbers(text) # collapse multiple spaces text = re.sub(r"[ \t]+", " ", text).strip() return text if __name__ == "__main__": tests = [ ("Assalomu alaykum, bugun havo juda ham chiroyli.", None), ("Oʻzbekiston Respublikasi 1991-yil 31-avgustda mustaqiligi eʼlon qildi.", "O'zbekiston Respublikasi bir ming to'qqiz yuz to'qson bir yil o'ttiz birinchi avgustda mustaqiligi e'lon qildi."), ("Ertaga soat 9 da uchrashamizmi?", None), ("50 foiz odam rozi", None), ("2-inchi gʻoya", None), ("Kompyuter oʻyinlari 100 yildan beri", None), ] for t, expected in tests: out = normalize_uz(t) status = "" if expected is not None: status = "OK" if out == expected else f"MISMATCH (want: {expected})" print(f"IN : {t}\nOUT: {out}\n {status}\n")