Text Processing Engineering: Unicode Segmentation, Locale-Aware Case Mapping, and Readability Metrics

Do NOT break:
  - Between a base character and combining marks (e + ◌́ = é)
  - Between Regional Indicator pairs (🇯 + 🇵 = 🇯🇵)
  - In emoji ZWJ sequences (👨 + ZWJ + 👩 + ZWJ + 👧 = 👨‍👩‍👧)
  - Between Hangul jamo in a syllable block
  - Between CR and LF

DO break:
  - Between every other pair of characters

// Correct: Intl.Segmenter (UAX #29 compliant)
function graphemes(str) {
  const segmenter = new Intl.Segmenter('en', { granularity: 'grapheme' });
  return [...segmenter.segment(str)].map(s => s.segment);
}

graphemes("👨‍👩‍👧‍👦")  // ["👨‍👩‍👧‍👦"] — 1 grapheme
graphemes("café")        // ["c", "a", "f", "é"] — 4 graphemes (NFC)

// Incorrect: spread operator (splits on code points, not graphemes)
[..."👨‍👩‍👧‍👦"]  // ["👨", "‍", "👩", "‍", "👧", "‍", "👦"] — 7 code points

const segmenter = new Intl.Segmenter('en', { granularity: 'word' });
const words = [...segmenter.segment("I can't believe it's 3:30pm")]
  .filter(s => s.isWordLike)
  .map(s => s.segment);
// ["I", "can't", "believe", "it's", "3:30pm"]

// For CJK: requires dictionary-based segmentation
const zhSegmenter = new Intl.Segmenter('zh', { granularity: 'word' });
const zhWords = [...zhSegmenter.segment("今天天气很好")]
  .filter(s => s.isWordLike)
  .map(s => s.segment);
// ["今天", "天气", "很", "好"] (dictionary-based)

"Dr. Smith earned $3.5M in Q4. Impressive!"
  → 2 sentences (not 4, despite 4 periods)

Rules:
  - Don't break after abbreviations (Dr., Mr., U.S., etc.)
  - Don't break inside numbers (3.14, $1,000.00)
  - Don't break inside ellipsis (...)
  - DO break after sentence-terminal punctuation followed by uppercase

Turkish rules:
  i → İ (U+0130, LATIN CAPITAL LETTER I WITH DOT ABOVE)
  I → ı (U+0131, LATIN SMALL LETTER DOTLESS I)

English rules:
  i → I
  I → i

// JavaScript: locale-sensitive
"FILE".toLocaleLowerCase('tr')  // "fıle" (dotless i)
"FILE".toLocaleLowerCase('en')  // "file"

// Python: casefold() is locale-independent
"straße".casefold()  // "strasse" (German ß → ss)
"straße".lower()     // "straße" (lower() preserves ß)

text = "straße"
print(len(text))               # 6
print(len(text.upper()))       # 7 ("STRASSE")
print(len(text.casefold()))    # 7 ("strasse")

1. Normalize to NFD
2. Apply case folding (toCasefold mapping)
3. Normalize to NFD again (case folding may introduce decomposable characters)
4. Compare code point by code point

import unicodedata

def case_insensitive_equal(a, b):
    def normalize(s):
        s = unicodedata.normalize('NFD', s)
        s = s.casefold()
        s = unicodedata.normalize('NFD', s)
        return s
    return normalize(a) == normalize(b)

case_insensitive_equal("straße", "STRASSE")  # True
case_insensitive_equal("file", "FILE")        # True (compatibility casefold)

import re

def split_identifier(name: str) -> list[str]:
    """Split an identifier into word components.
    
    Handles: camelCase, PascalCase, snake_case, kebab-case,
    SCREAMING_SNAKE, dot.case, and mixed patterns like HTMLParser.
    """
    # Insert boundary before uppercase letters following lowercase
    # But keep consecutive uppercase together (HTML → [HTML] not [H,T,M,L])
    name = re.sub(r'([a-z])([A-Z])', r'\1_\2', name)
    name = re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', name)
    
    # Split on non-alphanumeric characters
    return [w for w in re.split(r'[_\-.\s/]+', name) if w]

def to_camel(words: list[str]) -> str:
    return words[0].lower() + ''.join(w.capitalize() for w in words[1:])

def to_snake(words: list[str]) -> str:
    return '_'.join(w.lower() for w in words)

def to_kebab(words: list[str]) -> str:
    return '-'.join(w.lower() for w in words)

# Examples:
split_identifier("HTMLParser")      # ["HTML", "Parser"]
split_identifier("getHTTPResponse") # ["get", "HTTP", "Response"]
split_identifier("my-kebab-case")   # ["my", "kebab", "case"]

FKGL = 0.39 × (total words / total sentences)
     + 11.8 × (total syllables / total words)
     − 15.59

FRE = 206.835
    − 1.015 × (total words / total sentences)
    − 84.6 × (total syllables / total words)

CLI = 0.0588 × L − 0.296 × S − 15.8

Where:
  L = average number of characters per 100 words
  S = average number of sentences per 100 words

def count_syllables(word: str) -> int:
    word = word.lower().strip()
    if len(word) <= 3:
        return 1
    
    # Remove silent e
    if word.endswith('e') and not word.endswith('le'):
        word = word[:-1]
    
    # Count vowel groups
    vowels = 'aeiouy'
    count = 0
    prev_vowel = False
    for char in word:
        is_vowel = char in vowels
        if is_vowel and not prev_vowel:
            count += 1
        prev_vowel = is_vowel
    
    return max(1, count)

# Approximation accuracy: ~85% for English
# For production use: CMU Pronouncing Dictionary or phonetic lookup

import unicodedata
import re

def slugify(text: str, max_length: int = 80) -> str:
    # NFKD decomposition separates accents from base characters
    text = unicodedata.normalize('NFKD', text)
    
    # Remove combining marks (accents, diacritics)
    text = ''.join(c for c in text if not unicodedata.combining(c))
    
    # Transliterate common non-ASCII characters
    transliterations = {
        'ß': 'ss', 'æ': 'ae', 'œ': 'oe', 'ø': 'o',
        'đ': 'd', 'ð': 'd', 'þ': 'th', 'ł': 'l',
    }
    for src, dst in transliterations.items():
        text = text.replace(src, dst)
    
    # Lowercase
    text = text.lower()
    
    # Replace non-alphanumeric with hyphens
    text = re.sub(r'[^a-z0-9]+', '-', text)
    
    # Collapse multiple hyphens and strip from ends
    text = re.sub(r'-+', '-', text).strip('-')
    
    # Truncate at word boundary
    if len(text) > max_length:
        text = text[:max_length].rsplit('-', 1)[0]
    
    return text

slugify("Café au lait — Recipe")   # "cafe-au-lait-recipe"
slugify("Ångström unit ≈ 0.1nm")   # "angstrom-unit-0-1nm"
slugify("C++ für Anfänger")        # "c-fur-anfanger"

"银行" → "yín háng" (bank) or "yín xíng" (walk) depending on context
"长城" → "cháng chéng" (Great Wall)

"東京" → "tōkyō" (place name) — but kanji have multiple readings
"今日" → "kyō" (today) or "konnichi" (this day) depending on context

21 → "vingt et un" (masculine) / "vingt et une" (feminine)

21 → "einundzwanzig" (one-and-twenty, single compound word)

1 рубль, 2 рубля, 5 рублей (different noun forms for 1, 2-4, 5+)

1本 (ippon), 2本 (nihon), 3本 (sanbon) — counters change pronunciation

ONES = ['', 'one', 'two', 'three', 'four', 'five',
        'six', 'seven', 'eight', 'nine', 'ten',
        'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen',
        'sixteen', 'seventeen', 'eighteen', 'nineteen']
TENS = ['', '', 'twenty', 'thirty', 'forty',
        'fifty', 'sixty', 'seventy', 'eighty', 'ninety']
SCALES = ['', 'thousand', 'million', 'billion', 'trillion']

def number_to_words(n: int) -> str:
    if n == 0:
        return 'zero'
    if n < 0:
        return 'negative ' + number_to_words(-n)
    
    parts = []
    scale_idx = 0
    
    while n > 0:
        chunk = n % 1000
        if chunk:
            chunk_words = _chunk_to_words(chunk)
            if SCALES[scale_idx]:
                chunk_words += ' ' + SCALES[scale_idx]
            parts.append(chunk_words)
        n //= 1000
        scale_idx += 1
    
    return ' '.join(reversed(parts))

def _chunk_to_words(n: int) -> str:
    """Convert 1-999 to words."""
    if n >= 100:
        return ONES[n // 100] + ' hundred' + (
            ' ' + _chunk_to_words(n % 100) if n % 100 else '')
    if n >= 20:
        return TENS[n // 10] + ('-' + ONES[n % 10] if n % 10 else '')
    return ONES[n]

def amount_to_words(amount: float, currency: str = 'USD') -> str:
    currencies = {
        'USD': ('dollar', 'dollars', 'cent', 'cents'),
        'EUR': ('euro', 'euros', 'cent', 'cents'),
        'GBP': ('pound', 'pounds', 'penny', 'pence'),
    }
    singular_main, plural_main, singular_sub, plural_sub = currencies[currency]
    
    # Avoid floating point: work with integer cents
    cents = round(amount * 100)
    main_part = cents // 100
    sub_part = cents % 100
    
    main_unit = singular_main if main_part == 1 else plural_main
    main_words = number_to_words(main_part) + ' ' + main_unit
    
    if sub_part == 0:
        return main_words
    
    sub_unit = singular_sub if sub_part == 1 else plural_sub
    return main_words + ' and ' + number_to_words(sub_part) + ' ' + sub_unit

import re
from typing import NamedTuple

class TextStats(NamedTuple):
    characters: int
    characters_no_spaces: int
    words: int
    sentences: int
    paragraphs: int
    reading_time_minutes: float

def analyze_text(text: str, locale: str = 'en') -> TextStats:
    characters = len(text)
    characters_no_spaces = len(text.replace(' ', '').replace('\t', '').replace('\n', ''))
    
    # Word count: CJK characters each count as one word
    cjk_pattern = re.compile(r'[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]')
    cjk_chars = len(cjk_pattern.findall(text))
    
    # Non-CJK words: split on whitespace
    non_cjk_text = cjk_pattern.sub(' ', text)
    latin_words = len([w for w in non_cjk_text.split() if w.strip()])
    
    words = latin_words + cjk_chars
    
    # Sentences: terminal punctuation followed by space or end
    sentences = len(re.findall(r'[.!?。！？]+(?:\s|$)', text)) or 1
    
    # Paragraphs: blocks separated by blank lines
    paragraphs = len([p for p in text.split('\n\n') if p.strip()])
    
    # Reading time: 200 wpm for Latin, 300 cpm for CJK
    reading_time = (latin_words / 200) + (cjk_chars / 300)
    
    return TextStats(
        characters=characters,
        characters_no_spaces=characters_no_spaces,
        words=words,
        sentences=sentences,
        paragraphs=paragraphs,
        reading_time_minutes=round(reading_time, 1)
    )

def deduplicate(lines: list[str], ignore_case: bool = False) -> list[str]:
    seen = set()
    result = []
    for line in lines:
        key = line.casefold() if ignore_case else line
        if key not in seen:
            seen.add(key)
            result.append(line)
    return result

import mmh3
from bitarray import bitarray

class BloomFilter:
    def __init__(self, size: int, num_hashes: int):
        self.size = size
        self.num_hashes = num_hashes
        self.bits = bitarray(size)
        self.bits.setall(0)
    
    def add(self, item: str):
        for i in range(self.num_hashes):
            idx = mmh3.hash(item, i) % self.size
            self.bits[idx] = 1
    
    def might_contain(self, item: str) -> bool:
        return all(
            self.bits[mmh3.hash(item, i) % self.size]
            for i in range(self.num_hashes)
        )

# Unix one-liner for deduplication preserving order:
awk '!seen[$0]++' input.txt > output.txt

# Sort-based (doesn't preserve order, but handles huge files):
sort -u input.txt > output.txt

# Sort-based preserving order:
nl input.txt | sort -k2 -u | sort -n | cut -f2- > output.txt