Sochen
d0f4aea58d
Exposes the full word→rank dict for use by the upcoming sentence_difficulty module without requiring per-word lookups. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
92 lines
3 KiB
Python
92 lines
3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Hebrew word frequency lookup from hermitdave/FrequencyWords corpus.
|
|
Downloads he_50k.txt once; subsequent runs read from cache.
|
|
Exposed API: get_frequency_rank(word_no_nikkud) -> int | None
|
|
|
|
TODO: Rewrite to update words.json frequency field directly instead of
|
|
writing to a separate frequency_cache.json. Currently the migration script
|
|
bridges the gap. See Phase 5 in SPRINT_LOG.md.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
FREQ_URL = "https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2016/he/he_50k.txt"
|
|
CACHE_PATH = Path(__file__).parent / "data" / "frequency_cache.json"
|
|
CLEAN_CACHE_PATH = Path(__file__).parent / "data" / "frequency_clean.json"
|
|
REQUEST_TIMEOUT = 30
|
|
|
|
# Module-level cache: word_no_nikkud -> rank (1 = most common)
|
|
_freq: dict[str, int] = {}
|
|
|
|
|
|
def load(cache_path: Path = CACHE_PATH) -> None:
|
|
"""Load frequency data from cache, downloading if not present.
|
|
|
|
Prefers frequency_clean.json (YAP-filtered) over raw frequency_cache.json.
|
|
"""
|
|
global _freq
|
|
# Prefer YAP-cleaned frequency data if available
|
|
clean_path = cache_path.parent / "frequency_clean.json" if cache_path == CACHE_PATH else None
|
|
load_path = clean_path if clean_path and clean_path.exists() else cache_path
|
|
if load_path.exists():
|
|
with open(load_path, encoding="utf-8") as f:
|
|
_freq = json.load(f)
|
|
label = "clean" if load_path == clean_path else "raw"
|
|
logger.info(f"Frequency cache loaded ({label}): {len(_freq)} entries")
|
|
return
|
|
|
|
logger.info("Downloading FrequencyWords he_50k.txt …")
|
|
resp = requests.get(FREQ_URL, timeout=REQUEST_TIMEOUT)
|
|
resp.raise_for_status()
|
|
|
|
rank = 1
|
|
for line in resp.text.splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
word = line.split()[0]
|
|
if word and word not in _freq:
|
|
_freq[word] = rank
|
|
rank += 1
|
|
|
|
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(cache_path, "w", encoding="utf-8") as f:
|
|
json.dump(_freq, f, ensure_ascii=False)
|
|
logger.info(f"Frequency cache saved: {len(_freq)} entries → {cache_path}")
|
|
|
|
|
|
def get_frequency_rank(word_no_nikkud: str) -> int | None:
|
|
"""
|
|
Return the frequency rank of a word (1 = most common).
|
|
Returns None if not found in the corpus.
|
|
Expects ktiv male (no nikkud) input.
|
|
"""
|
|
if not _freq:
|
|
load()
|
|
clean = word_no_nikkud.strip()
|
|
return _freq.get(clean)
|
|
|
|
|
|
def get_freq_data() -> dict[str, int]:
|
|
"""Return the full frequency dict (word -> rank).
|
|
|
|
Auto-loads from cache if not yet loaded.
|
|
"""
|
|
if not _freq:
|
|
load()
|
|
return _freq
|
|
|
|
|
|
if __name__ == "__main__":
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
|
|
load()
|
|
tests = ["שלום", "ספר", "בית", "מים", "כלב"]
|
|
for w in tests:
|
|
print(f"{w}: rank {get_frequency_rank(w)}")
|