Averaged Perceptron on structured features — generalizes across scripts. Feature groups: - structural_*: 12 scores from structural feature extractors - unicode_*: 15 Unicode block proportions - keyword_*: 12 scores from keyword extractors - ngram_*: char n-gram features
| 48 | |
| 49 | |
| 50 | class ScriptAgnosticClassifier: |
| 51 | """Averaged Perceptron on structured features — generalizes across scripts. |
| 52 | |
| 53 | Feature groups: |
| 54 | - structural_*: 12 scores from structural feature extractors |
| 55 | - unicode_*: 15 Unicode block proportions |
| 56 | - keyword_*: 12 scores from keyword extractors |
| 57 | - ngram_*: char n-gram features (optional boost, low weight for unseen scripts) |
| 58 | """ |
| 59 | |
| 60 | TIERS = ("SIMPLE", "MEDIUM", "COMPLEX") |
| 61 | TIER_ORDER = {"SIMPLE": 0, "MEDIUM": 1, "COMPLEX": 2} |
| 62 | |
| 63 | def __init__(self, use_ngrams: bool = True) -> None: |
| 64 | self._weights: dict[str, dict[str, float]] = {t: defaultdict(float) for t in self.TIERS} |
| 65 | self._avg_weights: dict[str, dict[str, float]] = {t: defaultdict(float) for t in self.TIERS} |
| 66 | self._update_count = 0 |
| 67 | self._trained = False |
| 68 | self._use_ngrams = use_ngrams |
| 69 | |
| 70 | @classmethod |
| 71 | def _normalize_tier_label(cls, tier: str) -> str | None: |
| 72 | normalized = str(tier).strip().upper() |
| 73 | if normalized == "REASONING": |
| 74 | return "COMPLEX" |
| 75 | if normalized in cls.TIERS: |
| 76 | return normalized |
| 77 | return None |
| 78 | |
| 79 | def _build_features( |
| 80 | self, |
| 81 | structural_scores: dict[str, float], |
| 82 | unicode_blocks: dict[str, float], |
| 83 | keyword_scores: dict[str, float] | None = None, |
| 84 | prompt: str = "", |
| 85 | context_features: dict[str, float] | None = None, |
| 86 | ) -> dict[str, float]: |
| 87 | """Build the full feature vector from component parts. |
| 88 | |
| 89 | Keyword scores are accepted for backward compatibility but ignored |
| 90 | when the model is trained without them. N-gram features learn |
| 91 | equivalent patterns directly from data. |
| 92 | """ |
| 93 | features: dict[str, float] = {} |
| 94 | |
| 95 | for name, score in structural_scores.items(): |
| 96 | features[f"s_{name}"] = score |
| 97 | |
| 98 | for name, proportion in unicode_blocks.items(): |
| 99 | features[f"u_{name}"] = proportion |
| 100 | |
| 101 | if keyword_scores: |
| 102 | for name, score in keyword_scores.items(): |
| 103 | features[f"k_{name}"] = score |
| 104 | |
| 105 | if context_features: |
| 106 | for name, value in context_features.items(): |
| 107 | key = name if name.startswith("ctx_") else f"ctx_{name}" |
no outgoing calls