๐Ÿ— KeyzHub
19Keys ยท community archive
7242 bytes raw
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
"""Command-line interface for aave-compress.

Usage::

    aave-compress "some text" --level medium --model claude-sonnet-5
    aave-compress "some text" --level full --strip-filler --tiktoken-model gpt-4o --gate

Prints the original and compressed text, the markers that fired, and โ€” only
when a real tokenizer count is available via :mod:`aave_compress.tokenizers` โ€”
the measured token savings. There is deliberately NO estimated-savings
fallback based on characters or words: unverified compression claims are
exactly the failure mode this project exists to avoid.

``--gate`` needs a working tokenizer (from ``--model`` or the approximate
``--tiktoken-model`` fallback). If neither works, the run proceeds UNGATED
with a warning rather than crashing.
"""

from __future__ import annotations

import argparse
import sys
from collections.abc import Callable

from .engine import LEVELS, compress
from .tokenizers import count_tokens, count_tokens_tiktoken


def _resolve_counter(
    args: argparse.Namespace,
) -> tuple[Callable[[str], int] | None, str]:
    """Probe tokenizer availability once, up front.

    Returns ``(counter_fn, label)``. When no tokenizer works, ``counter_fn``
    is None and ``label`` carries the reason.
    """
    probe = "tokenizer availability probe"
    try:
        probed = count_tokens(probe, model=args.model)
        return (
            lambda text: count_tokens(text, model=args.model).tokens,
            f"{args.model}, method={probed.method}",
        )
    except Exception as exc:  # noqa: BLE001 - probe: any failure means "fall back", never crash
        # Covers missing-key RuntimeError/ValueError *and* API-side failures
        # (auth, zero credit balance, rate limit, network) raised by the
        # anthropic SDK, which are not RuntimeError/ValueError subclasses.
        reason = str(exc)
    if args.tiktoken_model:
        try:
            count_tokens_tiktoken(probe, model=args.tiktoken_model)
        except Exception as exc:  # noqa: BLE001 - same probe-never-crashes rule
            return None, f"{reason} (tiktoken fallback also failed: {exc})"
        return (
            lambda text: count_tokens_tiktoken(
                text, model=args.tiktoken_model
            ).tokens,
            f"{args.tiktoken_model}, method=tiktoken_approx โ€” GPT-family "
            "approximation, NOT valid for any Claude model",
        )
    return None, reason


def main(argv: list[str] | None = None) -> None:
    parser = argparse.ArgumentParser(
        prog="aave-compress",
        description=(
            "Compress LLM prompt text using documented AAVE grammar patterns "
            "and measure real token savings."
        ),
    )
    parser.add_argument("text", help="Text to compress")
    parser.add_argument(
        "--level",
        choices=sorted(LEVELS),
        default="medium",
        help="Compression level (default: medium)",
    )
    parser.add_argument(
        "--model",
        default="claude-sonnet-5",
        help=(
            "Model to count tokens for (default: claude-sonnet-5). claude-* "
            "models use the Anthropic count_tokens API (requires "
            "ANTHROPIC_API_KEY); gpt-* models use tiktoken."
        ),
    )
    parser.add_argument(
        "--tiktoken-model",
        default=None,
        metavar="GPT_MODEL",
        help=(
            "APPROXIMATE fallback: count tokens with tiktoken for this "
            "GPT-family model (e.g. gpt-4o) if the primary counter is "
            "unavailable. NEVER valid for Claude models."
        ),
    )
    parser.add_argument(
        "--strip-filler",
        action="store_true",
        help=(
            "Run the generic filler/hedge/politeness stripper "
            "(filters.strip_fillers) as a first pass before the AAVE markers."
        ),
    )
    parser.add_argument(
        "--gate",
        action="store_true",
        help=(
            "Keep each transform step only if it strictly reduces the token "
            "count. Needs a working tokenizer (--model or --tiktoken-model); "
            "without one the run proceeds ungated."
        ),
    )
    args = parser.parse_args(argv)
    if args.tiktoken_model and args.tiktoken_model.lower().startswith("claude"):
        parser.error(
            "--tiktoken-model must be a GPT-family model: tiktoken counts are "
            "never valid for Claude models (they undercount by 15-20%+). "
            "Set ANTHROPIC_API_KEY and use --model instead."
        )

    counter, counter_label = _resolve_counter(args)

    gate = args.gate
    if gate and counter is None:
        print(
            "Gating needs a working tokenizer, and none is available: "
            f"{counter_label}\nProceeding UNGATED.",
            file=sys.stderr,
        )
        gate = False

    result = compress(
        args.text,
        level=args.level,
        strip_filler=args.strip_filler,
        tokenizer=counter,
        gate=gate,
    )

    print(f"Level: {result.level}")
    if args.strip_filler:
        state = "fillers removed" if result.filler_stripped else "nothing to remove"
        print(f"Filler strip: on ({state})")
    else:
        print("Filler strip: off")
    if result.gated:
        print(
            "Gating: on โ€” each step kept only if it strictly reduced the "
            f"token count ({counter_label})"
        )
    elif args.gate:
        print("Gating: requested but ran UNGATED (no working tokenizer)")
    else:
        print("Gating: off")
    print()
    print("Original:")
    print(f"  {result.original}")
    print()
    print("Compressed:")
    print(f"  {result.compressed}")
    print()
    if result.markers_applied:
        print("Markers applied: " + ", ".join(result.markers_applied))
    else:
        print("Markers applied: (none)")
    print()

    if counter is None:
        print(
            "Token counts unavailable: set ANTHROPIC_API_KEY to see real "
            "savings โ€” do not trust word-count-based estimates as a "
            f"substitute. ({counter_label})",
            file=sys.stderr,
        )
        return

    try:
        original_tokens = counter(result.original)
        compressed_tokens = counter(result.compressed)
    except Exception as exc:
        print(
            f"Token counting failed mid-run ({type(exc).__name__}: {exc}); "
            "no counts reported.",
            file=sys.stderr,
        )
        return

    ratio = (
        compressed_tokens / original_tokens if original_tokens else float("nan")
    )
    print(f"Token counts ({counter_label}):")
    print(f"  Original:   {original_tokens}")
    print(f"  Compressed: {compressed_tokens}")
    print(f"  Ratio (compressed/original): {ratio:.3f}")
    print()
    print(
        "NOTE: this ratio is for THIS ONE INPUT ONLY โ€” a single example, NOT a "
        "benchmark. Do not quote it as a general savings figure. For the "
        "corpus-level number (currently 5.0% mean at level=full with "
        "--strip-filler, n=20 non-cherry-picked prompts, GPT-4o "
        "tiktoken-approximate โ€” not yet Claude-verified) run "
        "`python eval/run_eval.py`; see docs/methodology.md."
    )


if __name__ == "__main__":
    main()