import pytest
from aave_compress.tokenizers import TokenCount, count_tokens, count_tokens_tiktoken
def test_count_tokens_tiktoken_gpt4o():
pytest.importorskip("tiktoken")
result = count_tokens_tiktoken("This endpoint usually times out.", model="gpt-4o")
assert isinstance(result, TokenCount)
assert result.tokens > 0
assert result.model == "gpt-4o"
assert result.method == "tiktoken_approx"
def test_count_tokens_tiktoken_rejects_claude_models():
with pytest.raises(ValueError, match="never be used for Claude"):
count_tokens_tiktoken("some text", model="claude-sonnet-5")
def test_count_tokens_tiktoken_rejects_claude_models_case_insensitive():
with pytest.raises(ValueError):
count_tokens_tiktoken("some text", model="Claude-Opus-4-7")
def test_count_tokens_claude_without_api_key_raises_runtime_error(monkeypatch):
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
with pytest.raises(RuntimeError, match="ANTHROPIC_API_KEY"):
count_tokens("some text", model="claude-sonnet-5")
def test_count_tokens_dispatches_gpt_to_tiktoken():
pytest.importorskip("tiktoken")
result = count_tokens("This endpoint usually times out.", model="gpt-4o")
assert result.method == "tiktoken_approx"
assert result.tokens > 0