🗝 KeyzHub
19Keys · community archive
1301 bytes raw
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
import pytest

from aave_compress.tokenizers import TokenCount, count_tokens, count_tokens_tiktoken


def test_count_tokens_tiktoken_gpt4o():
    pytest.importorskip("tiktoken")
    result = count_tokens_tiktoken("This endpoint usually times out.", model="gpt-4o")
    assert isinstance(result, TokenCount)
    assert result.tokens > 0
    assert result.model == "gpt-4o"
    assert result.method == "tiktoken_approx"


def test_count_tokens_tiktoken_rejects_claude_models():
    with pytest.raises(ValueError, match="never be used for Claude"):
        count_tokens_tiktoken("some text", model="claude-sonnet-5")


def test_count_tokens_tiktoken_rejects_claude_models_case_insensitive():
    with pytest.raises(ValueError):
        count_tokens_tiktoken("some text", model="Claude-Opus-4-7")


def test_count_tokens_claude_without_api_key_raises_runtime_error(monkeypatch):
    monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
    with pytest.raises(RuntimeError, match="ANTHROPIC_API_KEY"):
        count_tokens("some text", model="claude-sonnet-5")


def test_count_tokens_dispatches_gpt_to_tiktoken():
    pytest.importorskip("tiktoken")
    result = count_tokens("This endpoint usually times out.", model="gpt-4o")
    assert result.method == "tiktoken_approx"
    assert result.tokens > 0