Function demo_token_estimation

tutorials/16_prompt_building_and_compaction.py:1010–1028

演示 token 估算

()

def demo_token_estimation():
    """演示 token 估算"""
    print("\n" + "=" * 60)
    print("Token 估算策略")
    print("=" * 60)

    test_cases = [
        ("hello", 1 + 1),            # 5 chars / 4 + 1 = 2
        ("a" * 100, 100 // 4 + 1),   # 100 / 4 + 1 = 26
        ("", 0 + 1),                   # 空字符串 = 1
        ("This is a typical sentence.", 27 // 4 + 1),
    ]

    print("\n  公式: len(text) // 4 + 1")
    print(f"  {'文本':<35} {'字符数':>6} {'估算tokens':>10}")
    print(f"  {'-'*35} {'-'*6} {'-'*10}")
    for text, expected in test_cases:
        display = text[:30] + "..." if len(text) > 30 else text
        print(f"  {display:<35} {len(text):>6} {expected:>10}")


# ============================================================