Code example

import unicodedata

# The fi ligature (U+FB01) and a circled digit (U+2460)
s = "of\uFB01ce \u2460"

# NFC keeps compatibility characters as-is
nfc = unicodedata.normalize("NFC", s)
# NFKC folds them to base forms: ligature -> 'fi', circled-1 -> '1'
nfkc = unicodedata.normalize("NFKC", s)

print(nfc == nfkc)            # False: they diverge on compatibility chars
print(repr(nfc), repr(nfkc))  # 'office ①'  vs  'office 1'

# A tokenizer trained on NFKC text must apply NFKC at inference too,
# or token IDs for these characters silently diverge from training.

# Right way: Anthropic messages.count_tokens
from anthropic import Anthropic
client = Anthropic()

messages = [
    {'role': 'user', 'content': 'Summarize this contract: ...'}
]
resp = client.messages.count_tokens(
    model='claude-sonnet-4-6',
    system='You are a legal summarization assistant.',
    messages=messages,
    tools=my_tool_defs,  # tool schema counts too
)
print(resp.input_tokens)  # official estimate, may differ slightly from billing

# Wrong: tiktoken on a Claude prompt
import tiktoken
enc = tiktoken.get_encoding('cl100k_base')
print(len(enc.encode('Summarize this contract: ...')))  # 10-30% off for Claude

# Wrong: len/4 heuristic on code
print(len(open('file.py').read()) // 4)  # underestimates badly on code

import tiktoken

# cl100k_base is the BPE vocab used by GPT-4 and GPT-3.5-turbo
enc = tiktoken.get_encoding("cl100k_base")

text = "Tokenization is the first layer of every LLM."
ids = enc.encode(text)

print(len(text), "chars ->", len(ids), "tokens")
print([enc.decode([i]) for i in ids])
# Common words stay whole, rare patterns split.
# 'Tokenization' splits into 'Token' + 'ization'; an unseen word like
# 'flibbertigibbet' splits into several known subwords.

import sentencepiece as spm

sp = spm.SentencePieceProcessor(model_file="spm.model")

pieces = sp.encode("the underworld", out_type=str)
print(pieces)
# e.g. ['\u2581the', '\u2581under', 'world']
# '\u2581under' is word-initial (has the marker); 'world' is word-internal (none)

text = sp.decode(pieces)
print(repr(text))  # 'the underworld' -> exact round trip
# Decode rule: replace U+2581 with a space, concatenate the rest.

import sentencepiece as spm
sp = spm.SentencePieceProcessor()
sp.load("spiece.model")  # T5's SentencePiece Unigram model

text = "tokenization"

# Deterministic Viterbi path: same result every call
print("viterbi:", sp.encode(text, out_type=str))

# Stochastic sampling: different valid splits across calls
for _ in range(4):
    print("sample:", sp.encode(text, out_type=str,
                               enable_sampling=True, alpha=0.2, nbest_size=-1))

from tokenizers import Tokenizer
from tokenizers.models import BPE
from tokenizers.trainers import BpeTrainer

# [UNK] covers symbols absent from the training corpus (rare IUPAC codes)
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))

# DNA has no word boundaries, so use no pre-tokenizer (None is the default).
# Each training line is one word: cut chromosomes into bounded chunks.
tokenizer.pre_tokenizer = None

# 4^6 = 4,096 hexamers is the productive ceiling; 8192 leaves headroom.
trainer = BpeTrainer(
    vocab_size=8192,
    special_tokens=["[UNK]", "[PAD]", "[MASK]", "[BOS]", "[EOS]",
                    "[STRAND+]", "[STRAND-]"],
    min_frequency=2,  # drop single-occurrence k-mers (often read errors)
    show_progress=True,
)

# Train on genomics corpora ONLY, never general text.
tokenizer.train(files=["hg38.txt", "bacterial.txt", "viral.txt"], trainer=trainer)
tokenizer.save("dna_tokenizer.json")

# Sanity check: GAATTC is the EcoRI restriction site, very common in hg38.
out = tokenizer.encode("ATCGTTACGCGATCGGAATTCATCG")
print(out.tokens)              # inspect how the GAATTC site is split
print(len(tokenizer.get_vocab()))

emoji = "🏳️‍🌈"  # rainbow flag, a ZWJ sequence

# Inspect the Unicode structure: 4 codepoints joined by ZWJ.
print([hex(ord(c)) for c in emoji])
# ['0x1f3f3', '0xfe0f', '0x200d', '0x1f308']
#   flag      VS-16     ZWJ       rainbow

print(len(emoji.encode("utf-8")))  # 14 bytes
# With no learned merges, a byte-level BPE tokenizer
# emits ~1 token per byte -> ~14 tokens.

import tiktoken

cl100k = tiktoken.get_encoding("cl100k_base")
o200k = tiktoken.get_encoding("o200k_base")

flag = "\U0001F3F3\uFE0F\u200D\U0001F308"  # rainbow flag (ZWJ sequence)

print(len(cl100k.encode(flag)))  # 9: one chunk, few emoji merges in the vocabulary
print(len(o200k.encode(flag)))   # 6: two chunks, but more byte merges in the vocabulary

# Always resolve the encoding from the model so the regex matches:
enc = tiktoken.encoding_for_model("gpt-4o")  # -> o200k_base

from transformers import AutoTokenizer

# Loads tokenizer.json + tokenizer_config.json from the checkpoint.
tok = AutoTokenizer.from_pretrained("meta-llama/Llama-4-Maverick-17B-128E-Instruct")

messages = [
    {"role": "system", "content": "You are concise."},
    {"role": "user", "content": "Define tokenization."},
]

# apply_chat_template uses the exact format the model was tuned on.
prompt = tok.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
ids = tok(prompt).input_ids  # special tokens placed correctly

import tiktoken
enc = tiktoken.get_encoding("cl100k_base")

# Semantically similar sentences across languages
samples = {
    "English": "The relationship between data and accuracy matters.",
    "Spanish": "La relacion entre datos y precision importa.",
    "Swahili": "Uhusiano kati ya data na usahihi ni muhimu.",
}
for lang, text in samples.items():
    n_tok = len(enc.encode(text))
    n_word = len(text.split())
    print(lang, n_tok, "tokens", "fertility", round(n_tok / n_word, 2))

import tiktoken

enc = tiktoken.get_encoding("cl100k_base")  # GPT-4's tokenizer

english = "Hello world"
arabic = "مرحبا بالعالم"  # same meaning

en_toks = enc.encode(english)
ar_toks = enc.encode(arabic)

print(len(en_toks))                  # 2
print(len(ar_toks))                  # 10
print(len(ar_toks) / len(en_toks))   # 5.0 (5x token tax)

import tiktoken

enc = tiktoken.encoding_for_model("text-embedding-3-small")

def budget(samples: list[str], total_docs: int, price_per_1m: float = 0.10):
    counts = [len(enc.encode(d)) for d in samples]
    mean = sum(counts) / len(counts)
    p95 = sorted(counts)[int(0.95 * len(counts)) - 1]
    est_tokens = mean * total_docs
    est_cost = est_tokens / 1_000_000 * price_per_1m
    return {"mean": mean, "p95": p95, "tokens": est_tokens, "cost": est_cost}

# Run per language on ~1000 real sampled descriptions each
es = budget(spanish_samples, 1_000_000)
fr = budget(french_samples, 1_000_000)

import tiktoken
enc = tiktoken.get_encoding("cl100k_base")

# Semantically similar sentences across languages
samples = {
    "English": "Training data and model accuracy matter for multilingual systems.",
    "Spanish": "Los datos y la precision del modelo importan en sistemas multilingues.",
    "Swahili": "Data ya mafunzo na usahihi wa mfano ni muhimu kwa mifumo ya lugha nyingi.",
}
baseline = None
for lang, text in samples.items():
    n_tok = len(enc.encode(text))
    n_word = len(text.split())
    baseline = baseline or n_tok
    print(lang, n_tok, "tokens", "fertility", round(n_tok / n_word, 2),
          "rel", round(n_tok / baseline, 1))

def vocab_memory_gb(vocab_size: int, d_model: int,
                    weight_tied: bool = False,
                    bytes_per_param: int = 2) -> float:
    """Vocabulary-related memory in GB; bfloat16 is 2 bytes per value."""
    n_matrices = 1 if weight_tied else 2  # embedding + lm_head unless tied
    total_params = n_matrices * vocab_size * d_model
    return total_params * bytes_per_param / 1e9

d_model = 4096
for vocab_size in [8_000, 50_000, 200_000]:
    untied = vocab_memory_gb(vocab_size, d_model)
    tied = vocab_memory_gb(vocab_size, d_model, weight_tied=True)
    print(f"{vocab_size:>7,}: {untied:5.2f} GB untied, {tied:5.2f} GB tied")

#   8,000:  0.13 GB untied,  0.07 GB tied
#  50,000:  0.82 GB untied,  0.41 GB tied
# 200,000:  3.28 GB untied,  1.64 GB tied