Types & classes — github.com/M4THYOU/TokenDagger

CodeHub MCPcopy Create free account

hub / github.com/M4THYOU/TokenDagger / types & classes

Types & classes198 in github.com/M4THYOU/TokenDagger

⨍ Functions 724◇ Types & classes 198

↓ 7 callersClassTokenDaggerError

Base exception for TokenDagger errors.

tokendagger/wrapper.py:23

↓ 5 callersClassdiyfp

src/nlohmann/json.hpp:16964

↓ 4 callersClassout_of_range

@brief exception indicating access out of the defined range @sa https://json.nlohmann.me/api/basic_json/out_of_range/

src/nlohmann/json.hpp:4557

↓ 3 callersClassTiktokenError

Exception class

src/tiktoken/tiktoken.hpp:32

↓ 3 callersClassTokenizer

High-level interface for the tokendagger tokenizer. This class provides a tiktoken-compatible API around the tokendagger.CoreBPE class.

tokendagger/wrapper.py:28

↓ 2 callersClassThroughputResult

Results from a throughput test.

tests/throughput_test.py:52

↓ 2 callersClassValueType

src/nlohmann/json.hpp:21524

↓ 2 callersClassinput_stream_adapter

! Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at beginning of input. Does not support changing the underlying std::streambuf

src/nlohmann/json.hpp:6225

↓ 2 callersClassiterator

src/tiktoken/hash_set8.hpp:112

↓ 1 callersClassBenchmarkResult

Results from a single benchmark run.

tests/performance_benchmark.py:45

↓ 1 callersClassCodeBenchmarkResult

Results from benchmarking a single code file.

tests/code_performance_benchmark.py:47

↓ 1 callersClassCodePerformanceBenchmark

Performance benchmark specifically for code tokenization.

tests/code_performance_benchmark.py:64

↓ 1 callersClassPerformanceBenchmark

Comprehensive performance benchmark suite.

tests/performance_benchmark.py:65

↓ 1 callersClassThroughputBenchmark

Multithreaded throughput benchmark suite.

tests/throughput_test.py:64

↓ 1 callersClassTokenDaggerVsTikTokenTest

Test suite comparing TokenDagger and TikToken tokenization.

tests/test_tokendagger_vs_tiktoken.py:35

↓ 1 callersClassiteration_proxy_value

src/nlohmann/json.hpp:5185

↓ 1 callersClassprimitive_iterator_t

@brief an iterator for primitive JSON types This class models an iterator for primitive JSON types (boolean, number, string). It's only purpose is to

src/nlohmann/json.hpp:12732

Core BPE implementation - using emhash8 for better performance

src/tiktoken/tiktoken.hpp:38

ClassCustomBuildExt

Custom build extension to build tiktoken library first.

ClassDefaultPolicy

src/tiktoken/hash_table8.hpp:71

src/nlohmann/json.hpp:3143

ClassExtend<integer_sequence<T, Ints...>, SeqSize, 0>

src/nlohmann/json.hpp:3147

ClassExtend<integer_sequence<T, Ints...>, SeqSize, 1>

src/nlohmann/json.hpp:3153

src/nlohmann/json.hpp:3161

src/nlohmann/json.hpp:3168

src/tiktoken/hash_table8.hpp:82

src/tiktoken/hash_set8.hpp:84

src/tiktoken/hash_table8.hpp:111

src/tiktoken/hash_set8.hpp:105

ClassInternalSpecialToken

src/main.cpp:27

ClassLlama4Tokenizer

src/main.cpp:46

ClassMistralTokenizer

src/mistral_main.cpp:58

ClassSpecialToken

src/main.cpp:38

ClassTokenizerConfig

src/mistral_main.cpp:26

ClassTokenizerConfig

src/main.cpp:42

ClassVectorHashEmhash

src/tiktoken/tiktoken.hpp:20

src/tiktoken/tiktoken.hpp:12

Classactual_object_comparator

src/nlohmann/json.hpp:3587

Classadl_serializer

src/nlohmann/json.hpp:5832

Classbinary_reader

src/nlohmann/json.hpp:9211

Classbinary_writer

src/nlohmann/json.hpp:15078

Classboundaries

src/nlohmann/json.hpp:17082

Classcached_power

src/nlohmann/json.hpp:17221

Enumcbor_tag_handler_t

how to treat CBOR tags

src/nlohmann/json.hpp:9184

Classchar_traits

src/nlohmann/json.hpp:3604

Classchar_traits<signed char>

src/nlohmann/json.hpp:3633

Classchar_traits<unsigned char>

src/nlohmann/json.hpp:3609

Classconjunction

src/nlohmann/json.hpp:3660

Classconjunction<B, Bn...>

src/nlohmann/json.hpp:3663

Classconjunction<B>

src/nlohmann/json.hpp:3661

Classconst_iterator

src/tiktoken/hash_table8.hpp:117

Classconst_iterator

src/tiktoken/hash_set8.hpp:111

Classcontainer_input_adapter_factory

src/nlohmann/json.hpp:6538

Classcontainer_input_adapter_factory< ContainerType, void_t<decltype(begin(std::declval<ContainerType>()), end(std::declval<ContainerType>()))>>

src/nlohmann/json.hpp:6541

src/nlohmann/json.hpp:23491

src/nlohmann/json.hpp:280

Classdetector<Default, void_t<Op<Args...>>, Op, Args...>

src/nlohmann/json.hpp:287

Enumerror_handler_t

how to treat decoding errors

src/nlohmann/json.hpp:18051

@brief general exception of the @ref basic_json class @sa https://json.nlohmann.me/api/basic_json/exception/

src/nlohmann/json.hpp:4371

Classexternal_constructor

src/nlohmann/json.hpp:5419

Classexternal_constructor<value_t::array>

src/nlohmann/json.hpp:5529

Classexternal_constructor<value_t::binary>

src/nlohmann/json.hpp:5468

Classexternal_constructor<value_t::boolean>

src/nlohmann/json.hpp:5422

Classexternal_constructor<value_t::number_float>

src/nlohmann/json.hpp:5490

Classexternal_constructor<value_t::number_integer>

src/nlohmann/json.hpp:5516

Classexternal_constructor<value_t::number_unsigned>

src/nlohmann/json.hpp:5503

Classexternal_constructor<value_t::object>

src/nlohmann/json.hpp:5599

Classexternal_constructor<value_t::string>

src/nlohmann/json.hpp:5435

Classfrom_json_fn

src/nlohmann/json.hpp:5098

Classhas_from_json

src/nlohmann/json.hpp:3527

Classhas_from_json < BasicJsonType, T, enable_if_t < !is_basic_json<T>::value >>

src/nlohmann/json.hpp:3540

Classhas_key_compare

src/nlohmann/json.hpp:3583

Classhas_non_default_from_json

src/nlohmann/json.hpp:3552

Classhas_non_default_from_json < BasicJsonType, T, enable_if_t < !is_basic_json<T>::value >>

src/nlohmann/json.hpp:3555

Classhas_to_json

src/nlohmann/json.hpp:3567

Classhas_to_json < BasicJsonType, T, enable_if_t < !is_basic_json<T>::value >>

src/nlohmann/json.hpp:3570

Classidentity_tag

src/nlohmann/json.hpp:4615

Enuminput_format_t

the supported input formats

src/nlohmann/json.hpp:6176

Classinteger_sequence

src/nlohmann/json.hpp:3122

Classinternal_iterator

src/nlohmann/json.hpp:12848

Classinvalid_iterator

@brief exception indicating errors with iterators @sa https://json.nlohmann.me/api/basic_json/invalid_iterator/

src/nlohmann/json.hpp:4522

Classis_basic_json

src/nlohmann/json.hpp:3463

Classis_basic_json_context

src/nlohmann/json.hpp:3472

Classis_c_string

src/nlohmann/json.hpp:4181

Classis_comparable

src/nlohmann/json.hpp:3977

Classis_comparable<Compare, A, B, void_t< decltype(std::declval<Compare>()(std::declval<A>(), std::declval<B>())), decltype(std::declval<Compare>()(std::declval<B>(), std::declval<A>())) >>

src/nlohmann/json.hpp:3980

Classis_compatible_array_type

src/nlohmann/json.hpp:3859

Classis_compatible_array_type_impl

src/nlohmann/json.hpp:3841

Classis_compatible_array_type_impl < BasicJsonType, CompatibleArrayType, enable_if_t < is_detected<iterator_t, CompatibleArrayType>::value&& is_iterator_traits<iterator_traits<detected_t<iterator_t, CompatibleArrayType>>>::value&& // special case for types like std::filesystem::path whose iterator's value_type are themselves // c.f. https://github.com/nlohmann/json/pull/3073 !std::is_same<CompatibleArrayType, detected_t<range_value_t, CompatibleArrayType>>::value >>

src/nlohmann/json.hpp:3844

Classis_compatible_integer_type

src/nlohmann/json.hpp:3929

Classis_compatible_integer_type_impl

src/nlohmann/json.hpp:3908

Classis_compatible_integer_type_impl < RealIntegerType, CompatibleNumberIntegerType, enable_if_t < std::is_integral<RealIntegerType>::value&& std::is_integral<CompatibleNumberIntegerType>::value&& !std::is_same<bool, CompatibleNumberIntegerType>::value >>

src/nlohmann/json.hpp:3911

Classis_compatible_object_type

src/nlohmann/json.hpp:3780

Classis_compatible_object_type_impl

src/nlohmann/json.hpp:3761

Classis_compatible_object_type_impl < BasicJsonType, CompatibleObjectType, enable_if_t < is_detected<mapped_type_t, CompatibleObjectType>::value&& is_detected<key_type_t, CompatibleObjectType>::value >>

src/nlohmann/json.hpp:3764

Classis_compatible_string_type

src/nlohmann/json.hpp:3817

Classis_compatible_type

src/nlohmann/json.hpp:3946

Classis_compatible_type_impl

src/nlohmann/json.hpp:3934

Classis_compatible_type_impl < BasicJsonType, CompatibleType, enable_if_t<is_complete_type<CompatibleType>::value >>

src/nlohmann/json.hpp:3937

Classis_complete_type

src/nlohmann/json.hpp:3754

next →1–100 of 198, ranked by callers