Functions — github.com/OpenNMT/Tokenizer

CodeHub MCPcopy Create free account

hub / github.com/OpenNMT/Tokenizer / functions

Functions261 in github.com/OpenNMT/Tokenizer

⨍ Functions 261◇ Types & classes 32

↓ 1 callersFunctionget_script_code

src/unicode/Unicode.cc:254

↓ 1 callersFunctionget_script_name

src/unicode/Unicode.cc:266

↓ 1 callersFunctionget_upper

src/unicode/Unicode.cc:212

↓ 1 callersFunctionhex_to_int

src/Utils.cc:82

↓ 1 callersMethodingest_file

bindings/python/pyonmttok/Python.cc:325

↓ 1 callersFunctionis_mark

src/unicode/Unicode.cc:197

↓ 1 callersMethodis_new_token

src/Tokenizer.cc:575

↓ 1 callersFunctionis_valid_language

src/unicode/Unicode.cc:240

↓ 1 callersMethodlearn

src/SentencePieceLearner.cc:70

↓ 1 callersMethodlearn

bindings/python/pyonmttok/Python.cc:348

↓ 1 callersFunctionmerge_consecutive_ranges

src/Tokenizer.cc:237

↓ 1 callersMethodnum_tokens

src/Tokenizer.cc:570

↓ 1 callersFunctionnumbers_only

Returns true if token only contains numbers.

src/Casing.cc:261

↓ 1 callersFunctionparse_kwargs

bindings/python/pyonmttok/Python.cc:391

↓ 1 callersFunctionread_case_markup

src/Casing.cc:198

↓ 1 callersFunctionreplace_pair

src/BPELearner.cc:202

↓ 1 callersFunctionrestore_token_casing

src/Casing.cc:116

↓ 1 callersFunctionset_random_generator_seed

src/Utils.cc:64

↓ 1 callersFunctionset_random_seed

src/Tokenizer.cc:12

↓ 1 callersMethodset_subword_encoder

src/Tokenizer.cc:1131

↓ 1 callersFunctionsupport_language_rules

src/unicode/Unicode.cc:235

↓ 1 callersMethodtokenize

src/ITokenizer.cc:141

↓ 1 callersFunctionunescape_characters

src/Tokenizer.cc:291

↓ 1 callersFunctionupdate_pair_statistics

src/BPELearner.cc:102

↓ 1 callersMethodupdate_tokenization_options

src/SentencePiece.cc:39

MethodBPELearnerWrapper

bindings/python/pyonmttok/Python.cc:370

src/BPELearner.cc:68

FunctionCharInfo

include/onmt/unicode/Unicode.h:61

src/Tokenizer.cc:80

include/onmt/Tokenizer.h:57

FunctionPYBIND11_MODULE

bindings/python/pyonmttok/Python.cc:498

MethodSentencePiece

src/SentencePiece.cc:21

MethodSentencePieceLearnerWrapper

bindings/python/pyonmttok/Python.cc:414

MethodSentencePieceTokenizerWrapper

bindings/python/pyonmttok/Python.cc:299

MethodSubwordLearner

src/SubwordLearner.cc:10

MethodSubwordLearnerWrapper

bindings/python/pyonmttok/Python.cc:316

test/test.cc:135

MethodTokenCaseMarkup

src/Casing.h:36

MethodTokenizerWrapper

bindings/python/pyonmttok/Python.cc:23

MethodTokensBuilder

src/Tokenizer.cc:557

Methodadd_alphabet_to_segment

src/Tokenizer.cc:148

Functionappend_feature

include/onmt/Token.h:59

Methodapply_merges

Functionbuild_vocab_from_lines

( lines, tokenizer=None, maximum_size=0, minimum_frequency=1, special_tokens=None, )

bindings/python/pyonmttok/__init__.py:51

Functionbuild_vocab_from_tokens

( tokens, maximum_size=0, minimum_frequency=1, special_tokens=None, )

bindings/python/pyonmttok/__init__.py:38

bindings/python/pyonmttok/Python.cc:133

Methodcheck_vocab_and_split

Functioncontains

include/onmt/Vocab.h:39

Methodcreate_subword_encoder

bindings/python/pyonmttok/Python.cc:384

Methodcreate_subword_encoder

bindings/python/pyonmttok/Python.cc:427

Functioncreate_vocab

bindings/python/pyonmttok/Python.cc:491

include/onmt/Token.h:51

Methodenable_regularization

src/SentencePiece.cc:72

src/SentencePiece.cc:78

Methodencode_and_annotate

src/SubwordEncoder.cc:74

Methodencode_and_annotate

src/SentencePiece.cc:90

Functionexplode_utf8

src/unicode/Unicode.cc:69

Functionexplode_utf8_with_marks

src/unicode/Unicode.cc:86

Functionexplode_utf8_with_marks

include/onmt/unicode/Unicode.h:112

Methodget_initial_pieces

Methodget_options

bindings/python/pyonmttok/Python.cc:106

Methodget_score

Functionhas_features

include/onmt/Token.h:64

Functionhash_token

bindings/python/pyonmttok/Python.cc:480

Methodin_vocabulary

src/SubwordLearner.cc:31

Methodingest_token

src/SubwordLearner.cc:18

Methodingest_token_impl

src/BPELearner.cc:54

Methodingest_token_impl

src/SentencePieceLearner.cc:63

Functionis_letter

src/unicode/Unicode.cc:187

Functionis_letter

include/onmt/unicode/Unicode.h:144

Functionis_number

src/unicode/Unicode.cc:192

Functionis_separator

src/unicode/Unicode.cc:182

src/SubwordLearner.cc:49

Methodload_from_dictionary

src/BPELearner.cc:40

Methodload_model

include/onmt/Vocab.h:24

cli/detokenize.cc:7

cli/learn.cc:13

cli/tokenize.cc:11

test/test.cc:1128

Methodmode_to_str

src/Tokenizer.cc:52

Methodoperator ()

src/BPELearner.cc:86

Functionoperator==

include/onmt/Token.h:69

Functionoperator==

include/onmt/unicode/Unicode.h:74

Methodparse_tokens

src/Tokenizer.cc:397

Functionprocess_stream

src/ITokenizer.cc:52

Methodpropagate_token_properties

src/SubwordEncoder.cc:96

Methodrecursive_split

Functionrepr_token

bindings/python/pyonmttok/Python.cc:458

Methodreset_vocabulary

Methodreset_vocabulary

src/SubwordEncoder.cc:70

Methodreset_vocabulary

src/SentencePiece.cc:67

Functionset_default_id

include/onmt/Vocab.h:69

Functionset_dropout

include/onmt/BPE.h:33

Methodset_input_filename

src/SentencePieceLearner.cc:56

Functionset_joiner

include/onmt/BPE.h:28

Methodset_vocabulary

src/SubwordEncoder.cc:66

← previous next →101–200 of 261, ranked by callers