MCPcopy Create free account

hub / github.com/OpenNMT/Tokenizer / functions

Functions261 in github.com/OpenNMT/Tokenizer

⨍ Functions 261◇ Types & classes 32
Methodset_vocabulary
src/SentencePiece.cc:51
Functionsplit_utf8
src/unicode/Unicode.cc:44
Methodstr_to_mode
src/Tokenizer.cc:37
Functiontest_bpe_case_insensitive_issue_147
()
bindings/python/test/test.py:302
Functiontest_bpe_learner
(tmpdir)
bindings/python/test/test.py:310
Functiontest_bpe_learner_escaped_character
(tmpdir)
bindings/python/test/test.py:344
Functiontest_bpe_learner_no_pairs
(tmpdir)
bindings/python/test/test.py:335
Functiontest_bpe_learner_tokens
(tmpdir)
bindings/python/test/test.py:323
Functiontest_call
()
bindings/python/test/test.py:36
Functiontest_custom_joiner
()
bindings/python/test/test.py:144
Functiontest_deepcopy
(use_constructor)
bindings/python/test/test.py:232
Functiontest_deprecated_lang
()
bindings/python/test/test.py:71
Functiontest_detok_with_ranges
()
bindings/python/test/test.py:247
Functiontest_empty
()
bindings/python/test/test.py:41
Functiontest_file
(tmpdir, tokens_delimiter)
bindings/python/test/test.py:95
Functiontest_invalid_annotation
()
bindings/python/test/test.py:85
Functiontest_invalid_bpe_model
()
bindings/python/test/test.py:80
Functiontest_invalid_files
(tmpdir)
bindings/python/test/test.py:125
Functiontest_invalid_lang
()
bindings/python/test/test.py:66
Functiontest_invalid_mode
()
bindings/python/test/test.py:61
Functiontest_invalid_sentencepiece_model
()
bindings/python/test/test.py:75
Functiontest_is_placeholder
()
bindings/python/test/test.py:15
Functiontest_is_valid_language
()
bindings/python/test/test.py:20
Functiontest_learner_with_invalid_files
(tmpdir, learner)
bindings/python/test/test.py:381
Functiontest_named_arguments
()
bindings/python/test/test.py:206
Functiontest_options
()
bindings/python/test/test.py:47
Functiontest_segment_alphabet
()
bindings/python/test/test.py:150
Functiontest_simple
()
bindings/python/test/test.py:26
Functiontest_sp_learner
(tmpdir, keep_vocab)
bindings/python/test/test.py:357
Functiontest_sp_tokenizer
()
bindings/python/test/test.py:166
Functiontest_sp_with_vocabulary
(tmpdir, as_file)
bindings/python/test/test.py:174
Functiontest_subword_regularization
()
bindings/python/test/test.py:268
Functiontest_token_api
()
bindings/python/test/test.py:391
Functiontest_token_api_features
()
bindings/python/test/test.py:475
Functiontest_token_api_with_subword
()
bindings/python/test/test.py:434
Functiontest_token_copy
()
bindings/python/test/test.py:504
Functiontest_token_deserialize_with_preserved_tokens
()
bindings/python/test/test.py:464
Functiontest_token_dict
()
bindings/python/test/test.py:513
Functiontest_token_length
()
bindings/python/test/test.py:500
Functiontest_token_pickle
()
bindings/python/test/test.py:555
Functiontest_token_repr
()
bindings/python/test/test.py:528
Functiontest_tokenize_batch
()
bindings/python/test/test.py:214
Functiontest_vocab
()
bindings/python/test/test.py:574
Functiontest_vocab_build_helpers
()
bindings/python/test/test.py:645
Functiontest_vocab_default_id
(tokens, default_id, expected_default_id)
bindings/python/test/test.py:666
Functiontest_vocab_from_file
(tmpdir)
bindings/python/test/test.py:634
Functiontest_vocab_from_text
()
bindings/python/test/test.py:623
Functiontest_vocab_pickle
()
bindings/python/test/test.py:674
Functiontest_with_separators
()
bindings/python/test/test.py:160
Methodtokenize_on_placeholders
src/Tokenizer.cc:659
Methodtokenize_text
src/Tokenizer.cc:783
Methodunset_annotate
src/Tokenizer.cc:1126
Methodupdate_tokenization_options
src/SubwordEncoder.cc:12
Functionutf8_to_cp
include/onmt/unicode/Unicode.h:90
Functionwork_loop
src/ITokenizer.cc:18
Function~ITokenizer
include/onmt/ITokenizer.h:22
Method~SentencePieceLearner
src/SentencePieceLearner.cc:49
Function~SubwordLearner
include/onmt/SubwordLearner.h:17
Method~SubwordLearnerWrapper
bindings/python/pyonmttok/Python.cc:323
Method~TokenizerWrapper
bindings/python/pyonmttok/Python.cc:104
Method~TokensBuilder
src/Tokenizer.cc:564
← previous201–261 of 261, ranked by callers