hub / github.com/google/sentencepiece / versions

github.com/google/sentencepiece @v0.2.2 → @1.0.0

indexed versions: v0.2.1 · v0.2.2 · 1.0.0

49 added 1,300 removed 0 signature changed

Removed / breaking in @v0.2.2, gone in @1.0.0

mainFunctionbenchmark/prepare_balanced_raw_multilingual.py

benchmark_workerFunctionbenchmark/run_bench.py

check_model_availableFunctionbenchmark/run_bench.py

get_tokenizerFunctionbenchmark/run_bench.py

load_dataFunctionbenchmark/run_bench.py

mainFunctionbenchmark/run_bench.py

run_single_benchmarkFunctionbenchmark/run_bench.py

RunFastBPEMergesFunctioncontrib/nlcodec/bpe_model_trainer_nlcodec.cc

WordSeqClasscontrib/nlcodec/bpe_model_trainer_nlcodec.cc

BigramHashClasscontrib/nlcodec/bpe_model_trainer_nlcodec.h

BigramHash.operator()Methodcontrib/nlcodec/bpe_model_trainer_nlcodec.h

LnNodeClasscontrib/nlcodec/bpe_model_trainer_nlcodec.h

LnNode.is_unlinkedMethodcontrib/nlcodec/bpe_model_trainer_nlcodec.h

LnNode.unlinkMethodcontrib/nlcodec/bpe_model_trainer_nlcodec.h

MaxHeapClasscontrib/nlcodec/bpe_model_trainer_nlcodec.h

MaxHeap.MaxHeapMethodcontrib/nlcodec/bpe_model_trainer_nlcodec.h

MaxHeap.emptyMethodcontrib/nlcodec/bpe_model_trainer_nlcodec.h

MaxHeap.popMethodcontrib/nlcodec/bpe_model_trainer_nlcodec.h

MaxHeap.pushMethodcontrib/nlcodec/bpe_model_trainer_nlcodec.h

NodeArenaClasscontrib/nlcodec/bpe_model_trainer_nlcodec.h

NodeArena.NodeArenaMethodcontrib/nlcodec/bpe_model_trainer_nlcodec.h

NodeArena.allocMethodcontrib/nlcodec/bpe_model_trainer_nlcodec.h

NodeArena.from_seqMethodcontrib/nlcodec/bpe_model_trainer_nlcodec.h

TESTFunctioncontrib/nlcodec/bpe_model_trainer_nlcodec_test.cc

TrainAndGetPiecesFunctioncontrib/nlcodec/bpe_model_trainer_nlcodec_test.cc

build_ext_unixClasspython/setup.py

build_ext_unix.build_extensionMethodpython/setup.py

build_ext_winClasspython/setup.py

build_ext_win.build_extensionMethodpython/setup.py

copy_package_dataFunctionpython/setup.py

find_abseil_libFunctionpython/setup.py

find_absl_includeFunctionpython/setup.py

find_targetsFunctionpython/setup.py

get_cflags_and_libsFunctionpython/setup.py

get_protobuf_includesFunctionpython/setup.py

get_win_archFunctionpython/setup.py

is_gil_disabledFunctionpython/setup.py

SentencePieceNormalizerClasspython/src/sentencepiece/__init__.py

SentencePieceNormalizer.DecompileMethodpython/src/sentencepiece/__init__.py

SentencePieceNormalizer.NormalizeMethodpython/src/sentencepiece/__init__.py

SentencePieceNormalizer.__getstate__Methodpython/src/sentencepiece/__init__.py

SentencePieceNormalizer.__init__Methodpython/src/sentencepiece/__init__.py

SentencePieceNormalizer.__setstate__Methodpython/src/sentencepiece/__init__.py

SentencePieceNormalizer.serialized_model_protoMethodpython/src/sentencepiece/__init__.py

SentencePieceNormalizer.serialized_normalizer_specMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessorClasspython/src/sentencepiece/__init__.py

SentencePieceProcessor.CalculateEntropyMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.DecodeMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.DecodeIdsMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.DecodeIdsAsImmutableProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.DecodeIdsAsProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.DecodeIdsAsSerializedProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.DecodePiecesMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.DecodePiecesAsImmutableProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.DecodePiecesAsProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.DecodePiecesAsSerializedProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.EncodeMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.EncodeAsIdsMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.EncodeAsImmutableProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.EncodeAsNumpyMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.EncodeAsOffsetMappingMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.EncodeAsPiecesMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.EncodeAsProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.EncodeAsSerializedProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.GetPieceSizeMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.GetScoreMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.IdToPieceMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.IsByteMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.IsControlMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.IsUnknownMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.IsUnusedMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.LoadMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.LoadFromFileMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.LoadFromSerializedProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.NBestEncodeMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.NBestEncodeAsIdsMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.NBestEncodeAsImmutableProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.NBestEncodeAsNumpyMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.NBestEncodeAsPiecesMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.NBestEncodeAsProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.NBestEncodeAsSerializedProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.NormalizeMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.OverrideNormalizerSpecMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.ParallelEncodeMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.ParallelEncodeAsIdsMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.ParallelEncodeAsImmutableProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.ParallelEncodeAsNumpyMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.ParallelEncodeAsPiecesMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.ParallelEncodeAsProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.ParallelEncodeAsSerializedProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.PieceToIdMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.SampleEncodeAndScoreMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.SampleEncodeAndScoreAsIdsMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.SampleEncodeAndScoreAsImmutableProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.SampleEncodeAndScoreAsPiecesMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.SampleEncodeAndScoreAsProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.SampleEncodeAndScoreAsSerializedProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.SampleEncodeAsIdsMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.SampleEncodeAsImmutableProtoMethodpython/src/sentencepiece/__init__.py

SentencePieceProcessor.SampleEncodeAsNumpyMethodpython/src/sentencepiece/__init__.py

… +1200 more

Signature changed

no signature changes

Added new API surface in @1.0.0

EncodeMethodsrc/bpe_model.cc

UpdateActiveSymbolsMethodsrc/bpe_model_trainer.cc

IgnoreErrorMethodsrc/deps/status.cc

StatusMethodsrc/deps/status.cc

RepClasssrc/deps/status.cc

ToStringMethodsrc/deps/status.cc

codeMethodsrc/deps/status.cc

messageMethodsrc/deps/status.cc

operator!=Methodsrc/deps/status.cc

operator==Methodsrc/deps/status.cc

set_error_messageMethodsrc/deps/status.cc

~StatusMethodsrc/deps/status.cc

CodeEnumsrc/deps/status.h

OkStatusFunctionsrc/deps/status.h

StatusClasssrc/deps/status.h

Status.RepClasssrc/deps/status.h

Status.okMethodsrc/deps/status.h

StatusBuilderClasssrc/deps/status_builder.h

StatusBuilder.StatusBuilderMethodsrc/deps/status_builder.h

ThreadPoolClasssrc/deps/threadpool.h

ThreadPool.ScheduleMethodsrc/deps/threadpool.h

ThreadPool.StartWorkersMethodsrc/deps/threadpool.h

ThreadPool.ThreadPoolMethodsrc/deps/threadpool.h

ThreadPool.~ThreadPoolMethodsrc/deps/threadpool.h

FreeList.operator[]Methodsrc/freelist.h

DecodeIdsAsSerializedProtoMethodsrc/sentencepiece_processor.cc

DecodePiecesAsSerializedProtoMethodsrc/sentencepiece_processor.cc

EncodeAsSerializedProtoMethodsrc/sentencepiece_processor.cc

NBestEncodeAsSerializedProtoMethodsrc/sentencepiece_processor.cc

SampleEncodeAsSerializedProtoMethodsrc/sentencepiece_processor.cc

mainFunctionsrc/spm_train_main.cc

~TrainerInterfaceMethodsrc/trainer_interface.cc

Model.max_scoreMethodsrc/unigram_model.h

RandomGeneratorStorageClasssrc/util.cc

RandomGeneratorStorage.DeleteMethodsrc/util.cc

RandomGeneratorStorage.GetMethodsrc/util.cc

RandomGeneratorStorage.RandomGeneratorStorageMethodsrc/util.cc

RandomGeneratorStorage.~RandomGeneratorStorageMethodsrc/util.cc

WideToUtf8Functionsrc/util.cc

FingerprintCatFunctionsrc/util.h

ItoaFunctionsrc/util.h

STLDeleteElementsFunctionsrc/util.h

lexical_castFunctionsrc/util.h

mixFunctionsrc/util.h

string_view_hashClasssrc/util.h

string_view_hash.operator()Methodsrc/util.h

ItemClasssrc/util_test.cc

Item.ItemMethodsrc/util_test.cc

Item.~ItemMethodsrc/util_test.cc