MCPcopy Create free account

hub / github.com/Langboat/Mengzi3 / functions

Functions78 in github.com/Langboat/Mengzi3

⨍ Functions 78◇ Types & classes 9
↓ 14 callersMethodwrite
(self, sizes, doc_idx)
finetune_demo/plugins/indexed_dataset.py:385
↓ 9 callersMethodexists
(path)
finetune_demo/plugins/indexed_dataset.py:217
↓ 7 callersFunctiondata_file_path
(prefix_path)
finetune_demo/plugins/indexed_dataset.py:130
↓ 7 callersFunctionprint_rank_0
If distributed is initialized, print only on rank 0.
finetune_demo/plugins/indexed_dataset.py:28
↓ 6 callersFunctionindex_file_path
(prefix_path)
finetune_demo/plugins/indexed_dataset.py:126
↓ 4 callersFunctionread_longs
(f, n)
finetune_demo/plugins/indexed_dataset.py:97
↓ 4 callersFunctionwrite_longs
(f, a)
finetune_demo/plugins/indexed_dataset.py:103
↓ 2 callersMethod__init__
(self, path)
finetune_demo/plugins/indexed_dataset.py:146
↓ 2 callersMethod_do_init
(self, path, skip_warmup)
finetune_demo/plugins/indexed_dataset.py:487
↓ 2 callersFunction_warmup_mmap_file
(path)
finetune_demo/plugins/indexed_dataset.py:351
↓ 2 callersMethodcheck_index
(self, i)
finetune_demo/plugins/indexed_dataset.py:172
↓ 2 callersFunctioncode
(dtype)
finetune_demo/plugins/indexed_dataset.py:119
↓ 2 callersMethodread_data
(self, path)
finetune_demo/plugins/indexed_dataset.py:169
↓ 2 callersFunctionset_seed
(seed)
examples/base_streaming_gen.py:42
↓ 2 callersMethodsize
(self, index)
finetune_demo/plugins/indexed_dataset.py:213
↓ 1 callersFunction__best_fitting_dtype
(vocab_size=None)
finetune_demo/plugins/indexed_dataset.py:37
↓ 1 callersMethod_get_pointers
(sizes)
finetune_demo/plugins/indexed_dataset.py:374
↓ 1 callersMethodadd_item
(self, tensor)
finetune_demo/plugins/indexed_dataset.py:299
↓ 1 callersMethodconcat_encode
(self, fin, file_path, stop_token_ids)
finetune_demo/finetune.py:324
↓ 1 callersMethoddtype
(self)
finetune_demo/plugins/indexed_dataset.py:454
↓ 1 callersMethodend_document
(self)
finetune_demo/plugins/indexed_dataset.py:307
↓ 1 callersMethodfinalize
(self, index_file)
finetune_demo/plugins/indexed_dataset.py:335
↓ 1 callersFunctiongenerate
(inputs)
examples/base_streaming_gen.py:49
↓ 1 callersFunctionget_num_lines
(file_path)
finetune_demo/finetune.py:291
↓ 1 callersFunctioninfer_dataset_impl
(path)
finetune_demo/plugins/indexed_dataset.py:45
↓ 1 callersFunctionmain
()
finetune_demo/finetune.py:423
↓ 1 callersFunctionmake_builder
(out_file, impl, vocab_size=None)
finetune_demo/plugins/indexed_dataset.py:63
↓ 1 callersFunctionparse_args
()
examples/base_streaming_gen.py:29
↓ 1 callersFunctionparse_args
()
finetune_demo/finetune.py:62
↓ 1 callersFunctionpreprocess_data
(args, key, level, dataset_impl='mmap')
finetune_demo/finetune.py:385
↓ 1 callersMethodread_index
(self, path)
finetune_demo/plugins/indexed_dataset.py:152
↓ 1 callersMethodwriter
(cls, path, dtype)
finetune_demo/plugins/indexed_dataset.py:362
Method__del__
(self)
finetune_demo/plugins/indexed_dataset.py:176
Method__del__
(self)
finetune_demo/plugins/indexed_dataset.py:449
Method__del__
(self)
finetune_demo/plugins/indexed_dataset.py:500
Method__enter__
(self)
finetune_demo/plugins/indexed_dataset.py:364
Method__exit__
(self, exc_type, exc_val, exc_tb)
finetune_demo/plugins/indexed_dataset.py:402
Method__getitem__
(self, idx)
finetune_demo/finetune.py:307
Method__getitem__
(self, idx)
finetune_demo/plugins/indexed_dataset.py:181
Method__getitem__
(self, idx)
finetune_demo/plugins/indexed_dataset.py:261
Method__getitem__
(self, i)
finetune_demo/plugins/indexed_dataset.py:466
Method__getitem__
(self, idx)
finetune_demo/plugins/indexed_dataset.py:509
Method__getstate__
(self)
finetune_demo/plugins/indexed_dataset.py:481
Method__init__
(self, indexed_dataset)
finetune_demo/finetune.py:301
Method__init__
(self, path, max_length=4096)
finetune_demo/finetune.py:317
Method__init__
(self, path)
finetune_demo/plugins/indexed_dataset.py:227
Method__init__
(self, out_file, dtype=np.int32)
finetune_demo/plugins/indexed_dataset.py:290
Method__init__
(self, path, skip_warmup=False)
finetune_demo/plugins/indexed_dataset.py:407
Method__init__
(self, path, skip_warmup=False)
finetune_demo/plugins/indexed_dataset.py:472
Method__init__
(self, out_file, dtype=np.int64)
finetune_demo/plugins/indexed_dataset.py:580
Method__len__
(self)
finetune_demo/finetune.py:304
Method__len__
(self)
finetune_demo/plugins/indexed_dataset.py:207
Method__len__
(self)
finetune_demo/plugins/indexed_dataset.py:469
Method__len__
(self)
finetune_demo/plugins/indexed_dataset.py:505
Method__setstate__
(self, state)
finetune_demo/plugins/indexed_dataset.py:484
Methodadd_doc
(self, tensor, sizes)
finetune_demo/plugins/indexed_dataset.py:591
Methodadd_item
(self, tensor)
finetune_demo/plugins/indexed_dataset.py:586
Functioncreate_doc_idx
(sizes)
finetune_demo/plugins/indexed_dataset.py:134
Functiondataset_exists
(path, impl)
finetune_demo/plugins/indexed_dataset.py:90
Methoddoc_idx
(self)
finetune_demo/plugins/indexed_dataset.py:462
Methoddoc_idx
(self)
finetune_demo/plugins/indexed_dataset.py:560
Methodend_document
(self)
finetune_demo/plugins/indexed_dataset.py:597
Methodexists
(path)
finetune_demo/plugins/indexed_dataset.py:574
Methodfinalize
(self, index_file)
finetune_demo/plugins/indexed_dataset.py:613
Methodget
Retrieves a single item from the dataset with the option to only return a portion of the item. get(idx) is the same as [idx] but get
finetune_demo/plugins/indexed_dataset.py:538
Functionget_available_dataset_impl
()
finetune_demo/plugins/indexed_dataset.py:41
Methodget_doc_idx
(self)
finetune_demo/plugins/indexed_dataset.py:563
Functionmake_dataset
(path, impl, skip_warmup=False)
finetune_demo/plugins/indexed_dataset.py:71
Methodmerge_file_
(self, another_file)
finetune_demo/plugins/indexed_dataset.py:310
Methodmerge_file_
(self, another_file)
finetune_demo/plugins/indexed_dataset.py:600
Methodnum_tokens
(self, index)
finetune_demo/plugins/indexed_dataset.py:210
Methodprefetch
(self, indices)
finetune_demo/plugins/indexed_dataset.py:236
Methodset_doc_idx
(self, doc_idx_)
finetune_demo/plugins/indexed_dataset.py:566
Methodsizes
(self)
finetune_demo/plugins/indexed_dataset.py:458
Methodsizes
(self)
finetune_demo/plugins/indexed_dataset.py:556
Methodsupports_prefetch
(self)
finetune_demo/plugins/indexed_dataset.py:222
Methodsupports_prefetch
(self)
finetune_demo/plugins/indexed_dataset.py:233
Methodsupports_prefetch
(self)
finetune_demo/plugins/indexed_dataset.py:570