Functions — github.com/JieShibo/MoLE

Method__init__

(self, input_resolution, dim, norm_layer=nn.LayerNorm)

pretrain/megatron/legacy/model/vision/swin_backbone.py:312

Method__init__

(self, dim, input_resolution, depth, num_heads, window_size, mlp_ratio=4., qkv_bias=True, qk_

pretrain/megatron/legacy/model/vision/swin_backbone.py:372

Method__init__

(self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None)

pretrain/megatron/legacy/model/vision/swin_backbone.py:434

Method__init__

(self, img_size=224, patch_size=4, in_chans=3, embed_dim=96, depths=[2, 2, 6, 2], num_heads=[

pretrain/megatron/legacy/model/vision/swin_backbone.py:496

Method__init__

(self, name, block_dataset, title_dataset, data_prefix, num_epochs, max_num_samples, max_seq_

pretrain/megatron/legacy/data/ict_dataset.py:50

Method__init__

(self, p=0.5, radius_min=0.1, radius_max=2.)

pretrain/megatron/legacy/data/vit_dataset.py:19

Method__init__

(self, p)

pretrain/megatron/legacy/data/vit_dataset.py:40

Method__init__

(self, image_size, train=True)

pretrain/megatron/legacy/data/vit_dataset.py:51

Method__init__

(self, image_size, train=True)

pretrain/megatron/legacy/data/vit_dataset.py:80

Method__init__

(self, image_size, train=True)

pretrain/megatron/legacy/data/vit_dataset.py:149

Method__init__

(self, task_name, dataset_name, datapath, tokenizer, max_seq_length)

pretrain/megatron/legacy/data/orqa_wiki_dataset.py:125

Method__init__

(self, name, data_prefix, indexed_dataset, num_samples, seq_length, seed, img_h, img_w)

pretrain/megatron/legacy/data/multimodal_dataset.py:30

Method__init__

( self, root: str, loader: Callable[[str], Any], extensions: O

pretrain/megatron/legacy/data/image_folder.py:142

Method__init__

(self, embedding_path=None, load_from_path=True, rank=None)

pretrain/megatron/legacy/data/realm_index.py:22

Method__init__

(self, embed_size, embed_data=None, use_gpu=False)

pretrain/megatron/legacy/data/realm_index.py:122

Method__init__

(self, start_idx, end_idx, doc_idx, block_idx)

pretrain/megatron/legacy/data/realm_dataset_utils.py:83

Method__init__

(self, mapping_array)

pretrain/megatron/legacy/data/realm_dataset_utils.py:97

Method__init__

(self, total_samples, consumed_samples, micro_batch_size, data_parallel_rank, data_parallel_s

pretrain/megatron/legacy/data/data_samplers.py:56

Method__init__

(self, dataset)

pretrain/megatron/legacy/data/data_samplers.py:105

Method__init__

(self, dataset, total_samples, consumed_samples, micro_batch_size, data_parallel_rank, data_p

pretrain/megatron/legacy/data/data_samplers.py:127

Method__init__

Initialize an ImageNetPolicy. Args: fillcolor (tuple): RGB color components of the color to be used for filling when

pretrain/megatron/legacy/data/autoaugment.py:62

Method__init__

Initialize a SubPolicy. Args: operation1 (str): Key specifying the first augmentation operation. There are fourteen k

pretrain/megatron/legacy/data/autoaugment.py:121

Method__init__

(self, start_idx, end_idx, doc_idx, block_idx)

pretrain/megatron/legacy/data/biencoder_dataset_utils.py:93

Method__init__

(self, mapping_array)

pretrain/megatron/legacy/data/biencoder_dataset_utils.py:107

Method__init__

(self, m, n, k)

pretrain/megatron/legacy/mpu/tests/test_layers.py:306

Method__init__

(self, size, scale=1.0)

pretrain/megatron/legacy/mpu/tests/commons.py:13

Method__init__

(self, name)

pretrain/megatron/core/timers.py:13

Method__init__

(self)

pretrain/megatron/core/timers.py:34

Method__init__

Initialize group of timers. Args: log_level (int): Log level to control what timers are enabled. log_opti

pretrain/megatron/core/timers.py:151

Method__init__

(self)

pretrain/megatron/core/utils.py:67

Method__init__

(self, max_batch_size, max_sequence_length)

pretrain/megatron/core/inference_params.py:5

Method__init__

( self, config: TransformerConfig, hidden_size: int, eps: float = 1e-5,

pretrain/megatron/core/fusions/fused_layer_norm.py:53

Method__init__

( self, input_in_fp16, input_in_bf16, attn_mask_type, scaled_masked_so

pretrain/megatron/core/fusions/fused_softmax.py:111

Method__init__

( self, num_embeddings: int, embedding_dim: int, *, init_method: Calla

pretrain/megatron/core/tensor_parallel/layers.py:160

Method__init__

( self, input_size, output_size, *, config: ModelParallelConfig,

pretrain/megatron/core/tensor_parallel/layers.py:610

Method__init__

(self)

pretrain/megatron/core/tensor_parallel/random.py:84

Method__init__

( self, params: List[torch.nn.Parameter], param_data: Optional[torch.Tensor],

pretrain/megatron/core/distributed/param_and_grad_buffer.py:58

Method__init__

( self, param_dtype: torch.dtype, grad_dtype: torch.dtype, params: List[torch.

pretrain/megatron/core/distributed/param_and_grad_buffer.py:205

Method__init__

( self, config: TransformerConfig, module: torch.nn.Module, data_parallel_grou

pretrain/megatron/core/distributed/distributed_data_parallel.py:40

Method__init__

(self, obj)

pretrain/megatron/core/dist_checkpointing/mapping.py:221

Method__init__

(self, *args, **kwargs)

pretrain/megatron/core/dist_checkpointing/strategies/filesystem_async.py:47

Method__init__

(self, data_parallel_group, cpu_transfer=True)

pretrain/megatron/core/dist_checkpointing/strategies/two_stage.py:94

Method__init__

(self, backend: str, version: int)

pretrain/megatron/core/dist_checkpointing/strategies/base.py:66

Method__init__

Adds parameters specific to PyT Distributed format Args: backend (str): format backend string version (int): format ve

pretrain/megatron/core/dist_checkpointing/strategies/torch.py:379

Method__init__

(self, load_directly_on_device: bool = False)

pretrain/megatron/core/dist_checkpointing/strategies/tensorstore.py:29

Method__init__

(self, *tokenizer_paths: str, **tokenizer_options: Any)

pretrain/megatron/core/datasets/megatron_tokenizer.py:21

Method__init__

( self, indexed_dataset: IndexedDataset, dataset_path: str, indexed_indices: n

pretrain/megatron/core/datasets/t5_dataset.py:62

Method__init__

( self, datasets: List[MegatronDataset], weights: List[float], size: int,

pretrain/megatron/core/datasets/blended_dataset.py:39

Method__init__

( self, indexed_dataset: IndexedDataset, dataset_path: str, indexed_indices: n

pretrain/megatron/core/datasets/bert_dataset.py:48

Method__init__

( self, cls: Type[MidLevelDataset], sizes: List[int], is_built_on_rank: Callab

pretrain/megatron/core/datasets/blended_megatron_dataset_builder.py:40

Method__init__

( self, dataset: Optional[LowLevelDataset], dataset_path: Optional[str], indic

pretrain/megatron/core/datasets/gpt_dataset.py:55

Method__init__

(self, idx_path: str, multimodal: bool)

pretrain/megatron/core/datasets/indexed_dataset.py:222

Method__init__

(self, path_prefix: str, multimodal: bool = False, mmap: bool = True)

pretrain/megatron/core/datasets/indexed_dataset.py:345

Method__init__

( self, bin_path: str, dtype: Type[numpy.number] = numpy.int32, multimodal: bool = False )

pretrain/megatron/core/datasets/indexed_dataset.py:617

Method__init__

( self, dataset: LowLevelDataset, dataset_path: str, indices: numpy.ndarray,

pretrain/megatron/core/datasets/megatron_dataset.py:36

Method__init__

( self, dataset: Optional[LowLevelDataset], dataset_path: Optional[str], indic

pretrain/megatron/core/datasets/megatron_dataset.py:173

Method__init__

( self, indexed_dataset: IndexedDataset, dataset_path: str, indexed_indices: n

pretrain/megatron/core/datasets/masked_dataset.py:100

Method__init__

(self, gpt_dataset: MultiSplitGPTDataset, gpt_tokenizer: Any)

pretrain/megatron/core/datasets/retro/utils.py:83

Method__init__

( self, indexed_dataset: IndexedDataset, dataset_path: str, indexed_indices: n

pretrain/megatron/core/datasets/retro/query/multi_split_gpt_dataset.py:67

Method__init__

( self, num_queried_samples: int, num_neighbors: int, num_retrieved_chunks: in

pretrain/megatron/core/datasets/retro/query/retro_dataset.py:50

Method__init__

(self, sample_dataset: GPTDataset, sample_length: int, chunk_length: int)

pretrain/megatron/core/datasets/retro/query/gpt_chunk_dataset.py:33

Method__init__

( self, db_path: str, indexed_datasets: List[IndexedDataset], chunks: np.ndarr

pretrain/megatron/core/datasets/retro/db/dataset.py:29

Method__init__

( self, config: TransformerConfig, parallel_output: bool, vocab_size: int,

pretrain/megatron/core/models/T5/t5_model.py:34

Method__init__

( self, hidden_size: int, init_method: callable, config: TransformerConfig,

pretrain/megatron/core/models/bert/pooler.py:23

Method__init__

( self, config: TransformerConfig, num_tokentypes: int, transformer_layer_spec

pretrain/megatron/core/models/bert/bert_model.py:43

Method__init__

( self, hidden_size: int, config: TransformerConfig, )

pretrain/megatron/core/models/bert/bert_lm_head.py:18

Method__init__

( self, config: TransformerConfig, transformer_layer_spec: ModuleSpec, vocab_s

pretrain/megatron/core/models/gpt/gpt_model.py:41

Method__init__

( self, config: TransformerConfig, submodules: MLPSubmodules, projector_type:

pretrain/megatron/core/models/vision/multimodal_projector.py:21

Method__init__

( self, transformer_config: TransformerConfig, transformer_layer_spec: ModuleSpec,

pretrain/megatron/core/models/vision/clip_vit_model.py:30

Method__init__

( self, config: TransformerConfig, vocab_size: int, max_sequence_length: int,

pretrain/megatron/core/models/common/embeddings/language_model_embedding.py:26

Method__init__

( self, kv_channels: int, rotary_percent: float, rotary_interleaved: bool = Fa

pretrain/megatron/core/models/common/embeddings/rotary_pos_embedding.py:58

Method__init__

(self, config: TransformerConfig)

pretrain/megatron/core/models/common/vision_module/vision_module.py:16

Method__init__

(self, config: TransformerConfig)

pretrain/megatron/core/models/common/language_module/language_module.py:21

Method__init__

( self, language_transformer_config: TransformerConfig, language_transformer_layer_spe

pretrain/megatron/core/models/multimodal/llava_model.py:31

Method__init__

( self, config: RetroConfig, )

pretrain/megatron/core/models/retro/encoder_attention.py:116

Method__init__

( self, config: RetroConfig, )

pretrain/megatron/core/models/retro/decoder_attention.py:223

Method__init__

( self, config: RetroConfig, submodules: CrossAttentionSubmodules, layer_numbe

pretrain/megatron/core/models/retro/base_attention.py:26

Method__init__

(self)

pretrain/megatron/core/transformer/transformer_layer.py:50

Method__init__

( self, config: TransformerConfig, submodules: TransformerLayerSubmodules, lay

pretrain/megatron/core/transformer/transformer_layer.py:61

Method__init__

(self, *args, **kwargs)

pretrain/megatron/core/transformer/identity_op.py:24

Method__init__

( self, config: TransformerConfig, submodules: SelfAttentionSubmodules, layer_

pretrain/megatron/core/transformer/attention.py:347

Method__init__

( self, config: TransformerConfig, submodules: CrossAttentionSubmodules, layer

pretrain/megatron/core/transformer/attention.py:522

Method__init__

( self, config: TransformerConfig, spec: Union[TransformerBlockSubmodules, ModuleSpec]

pretrain/megatron/core/transformer/transformer_block.py:94

Method__init__

( self, config: TransformerConfig, submodules: MLPSubmodules, is_expert: bool

pretrain/megatron/core/transformer/mlp.py:47

Method__init__

(self, config: TransformerConfig)

pretrain/megatron/core/transformer/module.py:38

Method__init__

( self, config: TransformerConfig, layer_number: int, attn_mask_type: AttnMask

pretrain/megatron/core/transformer/dot_product_attention.py:34

Method__init__

( self, config: TransformerConfig, submodules: MLPSubmodules, is_expert: bool

pretrain/megatron/core/transformer/vanillamlp.py:48

Method__init__

( self, input_size: int, output_size: int, *, config: TransformerConfi

pretrain/megatron/core/transformer/custom_layers/transformer_engine.py:173

Method__init__

( self, input_size: int, output_size: int, *, config: ModelParallelCon

pretrain/megatron/core/transformer/custom_layers/transformer_engine.py:281

Method__init__

( self, input_size: int, output_size: int, *, config: ModelParallelCon

pretrain/megatron/core/transformer/custom_layers/transformer_engine.py:327

Method__init__

( self, config: TransformerConfig, layer_number: int, attn_mask_type: AttnMask

pretrain/megatron/core/transformer/custom_layers/transformer_engine.py:380

Method__init__

Initialize the zero token dropping router.

pretrain/megatron/core/transformer/moe/token_dispatcher.py:62

Method__init__

Initialize the AlltoAll token dispatcher. Args: num_local_experts (int): Number of local experts on the current device.

pretrain/megatron/core/transformer/moe/token_dispatcher.py:282

Method__init__

(self, num_local_experts, config: TransformerConfig, submodules: MLPSubmodules)

pretrain/megatron/core/transformer/moe/experts.py:179

Method__init__

(self, config: TransformerConfig, layer_number: int = None)

pretrain/megatron/core/transformer/moe/moe_layer.py:26

Method__init__

Initialize the zero token dropping router. Args: config (TransformerConfig): The configuration for the transformer model.

pretrain/megatron/core/transformer/moe/router.py:95

Method__init__

Input optimizer is the base optimizer (e.g., Adam).

pretrain/megatron/core/optimizer/optimizer.py:76

Method__init__

( self, optimizer: torch.optim.Optimizer, config: OptimizerConfig, grad_scaler

pretrain/megatron/core/optimizer/optimizer.py:232

Method__init__

( self, optimizer: torch.optim.Optimizer, config: OptimizerConfig, grad_scaler

pretrain/megatron/core/optimizer/optimizer.py:390

Method__init__

(self, chained_optimizers: List[MegatronOptimizer])

pretrain/megatron/core/optimizer/optimizer.py:708

Functions2,110 in github.com/JieShibo/MoLE