Method __init__

funasr/models/lora/layers.py:43–73

Initialize Embedding. Args: num_embeddings: TODO. embedding_dim: Size/dimension parameter. r: TODO. lora_alpha: TODO. merge_weights: TODO. **kwargs: Additional keyword arguments.

(
        self,
        num_embeddings: int,
        embedding_dim: int,
        r: int = 0,
        lora_alpha: int = 1,
        merge_weights: bool = True,
        **kwargs
    )

class Embedding(nn.Embedding, LoRALayer):
    # LoRA implemented in a dense layer
    def __init__(
        self,
        num_embeddings: int,
        embedding_dim: int,
        r: int = 0,
        lora_alpha: int = 1,
        merge_weights: bool = True,
        **kwargs
    ):
        """Initialize Embedding.
        
            Args:
                num_embeddings: TODO.
                embedding_dim: Size/dimension parameter.
                r: TODO.
                lora_alpha: TODO.
                merge_weights: TODO.
                **kwargs: Additional keyword arguments.
            """
        nn.Embedding.__init__(self, num_embeddings, embedding_dim, **kwargs)
        LoRALayer.__init__(
            self, r=r, lora_alpha=lora_alpha, lora_dropout=0, merge_weights=merge_weights
        )
        # Actual trainable parameters
        if r > 0:
            self.lora_A = nn.Parameter(self.weight.new_zeros((r, num_embeddings)))
            self.lora_B = nn.Parameter(self.weight.new_zeros((embedding_dim, r)))
            self.scaling = self.lora_alpha / self.r
            # Freezing the pre-trained weight matrix
            self.weight.requires_grad = False
        self.reset_parameters()

    def reset_parameters(self):
        """Reset parameters."""