Class Text_Lora

code/st_model.py:142–319

class Text_Lora(nn.Module):
    def __init__(self, args):
        super(Text_Lora, self).__init__()
        self.args = args

        if args.text_encoder == 'SentenceBert':
            self.tokenizer = AutoTokenizer.from_pretrained(
                "sentence-transformers/multi-qa-distilbert-cos-v1")
            self.textmodel = AutoModel.from_pretrained(
                "sentence-transformers/multi-qa-distilbert-cos-v1")
            self.target_modules=["q_lin", "v_lin"]
        #Roberta
        if args.text_encoder == 'roberta':
            self.tokenizer = RobertaTokenizer.from_pretrained('roberta-base',)
            self.textmodel = RobertaModel.from_pretrained('roberta-base')
            self.target_modules = ["query", "value"]
        elif args.text_encoder == 'bert':
            self.tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
            self.textmodel = BertModel.from_pretrained('bert-base-uncased',device_map="auto")
            self.target_modules = ["query", "value"]
        elif args.text_encoder == 'bert-large':
            self.tokenizer = BertTokenizer.from_pretrained('bert-large-uncased')
            self.textmodel = BertModel.from_pretrained('bert-large-uncased',device_map="auto")
            self.target_modules = ["query", "value"]

        elif args.text_encoder == 't5':
            self.tokenizer = T5Tokenizer.from_pretrained("t5-large")
            self.textmodel = T5Model.from_pretrained("t5-large",device_map="auto")
            self.target_modules = ["q", "v"]
        elif args.text_encoder == 'llama':
            self.tokenizer = AutoTokenizer.from_pretrained("/data/yuhanli/Llama-2-7b-hf")
            self.textmodel = AutoModel.from_pretrained("/data/yuhanli/Llama-2-7b-hf", device_map="auto")
            # self.textmodel = AutoModel.from_pretrained("/data/yuhanli/Llama-2-7b-hf")
            self.target_modules = ["q_proj", "v_proj"]
            self.tokenizer.pad_token = self.tokenizer.eos_token
        
        self.config = LoraConfig(
            task_type="SEQ_CLS",
            r=4,
            lora_alpha=16,
            # target_modules=["q_lin", "v_lin"],
            target_modules=self.target_modules,
            lora_dropout=0.1,
        )
        self.lora_model = LoraModel(self.textmodel, self.config, "default")
        self.descriptions = {
            "Cora": "The Cora dataset is a fundamental resource in the field of graph learning, particularly within the realm of machine learning research. It represents a network of scientific publications. There are 7 categories in Cora: Theory: This category covers theoretical aspects of machine learning and AI. Reinforcement Learning: This category includes research on reinforcement learning, a type of machine learning where an agent learns to make decisions to achieve a goal, focusing on algorithms, methodologies, and applications in decision-making areas. Genetic Algorithms: This category deals with genetic algorithms, a type of optimization algorithm inspired by natural evolution. Neural Networks: This category focuses on artificial neural networks, a subset of machine learning mimicking the human brain, covering various architectures, training techniques, and applications. Probabilistic Methods: This category pertains to research on probabilistic methods in machine learning, using probability mathematics to handle uncertainty and make predictions. Case Based: This category focuses on case-based reasoning in AI, a method that solves new problems by referring to similar past cases. Rule Learning: This category is about rule-based learning in machine learning, involving the generation of rules for decision-making systems, focusing on algorithms, transparency, and applications in fields requiring interpretability. The average degree of Cora is 4.",
            "Citeseer": "The Citeseer dataset is a prominent academic resource in the field of computer science, categorizing publications into six distinct areas. These are Agents, focusing on intelligent agents; Machine Learning (ML), covering all aspects of learning techniques and applications; Information Retrieval (IR), dealing with data and text indexing and retrieval; Databases (DB), related to database management and data mining; Human-Computer Interaction (HCI), emphasizing computer technology interfaces for humans; and Artificial Intelligence (AI), a broad category encompassing general AI theory and applications, excluding certain subfields. The average degree of this graph is 2.",
            "Pubmed": "The PubMed dataset comprises three categories: Experimental studies on diabetes mechanisms and therapies, Type 1 Diabetes research focusing on autoimmune processes and treatments, and Type 2 Diabetes studies emphasizing insulin resistance and management strategies. Each category addresses specific aspects of diabetes research, aiding in understanding and treating this complex disease. The average degree of this graph is 4.5.",
            "Arxiv": "The arXiv dataset is a notable resource in the field of graph learning, particularly in the area of computer science research. This dataset forms a directed graph representing the citation network among all Computer Science papers on arXiv, as indexed by the Microsoft Academic Graph (MAG). Each node in this network corresponds to a paper, and directed edges indicate citations. The dataset's primary challenge is predicting the 40 subject areas of arXiv CS papers, such as cs.AI, cs.LG, and cs.OS. The task is structured as a 40-class classification problem.",
            "wikics": "The Wiki CS dataset is a comprehensive collection of Wikipedia entries, systematically categorized into ten distinct areas of computer science. These categories include Computational Linguistics, focusing on the intersection of computer science and linguistics; Databases, covering database technologies and theories; Operating Systems, detailing the software that manages computer hardware; Computer Architecture, exploring the design and structure of computer systems; Computer Security, addressing the protection of information systems; Internet Protocols, discussing the rules governing internet data exchange; Computer File Systems, about methods for storing and organizing computer files; Distributed Computing Architecture, concerning computations spread across multiple machines; Web Technology, focusing on the technologies underpinning the web; and Programming Language Topics, which includes various aspects of programming languages. This dataset serves as a valuable resource for understanding diverse computer science topics as represented in Wikipedia, reflecting the breadth and depth of the field.",
            "home": "This graph is of amazon products about home using. There are six categories: Baby Products, Appliances, All Beauty, Luxury Beauty, Office & School Supplies, and Home Improvement. The average degree of this graph is 30.39.",
            "tech": "This graph is of amazon products about technologies. There are three categories: Software, Video Games, and Industrial & Scientific. The average degree of this graph is 87.60.",
            "reddit":"Reddit is also a social network where each node denotes a user, the node features are the content of users’ historically published subreddits, and edges denote whether two users have replied to each other. The prediction task is to classify whether a user is in the top 50% popular (average score of all subreddits).",
            "instagram":"Instagram is a social network where edges represent following relationships, nodes represent users, and the prediction task is to classify commercial and normal users in this network.",
        }

        self.criteria = nn.CrossEntropyLoss()