| 18 | |
| 19 | |
| 20 | class LLMChainPostprocessing(APICallPostprocessing): |
| 21 | def __init__( |
| 22 | self, |
| 23 | start_tokens: List[int], |
| 24 | end_tokens: List[int], |
| 25 | minimum_percentage: float = 0.1, |
| 26 | ): |
| 27 | self.llmchain = langchain_llmchain |
| 28 | self.api_text = "LLMChain(" |
| 29 | super().__init__(start_tokens, end_tokens, minimum_percentage) |
| 30 | |
| 31 | def add_api_calls( |
| 32 | self, |
| 33 | candidate: int, |
| 34 | outputs: dict, |
| 35 | texts_to_test: List[str], |
| 36 | tokenizer: PreTrainedTokenizerBase, |
| 37 | input_tokens: torch.Tensor, |
| 38 | input_start: int, |
| 39 | nums_to_keep: List[int], |
| 40 | base_loss: float, |
| 41 | *args, |
| 42 | **kwargs |
| 43 | ): |
| 44 | generated_texts = list() |
| 45 | max_token_len = N |
| 46 | max_token_len_base = N |
| 47 | for j in range(len(outputs)): |
| 48 | outputs[j]["LLMChain"] = outputs[j]["generated_text"].replace( |
| 49 | texts_to_test[candidate], "" |
| 50 | ) |
| 51 | outputs[j]["Generated"] = outputs[j]["generated_text"].split("Output:")[-1] |
| 52 | if "]" in outputs[j]["LLMChain"]: |
| 53 | outputs[j]["LLMChain"] = ( |
| 54 | outputs[j]["LLMChain"].replace("LLMChain(", "").split("]")[0] |
| 55 | ) |
| 56 | if ")" in outputs[j]["LLMChain"]: |
| 57 | outputs[j]["LLMChain"] = outputs[j]["LLMChain"].split(")")[0] |
| 58 | if outputs[j]["LLMChain"][0] == "\"": |
| 59 | outputs[j]["LLMChain"] = outputs[j]["LLMChain"][1:] |
| 60 | if outputs[j]["LLMChain"][-1] == "\"": |
| 61 | outputs[j]["LLMChain"] = outputs[j]["LLMChain"][:-1] |
| 62 | outputs[j]["LLMChain_text"] = ( |
| 63 | "[LLMChain(" + outputs[j]["LLMChain"] + ")" |
| 64 | ) |
| 65 | base_inputs = tokenizer( |
| 66 | outputs[j]["LLMChain_text"] + "]" + "\n", |
| 67 | return_tensors="pt", |
| 68 | )["input_ids"].cuda() |
| 69 | outputs[j]["LLMChain"] = str(self.llmchain(outputs[j]["LLMChain"])) |
| 70 | outputs[j]["LLMChain_output"] = [outputs[j]["LLMChain_text"][1:], outputs[j]["LLMChain"]] |
| 71 | outputs[j]["LLMChain_text"] = ( |
| 72 | outputs[j]["LLMChain_text"] |
| 73 | + "->" |
| 74 | + outputs[j]["LLMChain"] |
| 75 | + "]" |
| 76 | ) |
| 77 | test_inputs = tokenizer( |
no outgoing calls
no test coverage detected