| 119 | return np.round(L / (1 + np.exp(-k * (x - x0))), 3) |
| 120 | |
| 121 | def read_context_files(self, n): |
| 122 | max_context_length = max(self.context_lengths) |
| 123 | contexts = [] |
| 124 | f = open_file(self.haystack_file, 'r') |
| 125 | for _ in range(n): |
| 126 | context = "" |
| 127 | toks = 0 |
| 128 | while toks < max_context_length: |
| 129 | text = json.loads(f.readline())['text'] |
| 130 | context += text |
| 131 | toks += len(self.enc.encode(text)) |
| 132 | contexts.append(context) |
| 133 | return contexts |
| 134 | |
| 135 | def encode_and_trim(self, context, context_length): |
| 136 | tokens = self.enc.encode(context) |