(self, prompts, max_n_frames)
| 108 | return encodings |
| 109 | |
| 110 | def construct_input(self, prompts, max_n_frames): |
| 111 | max_input_length = max_n_frames * self.n_tokens_per_frame + self.min_buffer_size |
| 112 | max_input_length = int(math.ceil(max_input_length / self.block_size) * self.block_size) |
| 113 | |
| 114 | vision_start = self.tokenizer.encode('<vision>') |
| 115 | vision_end = self.tokenizer.encode('</vision>') |
| 116 | |
| 117 | input_ids = np.zeros((len(prompts), max_input_length), dtype=int) |
| 118 | vision_masks = np.zeros((len(prompts), max_input_length), dtype=bool) |
| 119 | attention_mask = np.zeros((len(prompts), max_input_length), dtype=int) |
| 120 | for i, prompt in enumerate(tqdm(prompts)): |
| 121 | vision = self._read_process_vision(prompt['input_path'], max_n_frames) |
| 122 | text_1 = self.tokenizer.encode(f"<s>You are a helpful assistant. USER: {prompt['question']}\n") |
| 123 | tail = self.tokenizer.encode(" ASSISTANT:") |
| 124 | |
| 125 | tokens, vm = [], [] |
| 126 | tokens.extend(text_1) |
| 127 | vm.extend([False] * len(text_1)) |
| 128 | tokens.extend(vision_start) |
| 129 | vm.extend([False] * len(vision_start)) |
| 130 | tokens.extend(vision) |
| 131 | vm.extend([True] * len(vision)) |
| 132 | tokens.extend(vision_end) |
| 133 | vm.extend([False] * len(vision_end)) |
| 134 | tokens.extend(tail) |
| 135 | vm.extend([False] * len(tail)) |
| 136 | assert len(tokens) < max_input_length, (len(tokens), max_input_length) |
| 137 | assert len(tokens) == len(vm) |
| 138 | input_ids[i, -len(tokens):] = tokens |
| 139 | vision_masks[i, -len(tokens):] = vm |
| 140 | attention_mask[i, -len(tokens):] = 1 |
| 141 | return { |
| 142 | 'input_ids': input_ids, |
| 143 | 'vision_masks': vision_masks, |
| 144 | 'attention_mask': attention_mask |
| 145 | } |
| 146 | |
| 147 | |
| 148 | def _load_model(self): |
no test coverage detected