| 151 | |
| 152 | |
| 153 | class ClipDetector: |
| 154 | # fmt: off |
| 155 | plain_labels = ["bicycle", "boat", "bus", "car", "fire hydrant", "motorcycle", "traffic light", # YOLO TASKS |
| 156 | "bridge", "chimney", "crosswalk", "mountain", "palm tree", "stair", "tractor", "taxi"] |
| 157 | |
| 158 | all_labels = ["a bicycle", "a boat", "a bus", "a car", "a fire hydrant", "a motorcycle", "a traffic light", # YOLO TASKS |
| 159 | "the front or bottom or side of a concrete or steel bridge supported by concrete pillars over a street or highway", |
| 160 | "A close-up of a chimney on a house, with rooftops and ceiling below", |
| 161 | "striped pedestrian crossing with white/yellow of a crosswalk stretching over a gray ground of a street", |
| 162 | "An californian green or grey landscape with trees or a bridge or street or road connecting two mountain slopes", |
| 163 | "A feather-like warm palm growing behind to a tiled rooftop, with a californian road or street", |
| 164 | "a stairway for pedestrians in front of a house or building leading to a walkway", |
| 165 | "a tractor or agricultural vehicle driving on a street or field", |
| 166 | "a taxi or a yellow car", |
| 167 | "a house wall", |
| 168 | "an empty street"] |
| 169 | # fmt: on |
| 170 | |
| 171 | thresholds = { |
| 172 | "bridge": 0.7285372716747225, |
| 173 | "chimney": 0.7918647485226393, |
| 174 | "crosswalk": 0.8879293048381806, |
| 175 | "mountain": 0.5551278884819476, |
| 176 | "palm tree": 0.8093279512040317, |
| 177 | "stair": 0.9112694561691023, |
| 178 | "tractor": 0.9385110986077537, |
| 179 | "taxi": 0.7967491503432393, |
| 180 | } |
| 181 | |
| 182 | area_captcha_labels = { |
| 183 | "bridge": "A detailed perspective of a concrete bridge with cylindrical and rectangular supports spanning over a wide highway.", |
| 184 | "chimney": "A close-up of a chimney on a house, with rooftops and ceiling below", |
| 185 | "crosswalk": "striped pedestrian crossing with white/yellow of a crosswalk stretching over a gray ground of a street", |
| 186 | "mountain": "An californian green or grey landscape with trees or a bridge or street or road connecting two mountain slopes", |
| 187 | "palm tree": "A feather-like warm palm growing behind to a tiled rooftop, with a californian road or street", |
| 188 | "stair": "a stairway for pedestrians in front of a house or building leading to a walkway", |
| 189 | "tractor": "a tractor or agricultural vehicle", |
| 190 | "taxi": "a yellow car or taxi", |
| 191 | } |
| 192 | |
| 193 | def __init__(self) -> None: |
| 194 | pass |
| 195 | |
| 196 | def clip_detect_vit(self, images: List[cv2.typing.MatLike], task_type: str) -> List[bool]: |
| 197 | response = [] |
| 198 | inputs = detection_models.vit_processor(text=self.all_labels, images=images, return_tensors="pt", padding=True) |
| 199 | with no_grad(): |
| 200 | outputs = detection_models.vit_model(**inputs) |
| 201 | logits_per_image = outputs.logits_per_image # this is the image-text similarity score |
| 202 | probs = logits_per_image.softmax(dim=1) |
| 203 | results = probs.tolist() |
| 204 | |
| 205 | for result in results: |
| 206 | task_index = self.plain_labels.index(task_type) |
| 207 | prediction = result[task_index] |
| 208 | choice = prediction >= (self.thresholds[task_type] - 0.2) |
| 209 | |
| 210 | response.append(choice) |