(id, values)
| 412 | |
| 413 | |
| 414 | def pre_process_vector(id, values): |
| 415 | corrected_values = [float(v) for v in values] |
| 416 | |
| 417 | # Normalize the vector for cosine distance |
| 418 | vector_array = np.array(corrected_values).reshape(1, -1) |
| 419 | normalized_values = normalize(vector_array, norm='l2').flatten().tolist() |
| 420 | |
| 421 | result = { |
| 422 | "id": str(id), # Keep as string for server compatibility |
| 423 | "dense_values": normalized_values, |
| 424 | } |
| 425 | |
| 426 | # Only add document_id if id is an integer |
| 427 | if isinstance(id, int): |
| 428 | result["document_id"] = f"doc_{id // 10}" |
| 429 | |
| 430 | return result |
| 431 | |
| 432 | |
| 433 | def read_dataset_from_parquet(dataset_name, max_vectors=50000): |
no outgoing calls
no test coverage detected