(file_path)
| 79 | |
| 80 | |
| 81 | def process_csv_dask(file_path): |
| 82 | try: |
| 83 | df = pd.read_csv(file_path) |
| 84 | df["query_embedding"] = df["query"].apply( |
| 85 | lambda x: get_embedding(x, engine=openai_embedding_model) |
| 86 | ) |
| 87 | df["statement_embedding"] = df["statement"].apply( |
| 88 | lambda x: get_embedding(x, engine=openai_embedding_model) |
| 89 | ) |
| 90 | |
| 91 | # check if the output directory exists |
| 92 | if not os.path.exists(output_dir): |
| 93 | os.makedirs(output_dir, exist_ok=True) |
| 94 | |
| 95 | output_file_path = os.path.join(output_dir, os.path.basename(file_path)) |
| 96 | df.to_csv(output_file_path, index=False) |
| 97 | |
| 98 | return output_file_path |
| 99 | except Exception as e: |
| 100 | logger.error(f"Error processing {file_path}: {e}") |
| 101 | return None |
| 102 | |
| 103 | def process_all_csv_files(directory): |
| 104 | client = Client() |
nothing calls this directly
no test coverage detected