Process a single sample: search for relevant memories.
(
client, sample, sample_idx, frame, version, top_k, success_records, record_file, file_lock
)
| 52 | |
| 53 | |
| 54 | def process_sample( |
| 55 | client, sample, sample_idx, frame, version, top_k, success_records, record_file, file_lock |
| 56 | ): |
| 57 | """Process a single sample: search for relevant memories.""" |
| 58 | # Skip if already processed |
| 59 | if str(sample_idx) in success_records: |
| 60 | return None |
| 61 | |
| 62 | user_id = f"longbench_v2_{sample_idx}_{version}" |
| 63 | query = sample.get("question", "") |
| 64 | |
| 65 | if not query: |
| 66 | return None |
| 67 | |
| 68 | memories_used, duration_ms, search_results = memos_api_search( |
| 69 | client, query, user_id, top_k, frame |
| 70 | ) |
| 71 | |
| 72 | if not (isinstance(memories_used, list) and any(str(m).strip() for m in memories_used)): |
| 73 | return None |
| 74 | |
| 75 | result = { |
| 76 | "sample_idx": sample_idx, |
| 77 | "_id": sample.get("_id"), |
| 78 | "domain": sample.get("domain"), |
| 79 | "sub_domain": sample.get("sub_domain"), |
| 80 | "difficulty": sample.get("difficulty"), |
| 81 | "length": sample.get("length"), |
| 82 | "question": query, |
| 83 | "choice_A": sample.get("choice_A"), |
| 84 | "choice_B": sample.get("choice_B"), |
| 85 | "choice_C": sample.get("choice_C"), |
| 86 | "choice_D": sample.get("choice_D"), |
| 87 | "answer": sample.get("answer"), |
| 88 | # Raw memories used for RAG answering (aligned with longbench_stx) |
| 89 | "memories_used": memories_used, |
| 90 | # Preserve full search results payload for debugging / analysis |
| 91 | "search_results": search_results, |
| 92 | "search_duration_ms": duration_ms, |
| 93 | } |
| 94 | |
| 95 | # Record successful processing (thread-safe) |
| 96 | with file_lock, open(record_file, "a") as f: |
| 97 | f.write(f"{sample_idx}\n") |
| 98 | f.flush() |
| 99 | |
| 100 | return result |
| 101 | |
| 102 | |
| 103 | def load_dataset_from_local(): |
nothing calls this directly
no test coverage detected