Download URLs, extract text from the resulting HTML files, and return sandbox handles for both inputs and extracted outputs. Returns: { downloaded: [{ url, handle }], extracted: { outputs: [ ] } }
(
ctx: Context,
urls: List[str],
output_parent: Optional[str] = None,
)
| 81 | |
| 82 | |
| 83 | async def web_fetch_and_parse_urls( |
| 84 | ctx: Context, |
| 85 | urls: List[str], |
| 86 | output_parent: Optional[str] = None, |
| 87 | ) -> dict: |
| 88 | """ |
| 89 | Download URLs, extract text from the resulting HTML files, and |
| 90 | return sandbox handles for both inputs and extracted outputs. |
| 91 | |
| 92 | Returns: |
| 93 | { |
| 94 | downloaded: [{ url, handle }], |
| 95 | extracted: { |
| 96 | outputs: [<handles to .txt files>] |
| 97 | } |
| 98 | } |
| 99 | """ |
| 100 | rt = MANAGER.get(_sid(ctx)) |
| 101 | |
| 102 | downloaded = await web_fetch_urls(ctx, urls) |
| 103 | if "error" in downloaded: |
| 104 | return downloaded |
| 105 | |
| 106 | # Resolve input handles to real paths for the parser (server-side only) |
| 107 | input_paths: List[str] = [] |
| 108 | for item in downloaded["results"]: |
| 109 | try: |
| 110 | p = rt.resolve(item["handle"], need="r") |
| 111 | input_paths.append(str(p)) |
| 112 | except Exception: |
| 113 | continue |
| 114 | |
| 115 | if not input_paths: |
| 116 | return _err("NO_INPUTS", message="No inputs resolved for extraction") |
| 117 | |
| 118 | parser_kwargs: Dict[str, Any] = {} |
| 119 | if output_parent: |
| 120 | try: |
| 121 | out_dir = rt.resolve(output_parent, need="list") |
| 122 | out_dir.mkdir(parents=True, exist_ok=True) |
| 123 | parser_kwargs["output_dir"] = str(out_dir) |
| 124 | except Exception as e: |
| 125 | return _err(type(e).__name__, message=str(e)) |
| 126 | |
| 127 | parsed = await extract_text_from_files(input_paths, **parser_kwargs) |
| 128 | |
| 129 | outputs: List[str] = [] |
| 130 | if isinstance(parsed, dict): |
| 131 | filenames = parsed.get("filenames") or parsed.get("outputs") or [] |
| 132 | for fp in filenames: |
| 133 | try: |
| 134 | h = rt.publish_path(filename_to_path(fp), perm="r") |
| 135 | outputs.append(h) |
| 136 | except Exception: |
| 137 | continue |
| 138 | |
| 139 | rt.touch() |
| 140 | return _ok( |
nothing calls this directly
no test coverage detected