r"""Processes content from local file path, remote URL, string content, Element object, or a binary file object, divides it into chunks by using `Unstructured IO`, and stores their embeddings in the specified vector storage. Args: content (Union[str, Elem
(
self,
content: Union[str, "Element", IO[bytes]],
chunk_type: str = "chunk_by_title",
max_characters: int = 500,
embed_batch: int = 50,
should_chunk: bool = True,
extra_info: Optional[dict] = None,
metadata_filename: Optional[str] = None,
**kwargs: Any,
)
| 70 | self.uio: UnstructuredIO = UnstructuredIO() |
| 71 | |
| 72 | def process( |
| 73 | self, |
| 74 | content: Union[str, "Element", IO[bytes]], |
| 75 | chunk_type: str = "chunk_by_title", |
| 76 | max_characters: int = 500, |
| 77 | embed_batch: int = 50, |
| 78 | should_chunk: bool = True, |
| 79 | extra_info: Optional[dict] = None, |
| 80 | metadata_filename: Optional[str] = None, |
| 81 | **kwargs: Any, |
| 82 | ) -> None: |
| 83 | r"""Processes content from local file path, remote URL, string |
| 84 | content, Element object, or a binary file object, divides it into |
| 85 | chunks by using `Unstructured IO`, and stores their embeddings in the |
| 86 | specified vector storage. |
| 87 | |
| 88 | Args: |
| 89 | content (Union[str, Element, IO[bytes]]): Local file path, remote |
| 90 | URL, string content, Element object, or a binary file object. |
| 91 | chunk_type (str): Type of chunking going to apply. Defaults to |
| 92 | "chunk_by_title". |
| 93 | max_characters (int): Max number of characters in each chunk. |
| 94 | Defaults to `500`. |
| 95 | embed_batch (int): Size of batch for embeddings. Defaults to `50`. |
| 96 | should_chunk (bool): If True, divide the content into chunks, |
| 97 | otherwise skip chunking. Defaults to True. |
| 98 | extra_info (Optional[dict]): Extra information to be added |
| 99 | to the payload. Defaults to None. |
| 100 | metadata_filename (Optional[str]): The metadata filename to be |
| 101 | used for storing metadata. Defaults to None. |
| 102 | **kwargs (Any): Additional keyword arguments for content parsing. |
| 103 | """ |
| 104 | from unstructured.documents.elements import Element |
| 105 | |
| 106 | if isinstance(content, Element): |
| 107 | elements = [content] |
| 108 | elif isinstance(content, IOBase): |
| 109 | elements = ( |
| 110 | self.uio.parse_bytes( |
| 111 | file=content, metadata_filename=metadata_filename, **kwargs |
| 112 | ) |
| 113 | or [] |
| 114 | ) |
| 115 | elif isinstance(content, str): |
| 116 | # Check if the content is URL |
| 117 | parsed_url = urlparse(content) |
| 118 | is_url = all([parsed_url.scheme, parsed_url.netloc]) |
| 119 | if is_url or os.path.exists(content): |
| 120 | elements = ( |
| 121 | self.uio.parse_file_or_url( |
| 122 | input_path=content, |
| 123 | metadata_filename=metadata_filename, |
| 124 | **kwargs, |
| 125 | ) |
| 126 | or [] |
| 127 | ) |
| 128 | else: |
| 129 | elements = [ |
no test coverage detected