process_chunks splits the large text into chunks
(&self, file_name: &str)
| 100 | |
| 101 | // process_chunks splits the large text into chunks |
| 102 | pub fn process_chunks(&self, file_name: &str) -> Result<Vec<String>> { |
| 103 | let chunk_size = self |
| 104 | .settings |
| 105 | .embedding_model_chunk_size |
| 106 | .as_ref() |
| 107 | .expect("required chunk size"); |
| 108 | let content = |
| 109 | extract_file_content(file_name).context("failed to extract the file content")?; |
| 110 | let chunks = chunk_text(&content, *chunk_size); |
| 111 | Ok(chunks) |
| 112 | } |
| 113 | |
| 114 | // process_embedding generates the embeddings for different chunk texts parallely |
| 115 | pub async fn process_embeddings( |
no test coverage detected