Get Chunker configuration from environment variables.
()
| 86 | |
| 87 | |
| 88 | def get_chunker_config() -> dict[str, Any]: |
| 89 | """Get Chunker configuration from environment variables.""" |
| 90 | return { |
| 91 | "backend": "sentence", |
| 92 | "config": { |
| 93 | "tokenizer_or_token_counter": "gpt2", |
| 94 | "chunk_size": 512, |
| 95 | "chunk_overlap": 128, |
| 96 | "min_sentences_per_chunk": 1, |
| 97 | }, |
| 98 | } |
| 99 | |
| 100 | |
| 101 | def get_reader_config() -> dict[str, Any]: |