| 223 | } |
| 224 | |
| 225 | ExecutionStats score_file(const std::string& source_path, |
| 226 | const std::string& target_path, |
| 227 | const std::string& output_path, |
| 228 | size_t max_batch_size, |
| 229 | size_t read_batch_size, |
| 230 | const std::string& batch_type_str, |
| 231 | size_t max_input_length, |
| 232 | dim_t offset, |
| 233 | bool with_tokens_score, |
| 234 | const TokenizeFn& source_tokenize_fn, |
| 235 | const TokenizeFn& target_tokenize_fn, |
| 236 | const DetokenizeFn& target_detokenize_fn) { |
| 237 | if (bool(source_tokenize_fn) != bool(target_tokenize_fn) |
| 238 | || bool(target_tokenize_fn) != bool(target_detokenize_fn)) |
| 239 | throw std::invalid_argument("source_tokenize_fn, target_tokenize_fn, and target_detokenize_fn should all be set or none at all"); |
| 240 | |
| 241 | const auto batch_type = str_to_batch_type(batch_type_str); |
| 242 | ScoringOptions options; |
| 243 | options.max_input_length = max_input_length; |
| 244 | options.offset = offset; |
| 245 | std::shared_lock lock(_mutex); |
| 246 | assert_model_is_ready(); |
| 247 | |
| 248 | if (source_tokenize_fn) { |
| 249 | return _pool->score_raw_text_file(source_path, |
| 250 | target_path, |
| 251 | output_path, |
| 252 | source_tokenize_fn, |
| 253 | target_tokenize_fn, |
| 254 | target_detokenize_fn, |
| 255 | options, |
| 256 | max_batch_size, |
| 257 | read_batch_size, |
| 258 | batch_type, |
| 259 | with_tokens_score); |
| 260 | } else { |
| 261 | return _pool->score_text_file(source_path, |
| 262 | target_path, |
| 263 | output_path, |
| 264 | options, |
| 265 | max_batch_size, |
| 266 | read_batch_size, |
| 267 | batch_type, |
| 268 | with_tokens_score); |
| 269 | } |
| 270 | } |
| 271 | }; |
| 272 | |
| 273 | |