| 268 | |
| 269 | |
| 270 | int DiB_trainFromFiles(const char* dictFileName, unsigned maxDictSize, |
| 271 | const char** fileNamesTable, unsigned nbFiles, size_t chunkSize, |
| 272 | ZDICT_legacy_params_t* params, ZDICT_cover_params_t* coverParams, |
| 273 | ZDICT_fastCover_params_t* fastCoverParams, int optimize) |
| 274 | { |
| 275 | unsigned const displayLevel = params ? params->zParams.notificationLevel : |
| 276 | coverParams ? coverParams->zParams.notificationLevel : |
| 277 | fastCoverParams ? fastCoverParams->zParams.notificationLevel : |
| 278 | 0; /* should never happen */ |
| 279 | void* const dictBuffer = malloc(maxDictSize); |
| 280 | fileStats const fs = DiB_fileStats(fileNamesTable, nbFiles, chunkSize, displayLevel); |
| 281 | size_t* const sampleSizes = (size_t*)malloc(fs.nbSamples * sizeof(size_t)); |
| 282 | size_t const memMult = params ? MEMMULT : |
| 283 | coverParams ? COVER_MEMMULT: |
| 284 | FASTCOVER_MEMMULT; |
| 285 | size_t const maxMem = DiB_findMaxMem(fs.totalSizeToLoad * memMult) / memMult; |
| 286 | size_t loadedSize = (size_t) MIN ((unsigned long long)maxMem, fs.totalSizeToLoad); |
| 287 | void* const srcBuffer = malloc(loadedSize+NOISELENGTH); |
| 288 | int result = 0; |
| 289 | |
| 290 | /* Checks */ |
| 291 | if ((!sampleSizes) || (!srcBuffer) || (!dictBuffer)) |
| 292 | EXM_THROW(12, "not enough memory for DiB_trainFiles"); /* should not happen */ |
| 293 | if (fs.oneSampleTooLarge) { |
| 294 | DISPLAYLEVEL(2, "! Warning : some sample(s) are very large \n"); |
| 295 | DISPLAYLEVEL(2, "! Note that dictionary is only useful for small samples. \n"); |
| 296 | DISPLAYLEVEL(2, "! As a consequence, only the first %u bytes of each sample are loaded \n", SAMPLESIZE_MAX); |
| 297 | } |
| 298 | if (fs.nbSamples < 5) { |
| 299 | DISPLAYLEVEL(2, "! Warning : nb of samples too low for proper processing ! \n"); |
| 300 | DISPLAYLEVEL(2, "! Please provide _one file per sample_. \n"); |
| 301 | DISPLAYLEVEL(2, "! Alternatively, split files into fixed-size blocks representative of samples, with -B# \n"); |
| 302 | EXM_THROW(14, "nb of samples too low"); /* we now clearly forbid this case */ |
| 303 | } |
| 304 | if (fs.totalSizeToLoad < (unsigned long long)maxDictSize * 8) { |
| 305 | DISPLAYLEVEL(2, "! Warning : data size of samples too small for target dictionary size \n"); |
| 306 | DISPLAYLEVEL(2, "! Samples should be about 100x larger than target dictionary size \n"); |
| 307 | } |
| 308 | |
| 309 | /* init */ |
| 310 | if (loadedSize < fs.totalSizeToLoad) |
| 311 | DISPLAYLEVEL(1, "Not enough memory; training on %u MB only...\n", (unsigned)(loadedSize >> 20)); |
| 312 | |
| 313 | /* Load input buffer */ |
| 314 | DISPLAYLEVEL(3, "Shuffling input files\n"); |
| 315 | DiB_shuffle(fileNamesTable, nbFiles); |
| 316 | |
| 317 | DiB_loadFiles(srcBuffer, &loadedSize, sampleSizes, fs.nbSamples, fileNamesTable, nbFiles, chunkSize, displayLevel); |
| 318 | |
| 319 | { size_t dictSize; |
| 320 | if (params) { |
| 321 | DiB_fillNoise((char*)srcBuffer + loadedSize, NOISELENGTH); /* guard band, for end of buffer condition */ |
| 322 | dictSize = ZDICT_trainFromBuffer_unsafe_legacy(dictBuffer, maxDictSize, |
| 323 | srcBuffer, sampleSizes, fs.nbSamples, |
| 324 | *params); |
| 325 | } else if (coverParams) { |
| 326 | if (optimize) { |
| 327 | dictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, maxDictSize, |
no test coverage detected