Function ScoreCJKScriptSpan

cld2/internal/scoreonescriptspan.cc:1163–1214

Score RTypeCJK scriptspan into doc_tote and vec, updating scoringcontext

// Score RTypeCJK scriptspan into doc_tote and vec, updating scoringcontext
void ScoreCJKScriptSpan(const LangSpan& scriptspan,
                        ScoringContext* scoringcontext,
                        DocTote* doc_tote,
                        ResultChunkVector* vec) {
  // Allocate three parallel arrays of scoring hits
  ScoringHitBuffer* hitbuffer = new ScoringHitBuffer;
  hitbuffer->init();
  hitbuffer->ulscript = scriptspan.ulscript;

  scoringcontext->prior_chunk_lang = UNKNOWN_LANGUAGE;
  scoringcontext->oldest_distinct_boost = 0;

  // Incoming scriptspan has a single leading space at scriptspan.text[0]
  // and three trailing spaces then NUL at scriptspan.text[text_bytes + 0/1/2/3]

  int letter_offset = 1;        // Skip initial space
  hitbuffer->lowest_offset = letter_offset;
  int letter_limit = scriptspan.text_bytes;
  while (letter_offset < letter_limit) {
    if (scoringcontext->flags_cld2_verbose) {
      fprintf(scoringcontext->debug_file, " ScoreCJKScriptSpan[%d,%d)<br>\n",
              letter_offset, letter_limit);
    }
    //
    // Fill up one hitbuffer, possibly splicing onto previous fragment
    //
    // NOTE: GetUniHits deals with close repeats
    // NOTE: After last chunk there is always a hitbuffer entry with an offset
    // just off the end of the text = next_offset.
    int next_offset = GetUniHits(scriptspan.text, letter_offset, letter_limit,
                                  scoringcontext, hitbuffer);
    // NOTE: GetBiHitVectors deals with close repeats,
    // does one hash and two lookups (delta and distinct) per word
    GetBiHits(scriptspan.text, letter_offset, next_offset,
                scoringcontext, hitbuffer);

    //
    // Score one hitbuffer in chunks to summarybuffer
    //
    bool more_to_come = next_offset < letter_limit;
    bool score_cjk = true;
    ProcessHitBuffer(scriptspan, letter_offset, scoringcontext, doc_tote, vec,
                     more_to_come, score_cjk, hitbuffer);
    SpliceHitBuffer(hitbuffer, next_offset);

    letter_offset = next_offset;
  }

  delete hitbuffer;
  // Context across buffers is not connected yet
  scoringcontext->prior_chunk_lang = UNKNOWN_LANGUAGE;
}