Function ScoreQuadScriptSpan

cld2/internal/scoreonescriptspan.cc:1231–1277

Score RTypeMany scriptspan into doc_tote and vec, updating scoringcontext We have a scriptspan with all lowercase text in one script. Look up quadgrams and octagrams, saving the hits in three parallel vectors. Score from those vectors in chunks, toting each chunk to get a single language, and combining into the overall document score. The hit vectors in general are not big enough to handle and ent

// Returns updated doc_tote
// If vec != NULL, appends to that vector of ResultChunk's
void ScoreQuadScriptSpan(const LangSpan& scriptspan,
                         ScoringContext* scoringcontext,
                         DocTote* doc_tote,
                         ResultChunkVector* vec) {
  // Allocate three parallel arrays of scoring hits
  ScoringHitBuffer* hitbuffer = new ScoringHitBuffer;
  hitbuffer->init();
  hitbuffer->ulscript = scriptspan.ulscript;

  scoringcontext->prior_chunk_lang = UNKNOWN_LANGUAGE;
  scoringcontext->oldest_distinct_boost = 0;

  // Incoming scriptspan has a single leading space at scriptspan.text[0]
  // and three trailing spaces then NUL at scriptspan.text[text_bytes + 0/1/2/3]

  int letter_offset = 1;        // Skip initial space
  hitbuffer->lowest_offset = letter_offset;
  int letter_limit = scriptspan.text_bytes;
  while (letter_offset < letter_limit) {
    //
    // Fill up one hitbuffer, possibly splicing onto previous fragment
    //
    // NOTE: GetQuadHits deals with close repeats
    // NOTE: After last chunk there is always a hitbuffer entry with an offset
    // just off the end of the text = next_offset.
    int next_offset = GetQuadHits(scriptspan.text, letter_offset, letter_limit,
                                  scoringcontext, hitbuffer);
    // If true, there is more text to process in this scriptspan
    // NOTE: GetOctaHitVectors deals with close repeats,
    // does one hash and two lookups (delta and distinct) per word
    GetOctaHits(scriptspan.text, letter_offset, next_offset,
                scoringcontext, hitbuffer);

    //
    // Score one hitbuffer in chunks to summarybuffer
    //
    bool more_to_come = next_offset < letter_limit;
    bool score_cjk = false;
    ProcessHitBuffer(scriptspan, letter_offset, scoringcontext, doc_tote, vec,
                     more_to_come, score_cjk, hitbuffer);
    SpliceHitBuffer(hitbuffer, next_offset);

    letter_offset = next_offset;
  }

  delete hitbuffer;
}


// Score one scriptspan into doc_tote and vec, updating scoringcontext