Method tokenize

src/core/language.cpp:1356–1441

  }

  void language::tokenize(utterance& u) const
  {
    if(!lcfg.tok_sent)
      return;
    if(!u.has_relation("TokIn"))
      return;
    relation& in_rel=u.get_relation("TokIn");
    const std::string sb{"sb"};
    const std::string eb{"eb"};
    const std::string br{"br"};
    std::vector<std::string> input;
    for(auto it=in_rel.begin(); it!=in_rel.end(); ++it)
      {
        input.push_back(sb);
        const std::string& name=it->get("name").as<std::string>();
        str::utf8explode(name, std::back_inserter(input));
        input.push_back(eb);
        if(!it->has_next())
          break;
        if(it->has_feature("break"))
          {
            input.push_back(br);
            continue;
          }
        const std::string& whitespace=it->next().get("whitespace").as<std::string>();
        str::utf8explode(whitespace, std::back_inserter(input));
      }
    std::vector<std::string> output;
    if(!tok_fst.translate(input.begin(), input.end(), std::back_inserter(output)))
      throw tokenization_error("");
    auto in_it=input.cbegin();
    auto in_tok_it=in_rel.begin();
    auto out_tok_it=in_tok_it;
    std::string name;
    bool in_whitespace=false;
    bool first_sb=true;
    item* last_subtok{nullptr};
    for(const auto& out_sym: output)
      {
        if(in_it==input.end() || out_sym!=*in_it)
          {
	    if(out_sym.size()>1 && out_sym[0]=='+' && last_subtok)
	      {
		const std::string subtag=out_sym.substr(1);
		last_subtok->as("TokStructure").set<std::string>("tag_"+subtag, "1");
		continue;
	      }
            if(out_tok_it==in_rel.end())
              throw tokenization_error(name);
            if(name.empty())
              throw tokenization_error("");
	    auto tag=out_sym;
            append_subtoken(*out_tok_it, name, tag);
            name.clear();
	    last_subtok=out_tok_it->as("TokStructure").last_child_ptr();
            out_tok_it=in_tok_it;
            continue;
          }