Method Tokenize

be/src/runtime/datetime-iso-sql-format-tokenizer.cc:103–121

const int IsoSqlFormatTokenizer::FRACTIONAL_MAX_LEN = 9;

FormatTokenizationResult IsoSqlFormatTokenizer::Tokenize() {
  DCHECK(dt_ctx_ != nullptr);
  DCHECK(dt_ctx_->fmt != nullptr);
  DCHECK(dt_ctx_->fmt_len > 0);
  DCHECK(dt_ctx_->toks.size() == 0);
  DCHECK(used_tokens_.empty());
  if (dt_ctx_->fmt_len > MAX_FORMAT_LENGTH) return TOO_LONG_FORMAT_ERROR;
  const char* str_end = dt_ctx_->fmt + dt_ctx_->fmt_len;
  const char* current_pos = dt_ctx_->fmt;
  ProcessFXModifier(&current_pos);
  while (current_pos < str_end) {
    ProcessSeparators(&current_pos);
    if (current_pos == str_end) break;
    FormatTokenizationResult parse_result = ProcessNextToken(&current_pos);
    if (parse_result != SUCCESS) return parse_result;
  }
  if (dt_ctx_->toks.empty()) return GENERAL_ERROR;
  return CheckIncompatibilities();
}

void IsoSqlFormatTokenizer::ProcessSeparators(const char** current_pos) {
  DCHECK(current_pos != nullptr && *current_pos != nullptr);