| 1958 | } |
| 1959 | |
| 1960 | bool NyquistBase::Tokenizer::Tokenize( |
| 1961 | const wxString& line, bool eof, size_t trimStart, size_t trimEnd) |
| 1962 | { |
| 1963 | auto endToken = [&] { |
| 1964 | if (!tok.empty()) |
| 1965 | { |
| 1966 | tokens.push_back(tok); |
| 1967 | tok = wxT(""); |
| 1968 | } |
| 1969 | }; |
| 1970 | |
| 1971 | for (auto c : |
| 1972 | make_iterator_range(line.begin() + trimStart, line.end() - trimEnd)) |
| 1973 | { |
| 1974 | if (q && !sl && c == wxT('\\')) |
| 1975 | { |
| 1976 | // begin escaped character, only within quotes |
| 1977 | sl = true; |
| 1978 | continue; |
| 1979 | } |
| 1980 | |
| 1981 | if (!sl && c == wxT('"')) |
| 1982 | { |
| 1983 | // Unescaped quote |
| 1984 | if (!q) |
| 1985 | { |
| 1986 | // start of string |
| 1987 | if (!paren) |
| 1988 | // finish previous token |
| 1989 | endToken(); |
| 1990 | // Include the delimiter in the token |
| 1991 | tok += c; |
| 1992 | q = true; |
| 1993 | } |
| 1994 | else |
| 1995 | { |
| 1996 | // end of string |
| 1997 | // Include the delimiter in the token |
| 1998 | tok += c; |
| 1999 | if (!paren) |
| 2000 | endToken(); |
| 2001 | q = false; |
| 2002 | } |
| 2003 | } |
| 2004 | else if (!q && !paren && (c == wxT(' ') || c == wxT('\t'))) |
| 2005 | // Unenclosed whitespace |
| 2006 | // Separate tokens; don't accumulate this character |
| 2007 | endToken(); |
| 2008 | else if (!q && c == wxT(';')) |
| 2009 | // semicolon not in quotes, but maybe in parentheses |
| 2010 | // Lisp style comments with ; (but not with #| ... |#) are allowed |
| 2011 | // within a wrapped header multi-line, so that i18n hint comments may |
| 2012 | // be placed before strings and found by xgettext |
| 2013 | break; |
| 2014 | else if (!q && c == wxT('(')) |
| 2015 | { |
| 2016 | // Start of list or sublist |
| 2017 | if (++paren == 1) |
no test coverage detected