Method process_token_buff

sources/compiler/lexer.cpp:481–514

	};

	void
	compiler_type::process_token_buff(std::deque<token_base *> &tokens, std::deque<std::deque<token_base *>> &ast)
	{
		std::deque<token_base *> oldt, expr;
		std::swap(tokens, oldt);
		tokens.clear();
		for (auto &ptr : oldt) {
			if (ptr->get_type() == token_types::signal &&
			        static_cast<token_signal *>(ptr)->get_signal() == signal_types::endline_)
				ptr = new token_endline(ptr->get_line_num());
			if (ptr->get_type() == token_types::action || ptr->get_type() == token_types::endline) {
				if (!expr.empty()) {
					translator.match_grammar(context, expr);
					for (auto &it : expr)
						tokens.push_back(it);
					expr.clear();
				}
				tokens.push_back(ptr);
			}
			else
				expr.push_back(ptr);
		}
		std::deque<token_base *> tmp;
		for (auto &ptr : tokens) {
			tmp.push_back(ptr);
			if (ptr != nullptr && ptr->get_type() == token_types::endline) {
				if (tmp.size() > 1)
					ast.push_back(tmp);
				tmp.clear();
			}
		}
		if (tmp.size() > 1)
			ast.push_back(tmp);
	}

	void compiler_type::translate_into_tokens(const std::deque<char> &char_buff, std::deque<token_base *> &tokens,
	        charset encoding)