Function test_tokenizer

modules/gdscript/tests/test_gdscript.cpp:58–130

namespace GDScriptTests {

static void test_tokenizer(const String &p_code, const Vector<String> &p_lines) {
	GDScriptTokenizerText tokenizer;
	tokenizer.set_source_code(p_code);

	int tab_size = 4;
#ifdef TOOLS_ENABLED
	if (EditorSettings::get_singleton()) {
		tab_size = EditorSettings::get_singleton()->get_setting("text_editor/behavior/indent/size");
	}
#endif // TOOLS_ENABLED
	String tab = String(" ").repeat(tab_size);

	GDScriptTokenizer::Token current = tokenizer.scan();
	while (current.type != GDScriptTokenizer::Token::TK_EOF) {
		StringBuilder token;
		token += " --> "; // Padding for line number.

		if (current.start_line != current.end_line) {
			// Print "vvvvvv" to point at the token.
			StringBuilder pointer;
			pointer += "     "; // Padding for line number.

			int line_width = 0;
			if (current.start_line - 1 >= 0 && current.start_line - 1 < p_lines.size()) {
				line_width = p_lines[current.start_line - 1].replace("\t", tab).length();
			}

			const int offset = MAX(0, current.start_column - 1);
			const int width = MAX(0, line_width - current.start_column + 1);
			pointer += String::chr(' ').repeat(offset) + String::chr('v').repeat(width);

			print_line(pointer.as_string());
		}

		for (int l = current.start_line; l <= current.end_line && l <= p_lines.size(); l++) {
			print_line(vformat("%04d %s", l, p_lines[l - 1]).replace("\t", tab));
		}

		{
			// Print "^^^^^^" to point at the token.
			StringBuilder pointer;
			pointer += "     "; // Padding for line number.

			if (current.start_line == current.end_line) {
				const int offset = MAX(0, current.start_column - 1);
				const int width = MAX(0, current.end_column - current.start_column);
				pointer += String::chr(' ').repeat(offset) + String::chr('^').repeat(width);
			} else {
				const int width = MAX(0, current.end_column - 1);
				pointer += String::chr('^').repeat(width);
			}

			print_line(pointer.as_string());
		}

		token += current.get_name();

		if (current.type == GDScriptTokenizer::Token::ERROR || current.type == GDScriptTokenizer::Token::LITERAL || current.type == GDScriptTokenizer::Token::IDENTIFIER || current.type == GDScriptTokenizer::Token::ANNOTATION) {