Function do_ok_test

core/src/lexer.rs:807–820

Runs the lexer on the given `input` and expects the returned tokens to match `exp_token_spans`.

(input: &str, exp_token_spans: &[TokenSpan])

    /// Runs the lexer on the given `input` and expects the returned tokens to match
    /// `exp_token_spans`.
    fn do_ok_test(input: &str, exp_token_spans: &[TokenSpan]) {
        let mut input = input.as_bytes();
        let mut lexer = Lexer::from(&mut input);

        let mut token_spans: Vec<TokenSpan> = vec![];
        let mut eof = false;
        while !eof {
            let token_span = lexer.read().expect("Lexing failed");
            eof = token_span.token == Token::Eof;
            token_spans.push(token_span);
        }

        assert_eq!(exp_token_spans, token_spans.as_slice());
    }

    #[test]
    fn test_empty() {