Function test_basic_tokenize

regexer/src/lexer/ast.rs:244–280

()

    #[test]
    fn test_basic_tokenize() {
        let input = String::from(
            "letter(upcase=True) | letter(upcase=False) | glob(rest=True) | glob(rest=False) | whitespace | number",
        );

        assert_eq!(
            parse(tokens::tokenize(input)),
            vec![
                Function::Letter {
                    casing: Some(Casing::Upcase),
                    select: None,
                },
                Function::Letter {
                    casing: Some(Casing::Downcase),
                    select: None
                },
                Function::Glob { rest: true },
                Function::Glob { rest: false },
                Function::Whitespace,
                Function::Number { select: None },
            ]
        );

        let input = String::from("letters(upcase=True) | glob(rest=True) | whitespace | numbers");

        assert_eq!(
            parse(tokens::tokenize(input)),
            vec![
                Function::Letters {
                    casing: Some(Casing::Upcase),
                },
                Function::Glob { rest: true },
                Function::Whitespace,
                Function::Numbers,
            ]
        );
    }

    #[test]
    fn test_select_parameters() {