Function test_jsonlines_reading

python/pathway/tests/test_errors.py:1362–1400

(tmp_path)

def test_jsonlines_reading(tmp_path):
    class InputSchema(pw.Schema):
        a: int
        b: int
        c: int | None

    path = tmp_path / "input.jsonlines"
    generate_jsonlines(path)
    input = pw.io.jsonlines.read(path, schema=InputSchema, mode="static")
    result = input.with_columns(
        b=pw.fill_error(pw.this.b, 0), c=pw.fill_error(pw.this.c, 0)
    )
    expected = T(
        """
        a | b | c
        1 | 2 | 3
        2 | 0 | 3
        1 | 3 | 0
        6 | 0 | 0
        7 | 1 |
    """
    )
    expected_errors = T(
        """
        message
        failed to create a field "b" with type int from json payload: "x"
        failed to create a field "b" with type int from json payload: "1"
        failed to create a field "c" with type int / None from json payload: "t"
        failed to create a field "c" with type int / None from json payload: "y"
    """,
        split_on_whitespace=False,
    ).select(message=pw.this.message.str.replace("/", "|"))
    # can't use | because it's a column sep

    assert_table_equality_wo_index(
        (result, pw.global_error_log().select(pw.this.message)),
        (expected, expected_errors),
        terminate_on_error=False,
    )


def test_jsonlines_reading_pk(tmp_path):