Decodes each string into a sequence of codepoints.
(input, input_encoding, errors, replacement_char,
replace_control_characters, with_offsets)
| 397 | |
| 398 | |
| 399 | def _unicode_decode(input, input_encoding, errors, replacement_char, |
| 400 | replace_control_characters, with_offsets): |
| 401 | """Decodes each string into a sequence of codepoints.""" |
| 402 | input = ragged_tensor.convert_to_tensor_or_ragged_tensor(input, name="input") |
| 403 | input_ndims = input.shape.ndims |
| 404 | if input_ndims is None: |
| 405 | raise ValueError("Rank of `input` must be statically known.") |
| 406 | |
| 407 | if input_ndims > 1: |
| 408 | # Convert to a ragged tensor with ragged_rank = input_ndims - 1. |
| 409 | if not ragged_tensor.is_ragged(input): |
| 410 | input = ragged_tensor.RaggedTensor.from_tensor( |
| 411 | input, ragged_rank=input_ndims - 1) |
| 412 | elif input.ragged_rank < input_ndims - 1: |
| 413 | input = input.with_flat_values( |
| 414 | ragged_tensor.RaggedTensor.from_tensor( |
| 415 | input.flat_values, |
| 416 | ragged_rank=input_ndims - input.ragged_rank + 1)) |
| 417 | |
| 418 | # Reshape the input to a flat vector, and apply the gen_string_ops op. |
| 419 | if ragged_tensor.is_ragged(input): |
| 420 | flat_input = array_ops.reshape(input.flat_values, [-1]) |
| 421 | else: |
| 422 | flat_input = array_ops.reshape(input, [-1]) |
| 423 | |
| 424 | if with_offsets: |
| 425 | decode_op = gen_string_ops.unicode_decode_with_offsets |
| 426 | else: |
| 427 | decode_op = gen_string_ops.unicode_decode |
| 428 | flat_result = decode_op( |
| 429 | input=flat_input, |
| 430 | input_encoding=input_encoding, |
| 431 | errors=errors, |
| 432 | replacement_char=replacement_char, |
| 433 | replace_control_characters=replace_control_characters) |
| 434 | |
| 435 | if input_ndims == 0: |
| 436 | codepoints = flat_result.char_values |
| 437 | if with_offsets: |
| 438 | offsets = flat_result.char_to_byte_starts |
| 439 | else: |
| 440 | codepoints = ragged_tensor.RaggedTensor.from_row_splits( |
| 441 | flat_result.char_values, flat_result.row_splits, validate=False) |
| 442 | if input_ndims > 1: |
| 443 | codepoints = input.with_flat_values(codepoints) |
| 444 | if with_offsets: |
| 445 | offsets = ragged_tensor.RaggedTensor.from_row_splits( |
| 446 | flat_result.char_to_byte_starts, flat_result.row_splits, |
| 447 | validate=False) |
| 448 | if input_ndims > 1: |
| 449 | offsets = input.with_flat_values(offsets) |
| 450 | |
| 451 | if with_offsets: |
| 452 | return codepoints, offsets |
| 453 | else: |
| 454 | return codepoints |
| 455 | |
| 456 |
no test coverage detected