Class EncodedStringCoder

xarray/coding/strings.py:50–98

Transforms between unicode strings and fixed-width UTF-8 bytes.

class EncodedStringCoder(VariableCoder):
    """Transforms between unicode strings and fixed-width UTF-8 bytes."""

    def __init__(self, allows_unicode=True):
        self.allows_unicode = allows_unicode

    def encode(self, variable: Variable, name=None) -> Variable:
        dims, data, attrs, encoding = unpack_for_encoding(variable)

        # StringDType: replace nulls and convert to fixed-width unicode (U),
        # which all backends support natively (GH11199)
        if data.dtype.kind == "T":
            data = np.asarray(data, dtype=object)
            data[data == None] = ""  # noqa: E711
            data = np.asarray(data, dtype="U")
            variable = Variable(dims, data, attrs, encoding)

        contains_unicode = is_unicode_dtype(data.dtype)
        encode_as_char = encoding.get("dtype") == "S1"
        if encode_as_char:
            del encoding["dtype"]  # no longer relevant

        if contains_unicode and (encode_as_char or not self.allows_unicode):
            if "_FillValue" in attrs:
                raise NotImplementedError(
                    f"variable {name!r} has a _FillValue specified, but "
                    "_FillValue is not yet supported on unicode strings: "
                    "https://github.com/pydata/xarray/issues/1647"
                )

            string_encoding = encoding.pop("_Encoding", "utf-8")
            safe_setitem(attrs, "_Encoding", string_encoding, name=name)
            # TODO: figure out how to handle this in a lazy way with dask
            data = encode_string_array(data, string_encoding)

            return Variable(dims, data, attrs, encoding)
        else:
            variable.encoding = encoding
            return variable

    def decode(self, variable: Variable, name=None) -> Variable:
        dims, data, attrs, encoding = unpack_for_decoding(variable)

        if "_Encoding" in attrs:
            string_encoding = pop_to(attrs, encoding, "_Encoding")
            func = partial(decode_bytes_array, encoding=string_encoding)
            data = lazy_elemwise_func(data, func, np.dtype(object))

        return Variable(dims, data, attrs, encoding)


def decode_bytes_array(bytes_array, encoding="utf-8"):