Method Compute

tensorflow/core/kernels/unicode_ops.cc:527–572

* Encodes Unicode codepoints into the desired string representation. * * We lose a dimension while encoding, since a series of integer codepoints is * encoded into a single string. * * This accepts two input tensors: a rank 1 tensor of code point values and * a single rank 1 tensor of splits which determine where each string begins * and ends from the provided code points.

   * and ends from the provided code points.
   */
  void Compute(OpKernelContext* context) override {
    // Get inputs
    const Tensor& input_tensor = context->input(0);
    const auto input_tensor_flat = input_tensor.flat<int32>();
    const Tensor& input_splits = context->input(1);
    const auto input_splits_flat = input_splits.flat<SPLITS_TYPE>();

    OP_REQUIRES(
        context, input_splits.NumElements() > 0,
        errors::InvalidArgument("Input_splits should contain elements, but "
                                "given input_values has 0 elements"));
    // Since we limit to a 2-D input (flat_values of rank 1 and a single splits
    // tensor), our output dimension will be 1 with it's size equal to the
    // number of splits (outer dimension or ragged tensor).
    TensorShape output_shape({input_splits.dim_size(0) - 1});
    Tensor* output_tensor;
    OP_REQUIRES_OK(context, context->allocate_output("output", output_shape,
                                                     &output_tensor));
    auto output_tensor_flat = output_tensor->flat<tstring>();

    // Use a single index over the flattened input values tensor.
    int idx = 0;
    // Loop through our split dimension to create a new string at each split.
    for (int i = 1; i < input_splits_flat.size(); ++i) {
      icu::UnicodeString unicode_string;
      icu::UnicodeStringAppendable appendable_unicode_string(unicode_string);
      for (; idx < input_splits_flat(i); ++idx) {
        int32 code_point = input_tensor_flat(idx);
        // Check for invalid code point
        if (code_point > UCHAR_MAX_VALUE || code_point < UCHAR_MIN_VALUE) {
          if (error_options_.error_on_malformatting) {
            context->CtxFailure(errors::InvalidArgument(
                "Code point value out of valid Unicode range."));
            return;
          } else if (!error_options_.elide_replacement) {
            code_point = error_options_.subst;
          }
        }
        appendable_unicode_string.appendCodePoint(code_point);
      }
      // Encode our string and save in the output.
      string result;
      Encode(encoding_, unicode_string, &result);
      output_tensor_flat(i - 1) = result;
    }
  }

 private:
  UnicodeEncoding encoding_;