MCPcopy Create free account
hub / github.com/DeepRec-AI/DeepRec / Encode

Function Encode

tensorflow/core/kernels/unicode_ops.cc:54–84  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

52namespace {
53
54void Encode(const UnicodeEncoding encoding, const icu::UnicodeString& in,
55 string* out) {
56 if (encoding == UnicodeEncoding::UTF8) {
57 out->clear();
58 in.toUTF8String(*out);
59 } else if (encoding == UnicodeEncoding::UTF16BE) {
60 // TODO(gbillock): consider using the
61 // extract(char *dest, int32_t destCapacity, UConverter *cnv)
62 // for UTF16/32
63 out->clear(); // subtle: must come before reserve()
64 out->reserve(2 * in.length() + 1);
65 const char16_t* buf = in.getBuffer();
66 for (int i = 0; i < in.length(); ++i) {
67 // Emit big-endian encoding for UTF-16 always.
68 out->push_back((buf[i] & 0xFF00) >> 8);
69 out->push_back(buf[i] & 0x00FF);
70 }
71 } else if (encoding == UnicodeEncoding::UTF32BE) {
72 out->clear(); // subtle: must come before reserve()
73 out->reserve(4 * in.countChar32() + 1);
74 icu::StringCharacterIterator it(in);
75 UChar32 ch;
76 while (it.hasNext()) {
77 ch = it.next32PostInc();
78 out->push_back((ch & 0xFF000000) >> 24);
79 out->push_back((ch & 0x00FF0000) >> 16);
80 out->push_back((ch & 0x0000FF00) >> 8);
81 out->push_back((ch & 0x000000FF));
82 }
83 }
84}
85
86// This error callback is only useful for finding illegal encoding errors when
87// we want to be strict -- otherwise illegal encodings are replaced on read

Callers 2

TranscodeMethod · 0.70
ComputeMethod · 0.70

Calls 5

hasNextMethod · 0.80
clearMethod · 0.45
reserveMethod · 0.45
lengthMethod · 0.45
push_backMethod · 0.45

Tested by

no test coverage detected