MCPcopy Create free account
hub / github.com/google-research/language / build_text_inputs

Function build_text_inputs

language/capwap/utils/text_utils.py:249–294  ·  view source on GitHub ↗

Convert text to TextInputs. Args: text: , space-separated token string. length: Length to pad or truncate to. lookup_table: Instance of contrib.lookup.index_table_from_tensor. segment_id: Integer denoting segment type. start_token: Optional start token. end_token:

(
    text,
    length,
    lookup_table,
    segment_id=0,
    start_token=None,
    end_token=None,
)

Source from the content-addressed store, hash-verified

247
248
249def build_text_inputs(
250 text,
251 length,
252 lookup_table,
253 segment_id=0,
254 start_token=None,
255 end_token=None,
256):
257 """Convert text to TextInputs.
258
259 Args:
260 text: <string>, space-separated token string.
261 length: Length to pad or truncate to.
262 lookup_table: Instance of contrib.lookup.index_table_from_tensor.
263 segment_id: Integer denoting segment type.
264 start_token: Optional start token.
265 end_token: Optional end token.
266
267 Returns:
268 Instance of TextInputs.
269 """
270 # Tokenize and truncate.
271 tokens = tf.string_split([text]).values
272 length_offset = sum([0 if i is None else 1 for i in [start_token, end_token]])
273 tokens = tokens[:length - length_offset]
274 if start_token is not None:
275 tokens = tf.concat([[start_token], tokens], axis=0)
276 if end_token is not None:
277 tokens = tf.concat([tokens, [end_token]], axis=0)
278
279 token_ids = tf.cast(lookup_table.lookup(tokens), tf.int32)
280 mask = tf.ones_like(token_ids)
281 segment_ids = tf.fill(tf.shape(token_ids), segment_id)
282
283 pad = [[0, length - tf.size(token_ids)]]
284 token_ids = tf.pad(token_ids, pad)
285 mask = tf.pad(mask, pad)
286 segment_ids = tf.pad(segment_ids, pad)
287 positions = tf.range(length)
288 text_input = TextInputs(
289 token_ids=tf.ensure_shape(token_ids, [length]),
290 mask=tf.ensure_shape(mask, [length]),
291 segment_ids=tf.ensure_shape(segment_ids, [length]),
292 positions=tf.ensure_shape(positions, [length]))
293
294 return text_input
295
296
297def build_planner_inputs(question, answer, length, lookup_table):

Callers

nothing calls this directly

Calls 2

padMethod · 0.80
sizeMethod · 0.45

Tested by

no test coverage detected

Used in the wild real call sites across dependent graphs

searching dependent graphs…