Convert text to TextInputs. Args: text: , space-separated token string. length: Length to pad or truncate to. lookup_table: Instance of contrib.lookup.index_table_from_tensor. segment_id: Integer denoting segment type. start_token: Optional start token. end_token:
(
text,
length,
lookup_table,
segment_id=0,
start_token=None,
end_token=None,
)
| 247 | |
| 248 | |
| 249 | def build_text_inputs( |
| 250 | text, |
| 251 | length, |
| 252 | lookup_table, |
| 253 | segment_id=0, |
| 254 | start_token=None, |
| 255 | end_token=None, |
| 256 | ): |
| 257 | """Convert text to TextInputs. |
| 258 | |
| 259 | Args: |
| 260 | text: <string>, space-separated token string. |
| 261 | length: Length to pad or truncate to. |
| 262 | lookup_table: Instance of contrib.lookup.index_table_from_tensor. |
| 263 | segment_id: Integer denoting segment type. |
| 264 | start_token: Optional start token. |
| 265 | end_token: Optional end token. |
| 266 | |
| 267 | Returns: |
| 268 | Instance of TextInputs. |
| 269 | """ |
| 270 | # Tokenize and truncate. |
| 271 | tokens = tf.string_split([text]).values |
| 272 | length_offset = sum([0 if i is None else 1 for i in [start_token, end_token]]) |
| 273 | tokens = tokens[:length - length_offset] |
| 274 | if start_token is not None: |
| 275 | tokens = tf.concat([[start_token], tokens], axis=0) |
| 276 | if end_token is not None: |
| 277 | tokens = tf.concat([tokens, [end_token]], axis=0) |
| 278 | |
| 279 | token_ids = tf.cast(lookup_table.lookup(tokens), tf.int32) |
| 280 | mask = tf.ones_like(token_ids) |
| 281 | segment_ids = tf.fill(tf.shape(token_ids), segment_id) |
| 282 | |
| 283 | pad = [[0, length - tf.size(token_ids)]] |
| 284 | token_ids = tf.pad(token_ids, pad) |
| 285 | mask = tf.pad(mask, pad) |
| 286 | segment_ids = tf.pad(segment_ids, pad) |
| 287 | positions = tf.range(length) |
| 288 | text_input = TextInputs( |
| 289 | token_ids=tf.ensure_shape(token_ids, [length]), |
| 290 | mask=tf.ensure_shape(mask, [length]), |
| 291 | segment_ids=tf.ensure_shape(segment_ids, [length]), |
| 292 | positions=tf.ensure_shape(positions, [length])) |
| 293 | |
| 294 | return text_input |
| 295 | |
| 296 | |
| 297 | def build_planner_inputs(question, answer, length, lookup_table): |