Field
| 52 | |
| 53 | |
| 54 | class Field(RawField): |
| 55 | """Field""" |
| 56 | def __init__(self, |
| 57 | name, |
| 58 | pad=None, |
| 59 | unk=None, |
| 60 | bos=None, |
| 61 | eos=None, |
| 62 | lower=False, |
| 63 | use_vocab=True, |
| 64 | tokenize=None, |
| 65 | tokenizer=None, |
| 66 | fn=None): |
| 67 | self.name = name |
| 68 | self.pad = pad |
| 69 | self.unk = unk |
| 70 | self.bos = bos |
| 71 | self.eos = eos |
| 72 | self.lower = lower |
| 73 | self.use_vocab = use_vocab |
| 74 | self.tokenize = tokenize |
| 75 | self.tokenizer = tokenizer |
| 76 | self.fn = fn |
| 77 | |
| 78 | self.specials = [token for token in [pad, unk, bos, eos] if token is not None] |
| 79 | |
| 80 | def __repr__(self): |
| 81 | """repr""" |
| 82 | s, params = "({}): {}(".format(self.name, self.__class__.__name__), [] |
| 83 | if self.pad is not None: |
| 84 | params.append("pad={}".format(self.pad)) |
| 85 | if self.unk is not None: |
| 86 | params.append("unk={}".format(self.unk)) |
| 87 | if self.bos is not None: |
| 88 | params.append("bos={}".format(self.bos)) |
| 89 | if self.eos is not None: |
| 90 | params.append("eos={}".format(self.eos)) |
| 91 | if self.lower: |
| 92 | params.append("lower={}".format(self.lower)) |
| 93 | if not self.use_vocab: |
| 94 | params.append("use_vocab={}".format(self.use_vocab)) |
| 95 | s += ", ".join(params) |
| 96 | s += ")" |
| 97 | |
| 98 | return s |
| 99 | |
| 100 | @property |
| 101 | def pad_index(self): |
| 102 | """pad index""" |
| 103 | if self.pad is None: |
| 104 | return 0 |
| 105 | if hasattr(self, 'vocab'): |
| 106 | return self.vocab[self.pad] |
| 107 | return self.specials.index(self.pad) |
| 108 | |
| 109 | @property |
| 110 | def unk_index(self): |
| 111 | """unk index""" |
no outgoing calls
no test coverage detected