Add adds the file f to the index under the given name. It logs errors using package log.
(name string, f io.Reader)
| 116 | // Add adds the file f to the index under the given name. |
| 117 | // It logs errors using package log. |
| 118 | func (ix *IndexWriter) Add(name string, f io.Reader) { |
| 119 | ix.trigram.Reset() |
| 120 | var ( |
| 121 | c = byte(0) |
| 122 | i = 0 |
| 123 | buf = ix.inbuf[:0] |
| 124 | tv = uint32(0) |
| 125 | n = int64(0) |
| 126 | linelen = 0 |
| 127 | ) |
| 128 | for { |
| 129 | tv = (tv << 8) & (1<<24 - 1) |
| 130 | if i >= len(buf) { |
| 131 | n, err := f.Read(buf[:cap(buf)]) |
| 132 | if n == 0 { |
| 133 | if err != nil { |
| 134 | if err == io.EOF { |
| 135 | break |
| 136 | } |
| 137 | log.Printf("%s: %v\n", name, err) |
| 138 | return |
| 139 | } |
| 140 | log.Printf("%s: 0-length read\n", name) |
| 141 | return |
| 142 | } |
| 143 | buf = buf[:n] |
| 144 | i = 0 |
| 145 | } |
| 146 | c = buf[i] |
| 147 | i++ |
| 148 | tv |= uint32(c) |
| 149 | if n++; n >= 3 { |
| 150 | ix.trigram.Add(tv) |
| 151 | } |
| 152 | if !validUTF8((tv>>8)&0xFF, tv&0xFF) { |
| 153 | if ix.LogSkip { |
| 154 | log.Printf("%s: invalid UTF-8, ignoring\n", name) |
| 155 | } |
| 156 | return |
| 157 | } |
| 158 | if n > maxFileLen { |
| 159 | if ix.LogSkip { |
| 160 | log.Printf("%s: too long, ignoring\n", name) |
| 161 | } |
| 162 | return |
| 163 | } |
| 164 | if linelen++; linelen > maxLineLen { |
| 165 | if ix.LogSkip { |
| 166 | log.Printf("%s: very long lines, ignoring\n", name) |
| 167 | } |
| 168 | return |
| 169 | } |
| 170 | if c == '\n' { |
| 171 | linelen = 0 |
| 172 | } |
| 173 | } |
| 174 | if ix.trigram.Len() > maxTextTrigrams { |
| 175 | if ix.LogSkip { |