| 154 | ) |
| 155 | |
| 156 | func (w *Writer) AddFile(fn, name string) error { |
| 157 | w.set.Reset() |
| 158 | docid := uint32(len(w.docs)) |
| 159 | w.docs = append(w.docs, name) |
| 160 | f, err := os.Open(fn) |
| 161 | if err != nil { |
| 162 | return err |
| 163 | } |
| 164 | defer f.Close() |
| 165 | |
| 166 | st, err := f.Stat() |
| 167 | if err != nil { |
| 168 | return err |
| 169 | } |
| 170 | if st.Size() > maxFileLen { |
| 171 | return errors.New("too long, ignoring") |
| 172 | } |
| 173 | if st.Size() < 3 { |
| 174 | return errors.New("too short, ignoring") |
| 175 | } |
| 176 | |
| 177 | var ( |
| 178 | c byte |
| 179 | tv uint32 |
| 180 | i = 0 |
| 181 | n = 0 |
| 182 | linelen = 0 |
| 183 | buf = w.inbuf[:0] |
| 184 | entries = make([]uint64, st.Size()-2) |
| 185 | ) |
| 186 | for { |
| 187 | tv = (tv << 8) & (1<<24 - 1) |
| 188 | if i >= len(buf) { |
| 189 | n, err := f.Read(buf[:cap(buf)]) |
| 190 | if n == 0 { |
| 191 | if err != nil { |
| 192 | if err == io.EOF { |
| 193 | break |
| 194 | } |
| 195 | return err |
| 196 | } |
| 197 | return errors.New("0-length read") |
| 198 | } |
| 199 | buf = buf[:n] |
| 200 | i = 0 |
| 201 | } |
| 202 | c = buf[i] |
| 203 | i++ |
| 204 | tv |= uint32(c) |
| 205 | if linelen++; linelen > maxLineLen { |
| 206 | return errors.New("very long lines, ignoring") |
| 207 | } |
| 208 | if c == '\n' { |
| 209 | linelen = 0 |
| 210 | } |
| 211 | if !validUTF8((tv>>8)&0xFF, tv&0xFF) { |
| 212 | return errors.New("invalid UTF-8, ignoring") |
| 213 | } |