| 31 | } |
| 32 | |
| 33 | func CreateColumnIndex(c *column, column int, ofile string) (count int64, format string, err error) { |
| 34 | tmpFilePath := fmt.Sprintf("%s/temp/%d%d.idx", conf.PATH_DATA, rand.Int(), rand.Int()) |
| 35 | |
| 36 | f, err := os.Create(tmpFilePath) |
| 37 | if err != nil { |
| 38 | return |
| 39 | } |
| 40 | defer f.Close() |
| 41 | |
| 42 | format = "array" |
| 43 | eof := false // identifies EOF |
| 44 | curr := int64(0) // stores the offset position of the current index |
| 45 | count = 0 // stores the number of indexed positions and get returned |
| 46 | total_n := 0 // stores the number of bytes read for the current index record |
| 47 | line_count := 0 // stores the number of lines that have been read from the data file |
| 48 | prev_str := "" // keeps track of the string of the specified column of the previous line |
| 49 | buffer_pos := 0 // used to track the location in our byte array |
| 50 | |
| 51 | // Writing index file in 16MB chunks |
| 52 | var b [16777216]byte |
| 53 | for { |
| 54 | buf, er := c.r.ReadLine() |
| 55 | n := len(buf) |
| 56 | if er != nil { |
| 57 | if er != io.EOF { |
| 58 | err = er |
| 59 | return |
| 60 | } |
| 61 | eof = true |
| 62 | } |
| 63 | // skip empty line |
| 64 | if n <= 1 { |
| 65 | total_n += n |
| 66 | line_count += 1 |
| 67 | if eof { |
| 68 | break |
| 69 | } else { |
| 70 | continue |
| 71 | } |
| 72 | } |
| 73 | // split line by columns and test if column value has changed |
| 74 | slices := bytes.Split(buf, []byte("\t")) |
| 75 | if len(slices) < column-1 { |
| 76 | return 0, format, errors.New("Specified column does not exist for all lines in file.") |
| 77 | } |
| 78 | |
| 79 | str := string(slices[column-1]) |
| 80 | if prev_str != str && line_count != 0 { |
| 81 | // Calculating position in byte array |
| 82 | x := (buffer_pos * 16) |
| 83 | // Print byte array if it's full |
| 84 | if x == 16777216 { |
| 85 | f.Write(b[:]) |
| 86 | buffer_pos = 0 |
| 87 | x = 0 |
| 88 | } |
| 89 | // Adding next record to byte array |
| 90 | binary.LittleEndian.PutUint64(b[x:x+8], uint64(curr)) |