We anticipate that the input index format (iformat) will be either "array" or "matrix" and for subset indexes, the output format should be the same as the input format. Only input index format type "array" is currently supported.
(s *subset, oifile string, ifile string, iformat string, ilength int64)
| 34 | // subset indexes, the output format should be the same as the input format. Only input index format |
| 35 | // type "array" is currently supported. |
| 36 | func CreateSubsetIndex(s *subset, oifile string, ifile string, iformat string, ilength int64) (count int64, size int64, err error) { |
| 37 | if iformat == "array" { |
| 38 | tmpFilePath := fmt.Sprintf("%s/temp/%d%d.idx", conf.PATH_DATA, rand.Int(), rand.Int()) |
| 39 | |
| 40 | ifh, err := os.Open(ifile) |
| 41 | if err != nil { |
| 42 | return -1, -1, err |
| 43 | } |
| 44 | defer ifh.Close() |
| 45 | |
| 46 | ofh, err := os.Create(tmpFilePath) |
| 47 | if err != nil { |
| 48 | return -1, -1, err |
| 49 | } |
| 50 | defer ofh.Close() |
| 51 | |
| 52 | count = 0 |
| 53 | size = 0 |
| 54 | prev_int := int(0) |
| 55 | buffer_pos := 0 // used to track the location in our output byte array |
| 56 | |
| 57 | // Writing index file in 16MB chunks |
| 58 | var b [16777216]byte |
| 59 | for { |
| 60 | buf, er := s.r.ReadLine() |
| 61 | n := len(buf) |
| 62 | if er != nil { |
| 63 | if er != io.EOF { |
| 64 | err = er |
| 65 | return -1, -1, err |
| 66 | } |
| 67 | break |
| 68 | } |
| 69 | // skip empty line |
| 70 | if n <= 1 { |
| 71 | continue |
| 72 | } |
| 73 | // int from line |
| 74 | str := string(buf[:n-1]) |
| 75 | curr_int, er := strconv.Atoi(str) |
| 76 | if er != nil { |
| 77 | err = er |
| 78 | return -1, -1, err |
| 79 | } |
| 80 | |
| 81 | if curr_int <= prev_int { |
| 82 | err = errors.New(fmt.Sprintf("Subset indices must be numerically sorted and non-redundant, found value %d after value %d", curr_int, prev_int)) |
| 83 | return -1, -1, err |
| 84 | } |
| 85 | |
| 86 | if int64(curr_int) > ilength { |
| 87 | err = errors.New(fmt.Sprintf("Subset index: %d does not exist in parent index file.", curr_int)) |
| 88 | return -1, -1, err |
| 89 | } |
| 90 | |
| 91 | var ibuf [16]byte |
| 92 | _, er = ifh.ReadAt(ibuf[0:16], int64((curr_int-1)*16)) |
| 93 | if er != nil { |