We anticipate that the input index format (iformat) will be either "array" or "matrix" and for subset nodes, the output index format should be the same as the input format. Only input index format type "array" is currently supported.
(s *subset, cofile string, ofile string, ifile string, iformat string, ilength int64)
| 131 | // subset nodes, the output index format should be the same as the input format. Only input index |
| 132 | // format type "array" is currently supported. |
| 133 | func CreateSubsetNodeIndexes(s *subset, cofile string, ofile string, ifile string, iformat string, ilength int64) (coCount int64, oCount int64, oSize int64, err error) { |
| 134 | if iformat == "array" { |
| 135 | // create temporary output file (oTmpFilePath) for subset index and temporary output file (coTmpFilePath) for compressed subset index |
| 136 | oTmpFilePath := fmt.Sprintf("%s/temp/%d%d.idx", conf.PATH_DATA, rand.Int(), rand.Int()) |
| 137 | coTmpFilePath := fmt.Sprintf("%s/temp/%d%d.idx", conf.PATH_DATA, rand.Int(), rand.Int()) |
| 138 | |
| 139 | var ifh *os.File |
| 140 | ifh, err = os.Open(ifile) |
| 141 | if err != nil { |
| 142 | return |
| 143 | } |
| 144 | defer ifh.Close() |
| 145 | |
| 146 | var ofh *os.File |
| 147 | ofh, err = os.Create(oTmpFilePath) |
| 148 | if err != nil { |
| 149 | return |
| 150 | } |
| 151 | defer ofh.Close() |
| 152 | |
| 153 | var cofh *os.File |
| 154 | cofh, err = os.Create(coTmpFilePath) |
| 155 | if err != nil { |
| 156 | return |
| 157 | } |
| 158 | defer cofh.Close() |
| 159 | |
| 160 | // these are attributes of the subset index output, "matrix" format will be supported in the future |
| 161 | oCount = 0 |
| 162 | oSize = 0 |
| 163 | |
| 164 | // coCount is the number of indices created for the compressed output index |
| 165 | coCount = 0 |
| 166 | |
| 167 | //coOffset and coLength store the offset and length for a given compressed output index entry |
| 168 | coOffset := int64(0) |
| 169 | coLength := int64(0) |
| 170 | |
| 171 | // these store the previous offset and length for concatenating contiguous reads into one entry for compressed index |
| 172 | prevOffset := int64(0) |
| 173 | prevLength := int64(0) |
| 174 | |
| 175 | // stores previous integer from subset file line |
| 176 | prev_int := int(0) |
| 177 | |
| 178 | // used to track the location in our output byte array |
| 179 | buffer_pos1 := 0 |
| 180 | buffer_pos2 := 0 |
| 181 | |
| 182 | // Writing index files in 16MB chunks |
| 183 | var b1 [16777216]byte |
| 184 | var b2 [16777216]byte |
| 185 | |
| 186 | for { |
| 187 | buf, er := s.r.ReadLine() |
| 188 | n := len(buf) |
| 189 | if er != nil { |
| 190 | if er != io.EOF { |