Method getNextObjs

downloader/job.go:388–426

Reads the content of a remote bucket page by page until any objects to download found or the bucket list is over.

()

// Reads the content of a remote bucket page by page until any objects to
// download found or the bucket list is over.
func (j *backendDlJob) getNextObjs() error {
	var (
		sid     = j.t.SID()
		smap    = j.t.Sowner().Get()
		backend = j.t.Backend(j.bck)
	)
	j.objs = j.objs[:0]
	for len(j.objs) < downloadBatchSize {
		msg := &apc.ListObjsMsg{
			Prefix:            j.prefix,
			ContinuationToken: j.continuationToken,
			PageSize:          backend.MaxPageSize(),
		}
		bckList, _, err := backend.ListObjects(j.bck, msg)
		if err != nil {
			return err
		}
		j.continuationToken = bckList.ContinuationToken

		for _, entry := range bckList.Entries {
			if !j.checkObj(entry.Name) {
				continue
			}
			obj, err := makeDlObj(smap, sid, j.bck, entry.Name, "")
			if err != nil {
				if err == errInvalidTarget {
					continue
				}
				return err
			}
			j.objs = append(j.objs, obj)
		}
		if j.continuationToken == "" {
			j.done = true
			break
		}
	}
	return nil
}

/////////////////////
// downloadJobInfo //