Function SetFileFromParts

shock-server/node/fs.go:220–361

this runs asynchronously and uses FileLockMgr

(id string, compress string, numParts int, allowEmpty bool)

// this runs asynchronously and uses FileLockMgr
func SetFileFromParts(id string, compress string, numParts int, allowEmpty bool) {
	// use function closure to get current value of err at return time
	var err error
	defer func() {
		locker.FileLockMgr.Error(id, err)
	}()

	outf := fmt.Sprintf("%s/%s.data", getPath(id), id)

	var outh *os.File
	outh, err = os.Create(outf)
	if err != nil {
		return
	}
	defer outh.Close()

	pReader, pWriter := io.Pipe()
	defer pReader.Close()

	cError := make(chan error)
	cKill := make(chan bool)

	// goroutine to add file readers to pipe while reading pipe
	// allows us to only open one file at a time but stream the data as if it was one reader
	go func() {
		var ferr error
		killed := false
		for i := 1; i <= numParts; i++ {
			select {
			case <-cKill:
				killed = true
				break
			default:
			}
			filename := fmt.Sprintf("%s/parts/%d", getPath(id), i)
			// skip this portion unless either
			// 1. file exists, or
			// 2. file does not exist and allowEmpty == false
			if _, errf := os.Stat(filename); errf == nil || (errf != nil && allowEmpty == false) {
				var part *os.File
				part, ferr = os.Open(filename)
				if ferr != nil {
					break
				}
				_, ferr = io.Copy(pWriter, part)
				part.Close()
				if ferr != nil {
					break
				}
			}
		}
		pWriter.Close()
		select {
		case <-cKill:
			killed = true
		default:
		}
		if killed {