Function Test_SingleNodeSnapshot_CheckpointFailures

store/store_snapshot_test.go:718–842

Test_SingleNodeSnapshot_CheckpointFailures tests the Store responds correctly when checkpoint of the underlying database returns an error. This can happen depending on active readers of the database. The Store should handle active readers fine, aborting the snapshot, and handling it next time. This

(t *testing.T)

// readers fine, aborting the snapshot, and handling it next time. This tests
// critical code paths which are very rare, but are perfectly possible in production.
func Test_SingleNodeSnapshot_CheckpointFailures(t *testing.T) {
	s, ln := mustNewStore(t)
	defer ln.Close()

	if err := s.Open(); err != nil {
		t.Fatalf("failed to open single-node store: %s", err.Error())
	}
	defer s.Close(true)
	s.NoSnapshotOnClose = true
	if err := s.Bootstrap(NewServer(s.ID(), s.Addr(), true)); err != nil {
		t.Fatalf("failed to bootstrap single-node store: %s", err.Error())
	}
	if _, err := s.WaitForLeader(10 * time.Second); err != nil {
		t.Fatalf("Error waiting for leader: %s", err)
	}

	queries := []string{
		`CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)`,
		`INSERT INTO foo(name) VALUES("fiona")`,
	}
	mustExecute(t, s, queries)

	////////////////////////////////////////////////////////////////////////////
	// Start testing with stalled queries.

	startStalledQuery := func(srcDB *db.DB) context.CancelFunc {
		ctx, cancelFunc := context.WithCancel(context.Background())
		go func() {
			srcDB.QueryWithContext(ctx, mustCreateRequest(`SELECT * FROM foo`), false)
		}()
		time.Sleep(time.Second)
		return cancelFunc
	}

	srcDB, err := db.Open(s.dbPath, false, true)
	if err != nil {
		t.Fatalf("failed to open database: %s", err)
	}
	defer srcDB.Close()

	cancelFunc := startStalledQuery(srcDB)

	// First snapshot, which will be full, will fail due to the reader.
	if err := s.Snapshot(0); err == nil {
		t.Fatalf("expected error due to blocking reader when attempting first snapshot")
	}

	// Release the reader, full snapshot should work.
	cancelFunc()
	if err := s.Snapshot(0); err != nil {
		t.Fatalf("failed to snapshot after canceling reader: %s", err)
	}

	//////////////////////////////////////////////////////////////////////////////////
	// Insert a bunch of records, which will be added to the WAL. Then start
	// a reader which should be reading from the last page in the WAL. This will
	// mean snapshot will be OK.
	clear(queries)