Function TestAutoRetry

cmd/testwrapper/testwrapper_test.go:116–166

TestAutoRetry covers a non-Mark()'d test that fails on the first attempt and passes on retry: the wrapper must exit 0, emit a flakytest failures JSON line with a fake /issues/UNKNOWN URL, and have run the test twice.

(t *testing.T)

// passes on retry: the wrapper must exit 0, emit a flakytest failures JSON
// line with a fake /issues/UNKNOWN URL, and have run the test twice.
func TestAutoRetry(t *testing.T) {
	t.Parallel()

	testfile := filepath.Join(t.TempDir(), "autoretry_test.go")
	// Note: no flakytest.Mark call. The wrapper should retry anyway.
	code := []byte(`package autoretry_test

import (
	"os"
	"testing"
)

func TestAutoFlake(t *testing.T) {
	e := os.Getenv("TS_TESTWRAPPER_ATTEMPT")
	if e == "" {
		t.Skip("not running in testwrapper")
	}
	if e == "1" {
		t.Fatal("First run in testwrapper, failing so that test is retried. This is expected.")
	}
}
`)
	if err := os.WriteFile(testfile, code, 0o644); err != nil {
		t.Fatalf("writing package: %s", err)
	}

	out, err := cmdTestwrapper(t, "-v", testfile).CombinedOutput()
	if err != nil {
		t.Fatalf("testwrapper %s: %s with output:\n%s", testfile, err, out)
	}

	if !bytes.Contains(out, []byte("flakytest failures JSON:")) {
		t.Errorf("missing flakytest failures JSON line in output:\n%s", out)
	}
	if !bytes.Contains(out, []byte("/issues/UNKNOWN")) {
		t.Errorf("missing fake /issues/UNKNOWN URL in output:\n%s", out)
	}
	if !bytes.Contains(out, []byte("https://github.com/tailscale/tailscale/issues/UNKNOWN")) {
		t.Errorf("missing fake URL with detected repo in output:\n%s", out)
	}
	if bytes.Contains(out, []byte("permanent test failures JSON:")) {
		t.Errorf("unexpected permanent failures line in output:\n%s", out)
	}
	if runs := bytes.Count(out, []byte("=== RUN   TestAutoFlake")); runs != 2 {
		t.Errorf("expected TestAutoFlake to run twice but ran %d times in output:\n%s", runs, out)
	}

	if testing.Verbose() {
		t.Logf("success - output:\n%s", out)
	}
}

// TestPermanentFailure covers a test that always fails: the wrapper must exit
// non-zero, emit a permanent test failures JSON line, and NOT emit a flakytest