fix(recovery): recover active target after Player transport failure
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
9b99a8b363
commit
02a9b5c93a
2 files changed
+59
-5
No files matched your search
@@ -192,14 +192,15 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
||||
healthy := false
|
||||
if result.FrontendID != 0 {
|
||||
status, e := d.health(ctx)
|
||||
var rejection *managedHealthRejection
|
||||
if e != nil && !errors.As(e, &rejection) {
|
||||
var fatal *managedRecoveryFatal
|
||||
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
|
||||
return result, e
|
||||
}
|
||||
healthy = status == "OK"
|
||||
if e != nil {
|
||||
healthy = false
|
||||
if err := ctx.Err(); err != nil {
|
||||
return result, err
|
||||
}
|
||||
// A bounded Player request timeout is unhealthy, not job cancellation.
|
||||
healthy = e == nil && status == "OK"
|
||||
}
|
||||
if !healthy {
|
||||
if e := stop(); e != nil {
|
||||
|
||||
@@ -102,6 +102,59 @@ func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
|
||||
}
|
||||
|
||||
func TestManagedRecoveryHealthTransportErrorReusesSavedFrontend(t *testing.T) {
|
||||
for _, healthErr := range []error{errors.New("Player transport failed"), context.DeadlineExceeded} {
|
||||
t.Run(healthErr.Error(), func(t *testing.T) {
|
||||
d, _, events := recoveryFixture(t, true)
|
||||
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
|
||||
probes := 0
|
||||
d.health = func(context.Context) (string, error) {
|
||||
probes++
|
||||
if probes == 1 {
|
||||
return "", healthErr
|
||||
}
|
||||
return "OK", nil
|
||||
}
|
||||
d.discover = func(context.Context) (string, error) { return "", errors.New("unresolved") }
|
||||
result, err := recoverManagedTarget(context.Background(), d)
|
||||
if err != nil || result.FrontendID != savedID || probes != 2 {
|
||||
t.Fatalf("result=%+v err=%v probes=%d", result, err, probes)
|
||||
}
|
||||
want := []string{"stop", "check:" + managedFrontendHost + ":--lua-desync=saved", "restart"}
|
||||
if !reflect.DeepEqual(*events, want) {
|
||||
t.Fatalf("saved-first recovery events=%v want=%v", *events, want)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{savedID})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoveryInitialHealthFailsClosed(t *testing.T) {
|
||||
for _, mode := range []string{"fatal", "canceled", "context-canceled"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
d, front, events := recoveryFixture(t, true)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
var healthErr error = context.Canceled
|
||||
if mode == "fatal" {
|
||||
healthErr = &managedRecoveryFatal{Err: errors.New("unsafe target")}
|
||||
}
|
||||
d.health = func(context.Context) (string, error) {
|
||||
if mode == "context-canceled" {
|
||||
cancel()
|
||||
return "OK", nil
|
||||
}
|
||||
return "", healthErr
|
||||
}
|
||||
_, err := recoverManagedTarget(ctx, d)
|
||||
if !errors.Is(err, healthErr) || len(*events) != 0 {
|
||||
t.Fatalf("err=%v events=%v", err, *events)
|
||||
}
|
||||
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestManagedRecoverySavedFirstAndFailClosed(t *testing.T) {
|
||||
for _, mode := range []string{"pass", "nonpass", "malformed", "runtime", "checker-nonpass", "missing-report", "incomplete", "health-reject", "apply-infrastructure", "cancel", "cleanup-failure"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
|
||||
Reference in new issue
Block a user