fix(recovery): recover active target after Player transport failure
Co-authored-by: GPT-6 <gpt-6@openai.com>
This commit is contained in:
1 parent
9b99a8b363
commit
02a9b5c93a
2 files changed
+59
-5
No files matched your search
@@ -192,14 +192,15 @@ func recoverManagedTarget(ctx context.Context, d managedRecoveryDeps) (result ma
|
|||||||
healthy := false
|
healthy := false
|
||||||
if result.FrontendID != 0 {
|
if result.FrontendID != 0 {
|
||||||
status, e := d.health(ctx)
|
status, e := d.health(ctx)
|
||||||
var rejection *managedHealthRejection
|
var fatal *managedRecoveryFatal
|
||||||
if e != nil && !errors.As(e, &rejection) {
|
if errors.As(e, &fatal) || errors.Is(e, context.Canceled) {
|
||||||
return result, e
|
return result, e
|
||||||
}
|
}
|
||||||
healthy = status == "OK"
|
if err := ctx.Err(); err != nil {
|
||||||
if e != nil {
|
return result, err
|
||||||
healthy = false
|
|
||||||
}
|
}
|
||||||
|
// A bounded Player request timeout is unhealthy, not job cancellation.
|
||||||
|
healthy = e == nil && status == "OK"
|
||||||
}
|
}
|
||||||
if !healthy {
|
if !healthy {
|
||||||
if e := stop(); e != nil {
|
if e := stop(); e != nil {
|
||||||
|
|||||||
@@ -102,6 +102,59 @@ func TestManagedRecoveryBootstrapReusesSavedFrontend(t *testing.T) {
|
|||||||
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
|
assertActiveCandidates(t, d.transaction, d.db, []int64{result.FrontendID, result.GooglevideoID})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryHealthTransportErrorReusesSavedFrontend(t *testing.T) {
|
||||||
|
for _, healthErr := range []error{errors.New("Player transport failed"), context.DeadlineExceeded} {
|
||||||
|
t.Run(healthErr.Error(), func(t *testing.T) {
|
||||||
|
d, _, events := recoveryFixture(t, true)
|
||||||
|
savedID := managedCandidateFixture(t, d.db, "frontend", "youtube.com", "--lua-desync=saved", true)
|
||||||
|
probes := 0
|
||||||
|
d.health = func(context.Context) (string, error) {
|
||||||
|
probes++
|
||||||
|
if probes == 1 {
|
||||||
|
return "", healthErr
|
||||||
|
}
|
||||||
|
return "OK", nil
|
||||||
|
}
|
||||||
|
d.discover = func(context.Context) (string, error) { return "", errors.New("unresolved") }
|
||||||
|
result, err := recoverManagedTarget(context.Background(), d)
|
||||||
|
if err != nil || result.FrontendID != savedID || probes != 2 {
|
||||||
|
t.Fatalf("result=%+v err=%v probes=%d", result, err, probes)
|
||||||
|
}
|
||||||
|
want := []string{"stop", "check:" + managedFrontendHost + ":--lua-desync=saved", "restart"}
|
||||||
|
if !reflect.DeepEqual(*events, want) {
|
||||||
|
t.Fatalf("saved-first recovery events=%v want=%v", *events, want)
|
||||||
|
}
|
||||||
|
assertActiveCandidates(t, d.transaction, d.db, []int64{savedID})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestManagedRecoveryInitialHealthFailsClosed(t *testing.T) {
|
||||||
|
for _, mode := range []string{"fatal", "canceled", "context-canceled"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
d, front, events := recoveryFixture(t, true)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
var healthErr error = context.Canceled
|
||||||
|
if mode == "fatal" {
|
||||||
|
healthErr = &managedRecoveryFatal{Err: errors.New("unsafe target")}
|
||||||
|
}
|
||||||
|
d.health = func(context.Context) (string, error) {
|
||||||
|
if mode == "context-canceled" {
|
||||||
|
cancel()
|
||||||
|
return "OK", nil
|
||||||
|
}
|
||||||
|
return "", healthErr
|
||||||
|
}
|
||||||
|
_, err := recoverManagedTarget(ctx, d)
|
||||||
|
if !errors.Is(err, healthErr) || len(*events) != 0 {
|
||||||
|
t.Fatalf("err=%v events=%v", err, *events)
|
||||||
|
}
|
||||||
|
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestManagedRecoverySavedFirstAndFailClosed(t *testing.T) {
|
func TestManagedRecoverySavedFirstAndFailClosed(t *testing.T) {
|
||||||
for _, mode := range []string{"pass", "nonpass", "malformed", "runtime", "checker-nonpass", "missing-report", "incomplete", "health-reject", "apply-infrastructure", "cancel", "cleanup-failure"} {
|
for _, mode := range []string{"pass", "nonpass", "malformed", "runtime", "checker-nonpass", "missing-report", "incomplete", "health-reject", "apply-infrastructure", "cancel", "cleanup-failure"} {
|
||||||
t.Run(mode, func(t *testing.T) {
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
|||||||
Reference in new issue
Block a user