Require effective SYS_PTRACE only for native owned workers and distinguish denied FD inspection from foreign queue ownership. Keep inode proof authoritative and constrain deployment to its private PID namespace. Co-authored-by: GPT-6 <gpt-6@openai.com>
430 lines
15 KiB
Go
430 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func assetRuntimeFixture(t *testing.T) (managedAssetRuntime, *[]string, chan error) {
|
|
t.Helper()
|
|
events := []string{}
|
|
done := make(chan error, 1)
|
|
r := managedAssetRuntime{proxy: "socks5://127.0.0.1:1080", guard: func() error { return nil }}
|
|
r.nft = func(_ context.Context, input string) (string, error) {
|
|
if input == "list tables\n" {
|
|
events = append(events, "inventory")
|
|
return "", nil
|
|
}
|
|
if strings.HasPrefix(input, "delete") {
|
|
events = append(events, "delete")
|
|
return "", nil
|
|
}
|
|
if len(events) == 0 || events[len(events)-1] != "ready" {
|
|
t.Fatal("queue activated before READY", events)
|
|
}
|
|
if !strings.Contains(input, "hook input") || !strings.Contains(input, "hook output") || !strings.Contains(input, "meta mark & 0x10000000 == 0") {
|
|
t.Fatal(input)
|
|
}
|
|
events = append(events, "activate")
|
|
return "", nil
|
|
}
|
|
r.queue = func(pid int) (bool, error) {
|
|
if pid == 0 {
|
|
return false, nil
|
|
}
|
|
events = append(events, "ready")
|
|
return true, nil
|
|
}
|
|
r.start = func(args []string) (managedAssetWorker, error) {
|
|
if strings.Contains(strings.Join(args, " "), "--daemon") {
|
|
t.Fatal(args)
|
|
}
|
|
events = append(events, "start")
|
|
return managedAssetWorker{pid: 123, done: done, stop: func() error { events = append(events, "join"); done <- nil; return nil }}, nil
|
|
}
|
|
r.curl = func(_ context.Context, args []string, stdin io.Reader, out, diagnostic io.Writer) error {
|
|
events = append(events, "curl")
|
|
config, err := io.ReadAll(stdin)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
raw, err := strconv.Unquote(strings.TrimSuffix(strings.TrimPrefix(string(config), "url = "), "\n"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var diag bytes.Buffer
|
|
resourceWire(raw, "", out, &diag)
|
|
tls := diag.String()
|
|
if strings.Contains(strings.Join(args, " "), "--tlsv1.3") {
|
|
tls = strings.ReplaceAll(tls, "TLSv1.2", "TLSv1.3")
|
|
}
|
|
_, err = io.WriteString(diagnostic, tls)
|
|
return err
|
|
}
|
|
return r, &events, done
|
|
}
|
|
|
|
func TestManagedAssetTargetQueueOwnershipCapability(t *testing.T) {
|
|
// nfqws drops UID and becomes nondumpable. The queue ownership check must
|
|
// still be able to read its socket symlinks, without sharing host processes.
|
|
raw, err := os.ReadFile("../../deploy/managed-target/compose.yaml")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
manifest := string(raw)
|
|
_, capabilities, ok := strings.Cut(manifest, "cap_add: [")
|
|
if !ok {
|
|
t.Fatal("target capabilities missing")
|
|
}
|
|
capabilities, _, ok = strings.Cut(capabilities, "]")
|
|
if !ok || !strings.Contains(","+strings.ReplaceAll(capabilities, " ", "")+",", ",SYS_PTRACE,") {
|
|
t.Fatal("queue socket ownership requires SYS_PTRACE for dropped-UID workers")
|
|
}
|
|
if !strings.Contains(manifest, "privileged: false") || strings.Contains(manifest, "pid:") {
|
|
t.Fatal("queue inspection must remain confined to an unprivileged private PID namespace")
|
|
}
|
|
}
|
|
|
|
func TestManagedAssetCapabilityGuard(t *testing.T) {
|
|
const required uint64 = 1<<5 | 1<<12 | 1<<19
|
|
for _, bits := range []uint64{required, required | 1<<13, required &^ (1 << 5), required &^ (1 << 12), required &^ (1 << 19), 0} {
|
|
err := managedAssetCapabilityGuard("CapEff:\t" + strconv.FormatUint(bits, 16) + "\n")
|
|
if (err == nil) != (bits&required == required) {
|
|
t.Fatalf("capabilities %x: %v", bits, err)
|
|
}
|
|
}
|
|
for _, status := range []string{"", "CapEff: invalid", "CapPrm: 81020"} {
|
|
if err := managedAssetCapabilityGuard(status); err == nil {
|
|
t.Fatalf("accepted %q", status)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestManagedAssetQueueFDReady(t *testing.T) {
|
|
dir := t.TempDir()
|
|
for _, fd := range []string{"1", "2"} {
|
|
if err := os.WriteFile(filepath.Join(dir, fd), nil, 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
ioErr := errors.New("proc read failure")
|
|
for _, tc := range []struct {
|
|
name string
|
|
firstErr error
|
|
secondLink string
|
|
wantReady bool
|
|
wantErr string
|
|
}{
|
|
{"foreign", nil, "socket:[other]", false, "asset queue belongs to another process"},
|
|
{"denied", os.ErrPermission, "socket:[other]", false, "asset queue ownership inspection denied"},
|
|
{"missing transient", os.ErrNotExist, "socket:[owned]", true, ""},
|
|
{"match despite denial", os.ErrPermission, "socket:[owned]", true, ""},
|
|
{"missing foreign", os.ErrNotExist, "socket:[other]", false, "asset queue belongs to another process"},
|
|
{"read failure", ioErr, "socket:[other]", false, ioErr.Error()},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
ready, err := managedAssetQueueFDReady(dir, map[string]bool{"socket:[owned]": true}, func(path string) (string, error) {
|
|
if filepath.Base(path) == "1" {
|
|
return "socket:[other]", tc.firstErr
|
|
}
|
|
return tc.secondLink, nil
|
|
})
|
|
if ready != tc.wantReady || (err == nil) != (tc.wantErr == "") || err != nil && err.Error() != tc.wantErr {
|
|
t.Fatalf("ready=%v err=%v", ready, err)
|
|
}
|
|
})
|
|
}
|
|
if _, err := managedAssetQueueFDReady(filepath.Join(dir, "absent"), nil, os.Readlink); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("directory error lost: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestManagedAssetRuntimeOwnedCollector(t *testing.T) {
|
|
d, front, _ := recoveryFixture(t, true)
|
|
if err := d.transaction.Apply(d.db, []int64{front}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := map[string][]byte{}
|
|
entries, err := os.ReadDir(d.transaction.Dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, entry := range entries {
|
|
if entry.IsDir() {
|
|
continue
|
|
}
|
|
data, err := os.ReadFile(filepath.Join(d.transaction.Dir, entry.Name()))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before[entry.Name()] = data
|
|
}
|
|
r, events, _ := assetRuntimeFixture(t)
|
|
c := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com", "yt3.ggpht.com"}, Args: managedAssetSeeds[0]}
|
|
m, err := r.verify(context.Background(), c, managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}})
|
|
if err != nil || len(m) != 4 {
|
|
t.Fatal(m, err)
|
|
}
|
|
count := 0
|
|
for _, e := range *events {
|
|
if e == "curl" {
|
|
count++
|
|
}
|
|
}
|
|
if count != 12 || strings.Join((*events)[len(*events)-2:], ",") != "delete,join" {
|
|
t.Fatal(*events)
|
|
}
|
|
for _, measurement := range m {
|
|
if measurement.Args != c.Args {
|
|
t.Fatal("unbound strategy")
|
|
}
|
|
if _, _, err := validateAssetTransfers(measurement); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for name, data := range before {
|
|
after, err := os.ReadFile(filepath.Join(d.transaction.Dir, name))
|
|
if err != nil || !bytes.Equal(data, after) {
|
|
t.Fatal("probe mutated managed artifact", name, err)
|
|
}
|
|
}
|
|
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
|
}
|
|
|
|
func TestManagedAssetRuntimeFailedAtomicNFTInstallNeverDeletesForeignTable(t *testing.T) {
|
|
for _, mode := range []string{"preexisting", "raced-create", "install-failure"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
r, events, _ := assetRuntimeFixture(t)
|
|
r.nft = func(_ context.Context, input string) (string, error) {
|
|
if input == "list tables\n" {
|
|
if mode == "preexisting" {
|
|
return "table inet " + managedAssetTable, nil
|
|
}
|
|
return "", nil
|
|
}
|
|
if strings.HasPrefix(input, "delete") {
|
|
t.Fatal("deleted foreign table after failed atomic install")
|
|
}
|
|
if !strings.HasPrefix(input, "create table inet "+managedAssetTable+"\n") {
|
|
t.Fatal("table creation must reject racing existing table", input)
|
|
}
|
|
return "", errors.New(mode)
|
|
}
|
|
c := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com"}, Args: managedAssetSeeds[0]}
|
|
m, err := r.verify(context.Background(), c, managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}})
|
|
var fatal *managedRecoveryFatal
|
|
if len(m) != 0 || !errors.As(err, &fatal) || errors.Is(err, errCandidateNonpass) {
|
|
t.Fatal(m, err)
|
|
}
|
|
if mode == "preexisting" {
|
|
if len(*events) != 0 {
|
|
t.Fatal("started worker over preexisting table", *events)
|
|
}
|
|
} else if strings.Join(*events, ",") != "start,ready,join" {
|
|
t.Fatal("failed install did not join owned worker", *events)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedAssetRuntimeCancelFailureAndCleanup(t *testing.T) {
|
|
for _, mode := range []string{"cancel-ready", "cancel-curl", "worker", "nonpass", "cleanup"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
r, events, done := assetRuntimeFixture(t)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
if mode == "cancel-ready" {
|
|
r.queue = func(pid int) (bool, error) {
|
|
if pid != 0 {
|
|
cancel()
|
|
}
|
|
return false, nil
|
|
}
|
|
}
|
|
if mode == "cancel-curl" {
|
|
r.curl = func(context.Context, []string, io.Reader, io.Writer, io.Writer) error {
|
|
cancel()
|
|
return context.Canceled
|
|
}
|
|
}
|
|
if mode == "worker" {
|
|
r.curl = func(ctx context.Context, _ []string, _ io.Reader, _, _ io.Writer) error {
|
|
done <- errors.New("worker failure")
|
|
<-ctx.Done()
|
|
return ctx.Err()
|
|
}
|
|
}
|
|
if mode == "nonpass" {
|
|
r.curl = func(_ context.Context, _ []string, _ io.Reader, out, diagnostic io.Writer) error {
|
|
resourceWire("https://i.ytimg.com/image", "cn", out, diagnostic)
|
|
return nil
|
|
}
|
|
}
|
|
if mode == "cleanup" {
|
|
nft := r.nft
|
|
r.nft = func(ctx context.Context, input string) (string, error) {
|
|
out, err := nft(ctx, input)
|
|
if strings.HasPrefix(input, "delete") {
|
|
err = errors.New("cleanup failed")
|
|
}
|
|
return out, err
|
|
}
|
|
}
|
|
c := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com"}, Args: managedAssetSeeds[0]}
|
|
m, err := r.verify(ctx, c, managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}})
|
|
if err == nil || len(m) != 0 {
|
|
t.Fatal(m, err)
|
|
}
|
|
var fatal *managedRecoveryFatal
|
|
if (mode == "worker" || mode == "cleanup") && !errors.As(err, &fatal) {
|
|
t.Fatal("missing fatal", err)
|
|
}
|
|
if mode == "nonpass" && !errors.Is(err, errCandidateNonpass) {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.HasPrefix(mode, "cancel") && !errors.Is(err, context.Canceled) {
|
|
t.Fatal(err)
|
|
}
|
|
if mode == "cancel-ready" && (*events)[len(*events)-1] != "join" {
|
|
t.Fatal(*events)
|
|
}
|
|
if mode != "cancel-ready" && !strings.Contains(strings.Join(*events, ","), "delete") {
|
|
t.Fatal(*events)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedAssetSeedsFallbackAndNoQualification(t *testing.T) {
|
|
for _, pass := range []bool{true, false} {
|
|
t.Run(strconv.FormatBool(pass), func(t *testing.T) {
|
|
d, front, _ := recoveryFixture(t, true)
|
|
avatar := unitAssetCandidate(t, d.db, "yt3.ggpht.com", "--lua-desync=fake", true)
|
|
ggc := managedCandidateFixture(t, d.db, "googlevideo", "fresh.googlevideo.com", "--lua-desync=fake", true)
|
|
if err := d.transaction.Apply(d.db, []int64{front, avatar, ggc}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}}, nil
|
|
}
|
|
calls := 0
|
|
d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) {
|
|
calls++
|
|
if !pass || calls == 1 {
|
|
return nil, errCandidateNonpass
|
|
}
|
|
return []assetMeasurement{unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args)}, nil
|
|
}
|
|
err := repairManagedAssets(context.Background(), d, "i.ytimg.com")
|
|
if !pass {
|
|
if !errors.Is(err, errNoQualifiedStrategy) || calls != len(managedAssetSeeds) {
|
|
t.Fatal(calls, err)
|
|
}
|
|
assertActiveCandidates(t, d.transaction, d.db, []int64{front, avatar, ggc})
|
|
} else {
|
|
if err != nil || calls != 2 {
|
|
t.Fatal(calls, err)
|
|
}
|
|
ids, err := d.transaction.ActiveCandidates(d.db)
|
|
if err != nil || len(ids) != 4 {
|
|
t.Fatal(ids, err)
|
|
}
|
|
for _, required := range []int64{front, avatar, ggc} {
|
|
found := false
|
|
for _, id := range ids {
|
|
found = found || id == required
|
|
}
|
|
if !found {
|
|
t.Fatal(ids)
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestManagedAssetPartialOverlapAndStageHealth(t *testing.T) {
|
|
d, front, _ := recoveryFixture(t, true)
|
|
old := storedCandidate{Group: "assets", Endpoints: []string{"i.ytimg.com", "yt3.ggpht.com"}, Args: "--lua-desync=fake"}
|
|
id, err := saveManagedAssetCandidate(d.db, old, []assetMeasurement{
|
|
unitImageMeasurement(t, "i.ytimg.com", "tls12", old.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", old.Args),
|
|
unitImageMeasurement(t, "yt3.ggpht.com", "tls12", old.Args), unitImageMeasurement(t, "yt3.ggpht.com", "tls13", old.Args),
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
unitAssetCandidate(t, d.db, "i.ytimg.com", "--lua-desync=multisplit", true)
|
|
if err := d.transaction.Apply(d.db, []int64{front, id}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r, _, _ := assetRuntimeFixture(t)
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image", "yt3.ggpht.com": "https://yt3.ggpht.com/image"}, proxy: r.proxy, runner: r.curl, resourceErr: &managedResourceFailure{"media", "unknown", nil}}, nil
|
|
}
|
|
d.frontHealth = func(context.Context) (string, error) { return "OK", nil }
|
|
d.health = func(context.Context) (string, error) {
|
|
t.Fatal("unrelated media gated asset stage")
|
|
return "DEGRADED", nil
|
|
}
|
|
d.verifyAssets = func(_ context.Context, c storedCandidate, _ managedServiceInventory) ([]assetMeasurement, error) {
|
|
if c.ID != 0 {
|
|
return nil, errCandidateNonpass
|
|
}
|
|
if len(c.Endpoints) != 2 {
|
|
t.Fatal("partial overlap lost sibling", c)
|
|
}
|
|
return []assetMeasurement{
|
|
unitImageMeasurement(t, "i.ytimg.com", "tls12", c.Args), unitImageMeasurement(t, "i.ytimg.com", "tls13", c.Args),
|
|
unitImageMeasurement(t, "yt3.ggpht.com", "tls12", c.Args), unitImageMeasurement(t, "yt3.ggpht.com", "tls13", c.Args),
|
|
}, nil
|
|
}
|
|
if err := repairManagedAssets(context.Background(), d, "i.ytimg.com"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ids, err := d.transaction.ActiveCandidates(d.db)
|
|
if err != nil || len(ids) != 2 {
|
|
t.Fatal(ids, err)
|
|
}
|
|
replacement, err := readStrategyCandidate(d.db, ids[1])
|
|
if err != nil || len(replacement.Endpoints) != 2 {
|
|
t.Fatal(replacement, err)
|
|
}
|
|
}
|
|
|
|
func TestManagedAssetMalformedAndUnsafeCleanupNeverRestore(t *testing.T) {
|
|
for _, unsafe := range []bool{false, true} {
|
|
d, front, events := recoveryFixture(t, true)
|
|
if err := d.transaction.Apply(d.db, []int64{front}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.resources = func(context.Context) (managedServiceInventory, error) {
|
|
return managedServiceInventory{images: map[string]string{"i.ytimg.com": "https://i.ytimg.com/image"}}, nil
|
|
}
|
|
calls := 0
|
|
d.verifyAssets = func(context.Context, storedCandidate, managedServiceInventory) ([]assetMeasurement, error) {
|
|
calls++
|
|
err := errors.New("malformed collector metrics")
|
|
if unsafe {
|
|
return nil, &managedRecoveryFatal{Err: &managedAssetCleanupFailure{Err: err}}
|
|
}
|
|
return nil, &managedRecoveryFatal{Err: err}
|
|
}
|
|
err := repairManagedAssets(context.Background(), d, "i.ytimg.com")
|
|
var fatal *managedRecoveryFatal
|
|
if !errors.As(err, &fatal) || errors.Is(err, errNoQualifiedStrategy) || calls != 1 {
|
|
t.Fatal(calls, err)
|
|
}
|
|
if unsafe && (*events)[len(*events)-1] != "stop" {
|
|
t.Fatal("unsafe restoration", *events)
|
|
}
|
|
assertActiveCandidates(t, d.transaction, d.db, []int64{front})
|
|
}
|
|
}
|