231 lines
7.6 KiB
Go
231 lines
7.6 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/signal"
|
|
"strings"
|
|
"syscall"
|
|
"time"
|
|
)
|
|
|
|
func tuneYoutube(stdout, stderr io.Writer, scan func(context.Context, string, string, io.Writer, io.Writer) (string, error)) int {
|
|
return tuneYoutubeWithCheck(stdout, stderr, scan, runCombinedBlockcheck)
|
|
}
|
|
|
|
func tuneYoutubeWithCheck(stdout, stderr io.Writer, scan func(context.Context, string, string, io.Writer, io.Writer) (string, error), check func(context.Context, string, string, string, io.Writer, io.Writer) error) int {
|
|
started := time.Now()
|
|
db, err := openStore(databasePath())
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "error: tune youtube database:", err)
|
|
return 1
|
|
}
|
|
defer db.Close()
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
|
defer stop()
|
|
fmt.Fprintln(stderr, "Testing frontend youtube.com only; GGC and other frontend domains have not been tested. No options will be applied.")
|
|
fmt.Fprintln(stderr, "Full TLS1.2 and TLS1.3 scan with 8 workers, then real-data check with 32 workers, --take 10, 3 passes; no overall time limit; selection requires fully passing results for both tls12 and tls13.")
|
|
fmt.Fprintln(stderr, "The best individually verified TLS strategies are combined into one profile, then checked together for 3/3 passes on both TLS versions before output. No options will be applied.")
|
|
var raw bytes.Buffer
|
|
var report []byte
|
|
var args, output string
|
|
path, err := scan(ctx, "blockcheckw", "youtube.com", io.MultiWriter(&raw, stderr), stderr)
|
|
if path != "" {
|
|
fmt.Fprintln(stderr, "Raw blockcheck report retained at:", path)
|
|
fmt.Fprintln(stderr, "Blockcheck check JSON path:", path+".check.json")
|
|
if saveErr := os.WriteFile(path+".stdout", raw.Bytes(), 0600); saveErr != nil {
|
|
err = fmt.Errorf("retaining stdout: %w", saveErr)
|
|
}
|
|
}
|
|
if err == nil {
|
|
report, err = os.ReadFile(path + ".check.json")
|
|
if err == nil {
|
|
args, err = selectBlockcheckReport(bytes.NewReader(report))
|
|
if err == nil {
|
|
fmt.Fprintln(stderr, "Combined blockcheck JSON path:", path+".combined.json")
|
|
err = check(ctx, "blockcheckw", path, args, io.MultiWriter(&raw, stderr), stderr)
|
|
if saveErr := os.WriteFile(path+".stdout", raw.Bytes(), 0600); saveErr != nil {
|
|
err = fmt.Errorf("retaining stdout: %w", saveErr)
|
|
}
|
|
}
|
|
if err == nil {
|
|
report, err = os.ReadFile(path + ".combined.json")
|
|
}
|
|
if err == nil {
|
|
output, err = combinedBlockcheckOutput(bytes.NewReader(report), args)
|
|
}
|
|
if err == nil {
|
|
err = ctx.Err()
|
|
}
|
|
}
|
|
}
|
|
if saveErr := saveRun(db, started, args, output, report, err); saveErr != nil {
|
|
fmt.Fprintln(stderr, "error: tune youtube database:", saveErr)
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "error: tune youtube:", err)
|
|
}
|
|
return 1
|
|
}
|
|
if err == nil {
|
|
_, err = fmt.Fprintln(stdout, output)
|
|
}
|
|
if err != nil {
|
|
fmt.Fprintln(stderr, "error: tune youtube:", err)
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func normalizeProtocol(protocol string) string {
|
|
switch protocol {
|
|
case "HTTPS/TLS1.2":
|
|
return "tls12"
|
|
case "HTTPS/TLS1.3":
|
|
return "tls13"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func selectBlockcheckReport(r io.Reader) (string, error) {
|
|
return selectBlockcheckArgs(r, "")
|
|
}
|
|
|
|
func combinedBlockcheckOutput(r io.Reader, args string) (string, error) {
|
|
return combinedBlockcheckOutputForHost(r, args, "youtube.com")
|
|
}
|
|
|
|
func combinedBlockcheckOutputForHost(r io.Reader, args, host string) (string, error) {
|
|
if args == "" {
|
|
return "", fmt.Errorf("empty combined args")
|
|
}
|
|
if _, err := selectBlockcheckArgsForTarget(r, args, "frontend", host); err != nil {
|
|
return "", err
|
|
}
|
|
return "NFQWS2_OPT='--filter-tcp=443 --filter-l7=tls --hostlist-domains=" + host + " " + args + "'", nil
|
|
}
|
|
|
|
func selectBlockcheckArgs(r io.Reader, expected string) (string, error) {
|
|
return selectBlockcheckArgsForTarget(r, expected, "frontend", "youtube.com")
|
|
}
|
|
|
|
func selectBlockcheckArgsForTarget(r io.Reader, expected, group, host string) (string, error) {
|
|
return selectBlockcheckEvidence(r, expected, group, host, false)
|
|
}
|
|
|
|
func selectBlockcheckEvidence(r io.Reader, expected, group, host string, recovery bool) (string, error) {
|
|
d := json.NewDecoder(r)
|
|
var report struct {
|
|
Strategies []json.RawMessage `json:"strategies"`
|
|
}
|
|
if err := d.Decode(&report); err != nil {
|
|
return "", err
|
|
}
|
|
if err := d.Decode(new(json.RawMessage)); err != io.EOF {
|
|
return "", fmt.Errorf("expected exactly one JSON check report object")
|
|
}
|
|
if len(report.Strategies) == 0 {
|
|
return "", fmt.Errorf("check report requires a nonempty strategies array")
|
|
}
|
|
best := make(map[string]candidate)
|
|
seen := make(map[string]bool)
|
|
nonpass := false
|
|
for i, record := range report.Strategies {
|
|
var fields map[string]json.RawMessage
|
|
if err := json.Unmarshal(record, &fields); err != nil {
|
|
return "", err
|
|
}
|
|
for _, key := range []string{"protocol", "args", "success_rate", "coverage", "passes_ok", "passes_total", "median_latency_ms", "median_speed_kbps"} {
|
|
if v, ok := fields[key]; !ok || bytes.Equal(bytes.TrimSpace(v), []byte("null")) {
|
|
return "", fmt.Errorf("result %d: missing or null %s", i, key)
|
|
}
|
|
}
|
|
var row struct {
|
|
candidate
|
|
Protocol string `json:"protocol"`
|
|
}
|
|
// v0.9.0 includes additional report metadata. Required measurements
|
|
// are checked above; unrelated metadata is not interpreted as evidence.
|
|
if err := json.Unmarshal(record, &row); err != nil {
|
|
return "", fmt.Errorf("result %d: %w", i, err)
|
|
}
|
|
protocol := normalizeProtocol(row.Protocol)
|
|
if protocol == "" {
|
|
return "", fmt.Errorf("result %d: unsupported check protocol %q", i, row.Protocol)
|
|
}
|
|
if _, exists := fields["group"]; !exists {
|
|
row.Group = group
|
|
}
|
|
if _, exists := fields["domains"]; !exists {
|
|
row.Domains = []string{host}
|
|
}
|
|
if err := row.validateMeasurements(); err != nil {
|
|
return "", fmt.Errorf("result %d: %w", i, err)
|
|
}
|
|
if row.Group != group || len(row.Domains) != 1 || row.Domains[0] != host {
|
|
return "", fmt.Errorf("result %d: unexpected target", i)
|
|
}
|
|
key := protocol + "\x00" + row.Args
|
|
if seen[key] {
|
|
return "", fmt.Errorf("result %d: duplicate args/protocol", i)
|
|
}
|
|
seen[key] = true
|
|
if recovery {
|
|
if row.Args != expected || row.PassesTotal != 3 {
|
|
return "", fmt.Errorf("unexpected saved args or incomplete check")
|
|
}
|
|
if err := row.validateArgs(); err != nil {
|
|
return "", err
|
|
}
|
|
nonpass = nonpass || !fullyPassing(row.candidate)
|
|
} else if expected != "" && (row.Args != expected || !fullyPassing(row.candidate) || row.PassesTotal != 3) {
|
|
return "", fmt.Errorf("result %d: combined args must pass exactly 3/3 for both protocols", i)
|
|
}
|
|
// Valid check evidence may describe args our inert output cannot represent.
|
|
// Exclude that candidate only; malformed measurements still fail above.
|
|
if err := row.validateArgs(); err != nil {
|
|
continue
|
|
}
|
|
if !fullyPassing(row.candidate) || row.PassesTotal < 3 {
|
|
continue
|
|
}
|
|
if old, ok := best[protocol]; !ok || better(row.candidate, old) {
|
|
best[protocol] = row.candidate
|
|
}
|
|
}
|
|
if recovery {
|
|
if len(seen) != 2 || !seen["tls12\x00"+expected] || !seen["tls13\x00"+expected] {
|
|
return "", fmt.Errorf("incomplete combined TLS evidence")
|
|
}
|
|
if nonpass {
|
|
return "", errCandidateNonpass
|
|
}
|
|
return expected, nil
|
|
}
|
|
var args []string
|
|
payloads := make(map[string]bool)
|
|
for _, protocol := range []string{"tls12", "tls13"} {
|
|
c, ok := best[protocol]
|
|
if !ok {
|
|
return "", errNoQualifiedStrategy
|
|
}
|
|
if protocol == "tls13" && c.Args == best["tls12"].Args {
|
|
continue
|
|
}
|
|
for _, token := range strings.Fields(c.Args) {
|
|
if strings.HasPrefix(token, "--payload=") {
|
|
if payloads[token] {
|
|
continue
|
|
}
|
|
payloads[token] = true
|
|
}
|
|
args = append(args, token)
|
|
}
|
|
}
|
|
return strings.Join(args, " "), nil
|
|
}
|