降低云视频会议媒体节点弹性扩容冷启动延迟的预热镜像技巧

# Stage 1: Builder
FROM golang:1.22-alpine AS builder
RUN apk add --no-cache git make
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o /media-node .

# Stage 2: Runtime (Distroless)
FROM gcr.io/distroless/static-debian12:nonroot
COPY --from=builder /media-node /usr/local/bin/
COPY --from=builder /src/configs /etc/media-node/
# 预热：运行一次基准测试生成 PGO 数据
RUN /usr/local/bin/media-node --benchmark=once --output=/var/cache/pgo
ENTRYPOINT ["/usr/local/bin/media-node"]

# Deployment 关键字段
spec:
  replicas: 0  # 平时缩为 0，由 HPA/VPA 动态扩容
  template:
    spec:
      schedulerName: default-scheduler
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 80
            preference:
              matchExpressions:
              - key: topology.kubernetes.io/zone
                operator: In
                values: ["cn-hangzhou-a"]  # 就近调度
      tolerations:
      - key: "media-node-pool"
        operator: "Exists"
        effect: "NoSchedule"

stages:
  - build          # 多阶段构建 + 单元测试
  - prewarm        # 核心新增：基准跑分 + PGO 数据生成
  - scan           # 漏洞扫描 + SBOM 生成 + 签名
  - sync           # 多 Registry 同步 + 多架构 Manifest 推送
  - promote        # 语义化版本打 Tag + 策略准入

variables:
  DOCKER_HOST: tcp://docker:2376
  DOCKER_TLS_CERTDIR: "/certs"
  REGISTRY_PRIMARY: registry.cn-hangzhou.aliyuncs.com/${GROUP}
  REGISTRY_SECONDARY: registry.aws.internal/${GROUP}
  COSIGN_PASSWORD: ${COSIGN_KEY_PASS}  # 密钥托管在 Vault

build:amd64:
  stage: build
  image: docker:24-dind
  services: [docker:24-dind]
  script:
    - docker buildx build --platform linux/amd64 --target runtime -t $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA .
    - docker push $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA

prewarm:benchmark:
  stage: prewarm
  needs: [build:amd64]
  image: $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA  # 直接复用构建产物
  variables:
    ENTRYPOINT: ["/usr/local/bin/media-node", "--benchmark=once", "--output=/var/cache/pgo"]
  script:
    - mkdir -p /workspace/cache
    - docker run --rm -v /workspace/cache:/var/cache $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA
    - docker buildx build --platform linux/amd64 --target runtime 
        --cache-from $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA 
        -v /workspace/cache:/var/cache 
        --build-arg PGO_CACHE_DIR=/var/cache 
        -t $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA-pgo .
    - docker push $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA-pgo

scan:security:
  stage: scan
  needs: [prewarm:benchmark]
  image: aquasec/trivy:latest
  script:
    - trivy image --scanners vuln,secret,config --severity HIGH,CRITICAL 
        --format cyclonedx --output sbom-$CI_COMMIT_SHA.json $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA-pgo
    - syft $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA-pgo -o spdx-json=sbom-spdx-$CI_COMMIT_SHA.json
    - cosign sign --yes --annotations "sbom.cyclonedx=@sbom-$CI_COMMIT_SHA.json" 
        $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA-pgo
  artifacts:
    reports:
      container_scanning: gl-container-scanning-report.json
    paths: [sbom-*.json]
    expire_in: 90d

sync:multi-registry:
  stage: sync
  needs: [scan:security]
  image: google/go-containerregistry:latest
  script:
    - crane copy $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA-pgo $REGISTRY_SECONDARY/media-node:$CI_COMMIT_SHA-pgo
    - crane copy $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA-pgo $REGISTRY_PRIMARY/media-node:latest-stable
    - crane manifest push $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA-pgo 
        $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA-pgo-arm64 
        $REGISTRY_PRIMARY/media-node:$CI_COMMIT_SHA-pgo-amd64 
        --platform linux/arm64,linux/amd64  # 多架构 Manifest

# Kyverno ClusterPolicy 片段
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: require-signed-media-node
spec:
  validationFailureAction: Enforce
  rules:
  - name: verify-cosign-signature
    match:
      any:
      - resources:
          kinds: [Pod]
          selector:
            matchLabels:
              app: media-node
    verifyImages:
    - image: "registry.cn-hangzhou.aliyuncs.com/*/media-node:*"
      key: |-
        -----BEGIN PUBLIC KEY-----
        MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE...
        -----END PUBLIC KEY-----
      annotations:
        - "sbom.cyclonedx:*"  # 必须附带 SBOM
  - name: block-critical-cves
    match: *match
    validate:
      manifest:
        - (image): "registry.cn-hangzhou.aliyuncs.com/*/media-node:*"
          pattern: "{{ trivy_scan_result.critical_count == 0 }}"

[GitLab CI] → [Primary Registry: 阿里云 ACR EE 华东 2]
                    │
        ┌───────────┴───────────┐
        ▼                       ▼
[P2P Seed Peer: 华东 2]   [Cross-Region Replication: ACR EE 华北 2 / AWS ECR cn-north-1]
        │                       │
        ▼                       ▼
[Dragonfly Supernode]     [Dragonfly Supernode]
        │                       │
        └───────────┬───────────┘
                    ▼
         [各可用区 Worker Node DaemonSet: dfdaemon]

# 仅允许带有 `security.scan/passed: "true"` Label 的镜像启动
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: require-scan-label
spec:
  rules:
  - name: check-scan-label
    match:
      any:
      - resources:
          kinds: [Pod]
    validate:
      manifests:
      - (image): "*"
        pattern: "{{ metadata.labels['security.scan/passed'] == 'true' }}"
        message: "镜像未通过安全扫描或缺少扫描标签，禁止部署"

def calculate_optimal_warm_pool(
    forecast_qps: list[float],           # 未来 24h 分钟级 QPS 预测
    node_capacity: int,                  # 单节点承载并发数
    cold_start_p99: float,               # 当前冷启动 P99 (秒)
    sla_latency_budget: float,           # 业务可容忍扩容延迟 (秒)
    spot_price_ratio: float = 0.3,       # 抢占式实例折扣
    on_demand_hourly: float = 1.2        # 按量实例单价 (元/小时)
) -> int:
    """
    目标：最小化 总成本 = 就绪池成本 + 冷启动惩罚成本 + SLA 违约风险成本
    约束：冷启动 P99 * 扩容批次数 <= SLA 预算
    """
    import numpy as np
    from scipy.optimize import minimize_scalar

    def total_cost(warm_replicas: int) -> float:
        # 1. 就绪池成本（混合抢占式/按量）
        warm_cost = warm_replicas * on_demand_hourly * (1 - spot_price_ratio) * 24

        # 2. 冷启动惩罚：预测峰值超出就绪池的部分
        peak_surplus = max(0, max(forecast_qps) / node_capacity - warm_replicas)
        cold_penalty = peak_surplus * cold_start_p99 / 3600 * on_demand_hourly * 1.5  # 1.5 倍惩罚系数

        # 3. SLA 风险：若冷启动超 budget，按合同罚款估算
        sla_risk = max(0, cold_start_p99 - sla_latency_budget) * 1000  # 简化模型

        return warm_cost + cold_penalty + sla_risk

    res = minimize_scalar(total_cost, bounds=(0, max(forecast_qps)/node_capacity*1.2), method='bounded')
    return int(np.ceil(res.x))

# 实际接入：每日 02:00 由 Airflow DAG 运行，结果写入 ConfigMap，HPA Controller 读取调整 minReplicas

graph TD
    A[HPA 连续扩容] --> B[Kube-scheduler 批量绑定节点]
    B --> C[同可用区 40 节点并发拉取 1.2 GB 镜像]
    C --> D[ACR 单节点带宽限流 1 Gbps]
    D --> E[Dragonfly Supernode 磁盘 IOPS 打满]
    E --> F[P2C 回源雪崩 → Registry 504]
    F --> G[Kubelet 重试指数退避 → 总耗时 > 2 min]
    G --> H[Pod 超过 `progressDeadlineSeconds` 标记 Failed]
    H --> I[HPA 判定扩容失败 → 继续扩容 → 死循环]

# 编译 Rust 版 SFU 核心逻辑为 Wasm
cargo build --target wasm32-wasip1 --release
# 产物大小：4.2 MB (vs 容器镜像 68 MB)
# 启动基准：
$ time wasmedge --dir=. sfu.wasm --benchmark=once
real    0m0.087s   # 冷启动含模块实例化+预热
user    0m0.042s
sys     0m0.031s