提升媒体节点跨可用区调度的地域亲和性策略技巧

# 推荐标签规范（兼容 Kubernetes 标准拓扑键）
labels:
  # L1: 地域级 - 物理机房/公有云 Region
  topology.kubernetes.io/region: "cn-hangzhou"
  
  # L2: 可用区级 - 独立电力/网络故障域
  topology.kubernetes.io/zone: "cn-hangzhou-i"
  
  # L3: 机架/交换机级 - 可选，用于超大规模集群的网络微隔离
  topology.kubernetes.io/rack: "rack-03"
  
  # 业务自定义语义标签（便于策略组合）
  media.node/type: "sfu"              # 节点角色：sfu/mcu/transcoder/origin
  media.network/tier: "high-perf"     # 网络层级：标识高性能网卡/DPU节点
  media.hardware/gpu: "nvidia-t4"     # 硬件亲和性标识

apiVersion: apps/v1
kind: Deployment
metadata:
  name: media-sfu-cluster
spec:
  replicas: 12
  selector:
    matchLabels:
      app: media-sfu
  template:
    metadata:
      labels:
        app: media-sfu
    spec:
      topologySpreadConstraints:
      # 约束 1：跨可用区“软性均匀分布” - 保障高可用底线
      - maxSkew: 1
        topologyKey: topology.kubernetes.io/zone
        whenUnsatisfiable: ScheduleAnyway  # 关键：不满足时仍允许调度，避免资源碎片化导致 Pending
        labelSelector:
          matchLabels:
            app: media-sfu
        minDomains: 3  # 1.24+ 建议开启，显式声明预期覆盖的 AZ 数量
      
      # 约束 2：同可用区内“机架级反亲和” - 规避单交换机故障
      - maxSkew: 1
        topologyKey: topology.kubernetes.io/rack
        whenUnsatisfiable: DoNotSchedule   # 机架级故障域较小，建议硬性约束
        labelSelector:
          matchLabels:
            app: media-sfu
      
      # 约束 3：节点维度“单节点单实例” - 避免资源争抢
      - maxSkew: 1
        topologyKey: kubernetes.io/hostname
        whenUnsatisfiable: DoNotSchedule
        labelSelector:
          matchLabels:
            app: media-sfu
      
      # 亲和性补充：优先调度至高性能网络节点池
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 80
            preference:
              matchExpressions:
              - key: media.network/tier
                operator: In
                values: ["high-perf"]
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: media.node/type
                operator: In
                values: ["sfu"]  # 防止误调度至转码节点池

# CRD: networktopology.media.io/v1alpha1
apiVersion: media.io/v1alpha1
kind: NetworkTopology
metadata:
  name: cluster-mesh-topology
spec:
  zones:
  - name: cn-hangzhou-i
    region: cn-hangzhou
    nodes:
    - name: k8s-node-sfu-01
      # 实时遥测指标（由 Network Controller 定时更新，如每 30s）
      metrics:
        egressBandwidthGbps: 50
        avgRttMsToOtherZones:
          cn-hangzhou-j: 1.2
          cn-hangzhou-k: 1.5
        packetLossRate: 0.0001
    links:
    - targetZone: cn-hangzhou-j
      capacityGbps: 100
      currentUtilization: 0.35
      costPerGB: 0.05  # 单位：元/GB，用于成本感知评分

// pkg/scheduler/plugins/mediatopologyscore/score.go
func (p *MediaTopologyScore) Score(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
    nodeInfo, _ := p.handle.SnapshotSharedLister().NodeInfos().Get(nodeName)
    nodeZone := nodeInfo.Node().Labels[topologyZoneKey]
    podZone := getPodPreferredZone(pod) // 从 Pod Annotation 或 ConfigMap 解析业务期望 Zone
    
    // 1. 获取实时网络拓扑视图
    topo := p.networkTopologyCache.Get()
    if topo == nil { return 50, nil } // 降级：无拓扑数据时给中性分
    
    // 2. 计算多维得分 (0-100)
    var score float64
    
    // 维度 A: 就近性得分 (权重 40%) - 同 Zone 满分，跨 Zone 按 RTT 指数衰减
    proximityScore := calculateProximityScore(nodeZone, podZone, topo)
    
    // 维度 B: 成本得分 (权重 30%) - 优先走低成本链路
    costScore := calculateCostScore(nodeZone, podZone, topo)
    
    // 维度 C: 负载均衡得分 (权重 20%) - 避免热点 Zone
    balanceScore := calculateBalanceScore(nodeZone, p.handle.SnapshotSharedLister())
    
    // 维度 D: 硬件亲和得分 (权重 10%) - GPU/DPU 匹配度
    hwScore := calculateHardwareAffinityScore(pod, nodeInfo)
    
    score = 0.4*proximityScore + 0.3*costScore + 0.2*balanceScore + 0.1*hwScore
    
    // 3. 归一化输出
    return int64(score * 10), nil // framework 要求 0-1000
}

// 关键算法：跨 Zone RTT 指数衰减函数
func calculateProximityScore(nodeZone, targetZone string, topo *NetworkTopology) float64 {
    if nodeZone == targetZone { return 100 }
    link := topo.GetLink(nodeZone, targetZone)
    if link == nil { return 10 } // 无直连链路，极低分
    // RTT 1ms -> 95分, 2ms -> 85分, 5ms -> 50分, 10ms -> 10分
    return math.Max(10, 100 - 15*math.Log2(link.AvgRttMs))
}

func (p *MediaStatefulPermit) Permit(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (*framework.Status, time.Duration) {
    // 仅对标记为有状态的媒体负载生效
    if !isStatefulMediaPod(pod) { return framework.NewStatus(framework.Success), 0 }
    
    // 1. 检查目标节点是否已有同类 Pod 运行（反亲和兜底）
    // 2. 关键逻辑：等待新 Pod 通过健康检查并完成状态同步/信令注册
    //    通过监听 Pod Condition: `MediaStateSynced=True` (由业务 Sidecar 上报)
    
    waitCh := make(chan struct{})
    go func() {
        // 轮询或 Watch Pod Status，最长等待 PermitTimeout (建议 5-10min)
        if waitForMediaReady(pod.Name, pod.Namespace, p.permitTimeout) {
            close(waitCh) // 允许 Bind
        } else {
            // 超时拒绝，触发调度器重新评分选点
            p.reject(pod, "Stateful media pod sync timeout")
        }
    }()
    
    // 返回 "Wait" 状态，调度器挂起该 Pod 的 Bind 阶段
    return framework.NewStatus(framework.Wait, "Waiting for media state sync"), p.permitTimeout
}

# StorageClass 定义：强制拓扑感知
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: media-recording-ssd-topology
provisioner: disk.csi.aliyun.com # 或 aws-ebs, gcp-pd
volumeBindingMode: WaitForFirstConsumer # 关键：延迟绑定，由调度器决定 PV 创建 Zone
allowedTopologies:
- matchLabelExpressions:
  - key: topology.kubernetes.io/zone
    values: ["cn-hangzhou-i", "cn-hangzhou-j", "cn-hangzhou-k"]
parameters:
  type: cloud_essd
  performanceLevel: "PL2"
---
# PVC 使用
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: media-recording-pvc
spec:
  storageClassName: media-recording-ssd-topology
  accessModes: ["ReadWriteOnce"]
  resources:
    requests:
      storage: 500Gi

# Karmada PropagationPolicy: 定义媒体节点跨集群分布策略
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: media-sfu-propagation
spec:
  resourceSelectors:
  - apiVersion: apps/v1
    kind: Deployment
    name: media-sfu
    namespace: media-prod
  placement:
    clusterAffinity:
      # 硬性约束：核心转码必须有 GPU 的集群
      requiredDuringSchedulingIgnoredDuringExecution:
      - clusterNames: ["gpu-cluster-hz", "gpu-cluster-sh"]
    clusterSpreadConstraints:
    # 软性约束：SFU 接入层按用户地理分布就近部署
    - maxSkew: 1
      topologyKey: "region" # Karmada 自动聚合的集群标签
      whenUnsatisfiable: ScheduleAnyway
      weight: 80
    replicaScheduling: WeightedPreference
    replicaSchedulingType: Divided
    weightPreference:
    - targetCluster:
        clusterNames: ["edge-cluster-bj", "edge-cluster-sh"] # 边缘集群权重高
      weight: 100
    - targetCluster:
        clusterNames: ["central-cluster-hz"] # 中心集群兜底
      weight: 30
---
# OverridePolicy: 针对不同集群差异化注入配置 (如镜像仓库、资源限额、拓扑标签)
apiVersion: policy.karmada.io/v1alpha1
kind: OverridePolicy
metadata:
  name: media-sfu-override-edge
spec:
  resourceSelectors:
  - apiVersion: apps/v1
    kind: Deployment
    name: media-sfu
  overrideRules:
  - targetCluster:
      clusterNames: ["edge-cluster-*"]
    overriders:
      plaintext:
      - path: "/spec/template/spec/containers/0/image"
        operator: replace
        value: "registry.internal/edge-sfu:v1.2.0-arm64" # 边缘集群多为 ARM 架构
      - path: "/spec/template/spec/nodeSelector"
        operator: add
        value:
          node-type: "edge-media"
          network-tier: "5g-backhaul"

# 伪代码：动态成本权重控制器 (作为独立 Controller 运行，每 5 分钟调谐一次)
def tune_cost_weight():
    # 1. 获取过去 1 小时业务指标
    current_p99_latency = get_metric("media_e2e_latency_p99")
    current_cost_per_gb = get_metric("cross_az_bandwidth_cost_per_gb")
    budget_threshold = get_config("daily_budget_threshold") # 如 80% 预算线
    
    # 2. PID 控制器逻辑调整权重 w_cost
    # 目标：延迟 < SLA(300ms) 且 成本 < 预算线
    error_latency = max(0, current_p99_latency - 300)
    error_cost = max(0, current_cost_per_gb - budget_threshold)
    
    # 延迟超标 -> 降低成本权重 (w_cost -= 0.1)，强迫就近调度
    # 成本超标 -> 提高成本权重 (w_cost += 0.1)，允许跨 AZ 走低成本链路
    new_w_cost = clamp(base_w_cost - 0.1*error_latency + 0.15*error_cost, 0.1, 0.5)
    
    # 3. 热更新调度器插件配置 (通过 ConfigMap Reload 或 gRPC 动态下发)
    update_scheduler_configmap("media-scheduler-config", {"costWeight": new_w_cost})
    log.info(f"Dynamic cost weight tuned to: {new_w_cost}")

# ScaledObject: 当跨 AZ 流量成本激增时，自动修改 Deployment 调度策略为“强制同 AZ”
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: media-sfu-scheduler-policy-autoscaler
spec:
  scaleTargetRef:
    name: media-sfu # 目标 Deployment
  triggers:
  - type: prometheus
    metadata:
      serverAddress: http://prometheus.monitoring.svc
      metricName: cross_az_bandwidth_cost_rate_yuan_per_min
      threshold: "500" # 单位：元/分钟，触发阈值
      query: sum(rate(cross_az_traffic_bytes[5m])) * on(zone) group_left cost_per_gb / 1024/1024/1024
      # 当成本超阈值，KEDA 修改 Deployment Annotation 触发滚动更新
      # 需配合自定义 Controller 监听 Annotation 变更，Patch Deployment 的 topologySpreadConstraints
  advanced:
    restoreToOriginalReplicas: true
    horizontalPodAutoscalerConfig:
      behavior:
        scaleDown:
          stabilizationWindowSeconds: 300 # 成本下降后 5 分钟恢复弹性策略

# Makefile 片段
test-scheduler-policy:
    # 1. 启动模拟器，加载真实集群快照 (Node/Pod/PV/NetworkTopology CRD)
    kind load docker-image scheduler-simulator:latest
    kubectl run simulator --image=scheduler-simulator:latest -- 
      --snapshot-path=./testdata/cluster-snapshot.yaml 
      --policy-config=./config/scheduler-config.yaml 
      --test-scenarios=./testdata/scenarios.yaml
    
# scenarios.yaml 示例
scenarios:
- name: "SFU 扩容 10 副本，3 AZ 资源充足"
  initialPods: 20
  targetReplicas: 30
  expectedDistribution:
    cn-hangzhou-i: 10
    cn-hangzhou-j: 10
    cn-hangzhou-k: 10
  assertions:
    - "maxSkew <= 1"
    - "noPodPending"
    - "crossAzTrafficRatio < 0.05"

- name: "AZ-j 故障模拟，流量迁移至 AZ-i/k"
  initialPods: 30
  nodeTaints:
    - key: "node.kubernetes.io/unreachable"
      effect: "NoExecute"
      nodeSelector: "topology.kubernetes.io/zone=cn-hangzhou-j"
  expectedBehavior:
    - "rescheduleTo: [cn-hangzhou-i, cn-hangzhou-k]"
    - "permitPluginWaitForSync: true"
    - "maxDisruptionBudgetViolation: 0"

# Argo Rollout 管理 Scheduler ConfigMap
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: media-scheduler-config-rollout
spec:
  replicas: 1 # 调度器通常单副本或 Leader 选举，ConfigMap 灰度即生效
  strategy:
    canary:
      canaryMetadata:
        annotations:
          rollout.argoproj.io/track: "canary"
      stableMetadata:
        annotations:
          rollout.argoproj.io/track: "stable"
      steps:
      - setWeight: 10  # 10% 节点先应用新策略 (通过 NodeSelector 选取金丝雀节点池)
      - pause: {duration: 30m}
      - analysis:
          templates:
          - templateName: scheduler-policy-analysis
      - setWeight: 50
      - pause: {duration: 1h}
      - setWeight: 100
  selector:
    matchLabels:
      app: kube-scheduler
  template:
    # ... 调度器 Pod 模板，挂载 ConfigMap
---
# AnalysisTemplate: 自动化判定新策略优劣
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: scheduler-policy-analysis
spec:
  metrics:
  - name: scheduling-latency-p99
    interval: 5m
    successCondition: result[0] < 500 # ms
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus.monitoring.svc
        query: histogram_quantile(0.99, rate(scheduler_scheduling_duration_seconds_bucket[5m]))
  - name: cross-az-traffic-ratio
    interval: 10m
    successCondition: result[0] < 0.15
    provider:
      prometheus:
        query: |
          sum(rate(cross_az_media_traffic_bytes[10m])) / sum(rate(total_media_traffic_bytes[10m]))
  - name: pod-pending-rate
    successCondition: result[0] == 0
    provider:
      prometheus:
        query: |
          sum(kube_pod_status_phase{phase="Pending", namespace="media-prod"}) / sum(kube_pod_status_phase{namespace="media-prod"})

{
  "timestamp": "2024-05-20T10:30:00.123Z",
  "auditType": "SCHEDULING_DECISION",
  "pod": {"uid": "xxx", "name": "media-sfu-xxx", "namespace": "media-prod", "labels": {"roomId": "r_123"}},
  "decision": {
    "selectedNode": "k8s-node-sfu-05",
    "selectedZone": "cn-hangzhou-j",
    "scoreBreakdown": {"proximity": 95, "cost": 80, "balance": 70, "hardware": 100},
    "rejectedNodes": [
      {"node": "k8s-node-sfu-01", "zone": "cn-hangzhou-i", "reason": "Insufficient GPU", "score": 0},
      {"node": "k8s-node-sfu-09", "zone": "cn-hangzhou-k", "reason": "CrossAzCostPenalty", "score": 45}
    ]
  },
  "compliance": {
    "dataResidencyCheck": "PASSED", // 检查目标 Zone 是否满足数据驻留合规要求
    "crossBorderFlow": false
  },
  "trigger": "SCALE_OUT_HPA"
}

# Kyverno ClusterPolicy: 禁止媒体核心负载调度至非合规 Zone (如海外节点)
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: media-workload-zone-compliance
spec:
  validationFailureAction: Enforce
  rules:
  - name: check-media-node-zone
    match:
      any:
      - resources:
          kinds: ["Deployment", "StatefulSet", "Pod"]
          selector:
            matchLabels:
              media.node/type: "sfu" # 仅针对媒体核心节点
    validate:
      message: "Media core workloads must be scheduled in compliant zones (cn-*)."
      pattern:
        spec:
          template:
            spec:
              affinity:
                nodeAffinity:
                  requiredDuringSchedulingIgnoredDuringExecution:
                    nodeSelectorTerms:
                    - matchExpressions:
                      - key: topology.kubernetes.io/region
                        operator: In
                        values: ["cn-*"] # 仅允许中国大陆 Region