# 1. 确认硬件时间源质量
lscpu | grep -i "tsc|invariant"
dmesg | grep -i "tsc|clocksource"

# 2. 部署 chrony + 硬件时间戳
# /etc/chrony.conf 关键配置
refclock PHC /dev/ptp0 poll 3 dpoll -2 offset 0  # 物理网卡 PTP 硬件时间戳
server ntp.aliyun.com iburst minpoll 4 maxpoll 4  # 固定 16s 同步间隔
makestep 0.1 3                                    # 启动期快速跟随
log measurements statistics tracking

# Kubernetes Pod 级时钟同步最佳实践
spec:
  securityContext:
    sysctls:
    - name: kernel.sched_rt_runtime_us
      value: "-1"          # 允许实时调度，减少 vCPU 抢占抖动
  containers:
  - name: media-server
    securityContext:
      capabilities:
        add: ["SYS_TIME", "SYS_RESOURCE", "CAP_DAC_OVERRIDE"]
    volumeMounts:
    - name: ptp
      mountPath: /dev/ptp0   # 直通宿主机 PTP 硬件时钟
  volumes:
  - name: ptp
    hostPath:
      path: /dev/ptp0

// 伪代码：每帧回调中记录
struct FrameMeta {
  uint64_t ntp_ts;        // 业务层 NTP 时间戳（发送端打标/接收端解析）
  uint64_t local_mono;    // clock_gettime(CLOCK_MONOTONIC_RAW)
  uint64_t local_realtime;// clock_gettime(CLOCK_REALTIME)
  uint32_t stream_id;
  uint8_t  media_type;    // 0=audio, 1=video
};

// 定期上报至时序库，计算：
// delta_ntp_mono = ntp_ts - local_mono
// delta_ntp_real = ntp_ts - local_realtime
// 观测 delta 随时间的线性漂移斜率

offset = (T2 - T1 + T3 - T4) / 2
delay  = (T4 - T1) - (T3 - T2)
asymmetry = (T2 - T1) - (T3 - T4)  // 非零即不对称

# 接收端抓包对比 RTP timestamp 与 NTP timestamp 增长速率
tshark -r capture.pcap -Y "rtp" -T fields -e frame.time_relative -e rtp.timestamp -e rtp.ntp_time

# 宿主机
chronyc tracking | grep "System time"

# 容器内
chronyc tracking | grep "System time"

# 对比差值

# 从 Prometheus 拉取最近 2 小时 chrony_system_offset_seconds
import pandas as pd
df = pd.read_promql('chrony_system_offset_seconds[2h]')
df['drift_ppm'] = df['value'].diff() / df['timestamp'].diff() * 1e6
print(f"最大漂移率: {df['drift_ppm'].max():.1f} ppm")
print(f"累积偏移: {df['value'].iloc[-1] - df['value'].iloc[0]:.3f} s")

graph TB
    subgraph Core[核心时钟层 Stratum 0/1]
        GM1[(Grandmaster<br/>原子钟/GNSS)]
        GM2[(Grandmaster<br/>备用)]
        PTP_SW[PTP 透明时钟/边界时钟交换机]
    end

    subgraph Access[接入汇聚层 Stratum 2/3]
        NTP_E1[Chrony NTP Server<br/>Interleaved Mode + xleave]
        NTP_E2[Chrony NTP Server<br/>Interleaved Mode + xleave]
        PTP_BC[PTP Boundary Clock<br/>普通服务器双网卡]
    end

    subgraph Edge[业务边缘层 Stratum 3/4]
        K8S_MASTER[K8s Master/Etcd<br/>时间基准]
        MEDIA_NODE[媒体节点<br/>PTP 直通 / NTP Client]
        EDGE_GW[边缘网关<br/>PTP Slave Only]
        MOBILE_SDK[移动端 SDK<br/>NTP + 本地 Kalman 滤波]
    end

    GM1 --> PTP_SW
    GM2 --> PTP_SW
    PTP_SW --> NTP_E1
    PTP_SW --> NTP_E2
    PTP_SW --> PTP_BC
    NTP_E1 --> K8S_MASTER
    NTP_E2 --> K8S_MASTER
    PTP_BC --> MEDIA_NODE
    PTP_BC --> EDGE_GW
    NTP_E1 -.->|Interleaved| MOBILE_SDK

# /etc/chrony.conf 核心接入层配置
# 1. 启用 Interleaved 模式（需服务端/客户端均 ≥ 4.0）
server ntp-core-1.internal iburst minpoll 4 maxpoll 4 xleave
server ntp-core-2.internal iburst minpoll 4 maxpoll 4 xleave

# 2. 硬件时间戳（需网卡驱动支持 SO_TIMESTAMPING）
hwtimestamp *

# 3. 允许客户端使用 Interleaved
allow all interleaved

# 4. 严格的漂移控制
maxdrift 50              # 限制最大频率调整 50 ppm
makestep 0.5 1           # 启动期允许阶跃 0.5s，之后仅频率调整
logchange 0.1            # 记录 > 0.1 ms 的阶跃变化

# 5. 监测指标导出
log measurements statistics tracking

chronyc ntpdata | grep -A5 "Interleaved"
# 输出包含 "Interleaved : Yes" 即生效
# 此时 offset 计算公式改为：offset = (T2 - T1') / 2，消除请求/响应路径不对称误差

// 核心思路：融合「NTP 绝对时间（低频、高噪）」+「本地单调时钟（高频、低噪、漂移）」
// 状态向量 X = [offset, drift]^T
// 观测值 Z = NTP_offset_measurement

class ClockEstimator {
  private x = [0, 0];           // [offset_ms, drift_ppm]
  private P = [[1000, 0], [0, 100]]; // 协方差矩阵
  private readonly Q = [[0.01, 0], [0, 0.001]]; // 过程噪声
  private readonly R = 25;      // 观测噪声方差 (5ms^2)

  // 每次收到 NTP 响应调用
  update(ntpOffsetMs: number, localMonotonicMs: number) {
    const dt = (localMonotonicMs - this.lastLocalMs) / 1000; // 秒
    if (dt <= 0) return;

    // 1. 预测
    this.x[0] += this.x[1] * dt * 1e-3; // offset += drift * dt
    this.P[0][0] += dt * (2 * this.P[0][1] + dt * this.P[1][1]) + this.Q[0][0];
    this.P[0][1] += dt * this.P[1][1] + this.Q[0][1];
    this.P[1][0] = this.P[0][1];
    this.P[1][1] += this.Q[1][1];

    // 2. 更新
    const y = ntpOffsetMs - this.x[0]; // 创新量
    const S = this.P[0][0] + this.R;
    const K = [this.P[0][0] / S, this.P[1][0] / S]; // 卡尔曼增益

    this.x[0] += K[0] * y;
    this.x[1] += K[1] * y;

    this.P[0][0] -= K[0] * this.P[0][0];
    this.P[0][1] -= K[0] * this.P[0][1];
    this.P[1][0] -= K[1] * this.P[0][0];
    this.P[1][1] -= K[1] * this.P[0][1];

    this.lastLocalMs = localMonotonicMs;
  }

  // 业务层获取当前最佳 NTP 时间
  getEstimatedNtpMs(localMonotonicMs: number): number {
    const dt = (localMonotonicMs - this.lastLocalMs) / 1000;
    return localMonotonicMs + this.x[0] + this.x[1] * dt * 1e-3;
  }
}

// 仅在检测到系统时钟回拨/跳跃时修正
let lastPerf = performance.now();
let lastMonotonic = Date.now(); // 近似
setInterval(() => {
  const nowPerf = performance.now();
  const nowMono = Date.now();
  const deltaPerf = nowPerf - lastPerf;
  const deltaMono = nowMono - lastMonotonic;
  if (Math.abs(deltaPerf - deltaMono) > 50) { // 50ms 阈值
    // 上报异常，触发 NTP 重新同步
    reportClockAnomaly({ deltaPerf, deltaMono });
  }
  lastPerf = nowPerf;
  lastMonotonic = nowMono;
}, 5000);

# clock-recovery-operator CRD 示例
apiVersion: clock.example.com/v1alpha1
kind: ClockRecoveryPolicy
metadata:
  name: media-node-policy
spec:
  targetRef:
    kind: Deployment
    name: media-server
  metrics:
    - name: chrony_system_offset_seconds
      threshold: 0.005  # 5ms
      duration: 300s
    - name: chrony_rms_offset_seconds
      threshold: 0.001
      duration: 60s
  actions:
    - level: L1
      type: ChronyCommand
      command: "makestep"
      condition: "offset < 0.1"
    - level: L2
      type: ConfigPatch
      patch:
        - op: replace
          path: /spec/template/spec/containers/0/env/-
          value: {name: CHRONY_MINPOLL, value: "4"}
    - level: L3
      type: K8sAction
      action: CordonAndDrain
      postHook: "ansible-playbook fix_clock.yml -l {{.nodeName}}"

# Chrony Server (需证书)
ntsservercert /etc/chrony/nts-server.pem
ntsserverkey /etc/chrony/nts-server.key

# Chrony Client
server time.example.com iburst nts
# 验证：chronyc -N authdata | grep NTS

#!/bin/bash
# clock_health_check.sh - 单节点时钟健康度全景扫描
# 用法: curl -sSL https://raw.example.com/clock_health_check.sh | bash

set -euo pipefail
RED='33[0;31m'; GREEN='33[0;32m'; YELLOW='33[1;33m'; NC='33[0m'

echo "========== [$(date -Is)] 时钟健康度巡检 =========="

# 1. 硬件时间源
echo -e "n${YELLOW}>> 硬件时间源${NC}"
lscpu | grep -iE "tsc|invariant|constant" | sed 's/^/   /'
dmesg -T | grep -iE "tsc|clocksource|hpet" | tail -3 | sed 's/^/   /'

# 2. Chrony 状态
echo -e "n${YELLOW}>> Chrony 同步状态${NC}"
if command -v chronyc &>/dev/null; then
  chronyc tracking | grep -E "Reference ID|Stratum|System time|Last offset|RMS offset|Root dispersion|Leap status" | sed 's/^/   /'
  chronyc sources -v | awk '/^^|*/ {printf "   %-20s %s %s %sn", $1, $2, $3, $4}'
  chronyc ntpdata | grep -A2 "Interleaved" | head -3 | sed 's/^/   /'
else
  echo -e "   ${RED}chronyd 未安装${NC}"
fi

# 3. 容器时钟对比 (若在容器内)
if [ -f /.dockerenv ] || grep -qa container /proc/1/cgroup 2>/dev/null; then
  echo -e "n${YELLOW}>> 容器/宿主机时钟对比${NC}"
  HOST_OFFSET=$(nsenter -t 1 -m chronyc tracking 2>/dev/null | awk '/System time/ {print $4$5}' || echo "N/A")
  CONT_OFFSET=$(chronyc tracking 2>/dev/null | awk '/System time/ {print $4$5}' || echo "N/A")
  echo "   宿主机 System time: $HOST_OFFSET"
  echo "   容器内 System time: $CONT_OFFSET"
fi

# 4. 网络路径不对称快速探测
echo -e "n${YELLOW}>> NTP 路径不对称探测 (需 root)${NC}"
if [ "$EUID" -eq 0 ]; then
  SERVER=$(chronyc sources -n | awk '/^*/ {print $2}' | head -1)
  if [ -n "$SERVER" ]; then
    # 发送 4 个 NTP 请求，计算单向延迟差
    python3 -c "
import socket, struct, time, statistics
server = '$SERVER'
port = 123
diffs = []
for _ in range(4):
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    sock.settimeout(2)
    msg = b'x1b' + b'x00' * 47
    t1 = time.time()
    sock.sendto(msg, (server, port))
    data, _ = sock.recvfrom(1024)
    t4 = time.time()
    sock.close()
    if len(data) >= 48:
        t2 = struct.unpack('!I', data[32:36])[0] + struct.unpack('!I', data[36:40])[0] / 2**32
        t3 = struct.unpack('!I', data[40:44])[0] + struct.unpack('!I', data[44:48])[0] / 2**32
        # NTP 纪元转 Unix 纪元
        t2 -= 2208988800; t3 -= 2208988800
        asymmetry = (t2 - t1) - (t4 - t3)
        diffs.append(asymmetry * 1000)
print(f'   单向延迟不对称 (ms): {[round(d,3) for d in diffs]}')
print(f'   平均不对称: {round(statistics.mean(diffs),3)} ms' if diffs else '   探测失败')
"
  fi
else
  echo "   需 root 权限探测不对称，跳过"
fi

# 5. 关键指标阈值判定
echo -e "n${YELLOW}>> 关键指标判定${NC}"
OFFSET_MS=$(chronyc tracking 2>/dev/null | awk '/System time/ {print $4*1000}' | sed 's/[^0-9.-]//g')
RMS_MS=$(chronyc tracking 2>/dev/null | awk '/RMS offset/ {print $4*1000}' | sed 's/[^0-9.-]//g')
DISP_MS=$(chronyc tracking 2>/dev/null | awk '/Root dispersion/ {print $4*1000}' | sed 's/[^0-9.-]//g')

check_metric() {
  local name=$1 val=$2 warn=$3 crit=$4
  if (( $(echo "$val > $crit" | bc -l) )); then
    echo -e "   ${RED}[CRIT]${NC} $name = ${val} ms (阈值 > ${crit} ms)"
  elif (( $(echo "$val > $warn" | bc -l) )); then
    echo -e "   ${YELLOW}[WARN]${NC} $name = ${val} ms (阈值 > ${warn} ms)"
  else
    echo -e "   ${GREEN}[OK]${NC}   $name = ${val} ms"
  fi
}

check_metric "System Offset" "${OFFSET_MS:-999}" 5 20
check_metric "RMS Offset"    "${RMS_MS:-999}"    1 5
check_metric "Root Dispersion" "${DISP_MS:-999}" 10 50

echo -e "n========== 巡检完成 =========="