Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 89 additions & 61 deletions .github/scripts/run-performance-matrix.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,13 @@ RPC_PORT_START=19090
RPC_PORT_END=19120
BASE_PAYLOAD_BYTES="1024"
NACOS_ADDRESS="127.0.0.1:8848"
REPORT_RUN_TIME_UTC="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"

rm -rf "$MATRIX_DIR"
mkdir -p "$MATRIX_DIR" "$LOG_DIR"

if ! command -v tc >/dev/null 2>&1; then
echo "Linux tc is required for network emulation." >&2
echo "性能矩阵需要 Linux tc 命令。" >&2
exit 1
fi

Expand All @@ -26,8 +27,8 @@ cleanup() {
}
trap cleanup EXIT

# Resolve and compile everything before tc is applied so dependency downloads never pollute network scenarios.
# JaCoCo is intentionally disabled in this performance job; coverage remains in the unit-test job.
# 在应用 tc 前完成依赖解析与编译,避免下载依赖污染网络场景。
# 性能 job 显式关闭 JaCoCo;覆盖率仍由单元测试 job 负责。
mvn -B -ntp test-compile \
-pl rpc-consumer -am \
-DskipTests \
Expand Down Expand Up @@ -57,7 +58,7 @@ run_scenario() {
scenario_index=$((scenario_index + 1))
local rpc_port=$((RPC_PORT_START + scenario_index))
if (( rpc_port > RPC_PORT_END )); then
echo "Scenario count exceeded reserved tc RPC port range ${RPC_PORT_START}-${RPC_PORT_END}." >&2
echo "性能场景数量超过预留 RPC 端口范围 ${RPC_PORT_START}-${RPC_PORT_END}。" >&2
exit 1
fi

Expand All @@ -67,7 +68,7 @@ run_scenario() {
output=$(printf "%s/%02d-%s.md" "$MATRIX_DIR" "$scenario_index" "$scenario")
log="$LOG_DIR/$scenario.log"

echo "::group::Performance scenario: $scenario (RPC port $rpc_port)"
echo "::group::性能场景: $scenario (RPC 端口 $rpc_port)"
set +e
mvn -B -ntp test \
-pl rpc-consumer -am \
Expand Down Expand Up @@ -95,31 +96,47 @@ run_scenario() {

local summary="$SNAPSHOT_DIR/summary.md"
if [[ "$status" -eq 0 && ! -s "$summary" ]]; then
echo "Scenario $scenario completed without producing $summary" >&2
echo "场景 $scenario 已完成,但没有生成 $summary" >&2
status=2
fi

{
echo "### \`$scenario\`"
echo "### 场景:\`$scenario\`"
echo
echo "- Status: $([[ "$status" -eq 0 ]] && echo 'PASS' || echo 'FAIL')"
echo "- Matrix dimension: \`$category\`"
echo "- Registry / Protocol / Serializer: \`$registry / $protocol / $serializer\`"
echo "- Request payload: \`${payload_bytes} bytes\`"
echo "- RPC port: \`$rpc_port\`"
echo "- Network profile: \`$network\`"
echo "- Strict success requirement: \`$require_all_success\`"
echo "- 状态:$([[ "$status" -eq 0 ]] && echo '通过' || echo '失败')"
echo "- 矩阵维度:\`$category\`"
echo "- 注册中心 / 协议 / 序列化器:\`$registry / $protocol / $serializer\`"
echo "- 请求载荷:\`${payload_bytes} bytes\`"
echo "- RPC 端口:\`$rpc_port\`"
echo "- 网络环境:\`$network\`"
echo "- 是否要求全部请求成功:\`$require_all_success\`"
echo

if [[ "$status" -eq 0 ]]; then
sed \
-e '1{/^# RPC CI Performance Snapshot$/d;}' \
-e '/^> Observational snapshot only\./d' \
-e '/^Raw attempt samples are available/d' \
-e 's/^## Framework metrics$/#### Framework metrics/' \
-e 's/^- Commit:/- 提交:/' \
-e 's/^- Protocol:/- 协议:/' \
-e 's/^- Serializer:/- 序列化器:/' \
-e 's/^- Registry:/- 注册中心:/' \
-e 's/^- RPC port:/- RPC 端口:/' \
-e 's/^- Request payload:/- 请求载荷:/' \
-e 's/^- Require all requests to succeed:/- 是否要求全部请求成功:/' \
-e 's/^- Java:/- Java:/' \
-e 's/^- CPU visible to JVM:/- JVM 可见 CPU:/' \
-e 's#^| Phase | Attempts | Success | Failed | Success % | Concurrency | Attempt req/s | Success req/s | Avg ms | P50 ms | P95 ms | P99 ms | Max ms |#| 阶段 | 尝试数 | 成功 | 失败 | 成功率 % | 并发度 | 尝试吞吐 req/s | 成功吞吐 req/s | 平均 ms | P50 ms | P95 ms | P99 ms | 最大 ms |#' \
-e 's/^| sequential |/| 顺序调用 |/' \
-e 's/^| concurrent |/| 并发调用 |/' \
-e 's/^## Framework metrics$/#### 框架指标/' \
-e 's#^| Phase | Side | Total | Success | Failed | Timeout | Active | Metric avg ms | Metric max ms |#| 阶段 | 侧别 | 总请求 | 成功 | 失败 | 超时 | 活跃 | 指标平均 ms | 指标最大 ms |#' \
-e 's/| CLIENT |/| 客户端 |/' \
-e 's/| SERVER |/| 服务端 |/' \
-e 's/^Latency percentiles include successful RPCs only\. Low sample counts do not establish a reliable tail-latency SLA\.$/延迟分位数仅统计成功 RPC;样本量较小时不能据此得出可靠的尾延迟 SLA。/' \
-e 's/^Nacos visibility polling and warmup are excluded from timing\. In lossy profiles, server observation intervals may include late requests\.$/Nacos 可见性轮询和预热阶段不计入性能计时;在丢包网络下,服务端观察窗口可能包含晚到请求。/' \
"$summary"
else
echo '> Scenario failed. The tail of its Maven log is included below so unsupported or broken modes remain visible.'
echo '> 该场景执行失败。下面保留 Maven 日志尾部,便于继续诊断不支持或异常的模式。'
echo
echo '```text'
tail -n 60 "$log"
Expand All @@ -129,9 +146,9 @@ run_scenario() {

if [[ "$status" -ne 0 ]]; then
scenario_failures=$((scenario_failures + 1))
echo "Scenario $scenario FAILED; continuing so the rest of the matrix still runs." >&2
echo "场景 $scenario 失败;继续执行剩余矩阵。" >&2
else
echo "Scenario $scenario passed."
echo "场景 $scenario 通过。"
fi
}

Expand All @@ -150,14 +167,14 @@ start_nacos() {

for _ in $(seq 1 60); do
if curl -fsS "http://127.0.0.1:8848/nacos/v1/console/health/readiness" >/dev/null; then
echo "Nacos is ready."
echo "Nacos 已就绪。"
return
fi
sleep 2
done

docker logs rpc-perf-nacos || true
echo "Nacos did not become ready in time." >&2
echo "Nacos 未能在规定时间内就绪。" >&2
exit 1
}

Expand All @@ -168,9 +185,8 @@ reset_network() {
shape_rpc_ports() {
reset_network

# Every scenario gets a unique RPC endpoint in a reserved local port range. This prevents a previous Nacos
# ephemeral instance shutdown from racing a new JVM that re-registers the exact same ip:port identity.
# Only this reserved RPC data-plane range is shaped; Nacos 8848/9848/9849 stays untouched.
# 每个场景使用独立 RPC 端口,避免上一个 Nacos 临时实例退出时与新 JVM 复用同一 ip:port 产生竞态。
# 仅塑形预留 RPC 数据面端口;Nacos 8848/9848/9849 不受影响。
sudo tc qdisc add dev lo root handle 1: prio bands 3
sudo tc qdisc add dev lo parent 1:1 handle 10: netem "$@"

Expand All @@ -191,84 +207,96 @@ apply_lan_profile() {

start_nacos

# Production-like baseline: gRPC + Nacos + Protobuf + synthetic LAN conditions.
# On loopback, request and response packets are both shaped, so the configured one-way delay roughly doubles
# into added request/response RTT before framework processing time is included.
# 生产型基线:gRPC + Nacos + Protobuf + 合成内网条件。
apply_lan_profile
run_scenario "baseline" "baseline-grpc-protobuf-nacos-lan" \
"nacos" "grpc" "protobuf" "tc LAN: 1ms +/-0.2ms one-way, ~2ms added RTT, 1gbit" \
run_scenario "基线" "baseline-grpc-protobuf-nacos-lan" \
"nacos" "grpc" "protobuf" "tc 内网:单向 1ms ±0.2ms,约增加 2ms RTT,1gbit" \
"$BASE_PAYLOAD_BYTES" 20 40 120 8 8000 10000 true

# Payload matrix: keep the production baseline stack and LAN network profile fixed.
# Payload 矩阵:其余条件固定为生产基线。
for payload in 64 16384 262144; do
run_scenario "payload" "payload-${payload}b" \
"nacos" "grpc" "protobuf" "tc LAN: 1ms +/-0.2ms one-way, ~2ms added RTT, 1gbit" \
run_scenario "请求载荷" "payload-${payload}b" \
"nacos" "grpc" "protobuf" "tc 内网:单向 1ms ±0.2ms,约增加 2ms RTT,1gbit" \
"$payload" 10 20 60 4 10000 12000 true
done

# Serializer matrix: gRPC + Nacos + LAN remain fixed; Protobuf is represented by the baseline.
# 序列化矩阵:gRPC + Nacos + 内网固定;Protobuf 由基线代表。
for serializer in kryo java json; do
run_scenario "serializer" "serializer-${serializer}" \
"nacos" "grpc" "$serializer" "tc LAN: 1ms +/-0.2ms one-way, ~2ms added RTT, 1gbit" \
run_scenario "序列化器" "serializer-${serializer}" \
"nacos" "grpc" "$serializer" "tc 内网:单向 1ms ±0.2ms,约增加 2ms RTT,1gbit" \
"$BASE_PAYLOAD_BYTES" 10 20 60 4 8000 10000 true
done

# Registry matrix: Nacos is represented by the baseline; Local is retained only as a comparison control.
run_scenario "registry" "registry-local-control" \
"local" "grpc" "protobuf" "tc LAN: 1ms +/-0.2ms one-way, ~2ms added RTT, 1gbit" \
# 注册中心矩阵:Nacos 由基线代表;Local 仅作对照。
run_scenario "注册中心" "registry-local-control" \
"local" "grpc" "protobuf" "tc 内网:单向 1ms ±0.2ms,约增加 2ms RTT,1gbit" \
"$BASE_PAYLOAD_BYTES" 10 20 60 4 8000 10000 true

# Network matrix: gRPC + Nacos + Protobuf + 1 KiB payload remain fixed.
# 网络矩阵:固定 gRPC + Nacos + Protobuf + 1 KiB。
shape_rpc_ports delay 3ms 1ms distribution normal rate 500mbit
run_scenario "network" "network-cross-az" \
"nacos" "grpc" "protobuf" "tc cross-AZ/private: 3ms +/-1ms one-way, ~6ms added RTT, 500mbit" \
run_scenario "网络" "network-cross-az" \
"nacos" "grpc" "protobuf" "tc 跨 AZ/私网:单向 3ms ±1ms,约增加 6ms RTT,500mbit" \
"$BASE_PAYLOAD_BYTES" 10 20 60 4 10000 12000 true

shape_rpc_ports delay 25ms 5ms distribution normal loss 0.05% rate 100mbit
run_scenario "network" "network-public-internet" \
"nacos" "grpc" "protobuf" "tc public internet: 25ms +/-5ms one-way, ~50ms added RTT, 0.05% loss, 100mbit" \
run_scenario "网络" "network-public-internet" \
"nacos" "grpc" "protobuf" "tc 普通公网:单向 25ms ±5ms,约增加 50ms RTT,丢包 0.05%,100mbit" \
"$BASE_PAYLOAD_BYTES" 10 20 60 4 12000 14000 false

shape_rpc_ports delay 60ms 10ms distribution normal loss 0.1% rate 50mbit
run_scenario "network" "network-cross-region" \
"nacos" "grpc" "protobuf" "tc cross-region: 60ms +/-10ms one-way, ~120ms added RTT, 0.1% loss, 50mbit" \
run_scenario "网络" "network-cross-region" \
"nacos" "grpc" "protobuf" "tc 跨地域公网:单向 60ms ±10ms,约增加 120ms RTT,丢包 0.1%,50mbit" \
"$BASE_PAYLOAD_BYTES" 8 16 48 4 15000 17000 false

shape_rpc_ports delay 120ms 40ms distribution normal loss 1% rate 5mbit
run_scenario "network" "network-weak-mobile" \
"nacos" "grpc" "protobuf" "tc weak/mobile: 120ms +/-40ms one-way, ~240ms added RTT, 1% loss, 5mbit" \
run_scenario "网络" "network-weak-mobile" \
"nacos" "grpc" "protobuf" "tc 弱网/移动网络:单向 120ms ±40ms,约增加 240ms RTT,丢包 1%,5mbit" \
"$BASE_PAYLOAD_BYTES" 6 12 36 3 20000 22000 false

# Protocol matrix runs last so a currently broken protocol cannot hide the baseline/network/serializer data.
# 协议矩阵放在最后,避免单个异常协议遮住其它维度数据。
apply_lan_profile
for protocol in netty http http2; do
run_scenario "protocol" "protocol-${protocol}" \
"nacos" "$protocol" "protobuf" "tc LAN: 1ms +/-0.2ms one-way, ~2ms added RTT, 1gbit" \
run_scenario "协议" "protocol-${protocol}" \
"nacos" "$protocol" "protobuf" "tc 内网:单向 1ms ±0.2ms,约增加 2ms RTT,1gbit" \
"$BASE_PAYLOAD_BYTES" 10 20 60 4 10000 12000 true
done

reset_network

MATRIX_FILE="$MATRIX_DIR/matrix.md"
SHORT_SHA="${GITHUB_SHA:-local}"
SHORT_SHA="${SHORT_SHA:0:7}"
REPORT_DATE_UTC="${REPORT_RUN_TIME_UTC:0:10}"
REPORT_CLOCK_UTC="${REPORT_RUN_TIME_UTC:11:8}"
REPORT_CLOCK_UTC="${REPORT_CLOCK_UTC//:/-}"
REPORT_RELATIVE_PATH="docs/performance/${REPORT_DATE_UTC}/${REPORT_DATE_UTC}_${REPORT_CLOCK_UTC}Z_${SHORT_SHA}.md"

{
echo "## CI Matrix — $(date -u '+%Y-%m-%d %H:%M:%S UTC') — \`${GITHUB_SHA:-local}\`"
echo "# RPC CI 性能矩阵报告"
echo
echo "- Runner: \`${RUNNER_OS:-local} ${RUNNER_ARCH:-unknown}\`"
echo "- Java: \`$(java -version 2>&1 | head -n 1 | tr -d '"')\`"
echo "- CPU visible: \`$(nproc)\`"
echo "- Baseline: \`gRPC + Nacos + Protobuf + 1 KiB + tc LAN\`"
echo "- Strategy: orthogonal matrix around the production-like baseline; Local Registry is a control only"
echo "- Network simulation: Linux \`tc netem\` on \`lo\`, filtered to reserved RPC ports \`$((RPC_PORT_START + 1))-$RPC_PORT_END\` only"
echo "- Nacos isolation: every scenario gets a unique RPC ip:port instance identity"
echo "- Coverage instrumentation: disabled for performance scenarios"
echo "- Scenario failures: \`$scenario_failures\`"
echo "- 运行时间(UTC):\`$REPORT_RUN_TIME_UTC\`"
echo "- 提交:\`${GITHUB_SHA:-local}\`"
echo "- Runner:\`${RUNNER_OS:-local} ${RUNNER_ARCH:-unknown}\`"
echo "- Java:\`$(java -version 2>&1 | head -n 1 | tr -d '"')\`"
echo "- 可用 CPU:\`$(nproc)\`"
echo "- 生产基线:\`gRPC + Nacos + Protobuf + 1 KiB + tc 内网\`"
echo "- 测试策略:围绕生产基线做正交矩阵,一次只改变一个主要变量;Local Registry 仅作对照"
echo "- 网络模拟:Linux \`tc netem\` 作用于 \`lo\`,仅过滤预留 RPC 端口 \`$((RPC_PORT_START + 1))-$RPC_PORT_END\`"
echo "- Nacos 隔离:每个场景使用独立 RPC ip:port 实例身份"
echo "- 覆盖率插桩:性能场景关闭 JaCoCo"
echo "- 失败场景数:\`$scenario_failures\`"
echo
cat "$MATRIX_DIR"/[0-9][0-9]-*.md
} > "$MATRIX_FILE"

cat > "$MATRIX_DIR/report-meta.env" <<EOF
REPORT_RELATIVE_PATH=$REPORT_RELATIVE_PATH
REPORT_RUN_TIME_UTC=$REPORT_RUN_TIME_UTC
EOF

cat "$MATRIX_FILE"

if [[ "$scenario_failures" -ne 0 ]]; then
echo "$scenario_failures performance scenario(s) failed; matrix data was still generated for diagnosis." >&2
echo "$scenario_failures 个性能场景失败;诊断数据已完整生成。" >&2
exit 1
fi
52 changes: 38 additions & 14 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ on:
branches: [ main ]
paths-ignore:
- 'docs/PERFORMANCE.md'
- 'docs/performance/**'
pull_request:
branches: [ main ]

Expand Down Expand Up @@ -97,6 +98,8 @@ jobs:
timeout-minutes: 35
outputs:
performance_markdown: ${{ steps.performance-record.outputs.markdown }}
report_path: ${{ steps.performance-record.outputs.report_path }}
report_time_utc: ${{ steps.performance-record.outputs.report_time_utc }}
steps:
- name: Checkout code
uses: actions/checkout@v7
Expand All @@ -121,10 +124,22 @@ jobs:
shell: bash
run: |
matrix='rpc-consumer/target/performance-matrix/matrix.md'
meta='rpc-consumer/target/performance-matrix/report-meta.env'
if [[ ! -s "$matrix" ]]; then
echo 'Performance matrix file was not produced.' >&2
echo '没有生成性能矩阵报告。' >&2
exit 1
fi
if [[ ! -s "$meta" ]]; then
echo '没有生成性能报告元数据。' >&2
exit 1
fi
source "$meta"
case "$REPORT_RELATIVE_PATH" in
docs/performance/*.md) ;;
*) echo "非法性能报告路径: $REPORT_RELATIVE_PATH" >&2; exit 1 ;;
esac
echo "report_path=$REPORT_RELATIVE_PATH" >> "$GITHUB_OUTPUT"
echo "report_time_utc=$REPORT_RUN_TIME_UTC" >> "$GITHUB_OUTPUT"
delimiter="RPC_PERFORMANCE_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}"
{
echo "markdown<<${delimiter}"
Expand Down Expand Up @@ -165,30 +180,39 @@ jobs:
with:
ref: main
fetch-depth: 0
- name: Append performance matrix
- name: Write standalone performance report
shell: bash
env:
PERFORMANCE_MARKDOWN: ${{ needs.performance-matrix.outputs.performance_markdown }}
REPORT_PATH: ${{ needs.performance-matrix.outputs.report_path }}
REPORT_TIME_UTC: ${{ needs.performance-matrix.outputs.report_time_utc }}
run: |
test -n "$PERFORMANCE_MARKDOWN"
test -n "$REPORT_PATH"
test -n "$REPORT_TIME_UTC"
case "$REPORT_PATH" in
docs/performance/*.md) ;;
*) echo "非法性能报告路径: $REPORT_PATH" >&2; exit 1 ;;
esac
git pull --ff-only origin main
mkdir -p docs
test -f docs/PERFORMANCE.md
{
echo
echo '---'
echo
printf '%s\n' "$PERFORMANCE_MARKDOWN"
} >> docs/PERFORMANCE.md
- name: Commit performance history
mkdir -p "$(dirname "$REPORT_PATH")"
if [[ -e "$REPORT_PATH" ]]; then
echo "性能报告已存在,不允许覆盖: $REPORT_PATH" >&2
exit 1
fi
printf '%s\n' "$PERFORMANCE_MARKDOWN" > "$REPORT_PATH"
echo "已生成独立性能报告: $REPORT_PATH"
- name: Commit performance report
shell: bash
env:
REPORT_PATH: ${{ needs.performance-matrix.outputs.report_path }}
run: |
git config user.name 'github-actions[bot]'
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
git add docs/PERFORMANCE.md
git add "$REPORT_PATH"
if git diff --cached --quiet; then
echo "No performance history change."
echo "没有新的性能报告。"
exit 0
fi
git commit -m 'docs: record RPC performance matrix [skip ci]'
git commit -m 'docs: 记录 RPC 性能测试报告 [skip ci]'
git push origin HEAD:main
Loading