From f09b772d7c7faa4f849db25adc64d30e4077d3b5 Mon Sep 17 00:00:00 2001 From: dallas Date: Wed, 9 Sep 2026 15:15:30 +0800 Subject: [PATCH 1/3] update zone1 control mode --- jenkins/board_flow.py | 282 +++++++++++++----- jenkins/board_stage.sh | 5 - jenkins/ci.yaml | 5 +- jenkins/ci_runner.py | 34 ++- jenkins/prepare.sh | 0 jenkins/run_ci.sh | 226 ++++++++++++++ .../qemu-gicv3/configs/zone1-linux.json | 4 +- 7 files changed, 463 insertions(+), 93 deletions(-) mode change 100644 => 100755 jenkins/prepare.sh create mode 100755 jenkins/run_ci.sh diff --git a/jenkins/board_flow.py b/jenkins/board_flow.py index ea13c154a..cd20a97e0 100644 --- a/jenkins/board_flow.py +++ b/jenkins/board_flow.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Board CI flow helpers (zone0, login, network_and_trans, zone1).""" +"""Board CI flow helpers (zone0, login, network_and_trans, zone1 screen attach).""" from __future__ import annotations @@ -15,8 +15,14 @@ from terminal import Terminal, TerminalCommandError, TerminalTimeoutError -ZONE0_READY_PATTERN = r"root@[^\r\n]*#\s?|(?:\r?\n)#\s*(?:\r?\n|$)" +ZONE0_READY_PATTERN = r"root@[^\r\n]*#\s?|(?:\r?\n)#\s?" +# Match root shell, busybox "#", or getty "login:" on zone1 virtio-console. +ZONE1_READY_PATTERN = ( + r"root@[^\r\n]*[#$]\s?|(?:\r?\n)#\s?|(?:^|\r?\n)\s*#\s*(?:\r?\n|$)|login:" +) ZONE1_INNER_PROMPT_TIMEOUT = 60.0 +RETRY_FIND_PTS = 3 +RETRY_SCREEN = 3 ZONE1_PTS_PATTERN = r"/dev/pts/\d+" GUNZIP_ARTIFACTS = {"hvisor.gz": "hvisor"} SPLIT_PART_RE = re.compile(r"^(.+)\.part\.[a-z]{2}$") @@ -338,9 +344,10 @@ def board_path(work_dir: str, *parts: str) -> str: def ping_success(output: str) -> bool: - if re.search(r"0%\s*(?:packet\s*)?loss\b", output, re.I): + # Require a non-digit before 0% so "100% packet loss" does not match. + if re.search(r"(? 0: @@ -593,7 +600,7 @@ def install_tmp_to_workdir(cfg: dict[str, Any], term: Terminal, names: list[str] board_run(term, f"mkdir -p {work_dir}", timeout=15.0) for name in names: board_run_check(term, f"cp {tmp}/{name} {board_path(work_dir, name)}", timeout=60.0) - for script in ("boot_zone1.sh", "hvisor", "check_serial.sh"): + for script in ("boot_zone1.sh", "hvisor"): board_run(term, f"chmod +x {board_path(work_dir, script)} 2>/dev/null || true", timeout=15.0) print(f"[trans] installed {len(names)} file(s) to {work_dir}", flush=True) @@ -663,6 +670,82 @@ def parse_boot_script_lines(content: str) -> list[str]: return lines +def zone1_ready_pattern(cfg: dict[str, Any]) -> str: + custom = str(cfg.get("zone1_ready_pattern", "")).strip() + return custom or ZONE1_READY_PATTERN + + +def zone1_cmd_list(cfg: dict[str, Any]) -> list[str]: + raw = cfg.get("zone1_cmds") or [] + if not isinstance(raw, list): + cmds: list[str] = [] + else: + cmds = [str(item).strip() for item in raw if str(item).strip()] + if not cmds or cmds[0] != "ls": + cmds = ["ls", *cmds] + return cmds + + +def line_timeout(line: str) -> float: + if "&" in line or "nohup" in line: + return 30.0 + return 120.0 + + +def boot_line_is_background(line: str) -> bool: + return "&" in line or "nohup" in line + + +def wait_zone0_prompt_after_command( + cfg: dict[str, Any], + term: Terminal, + timeout: float, +) -> bool: + """Wait for zone0 prompt; nudge with Enter if logs glued onto '#'.""" + pattern = zone0_ready_pattern(cfg) + deadline = time.monotonic() + timeout + first = True + while time.monotonic() < deadline: + remaining = deadline - time.monotonic() + slice_timeout = min(3.0 if first else 5.0, remaining) + if slice_timeout <= 0: + break + if term.wait_pattern(pattern, timeout=slice_timeout): + return True + first = False + term.send("") + return False + + +def boot_zone1_from_script( + cfg: dict[str, Any], + term: Terminal, + boot_script: str = "boot_zone1.sh", +) -> None: + """Send boot_zone1.sh lines via send_one_by_one; wait prompt/quiet after each.""" + work_dir = str(cfg.get("zone1_work_dir", "/root")).strip() or "/root" + script_path = board_path(work_dir, boot_script) + _rc, content = term.run("cat_boot", f"cat {script_path}", timeout=30.0) + lines = parse_boot_script_lines(content) + if not lines: + raise TerminalCommandError(f"no executable lines in {boot_script}") + print(f"[zone1] running {len(lines)} line(s) from {boot_script} via send_one_by_one", flush=True) + term.send_one_by_one(f"cd {work_dir}") + if not wait_zone0_prompt_after_command(cfg, term, timeout=15.0): + raise TerminalTimeoutError(f"timed out waiting for zone0 prompt after cd {work_dir}") + for index, line in enumerate(lines, start=1): + print(f"[zone1] boot line {index}/{len(lines)}: {line}", flush=True) + term.send_one_by_one(line) + timeout = line_timeout(line) + if boot_line_is_background(line): + term.read_until_quiet(quiet_seconds=1.0, max_duration=timeout) + continue + if not wait_zone0_prompt_after_command(cfg, term, timeout=timeout): + raise TerminalTimeoutError( + f"timed out waiting for zone0 prompt after boot line {index}: {line}" + ) + + def board_zone_list_shows_running( term: Terminal, zone_name: str = "linux2", @@ -675,29 +758,15 @@ def board_zone_list_shows_running( ) -def boot_line_command(work_dir: str, line: str) -> str: - stripped = line.rstrip() - if stripped.startswith("cd "): - inner = stripped - else: - inner = f"cd {work_dir} && {stripped}" - if stripped.endswith("&"): - return f"( {inner} )" - return inner - - -def line_timeout(line: str) -> float: - if "&" in line or "nohup" in line: - return 30.0 - return 120.0 - - -def find_zone1_pts(term: Terminal, timeout: float = 60.0) -> int: +def find_zone1_pts(term: Terminal, timeout: float = 20.0) -> int: """Return the newest virtio-console pts number (poll ls until it appears).""" deadline = time.monotonic() + timeout last_output = "" while time.monotonic() < deadline: - _, pts_output = board_run(term, "ls -1 /dev/pts/[0-9]*", timeout=15.0) + try: + _rc, pts_output = term.run("ls_pts", "ls -1 /dev/pts/[0-9]*", timeout=15.0) + except TerminalTimeoutError as exc: + pts_output = exc.partial_output last_output = pts_output pts_numbers = sorted(int(m) for m in re.findall(r"/dev/pts/(\d+)", pts_output)) if pts_numbers: @@ -708,24 +777,116 @@ def find_zone1_pts(term: Terminal, timeout: float = 60.0) -> int: ) -def run_boot_script_lines(cfg: dict[str, Any], term: Terminal) -> None: - work_dir = cfg["zone1_work_dir"] - boot_script = "boot_zone1.sh" - _, content = board_run(term, f"cat {board_path(work_dir, boot_script)}", timeout=30.0) - lines = parse_boot_script_lines(content) - if not lines: - raise TerminalCommandError(f"no executable lines in {boot_script}") - print(f"[zone1] running {len(lines)} line(s) from {boot_script}", flush=True) - for index, line in enumerate(lines, start=1): - cmd = boot_line_command(work_dir, line) - if len(cmd) > MAX_BOARD_CMD_LEN: - raise TerminalCommandError( - f"boot line {index} too long ({len(cmd)} > {MAX_BOARD_CMD_LEN}): {line!r}" +def retry_find_zone1_pts(cfg: dict[str, Any], term: Terminal) -> int: + found: dict[str, int] = {"pts": -1} + + def do_find() -> None: + found["pts"] = find_zone1_pts(term) + + retry_step( + "find_pts", + do_find, + retries=RETRY_FIND_PTS, + ) + return found["pts"] + + +def close_zone1_screen(cfg: dict[str, Any], term: Terminal) -> None: + """Quit GNU screen entirely so the pts is released. Do not detach.""" + print("[zone1] quit screen (Ctrl-A :quit)", flush=True) + # Prefer colon-command quit; Ctrl-A \\ is easy to lose on lossy serial. + term.backend.write(b"\x01") + time.sleep(0.2) + term.backend.write(b":quit\n") + time.sleep(1.0) + term.backend.write(b"\x01\\") + time.sleep(0.3) + term.backend.write(b"y") + time.sleep(1.0) + term.send("") + term.read_until_quiet(quiet_seconds=1.0, max_duration=8.0) + print("[zone1] kill leftover screen sessions", flush=True) + term.send("pkill -9 screen; screen -wipe") + term.read_until_quiet(quiet_seconds=1.0, max_duration=8.0) + term.send("") + wait_zone0_prompt_after_command(cfg, term, timeout=15.0) + + +def wait_zone1_ready( + cfg: dict[str, Any], + term: Terminal, + timeout: float, + *, + from_offset: int | None = None, +) -> bool: + """Wait for zone1 console ready. + + Prefer passive wait first (phytium-pi often already shows '#'). + Nudge with CR (\\r), not LF — matches serial Enter and avoids screen + mis-handling seen on Ubuntu/xterm zone0 (rk3568). + """ + pattern = zone1_ready_pattern(cfg) + deadline = time.monotonic() + timeout + nudged = False + while time.monotonic() < deadline: + remaining = deadline - time.monotonic() + # First slice: wait longer without keypress (pi path). + slice_timeout = min(8.0 if not nudged else 5.0, remaining) + if slice_timeout <= 0: + break + if term.wait_pattern(pattern, timeout=slice_timeout, from_offset=from_offset): + return True + term.backend.write(b"\r") + nudged = True + return False + + +def attach_zone1_screen(cfg: dict[str, Any], term: Terminal, pts: int) -> None: + session = str(cfg.get("zone1_screen_session", "hvisor-zone1")).strip() or "hvisor-zone1" + # Align with phytium-pi: simple TERM so screen skips xterm app-keypad modes + # that interact badly with automated Enter on rk3568 Ubuntu zone0. + print("[zone1] export TERM=linux before screen", flush=True) + term.send_one_by_one("export TERM=linux") + term.read_until_quiet(quiet_seconds=0.5, max_duration=5.0) + cmd = f"screen -S {session} /dev/pts/{pts}" + print(f"[zone1] {cmd}", flush=True) + offset = term.offset() + term.send(cmd) + # Let screen finish init / zone1 logs settle (pi is already at '#'). + term.read_for(duration=5.0) + print("[zone1] send CR after screen attach", flush=True) + term.backend.write(b"\r") + timeout = float(cfg.get("zone1_shell_timeout", ZONE1_INNER_PROMPT_TIMEOUT)) + if not wait_zone1_ready(cfg, term, timeout=timeout, from_offset=offset): + raise TerminalTimeoutError( + f"timed out waiting for zone1 prompt after {cmd}" + ) + + +def retry_attach_zone1_screen(cfg: dict[str, Any], term: Terminal, pts: int) -> None: + retry_step( + "screen", + lambda: attach_zone1_screen(cfg, term, pts), + retries=RETRY_SCREEN, + on_retry=lambda: close_zone1_screen(cfg, term), + ) + + +def run_zone1_inner_cmds(cfg: dict[str, Any], term: Terminal) -> None: + """Run cmds inside screen-attached zone1 (no zone0 __HV_M_ markers).""" + timeout = float(cfg.get("zone1_shell_timeout", ZONE1_INNER_PROMPT_TIMEOUT)) + for cmd in zone1_cmd_list(cfg): + print(f"[zone1] inner cmd: {cmd}", flush=True) + offset = term.offset() + # Type command with LF-free finish: CR like a real serial Enter. + for char in cmd.rstrip("\n"): + term.backend.write(char.encode("utf-8", errors="replace")) + time.sleep(0.02) + term.backend.write(b"\r") + if not wait_zone1_ready(cfg, term, timeout=timeout, from_offset=offset): + raise TerminalTimeoutError( + f"timed out waiting for zone1 prompt after cmd: {cmd}" ) - try: - board_run_check(term, cmd, timeout=line_timeout(line)) - except TerminalCommandError as exc: - raise TerminalCommandError(f"boot line {index} failed: {line}\n{exc}") from exc def boot_board_zone0_with_retry(cfg: dict[str, Any], term: Terminal) -> None: @@ -777,33 +938,18 @@ def board_zone1_stop(cfg: dict[str, Any], term: Terminal) -> None: def board_zone1_start(cfg: dict[str, Any], term: Terminal) -> int: work_dir = cfg["zone1_work_dir"] - inner_log = "/tmp/zone1_inner.log" - - def run_zone1() -> None: - board_run(term, f"cd {work_dir}", timeout=15.0) - board_run(term, "ls", timeout=15.0) - run_boot_script_lines(cfg, term) - board_zone_list_shows_running( - term, str(cfg.get("zone1_name", "linux2")) - ) - max_pts = find_zone1_pts(term) - board_run(term, f"cd {work_dir}", timeout=15.0) - check_rc, _ = board_run( - term, - f"./check_serial.sh /dev/pts/{max_pts} {inner_log} {int(ZONE1_INNER_PROMPT_TIMEOUT)}", - timeout=ZONE1_INNER_PROMPT_TIMEOUT + 30.0, + board_run(term, f"cd {work_dir}", timeout=15.0) + board_run(term, "ls", timeout=15.0) + boot_zone1_from_script(cfg, term) + board_zone_list_shows_running(term, str(cfg.get("zone1_name", "linux2"))) + max_pts = retry_find_zone1_pts(cfg, term) + print("[zone1] script /dev/null before screen", flush=True) + term.send_one_by_one("script /dev/null") + if not wait_zone0_prompt_after_command(cfg, term, timeout=15.0): + raise TerminalTimeoutError( + "timed out waiting for zone0 prompt after script /dev/null" ) - if check_rc != 0: - raise TerminalCommandError("check_serial.sh failed (no console prompt)") - _, inner_output = board_run(term, f"tail -c 131072 {inner_log}", timeout=30.0) - if inner_output: - save_case_log(cfg, "zone1_inner_serial.log", inner_output) - - retry_step( - "zone1_start", - run_zone1, - retries=int(cfg.get("retry_zone1", 1)), - on_retry=lambda: board_zone1_stop(cfg, term), - ) + retry_attach_zone1_screen(cfg, term, max_pts) + run_zone1_inner_cmds(cfg, term) print("zone1_started successfully", flush=True) return 0 diff --git a/jenkins/board_stage.sh b/jenkins/board_stage.sh index 8880f738f..265f46d4e 100755 --- a/jenkins/board_stage.sh +++ b/jenkins/board_stage.sh @@ -20,7 +20,6 @@ CONFIGS_DIR="${PLATFORM_DIR}/configs" IMAGE_DIR="${PLATFORM_DIR}/image" SCRIPTS_DIR="${PLATFORM_DIR}/scripts" ZONE1_BOOT_SCRIPT="${SCRIPTS_DIR}/boot_zone1.sh" -CHECK_SERIAL_SCRIPT="${WORKSPACE_ROOT}/jenkins/check_serial.sh" if [ -z "${ZONE1_DTB:-}" ]; then for candidate in \ @@ -71,10 +70,6 @@ else echo "warning: zone1 dtb unavailable, skip copying ${ZONE1_DTB}" fi -if [ -f "${CHECK_SERIAL_SCRIPT}" ]; then - cp "${CHECK_SERIAL_SCRIPT}" "${STAGING_DIR}/" -fi - chmod -R a+rX "${STAGING_DIR}" chmod +x "${STAGING_DIR}/boot_zone1.sh" diff --git a/jenkins/ci.yaml b/jenkins/ci.yaml index 9542f239a..a236dbcd3 100644 --- a/jenkins/ci.yaml +++ b/jenkins/ci.yaml @@ -32,7 +32,7 @@ bids: - platform/aarch64/rk3568/image/dts/rk3568_limit_zone0.dtb zone0_image: /home/light/DEMO/sdk/rk356x-up4-2c/kernel/arch/arm64/boot/Image uboot_ready_pattern: '=>' - zone0_ready_pattern: 'root@linux:[^\r\n]*#\s?' + zone0_ready_pattern: 'root@(linux|localhost):[^\r\n]*#\s?' zone0_shell_timeout: 180 uboot_step_timeout: 90 uboot_cmds: @@ -44,7 +44,8 @@ bids: deploy: board_ip: 192.168.1.240 board_iface: eth0 - link_wait: 3 + link_wait: 8 + ping_retries: 3 host_ip: 192.168.1.181 host_user: light zone1_dtb: platform/aarch64/rk3568/image/dts/rk3568_limit_zone1.dtb diff --git a/jenkins/ci_runner.py b/jenkins/ci_runner.py index 877e179de..ba10e5ccd 100755 --- a/jenkins/ci_runner.py +++ b/jenkins/ci_runner.py @@ -17,10 +17,14 @@ board_power_off, board_zone1_start, boot_board_zone0_with_retry, + boot_zone1_from_script, close_board_terminal, get_board_terminal, logs_dir, release_logs_ownership, + retry_attach_zone1_screen, + retry_find_zone1_pts, + run_zone1_inner_cmds, ) from ci_config import get_bid_entry, load_ci, parse_bid from terminal import Terminal, TerminalCommandError, TerminalTimeoutError @@ -212,13 +216,7 @@ def zone1_start(cfg: dict[str, Any], term: Terminal | None) -> int: if term is None: raise SystemExit("terminal backend is required") _, _ = run_and_print_quiet(term, "cd /root", quiet_seconds=1.0, max_duration=15.0) - _, _ = run_and_print_quiet( - term, - "./boot_zone1.sh", - quiet_seconds=15, - max_duration=120.0, - check_exit=False, - ) + boot_zone1_from_script(cfg, term) zone_list_out, _ = run_and_print_quiet( term, "./hvisor zone list", @@ -227,15 +225,11 @@ def zone1_start(cfg: dict[str, Any], term: Terminal | None) -> int: check_exit=False, ) zone_list_shows_running(zone_list_out, str(cfg.get("zone1_name", "linux2"))) + max_pts = retry_find_zone1_pts(cfg, term) if cfg["arch"] != "x86_64": _ = run_and_print_quiet_raw(term, "script /dev/null", quiet_seconds=1.0, max_duration=15.0) - pts_output, _ = run_and_print_quiet(term, "ls -1 /dev/pts/[0-9]*", quiet_seconds=1.0, max_duration=15.0) - pts_numbers = sorted(int(match) for match in re.findall(r"/dev/pts/(\d+)", pts_output)) - if not pts_numbers: - raise TerminalCommandError("failed to find numeric pts from 'ls -1 /dev/pts/[0-9]*'") - max_pts = pts_numbers[-1] - _ = run_and_print_send_only(term, f"screen /dev/pts/{max_pts}", read_duration=20.0) - _ = run_and_print_send_only(term, "\n", read_duration=2.0) + retry_attach_zone1_screen(cfg, term, max_pts) + run_zone1_inner_cmds(cfg, term) print("zone1_started successfully", flush=True) return 0 @@ -278,7 +272,7 @@ def asterinas_zone1_regression(cfg: dict[str, Any], term: Terminal | None) -> in raise TerminalCommandError("failed to find numeric pts from 'ls -1 /dev/pts/[0-9]*'") max_pts = pts_numbers[-1] - _ = run_and_print_send_only(term, f"screen /dev/pts/{max_pts}", read_duration=20.0) + _ = run_and_print_send_only(term, f"screen -S hvisor-zone1 /dev/pts/{max_pts}", read_duration=20.0) _ = read_and_print_until_quiet(term, quiet_seconds=3.0, max_duration=30.0) regression_marker = "__HV_REGRESSION_RC_" @@ -399,11 +393,19 @@ def load_runtime_config(args: argparse.Namespace) -> dict[str, Any]: ), "zone1_work_dir": str(deploy.get("zone1_work_dir", "/root")).strip() or "/root", "zone1_dtb": str(deploy.get("zone1_dtb", "")).strip(), + "zone1_cmds": [ + str(item).strip() + for item in (tests.get("zone1_cmds") or []) + if str(item).strip() + ] + if isinstance(tests.get("zone1_cmds"), list) + else [], + "zone1_ready_pattern": str(tests.get("zone1_ready_pattern", "")).strip(), + "zone1_shell_timeout": float(tests.get("zone1_shell_timeout", 60.0)), "scp_tmp_dir": scp_tmp_dir, "scp_tmp_file": f"{scp_tmp_dir}/f", "retry_zone0": 2, "retry_network": 2, - "retry_zone1": 1, } diff --git a/jenkins/prepare.sh b/jenkins/prepare.sh old mode 100644 new mode 100755 diff --git a/jenkins/run_ci.sh b/jenkins/run_ci.sh new file mode 100755 index 000000000..f31d86087 --- /dev/null +++ b/jenkins/run_ci.sh @@ -0,0 +1,226 @@ +#!/bin/sh +# Run one BID locally: build workspace hvisor-tool (clone if missing), +# prepare qemu rootfs (already under platform/) or deploy board TFTP, +# then start ci_runner. +# Usage: jenkins/run_ci.sh +# jenkins/run_ci.sh --list + +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "${ROOT}" + +if [ "${1:-}" = "--list" ] || [ "${1:-}" = "-l" ]; then + python3 jenkins/ci_config.py list-bids + exit 0 +fi + +BID=${1:?usage: $0 (or $0 --list)} + +eval "$( + python3 - "${BID}" <<'PY' +import shlex +import sys +from pathlib import Path + +sys.path.insert(0, str(Path("jenkins").resolve())) +from ci_config import get_bid_entry, load_ci, parse_bid + +bid = sys.argv[1] +entry = get_bid_entry(load_ci(), bid) +arch, bid_board = parse_bid(bid) +board = (entry.get("platform_board") or "").strip() or bid_board +tests = entry.get("tests") or {} +build_args = entry.get("build_args") or {} +mode = (entry.get("mode") or "").strip() +kdir = str(build_args.get("KDIR") or "").strip() +tftp_dir = str(tests.get("tftp_dir") or "").strip() +zone0_image = str(tests.get("zone0_image") or "").strip() +dtbs = tests.get("zone0_dtbs") or [] +if tests.get("zone0_dtb"): + dtbs = [tests.get("zone0_dtb")] +dtb_list = [str(x).strip() for x in dtbs if str(x).strip()] + +def emit(key, value): + print(f"{key}={shlex.quote(str(value))}") + +emit("MODE", mode) +emit("ARCH", arch) +emit("BOARD", board) +emit("KDIR", kdir) +emit("TFTP_DIR", tftp_dir) +emit("ZONE0_IMAGE", zone0_image) +emit("ZONE0_DTBS", "\n".join(dtb_list)) +emit("SERIAL", str(tests.get("serial") or "").strip()) +PY +)" + +if [ -z "${MODE}" ]; then + echo "error: BID ${BID} has no tests.mode in jenkins/ci.yaml" >&2 + exit 1 +fi +if [ -z "${KDIR}" ]; then + echo "error: BID ${BID} is missing build_args KDIR" >&2 + exit 1 +fi + +HVISOR_TOOL_PATH=${HVISOR_TOOL_PATH:-${ROOT}/hvisor-tool} +HVISOR_TOOL_URL=${HVISOR_TOOL_URL:-https://github.com/syswonder/hvisor-tool} +export TERM="${TERM:-xterm}" +export PYTHONDONTWRITEBYTECODE=1 + +append_path() { + if [ -d "$1" ]; then + PATH="$1:${PATH}" + fi +} + +append_path "${QEMU_PATH:-/home/light/DEMO/qemu-10.1.0/build}" +append_path "${CARGO_HOME:-/usr/local/cargo}/bin" +append_path "${RISCV_TOOLCHAIN_PATH:-/home/light/DEMO/toolchain/riscv64-glibc-ubuntu-24.04-gcc}/bin" +append_path "${AARCH64_TOOLCHAIN_PATH:-/home/light/DEMO/toolchain/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu}/bin" +append_path "${LOONGARCH64_TOOLCHAIN_PATH:-/home/light/DEMO/toolchain/loongarch_cross_tools}/bin" +export PATH + +tool_arch() { + case "$1" in + aarch64|arm64) echo arm64 ;; + riscv64|riscv) echo riscv ;; + loongarch64|loongarch) echo loongarch ;; + x86_64) echo x86_64 ;; + *) echo "$1" ;; + esac +} + +ensure_hvisor_tool() { + if [ -f "${HVISOR_TOOL_PATH}/Makefile" ]; then + echo "hvisor-tool already present: ${HVISOR_TOOL_PATH}" + return + fi + echo "Clone hvisor-tool from ${HVISOR_TOOL_URL} -> ${HVISOR_TOOL_PATH}" + git clone --depth 1 --branch main "${HVISOR_TOOL_URL}" "${HVISOR_TOOL_PATH}" +} + +build_hvisor_tool() { + tarch=$(tool_arch "${ARCH}") + echo "Build hvisor-tool [BID=${BID}, ARCH=${tarch}, KDIR=${KDIR}]" + make -C "${HVISOR_TOOL_PATH}" all ARCH="${tarch}" KDIR="${KDIR}" + test -f "${HVISOR_TOOL_PATH}/output/hvisor" + test -f "${HVISOR_TOOL_PATH}/output/hvisor.ko" +} + +prepare_qemu() { + platform_dir="${ROOT}/platform/${ARCH}/${BOARD}" + virtdisk="${platform_dir}/image/virtdisk" + if [ ! -f "${virtdisk}/rootfs1.ext4" ] && [ ! -f "${virtdisk}/rootfs1.img" ]; then + echo "error: qemu rootfs missing under ${virtdisk} (expected rootfs1.ext4 or rootfs1.img)" >&2 + exit 1 + fi + echo "Prepare qemu rootfs [BID=${BID}, ARCH=${ARCH}, BOARD=${BOARD}]" + sudo -E env \ + ARCH="${ARCH}" \ + BOARD="${BOARD}" \ + KDIR="${KDIR}" \ + WORKSPACE_ROOT="${ROOT}" \ + HVISOR_TOOL_PATH="${HVISOR_TOOL_PATH}" \ + "${ROOT}/jenkins/prepare.sh" +} + +deploy_board_tftp() { + if [ -z "${TFTP_DIR}" ]; then + echo "error: BID ${BID} is missing tests.tftp_dir" >&2 + exit 1 + fi + zone0_image="${ZONE0_IMAGE}" + if [ -z "${zone0_image}" ]; then + zone0_image="${KDIR}/arch/arm64/boot/Image" + fi + echo "Deploy TFTP [BID=${BID}, TFTP_DIR=${TFTP_DIR}]" + tftp_staging="${ROOT}/.tftp-staging" + rm -rf "${tftp_staging}" + make cp ARCH="${ARCH}" BOARD="${BOARD}" MODE=release TFTP_DIR="${tftp_staging}" + test -f "${tftp_staging}/hvisor.bin" + sudo mkdir -p "${TFTP_DIR}" + sudo find "${TFTP_DIR}" -mindepth 1 -maxdepth 1 -type f -delete + sudo cp "${tftp_staging}/hvisor.bin" "${TFTP_DIR}/" + test -f "${TFTP_DIR}/hvisor.bin" + + if [ -n "${ZONE0_DTBS}" ]; then + printf '%s\n' "${ZONE0_DTBS}" | while IFS= read -r dtb; do + [ -n "${dtb}" ] || continue + test -f "${dtb}" + sudo cp "${dtb}" "${TFTP_DIR}/" + done + fi + + test -f "${zone0_image}" || { + echo "error: zone0 kernel Image not found: ${zone0_image}" >&2 + exit 1 + } + sudo cp "${zone0_image}" "${TFTP_DIR}/Image" + sudo chmod -R a+rX "${TFTP_DIR}" + ls -la "${TFTP_DIR}" +} + +free_board_serial() { + echo "Free board serial [BID=${BID}]" + # Kill leftover interactive consoles / prior ci_runner only (not this run_ci.sh). + sudo pkill -f "jenkins/ci_runner.py --bid ${BID}" 2>/dev/null || true + sudo pkill -f "screen ${SERIAL}" 2>/dev/null || true + if [ -n "${SERIAL}" ] && [ -e "${SERIAL}" ]; then + resolved=$(readlink -f "${SERIAL}" 2>/dev/null || true) + if [ -n "${resolved}" ]; then + sudo pkill -f "screen ${resolved}" 2>/dev/null || true + sudo pkill -f "picocom.*${resolved}" 2>/dev/null || true + fi + echo " fuser -k ${SERIAL}" + sudo fuser -k "${SERIAL}" 2>/dev/null || true + if [ -n "${resolved}" ] && [ "${resolved}" != "${SERIAL}" ] && [ -e "${resolved}" ]; then + echo " fuser -k ${resolved}" + sudo fuser -k "${resolved}" 2>/dev/null || true + fi + else + echo " skip: serial empty or missing (${SERIAL:-unset})" + fi + sleep 1 + if [ -n "${SERIAL}" ] && [ -e "${SERIAL}" ]; then + if sudo fuser "${SERIAL}" >/dev/null 2>&1; then + echo "warning: ${SERIAL} still busy after free attempt" >&2 + else + echo " serial free: ${SERIAL}" + fi + fi +} + +run_ci() { + echo "Run ci_runner [BID=${BID}, mode=${MODE}]" + if [ "${MODE}" = "board" ]; then + sudo -E env \ + TERM="${TERM}" \ + HVISOR_TOOL_PATH="${HVISOR_TOOL_PATH}" \ + python3 "${ROOT}/jenkins/ci_runner.py" --bid "${BID}" + else + python3 "${ROOT}/jenkins/ci_runner.py" --bid "${BID}" + fi +} + +case "${MODE}" in + qemu) + ensure_hvisor_tool + build_hvisor_tool + prepare_qemu + run_ci + ;; + board) + free_board_serial + ensure_hvisor_tool + build_hvisor_tool + deploy_board_tftp + free_board_serial + run_ci + ;; + *) + echo "error: unsupported tests.mode='${MODE}' for BID ${BID}" >&2 + exit 1 + ;; +esac diff --git a/platform/aarch64/qemu-gicv3/configs/zone1-linux.json b/platform/aarch64/qemu-gicv3/configs/zone1-linux.json index b73f0bc98..514c16f91 100644 --- a/platform/aarch64/qemu-gicv3/configs/zone1-linux.json +++ b/platform/aarch64/qemu-gicv3/configs/zone1-linux.json @@ -2,8 +2,8 @@ "name": "linux2", "zone_id": 1, "cpus": [ - 3, - 4 + 2, + 3 ], "memory_regions": [ { From 80e53efe2a1fb978304eed90016c31224800d302 Mon Sep 17 00:00:00 2001 From: dallas Date: Tue, 15 Sep 2026 14:43:59 +0800 Subject: [PATCH 2/3] add error and panic check when ci down --- .gitignore | 1 + Jenkinsfile | 12 +++++++ jenkins/board_flow.py | 10 ++++++ jenkins/check_log_severity.py | 68 +++++++++++++++++++++++++++++++++++ jenkins/ci_runner.py | 19 +++++----- jenkins/run_ci.sh | 7 ++++ 6 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 jenkins/check_log_severity.py diff --git a/.gitignore b/.gitignore index 245fbf3f1..b4ec68950 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,4 @@ jenkins-cli.jar tools/kconfig/.venv/ tools/kconfig/__pycache__/ kernel_build/ +.tftp-staging \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 9215383d8..cde61149b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -482,6 +482,18 @@ pipeline { } } } + + stage('Check console log severity') { + when { + expression { return hasCiTests() } + } + steps { + dir(matrixCellDir()) { + echo "Scan console logs for WARN/ERROR/panic [BID=${env.BID}]" + sh "python3 jenkins/check_log_severity.py --bid '${env.BID}'" + } + } + } } post { diff --git a/jenkins/board_flow.py b/jenkins/board_flow.py index cd20a97e0..de1ec76eb 100644 --- a/jenkins/board_flow.py +++ b/jenkins/board_flow.py @@ -50,6 +50,16 @@ def logs_dir(cfg: dict[str, Any]) -> Path: return path +def bid_log_name(bid: str) -> str: + """Map BID arch/board -> logs/__.log.""" + return f"{bid.strip().replace('/', '__')}.log" + + +def bid_log_path(cfg: dict[str, Any]) -> Path: + """Ensure logs/ exists and return the run console log path for this BID.""" + return logs_dir(cfg) / bid_log_name(str(cfg["bid"])) + + def release_logs_ownership(cfg: dict[str, Any]) -> None: """Return logs/ to the invoking user when ci_runner ran under sudo.""" if os.geteuid() != 0: diff --git a/jenkins/check_log_severity.py b/jenkins/check_log_severity.py new file mode 100644 index 000000000..ebcb77a53 --- /dev/null +++ b/jenkins/check_log_severity.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Scan BID console log for WARN (report) / ERROR+panic (fail). + +Usage (from workspace cell root): + python3 jenkins/check_log_severity.py --bid aarch64/rk3568 +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +from board_flow import bid_log_name + +ANSI_RE = re.compile( + r"\x1b\[[0-9;?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*\x07|\x1b[()][\w0-9]?|\r" +) +WARN_RE = re.compile(r"\[\s*WARN\b") +ERROR_RE = re.compile(r"\[\s*(ERROR|ERR|CRIT)\b") +PANIC_RE = re.compile( + r"\b(Kernel panic|panicked at|PANIC|Oops:|BUG:)\b", re.IGNORECASE +) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bid", required=True, help="arch/board BID") + args = parser.parse_args() + + path = Path("logs") / bid_log_name(args.bid) + if not path.is_file(): + print(f"[logcheck] missing {path.as_posix()}, skip") + return 0 + + print(f"[logcheck] file: {path.as_posix()}") + text = ANSI_RE.sub("", path.read_text(encoding="utf-8", errors="replace")) + warns, errors, panics = [], [], [] + for n, line in enumerate(text.splitlines(), 1): + line = line.strip() + if not line: + continue + hit = f"{path.as_posix()}:{n}: {line}" + if PANIC_RE.search(line): + panics.append(hit) + elif ERROR_RE.search(line): + errors.append(hit) + elif WARN_RE.search(line): + warns.append(hit) + + for label, items in (("WARN", warns), ("ERROR", errors), ("PANIC", panics)): + if items: + print(f"[logcheck] {len(items)} {label}") + for item in items[:20]: + print(f" {item}") + if len(items) > 20: + print(f" ... {len(items) - 20} more") + + if errors or panics: + print("[logcheck] FAILED") + return 1 + print("[logcheck] OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/jenkins/ci_runner.py b/jenkins/ci_runner.py index ba10e5ccd..e3cf8189e 100755 --- a/jenkins/ci_runner.py +++ b/jenkins/ci_runner.py @@ -12,6 +12,7 @@ from typing import Any, Callable from board_flow import ( + bid_log_path, board_login, board_network_and_trans, board_power_off, @@ -20,7 +21,6 @@ boot_zone1_from_script, close_board_terminal, get_board_terminal, - logs_dir, release_logs_ownership, retry_attach_zone1_screen, retry_find_zone1_pts, @@ -177,9 +177,7 @@ def zone0_start(cfg: dict[str, Any], term: Terminal | None) -> int: ) return 0 if cfg["mode"] == "board": - log_path = logs_dir(cfg) / "zone0_console.log" - log_path.write_text("", encoding="utf-8") - board_term = build_terminal(cfg, log_path) + board_term = build_terminal(cfg) board_term.open() cfg["_board_term"] = board_term boot_board_zone0_with_retry(cfg, board_term) @@ -410,18 +408,23 @@ def load_runtime_config(args: argparse.Namespace) -> dict[str, Any]: def build_terminal(cfg: dict[str, Any], log_path: Path | None = None) -> Terminal: + path = log_path or cfg["log_path"] if cfg["mode"] == "qemu": - return Terminal.from_qemu_socket(path=cfg["socket_path"], log_path=log_path) + return Terminal.from_qemu_socket(path=cfg["socket_path"], log_path=path) return Terminal.from_serial( port=cfg["serial_port"], baudrate=cfg["baudrate"], - log_path=log_path, + log_path=path, ) def main() -> int: args = parse_args() cfg = load_runtime_config(args) + log_path = bid_log_path(cfg) + log_path.write_text("", encoding="utf-8") + cfg["log_path"] = log_path + print(f"[ci_runner] console log -> {log_path}", flush=True) try: for case_name in cfg["cases"]: case_fn = CASE_HANDLERS.get(case_name) @@ -442,9 +445,7 @@ def main() -> int: term = get_board_terminal(cfg) if cfg["mode"] == "board" else None if term is None and cfg["mode"] == "board": - log_path = logs_dir(cfg) / "board_console.log" - log_path.write_text("", encoding="utf-8") - term = build_terminal(cfg, log_path) + term = build_terminal(cfg) term.open() cfg["_board_term"] = term diff --git a/jenkins/run_ci.sh b/jenkins/run_ci.sh index f31d86087..b2c3ca876 100755 --- a/jenkins/run_ci.sh +++ b/jenkins/run_ci.sh @@ -204,12 +204,18 @@ run_ci() { fi } +check_log_severity() { + echo "Check console log severity [BID=${BID}]" + python3 "${ROOT}/jenkins/check_log_severity.py" --bid "${BID}" +} + case "${MODE}" in qemu) ensure_hvisor_tool build_hvisor_tool prepare_qemu run_ci + check_log_severity ;; board) free_board_serial @@ -218,6 +224,7 @@ case "${MODE}" in deploy_board_tftp free_board_serial run_ci + check_log_severity ;; *) echo "error: unsupported tests.mode='${MODE}' for BID ${BID}" >&2 From f5f3ddff950bca93b34e232cdb0453a235d6d447 Mon Sep 17 00:00:00 2001 From: dallas Date: Thu, 17 Sep 2026 15:14:29 +0800 Subject: [PATCH 3/3] Clean up the code --- jenkins/board_flow.py | 374 ++++++++++++++++-------------------------- jenkins/ci_runner.py | 26 +-- 2 files changed, 140 insertions(+), 260 deletions(-) diff --git a/jenkins/board_flow.py b/jenkins/board_flow.py index de1ec76eb..f6d8f016c 100644 --- a/jenkins/board_flow.py +++ b/jenkins/board_flow.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Board CI flow helpers (zone0, login, network_and_trans, zone1 screen attach).""" +"""Board CI flow helpers (zone0, login, network_and_trans, zone1).""" from __future__ import annotations @@ -21,9 +21,6 @@ r"root@[^\r\n]*[#$]\s?|(?:\r?\n)#\s?|(?:^|\r?\n)\s*#\s*(?:\r?\n|$)|login:" ) ZONE1_INNER_PROMPT_TIMEOUT = 60.0 -RETRY_FIND_PTS = 3 -RETRY_SCREEN = 3 -ZONE1_PTS_PATTERN = r"/dev/pts/\d+" GUNZIP_ARTIFACTS = {"hvisor.gz": "hvisor"} SPLIT_PART_RE = re.compile(r"^(.+)\.part\.[a-z]{2}$") BOARD_PUBKEY_LINE = re.compile(r"^ssh-(?:ed25519|rsa)\s+\S+") @@ -665,7 +662,7 @@ def strip_board_markers(text: str) -> str: return BOARD_CMD_MARKER_RE.sub("", text) -def parse_boot_script_lines(content: str) -> list[str]: +def parse_script_lines(content: str) -> list[str]: content = strip_board_markers(content) lines: list[str] = [] for raw in content.splitlines(): @@ -680,225 +677,6 @@ def parse_boot_script_lines(content: str) -> list[str]: return lines -def zone1_ready_pattern(cfg: dict[str, Any]) -> str: - custom = str(cfg.get("zone1_ready_pattern", "")).strip() - return custom or ZONE1_READY_PATTERN - - -def zone1_cmd_list(cfg: dict[str, Any]) -> list[str]: - raw = cfg.get("zone1_cmds") or [] - if not isinstance(raw, list): - cmds: list[str] = [] - else: - cmds = [str(item).strip() for item in raw if str(item).strip()] - if not cmds or cmds[0] != "ls": - cmds = ["ls", *cmds] - return cmds - - -def line_timeout(line: str) -> float: - if "&" in line or "nohup" in line: - return 30.0 - return 120.0 - - -def boot_line_is_background(line: str) -> bool: - return "&" in line or "nohup" in line - - -def wait_zone0_prompt_after_command( - cfg: dict[str, Any], - term: Terminal, - timeout: float, -) -> bool: - """Wait for zone0 prompt; nudge with Enter if logs glued onto '#'.""" - pattern = zone0_ready_pattern(cfg) - deadline = time.monotonic() + timeout - first = True - while time.monotonic() < deadline: - remaining = deadline - time.monotonic() - slice_timeout = min(3.0 if first else 5.0, remaining) - if slice_timeout <= 0: - break - if term.wait_pattern(pattern, timeout=slice_timeout): - return True - first = False - term.send("") - return False - - -def boot_zone1_from_script( - cfg: dict[str, Any], - term: Terminal, - boot_script: str = "boot_zone1.sh", -) -> None: - """Send boot_zone1.sh lines via send_one_by_one; wait prompt/quiet after each.""" - work_dir = str(cfg.get("zone1_work_dir", "/root")).strip() or "/root" - script_path = board_path(work_dir, boot_script) - _rc, content = term.run("cat_boot", f"cat {script_path}", timeout=30.0) - lines = parse_boot_script_lines(content) - if not lines: - raise TerminalCommandError(f"no executable lines in {boot_script}") - print(f"[zone1] running {len(lines)} line(s) from {boot_script} via send_one_by_one", flush=True) - term.send_one_by_one(f"cd {work_dir}") - if not wait_zone0_prompt_after_command(cfg, term, timeout=15.0): - raise TerminalTimeoutError(f"timed out waiting for zone0 prompt after cd {work_dir}") - for index, line in enumerate(lines, start=1): - print(f"[zone1] boot line {index}/{len(lines)}: {line}", flush=True) - term.send_one_by_one(line) - timeout = line_timeout(line) - if boot_line_is_background(line): - term.read_until_quiet(quiet_seconds=1.0, max_duration=timeout) - continue - if not wait_zone0_prompt_after_command(cfg, term, timeout=timeout): - raise TerminalTimeoutError( - f"timed out waiting for zone0 prompt after boot line {index}: {line}" - ) - - -def board_zone_list_shows_running( - term: Terminal, - zone_name: str = "linux2", -) -> None: - # hvisor-tool returns zone count (non-zero) from zone list; validate output instead. - _, out = board_run(term, "./hvisor zone list", timeout=15.0) - if zone_name not in out or "running" not in out: - raise TerminalCommandError( - f"zone list missing running {zone_name!r}:\n{out.strip()}" - ) - - -def find_zone1_pts(term: Terminal, timeout: float = 20.0) -> int: - """Return the newest virtio-console pts number (poll ls until it appears).""" - deadline = time.monotonic() + timeout - last_output = "" - while time.monotonic() < deadline: - try: - _rc, pts_output = term.run("ls_pts", "ls -1 /dev/pts/[0-9]*", timeout=15.0) - except TerminalTimeoutError as exc: - pts_output = exc.partial_output - last_output = pts_output - pts_numbers = sorted(int(m) for m in re.findall(r"/dev/pts/(\d+)", pts_output)) - if pts_numbers: - return pts_numbers[-1] - time.sleep(1.0) - raise TerminalCommandError( - f"timed out waiting for zone1 pts device (last ls output: {last_output.strip()!r})" - ) - - -def retry_find_zone1_pts(cfg: dict[str, Any], term: Terminal) -> int: - found: dict[str, int] = {"pts": -1} - - def do_find() -> None: - found["pts"] = find_zone1_pts(term) - - retry_step( - "find_pts", - do_find, - retries=RETRY_FIND_PTS, - ) - return found["pts"] - - -def close_zone1_screen(cfg: dict[str, Any], term: Terminal) -> None: - """Quit GNU screen entirely so the pts is released. Do not detach.""" - print("[zone1] quit screen (Ctrl-A :quit)", flush=True) - # Prefer colon-command quit; Ctrl-A \\ is easy to lose on lossy serial. - term.backend.write(b"\x01") - time.sleep(0.2) - term.backend.write(b":quit\n") - time.sleep(1.0) - term.backend.write(b"\x01\\") - time.sleep(0.3) - term.backend.write(b"y") - time.sleep(1.0) - term.send("") - term.read_until_quiet(quiet_seconds=1.0, max_duration=8.0) - print("[zone1] kill leftover screen sessions", flush=True) - term.send("pkill -9 screen; screen -wipe") - term.read_until_quiet(quiet_seconds=1.0, max_duration=8.0) - term.send("") - wait_zone0_prompt_after_command(cfg, term, timeout=15.0) - - -def wait_zone1_ready( - cfg: dict[str, Any], - term: Terminal, - timeout: float, - *, - from_offset: int | None = None, -) -> bool: - """Wait for zone1 console ready. - - Prefer passive wait first (phytium-pi often already shows '#'). - Nudge with CR (\\r), not LF — matches serial Enter and avoids screen - mis-handling seen on Ubuntu/xterm zone0 (rk3568). - """ - pattern = zone1_ready_pattern(cfg) - deadline = time.monotonic() + timeout - nudged = False - while time.monotonic() < deadline: - remaining = deadline - time.monotonic() - # First slice: wait longer without keypress (pi path). - slice_timeout = min(8.0 if not nudged else 5.0, remaining) - if slice_timeout <= 0: - break - if term.wait_pattern(pattern, timeout=slice_timeout, from_offset=from_offset): - return True - term.backend.write(b"\r") - nudged = True - return False - - -def attach_zone1_screen(cfg: dict[str, Any], term: Terminal, pts: int) -> None: - session = str(cfg.get("zone1_screen_session", "hvisor-zone1")).strip() or "hvisor-zone1" - # Align with phytium-pi: simple TERM so screen skips xterm app-keypad modes - # that interact badly with automated Enter on rk3568 Ubuntu zone0. - print("[zone1] export TERM=linux before screen", flush=True) - term.send_one_by_one("export TERM=linux") - term.read_until_quiet(quiet_seconds=0.5, max_duration=5.0) - cmd = f"screen -S {session} /dev/pts/{pts}" - print(f"[zone1] {cmd}", flush=True) - offset = term.offset() - term.send(cmd) - # Let screen finish init / zone1 logs settle (pi is already at '#'). - term.read_for(duration=5.0) - print("[zone1] send CR after screen attach", flush=True) - term.backend.write(b"\r") - timeout = float(cfg.get("zone1_shell_timeout", ZONE1_INNER_PROMPT_TIMEOUT)) - if not wait_zone1_ready(cfg, term, timeout=timeout, from_offset=offset): - raise TerminalTimeoutError( - f"timed out waiting for zone1 prompt after {cmd}" - ) - - -def retry_attach_zone1_screen(cfg: dict[str, Any], term: Terminal, pts: int) -> None: - retry_step( - "screen", - lambda: attach_zone1_screen(cfg, term, pts), - retries=RETRY_SCREEN, - on_retry=lambda: close_zone1_screen(cfg, term), - ) - - -def run_zone1_inner_cmds(cfg: dict[str, Any], term: Terminal) -> None: - """Run cmds inside screen-attached zone1 (no zone0 __HV_M_ markers).""" - timeout = float(cfg.get("zone1_shell_timeout", ZONE1_INNER_PROMPT_TIMEOUT)) - for cmd in zone1_cmd_list(cfg): - print(f"[zone1] inner cmd: {cmd}", flush=True) - offset = term.offset() - # Type command with LF-free finish: CR like a real serial Enter. - for char in cmd.rstrip("\n"): - term.backend.write(char.encode("utf-8", errors="replace")) - time.sleep(0.02) - term.backend.write(b"\r") - if not wait_zone1_ready(cfg, term, timeout=timeout, from_offset=offset): - raise TerminalTimeoutError( - f"timed out waiting for zone1 prompt after cmd: {cmd}" - ) - - def boot_board_zone0_with_retry(cfg: dict[str, Any], term: Terminal) -> None: retry_step( "zone0_boot", @@ -947,19 +725,145 @@ def board_zone1_stop(cfg: dict[str, Any], term: Terminal) -> None: def board_zone1_start(cfg: dict[str, Any], term: Terminal) -> int: - work_dir = cfg["zone1_work_dir"] + """Boot zone1 from script, attach screen to its console, run smoke cmds.""" + work_dir = str(cfg.get("zone1_work_dir", "/root")).strip() or "/root" + zone_name = str(cfg.get("zone1_name", "linux2")) + session = str(cfg.get("zone1_screen_session", "hvisor-zone1")).strip() or "hvisor-zone1" + zone0_pat = zone0_ready_pattern(cfg) + zone1_pat = str(cfg.get("zone1_ready_pattern", "")).strip() or ZONE1_READY_PATTERN + shell_timeout = float(cfg.get("zone1_shell_timeout", ZONE1_INNER_PROMPT_TIMEOUT)) + + def wait_prompt(pattern: str, timeout: float, *, from_offset: int | None = None, cr: bool = False) -> bool: + deadline = time.monotonic() + timeout + nudged = False + while time.monotonic() < deadline: + remaining = deadline - time.monotonic() + first_slice = 8.0 if cr else 3.0 + slice_timeout = min(first_slice if not nudged else 5.0, remaining) + if slice_timeout <= 0: + break + if term.wait_pattern(pattern, timeout=slice_timeout, from_offset=from_offset): + return True + if cr: + term.backend.write(b"\r") + else: + term.send("") + nudged = True + return False + + # --- boot zone1 (send boot_zone1.sh line by line) --- board_run(term, f"cd {work_dir}", timeout=15.0) board_run(term, "ls", timeout=15.0) - boot_zone1_from_script(cfg, term) - board_zone_list_shows_running(term, str(cfg.get("zone1_name", "linux2"))) - max_pts = retry_find_zone1_pts(cfg, term) - print("[zone1] script /dev/null before screen", flush=True) - term.send_one_by_one("script /dev/null") - if not wait_zone0_prompt_after_command(cfg, term, timeout=15.0): - raise TerminalTimeoutError( - "timed out waiting for zone0 prompt after script /dev/null" + script_path = board_path(work_dir, "boot_zone1.sh") + _rc, content = term.run("cat_boot", f"cat {script_path}", timeout=30.0) + boot_lines = parse_script_lines(content) + if not boot_lines: + raise TerminalCommandError("no executable lines in boot_zone1.sh") + print(f"[zone1] sending {len(boot_lines)} boot line(s)", flush=True) + for index, line in enumerate(boot_lines, start=1): + print(f"[zone1] boot {index}/{len(boot_lines)}: {line}", flush=True) + term.send_one_by_one(line) + bg = "&" in line or "nohup" in line + timeout = 30.0 if bg else 120.0 + if bg: + term.read_until_quiet(quiet_seconds=1.0, max_duration=timeout) + continue + if not wait_prompt(zone0_pat, timeout): + raise TerminalTimeoutError(f"timed out after boot line {index}: {line}") + + # --- confirm zone is running --- + _, zone_list_out = board_run(term, "./hvisor zone list", timeout=15.0) + if zone_name not in zone_list_out or "running" not in zone_list_out: + raise TerminalCommandError( + f"zone list missing running {zone_name!r}:\n{zone_list_out.strip()}" + ) + + # --- find virtio-console pts --- + pts = -1 + last_pts_out = "" + for attempt in range(3): + if attempt > 0: + print(f"[zone1] retry find_pts ({attempt}/2)", flush=True) + time.sleep(3.0) + deadline = time.monotonic() + 20.0 + while time.monotonic() < deadline: + try: + _rc, last_pts_out = term.run("ls_pts", "ls -1 /dev/pts/[0-9]*", timeout=15.0) + except TerminalTimeoutError as exc: + last_pts_out = exc.partial_output + pts_numbers = sorted(int(m) for m in re.findall(r"/dev/pts/(\d+)", last_pts_out)) + if pts_numbers: + pts = pts_numbers[-1] + break + time.sleep(1.0) + if pts >= 0: + break + if pts < 0: + raise TerminalCommandError( + f"timed out waiting for zone1 pts (last ls: {last_pts_out.strip()!r})" ) - retry_attach_zone1_screen(cfg, term, max_pts) - run_zone1_inner_cmds(cfg, term) + print(f"[zone1] pts={pts}", flush=True) + + # --- attach screen (retry with quit+pkill cleanup) --- + if cfg.get("arch") != "x86_64": + print("[zone1] script /dev/null before screen", flush=True) + term.send_one_by_one("script /dev/null") + if not wait_prompt(zone0_pat, 15.0): + raise TerminalTimeoutError("timed out after script /dev/null") + + screen_ok = False + last_screen_exc: Exception | None = None + for attempt in range(3): + if attempt > 0: + print(f"[zone1] retry screen ({attempt}/2)", flush=True) + print("[zone1] quit leftover screen", flush=True) + term.backend.write(b"\x01") + time.sleep(0.2) + term.backend.write(b":quit\n") + time.sleep(1.0) + term.backend.write(b"\x01\\") + time.sleep(0.3) + term.backend.write(b"y") + time.sleep(1.0) + term.send("pkill -9 screen; screen -wipe") + term.read_until_quiet(quiet_seconds=1.0, max_duration=8.0) + term.send("") + wait_prompt(zone0_pat, 15.0) + time.sleep(3.0) + try: + # TERM=linux avoids xterm keypad modes that break automated Enter on rk3568. + term.send_one_by_one("export TERM=linux") + term.read_until_quiet(quiet_seconds=0.5, max_duration=5.0) + cmd = f"screen -S {session} /dev/pts/{pts}" + print(f"[zone1] {cmd}", flush=True) + offset = term.offset() + term.send(cmd) + term.read_for(duration=5.0) + term.backend.write(b"\r") + if not wait_prompt(zone1_pat, shell_timeout, from_offset=offset, cr=True): + raise TerminalTimeoutError(f"timed out waiting for zone1 prompt after {cmd}") + screen_ok = True + break + except (TerminalTimeoutError, TerminalCommandError) as exc: + last_screen_exc = exc + if not screen_ok: + assert last_screen_exc is not None + raise last_screen_exc + + # --- smoke cmds inside zone1 --- + raw_cmds = cfg.get("zone1_cmds") or [] + cmds = [str(item).strip() for item in raw_cmds if str(item).strip()] if isinstance(raw_cmds, list) else [] + if not cmds or cmds[0] != "ls": + cmds = ["ls", *cmds] + for cmd in cmds: + print(f"[zone1] inner cmd: {cmd}", flush=True) + offset = term.offset() + for char in cmd.rstrip("\n"): + term.backend.write(char.encode("utf-8", errors="replace")) + time.sleep(0.02) + term.backend.write(b"\r") + if not wait_prompt(zone1_pat, shell_timeout, from_offset=offset, cr=True): + raise TerminalTimeoutError(f"timed out waiting for zone1 prompt after cmd: {cmd}") + print("zone1_started successfully", flush=True) return 0 diff --git a/jenkins/ci_runner.py b/jenkins/ci_runner.py index e3cf8189e..acf390c0d 100755 --- a/jenkins/ci_runner.py +++ b/jenkins/ci_runner.py @@ -18,13 +18,9 @@ board_power_off, board_zone1_start, boot_board_zone0_with_retry, - boot_zone1_from_script, close_board_terminal, get_board_terminal, release_logs_ownership, - retry_attach_zone1_screen, - retry_find_zone1_pts, - run_zone1_inner_cmds, ) from ci_config import get_bid_entry, load_ci, parse_bid from terminal import Terminal, TerminalCommandError, TerminalTimeoutError @@ -207,29 +203,9 @@ def network_and_trans(cfg: dict[str, Any], term: Terminal | None) -> int: def zone1_start(cfg: dict[str, Any], term: Terminal | None) -> int: print("————————————————\ncase: zone1_start\n————————————————\n", flush=True) - if cfg["mode"] == "board": - if term is None: - raise SystemExit("terminal backend is required") - return board_zone1_start(cfg, term) if term is None: raise SystemExit("terminal backend is required") - _, _ = run_and_print_quiet(term, "cd /root", quiet_seconds=1.0, max_duration=15.0) - boot_zone1_from_script(cfg, term) - zone_list_out, _ = run_and_print_quiet( - term, - "./hvisor zone list", - quiet_seconds=1.0, - max_duration=15.0, - check_exit=False, - ) - zone_list_shows_running(zone_list_out, str(cfg.get("zone1_name", "linux2"))) - max_pts = retry_find_zone1_pts(cfg, term) - if cfg["arch"] != "x86_64": - _ = run_and_print_quiet_raw(term, "script /dev/null", quiet_seconds=1.0, max_duration=15.0) - retry_attach_zone1_screen(cfg, term, max_pts) - run_zone1_inner_cmds(cfg, term) - print("zone1_started successfully", flush=True) - return 0 + return board_zone1_start(cfg, term) def asterinas_zone1_regression(cfg: dict[str, Any], term: Terminal | None) -> int: