From e7fc9ed489891d9eaabcf59c88af2adc54fc8e66 Mon Sep 17 00:00:00 2001 From: laride <198868291+laride@users.noreply.github.com> Date: Fri, 26 Sep 2025 05:09:54 +0800 Subject: [PATCH] ci: support multiple variants per board (#1036) --- .github/workflows/build.yml | 49 +++--- scripts/release.py | 288 ++++++++++++++++++++++++------------ 2 files changed, 224 insertions(+), 113 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f7f365ed..d9659192 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,10 +14,10 @@ permissions: jobs: prepare: - name: Determine boards to build + name: Determine variants to build runs-on: ubuntu-latest outputs: - boards: ${{ steps.select.outputs.boards }} + variants: ${{ steps.select.outputs.variants }} steps: - name: Checkout uses: actions/checkout@v4 @@ -28,30 +28,30 @@ jobs: run: sudo apt-get update && sudo apt-get install -y jq - id: list - name: Get all board list + name: Get all variant list run: | - echo "all_boards=$(python scripts/release.py --list-boards --json)" >> $GITHUB_OUTPUT + echo "all_variants=$(python scripts/release.py --list-boards --json)" >> $GITHUB_OUTPUT - id: select - name: Select boards based on changes + name: Select variants based on changes env: - ALL_BOARDS: ${{ steps.list.outputs.all_boards }} + ALL_VARIANTS: ${{ steps.list.outputs.all_variants }} run: | EVENT_NAME="${{ github.event_name }}" - # For push to main branch, build all boards + # push 到 main 分支，编译全部变体 if [[ "$EVENT_NAME" == "push" ]]; then - echo "boards=$ALL_BOARDS" >> $GITHUB_OUTPUT + echo "variants=$ALL_VARIANTS" >> $GITHUB_OUTPUT exit 0 fi - # For pull_request + # pull_request 场景 BASE_SHA="${{ github.event.pull_request.base.sha }}" HEAD_SHA="${{ github.event.pull_request.head.sha }}" echo "Base: $BASE_SHA, Head: $HEAD_SHA" CHANGED=$(git diff --name-only $BASE_SHA $HEAD_SHA || true) - echo "Changed files:\n$CHANGED" + echo -e "Changed files:\n$CHANGED" NEED_ALL=0 declare -A AFFECTED @@ -60,6 +60,10 @@ jobs: NEED_ALL=1 fi + if [[ "$file" == main/boards/common/* ]]; then + NEED_ALL=1 + fi + if [[ "$file" == main/boards/* ]]; then board=$(echo "$file" | cut -d '/' -f3) AFFECTED[$board]=1 @@ -67,24 +71,25 @@ jobs: done <<< "$CHANGED" if [[ "$NEED_ALL" -eq 1 ]]; then - echo "boards=$ALL_BOARDS" >> $GITHUB_OUTPUT + echo "variants=$ALL_VARIANTS" >> $GITHUB_OUTPUT else if [[ ${#AFFECTED[@]} -eq 0 ]]; then - echo "boards=[]" >> $GITHUB_OUTPUT + echo "variants=[]" >> $GITHUB_OUTPUT else - JSON=$(printf '%s\n' "${!AFFECTED[@]}" | sort -u | jq -R -s -c 'split("\n")[:-1]') - echo "boards=$JSON" >> $GITHUB_OUTPUT + BOARDS_JSON=$(printf '%s\n' "${!AFFECTED[@]}" | sort -u | jq -R -s -c 'split("\n")[:-1]') + FILTERED=$(echo "$ALL_VARIANTS" | jq -c --argjson boards "$BOARDS_JSON" 'map(select(.board as $b | $boards | index($b)))') + echo "variants=$FILTERED" >> $GITHUB_OUTPUT fi fi build: - name: Build ${{ matrix.board }} + name: Build ${{ matrix.name }} needs: prepare - if: ${{ needs.prepare.outputs.boards != '[]' }} + if: ${{ needs.prepare.outputs.variants != '[]' }} strategy: - fail-fast: false # 单个 board 失败不影响其它 board + fail-fast: false # 单个变体失败不影响其它变体 matrix: - board: ${{ fromJson(needs.prepare.outputs.boards) }} + include: ${{ fromJson(needs.prepare.outputs.variants) }} runs-on: ubuntu-latest container: image: espressif/idf:release-v5.4 @@ -92,15 +97,15 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Build current board + - name: Build current variant shell: bash run: | source $IDF_PATH/export.sh - python scripts/release.py ${{ matrix.board }} + python scripts/release.py ${{ matrix.board }} --name ${{ matrix.name }} - name: Upload artifacts uses: actions/upload-artifact@v4 with: - name: xiaozhi_${{ matrix.board }}_${{ github.sha }}.bin + name: xiaozhi_${{ matrix.name }}_${{ github.sha }}.bin path: build/merged-binary.bin - if-no-files-found: error \ No newline at end of file + if-no-files-found: error diff --git a/scripts/release.py b/scripts/release.py index 28d32c3f..e976baad 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -3,107 +3,184 @@ import os import json import zipfile import argparse +from pathlib import Path +from typing import Optional -# 切换到项目根目录 -os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# Switch to project root directory +os.chdir(Path(__file__).resolve().parent.parent) -def get_board_type(): - with open("build/compile_commands.json") as f: +################################################################################ +# Common utility functions +################################################################################ + +def get_board_type_from_compile_commands() -> Optional[str]: + """Parse the current compiled BOARD_TYPE from build/compile_commands.json""" + compile_file = Path("build/compile_commands.json") + if not compile_file.exists(): + return None + with compile_file.open() as f: data = json.load(f) - for item in data: - if not item["file"].endswith("main.cc"): - continue - command = item["command"] - # extract -DBOARD_TYPE=xxx - board_type = command.split("-DBOARD_TYPE=\\\"")[1].split("\\\"")[0].strip() - return board_type + for item in data: + if not item["file"].endswith("main.cc"): + continue + cmd = item["command"] + if "-DBOARD_TYPE=\\\"" in cmd: + return cmd.split("-DBOARD_TYPE=\\\"")[1].split("\\\"")[0].strip() return None -def get_project_version(): - with open("CMakeLists.txt") as f: + +def get_project_version() -> Optional[str]: + """Read set(PROJECT_VER "x.y.z") from root CMakeLists.txt""" + with Path("CMakeLists.txt").open() as f: for line in f: if line.startswith("set(PROJECT_VER"): - return line.split("\"")[1].split("\"")[0].strip() + return line.split("\"")[1] return None -def merge_bin(): + +def merge_bin() -> None: if os.system("idf.py merge-bin") != 0: - print("merge bin failed") + print("merge-bin failed", file=sys.stderr) sys.exit(1) -def zip_bin(board_type, project_version): - if not os.path.exists("releases"): - os.makedirs("releases") - output_path = f"releases/v{project_version}_{board_type}.zip" - if os.path.exists(output_path): - os.remove(output_path) - with zipfile.ZipFile(output_path, 'w', compression=zipfile.ZIP_DEFLATED) as zipf: + +def zip_bin(name: str, version: str) -> None: + """Zip build/merged-binary.bin to releases/v{version}_{name}.zip""" + out_dir = Path("releases") + out_dir.mkdir(exist_ok=True) + output_path = out_dir / f"v{version}_{name}.zip" + + if output_path.exists(): + output_path.unlink() + + with zipfile.ZipFile(output_path, "w", compression=zipfile.ZIP_DEFLATED) as zipf: zipf.write("build/merged-binary.bin", arcname="merged-binary.bin") print(f"zip bin to {output_path} done") - -def release_current(): - merge_bin() - board_type = get_board_type() - print("board type:", board_type) - project_version = get_project_version() - print("project version:", project_version) - zip_bin(board_type, project_version) +################################################################################ +# board / variant related functions +################################################################################ -def get_all_board_types(): - board_configs = {} - with open("main/CMakeLists.txt", encoding='utf-8') as f: - lines = f.readlines() - for i, line in enumerate(lines): - # 查找 if(CONFIG_BOARD_TYPE_*) 行 - if "if(CONFIG_BOARD_TYPE_" in line: - config_name = line.strip().split("if(")[1].split(")")[0] - # 查找下一行的 set(BOARD_TYPE "xxx") - next_line = lines[i + 1].strip() +_BOARDS_DIR = Path("main/boards") + + +def _collect_variants(config_filename: str = "config.json") -> list[dict[str, str]]: + """Traverse all boards under main/boards, collect variant information. + + Return example: + [{"board": "bread-compact-ml307", "name": "bread-compact-ml307"}, ...] + """ + variants: list[dict[str, str]] = [] + for board_path in _BOARDS_DIR.iterdir(): + if not board_path.is_dir(): + continue + if board_path.name == "common": + continue + cfg_path = board_path / config_filename + if not cfg_path.exists(): + print(f"[WARN] {cfg_path} does not exist, skip", file=sys.stderr) + continue + try: + with cfg_path.open() as f: + cfg = json.load(f) + for build in cfg.get("builds", []): + variants.append({"board": board_path.name, "name": build["name"]}) + except Exception as e: + print(f"[ERROR] 解析 {cfg_path} 失败: {e}", file=sys.stderr) + return variants + + +def _parse_board_config_map() -> dict[str, str]: + """Build the mapping of CONFIG_BOARD_TYPE_xxx and board_type from main/CMakeLists.txt""" + cmake_file = Path("main/CMakeLists.txt") + mapping: dict[str, str] = {} + lines = cmake_file.read_text(encoding="utf-8").splitlines() + for idx, line in enumerate(lines): + if "if(CONFIG_BOARD_TYPE_" in line: + config_name = line.strip().split("if(")[1].split(")")[0] + if idx + 1 < len(lines): + next_line = lines[idx + 1].strip() if next_line.startswith("set(BOARD_TYPE"): board_type = next_line.split('"')[1] - board_configs[config_name] = board_type - return board_configs + mapping[config_name] = board_type + return mapping -def release(board_type, board_config, config_filename="config.json"): - config_path = f"main/boards/{board_type}/{config_filename}" - if not os.path.exists(config_path): - print(f"跳过 {board_type} 因为 {config_filename} 不存在") + +def _find_board_config(board_type: str) -> Optional[str]: + """Find the corresponding CONFIG_BOARD_TYPE_xxx for the given board_type""" + for config, b_type in _parse_board_config_map().items(): + if b_type == board_type: + return config + return None + +################################################################################ +# Check board_type in CMakeLists +################################################################################ + +def _board_type_exists(board_type: str) -> bool: + cmake_file = Path("main/CMakeLists.txt") + pattern = f'set(BOARD_TYPE "{board_type}")' + return pattern in cmake_file.read_text(encoding="utf-8") + +################################################################################ +# Compile implementation +################################################################################ + +def release(board_type: str, config_filename: str = "config.json", *, filter_name: Optional[str] = None) -> None: + """Compile and package all/specified variants of the specified board_type + + Args: + board_type: directory name under main/boards + config_filename: config.json name (default: config.json) + filter_name: if specified, only compile the build["name"] that matches + """ + cfg_path = _BOARDS_DIR / board_type / config_filename + if not cfg_path.exists(): + print(f"[WARN] {cfg_path} 不存在，跳过 {board_type}") return - # Print Project Version project_version = get_project_version() - print(f"Project Version: {project_version}", config_path) + print(f"Project Version: {project_version} ({cfg_path})") + + with cfg_path.open() as f: + cfg = json.load(f) + target = cfg["target"] + + builds = cfg.get("builds", []) + if filter_name: + builds = [b for b in builds if b["name"] == filter_name] + if not builds: + print(f"[ERROR] 未在 {board_type} 的 {config_filename} 中找到变体 {filter_name}", file=sys.stderr) + sys.exit(1) - with open(config_path, "r") as f: - config = json.load(f) - target = config["target"] - builds = config["builds"] - for build in builds: name = build["name"] if not name.startswith(board_type): - raise ValueError(f"name {name} 必须以 {board_type} 开头") - output_path = f"releases/v{project_version}_{name}.zip" - if os.path.exists(output_path): - print(f"跳过 {board_type} 因为 {output_path} 已存在") + raise ValueError(f"build.name {name} 必须以 {board_type} 开头") + + output_path = Path("releases") / f"v{project_version}_{name}.zip" + if output_path.exists(): + print(f"跳过 {name} 因为 {output_path} 已存在") continue - sdkconfig_append = [f"{board_config}=y"] - for append in build.get("sdkconfig_append", []): - sdkconfig_append.append(append) + # Process sdkconfig_append + sdkconfig_append = build.get("sdkconfig_append", []) + + print("-" * 80) print(f"name: {name}") print(f"target: {target}") - for append in sdkconfig_append: - print(f"sdkconfig_append: {append}") - # unset IDF_TARGET + for item in sdkconfig_append: + print(f"sdkconfig_append: {item}") + os.environ.pop("IDF_TARGET", None) + # Call set-target if os.system(f"idf.py set-target {target}") != 0: - print("set-target failed") + print("set-target failed", file=sys.stderr) sys.exit(1) + # Append sdkconfig - with open("sdkconfig", "a") as f: + with Path("sdkconfig").open("a") as f: f.write("\n") f.write("# Append by release.py\n") for append in sdkconfig_append: @@ -112,43 +189,72 @@ def release(board_type, board_config, config_filename="config.json"): if os.system(f"idf.py -DBOARD_NAME={name} build") != 0: print("build failed") sys.exit(1) - # Call merge-bin - if os.system("idf.py merge-bin") != 0: - print("merge-bin failed") - sys.exit(1) - # Zip bin + + # merge-bin + merge_bin() + + # Zip zip_bin(name, project_version) - print("-" * 80) + +################################################################################ +# CLI entry +################################################################################ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("board", nargs="?", default=None, help="板子类型或 all") parser.add_argument("-c", "--config", default="config.json", help="指定 config 文件名，默认 config.json") - parser.add_argument("--list-boards", action="store_true", help="列出所有支持的 board 列表") + parser.add_argument("--list-boards", action="store_true", help="列出所有支持的 board 及变体列表") parser.add_argument("--json", action="store_true", help="配合 --list-boards，JSON 格式输出") + parser.add_argument("--name", help="指定变体名称，仅编译匹配的变体") + args = parser.parse_args() + # List mode if args.list_boards: - board_configs = get_all_board_types() - boards = list(board_configs.values()) + variants = _collect_variants(config_filename=args.config) if args.json: - print(json.dumps(boards)) + print(json.dumps(variants)) else: - for board in boards: - print(board) + for v in variants: + print(f"{v['board']}: {v['name']}") sys.exit(0) - if args.board: - board_configs = get_all_board_types() - found = False - for board_config, board_type in board_configs.items(): - if args.board == 'all' or board_type == args.board: - release(board_type, board_config, config_filename=args.config) - found = True - if not found: - print(f"未找到板子类型: {args.board}") - print("可用的板子类型:") - for board_type in board_configs.values(): - print(f" {board_type}") + # Current directory firmware packaging mode + if args.board is None: + merge_bin() + curr_board_type = get_board_type_from_compile_commands() + if curr_board_type is None: + print("未能从 compile_commands.json 解析 board_type", file=sys.stderr) + sys.exit(1) + project_ver = get_project_version() + zip_bin(curr_board_type, project_ver) + sys.exit(0) + + # Compile mode + board_type_input: str = args.board + name_filter: str | None = args.name + + # Check board_type in CMakeLists + if board_type_input != "all" and not _board_type_exists(board_type_input): + print(f"[ERROR] main/CMakeLists.txt 中未找到 board_type {board_type_input}", file=sys.stderr) + sys.exit(1) + + variants_all = _collect_variants(config_filename=args.config) + + # Filter board_type list + target_board_types: set[str] + if board_type_input == "all": + target_board_types = {v["board"] for v in variants_all} else: - release_current() + target_board_types = {board_type_input} + + for bt in sorted(target_board_types): + if not _board_type_exists(bt): + print(f"[ERROR] main/CMakeLists.txt 中未找到 board_type {bt}", file=sys.stderr) + sys.exit(1) + cfg_path = _BOARDS_DIR / bt / args.config + if bt == board_type_input and not cfg_path.exists(): + print(f"开发板 {bt} 未定义 {args.config} 配置文件，跳过") + sys.exit(0) + release(bt, config_filename=args.config, filter_name=name_filter if bt == board_type_input else None)