ci: support multiple variants per board (#1036)

.github/workflows/build.yml

@@ -14,10 +14,10 @@ permissions:
 
 jobs:
   prepare:
-    name: Determine boards to build
+    name: Determine variants to build
     runs-on: ubuntu-latest
     outputs:
-      boards: ${{ steps.select.outputs.boards }}
+      variants: ${{ steps.select.outputs.variants }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -28,30 +28,30 @@ jobs:
         run: sudo apt-get update && sudo apt-get install -y jq
 
       - id: list
-        name: Get all board list
+        name: Get all variant list
         run: |
-          echo "all_boards=$(python scripts/release.py --list-boards --json)" >> $GITHUB_OUTPUT
+          echo "all_variants=$(python scripts/release.py --list-boards --json)" >> $GITHUB_OUTPUT
 
       - id: select
-        name: Select boards based on changes
+        name: Select variants based on changes
         env:
-          ALL_BOARDS: ${{ steps.list.outputs.all_boards }}
+          ALL_VARIANTS: ${{ steps.list.outputs.all_variants }}
         run: |
           EVENT_NAME="${{ github.event_name }}"
 
-          # For push to main branch, build all boards
+          # push 到 main 分支，编译全部变体
           if [[ "$EVENT_NAME" == "push" ]]; then
-            echo "boards=$ALL_BOARDS" >> $GITHUB_OUTPUT
+            echo "variants=$ALL_VARIANTS" >> $GITHUB_OUTPUT
             exit 0
           fi
 
-          # For pull_request
+          # pull_request 场景
           BASE_SHA="${{ github.event.pull_request.base.sha }}"
           HEAD_SHA="${{ github.event.pull_request.head.sha }}"
           echo "Base: $BASE_SHA, Head: $HEAD_SHA"
 
           CHANGED=$(git diff --name-only $BASE_SHA $HEAD_SHA || true)
-          echo "Changed files:\n$CHANGED"
+          echo -e "Changed files:\n$CHANGED"
 
           NEED_ALL=0
           declare -A AFFECTED
@@ -60,6 +60,10 @@ jobs:
               NEED_ALL=1
             fi
 
+            if [[ "$file" == main/boards/common/* ]]; then
+              NEED_ALL=1
+            fi
+
             if [[ "$file" == main/boards/* ]]; then
               board=$(echo "$file" | cut -d '/' -f3)
               AFFECTED[$board]=1
@@ -67,24 +71,25 @@ jobs:
           done <<< "$CHANGED"
 
           if [[ "$NEED_ALL" -eq 1 ]]; then
-            echo "boards=$ALL_BOARDS" >> $GITHUB_OUTPUT
+            echo "variants=$ALL_VARIANTS" >> $GITHUB_OUTPUT
           else
             if [[ ${#AFFECTED[@]} -eq 0 ]]; then
-              echo "boards=[]" >> $GITHUB_OUTPUT
+              echo "variants=[]" >> $GITHUB_OUTPUT
             else
-              JSON=$(printf '%s\n' "${!AFFECTED[@]}" | sort -u | jq -R -s -c 'split("\n")[:-1]')
+              BOARDS_JSON=$(printf '%s\n' "${!AFFECTED[@]}" | sort -u | jq -R -s -c 'split("\n")[:-1]')
-              echo "boards=$JSON" >> $GITHUB_OUTPUT
+              FILTERED=$(echo "$ALL_VARIANTS" | jq -c --argjson boards "$BOARDS_JSON" 'map(select(.board as $b | $boards | index($b)))')
+              echo "variants=$FILTERED" >> $GITHUB_OUTPUT
             fi
           fi
 
   build:
-    name: Build ${{ matrix.board }}
+    name: Build ${{ matrix.name }}
     needs: prepare
-    if: ${{ needs.prepare.outputs.boards != '[]' }}
+    if: ${{ needs.prepare.outputs.variants != '[]' }}
     strategy:
-      fail-fast: false  # 单个 board 失败不影响其它 board
+      fail-fast: false  # 单个变体失败不影响其它变体
       matrix:
-        board: ${{ fromJson(needs.prepare.outputs.boards) }}
+        include: ${{ fromJson(needs.prepare.outputs.variants) }}
     runs-on: ubuntu-latest
     container:
       image: espressif/idf:release-v5.4
@@ -92,15 +97,15 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Build current board
+      - name: Build current variant
         shell: bash
         run: |
           source $IDF_PATH/export.sh
-          python scripts/release.py ${{ matrix.board }}
+          python scripts/release.py ${{ matrix.board }} --name ${{ matrix.name }}
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: xiaozhi_${{ matrix.board }}_${{ github.sha }}.bin
+          name: xiaozhi_${{ matrix.name }}_${{ github.sha }}.bin
           path: build/merged-binary.bin
           if-no-files-found: error

scripts/release.py

@@ -3,107 +3,184 @@ import os
 import json
 import zipfile
 import argparse
+from pathlib import Path
+from typing import Optional
 
-# 切换到项目根目录
+# Switch to project root directory
-os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+os.chdir(Path(__file__).resolve().parent.parent)
 
-def get_board_type():
+################################################################################
-    with open("build/compile_commands.json") as f:
+# Common utility functions
+################################################################################
+
+def get_board_type_from_compile_commands() -> Optional[str]:
+    """Parse the current compiled BOARD_TYPE from build/compile_commands.json"""
+    compile_file = Path("build/compile_commands.json")
+    if not compile_file.exists():
+        return None
+    with compile_file.open() as f:
         data = json.load(f)
     for item in data:
         if not item["file"].endswith("main.cc"):
             continue
-            command = item["command"]
+        cmd = item["command"]
-            # extract -DBOARD_TYPE=xxx
+        if "-DBOARD_TYPE=\\\"" in cmd:
-            board_type = command.split("-DBOARD_TYPE=\\\"")[1].split("\\\"")[0].strip()
+            return cmd.split("-DBOARD_TYPE=\\\"")[1].split("\\\"")[0].strip()
-            return board_type
     return None
 
-def get_project_version():
+
-    with open("CMakeLists.txt") as f:
+def get_project_version() -> Optional[str]:
+    """Read set(PROJECT_VER "x.y.z") from root CMakeLists.txt"""
+    with Path("CMakeLists.txt").open() as f:
         for line in f:
             if line.startswith("set(PROJECT_VER"):
-                return line.split("\"")[1].split("\"")[0].strip()
+                return line.split("\"")[1]
     return None
 
-def merge_bin():
+
+def merge_bin() -> None:
     if os.system("idf.py merge-bin") != 0:
-        print("merge bin failed")
+        print("merge-bin failed", file=sys.stderr)
         sys.exit(1)
 
-def zip_bin(board_type, project_version):
+
-    if not os.path.exists("releases"):
+def zip_bin(name: str, version: str) -> None:
-        os.makedirs("releases")
+    """Zip build/merged-binary.bin to releases/v{version}_{name}.zip"""
-    output_path = f"releases/v{project_version}_{board_type}.zip"
+    out_dir = Path("releases")
-    if os.path.exists(output_path):
+    out_dir.mkdir(exist_ok=True)
-        os.remove(output_path)
+    output_path = out_dir / f"v{version}_{name}.zip"
-    with zipfile.ZipFile(output_path, 'w', compression=zipfile.ZIP_DEFLATED) as zipf:
+
+    if output_path.exists():
+        output_path.unlink()
+
+    with zipfile.ZipFile(output_path, "w", compression=zipfile.ZIP_DEFLATED) as zipf:
         zipf.write("build/merged-binary.bin", arcname="merged-binary.bin")
     print(f"zip bin to {output_path} done")
 
+################################################################################
+# board / variant related functions
+################################################################################
 
-def release_current():
+_BOARDS_DIR = Path("main/boards")
-    merge_bin()
-    board_type = get_board_type()
-    print("board type:", board_type)
-    project_version = get_project_version()
-    print("project version:", project_version)
-    zip_bin(board_type, project_version)
 
-def get_all_board_types():
+
-    board_configs = {}
+def _collect_variants(config_filename: str = "config.json") -> list[dict[str, str]]:
-    with open("main/CMakeLists.txt", encoding='utf-8') as f:
+    """Traverse all boards under main/boards, collect variant information.
-        lines = f.readlines()
+
-        for i, line in enumerate(lines):
+    Return example:
-            # 查找 if(CONFIG_BOARD_TYPE_*) 行
+        [{"board": "bread-compact-ml307", "name": "bread-compact-ml307"}, ...]
+    """
+    variants: list[dict[str, str]] = []
+    for board_path in _BOARDS_DIR.iterdir():
+        if not board_path.is_dir():
+            continue
+        if board_path.name == "common":
+            continue
+        cfg_path = board_path / config_filename
+        if not cfg_path.exists():
+            print(f"[WARN] {cfg_path} does not exist, skip", file=sys.stderr)
+            continue
+        try:
+            with cfg_path.open() as f:
+                cfg = json.load(f)
+            for build in cfg.get("builds", []):
+                variants.append({"board": board_path.name, "name": build["name"]})
+        except Exception as e:
+            print(f"[ERROR] 解析 {cfg_path} 失败: {e}", file=sys.stderr)
+    return variants
+
+
+def _parse_board_config_map() -> dict[str, str]:
+    """Build the mapping of CONFIG_BOARD_TYPE_xxx and board_type from main/CMakeLists.txt"""
+    cmake_file = Path("main/CMakeLists.txt")
+    mapping: dict[str, str] = {}
+    lines = cmake_file.read_text(encoding="utf-8").splitlines()
+    for idx, line in enumerate(lines):
         if "if(CONFIG_BOARD_TYPE_" in line:
             config_name = line.strip().split("if(")[1].split(")")[0]
-                # 查找下一行的 set(BOARD_TYPE "xxx") 
+            if idx + 1 < len(lines):
-                next_line = lines[i + 1].strip()
+                next_line = lines[idx + 1].strip()
                 if next_line.startswith("set(BOARD_TYPE"):
                     board_type = next_line.split('"')[1]
-                    board_configs[config_name] = board_type
+                    mapping[config_name] = board_type
-    return board_configs
+    return mapping
 
-def release(board_type, board_config, config_filename="config.json"):
+
-    config_path = f"main/boards/{board_type}/{config_filename}"
+def _find_board_config(board_type: str) -> Optional[str]:
-    if not os.path.exists(config_path):
+    """Find the corresponding CONFIG_BOARD_TYPE_xxx for the given board_type"""
-        print(f"跳过 {board_type} 因为 {config_filename} 不存在")
+    for config, b_type in _parse_board_config_map().items():
+        if b_type == board_type:
+            return config
+    return None
+
+################################################################################
+# Check board_type in CMakeLists
+################################################################################
+
+def _board_type_exists(board_type: str) -> bool:
+    cmake_file = Path("main/CMakeLists.txt")
+    pattern = f'set(BOARD_TYPE "{board_type}")'
+    return pattern in cmake_file.read_text(encoding="utf-8")
+
+################################################################################
+# Compile implementation
+################################################################################
+
+def release(board_type: str, config_filename: str = "config.json", *, filter_name: Optional[str] = None) -> None:
+    """Compile and package all/specified variants of the specified board_type
+
+    Args:
+        board_type: directory name under main/boards
+        config_filename: config.json name (default: config.json)
+        filter_name: if specified, only compile the build["name"] that matches
+    """
+    cfg_path = _BOARDS_DIR / board_type / config_filename
+    if not cfg_path.exists():
+        print(f"[WARN] {cfg_path} 不存在，跳过 {board_type}")
         return
 
-    # Print Project Version
     project_version = get_project_version()
-    print(f"Project Version: {project_version}", config_path)
+    print(f"Project Version: {project_version} ({cfg_path})")
 
-    with open(config_path, "r") as f:
+    with cfg_path.open() as f:
-        config = json.load(f)
+        cfg = json.load(f)
-    target = config["target"]
+    target = cfg["target"]
-    builds = config["builds"]
+
+    builds = cfg.get("builds", [])
+    if filter_name:
+        builds = [b for b in builds if b["name"] == filter_name]
+        if not builds:
+            print(f"[ERROR] 未在 {board_type} 的 {config_filename} 中找到变体 {filter_name}", file=sys.stderr)
+            sys.exit(1)
 
     for build in builds:
         name = build["name"]
         if not name.startswith(board_type):
-            raise ValueError(f"name {name} 必须以 {board_type} 开头")
+            raise ValueError(f"build.name {name} 必须以 {board_type} 开头")
-        output_path = f"releases/v{project_version}_{name}.zip"
+
-        if os.path.exists(output_path):
+        output_path = Path("releases") / f"v{project_version}_{name}.zip"
-            print(f"跳过 {board_type} 因为 {output_path} 已存在")
+        if output_path.exists():
+            print(f"跳过 {name} 因为 {output_path} 已存在")
             continue
 
-        sdkconfig_append = [f"{board_config}=y"]
+        # Process sdkconfig_append
-        for append in build.get("sdkconfig_append", []):
+        sdkconfig_append = build.get("sdkconfig_append", [])
-            sdkconfig_append.append(append)
+
+        print("-" * 80)
         print(f"name: {name}")
         print(f"target: {target}")
-        for append in sdkconfig_append:
+        for item in sdkconfig_append:
-            print(f"sdkconfig_append: {append}")
+            print(f"sdkconfig_append: {item}")
-        # unset IDF_TARGET
+
         os.environ.pop("IDF_TARGET", None)
+
         # Call set-target
         if os.system(f"idf.py set-target {target}") != 0:
-            print("set-target failed")
+            print("set-target failed", file=sys.stderr)
             sys.exit(1)
+
         # Append sdkconfig
-        with open("sdkconfig", "a") as f:
+        with Path("sdkconfig").open("a") as f:
             f.write("\n")
             f.write("# Append by release.py\n")
             for append in sdkconfig_append:
@@ -112,43 +189,72 @@ def release(board_type, board_config, config_filename="config.json"):
         if os.system(f"idf.py -DBOARD_NAME={name} build") != 0:
             print("build failed")
             sys.exit(1)
-        # Call merge-bin
+
-        if os.system("idf.py merge-bin") != 0:
+        # merge-bin
-            print("merge-bin failed")
+        merge_bin()
-            sys.exit(1)
+
-        # Zip bin
+        # Zip
         zip_bin(name, project_version)
-        print("-" * 80)
+
+################################################################################
+# CLI entry
+################################################################################
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("board", nargs="?", default=None, help="板子类型或 all")
     parser.add_argument("-c", "--config", default="config.json", help="指定 config 文件名，默认 config.json")
-    parser.add_argument("--list-boards", action="store_true", help="列出所有支持的 board 列表")
+    parser.add_argument("--list-boards", action="store_true", help="列出所有支持的 board 及变体列表")
     parser.add_argument("--json", action="store_true", help="配合 --list-boards，JSON 格式输出")
+    parser.add_argument("--name", help="指定变体名称，仅编译匹配的变体")
+
     args = parser.parse_args()
 
+    # List mode
     if args.list_boards:
-        board_configs = get_all_board_types()
+        variants = _collect_variants(config_filename=args.config)
-        boards = list(board_configs.values())
         if args.json:
-            print(json.dumps(boards))
+            print(json.dumps(variants))
         else:
-            for board in boards:
+            for v in variants:
-                print(board)
+                print(f"{v['board']}: {v['name']}")
         sys.exit(0)
 
-    if args.board:
+    # Current directory firmware packaging mode
-        board_configs = get_all_board_types()
+    if args.board is None:
-        found = False
+        merge_bin()
-        for board_config, board_type in board_configs.items():
+        curr_board_type = get_board_type_from_compile_commands()
-            if args.board == 'all' or board_type == args.board:
+        if curr_board_type is None:
-                release(board_type, board_config, config_filename=args.config)
+            print("未能从 compile_commands.json 解析 board_type", file=sys.stderr)
-                found = True
+            sys.exit(1)
-        if not found:
+        project_ver = get_project_version()
-            print(f"未找到板子类型: {args.board}")
+        zip_bin(curr_board_type, project_ver)
-            print("可用的板子类型:")
+        sys.exit(0)
-            for board_type in board_configs.values():
+
-                print(f"  {board_type}")
+    # Compile mode
+    board_type_input: str = args.board
+    name_filter: str | None = args.name
+
+    # Check board_type in CMakeLists
+    if board_type_input != "all" and not _board_type_exists(board_type_input):
+        print(f"[ERROR] main/CMakeLists.txt 中未找到 board_type {board_type_input}", file=sys.stderr)
+        sys.exit(1)
+
+    variants_all = _collect_variants(config_filename=args.config)
+
+    # Filter board_type list
+    target_board_types: set[str]
+    if board_type_input == "all":
+        target_board_types = {v["board"] for v in variants_all}
     else:
-        release_current()
+        target_board_types = {board_type_input}
+
+    for bt in sorted(target_board_types):
+        if not _board_type_exists(bt):
+            print(f"[ERROR] main/CMakeLists.txt 中未找到 board_type {bt}", file=sys.stderr)
+            sys.exit(1)
+        cfg_path = _BOARDS_DIR / bt / args.config
+        if bt == board_type_input and not cfg_path.exists():
+            print(f"开发板 {bt} 未定义 {args.config} 配置文件，跳过")
+            sys.exit(0)
+        release(bt, config_filename=args.config, filter_name=name_filter if bt == board_type_input else None)