CI: full rewrite - moved logic into ci.py and kept workflow scripts mostly dummy

.github/workflows/alpha.yml

@@ -1,28 +1,47 @@
-name: Alpha SideStore build
+name: Alpha SideStore Build
+
 on:
   push:
-    branches:
+    branches: [develop-alpha]
-      - develop-alpha
 
-# cancel duplicate run if from same branch
 concurrency:
   group: ${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  Reusable-build:
+  build:
-    uses: ./.github/workflows/reusable-sidestore-build.yml
+    runs-on: macos-15
+
+    steps:
+      - uses: actions/checkout@v4
         with:
-      # bundle_id: "com.SideStore.SideStore.Alpha"
+          submodules: recursive
-      bundle_id: "com.SideStore.SideStore"
+          fetch-depth: 0
-      # bundle_id_suffix: ".Alpha"
+
-      is_beta: true
+      - run: brew install ldid xcbeautify
-      publish: ${{ vars.PUBLISH_ALPHA_UPDATES == 'true' }} 
+
-      is_shared_build_num: false
+      - name: Shared
-      release_tag: "alpha"
+        id: shared
-      release_name: "Alpha"
+        run: python3 scripts/ci.py shared
-      upstream_tag: "nightly"
+
-      upstream_name: "Nightly"
+      - name: Beta bump
-    secrets:
+        env:
-      CROSS_REPO_PUSH_KEY: ${{ secrets.CROSS_REPO_PUSH_KEY }}
+          RELEASE_CHANNEL: alpha
+        run: python3 scripts/ci.py bump-beta
+
+      - name: Version
+        id: version
+        run: python3 scripts/ci.py version
+
+      - name: Build
+        run: python3 scripts/ci.py build
+
+      - name: Encrypt logs
+        env:
           BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}
+        run: python3 scripts/ci.py encrypt-build
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: SideStore-${{ steps.version.outputs.version }}.ipa
+          path: SideStore.ipa

.github/workflows/nightly.yml

@@ -1,82 +1,128 @@
 name: Nightly SideStore Build
+
 on:
   push:
-    branches:
+    branches: [develop]
-      - develop
   schedule:
-    - cron: '0 0 * * *' # Runs every night at midnight UTC
+    - cron: '0 0 * * *'
-  workflow_dispatch: # Allows manual trigger
+  workflow_dispatch:
 
-# cancel duplicate run if from same branch
 concurrency:
   group: ${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  check-changes:
+  build:
-    if: github.event_name == 'schedule'
+    runs-on: macos-26
-    runs-on: ubuntu-latest
+
-    outputs:
-      has_changes: ${{ steps.check.outputs.has_changes }}
     steps:
-      - name: Checkout repository
+      - uses: actions/checkout@v4
-        uses: actions/checkout@v4
         with:
-          fetch-depth: 0 # Ensure full history
+          submodules: recursive
+          fetch-depth: 0
 
-      - name: Get last successful workflow run
+      - run: brew install ldid xcbeautify
-        id: get_last_success
-        run: |
-          LAST_SUCCESS=$(gh run list --workflow "Nightly SideStore Build" --json createdAt,conclusion \
-            --jq '[.[] | select(.conclusion=="success")][0].createdAt' || echo "")
-          echo "Last successful run: $LAST_SUCCESS"
-          echo "last_success=$LAST_SUCCESS" >> $GITHUB_ENV
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Check for new commits since last successful build
+      - name: Restore Xcode/SwiftPM Cache (Exact match)
-        id: check
+        id: xcode-cache-restore
-        run: |
+        uses: actions/cache/restore@v3
-          if [ -n "$LAST_SUCCESS" ]; then
-            NEW_COMMITS=$(git rev-list --count --since="$LAST_SUCCESS" origin/develop)
-            COMMIT_LOG=$(git log --since="$LAST_SUCCESS" --pretty=format:"%h %s" origin/develop)
-          else
-            NEW_COMMITS=1
-            COMMIT_LOG=$(git log -n 10 --pretty=format:"%h %s" origin/develop) # Show last 10 commits if no history
-          fi
-          
-          echo "Has changes: $NEW_COMMITS"
-          echo "New commits since last successful build:"
-          echo "$COMMIT_LOG"
-          
-          if [ "$NEW_COMMITS" -gt 0 ]; then
-            echo "has_changes=true" >> $GITHUB_OUTPUT
-          else
-            echo "has_changes=false" >> $GITHUB_OUTPUT
-          fi
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          LAST_SUCCESS: ${{ env.last_success }}
-
-  Reusable-build:
-    if: |
-      always() &&
-      (github.event_name == 'push' || 
-      (github.event_name == 'schedule' && needs.check-changes.result == 'success' && needs.check-changes.outputs.has_changes == 'true'))
-    needs: check-changes
-    uses: ./.github/workflows/reusable-sidestore-build.yml
         with:
-      # bundle_id: "com.SideStore.SideStore.Nightly"
+          path: |
-      bundle_id: "com.SideStore.SideStore"
+            ~/Library/Developer/Xcode/DerivedData
-      # bundle_id_suffix: ".Nightly"
+            ~/Library/Caches/org.swift.swiftpm
-      is_beta: true
+          key: xcode-cache-build-${{ github.ref_name }}-${{ github.sha }}
-      publish: ${{ vars.PUBLISH_NIGHTLY_UPDATES == 'true' }} 
+
-      is_shared_build_num: false
+      - name: Restore Xcode/SwiftPM Cache (Last Available)
-      release_tag: "nightly"
+        id: xcode-cache-restore-recent
-      release_name: "Nightly"
+        uses: actions/cache/restore@v3
-      upstream_tag: "0.5.10"
+        with:
-      upstream_name: "Stable"
+          path: |
-    secrets:
+            ~/Library/Developer/Xcode/DerivedData
-      CROSS_REPO_PUSH_KEY: ${{ secrets.CROSS_REPO_PUSH_KEY }}
+            ~/Library/Caches/org.swift.swiftpm
+          key: xcode-cache-build-${{ github.ref_name }}-
+
+      - name: Short Commit SHA
+        id: shared
+        run: python3 scripts/ci.py commid-id
+
+      - name: Nightly Version bump
+        env:
+          RELEASE_CHANNEL: nightly
+        run: python3 scripts/ci.py bump-beta
+
+      - name: Version
+        id: version
+        run: python3 scripts/ci.py version
+
+      - name: Clean previous build artifacts
+        run: python3 scripts/ci.py clean
+
+      - name: Build
+        run: python3 scripts/ci.py build
+
+      - name: Tests Build
+        if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_BUILD == '1' }}
+        run: python3 scripts/ci.py tests-build
+
+      - name: Save Xcode & SwiftPM Cache
+        id: cache-save
+        if: ${{ steps.xcode-cache-restore.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@v3
+        with:
+          path: |
+            ~/Library/Developer/Xcode/DerivedData
+            ~/Library/Caches/org.swift.swiftpm
+          key: xcode-cache-build-${{ github.ref_name }}-${{ github.sha }}
+
+      - name: Tests Run
+        if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_RUN == '1' }}
+        run: python3 scripts/ci.py tests-run
+
+      - name: Encrypt build logs
+        env:
           BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}
+        run: python3 scripts/ci.py encrypt-build
 
+      - name: Encrypt tests-build logs
+        if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_BUILD == '1' }}
+        env:
+          BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}
+        run: python3 scripts/ci.py encrypt-tests-build
+
+      - name: Encrypt tests-run logs
+        if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_RUN == '1' }}
+        env:
+          BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}
+        run: python3 scripts/ci.py encrypt-tests-run
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: encrypted-build-logs-${{ steps.version.outputs.version }}.zip
+          path: encrypted-build-logs.zip
+
+      - uses: actions/upload-artifact@v4
+        if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_BUILD == '1' }}
+        with:
+          name: encrypted-tests-build-logs-${{ steps.shared.outputs.short_commit }}.zip
+          path: encrypted-tests-build-logs.zip
+
+      - uses: actions/upload-artifact@v4
+        if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_RUN == '1' }}
+        with:
+          name: encrypted-tests-run-logs-${{ steps.shared.outputs.short_commit }}.zip
+          path: encrypted-tests-run-logs.zip
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: SideStore-${{ steps.version.outputs.version }}.ipa
+          path: SideStore.ipa
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: SideStore-${{ steps.version.outputs.version }}-dSYMs.zip
+          path: SideStore.dSYMs.zip
+
+      - name: Deploy
+        run: python3 scripts/ci.py deploy \
+          --release-tag nightly \
+          --release-name Nightly

.github/workflows/obsolete/alpha.yml

@@ -0,0 +1,28 @@
+name: Alpha SideStore build
+on:
+  push:
+    branches:
+      - develop-alpha
+
+# cancel duplicate run if from same branch
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  Reusable-build:
+    uses: ./.github/workflows/reusable-sidestore-build.yml
+    with:
+      # bundle_id: "com.SideStore.SideStore.Alpha"
+      bundle_id: "com.SideStore.SideStore"
+      # bundle_id_suffix: ".Alpha"
+      is_beta: true
+      publish: ${{ vars.PUBLISH_ALPHA_UPDATES == 'true' }} 
+      is_shared_build_num: false
+      release_tag: "alpha"
+      release_name: "Alpha"
+      upstream_tag: "nightly"
+      upstream_name: "Nightly"
+    secrets:
+      CROSS_REPO_PUSH_KEY: ${{ secrets.CROSS_REPO_PUSH_KEY }}
+      BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}

.github/workflows/obsolete/nightly.yml

@@ -0,0 +1,82 @@
+name: Nightly SideStore Build
+on:
+  push:
+    branches:
+      - develop
+  schedule:
+    - cron: '0 0 * * *' # Runs every night at midnight UTC
+  workflow_dispatch: # Allows manual trigger
+
+# cancel duplicate run if from same branch
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  check-changes:
+    if: github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    outputs:
+      has_changes: ${{ steps.check.outputs.has_changes }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Ensure full history
+
+      - name: Get last successful workflow run
+        id: get_last_success
+        run: |
+          LAST_SUCCESS=$(gh run list --workflow "Nightly SideStore Build" --json createdAt,conclusion \
+            --jq '[.[] | select(.conclusion=="success")][0].createdAt' || echo "")
+          echo "Last successful run: $LAST_SUCCESS"
+          echo "last_success=$LAST_SUCCESS" >> $GITHUB_ENV
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check for new commits since last successful build
+        id: check
+        run: |
+          if [ -n "$LAST_SUCCESS" ]; then
+            NEW_COMMITS=$(git rev-list --count --since="$LAST_SUCCESS" origin/develop)
+            COMMIT_LOG=$(git log --since="$LAST_SUCCESS" --pretty=format:"%h %s" origin/develop)
+          else
+            NEW_COMMITS=1
+            COMMIT_LOG=$(git log -n 10 --pretty=format:"%h %s" origin/develop) # Show last 10 commits if no history
+          fi
+          
+          echo "Has changes: $NEW_COMMITS"
+          echo "New commits since last successful build:"
+          echo "$COMMIT_LOG"
+          
+          if [ "$NEW_COMMITS" -gt 0 ]; then
+            echo "has_changes=true" >> $GITHUB_OUTPUT
+          else
+            echo "has_changes=false" >> $GITHUB_OUTPUT
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          LAST_SUCCESS: ${{ env.last_success }}
+
+  Reusable-build:
+    if: |
+      always() &&
+      (github.event_name == 'push' || 
+      (github.event_name == 'schedule' && needs.check-changes.result == 'success' && needs.check-changes.outputs.has_changes == 'true'))
+    needs: check-changes
+    uses: ./.github/workflows/reusable-sidestore-build.yml
+    with:
+      # bundle_id: "com.SideStore.SideStore.Nightly"
+      bundle_id: "com.SideStore.SideStore"
+      # bundle_id_suffix: ".Nightly"
+      is_beta: true
+      publish: ${{ vars.PUBLISH_NIGHTLY_UPDATES == 'true' }} 
+      is_shared_build_num: false
+      release_tag: "nightly"
+      release_name: "Nightly"
+      upstream_tag: "0.5.10"
+      upstream_name: "Stable"
+    secrets:
+      CROSS_REPO_PUSH_KEY: ${{ secrets.CROSS_REPO_PUSH_KEY }}
+      BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}
+

scripts/ci.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+import os
+import sys
+import subprocess
+import datetime
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+# ----------------------------------------------------------
+# helpers
+# ----------------------------------------------------------
+
+def run(cmd, check=True):
+    print(f"$ {cmd}", flush=True)
+    subprocess.run(cmd, shell=True, cwd=ROOT, check=check)
+
+
+def output(name, value):
+    print(f"{name}={value}")
+    out = os.environ.get("GITHUB_OUTPUT")
+    if out:
+        with open(out, "a") as f:
+            f.write(f"{name}={value}\n")
+
+
+def getenv(name, default=""):
+    return os.environ.get(name, default)
+
+
+# ----------------------------------------------------------
+# SHARED
+# ----------------------------------------------------------
+
+def short_commit():
+    sha = subprocess.check_output(
+        "git rev-parse --short HEAD",
+        shell=True,
+        cwd=ROOT
+    ).decode().strip()
+
+    output("short_commit", sha)
+    return sha
+
+
+# ----------------------------------------------------------
+# VERSION BUMP
+# ----------------------------------------------------------
+
+def bump_beta():
+    date = datetime.datetime.utcnow().strftime("%Y.%m.%d")
+    release_channel = getenv("RELEASE_CHANNEL", "beta")
+
+    build_file = ROOT / "build_number.txt"
+    xcconfig = ROOT / "Build.xcconfig"
+
+    short = subprocess.check_output(
+        "git rev-parse --short HEAD",
+        shell=True,
+        cwd=ROOT
+    ).decode().strip()
+
+    def write(num):
+        run(
+            f"""sed -e "/MARKETING_VERSION = .*/s/$/-{release_channel}.{date}.{num}+{short}/" -i '' Build.xcconfig"""
+        )
+        build_file.write_text(f"{date},{num}")
+
+    if not build_file.exists():
+        write(1)
+        return
+
+    last = build_file.read_text().strip().split(",")[1]
+    write(int(last) + 1)
+
+
+# ----------------------------------------------------------
+# VERSION EXTRACTION
+# ----------------------------------------------------------
+
+def extract_version():
+    v = subprocess.check_output(
+        "grep MARKETING_VERSION Build.xcconfig | sed -e 's/MARKETING_VERSION = //g'",
+        shell=True,
+        cwd=ROOT
+    ).decode().strip()
+
+    output("version", v)
+    return v
+
+
+# ----------------------------------------------------------
+# CLEAN
+# ----------------------------------------------------------
+def clean():
+    run("make clean")
+    clean_derived_data()
+
+def clean_derived_data():
+    run("rm -rf ~/Library/Developer/Xcode/DerivedData/*", check=False)
+
+# ----------------------------------------------------------
+# BUILD
+# ----------------------------------------------------------
+
+def build():
+    run("make clean")
+    run("rm -rf ~/Library/Developer/Xcode/DerivedData/*", check=False)
+    run("mkdir -p build/logs")
+
+    run(
+        "set -o pipefail && "
+        "NSUnbufferedIO=YES make -B build "
+        "2>&1 | tee -a build/logs/build.log | xcbeautify --renderer github-actions"
+    )
+
+    run("make fakesign | tee -a build/logs/build.log")
+    run("make ipa | tee -a build/logs/build.log")
+
+    run("zip -r -9 ./SideStore.dSYMs.zip ./SideStore.xcarchive/dSYMs")
+
+
+# ----------------------------------------------------------
+# TESTS BUILD
+# ----------------------------------------------------------
+
+def tests_build():
+    run("mkdir -p build/logs")
+    run(
+        "NSUnbufferedIO=YES make -B build-tests "
+        "2>&1 | tee -a build/logs/tests-build.log | xcbeautify --renderer github-actions"
+    )
+
+
+# ----------------------------------------------------------
+# TESTS RUN
+# ----------------------------------------------------------
+
+def tests_run():
+    run("mkdir -p build/logs")
+    run("nohup make -B boot-sim-async </dev/null >> build/logs/tests-run.log 2>&1 &")
+
+    run("make -B sim-boot-check | tee -a build/logs/tests-run.log")
+
+    run("make run-tests 2>&1 | tee -a build/logs/tests-run.log")
+
+    run("zip -r -9 ./test-results.zip ./build/tests")
+
+
+# ----------------------------------------------------------
+# LOG ENCRYPTION
+# ----------------------------------------------------------
+
+def encrypt_logs(name):
+    pwd = getenv("BUILD_LOG_ZIP_PASSWORD", "12345")
+    run(
+        f'cd build/logs && zip -e -P "{pwd}" ../../{name}.zip *'
+    )
+
+
+# ----------------------------------------------------------
+# RELEASE NOTES
+# ----------------------------------------------------------
+
+def release_notes(tag):
+    last = subprocess.check_output(
+        "gh run list --branch $(git branch --show-current) "
+        "--status success --json headSha --jq '.[0].headSha'",
+        shell=True,
+        cwd=ROOT
+    ).decode().strip()
+
+    if not last or last == "null":
+        last = subprocess.check_output(
+            "git rev-list --max-parents=0 HEAD",
+            shell=True,
+            cwd=ROOT
+        ).decode().strip()
+
+    run(f"python3 update_release_notes.py {last} {tag}")
+
+
+# ----------------------------------------------------------
+# PUBLISH SOURCE.JSON
+# ----------------------------------------------------------
+
+def publish_apps(short_commit):
+    repo = ROOT / "SideStore/apps-v2.json"
+
+    run("git config user.name 'GitHub Actions'", check=False)
+    run("git config user.email 'github-actions@github.com'", check=False)
+
+    run("python3 scripts/update_apps.py './_includes/source.json'", check=False)
+
+    run("git add ./_includes/source.json", check=False)
+    run(
+        f"git commit -m ' - updated for {short_commit} deployment' || true",
+        check=False
+    )
+    run("git push", check=False)
+
+
+# ----------------------------------------------------------
+# ENTRYPOINT
+# ----------------------------------------------------------
+
+def main():
+    cmd = sys.argv[1]
+
+    if cmd == "commid-id":
+        short_commit()
+
+    elif cmd == "bump-beta":
+        bump_beta()
+
+    elif cmd == "version":
+        extract_version()
+
+    elif cmd == "clean":
+        clean()
+
+    elif cmd == "cleanDerivedData":
+        clean_derived_data()
+
+    elif cmd == "build":
+        build()
+
+    elif cmd == "tests-build":
+        tests_build()
+
+    elif cmd == "tests-run":
+        tests_run()
+
+    elif cmd == "encrypt-build":
+        encrypt_logs("encrypted-build-logs")
+
+    elif cmd == "encrypt-tests-build":
+        encrypt_logs("encrypted-tests-build-logs")
+
+    elif cmd == "encrypt-tests-run":
+        encrypt_logs("encrypted-tests-run-logs")
+
+    elif cmd == "release-notes":
+        release_notes(sys.argv[2])
+
+    elif cmd == "publish":
+        publish_apps(sys.argv[2])
+
+    else:
+        raise SystemExit(f"Unknown command {cmd}")
+
+
+if __name__ == "__main__":
+    main()