From cf0a1748829448919b6401f83ca25b533a4f1ab4 Mon Sep 17 00:00:00 2001 From: mahee96 <47920326+mahee96@users.noreply.github.com> Date: Mon, 23 Feb 2026 17:06:31 +0530 Subject: [PATCH] CI: full rewrite - moved logic into ci.py and kept workflow scripts mostly dummy --- .github/workflows/alpha.yml | 59 ++-- .github/workflows/nightly.yml | 174 +++++++----- .github/workflows/obsolete/alpha.yml | 28 ++ .github/workflows/{ => obsolete}/beta.yml | 0 .../{ => obsolete}/increase-beta-build-num.sh | 0 .github/workflows/obsolete/nightly.yml | 82 ++++++ .../reusable-sidestore-build.yml | 0 .../{ => obsolete}/sidestore-build.yml | 0 .../{ => obsolete}/sidestore-deploy.yml | 0 .../{ => obsolete}/sidestore-shared.yml | 0 .../{ => obsolete}/sidestore-tests-build.yml | 0 .../{ => obsolete}/sidestore-tests-run.yml | 0 scripts/ci.py | 255 ++++++++++++++++++ update_apps.py => scripts/update_apps.py | 0 .../update_release_notes.py | 0 15 files changed, 514 insertions(+), 84 deletions(-) create mode 100644 .github/workflows/obsolete/alpha.yml rename .github/workflows/{ => obsolete}/beta.yml (100%) rename .github/workflows/{ => obsolete}/increase-beta-build-num.sh (100%) create mode 100644 .github/workflows/obsolete/nightly.yml rename .github/workflows/{ => obsolete}/reusable-sidestore-build.yml (100%) rename .github/workflows/{ => obsolete}/sidestore-build.yml (100%) rename .github/workflows/{ => obsolete}/sidestore-deploy.yml (100%) rename .github/workflows/{ => obsolete}/sidestore-shared.yml (100%) rename .github/workflows/{ => obsolete}/sidestore-tests-build.yml (100%) rename .github/workflows/{ => obsolete}/sidestore-tests-run.yml (100%) create mode 100644 scripts/ci.py rename update_apps.py => scripts/update_apps.py (100%) rename update_release_notes.py => scripts/update_release_notes.py (100%) diff --git a/.github/workflows/alpha.yml b/.github/workflows/alpha.yml index 9066ff6a..84d6bcf4 100644 --- a/.github/workflows/alpha.yml +++ b/.github/workflows/alpha.yml @@ -1,28 +1,47 @@ -name: Alpha SideStore build +name: Alpha SideStore Build + on: push: - branches: - - develop-alpha + branches: [develop-alpha] -# cancel duplicate run if from same branch concurrency: group: ${{ github.ref }} cancel-in-progress: true jobs: - Reusable-build: - uses: ./.github/workflows/reusable-sidestore-build.yml - with: - # bundle_id: "com.SideStore.SideStore.Alpha" - bundle_id: "com.SideStore.SideStore" - # bundle_id_suffix: ".Alpha" - is_beta: true - publish: ${{ vars.PUBLISH_ALPHA_UPDATES == 'true' }} - is_shared_build_num: false - release_tag: "alpha" - release_name: "Alpha" - upstream_tag: "nightly" - upstream_name: "Nightly" - secrets: - CROSS_REPO_PUSH_KEY: ${{ secrets.CROSS_REPO_PUSH_KEY }} - BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }} \ No newline at end of file + build: + runs-on: macos-15 + + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + + - run: brew install ldid xcbeautify + + - name: Shared + id: shared + run: python3 scripts/ci.py shared + + - name: Beta bump + env: + RELEASE_CHANNEL: alpha + run: python3 scripts/ci.py bump-beta + + - name: Version + id: version + run: python3 scripts/ci.py version + + - name: Build + run: python3 scripts/ci.py build + + - name: Encrypt logs + env: + BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }} + run: python3 scripts/ci.py encrypt-build + + - uses: actions/upload-artifact@v4 + with: + name: SideStore-${{ steps.version.outputs.version }}.ipa + path: SideStore.ipa \ No newline at end of file diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 65662b62..3b8b5cc7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,82 +1,128 @@ name: Nightly SideStore Build + on: push: - branches: - - develop + branches: [develop] schedule: - - cron: '0 0 * * *' # Runs every night at midnight UTC - workflow_dispatch: # Allows manual trigger + - cron: '0 0 * * *' + workflow_dispatch: -# cancel duplicate run if from same branch concurrency: group: ${{ github.ref }} cancel-in-progress: true jobs: - check-changes: - if: github.event_name == 'schedule' - runs-on: ubuntu-latest - outputs: - has_changes: ${{ steps.check.outputs.has_changes }} + build: + runs-on: macos-26 + steps: - - name: Checkout repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: - fetch-depth: 0 # Ensure full history + submodules: recursive + fetch-depth: 0 - - name: Get last successful workflow run - id: get_last_success - run: | - LAST_SUCCESS=$(gh run list --workflow "Nightly SideStore Build" --json createdAt,conclusion \ - --jq '[.[] | select(.conclusion=="success")][0].createdAt' || echo "") - echo "Last successful run: $LAST_SUCCESS" - echo "last_success=$LAST_SUCCESS" >> $GITHUB_ENV + - run: brew install ldid xcbeautify + + - name: Restore Xcode/SwiftPM Cache (Exact match) + id: xcode-cache-restore + uses: actions/cache/restore@v3 + with: + path: | + ~/Library/Developer/Xcode/DerivedData + ~/Library/Caches/org.swift.swiftpm + key: xcode-cache-build-${{ github.ref_name }}-${{ github.sha }} + + - name: Restore Xcode/SwiftPM Cache (Last Available) + id: xcode-cache-restore-recent + uses: actions/cache/restore@v3 + with: + path: | + ~/Library/Developer/Xcode/DerivedData + ~/Library/Caches/org.swift.swiftpm + key: xcode-cache-build-${{ github.ref_name }}- + + - name: Short Commit SHA + id: shared + run: python3 scripts/ci.py commid-id + + - name: Nightly Version bump env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_CHANNEL: nightly + run: python3 scripts/ci.py bump-beta - - name: Check for new commits since last successful build - id: check - run: | - if [ -n "$LAST_SUCCESS" ]; then - NEW_COMMITS=$(git rev-list --count --since="$LAST_SUCCESS" origin/develop) - COMMIT_LOG=$(git log --since="$LAST_SUCCESS" --pretty=format:"%h %s" origin/develop) - else - NEW_COMMITS=1 - COMMIT_LOG=$(git log -n 10 --pretty=format:"%h %s" origin/develop) # Show last 10 commits if no history - fi - - echo "Has changes: $NEW_COMMITS" - echo "New commits since last successful build:" - echo "$COMMIT_LOG" - - if [ "$NEW_COMMITS" -gt 0 ]; then - echo "has_changes=true" >> $GITHUB_OUTPUT - else - echo "has_changes=false" >> $GITHUB_OUTPUT - fi + - name: Version + id: version + run: python3 scripts/ci.py version + + - name: Clean previous build artifacts + run: python3 scripts/ci.py clean + + - name: Build + run: python3 scripts/ci.py build + + - name: Tests Build + if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_BUILD == '1' }} + run: python3 scripts/ci.py tests-build + + - name: Save Xcode & SwiftPM Cache + id: cache-save + if: ${{ steps.xcode-cache-restore.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v3 + with: + path: | + ~/Library/Developer/Xcode/DerivedData + ~/Library/Caches/org.swift.swiftpm + key: xcode-cache-build-${{ github.ref_name }}-${{ github.sha }} + + - name: Tests Run + if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_RUN == '1' }} + run: python3 scripts/ci.py tests-run + + - name: Encrypt build logs env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - LAST_SUCCESS: ${{ env.last_success }} + BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }} + run: python3 scripts/ci.py encrypt-build - Reusable-build: - if: | - always() && - (github.event_name == 'push' || - (github.event_name == 'schedule' && needs.check-changes.result == 'success' && needs.check-changes.outputs.has_changes == 'true')) - needs: check-changes - uses: ./.github/workflows/reusable-sidestore-build.yml - with: - # bundle_id: "com.SideStore.SideStore.Nightly" - bundle_id: "com.SideStore.SideStore" - # bundle_id_suffix: ".Nightly" - is_beta: true - publish: ${{ vars.PUBLISH_NIGHTLY_UPDATES == 'true' }} - is_shared_build_num: false - release_tag: "nightly" - release_name: "Nightly" - upstream_tag: "0.5.10" - upstream_name: "Stable" - secrets: - CROSS_REPO_PUSH_KEY: ${{ secrets.CROSS_REPO_PUSH_KEY }} - BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }} + - name: Encrypt tests-build logs + if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_BUILD == '1' }} + env: + BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }} + run: python3 scripts/ci.py encrypt-tests-build + - name: Encrypt tests-run logs + if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_RUN == '1' }} + env: + BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }} + run: python3 scripts/ci.py encrypt-tests-run + + - uses: actions/upload-artifact@v4 + with: + name: encrypted-build-logs-${{ steps.version.outputs.version }}.zip + path: encrypted-build-logs.zip + + - uses: actions/upload-artifact@v4 + if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_BUILD == '1' }} + with: + name: encrypted-tests-build-logs-${{ steps.shared.outputs.short_commit }}.zip + path: encrypted-tests-build-logs.zip + + - uses: actions/upload-artifact@v4 + if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_RUN == '1' }} + with: + name: encrypted-tests-run-logs-${{ steps.shared.outputs.short_commit }}.zip + path: encrypted-tests-run-logs.zip + + - uses: actions/upload-artifact@v4 + with: + name: SideStore-${{ steps.version.outputs.version }}.ipa + path: SideStore.ipa + + - uses: actions/upload-artifact@v4 + with: + name: SideStore-${{ steps.version.outputs.version }}-dSYMs.zip + path: SideStore.dSYMs.zip + + - name: Deploy + run: python3 scripts/ci.py deploy \ + --release-tag nightly \ + --release-name Nightly \ No newline at end of file diff --git a/.github/workflows/obsolete/alpha.yml b/.github/workflows/obsolete/alpha.yml new file mode 100644 index 00000000..9066ff6a --- /dev/null +++ b/.github/workflows/obsolete/alpha.yml @@ -0,0 +1,28 @@ +name: Alpha SideStore build +on: + push: + branches: + - develop-alpha + +# cancel duplicate run if from same branch +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + Reusable-build: + uses: ./.github/workflows/reusable-sidestore-build.yml + with: + # bundle_id: "com.SideStore.SideStore.Alpha" + bundle_id: "com.SideStore.SideStore" + # bundle_id_suffix: ".Alpha" + is_beta: true + publish: ${{ vars.PUBLISH_ALPHA_UPDATES == 'true' }} + is_shared_build_num: false + release_tag: "alpha" + release_name: "Alpha" + upstream_tag: "nightly" + upstream_name: "Nightly" + secrets: + CROSS_REPO_PUSH_KEY: ${{ secrets.CROSS_REPO_PUSH_KEY }} + BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/beta.yml b/.github/workflows/obsolete/beta.yml similarity index 100% rename from .github/workflows/beta.yml rename to .github/workflows/obsolete/beta.yml diff --git a/.github/workflows/increase-beta-build-num.sh b/.github/workflows/obsolete/increase-beta-build-num.sh similarity index 100% rename from .github/workflows/increase-beta-build-num.sh rename to .github/workflows/obsolete/increase-beta-build-num.sh diff --git a/.github/workflows/obsolete/nightly.yml b/.github/workflows/obsolete/nightly.yml new file mode 100644 index 00000000..65662b62 --- /dev/null +++ b/.github/workflows/obsolete/nightly.yml @@ -0,0 +1,82 @@ +name: Nightly SideStore Build +on: + push: + branches: + - develop + schedule: + - cron: '0 0 * * *' # Runs every night at midnight UTC + workflow_dispatch: # Allows manual trigger + +# cancel duplicate run if from same branch +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + check-changes: + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + outputs: + has_changes: ${{ steps.check.outputs.has_changes }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Ensure full history + + - name: Get last successful workflow run + id: get_last_success + run: | + LAST_SUCCESS=$(gh run list --workflow "Nightly SideStore Build" --json createdAt,conclusion \ + --jq '[.[] | select(.conclusion=="success")][0].createdAt' || echo "") + echo "Last successful run: $LAST_SUCCESS" + echo "last_success=$LAST_SUCCESS" >> $GITHUB_ENV + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Check for new commits since last successful build + id: check + run: | + if [ -n "$LAST_SUCCESS" ]; then + NEW_COMMITS=$(git rev-list --count --since="$LAST_SUCCESS" origin/develop) + COMMIT_LOG=$(git log --since="$LAST_SUCCESS" --pretty=format:"%h %s" origin/develop) + else + NEW_COMMITS=1 + COMMIT_LOG=$(git log -n 10 --pretty=format:"%h %s" origin/develop) # Show last 10 commits if no history + fi + + echo "Has changes: $NEW_COMMITS" + echo "New commits since last successful build:" + echo "$COMMIT_LOG" + + if [ "$NEW_COMMITS" -gt 0 ]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + else + echo "has_changes=false" >> $GITHUB_OUTPUT + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LAST_SUCCESS: ${{ env.last_success }} + + Reusable-build: + if: | + always() && + (github.event_name == 'push' || + (github.event_name == 'schedule' && needs.check-changes.result == 'success' && needs.check-changes.outputs.has_changes == 'true')) + needs: check-changes + uses: ./.github/workflows/reusable-sidestore-build.yml + with: + # bundle_id: "com.SideStore.SideStore.Nightly" + bundle_id: "com.SideStore.SideStore" + # bundle_id_suffix: ".Nightly" + is_beta: true + publish: ${{ vars.PUBLISH_NIGHTLY_UPDATES == 'true' }} + is_shared_build_num: false + release_tag: "nightly" + release_name: "Nightly" + upstream_tag: "0.5.10" + upstream_name: "Stable" + secrets: + CROSS_REPO_PUSH_KEY: ${{ secrets.CROSS_REPO_PUSH_KEY }} + BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }} + diff --git a/.github/workflows/reusable-sidestore-build.yml b/.github/workflows/obsolete/reusable-sidestore-build.yml similarity index 100% rename from .github/workflows/reusable-sidestore-build.yml rename to .github/workflows/obsolete/reusable-sidestore-build.yml diff --git a/.github/workflows/sidestore-build.yml b/.github/workflows/obsolete/sidestore-build.yml similarity index 100% rename from .github/workflows/sidestore-build.yml rename to .github/workflows/obsolete/sidestore-build.yml diff --git a/.github/workflows/sidestore-deploy.yml b/.github/workflows/obsolete/sidestore-deploy.yml similarity index 100% rename from .github/workflows/sidestore-deploy.yml rename to .github/workflows/obsolete/sidestore-deploy.yml diff --git a/.github/workflows/sidestore-shared.yml b/.github/workflows/obsolete/sidestore-shared.yml similarity index 100% rename from .github/workflows/sidestore-shared.yml rename to .github/workflows/obsolete/sidestore-shared.yml diff --git a/.github/workflows/sidestore-tests-build.yml b/.github/workflows/obsolete/sidestore-tests-build.yml similarity index 100% rename from .github/workflows/sidestore-tests-build.yml rename to .github/workflows/obsolete/sidestore-tests-build.yml diff --git a/.github/workflows/sidestore-tests-run.yml b/.github/workflows/obsolete/sidestore-tests-run.yml similarity index 100% rename from .github/workflows/sidestore-tests-run.yml rename to .github/workflows/obsolete/sidestore-tests-run.yml diff --git a/scripts/ci.py b/scripts/ci.py new file mode 100644 index 00000000..18b866a4 --- /dev/null +++ b/scripts/ci.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +import os +import sys +import subprocess +import datetime +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +# ---------------------------------------------------------- +# helpers +# ---------------------------------------------------------- + +def run(cmd, check=True): + print(f"$ {cmd}", flush=True) + subprocess.run(cmd, shell=True, cwd=ROOT, check=check) + + +def output(name, value): + print(f"{name}={value}") + out = os.environ.get("GITHUB_OUTPUT") + if out: + with open(out, "a") as f: + f.write(f"{name}={value}\n") + + +def getenv(name, default=""): + return os.environ.get(name, default) + + +# ---------------------------------------------------------- +# SHARED +# ---------------------------------------------------------- + +def short_commit(): + sha = subprocess.check_output( + "git rev-parse --short HEAD", + shell=True, + cwd=ROOT + ).decode().strip() + + output("short_commit", sha) + return sha + + +# ---------------------------------------------------------- +# VERSION BUMP +# ---------------------------------------------------------- + +def bump_beta(): + date = datetime.datetime.utcnow().strftime("%Y.%m.%d") + release_channel = getenv("RELEASE_CHANNEL", "beta") + + build_file = ROOT / "build_number.txt" + xcconfig = ROOT / "Build.xcconfig" + + short = subprocess.check_output( + "git rev-parse --short HEAD", + shell=True, + cwd=ROOT + ).decode().strip() + + def write(num): + run( + f"""sed -e "/MARKETING_VERSION = .*/s/$/-{release_channel}.{date}.{num}+{short}/" -i '' Build.xcconfig""" + ) + build_file.write_text(f"{date},{num}") + + if not build_file.exists(): + write(1) + return + + last = build_file.read_text().strip().split(",")[1] + write(int(last) + 1) + + +# ---------------------------------------------------------- +# VERSION EXTRACTION +# ---------------------------------------------------------- + +def extract_version(): + v = subprocess.check_output( + "grep MARKETING_VERSION Build.xcconfig | sed -e 's/MARKETING_VERSION = //g'", + shell=True, + cwd=ROOT + ).decode().strip() + + output("version", v) + return v + + +# ---------------------------------------------------------- +# CLEAN +# ---------------------------------------------------------- +def clean(): + run("make clean") + clean_derived_data() + +def clean_derived_data(): + run("rm -rf ~/Library/Developer/Xcode/DerivedData/*", check=False) + +# ---------------------------------------------------------- +# BUILD +# ---------------------------------------------------------- + +def build(): + run("make clean") + run("rm -rf ~/Library/Developer/Xcode/DerivedData/*", check=False) + run("mkdir -p build/logs") + + run( + "set -o pipefail && " + "NSUnbufferedIO=YES make -B build " + "2>&1 | tee -a build/logs/build.log | xcbeautify --renderer github-actions" + ) + + run("make fakesign | tee -a build/logs/build.log") + run("make ipa | tee -a build/logs/build.log") + + run("zip -r -9 ./SideStore.dSYMs.zip ./SideStore.xcarchive/dSYMs") + + +# ---------------------------------------------------------- +# TESTS BUILD +# ---------------------------------------------------------- + +def tests_build(): + run("mkdir -p build/logs") + run( + "NSUnbufferedIO=YES make -B build-tests " + "2>&1 | tee -a build/logs/tests-build.log | xcbeautify --renderer github-actions" + ) + + +# ---------------------------------------------------------- +# TESTS RUN +# ---------------------------------------------------------- + +def tests_run(): + run("mkdir -p build/logs") + run("nohup make -B boot-sim-async > build/logs/tests-run.log 2>&1 &") + + run("make -B sim-boot-check | tee -a build/logs/tests-run.log") + + run("make run-tests 2>&1 | tee -a build/logs/tests-run.log") + + run("zip -r -9 ./test-results.zip ./build/tests") + + +# ---------------------------------------------------------- +# LOG ENCRYPTION +# ---------------------------------------------------------- + +def encrypt_logs(name): + pwd = getenv("BUILD_LOG_ZIP_PASSWORD", "12345") + run( + f'cd build/logs && zip -e -P "{pwd}" ../../{name}.zip *' + ) + + +# ---------------------------------------------------------- +# RELEASE NOTES +# ---------------------------------------------------------- + +def release_notes(tag): + last = subprocess.check_output( + "gh run list --branch $(git branch --show-current) " + "--status success --json headSha --jq '.[0].headSha'", + shell=True, + cwd=ROOT + ).decode().strip() + + if not last or last == "null": + last = subprocess.check_output( + "git rev-list --max-parents=0 HEAD", + shell=True, + cwd=ROOT + ).decode().strip() + + run(f"python3 update_release_notes.py {last} {tag}") + + +# ---------------------------------------------------------- +# PUBLISH SOURCE.JSON +# ---------------------------------------------------------- + +def publish_apps(short_commit): + repo = ROOT / "SideStore/apps-v2.json" + + run("git config user.name 'GitHub Actions'", check=False) + run("git config user.email 'github-actions@github.com'", check=False) + + run("python3 scripts/update_apps.py './_includes/source.json'", check=False) + + run("git add ./_includes/source.json", check=False) + run( + f"git commit -m ' - updated for {short_commit} deployment' || true", + check=False + ) + run("git push", check=False) + + +# ---------------------------------------------------------- +# ENTRYPOINT +# ---------------------------------------------------------- + +def main(): + cmd = sys.argv[1] + + if cmd == "commid-id": + short_commit() + + elif cmd == "bump-beta": + bump_beta() + + elif cmd == "version": + extract_version() + + elif cmd == "clean": + clean() + + elif cmd == "cleanDerivedData": + clean_derived_data() + + elif cmd == "build": + build() + + elif cmd == "tests-build": + tests_build() + + elif cmd == "tests-run": + tests_run() + + elif cmd == "encrypt-build": + encrypt_logs("encrypted-build-logs") + + elif cmd == "encrypt-tests-build": + encrypt_logs("encrypted-tests-build-logs") + + elif cmd == "encrypt-tests-run": + encrypt_logs("encrypted-tests-run-logs") + + elif cmd == "release-notes": + release_notes(sys.argv[2]) + + elif cmd == "publish": + publish_apps(sys.argv[2]) + + else: + raise SystemExit(f"Unknown command {cmd}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/update_apps.py b/scripts/update_apps.py similarity index 100% rename from update_apps.py rename to scripts/update_apps.py diff --git a/update_release_notes.py b/scripts/update_release_notes.py similarity index 100% rename from update_release_notes.py rename to scripts/update_release_notes.py