CI: improve more ci worflow

This commit is contained in:
mahee96
2026-02-24 00:41:33 +05:30
parent 97ee0b2dac
commit e55351dbb0
2 changed files with 281 additions and 156 deletions

View File

@@ -23,87 +23,103 @@ jobs:
- run: brew install ldid xcbeautify
- name: Restore Xcode/SwiftPM Cache (Exact match)
id: xcode-cache-restore
uses: actions/cache/restore@v3
with:
path: |
~/Library/Developer/Xcode/DerivedData
~/Library/Caches/org.swift.swiftpm
key: xcode-cache-build-${{ github.ref_name }}-${{ github.sha }}
- name: Restore Xcode/SwiftPM Cache (Last Available)
uses: actions/cache/restore@v3
with:
path: |
~/Library/Developer/Xcode/DerivedData
~/Library/Caches/org.swift.swiftpm
key: xcode-cache-build-${{ github.ref_name }}-
# --------------------------------------------------
# runtime env setup
# --------------------------------------------------
- uses: actions/checkout@v4
if: ${{ inputs.is_beta }}
with:
repository: 'SideStore/beta-build-num'
ref: ${{ env.ref }}
token: ${{ secrets.CROSS_REPO_PUSH_KEY }}
path: 'Dependencies/beta-build-num'
fetch-depth: 1
- name: Short Commit SHA
- name: Setup
run: |
echo "SHORT_COMMIT=$(python3 scripts/ci/workflow.py commid-id)" >> $GITHUB_ENV
BUILD_NUM=$(python3 scripts/ci/workflow.py reserve_build_number 'Dependencies/beta-build-num')
MARKETING_VERSION=$(python3 scripts/ci/workflow.py get-marketing-version)
SHORT_COMMIT=$(python3 scripts/ci/workflow.py commid-id)
- name: Version
run: |
echo "VERSION=$(python3 scripts/ci/workflow.py version)" >> $GITHUB_ENV
QUALIFIED_VERSION=$(python3 scripts/ci/workflow.py compute-qualified \
"$MARKETING_VERSION" \
"$BUILD_NUM" \
"${{ env.ref }}" \
"$SHORT_COMMIT")
echo "BUILD_NUM=$BUILD_NUM" >> $GITHUB_ENV
echo "MARKETING_VERSION=$MARKETING_VERSION" >> $GITHUB_ENV
echo "SHORT_COMMIT=$SHORT_COMMIT" >> $GITHUB_ENV
echo "VERSION=$QUALIFIED_VERSION" >> $GITHUB_ENV
- name: Restore Cache
id: xcode-cache
uses: actions/cache/restore@v3
with:
path: |
~/Library/Developer/Xcode/DerivedData
~/Library/Caches/org.swift.swiftpm
key: xcode-build-cache-${{ github.ref_name }}-${{ github.sha }}
restore-keys: |
xcode-build-cache-${{ github.ref_name }}-
# --------------------------------------------------
# build and test
# --------------------------------------------------
- name: Clean previous build artifacts
- name: Clean
if: contains(github.event.head_commit.message, '[--clean-build]')
run: |
python3 scripts/ci/workflow.py clean
python3 scripts/ci/workflow.py clean-derived-data
python3 scripts/ci/workflow.py clean-spm-cache
- name: Boot simulator (async)
run: |
mkdir -p build/logs
python3 scripts/ci/workflow.py boot-sim-async "iPhone 17 Pro"
- name: Build
run: python3 scripts/ci/workflow.py build
id: build
env:
BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}
run: |
python3 scripts/ci/workflow.py build; STATUS=$?
python3 scripts/ci/workflow.py encrypt-build
echo "encrypted=true" >> $GITHUB_OUTPUT
exit $STATUS
- name: Tests Build
id: test-build
if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_BUILD == '1' }}
run: python3 scripts/ci/workflow.py tests-build
env:
BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}
run: |
python3 scripts/ci/workflow.py tests-build; STATUS=$?
python3 scripts/ci/workflow.py encrypt-tests-build
exit $STATUS
- name: Save Xcode & SwiftPM Cache
if: ${{ steps.xcode-cache-restore.outputs.cache-hit != 'true' }}
- name: Save Cache
if: ${{ steps.xcode-cache.outputs.cache-hit != 'true' }}
uses: actions/cache/save@v3
with:
path: |
~/Library/Developer/Xcode/DerivedData
~/Library/Caches/org.swift.swiftpm
key: xcode-cache-build-${{ github.ref_name }}-${{ github.sha }}
key: xcode-build-cache-${{ github.ref_name }}-${{ github.sha }}
- name: Tests Run
if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_RUN == '1' }}
run: python3 scripts/ci/workflow.py tests-run
- name: Encrypt build logs
env:
BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}
run: python3 scripts/ci/workflow.py encrypt-build
- name: Encrypt tests-build logs
if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_BUILD == '1' }}
env:
BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}
run: python3 scripts/ci/workflow.py encrypt-tests-build
- name: Encrypt tests-run logs
id: test-run
if: ${{ vars.ENABLE_TESTS == '1' && vars.ENABLE_TESTS_RUN == '1' }}
env:
BUILD_LOG_ZIP_PASSWORD: ${{ secrets.BUILD_LOG_ZIP_PASSWORD }}
run: python3 scripts/ci/workflow.py encrypt-tests-run
run: |
python3 scripts/ci/workflow.py tests-run "iPhone 17 Pro"; STATUS=$?
python3 scripts/ci/workflow.py encrypt-tests-run
exit $STATUS
# --------------------------------------------------
# artifacts
# --------------------------------------------------
- uses: actions/upload-artifact@v4
with:
name: encrypted-build-logs-${{ env.VERSION }}.zip
@@ -125,14 +141,19 @@ jobs:
with:
name: SideStore-${{ env.VERSION }}.ipa
path: SideStore.ipa
- uses: actions/upload-artifact@v4
with:
name: SideStore-${{ env.VERSION }}-dSYMs.zip
path: SideStore.dSYMs.zip
# --------------------------------------------------
# deploy
# --------------------------------------------------
- name: Deploy
run: |
python3 scripts/ci/workflow.py deploy nightly $SHORT_COMMIT
python3 scripts/ci/workflow.py deploy \
Dependencies/apps-v2.json \
nightly \
"$SHORT_COMMIT" \
"$MARKETING_VERSION" \
"$VERSION"

View File

@@ -4,83 +4,114 @@ import sys
import subprocess
import datetime
from pathlib import Path
import time
import json
# REPO ROOT relative to script dir
ROOT = Path(__file__).resolve().parents[2]
# ----------------------------------------------------------
# helpers
# ----------------------------------------------------------
def run(cmd, check=True):
def run(cmd, check=True, cwd=None):
wd = cwd if cwd is not None else ROOT
print(f"$ {cmd}", flush=True)
subprocess.run(cmd, shell=True, cwd=ROOT, check=check)
subprocess.run(cmd, shell=True, cwd=wd, check=check)
print("", flush=True)
def runAndGet(cmd, cwd=None):
wd = cwd if cwd is not None else ROOT
print(f"$ {cmd}", flush=True)
out = subprocess.check_output(
cmd,
shell=True,
cwd=wd,
text=True,
).strip()
print(out, flush=True)
print("", flush=True)
return out
def getenv(name, default=""):
return os.environ.get(name, default)
# ----------------------------------------------------------
# SHARED
# ----------------------------------------------------------
def short_commit():
sha = subprocess.check_output(
"git rev-parse --short HEAD",
shell=True,
cwd=ROOT
).decode().strip()
return sha
return runAndGet("git rev-parse --short HEAD")
# ----------------------------------------------------------
# VERSION BUMP
# BUILD NUMBER RESERVATION
# ----------------------------------------------------------
def bump_beta():
date = datetime.datetime.now(datetime.UTC).strftime("%Y.%m.%d")
release_channel = getenv("RELEASE_CHANNEL", "beta")
build_file = ROOT / "build_number.txt"
def reserve_build_number(repo, max_attempts=5):
repo = Path(repo).resolve()
version_json = repo / "version.json"
short = subprocess.check_output(
"git rev-parse --short HEAD",
shell=True,
cwd=ROOT
).decode().strip()
def utc_now():
return datetime.datetime.now(datetime.UTC)\
.strftime("%Y-%m-%dT%H:%M:%SZ")
def write(num):
def read():
if not version_json.exists():
return {"build": 0, "issued_at": utc_now()}
return json.loads(version_json.read_text())
def write(data):
version_json.write_text(json.dumps(data, indent=2) + "\n")
for _ in range(max_attempts):
run("git pull --rebase", check=False, cwd=repo)
data = read()
data["build"] += 1
data["issued_at"] = utc_now()
write(data)
run("git add version.json", check=False, cwd=repo)
run(
f"""sed -e "/MARKETING_VERSION = .*/s/$/-{release_channel}.{date}.{num}+{short}/" -i '' {ROOT}/Build.xcconfig"""
f"git commit -m '{data.get('tag','build')} build no - {data['build']}' || true",
check=False,
cwd=repo,
)
build_file.write_text(f"{date},{num}")
if not build_file.exists():
write(1)
return
rc = subprocess.call("git push", shell=True, cwd=repo)
last = build_file.read_text().strip().split(",")[1]
write(int(last) + 1)
if rc == 0:
print(f"Reserved build #{data['build']}")
return data["build"]
print("Push rejected, retrying...")
time.sleep(2)
raise SystemExit("Failed reserving build number")
# ----------------------------------------------------------
# VERSION EXTRACTION
# MARKETING VERSION
# ----------------------------------------------------------
def extract_version():
v = subprocess.check_output(
"grep MARKETING_VERSION Build.xcconfig | sed -e 's/MARKETING_VERSION = //g'",
shell=True,
cwd=ROOT
).decode().strip()
return v
def get_marketing_version():
return runAndGet("grep MARKETING_VERSION Build.xcconfig | sed -e 's/MARKETING_VERSION = //g'")
def set_marketing_version(qualified):
run(
f"sed -E "
f"'s/^MARKETING_VERSION = .*/MARKETING_VERSION = {qualified}/' "
f"-i '' {ROOT}/Build.xcconfig"
)
def compute_qualified_version(marketing, build_num, channel, short):
date = datetime.datetime.now(datetime.UTC).strftime("%Y.%m.%d")
return f"{marketing}-{channel}.{date}.{build_num}+{short}"
# ----------------------------------------------------------
# CLEAN
# ----------------------------------------------------------
def clean():
run("make clean")
@@ -107,10 +138,8 @@ def build():
run("make fakesign | tee -a build/logs/build.log")
run("make ipa | tee -a build/logs/build.log")
run("zip -r -9 ./SideStore.dSYMs.zip ./SideStore.xcarchive/dSYMs")
# ----------------------------------------------------------
# TESTS BUILD
# ----------------------------------------------------------
@@ -122,117 +151,192 @@ def tests_build():
"2>&1 | tee -a build/logs/tests-build.log | xcbeautify --renderer github-actions"
)
# ----------------------------------------------------------
# TESTS RUN
# ----------------------------------------------------------
def tests_run():
run("mkdir -p build/logs")
run("nohup make -B boot-sim-async </dev/null >> build/logs/tests-run.log 2>&1 &")
def is_sim_booted(model):
out = runAndGet(f'xcrun simctl list devices "{model}"')
return "Booted" in out
run("make -B sim-boot-check | tee -a build/logs/tests-run.log")
def boot_sim_async(model):
log = ROOT / "build/logs/tests-run.log"
log.parent.mkdir(parents=True, exist_ok=True)
if is_sim_booted(model):
run(f'echo "Simulator {model} already booted." | tee -a {log}')
return
run(f'echo "Booting simulator {model} asynchronously..." | tee -a {log}')
with open(log, "a") as f:
subprocess.Popen(
["xcrun", "simctl", "boot", model],
cwd=ROOT,
stdout=f,
stderr=subprocess.STDOUT,
start_new_session=True,
)
def boot_sim_sync(model):
run("mkdir -p build/logs")
for i in range(1, 7):
if is_sim_booted(model):
run('echo "Simulator booted." | tee -a build/logs/tests-run.log')
return
run(f'echo "Simulator not ready (attempt {i}/6), retrying in 10s..." | tee -a build/logs/tests-run.log')
time.sleep(10)
raise SystemExit("Simulator failed to boot")
def tests_run(model):
run("mkdir -p build/logs")
if not is_sim_booted(model):
boot_sim_sync(model)
run("make run-tests 2>&1 | tee -a build/logs/tests-run.log")
run("zip -r -9 ./test-results.zip ./build/tests")
# ----------------------------------------------------------
# LOG ENCRYPTION
# ----------------------------------------------------------
def encrypt_logs(name):
pwd = getenv("BUILD_LOG_ZIP_PASSWORD", "12345")
run(
f'cd build/logs && zip -e -P "{pwd}" ../../{name}.zip *'
)
default_pwd = "12345"
pwd = getenv("BUILD_LOG_ZIP_PASSWORD", default_pwd)
if pwd == default_pwd:
print("Warning: BUILD_LOG_ZIP_PASSWORD not set, using fallback password")
run(f'cd build/logs && zip -e -P "{pwd}" ../../{name}.zip *')
# ----------------------------------------------------------
# RELEASE NOTES
# ----------------------------------------------------------
def release_notes(tag):
run(f"python3 generate_release_notes.py {tag}")
# ----------------------------------------------------------
# PUBLISH SOURCE.JSON
# ----------------------------------------------------------
def publish_apps(release_tag, short_commit):
repo = ROOT / "Dependencies/apps-v2.json"
if not repo.exists():
raise SystemExit("Dependencies/apps-v2.json repo missing")
# generate metadata + release notes
run(
f"python3 generate_source_metadata.py "
f"--release-tag {release_tag} "
f"--short-commit {short_commit}"
f"python3 generate_release_notes.py "
f"{tag} "
f"--repo-root {ROOT} "
f"--output-dir {ROOT}"
)
# update source.json using generated metadata
run("pushd Dependencies/apps-v2.json", check=False)
# ----------------------------------------------------------
# DEPLOY SOURCE.JSON
# ----------------------------------------------------------
run("git config user.name 'GitHub Actions'", check=False)
run("git config user.email 'github-actions@github.com'", check=False)
def deploy(repo, release_tag, short_commit, marketing_version, version):
repo = Path(repo).resolve()
run("python3 ../../scripts/update_source_metadata.py './_includes/source.json'")
if not repo.exists():
raise SystemExit(f"{repo} repo missing")
run("git add --verbose ./_includes/source.json", check=False)
run(f"git commit -m ' - updated for {short_commit} deployment' || true",check=False)
run("git push --verbose", check=False)
run(
f"python3 generate_source_metadata.py "
f"--repo-root {ROOT} "
f"--ipa {ROOT}/SideStore.ipa "
f"--output-dir {repo} "
f"--release-notes-dir {repo} "
f"--release-tag {release_tag} "
f"--version {version} "
f"--marketing-version {marketing_version} "
f"--short-commit {short_commit} "
f"--release-channel nightly "
f"--bundle-id com.SideStore.SideStore"
)
try:
run("git config user.name 'GitHub Actions'", check=False, cwd=repo)
run("git config user.email 'github-actions@github.com'", check=False, cwd=repo)
run(
"python3 ../../scripts/update_source_metadata.py './_includes/source.json'",
cwd=repo,
)
run("git add --verbose ./_includes/source.json", check=False, cwd=repo)
run(
f"git commit -m ' - updated for {short_commit} deployment' || true",
check=False,
cwd=repo,
)
run("git push --verbose", check=False, cwd=repo)
finally:
pass
run("popd", check=False)
# ----------------------------------------------------------
# ENTRYPOINT
# ----------------------------------------------------------
COMMANDS = {
"commid-id" : (short_commit, 0, ""),
"bump-beta" : (bump_beta, 0, ""),
"version" : (extract_version, 0, ""),
"clean" : (clean, 0, ""),
"clean-derived-data" : (clean_derived_data, 0, ""),
"clean-spm-cache" : (clean_spm_cache, 0, ""),
"build" : (build, 0, ""),
"tests-build" : (tests_build, 0, ""),
"tests-run" : (tests_run, 0, ""),
"encrypt-build" : (lambda: encrypt_logs("encrypted-build-logs"), 0, ""),
"encrypt-tests-build": (lambda: encrypt_logs("encrypted-tests-build-logs"), 0, ""),
"encrypt-tests-run" : (lambda: encrypt_logs("encrypted-tests-run-logs"), 0, ""),
"release-notes" : (release_notes, 1, "<tag>"),
"deploy" : (publish_apps, 2, "<release_tag> <short_commit>"),
# ----------------------------------------------------------
# SHARED
# ----------------------------------------------------------
"commid-id" : (short_commit, 0, ""),
# ----------------------------------------------------------
# VERSION / MARKETING
# ----------------------------------------------------------
"bump-beta" : (bump_beta, 0, ""),
"version" : (get_marketing_version, 0, ""),
"set-marketing-version" : (set_marketing_version, 1, "<qualified_version>"),
"compute-qualified" : (compute_qualified_version, 4, "<marketing> <build_num> <channel> <short_commit>"),
"reserve_build_number" : (reserve_build_number, 1, "<repo>"),
# ----------------------------------------------------------
# CLEAN
# ----------------------------------------------------------
"clean" : (clean, 0, ""),
"clean-derived-data" : (clean_derived_data, 0, ""),
"clean-spm-cache" : (clean_spm_cache, 0, ""),
# ----------------------------------------------------------
# BUILD
# ----------------------------------------------------------
"build" : (build, 0, ""),
# ----------------------------------------------------------
# TESTS
# ----------------------------------------------------------
"tests-build" : (tests_build, 0, ""),
"tests-run" : (tests_run, 1, "<model>"),
"boot-sim-async" : (boot_sim_async, 1, "<model>"),
"boot-sim-sync" : (boot_sim_sync, 0, ""),
# ----------------------------------------------------------
# LOG ENCRYPTION
# ----------------------------------------------------------
"encrypt-build" : (lambda: encrypt_logs("encrypted-build-logs"), 0, ""),
"encrypt-tests-build" : (lambda: encrypt_logs("encrypted-tests-build-logs"), 0, ""),
"encrypt-tests-run" : (lambda: encrypt_logs("encrypted-tests-run-logs"), 0, ""),
# ----------------------------------------------------------
# RELEASE / DEPLOY
# ----------------------------------------------------------
"release-notes" : (release_notes, 1, "<tag>"),
"deploy" : (deploy, 5, "<repo> <release_tag> <short_commit> <marketing_version> <version>"),
}
def main():
def usage():
lines = ["Available commands:"]
for name, (_, argc, arg_usage) in COMMANDS.items():
suffix = f" {arg_usage}" if arg_usage else ""
lines.append(f" - {name}{suffix}")
return "\n".join(lines)
if len(sys.argv) < 2:
raise SystemExit(usage())
raise SystemExit("No command")
cmd = sys.argv[1]
if cmd not in COMMANDS:
raise SystemExit(
f"Unknown command '{cmd}'.\n\n{usage()}"
)
raise SystemExit(f"Unknown command '{cmd}'")
func, argc, arg_usage = COMMANDS[cmd]
if len(sys.argv) - 2 < argc:
suffix = f" {arg_usage}" if arg_usage else ""
raise SystemExit(f"Usage: workflow.py {cmd}{suffix}")
func, argc, _ = COMMANDS[cmd]
args = sys.argv[2:2 + argc]
func(*args) if argc else func()
result = func(*args) if argc else func()
if result is not None:
print(result)
if __name__ == "__main__":
main()