mirror of
https://github.com/AvdLee/Xcode-Build-Optimization-Agent-Skill.git
synced 2026-09-14 13:59:59 +08:00
Antoine van der Lee
d977023cb1
Standard clean builds show cache-population overhead without the corresponding cache-hit benefit, making COMPILATION_CACHING appear to regress performance. The new "cached clean" phase measures clean builds with a warm compilation cache by deleting DerivedData (not the system compilation cache) between runs. This captures the realistic developer workflow: branch switching, pulling changes, and Clean Build Folder. Changes: - benchmark_builds.py: auto-detect COMPILATION_CACHING and run cached clean builds using a controlled DerivedData path - Schema bumped to 1.2.0 with optional cached_clean runs/summary - Report generator shows Cached Clean column when data is present - All skill docs and references updated to use the cached clean metric as the primary comparison for COMPILATION_CACHING impact
309 lines
12 KiB
Python
Executable File
309 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Benchmark Xcode clean and incremental builds.")
|
|
group = parser.add_mutually_exclusive_group(required=True)
|
|
group.add_argument("--workspace", help="Path to the .xcworkspace file")
|
|
group.add_argument("--project", help="Path to the .xcodeproj file")
|
|
parser.add_argument("--scheme", required=True, help="Scheme to build")
|
|
parser.add_argument("--configuration", default="Debug", help="Build configuration")
|
|
parser.add_argument("--destination", help="xcodebuild destination string")
|
|
parser.add_argument("--derived-data-path", help="DerivedData path override")
|
|
parser.add_argument("--output-dir", default=".build-benchmark", help="Output directory for artifacts")
|
|
parser.add_argument("--repeats", type=int, default=3, help="Measured runs per build type")
|
|
parser.add_argument("--skip-warmup", action="store_true", help="Skip the validation build")
|
|
parser.add_argument(
|
|
"--touch-file",
|
|
help="Path to a source file to touch before each incremental build. "
|
|
"When provided, measures a real edit-rebuild loop instead of a zero-change build.",
|
|
)
|
|
parser.add_argument(
|
|
"--no-cached-clean",
|
|
action="store_true",
|
|
help="Skip cached clean builds even when COMPILATION_CACHING is detected.",
|
|
)
|
|
parser.add_argument(
|
|
"--extra-arg",
|
|
action="append",
|
|
default=[],
|
|
help="Additional xcodebuild argument to append. Can be passed multiple times.",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def command_base(args: argparse.Namespace) -> List[str]:
|
|
command = ["xcodebuild"]
|
|
if args.workspace:
|
|
command.extend(["-workspace", args.workspace])
|
|
if args.project:
|
|
command.extend(["-project", args.project])
|
|
command.extend(["-scheme", args.scheme, "-configuration", args.configuration])
|
|
if args.destination:
|
|
command.extend(["-destination", args.destination])
|
|
if args.derived_data_path:
|
|
command.extend(["-derivedDataPath", args.derived_data_path])
|
|
command.extend(args.extra_arg)
|
|
return command
|
|
|
|
|
|
def shell_join(parts: List[str]) -> str:
|
|
return " ".join(subprocess.list2cmdline([part]) for part in parts)
|
|
|
|
|
|
_TASK_COUNT_RE = re.compile(r"^(.+?)\s*\((\d+)\s+tasks?\)$")
|
|
|
|
|
|
def _extract_task_count(name: str) -> tuple[str, Optional[int]]:
|
|
"""Split 'Category (N tasks)' into ('Category', N)."""
|
|
match = _TASK_COUNT_RE.match(name)
|
|
if match:
|
|
return match.group(1).strip(), int(match.group(2))
|
|
return name, None
|
|
|
|
|
|
def parse_timing_summary(output: str) -> List[Dict]:
|
|
categories: Dict[str, float] = {}
|
|
task_counts: Dict[str, Optional[int]] = {}
|
|
for raw_line in output.splitlines():
|
|
line = raw_line.strip()
|
|
if not line:
|
|
continue
|
|
for suffix in (" seconds", " second", " sec"):
|
|
if not line.endswith(suffix):
|
|
continue
|
|
trimmed = line[: -len(suffix)]
|
|
if "|" in trimmed:
|
|
name_part, _, seconds_text = trimmed.rpartition("|")
|
|
else:
|
|
name_part, _, seconds_text = trimmed.rpartition(" ")
|
|
try:
|
|
seconds = float(seconds_text.strip())
|
|
except ValueError:
|
|
continue
|
|
cleaned_name = name_part.replace(" ", " ").strip(" -:")
|
|
if len(cleaned_name) < 3:
|
|
continue
|
|
base_name, count = _extract_task_count(cleaned_name)
|
|
categories[base_name] = categories.get(base_name, 0.0) + seconds
|
|
if count is not None:
|
|
task_counts[base_name] = (task_counts.get(base_name) or 0) + count
|
|
break
|
|
result: List[Dict] = []
|
|
for name, seconds in sorted(categories.items(), key=lambda item: item[1], reverse=True):
|
|
entry: Dict = {"name": name, "seconds": round(seconds, 3)}
|
|
if name in task_counts:
|
|
entry["task_count"] = task_counts[name]
|
|
result.append(entry)
|
|
return result
|
|
|
|
|
|
def run_command(command: List[str]) -> subprocess.CompletedProcess:
|
|
return subprocess.run(command, capture_output=True, text=True)
|
|
|
|
|
|
def stats_for(runs: List[Dict[str, object]]) -> Dict[str, float]:
|
|
durations = [run["duration_seconds"] for run in runs if run.get("success")]
|
|
if not durations:
|
|
return {
|
|
"count": 0,
|
|
"min_seconds": 0.0,
|
|
"max_seconds": 0.0,
|
|
"median_seconds": 0.0,
|
|
"average_seconds": 0.0,
|
|
}
|
|
return {
|
|
"count": len(durations),
|
|
"min_seconds": round(min(durations), 3),
|
|
"max_seconds": round(max(durations), 3),
|
|
"median_seconds": round(statistics.median(durations), 3),
|
|
"average_seconds": round(statistics.fmean(durations), 3),
|
|
}
|
|
|
|
|
|
def xcode_version() -> str:
|
|
result = run_command(["xcodebuild", "-version"])
|
|
return result.stdout.strip() if result.returncode == 0 else "unknown"
|
|
|
|
|
|
def detect_compilation_caching(base_command: List[str]) -> bool:
|
|
"""Check whether COMPILATION_CACHING is enabled in the resolved build settings."""
|
|
result = run_command([*base_command, "-showBuildSettings"])
|
|
if result.returncode != 0:
|
|
return False
|
|
for line in result.stdout.splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith("COMPILATION_CACHING") and "=" in stripped:
|
|
value = stripped.split("=", 1)[1].strip()
|
|
return value == "YES"
|
|
return False
|
|
|
|
|
|
def measure_build(
|
|
base_command: List[str],
|
|
artifact_stem: str,
|
|
output_dir: Path,
|
|
build_type: str,
|
|
run_index: int,
|
|
) -> Dict[str, object]:
|
|
build_command = [*base_command, "build", "-showBuildTimingSummary"]
|
|
started = time.perf_counter()
|
|
result = run_command(build_command)
|
|
elapsed = round(time.perf_counter() - started, 3)
|
|
log_path = output_dir / f"{artifact_stem}-{build_type}-{run_index}.log"
|
|
log_path.write_text(result.stdout + result.stderr)
|
|
return {
|
|
"id": f"{build_type}-{run_index}",
|
|
"build_type": build_type,
|
|
"duration_seconds": elapsed,
|
|
"success": result.returncode == 0,
|
|
"exit_code": result.returncode,
|
|
"command": shell_join(build_command),
|
|
"raw_log_path": str(log_path),
|
|
"timing_summary_categories": parse_timing_summary(result.stdout + result.stderr),
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
output_dir = Path(args.output_dir)
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
artifact_stem = f"{timestamp}-{args.scheme.replace(' ', '-').lower()}"
|
|
base_command = command_base(args)
|
|
|
|
if not args.skip_warmup:
|
|
warmup = run_command([*base_command, "build"])
|
|
if warmup.returncode != 0:
|
|
sys.stderr.write(warmup.stdout + warmup.stderr)
|
|
return warmup.returncode
|
|
warmup_clean = run_command([*base_command, "clean"])
|
|
if warmup_clean.returncode != 0:
|
|
sys.stderr.write(warmup_clean.stdout + warmup_clean.stderr)
|
|
return warmup_clean.returncode
|
|
warmup_rebuild = run_command([*base_command, "build"])
|
|
if warmup_rebuild.returncode != 0:
|
|
sys.stderr.write(warmup_rebuild.stdout + warmup_rebuild.stderr)
|
|
return warmup_rebuild.returncode
|
|
|
|
runs: Dict[str, list] = {"clean": [], "incremental": []}
|
|
|
|
for index in range(1, args.repeats + 1):
|
|
clean_result = run_command([*base_command, "clean"])
|
|
clean_log_path = output_dir / f"{artifact_stem}-clean-prep-{index}.log"
|
|
clean_log_path.write_text(clean_result.stdout + clean_result.stderr)
|
|
if clean_result.returncode != 0:
|
|
sys.stderr.write(clean_result.stdout + clean_result.stderr)
|
|
return clean_result.returncode
|
|
runs["clean"].append(measure_build(base_command, artifact_stem, output_dir, "clean", index))
|
|
|
|
# --- Cached clean builds ---------------------------------------------------
|
|
# When COMPILATION_CACHING is enabled, the compilation cache lives outside
|
|
# DerivedData and survives product deletion. We measure "cached clean"
|
|
# builds by pointing DerivedData at a temp directory, warming the cache with
|
|
# one build, then deleting the DerivedData directory (but not the cache)
|
|
# before each measured rebuild. This captures the realistic scenario:
|
|
# branch switching, pulling changes, or Clean Build Folder.
|
|
should_cached_clean = not args.no_cached_clean and detect_compilation_caching(base_command)
|
|
if should_cached_clean:
|
|
dd_path = Path(args.derived_data_path) if args.derived_data_path else Path(
|
|
tempfile.mkdtemp(prefix="xcode-bench-dd-")
|
|
)
|
|
cached_cmd = list(base_command)
|
|
if not args.derived_data_path:
|
|
cached_cmd.extend(["-derivedDataPath", str(dd_path)])
|
|
|
|
cache_warmup = run_command([*cached_cmd, "build"])
|
|
if cache_warmup.returncode != 0:
|
|
sys.stderr.write("Warning: cached clean warmup build failed, skipping cached clean benchmarks.\n")
|
|
sys.stderr.write(cache_warmup.stdout + cache_warmup.stderr)
|
|
should_cached_clean = False
|
|
|
|
if should_cached_clean:
|
|
runs["cached_clean"] = []
|
|
for index in range(1, args.repeats + 1):
|
|
shutil.rmtree(dd_path, ignore_errors=True)
|
|
runs["cached_clean"].append(
|
|
measure_build(cached_cmd, artifact_stem, output_dir, "cached-clean", index)
|
|
)
|
|
shutil.rmtree(dd_path, ignore_errors=True)
|
|
|
|
# --- Incremental / zero-change builds --------------------------------------
|
|
incremental_label = "incremental"
|
|
if args.touch_file:
|
|
touch_path = Path(args.touch_file)
|
|
if not touch_path.exists():
|
|
sys.stderr.write(f"--touch-file path does not exist: {touch_path}\n")
|
|
return 1
|
|
incremental_label = "incremental"
|
|
else:
|
|
incremental_label = "zero-change"
|
|
|
|
for index in range(1, args.repeats + 1):
|
|
if args.touch_file:
|
|
touch_path.touch()
|
|
runs["incremental"].append(
|
|
measure_build(base_command, artifact_stem, output_dir, incremental_label, index)
|
|
)
|
|
|
|
summary: Dict[str, object] = {
|
|
"clean": stats_for(runs["clean"]),
|
|
"incremental": stats_for(runs["incremental"]),
|
|
}
|
|
if "cached_clean" in runs:
|
|
summary["cached_clean"] = stats_for(runs["cached_clean"])
|
|
|
|
artifact = {
|
|
"schema_version": "1.2.0" if "cached_clean" in runs else "1.1.0",
|
|
"created_at": datetime.now(timezone.utc).isoformat(),
|
|
"build": {
|
|
"entrypoint": "workspace" if args.workspace else "project",
|
|
"path": args.workspace or args.project,
|
|
"scheme": args.scheme,
|
|
"configuration": args.configuration,
|
|
"destination": args.destination or "",
|
|
"derived_data_path": args.derived_data_path or "",
|
|
"command": shell_join(base_command),
|
|
},
|
|
"environment": {
|
|
"host": platform.node(),
|
|
"macos_version": platform.platform(),
|
|
"xcode_version": xcode_version(),
|
|
"cwd": os.getcwd(),
|
|
},
|
|
"runs": runs,
|
|
"summary": summary,
|
|
"notes": [f"touch-file: {args.touch_file}"] if args.touch_file else [],
|
|
}
|
|
|
|
artifact_path = output_dir / f"{artifact_stem}.json"
|
|
artifact_path.write_text(json.dumps(artifact, indent=2) + "\n")
|
|
|
|
print(f"Saved benchmark artifact: {artifact_path}")
|
|
print(f"Clean median: {artifact['summary']['clean']['median_seconds']}s")
|
|
if "cached_clean" in artifact["summary"]:
|
|
print(f"Cached clean median: {artifact['summary']['cached_clean']['median_seconds']}s")
|
|
inc_label = "Incremental" if args.touch_file else "Zero-change"
|
|
print(f"{inc_label} median: {artifact['summary']['incremental']['median_seconds']}s")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|