1#!/usr/bin/env python32"""Prepare immutable bare Git benchmark fixtures and inventory corpus metadata.34Usage: prepare-git-performance.py --source-directory ~/Developer --out /tmp/tinyrelay-corpus [--repos a,b]5Every Git repository directly under the source directory is selected unless6--repos names a subset. The output directory must not already exist. Each selected repository is cloned7with --bare --no-local; the source working trees are never modified.8"""9from __future__ import annotations10import argparse, json, os, shutil, subprocess, tempfile11from pathlib import Path1213def is_repository(path: Path) -> bool:14 return (path / ".git").exists() or (path / "HEAD").exists()1516def select(source: Path, names: list[str] | None) -> list[str]:17 if names: return names18 return sorted(p.name for p in source.iterdir() if p.is_dir() and is_repository(p))1920def run(*args: str, cwd: Path | None = None, input: str | None = None) -> str:21 p = subprocess.run(args, cwd=cwd, input=input, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)22 if p.returncode:23 raise RuntimeError(f"{' '.join(args)}: {p.stderr.strip()}")24 return p.stdout.strip()2526def inventory(repo: Path, name: str) -> dict:27 head_branch = run("git", "--git-dir", str(repo), "symbolic-ref", "HEAD")28 head = run("git", "--git-dir", str(repo), "rev-parse", head_branch)29 refs_lines = run("git", "--git-dir", str(repo), "for-each-ref", "--format=%(refname) %(objectname)").splitlines()30 tags = [line for line in refs_lines if line.startswith("refs/tags/")]31 head_commits = int(run("git", "--git-dir", str(repo), "rev-list", "--count", head_branch))32 all_commits = int(run("git", "--git-dir", str(repo), "rev-list", "--all", "--count"))33 stats = {}34 for line in run("git", "--git-dir", str(repo), "count-objects", "-v").splitlines():35 k, _, v = line.partition(" ")36 if k and v.isdigit(): stats[k] = int(v)37 object_ids = run("git", "--git-dir", str(repo), "rev-list", "--objects", "--all").splitlines()38 ids = [line.split()[0] for line in object_ids]39 objects = {}40 object_bytes = 041 checked = run("git", "--git-dir", str(repo), "cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype) %(objectsize)")42 sizes = {}43 for line in checked.splitlines():44 fields = line.split()45 if len(fields) != 3: continue46 oid, kind, size = fields[0], fields[1], int(fields[2])47 objects[kind] = objects.get(kind, 0) + 148 object_bytes += size49 sizes[oid] = (kind, size)50 blob_sizes = []51 for line in object_ids:52 fields = line.split(maxsplit=1)53 if len(fields) == 2 and fields[0] in sizes and sizes[fields[0]][0] == "blob":54 blob_sizes.append((sizes[fields[0]][1], fields[1]))55 blob_sizes.sort(reverse=True)56 return {57 "name": name, "head": head, "head_branch": head_branch,58 "head_commits": head_commits, "all_commits": all_commits,59 "refs": len(refs_lines), "tags": len(tags),60 "objects": objects,61 "object_bytes": object_bytes,62 "pack_bytes": stats.get("size-pack", 0) * 1024,63 "largest_blobs": [{"bytes": size, "path": path} for size, path in blob_sizes[:10]],64 "blobs_over_1MiB": sum(size > 1024 * 1024 for size, _ in blob_sizes),65 "blobs_over_10MiB": sum(size > 10 * 1024 * 1024 for size, _ in blob_sizes),66 }6768def prepare(source: Path, out: Path, names: list[str] | None = None) -> list[dict]:69 if out.exists(): raise FileExistsError(f"refusing existing destination: {out}")70 selected = select(source, names)71 if not selected: raise FileNotFoundError(f"no repositories under {source}")72 out.mkdir(parents=True)73 rows = []74 try:75 for name in selected:76 src = source / name77 if not is_repository(src): raise FileNotFoundError(f"missing source repository: {src}")78 dest = out / (name + ".git")79 run("git", "clone", "--bare", "--no-local", str(src), str(dest))80 rows.append(inventory(dest, name))81 (out / "corpus.json").write_text(json.dumps(rows, indent=2) + "\n", encoding="utf-8")82 return rows83 except Exception:84 shutil.rmtree(out, ignore_errors=True)85 raise8687def self_test() -> None:88 with tempfile.TemporaryDirectory(prefix="prepare-git-performance-") as d:89 root, src, out = Path(d), Path(d) / "src", Path(d) / "out"90 src.mkdir(); run("git", "init", "-b", "trunk", str(src)); run("git", "-C", str(src), "config", "user.email", "test@example.invalid"); run("git", "-C", str(src), "config", "user.name", "test")91 (src / "tracked").write_text("tracked\n"); run("git", "-C", str(src), "add", "tracked"); run("git", "-C", str(src), "commit", "-m", "one"); run("git", "-C", str(src), "branch", "feature"); run("git", "-C", str(src), "tag", "v1")92 (src / "uncommitted").write_text("must not be copied\n")93 # Exercise prepare() itself without copying a real corpus: three94 # fixture names all point at the same tiny synthetic repository.95 names = ("alpha", "beta", "gamma")96 for name in names: os.symlink(src, root / name, target_is_directory=True)97 rows = prepare(root, out, list(names))98 assert len(rows) == len(names)99 row = rows[0]100 assert row["head_branch"] == "refs/heads/trunk" and row["head"] == run("git", "-C", str(src), "rev-parse", "HEAD")101 assert row["tags"] == 1 and row["all_commits"] == 1102 assert "uncommitted" not in run("git", "--git-dir", str(out / "alpha.git"), "ls-tree", "-r", "--name-only", "HEAD")103104if __name__ == "__main__":105 parser = argparse.ArgumentParser(description=__doc__)106 parser.add_argument("--source-directory", type=Path, default=Path.home() / "Developer")107 parser.add_argument("--out", type=Path)108 parser.add_argument("--repos", help="comma-separated repository directory names to select")109 parser.add_argument("--self-test", action="store_true")110 args = parser.parse_args()111 if args.self_test: self_test()112 else:113 if args.out is None: parser.error("--out is required unless --self-test is used")114 rows = prepare(args.source_directory.expanduser(), args.out.expanduser(), args.repos.split(",") if args.repos else None)115 print(json.dumps(rows, indent=2))