honestly its been months, i found unstaged changes and forgot

scripts/vendor-upstream-snapshots.py

@@ -22,6 +22,9 @@ from urllib.request import Request, urlopen
 ROOT = Path(__file__).resolve().parents[1]
 INVENTORY = ROOT / "docs" / "upstream-sources.yml"
 COMPOSE_BLOCK_LANGS = {"", "yaml", "yml", "docker-compose", "compose"}
+NORMALIZED_COMPOSE_NAME = "docker-compose.yml"
+DERIVED_TEMPLATE_MARKER = "nbops: derived-from-upstream"
+DERIVED_TEMPLATE_NAMES = ("compose.yaml.j2", "compose.override.yaml.j2")
 
 
 def parse_inline_list(value: str) -> list[str]:
@@ -137,6 +140,21 @@ def copy_if_present(src_root: Path, relative_path: str, dest_root: Path, apply:
     return "copied"
 
 
+def copy_compose_if_present(src_root: Path, relative_path: str, dest_root: Path, apply: bool) -> str:
+    src = src_root / relative_path
+    if not src.is_file():
+        return "missing"
+    if not looks_like_compose(src.read_text(errors="replace")):
+        return "invalid"
+    if not apply:
+        return "found"
+
+    dest = dest_root / NORMALIZED_COMPOSE_NAME
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(src, dest)
+    return "copied"
+
+
 def copy_local_file_if_present(relative_path: str, dest_root: Path, apply: bool) -> str:
     src = ROOT / relative_path
     if not src.is_file():
@@ -152,6 +170,17 @@ def copy_local_file_if_present(relative_path: str, dest_root: Path, apply: bool)
     return "copied"
 
 
+def replace_directory(src: Path, dest: Path) -> None:
+    if dest.exists():
+        shutil.rmtree(dest)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    shutil.move(str(src), str(dest))
+
+
+def preferred_override_exists(snapshot_dir: Path) -> bool:
+    return (snapshot_dir / "docker-compose.override.yml").is_file()
+
+
 def fetch_compose_url(url: str) -> tuple[str, str, str]:
     parsed = urlparse(url)
     if parsed.scheme not in {"http", "https"}:
@@ -173,12 +202,6 @@ def fetch_compose_url(url: str) -> tuple[str, str, str]:
     return "found", body, ""
 
 
-def compose_url_filename(index: int, total: int) -> str:
-    if total == 1:
-        return "compose.from-url.yaml"
-    return f"compose.from-url-{index}.yaml"
-
-
 def extract_fenced_blocks(markdown: str) -> list[tuple[str, str]]:
     pattern = re.compile(r"^```([^\n`]*)\n(.*?)^```", re.MULTILINE | re.DOTALL)
     blocks = []
@@ -226,6 +249,61 @@ def print_table(rows: list[list[str]]) -> None:
         print(fmt(row))
 
 
+def progress_enabled() -> bool:
+    return sys.stderr.isatty()
+
+
+def render_progress(current: int, total: int, label: str) -> None:
+    if not progress_enabled() or total <= 0:
+        return
+    width = 28
+    filled = min(width, int(width * current / total))
+    bar = "#" * filled + "-" * (width - filled)
+    sys.stderr.write(f"\r\033[K[{bar}] {current}/{total} {label}")
+    sys.stderr.flush()
+
+
+def clear_progress() -> None:
+    if progress_enabled():
+        sys.stderr.write("\r\033[K")
+        sys.stderr.flush()
+
+
+def snapshot_has_content(path: Path) -> bool:
+    return path.exists() and any(path.iterdir())
+
+
+def split_filter_values(values: list[str], csv_value: str) -> set[str]:
+    filters: set[str] = set()
+    for value in values:
+        filters.update(item.strip() for item in value.split(",") if item.strip())
+    filters.update(item.strip() for item in csv_value.split(",") if item.strip())
+    return filters
+
+
+def filtered_repositories(config: dict, upstream_filter: set[str]) -> list[dict]:
+    repos = config["repositories"]
+    if not upstream_filter:
+        return repos
+    return [repo for repo in repos if repo["purpose"] in upstream_filter]
+
+
+def clean_derived_templates(role: str) -> list[list[str]]:
+    rows: list[list[str]] = []
+    template_dir = ROOT / "roles" / role / "templates"
+    for name in DERIVED_TEMPLATE_NAMES:
+        path = template_dir / name
+        if not path.is_file():
+            continue
+        text = path.read_text(errors="replace")
+        if DERIVED_TEMPLATE_MARKER not in text:
+            rows.append([role, "template", str(path), "kept: unmarked"])
+            continue
+        path.unlink()
+        rows.append([role, "template", str(path), "removed"])
+    return rows
+
+
 def snapshot_source(vendor_dir: Path, role: str, status: str, notes: list[str], apply: bool) -> None:
     if not apply:
         return
@@ -246,11 +324,73 @@ def snapshot_source(vendor_dir: Path, role: str, status: str, notes: list[str],
     (vendor_dir / "SOURCE.md").write_text("\n".join(text))
 
 
+def clean_upstream_directories(selected: dict[str, dict], upstream_filter: set[str]) -> int:
+    rows: list[list[str]] = []
+    removed = 0
+    total = sum(len(filtered_repositories(config, upstream_filter)) for config in selected.values())
+    current = 0
+
+    for role, config in selected.items():
+        vendor_dir = ROOT / config["vendor_dir"]
+        repos = filtered_repositories(config, upstream_filter)
+        for repo in repos:
+            current += 1
+            purpose_dir = vendor_dir / repo["purpose"]
+            render_progress(current, total, f"clean {role}/{repo['purpose']}")
+            if purpose_dir.is_dir():
+                shutil.rmtree(purpose_dir)
+                removed += 1
+                result = "removed"
+            else:
+                result = "missing"
+            rows.append([role, repo["purpose"], str(purpose_dir), result])
+        rows.extend(clean_derived_templates(role))
+
+    clear_progress()
+    headers = ["Role", "Purpose", "Path", "Result"]
+    widths = [len(header) for header in headers]
+    for row in rows:
+        for index, value in enumerate(row):
+            widths[index] = max(widths[index], len(value))
+
+    def fmt(row: list[str]) -> str:
+        return "  ".join(value.ljust(widths[index]) for index, value in enumerate(row))
+
+    print(fmt(headers))
+    print("  ".join("-" * width for width in widths))
+    for row in rows:
+        print(fmt(row))
+    print(f"\nRemoved {removed} upstream purpose directories.")
+    return 0
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description="Test or vendor upstream README/Compose snapshots.")
     parser.add_argument("roles", nargs="*", help="Role names to process. Defaults to all roles.")
     parser.add_argument("--apply", action="store_true", help="Copy fetched files into role upstream directories.")
     parser.add_argument("--no-parse-readme", action="store_true", help="Do not extract Compose YAML from README blocks.")
+    parser.add_argument(
+        "--clean-upstreams",
+        action="store_true",
+        help="Delete upstream/<purpose> directories and marked derived templates for selected roles, then exit.",
+    )
+    parser.add_argument(
+        "--clean-purposes",
+        dest="clean_upstreams",
+        action="store_true",
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--upstream",
+        action="append",
+        default=[],
+        help="Limit processing to an upstream purpose name such as container or application. Repeatable.",
+    )
+    parser.add_argument(
+        "--upstreams",
+        default="",
+        help="Comma-separated upstream purpose names to process.",
+    )
     parser.add_argument("--list", action="store_true", help="List roles in the upstream inventory and exit.")
     parser.add_argument("--validate-inventory", action="store_true", help="Parse the inventory and exit without cloning.")
     args = parser.parse_args()
@@ -269,9 +409,22 @@ def main() -> int:
     if not selected:
         print(f"no matching roles found for: {', '.join(args.roles)}", file=sys.stderr)
         return 2
+    upstream_filter = split_filter_values(args.upstream, args.upstreams)
+    if upstream_filter:
+        known_upstreams = {repo["purpose"] for config in selected.values() for repo in config["repositories"]}
+        unknown_upstreams = sorted(upstream_filter - known_upstreams)
+        if unknown_upstreams:
+            print(f"no matching upstream purpose found for: {', '.join(unknown_upstreams)}", file=sys.stderr)
+            return 2
+    if args.clean_upstreams:
+        return clean_upstream_directories(selected, upstream_filter)
 
     rows: list[list[str]] = []
     failures: list[str] = []
+    total_steps = sum(len(filtered_repositories(config, upstream_filter)) for config in selected.values())
+    if not upstream_filter:
+        total_steps += sum(1 for config in selected.values() if config["local_compose_file"])
+    current_step = 0
 
     for role, config in selected.items():
         vendor_dir = ROOT / config["vendor_dir"]
@@ -285,20 +438,29 @@ def main() -> int:
         notes: list[str] = []
         snapshot_notes: list[str] = []
 
-        for repo in config["repositories"]:
+        for repo in filtered_repositories(config, upstream_filter):
+            current_step += 1
+            render_progress(current_step, total_steps, f"{role}/{repo['purpose']}")
             repo_count += 1
             with tempfile.TemporaryDirectory(prefix=f"nbops-upstream-{role}-") as tmp:
                 tmpdir = Path(tmp)
+                clone_dir = tmpdir / "repo"
+                purpose_dest = vendor_dir / repo["purpose"]
+                purpose_snapshot_dir = purpose_dest
+                if args.apply:
+                    purpose_snapshot_dir = tmpdir / "__snapshot__" / repo["purpose"]
+                    purpose_snapshot_dir.mkdir(parents=True, exist_ok=True)
+
                 ok = False
                 if "url" in repo:
-                    ok, output = run_capture(["git", "clone", "--depth", "1", "--branch", repo["ref"], repo["url"], str(tmpdir)])
+                    ok, output = run_capture(["git", "clone", "--depth", "1", "--branch", repo["ref"], repo["url"], str(clone_dir)])
                     if not ok:
                         notes.append(f"{repo['purpose']}: clone failed")
                         failures.append(f"{role}/{repo['purpose']}: {output.splitlines()[-1] if output.splitlines() else 'clone failed'}")
 
                 if ok:
                     for path in repo["readme_paths"]:
-                        result = copy_if_present(tmpdir, path, vendor_dir / repo["purpose"], args.apply)
+                        result = copy_if_present(clone_dir, path, purpose_snapshot_dir, args.apply)
                         if result == "missing":
                             readme_missing += 1
                         else:
@@ -307,30 +469,38 @@ def main() -> int:
 
                         if args.no_parse_readme:
                             continue
-                        candidates = readme_compose_candidates(tmpdir / path)
+                        candidates = readme_compose_candidates(clone_dir / path)
                         readme_candidates += len(candidates)
                         if len(candidates) > 1:
                             readme_ambiguous += 1
                         if args.apply and len(candidates) == 1:
-                            dest = vendor_dir / repo["purpose"] / "compose.from-readme.yaml"
+                            dest = purpose_snapshot_dir / NORMALIZED_COMPOSE_NAME
                             dest.parent.mkdir(parents=True, exist_ok=True)
                             dest.write_text(candidates[0]["body"])
 
                     for path in repo["compose_paths"]:
-                        result = copy_if_present(tmpdir, path, vendor_dir / repo["purpose"], args.apply)
+                        result = copy_compose_if_present(clone_dir, path, purpose_snapshot_dir, args.apply)
                         if result == "missing":
                             compose_missing += 1
+                        elif result == "invalid":
+                            compose_missing += 1
+                            notes.append(f"{repo['purpose']}: Compose `{path}` invalid")
                         else:
                             compose_found += 1
                         snapshot_notes.append(f"- {repo['purpose']} Compose `{path}`: {result}")
 
                 compose_urls = repo["compose_urls"]
                 for index, url in enumerate(compose_urls, start=1):
+                    if args.apply and (purpose_snapshot_dir / NORMALIZED_COMPOSE_NAME).is_file():
+                        snapshot_notes.append(
+                            f"- {repo['purpose']} Compose URL `{url}`: skipped; `{NORMALIZED_COMPOSE_NAME}` already staged"
+                        )
+                        continue
                     result, body, reason = fetch_compose_url(url)
                     if result == "found":
                         compose_found += 1
                         if args.apply:
-                            dest = vendor_dir / repo["purpose"] / compose_url_filename(index, len(compose_urls))
+                            dest = purpose_snapshot_dir / NORMALIZED_COMPOSE_NAME
                             dest.parent.mkdir(parents=True, exist_ok=True)
                             dest.write_text(body)
                             result = "copied"
@@ -340,7 +510,21 @@ def main() -> int:
                     suffix = f" ({reason})" if reason else ""
                     snapshot_notes.append(f"- {repo['purpose']} Compose URL `{url}`: {result}{suffix}")
 
-        if config["local_compose_file"]:
+                if args.apply and snapshot_has_content(purpose_snapshot_dir):
+                    if repo["purpose"] == "container" and preferred_override_exists(purpose_snapshot_dir):
+                        notes.append("container override preferred")
+                        snapshot_notes.append(
+                            "- container `docker-compose.override.yml`: prefer this when updating `templates/compose.yaml.j2`"
+                        )
+                    replace_directory(purpose_snapshot_dir, purpose_dest)
+                elif args.apply:
+                    snapshot_notes.append(f"- {repo['purpose']} snapshot: no files staged; existing purpose directory left unchanged")
+                elif repo["purpose"] == "container" and preferred_override_exists(purpose_dest):
+                    notes.append("container override preferred")
+
+        if config["local_compose_file"] and not upstream_filter:
+            current_step += 1
+            render_progress(current_step, total_steps, f"{role}/local")
             result = copy_local_file_if_present(config["local_compose_file"], vendor_dir, args.apply)
             if result in {"missing", "invalid"}:
                 compose_missing += 1
@@ -371,6 +555,7 @@ def main() -> int:
             ]
         )
 
+    clear_progress()
     print_table(rows)
 
     if failures: