honestly its been months, i found unstaged changes and forgot

.gitignore

@@ -12,4 +12,7 @@
 secrets.yml
 vault.yml
 
+roles/*/upstream/container/
+roles/*/upstream/application/
+
 TODO.md

docs/.gitignore

@@ -0,0 +1 @@
+demo_files_do_not_delete/

docs/compose-update-policy.md

@@ -7,8 +7,7 @@ upstream files are reference snapshots only and must not be deployed directly.
 
 Each Docker-backed role has:
 
-- `templates/compose.yaml.j2`: base deployed Compose file
-- `templates/compose.override.yaml.j2`: NBOPS-specific local override
+- `templates/compose.yaml.j2`: deployed Compose template
 - `templates/env.j2`: generated environment file
 - `upstream/`: copied upstream reference files
 
@@ -49,19 +48,28 @@ Entries that use `readme_paths` or `compose_paths` must include both `url` and
 
    The script also looks for Compose-shaped fenced YAML blocks in README files.
    When exactly one candidate is found, `--apply` writes it as
-   `compose.from-readme.yaml`. Compose YAML fetched from `compose_urls` is
-   written as `compose.from-url.yaml`, or `compose.from-url-N.yaml` when a
-   repository has multiple fallback URLs. A role-level `local_compose_file` is
-   copied under `roles/<role>/upstream/local/`.
+   `docker-compose.yml` inside that upstream purpose directory. Compose YAML
+   copied from `compose_paths` or fetched from `compose_urls` is also normalized
+   to `docker-compose.yml`. URL sources are fallback material and do not
+   overwrite a Compose file already staged from the repository or README parse.
+   A role-level `local_compose_file` is copied under `roles/<role>/upstream/local/`.
+
+   Repository snapshots are written under `roles/<role>/upstream/<purpose>/`.
+   On `--apply`, each `upstream/<purpose>/` directory is replaced with a fresh
+   snapshot for that source, so stale README and Compose files are removed.
+   The script displays a single-line progress bar on interactive terminals while
+   processing sources.
 4. Update `roles/<role>/upstream/SOURCE.md` with source URL, ref, snapshot date,
    fetched paths, and any manual verification notes.
 5. Commit upstream snapshot changes separately from NBOPS deployment template
    changes.
 6. Diff upstream snapshots against current role templates.
-7. Apply upstream structural changes to `templates/compose.yaml.j2`.
-8. Keep NBOPS-local changes in `templates/compose.override.yaml.j2`, including:
-   Traefik labels, internal networks, host paths, ports, restart policy, logging,
-   resource limits, and deployment-specific environment wiring.
+7. Apply upstream structural changes to `templates/compose.yaml.j2`. If a
+   role-level `upstream/docker-compose.override.yml` exists, treat it as the
+   primary source for the template; otherwise use `upstream/docker-compose.yml`.
+8. Keep NBOPS-local changes in the deployed template, including Traefik labels,
+   internal networks, host paths, ports, restart policy, logging, resource
+   limits, and deployment-specific environment wiring.
 9. Replace mutable image tags with pinned versions where practical.
 10. Document breaking changes or operator actions in the role README.
 
@@ -70,6 +78,27 @@ Entries that use `readme_paths` or `compose_paths` must include both `url` and
 - Do not edit generated Compose files on VMs.
 - Do not deploy files from `upstream/` directly.
 - Do not treat `status: verify` sources as trusted until manually checked.
+- Treat `upstream/<purpose>/` as generated snapshot output; do not keep manual
+  edits there.
+- To remove generated purpose snapshots and any template files explicitly marked
+  with `nbops: derived-from-upstream`, run:
+
+  ```sh
+  scripts/vendor-upstream-snapshots.py --clean-upstreams <role>
+  ```
+
+  This deletes directories like `upstream/application/` and
+  `upstream/container/`, but leaves files such as `upstream/SOURCE.md` and
+  root-level override YAMLs alone. The old `--clean-purposes` flag is retained
+  as a compatibility alias.
+- To target a specific upstream purpose, add `--upstream`:
+
+  ```sh
+  scripts/vendor-upstream-snapshots.py --clean-upstreams --upstream container <role>
+  ```
+
+  `--upstream` may be repeated, or `--upstreams container,application` may be
+  used for comma-separated values.
 - Prefer pinned tags or digests over `latest` and other mutable tags.
 - Keep HTTP application ports unpublished in Traefik mode; document internal
   service ports as comments in generated Compose.

docs/proxmox-template.md

@@ -10,11 +10,67 @@ Expected VM defaults:
 - CPU type `host`
 - QEMU Guest Agent enabled
 - Cloud-Init drive attached
+- Serial console attached as `serial0: socket`
+- Display set to `vga: serial0` for Proxmox xterm.js console access
 - 2 vCPU, 2 GB RAM, 32 GB disk unless a service needs more
 
+Use the helper script on a Proxmox node to create the base template:
+
+```sh
+scripts/create-proxmox-template.sh
+```
+
+The script prompts for VMID, name, storage, bridge, CPU, memory, disk size, and
+image source. Press Enter to accept the displayed defaults. It downloads the
+Debian 13 GenericCloud qcow2 image unless a local `--image-file` is provided,
+imports it into Proxmox storage, attaches a Cloud-Init drive, configures serial
+console support, and converts the VM to a template.
+
+For automation or previews:
+
+```sh
+scripts/create-proxmox-template.sh --dry-run --yes
+scripts/create-proxmox-template.sh --non-interactive --vmid 9000 --storage local-lvm --bridge vmbr0
+```
+
+If a PBS storage/datastore ID is supplied, the template's main disk is created
+with the Proxmox disk `backup=1` flag. Leaving it empty keeps the main disk at
+`backup=0`.
+
+```sh
+scripts/create-proxmox-template.sh --pbs-storage pbs-main
+```
+
+To initialise all VMs declared in `host_vars/*.yml`, run:
+
+```sh
+scripts/init-proxmox-vms.sh
+```
+
+The VM initialiser checks whether the template VMID exists. If it does not, it
+runs `scripts/create-proxmox-template.sh` first. Each host var file with a
+`proxmox_vmid` is cloned from the template and configured from its `proxmox_*`
+hardware settings.
+
 Cloud-Init is only responsible for Day 0 bootstrap: user creation, SSH
 hardening, Docker installation, baseline packages, firewall defaults, directory
-layout, PKI directory preparation, and `ansible-pull`.
+layout, PKI directory preparation, enabling `serial-getty@ttyS0.service`, and
+`ansible-pull`.
+
+Proxmox hardware settings must be applied outside Cloud-Init. For a VM or
+template, set:
+
+```sh
+qm set <vmid> --serial0 socket --vga serial0
+```
+
+The Cloud-Init template enables `serial-getty@ttyS0.service` by default and sets
+`TERM=xterm-256color` for that getty. Override these variables if needed:
+
+```yaml
+cloud_init_serial_console_enabled: true
+cloud_init_serial_console_tty: ttyS0
+cloud_init_serial_console_term: xterm-256color
+```
 
 Applications and durable service configuration belong in Ansible roles.
-

docs/upstream-sources.yml

@@ -65,7 +65,7 @@ sources:
     vendor_dir: roles/authentik/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/goauthentik/authentik.git
         ref: main
         readme_paths: [README.md]
@@ -77,7 +77,7 @@ sources:
     vendor_dir: roles/authentik_ldap/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/goauthentik/authentik.git
         ref: main
         readme_paths: [README.md]
@@ -89,7 +89,7 @@ sources:
     vendor_dir: roles/authentik_proxy/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/goauthentik/authentik.git
         ref: main
         readme_paths: [README.md]
@@ -101,7 +101,7 @@ sources:
     vendor_dir: roles/authentik_rac/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/goauthentik/authentik.git
         ref: main
         readme_paths: [README.md]
@@ -113,7 +113,7 @@ sources:
     vendor_dir: roles/cleanuparr/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/cleanuparr/cleanuparr.git
         ref: main
         readme_paths: [README.md]
@@ -167,7 +167,7 @@ sources:
     vendor_dir: roles/flaresolverr/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/FlareSolverr/FlareSolverr.git
         ref: master
         readme_paths: [README.md]
@@ -179,7 +179,7 @@ sources:
     vendor_dir: roles/gitea/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/go-gitea/gitea.git
         ref: main
         readme_paths: [README.md]
@@ -191,7 +191,7 @@ sources:
     vendor_dir: roles/gitea_act_runner/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://gitea.com/gitea/act_runner.git
         ref: main
         readme_paths: [README.md]
@@ -203,7 +203,7 @@ sources:
     vendor_dir: roles/grafana/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/grafana/grafana.git
         ref: main
         readme_paths: [README.md]
@@ -215,7 +215,7 @@ sources:
     vendor_dir: roles/grafana_renderer/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/grafana/grafana-image-renderer.git
         ref: master
         readme_paths: [README.md]
@@ -227,7 +227,7 @@ sources:
     vendor_dir: roles/homarr/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/homarr-labs/homarr.git
         ref: dev
         readme_paths: [README.md]
@@ -239,7 +239,7 @@ sources:
     vendor_dir: roles/immich/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/immich-app/immich.git
         ref: main
         readme_paths: [README.md]
@@ -281,7 +281,7 @@ sources:
     vendor_dir: roles/jellyseerr/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/Fallenbagel/jellyseerr.git
         ref: develop
         readme_paths: [README.md]
@@ -305,7 +305,7 @@ sources:
     vendor_dir: roles/kiwix_serve/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/kiwix/kiwix-tools.git
         ref: main
         readme_paths: [README.md]
@@ -317,7 +317,7 @@ sources:
     vendor_dir: roles/loki/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/grafana/loki.git
         ref: main
         readme_paths: [README.md]
@@ -341,7 +341,7 @@ sources:
     vendor_dir: roles/ntfy/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/binwiederhier/ntfy.git
         ref: main
         readme_paths: [README.md]
@@ -353,7 +353,7 @@ sources:
     vendor_dir: roles/ocis/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/owncloud/ocis.git
         ref: master
         readme_paths: [README.md]
@@ -365,7 +365,7 @@ sources:
     vendor_dir: roles/paperless_ngx/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/paperless-ngx/paperless-ngx.git
         ref: main
         readme_paths: [README.md]
@@ -377,7 +377,7 @@ sources:
     vendor_dir: roles/pelican_panel/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/pelican-dev/panel.git
         ref: main
         readme_paths: [README.md]
@@ -401,7 +401,7 @@ sources:
     vendor_dir: roles/pinchflat/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/kieraneglin/pinchflat.git
         ref: main
         readme_paths: [README.md]
@@ -437,7 +437,7 @@ sources:
     vendor_dir: roles/pterodactyl/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/pterodactyl/panel.git
         ref: 1.0-develop
         readme_paths: [README.md]
@@ -461,7 +461,7 @@ sources:
     vendor_dir: roles/requarks/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/requarks/wiki.git
         ref: master
         readme_paths: [README.md]
@@ -485,7 +485,7 @@ sources:
     vendor_dir: roles/traefik/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/traefik/traefik.git
         ref: master
         readme_paths: [README.md]
@@ -509,7 +509,7 @@ sources:
     vendor_dir: roles/uptime_kuma/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/louislam/uptime-kuma.git
         ref: master
         readme_paths: [README.md]
@@ -521,7 +521,7 @@ sources:
     vendor_dir: roles/valkey/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/valkey-io/valkey.git
         ref: unstable
         readme_paths: [README.md]
@@ -533,7 +533,7 @@ sources:
     vendor_dir: roles/vaultwarden/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/dani-garcia/vaultwarden.git
         ref: main
         readme_paths: [README.md]
@@ -545,7 +545,7 @@ sources:
     vendor_dir: roles/wizarr/upstream
     local_compose_file: ""
     repositories:
-      - purpose: application
+      - purpose: container
         url: https://github.com/Wizarrrr/wizarr.git
         ref: main
         readme_paths: [README.md]

host_vars/README.md

@@ -5,3 +5,25 @@ Create one file per VM, for example `host_vars/media-01.yml`.
 Use host variables for hostnames, service enablement, storage paths, and port
 assignments. Keep credentials and TLS private material outside Git.
 
+## Proxmox VM Hardware
+
+`scripts/init-proxmox-vms.sh` reads top-level `proxmox_*` keys from each
+`host_vars/*.yml` file. Files without `proxmox_vmid` are skipped.
+
+Example:
+
+```yaml
+proxmox_vmid: 101
+proxmox_name: cloud
+proxmox_template_vmid: 9000
+proxmox_storage: local-lvm
+proxmox_bridge: vmbr0
+proxmox_memory: 4096
+proxmox_cores: 2
+proxmox_cpu: host
+proxmox_disk_size: 64G
+proxmox_backup_enabled: true
+proxmox_ipconfig0: ip=dhcp
+proxmox_onboot: true
+proxmox_start: false
+```

roles/alloy/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `alloy` role.
+Role: `alloy`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/arr/upstream/SOURCE.md

@@ -1,7 +1,15 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `arr` role.
+Role: `arr`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- sonarr-container README `README.md`: copied
+- radarr-container README `README.md`: copied
+- lidarr-container README `README.md`: copied
+- bazarr-container README `README.md`: copied
+README Compose candidates: 4
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/authentik/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `authentik` role.
+Role: `authentik`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose URL `https://docs.goauthentik.io/compose.yml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/authentik_ldap/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `authentik_ldap` role.
+Role: `authentik_ldap`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose URL `https://docs.goauthentik.io/add-secure-apps/outposts/manual-deploy-docker-compose/`: invalid (response is not Compose-shaped YAML)
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/authentik_ldap/upstream/docker-compose.override.yml

@@ -0,0 +1,13 @@
+  authentik-ldap:
+    container_name: authentik-ldap
+    restart: unless-stopped
+    image: ${AUTHENTIK_IMAGE:-ghcr.io/goauthentik/ldap}:latest
+    ports:
+      - 389:3389
+      - 636:6636
+    environment:
+      AUTHENTIK_HOST: https://example.com
+      AUTHENTIK_INSECURE: "false"
+      AUTHENTIK_TOKEN: 
+    depends_on:
+      - authentik-server

roles/authentik_proxy/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `authentik_proxy` role.
+Role: `authentik_proxy`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose URL `https://docs.goauthentik.io/add-secure-apps/outposts/manual-deploy-docker-compose/`: invalid (response is not Compose-shaped YAML)
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/authentik_proxy/upstream/docker-compose.override.yml

@@ -0,0 +1,17 @@
+services:
+  authentik_proxy:
+    container_name: authentik-proxy-outpost
+    image: ${AUTHENTIK_IMAGE:-ghcr.io/goauthentik/server}:latest
+    restart: unless-stopped
+    # Optionally specify the container's network, which must be able to reach the core authentik server.
+    # networks:
+    #   - foo
+    ports:
+      - 9000:9000
+      - 9443:9443
+    environment:
+      AUTHENTIK_HOST: https://example.com
+      AUTHENTIK_INSECURE: "false"
+      AUTHENTIK_TOKEN: 
+      # Optional setting to be used when `authentik_host` for internal communication doesn't match the public URL.
+      # AUTHENTIK_HOST_BROWSER: https://external-domain.tld

roles/authentik_rac/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `authentik_rac` role.
+Role: `authentik_rac`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose URL `https://docs.goauthentik.io/add-secure-apps/outposts/manual-deploy-docker-compose/`: invalid (response is not Compose-shaped YAML)
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/authentik_rac/upstream/docker-compose.override.yml

@@ -0,0 +1,10 @@
+  authentik-rac:
+    container_name: authentik-rac
+    restart: unless-stopped
+    image: ${AUTHENTIK_IMAGE:-ghcr.io/goauthentik/rac}:latest
+    environment:
+      AUTHENTIK_HOST: https://example.com
+      AUTHENTIK_INSECURE: "false"
+      AUTHENTIK_TOKEN: changeme
+    depends_on:
+      - authentik-server

roles/cleanuparr/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `cleanuparr` role.
+Role: `cleanuparr`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/code_server/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `code_server` role.
+Role: `code_server`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+README Compose candidates: 1
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/filebrowser_quantum/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `filebrowser_quantum` role.
+Role: `filebrowser_quantum`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `verify`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+- container Compose URL `https://filebrowserquantum.com/en/docs/user-guides/other/standalone/`: invalid (response is not Compose-shaped YAML)
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/fiveetools/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `fiveetools` role.
+Role: `fiveetools`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `verify`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+- container Compose `docker-compose.yml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/flaresolverr/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `flaresolverr` role.
+Role: `flaresolverr`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `docker-compose.yml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/gitea/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `gitea` role.
+Role: `gitea`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose URL `https://docs.gitea.com/installation/install-with-docker-rootless`: invalid (response is not Compose-shaped YAML)
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/gitea_act_runner/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `gitea_act_runner` role.
+Role: `gitea_act_runner`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose URL `https://docs.gitea.com/usage/actions/act-runner#start-the-runner-using-docker-compose`: invalid (response is not Compose-shaped YAML)
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/grafana/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `grafana` role.
+Role: `grafana`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose URL `[] # Host own, all examples are bad`: invalid (unsupported URL scheme)
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/grafana_renderer/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `grafana_renderer` role.
+Role: `grafana_renderer`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/homarr/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `homarr` role.
+Role: `homarr`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: missing
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/immich/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `immich` role.
+Role: `immich`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `docker/docker-compose.yml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/ipv6nat/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `ipv6nat` role.
+Role: `ipv6nat`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/jellyfin/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `jellyfin` role.
+Role: `jellyfin`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+- application README `README.md`: copied
+README Compose candidates: 1
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/jellyseerr/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `jellyseerr` role.
+Role: `jellyseerr`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `docker-compose.yml`: missing
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/kavita/upstream/SOURCE.md

@@ -1,7 +1,11 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `kavita` role.
+Role: `kavita`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/kiwix_serve/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `kiwix_serve` role.
+Role: `kiwix_serve`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/loki/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `loki` role.
+Role: `loki`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `production/docker-compose.yaml`: missing
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/nginx_content/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `nginx_content` role.
+Role: `nginx_content`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/ntfy/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `ntfy` role.
+Role: `ntfy`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `docker-compose.yml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/ocis/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `ocis` role.
+Role: `ocis`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `deployments/examples/ocis_full/docker-compose.yml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/paperless_ngx/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `paperless_ngx` role.
+Role: `paperless_ngx`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `docker/compose/docker-compose.postgres.yml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/pelican_panel/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `pelican_panel` role.
+Role: `pelican_panel`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `compose-full-stack.yml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/php_fpm/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `php_fpm` role.
+Role: `php_fpm`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/pinchflat/upstream/SOURCE.md

@@ -1,7 +1,11 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `pinchflat` role.
+Role: `pinchflat`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/postgres/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `postgres` role.
+Role: `postgres`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/prometheus/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `prometheus` role.
+Role: `prometheus`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `documentation/examples/prometheus.yml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/pterodactyl/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `pterodactyl` role.
+Role: `pterodactyl`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/qbittorrent/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `qbittorrent` role.
+Role: `qbittorrent`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+README Compose candidates: 1
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/requarks/upstream/SOURCE.md

@@ -1,7 +1,11 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `requarks` role.
+Role: `requarks`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/syncthing/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `syncthing` role.
+Role: `syncthing`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+README Compose candidates: 1
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/traefik/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `traefik` role.
+Role: `traefik`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/transmission/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `transmission` role.
+Role: `transmission`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- container README `README.md`: copied
+README Compose candidates: 1
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/uptime_kuma/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `uptime_kuma` role.
+Role: `uptime_kuma`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `compose.yaml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/valkey/upstream/SOURCE.md

@@ -1,7 +1,12 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `valkey` role.
+Role: `valkey`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/vaultwarden/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `vaultwarden` role.
+Role: `vaultwarden`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `docker/docker-compose.yml`: missing
+README Compose candidates: 1
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

roles/wizarr/upstream/SOURCE.md

@@ -1,7 +1,13 @@
 # Upstream Snapshot
 
-This directory stores copied upstream reference files for the `wizarr` role.
+Role: `wizarr`
+Snapshot source inventory: `docs/upstream-sources.yml`
+Snapshot status: `tracked`
 
-See `docs/upstream-sources.yml` for repository URLs, refs, README paths, Compose paths, and verification status.
+Fetched repositories:
+- application README `README.md`: copied
+- application Compose `docker-compose.yml`: copied
+README Compose candidates: 0
+Ambiguous README files: 0
 
-Snapshot files are reference material only. Deployments are generated from this role's `templates/` directory.
+These files are reference snapshots only. Deployments are generated from role templates.

scripts/create-proxmox-template.sh

@@ -0,0 +1,425 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+VMID=9000
+NAME=debian-13-genericcloud
+IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-amd64.qcow2
+IMAGE_FILE=
+CACHE_DIR=/var/lib/vz/template/qcow2
+STORAGE=local-lvm
+CLOUDINIT_STORAGE=
+BRIDGE=vmbr0
+MEMORY=2048
+CORES=2
+CPU=host
+DISK_SIZE=32G
+MACHINE=q35
+BIOS=ovmf
+SCSI_HW=virtio-scsi-pci
+NET_MODEL=virtio
+PBS_STORAGE=
+DRY_RUN=0
+ASSUME_YES=0
+INTERACTIVE=1
+VMID_SET=0
+NAME_SET=0
+IMAGE_URL_SET=0
+IMAGE_FILE_SET=0
+CACHE_DIR_SET=0
+STORAGE_SET=0
+CLOUDINIT_STORAGE_SET=0
+BRIDGE_SET=0
+MEMORY_SET=0
+CORES_SET=0
+CPU_SET=0
+DISK_SIZE_SET=0
+PBS_STORAGE_SET=0
+
+usage() {
+  cat <<'USAGE'
+Create a Proxmox VM template from the Debian 13 GenericCloud image.
+
+Run this on a Proxmox node as root.
+
+Usage:
+  scripts/create-proxmox-template.sh [options]
+
+Options:
+  --vmid ID                 Template VMID. Default: 9000
+  --name NAME               Template name. Default: debian-13-genericcloud
+  --storage STORAGE         Disk storage. Default: local-lvm
+  --cloudinit-storage NAME  Cloud-Init storage. Default: same as --storage
+  --bridge BRIDGE           Network bridge. Default: vmbr0
+  --memory MB               Memory in MB. Default: 2048
+  --cores N                 vCPU count. Default: 2
+  --cpu TYPE                CPU type. Default: host
+  --disk-size SIZE          Resize imported disk to this size. Default: 32G
+  --image-url URL           Debian GenericCloud qcow2 URL.
+  --image-file PATH         Use an existing local qcow2 instead of downloading.
+  --cache-dir PATH          Download cache directory. Default: /var/lib/vz/template/qcow2
+  --pbs-storage NAME        PBS storage/datastore ID. Enables main disk backup flag when set.
+  --dry-run                 Print commands without executing them.
+  --yes                     Accept defaults for any unanswered prompts.
+  --non-interactive         Do not prompt; use defaults and provided flags.
+  -h, --help                Show this help.
+
+The template is configured with:
+  - Q35 machine type
+  - OVMF / UEFI
+  - VirtIO SCSI
+  - VirtIO network
+  - QEMU guest agent
+  - Cloud-Init drive
+  - serial0 socket
+  - vga serial0 for Proxmox xterm.js console support
+USAGE
+}
+
+prompt_value() {
+  local label=$1
+  local default_value=$2
+  local value
+
+  if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
+    printf '%s\n' "$default_value"
+    return 0
+  fi
+
+  printf '%s [%s]: ' "$label" "$default_value" >&2
+  read -r value
+  if [ -z "$value" ]; then
+    printf '%s\n' "$default_value"
+  else
+    printf '%s\n' "$value"
+  fi
+}
+
+prompt_optional_value() {
+  local label=$1
+  local value
+
+  if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
+    printf '\n'
+    return 0
+  fi
+
+  printf '%s [leave empty]: ' "$label" >&2
+  read -r value
+  printf '%s\n' "$value"
+}
+
+prompt_confirm() {
+  local label=$1
+  local value
+
+  if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
+    return 0
+  fi
+
+  printf '%s [Y/n]: ' "$label" >&2
+  read -r value
+  case "$value" in
+    ""|y|Y|yes|YES)
+      return 0
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+}
+
+log() {
+  printf '%s\n' "$*"
+}
+
+die() {
+  printf 'ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+run() {
+  if [ "$DRY_RUN" -eq 1 ]; then
+    printf '+'
+    printf ' %q' "$@"
+    printf '\n'
+    return 0
+  fi
+  "$@"
+}
+
+need_command() {
+  command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
+}
+
+download_image() {
+  local url=$1
+  local dest=$2
+
+  if [ -s "$dest" ]; then
+    log "Using cached image: $dest"
+    return 0
+  fi
+
+  run mkdir -p "$(dirname "$dest")"
+  if command -v curl >/dev/null 2>&1; then
+    run curl -fL --retry 3 --retry-delay 2 -o "$dest" "$url"
+  elif command -v wget >/dev/null 2>&1; then
+    run wget -O "$dest" "$url"
+  else
+    die "curl or wget is required to download the image"
+  fi
+}
+
+parse_args() {
+  while [ "$#" -gt 0 ]; do
+    case "$1" in
+      --vmid)
+        VMID=${2:?missing value for --vmid}
+        VMID_SET=1
+        shift 2
+        ;;
+      --name)
+        NAME=${2:?missing value for --name}
+        NAME_SET=1
+        shift 2
+        ;;
+      --storage)
+        STORAGE=${2:?missing value for --storage}
+        STORAGE_SET=1
+        shift 2
+        ;;
+      --cloudinit-storage)
+        CLOUDINIT_STORAGE=${2:?missing value for --cloudinit-storage}
+        CLOUDINIT_STORAGE_SET=1
+        shift 2
+        ;;
+      --bridge)
+        BRIDGE=${2:?missing value for --bridge}
+        BRIDGE_SET=1
+        shift 2
+        ;;
+      --memory)
+        MEMORY=${2:?missing value for --memory}
+        MEMORY_SET=1
+        shift 2
+        ;;
+      --cores)
+        CORES=${2:?missing value for --cores}
+        CORES_SET=1
+        shift 2
+        ;;
+      --cpu)
+        CPU=${2:?missing value for --cpu}
+        CPU_SET=1
+        shift 2
+        ;;
+      --disk-size)
+        DISK_SIZE=${2:?missing value for --disk-size}
+        DISK_SIZE_SET=1
+        shift 2
+        ;;
+      --image-url)
+        IMAGE_URL=${2:?missing value for --image-url}
+        IMAGE_URL_SET=1
+        shift 2
+        ;;
+      --image-file)
+        IMAGE_FILE=${2:?missing value for --image-file}
+        IMAGE_FILE_SET=1
+        shift 2
+        ;;
+      --cache-dir)
+        CACHE_DIR=${2:?missing value for --cache-dir}
+        CACHE_DIR_SET=1
+        shift 2
+        ;;
+      --pbs-storage)
+        PBS_STORAGE=${2:?missing value for --pbs-storage}
+        PBS_STORAGE_SET=1
+        shift 2
+        ;;
+      --dry-run)
+        DRY_RUN=1
+        shift
+        ;;
+      --yes)
+        ASSUME_YES=1
+        shift
+        ;;
+      --non-interactive)
+        INTERACTIVE=0
+        shift
+        ;;
+      -h|--help)
+        usage
+        exit 0
+        ;;
+      *)
+        die "unknown option: $1"
+        ;;
+    esac
+  done
+}
+
+collect_interactive_config() {
+  if [ "$VMID_SET" -eq 0 ]; then
+    VMID=$(prompt_value "Template VMID" "$VMID")
+  fi
+  if [ "$NAME_SET" -eq 0 ]; then
+    NAME=$(prompt_value "Template name" "$NAME")
+  fi
+  if [ "$STORAGE_SET" -eq 0 ]; then
+    STORAGE=$(prompt_value "Disk storage" "$STORAGE")
+  fi
+  if [ "$CLOUDINIT_STORAGE_SET" -eq 0 ]; then
+    CLOUDINIT_STORAGE=$(prompt_value "Cloud-Init storage" "${CLOUDINIT_STORAGE:-$STORAGE}")
+  fi
+  if [ "$BRIDGE_SET" -eq 0 ]; then
+    BRIDGE=$(prompt_value "Network bridge" "$BRIDGE")
+  fi
+  if [ "$MEMORY_SET" -eq 0 ]; then
+    MEMORY=$(prompt_value "Memory MB" "$MEMORY")
+  fi
+  if [ "$CORES_SET" -eq 0 ]; then
+    CORES=$(prompt_value "vCPU cores" "$CORES")
+  fi
+  if [ "$CPU_SET" -eq 0 ]; then
+    CPU=$(prompt_value "CPU type" "$CPU")
+  fi
+  if [ "$DISK_SIZE_SET" -eq 0 ]; then
+    DISK_SIZE=$(prompt_value "Disk size after resize" "$DISK_SIZE")
+  fi
+  if [ "$CACHE_DIR_SET" -eq 0 ]; then
+    CACHE_DIR=$(prompt_value "Image cache directory" "$CACHE_DIR")
+  fi
+  if [ "$IMAGE_FILE_SET" -eq 0 ]; then
+    IMAGE_FILE=$(prompt_optional_value "Local qcow2 image file")
+  fi
+  if [ -z "$IMAGE_FILE" ] && [ "$IMAGE_URL_SET" -eq 0 ]; then
+    IMAGE_URL=$(prompt_value "Debian GenericCloud image URL" "$IMAGE_URL")
+  fi
+  if [ "$PBS_STORAGE_SET" -eq 0 ]; then
+    PBS_STORAGE=$(prompt_optional_value "PBS storage/datastore ID for backup flag")
+  fi
+}
+
+print_summary() {
+  cat <<SUMMARY
+
+Template configuration:
+  VMID:               $VMID
+  Name:               $NAME
+  Disk storage:       $STORAGE
+  Cloud-Init storage: $CLOUDINIT_STORAGE
+  Bridge:             $BRIDGE
+  Memory:             $MEMORY MB
+  Cores:              $CORES
+  CPU:                $CPU
+  Disk size:          $DISK_SIZE
+  Machine:            $MACHINE
+  BIOS:               $BIOS
+  Serial console:     serial0 socket, vga serial0
+SUMMARY
+
+  if [ -n "$IMAGE_FILE" ]; then
+    printf '  Image file:         %s\n' "$IMAGE_FILE"
+  else
+    printf '  Image URL:          %s\n' "$IMAGE_URL"
+    printf '  Cache directory:    %s\n' "$CACHE_DIR"
+  fi
+  if [ "$DRY_RUN" -eq 1 ]; then
+    printf '  Mode:               dry-run\n'
+  fi
+  if [ -n "$PBS_STORAGE" ]; then
+    printf '  Backup flag:        enabled for main disk; PBS storage: %s\n' "$PBS_STORAGE"
+  else
+    printf '  Backup flag:        disabled for main disk\n'
+  fi
+  printf '\n'
+}
+
+main() {
+  parse_args "$@"
+
+  collect_interactive_config
+
+  if [ -z "$CLOUDINIT_STORAGE" ]; then
+    CLOUDINIT_STORAGE=$STORAGE
+  fi
+
+  print_summary
+  if ! prompt_confirm "Create this Proxmox template?"; then
+    die "cancelled"
+  fi
+
+  if [ "$DRY_RUN" -eq 0 ]; then
+    need_command qm
+    need_command awk
+  fi
+
+  if [ "$(id -u)" -ne 0 ] && [ "$DRY_RUN" -eq 0 ]; then
+    die "run this script as root on a Proxmox node"
+  fi
+
+  if [ "$DRY_RUN" -eq 0 ] && qm status "$VMID" >/dev/null 2>&1; then
+    die "VMID $VMID already exists; choose another --vmid or remove it manually"
+  fi
+
+  local image_path
+  if [ -n "$IMAGE_FILE" ]; then
+    image_path=$IMAGE_FILE
+    [ "$DRY_RUN" -eq 1 ] || [ -f "$image_path" ] || die "image file not found: $image_path"
+  else
+    image_path="$CACHE_DIR/$(basename "$IMAGE_URL")"
+    download_image "$IMAGE_URL" "$image_path"
+  fi
+
+  run qm create "$VMID" \
+    --name "$NAME" \
+    --ostype l26 \
+    --machine "$MACHINE" \
+    --bios "$BIOS" \
+    --memory "$MEMORY" \
+    --cores "$CORES" \
+    --cpu "$CPU" \
+    --agent enabled=1 \
+    --scsihw "$SCSI_HW" \
+    --net0 "$NET_MODEL,bridge=$BRIDGE" \
+    --serial0 socket \
+    --vga serial0
+
+  if [ "$BIOS" = "ovmf" ]; then
+    run qm set "$VMID" --efidisk0 "$STORAGE:0,pre-enrolled-keys=0"
+  fi
+
+  run qm importdisk "$VMID" "$image_path" "$STORAGE"
+
+  local imported_disk
+  if [ "$DRY_RUN" -eq 1 ]; then
+    imported_disk="$STORAGE:vm-$VMID-disk-0"
+    log "Dry-run assumption: imported disk will be $imported_disk"
+  else
+    imported_disk=$(qm config "$VMID" | awk -F': ' '/^unused[0-9]+:/{print $2; exit}')
+    [ -n "$imported_disk" ] || die "could not find imported unused disk in qm config for VMID $VMID"
+  fi
+
+  local disk_backup=0
+  if [ -n "$PBS_STORAGE" ]; then
+    disk_backup=1
+  fi
+
+  run qm set "$VMID" --scsi0 "$imported_disk,discard=on,ssd=1,backup=$disk_backup"
+  run qm set "$VMID" --ide2 "$CLOUDINIT_STORAGE:cloudinit"
+  run qm set "$VMID" --boot order=scsi0
+  run qm set "$VMID" --ipconfig0 ip=dhcp
+
+  if [ -n "$DISK_SIZE" ]; then
+    run qm resize "$VMID" scsi0 "$DISK_SIZE"
+  fi
+
+  run qm template "$VMID"
+
+  log "Created Proxmox template $NAME ($VMID)"
+}
+
+main "$@"

scripts/init-proxmox-vms.sh

@@ -0,0 +1,367 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+HOST_VARS_DIR=host_vars
+TEMPLATE_VMID=9000
+TEMPLATE_STORAGE=local-lvm
+TEMPLATE_BRIDGE=vmbr0
+PBS_STORAGE=
+DRY_RUN=0
+ASSUME_YES=0
+INTERACTIVE=1
+
+usage() {
+  cat <<'USAGE'
+Initialise Proxmox VMs from host_vars/*.yml.
+
+Run this on a Proxmox node as root from the repository root.
+
+Each host var file must set proxmox_vmid to be managed by this script.
+Files without proxmox_vmid are skipped.
+
+Supported host vars:
+  proxmox_vmid: 101
+  proxmox_name: cloud
+  proxmox_template_vmid: 9000
+  proxmox_storage: local-lvm
+  proxmox_bridge: vmbr0
+  proxmox_memory: 4096
+  proxmox_cores: 2
+  proxmox_cpu: host
+  proxmox_disk_size: 64G
+  proxmox_backup_enabled: true
+  proxmox_ipconfig0: ip=dhcp
+  proxmox_onboot: true
+  proxmox_start: false
+
+Options:
+  --host-vars-dir PATH      Host vars directory. Default: host_vars
+  --template-vmid ID        Default template VMID. Default: 9000
+  --template-storage NAME   Storage used if template must be created. Default: local-lvm
+  --template-bridge NAME    Bridge used if template must be created. Default: vmbr0
+  --pbs-storage NAME        PBS storage/datastore ID. Passed to template script and used as default backup target note.
+  --dry-run                 Print commands without executing them.
+  --yes                     Accept defaults for prompts.
+  --non-interactive         Do not prompt.
+  -h, --help                Show this help.
+USAGE
+}
+
+log() {
+  printf '%s\n' "$*"
+}
+
+die() {
+  printf 'ERROR: %s\n' "$*" >&2
+  exit 1
+}
+
+run() {
+  if [ "$DRY_RUN" -eq 1 ]; then
+    printf '+'
+    printf ' %q' "$@"
+    printf '\n'
+    return 0
+  fi
+  "$@"
+}
+
+need_command() {
+  command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
+}
+
+prompt_value() {
+  local label=$1
+  local default_value=$2
+  local value
+
+  if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
+    printf '%s\n' "$default_value"
+    return 0
+  fi
+
+  printf '%s [%s]: ' "$label" "$default_value" >&2
+  read -r value
+  if [ -z "$value" ]; then
+    printf '%s\n' "$default_value"
+  else
+    printf '%s\n' "$value"
+  fi
+}
+
+prompt_optional_value() {
+  local label=$1
+  local value
+
+  if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
+    printf '\n'
+    return 0
+  fi
+
+  printf '%s [leave empty]: ' "$label" >&2
+  read -r value
+  printf '%s\n' "$value"
+}
+
+prompt_confirm() {
+  local label=$1
+  local value
+
+  if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
+    return 0
+  fi
+
+  printf '%s [Y/n]: ' "$label" >&2
+  read -r value
+  case "$value" in
+    ""|y|Y|yes|YES)
+      return 0
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+}
+
+yaml_scalar() {
+  local file=$1
+  local key=$2
+  awk -v key="$key" '
+    $0 ~ "^[[:space:]]*" key ":" {
+      sub("^[[:space:]]*" key ":[[:space:]]*", "", $0)
+      sub("[[:space:]]+#.*$", "", $0)
+      gsub(/^["'\'']|["'\'']$/, "", $0)
+      print $0
+      exit
+    }
+  ' "$file"
+}
+
+bool_true() {
+  case "${1:-}" in
+    true|True|TRUE|yes|Yes|YES|1|on|On|ON)
+      return 0
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+}
+
+parse_args() {
+  while [ "$#" -gt 0 ]; do
+    case "$1" in
+      --host-vars-dir)
+        HOST_VARS_DIR=${2:?missing value for --host-vars-dir}
+        shift 2
+        ;;
+      --template-vmid)
+        TEMPLATE_VMID=${2:?missing value for --template-vmid}
+        shift 2
+        ;;
+      --template-storage)
+        TEMPLATE_STORAGE=${2:?missing value for --template-storage}
+        shift 2
+        ;;
+      --template-bridge)
+        TEMPLATE_BRIDGE=${2:?missing value for --template-bridge}
+        shift 2
+        ;;
+      --pbs-storage)
+        PBS_STORAGE=${2:?missing value for --pbs-storage}
+        shift 2
+        ;;
+      --dry-run)
+        DRY_RUN=1
+        shift
+        ;;
+      --yes)
+        ASSUME_YES=1
+        shift
+        ;;
+      --non-interactive)
+        INTERACTIVE=0
+        shift
+        ;;
+      -h|--help)
+        usage
+        exit 0
+        ;;
+      *)
+        die "unknown option: $1"
+        ;;
+    esac
+  done
+}
+
+collect_config() {
+  TEMPLATE_VMID=$(prompt_value "Default template VMID" "$TEMPLATE_VMID")
+  TEMPLATE_STORAGE=$(prompt_value "Default template/storage name" "$TEMPLATE_STORAGE")
+  TEMPLATE_BRIDGE=$(prompt_value "Default bridge" "$TEMPLATE_BRIDGE")
+  if [ -z "$PBS_STORAGE" ]; then
+    PBS_STORAGE=$(prompt_optional_value "PBS storage/datastore ID")
+  fi
+}
+
+template_exists() {
+  if [ "$DRY_RUN" -eq 1 ]; then
+    return 1
+  fi
+  qm status "$TEMPLATE_VMID" >/dev/null 2>&1
+}
+
+vm_exists() {
+  local vmid=$1
+  if [ "$DRY_RUN" -eq 1 ]; then
+    return 1
+  fi
+  qm status "$vmid" >/dev/null 2>&1
+}
+
+ensure_template() {
+  if template_exists; then
+    log "Template VMID $TEMPLATE_VMID already exists."
+    return 0
+  fi
+
+  log "Template VMID $TEMPLATE_VMID not found; creating it first."
+  local args=(
+    scripts/create-proxmox-template.sh
+    --vmid "$TEMPLATE_VMID"
+    --storage "$TEMPLATE_STORAGE"
+    --bridge "$TEMPLATE_BRIDGE"
+    --yes
+    --non-interactive
+  )
+  if [ -n "$PBS_STORAGE" ]; then
+    args+=(--pbs-storage "$PBS_STORAGE")
+  fi
+  if [ "$DRY_RUN" -eq 1 ]; then
+    args+=(--dry-run)
+  fi
+  run "${args[@]}"
+}
+
+disk_volume() {
+  local vmid=$1
+  local storage=$2
+  if [ "$DRY_RUN" -eq 1 ]; then
+    printf '%s:vm-%s-disk-0\n' "$storage" "$vmid"
+    return 0
+  fi
+  qm config "$vmid" | awk -F': ' '/^scsi0:/{split($2, parts, ","); print parts[1]; exit}'
+}
+
+init_vm_from_host_vars() {
+  local file=$1
+  local stem
+  stem=$(basename "$file" .yml)
+
+  local vmid
+  vmid=$(yaml_scalar "$file" proxmox_vmid)
+  if [ -z "$vmid" ]; then
+    log "Skipping $file: proxmox_vmid not set"
+    return 0
+  fi
+
+  local name template_vmid storage bridge memory cores cpu disk_size backup_enabled ipconfig0 onboot start_vm
+  name=$(yaml_scalar "$file" proxmox_name)
+  [ -n "$name" ] || name=$(yaml_scalar "$file" nbops_hostname)
+  [ -n "$name" ] || name=$stem
+  template_vmid=$(yaml_scalar "$file" proxmox_template_vmid)
+  [ -n "$template_vmid" ] || template_vmid=$TEMPLATE_VMID
+  storage=$(yaml_scalar "$file" proxmox_storage)
+  [ -n "$storage" ] || storage=$TEMPLATE_STORAGE
+  bridge=$(yaml_scalar "$file" proxmox_bridge)
+  [ -n "$bridge" ] || bridge=$TEMPLATE_BRIDGE
+  memory=$(yaml_scalar "$file" proxmox_memory)
+  [ -n "$memory" ] || memory=2048
+  cores=$(yaml_scalar "$file" proxmox_cores)
+  [ -n "$cores" ] || cores=2
+  cpu=$(yaml_scalar "$file" proxmox_cpu)
+  [ -n "$cpu" ] || cpu=host
+  disk_size=$(yaml_scalar "$file" proxmox_disk_size)
+  [ -n "$disk_size" ] || disk_size=32G
+  backup_enabled=$(yaml_scalar "$file" proxmox_backup_enabled)
+  [ -n "$backup_enabled" ] || backup_enabled=false
+  ipconfig0=$(yaml_scalar "$file" proxmox_ipconfig0)
+  [ -n "$ipconfig0" ] || ipconfig0=ip=dhcp
+  onboot=$(yaml_scalar "$file" proxmox_onboot)
+  [ -n "$onboot" ] || onboot=true
+  start_vm=$(yaml_scalar "$file" proxmox_start)
+  [ -n "$start_vm" ] || start_vm=false
+
+  if vm_exists "$vmid"; then
+    log "Skipping $name ($vmid): VM already exists"
+    return 0
+  fi
+
+  log "Initialising $name ($vmid) from $file"
+  run qm clone "$template_vmid" "$vmid" --name "$name" --full true --storage "$storage"
+  run qm set "$vmid" --memory "$memory" --cores "$cores" --cpu "$cpu"
+  run qm set "$vmid" --net0 "virtio,bridge=$bridge"
+  run qm set "$vmid" --ipconfig0 "$ipconfig0"
+
+  if bool_true "$onboot"; then
+    run qm set "$vmid" --onboot 1
+  else
+    run qm set "$vmid" --onboot 0
+  fi
+
+  local volume backup_flag
+  volume=$(disk_volume "$vmid" "$storage")
+  backup_flag=0
+  if bool_true "$backup_enabled"; then
+    backup_flag=1
+  fi
+  run qm set "$vmid" --scsi0 "$volume,discard=on,ssd=1,backup=$backup_flag"
+
+  if [ -n "$disk_size" ]; then
+    run qm resize "$vmid" scsi0 "$disk_size"
+  fi
+
+  if bool_true "$start_vm"; then
+    run qm start "$vmid"
+  fi
+}
+
+main() {
+  parse_args "$@"
+  collect_config
+
+  if [ "$DRY_RUN" -eq 0 ]; then
+    need_command qm
+    need_command awk
+    [ "$(id -u)" -eq 0 ] || die "run this script as root on a Proxmox node"
+  fi
+
+  [ -d "$HOST_VARS_DIR" ] || die "host vars directory not found: $HOST_VARS_DIR"
+
+  cat <<SUMMARY
+
+VM initialisation:
+  Host vars:       $HOST_VARS_DIR/*.yml
+  Template VMID:   $TEMPLATE_VMID
+  Default storage: $TEMPLATE_STORAGE
+  Default bridge:  $TEMPLATE_BRIDGE
+  PBS storage:     ${PBS_STORAGE:-not set}
+  Mode:            $([ "$DRY_RUN" -eq 1 ] && printf 'dry-run' || printf 'apply')
+
+SUMMARY
+
+  if ! prompt_confirm "Initialise VMs from host vars?"; then
+    die "cancelled"
+  fi
+
+  ensure_template
+
+  local found=0
+  for file in "$HOST_VARS_DIR"/*.yml; do
+    [ -e "$file" ] || continue
+    found=1
+    init_vm_from_host_vars "$file"
+  done
+  [ "$found" -eq 1 ] || die "no .yml files found in $HOST_VARS_DIR"
+}
+
+main "$@"

scripts/vendor-upstream-snapshots.py

@@ -22,6 +22,9 @@ from urllib.request import Request, urlopen
 ROOT = Path(__file__).resolve().parents[1]
 INVENTORY = ROOT / "docs" / "upstream-sources.yml"
 COMPOSE_BLOCK_LANGS = {"", "yaml", "yml", "docker-compose", "compose"}
+NORMALIZED_COMPOSE_NAME = "docker-compose.yml"
+DERIVED_TEMPLATE_MARKER = "nbops: derived-from-upstream"
+DERIVED_TEMPLATE_NAMES = ("compose.yaml.j2", "compose.override.yaml.j2")
 
 
 def parse_inline_list(value: str) -> list[str]:
@@ -137,6 +140,21 @@ def copy_if_present(src_root: Path, relative_path: str, dest_root: Path, apply:
     return "copied"
 
 
+def copy_compose_if_present(src_root: Path, relative_path: str, dest_root: Path, apply: bool) -> str:
+    src = src_root / relative_path
+    if not src.is_file():
+        return "missing"
+    if not looks_like_compose(src.read_text(errors="replace")):
+        return "invalid"
+    if not apply:
+        return "found"
+
+    dest = dest_root / NORMALIZED_COMPOSE_NAME
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    shutil.copy2(src, dest)
+    return "copied"
+
+
 def copy_local_file_if_present(relative_path: str, dest_root: Path, apply: bool) -> str:
     src = ROOT / relative_path
     if not src.is_file():
@@ -152,6 +170,17 @@ def copy_local_file_if_present(relative_path: str, dest_root: Path, apply: bool)
     return "copied"
 
 
+def replace_directory(src: Path, dest: Path) -> None:
+    if dest.exists():
+        shutil.rmtree(dest)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    shutil.move(str(src), str(dest))
+
+
+def preferred_override_exists(snapshot_dir: Path) -> bool:
+    return (snapshot_dir / "docker-compose.override.yml").is_file()
+
+
 def fetch_compose_url(url: str) -> tuple[str, str, str]:
     parsed = urlparse(url)
     if parsed.scheme not in {"http", "https"}:
@@ -173,12 +202,6 @@ def fetch_compose_url(url: str) -> tuple[str, str, str]:
     return "found", body, ""
 
 
-def compose_url_filename(index: int, total: int) -> str:
-    if total == 1:
-        return "compose.from-url.yaml"
-    return f"compose.from-url-{index}.yaml"
-
-
 def extract_fenced_blocks(markdown: str) -> list[tuple[str, str]]:
     pattern = re.compile(r"^```([^\n`]*)\n(.*?)^```", re.MULTILINE | re.DOTALL)
     blocks = []
@@ -226,6 +249,61 @@ def print_table(rows: list[list[str]]) -> None:
         print(fmt(row))
 
 
+def progress_enabled() -> bool:
+    return sys.stderr.isatty()
+
+
+def render_progress(current: int, total: int, label: str) -> None:
+    if not progress_enabled() or total <= 0:
+        return
+    width = 28
+    filled = min(width, int(width * current / total))
+    bar = "#" * filled + "-" * (width - filled)
+    sys.stderr.write(f"\r\033[K[{bar}] {current}/{total} {label}")
+    sys.stderr.flush()
+
+
+def clear_progress() -> None:
+    if progress_enabled():
+        sys.stderr.write("\r\033[K")
+        sys.stderr.flush()
+
+
+def snapshot_has_content(path: Path) -> bool:
+    return path.exists() and any(path.iterdir())
+
+
+def split_filter_values(values: list[str], csv_value: str) -> set[str]:
+    filters: set[str] = set()
+    for value in values:
+        filters.update(item.strip() for item in value.split(",") if item.strip())
+    filters.update(item.strip() for item in csv_value.split(",") if item.strip())
+    return filters
+
+
+def filtered_repositories(config: dict, upstream_filter: set[str]) -> list[dict]:
+    repos = config["repositories"]
+    if not upstream_filter:
+        return repos
+    return [repo for repo in repos if repo["purpose"] in upstream_filter]
+
+
+def clean_derived_templates(role: str) -> list[list[str]]:
+    rows: list[list[str]] = []
+    template_dir = ROOT / "roles" / role / "templates"
+    for name in DERIVED_TEMPLATE_NAMES:
+        path = template_dir / name
+        if not path.is_file():
+            continue
+        text = path.read_text(errors="replace")
+        if DERIVED_TEMPLATE_MARKER not in text:
+            rows.append([role, "template", str(path), "kept: unmarked"])
+            continue
+        path.unlink()
+        rows.append([role, "template", str(path), "removed"])
+    return rows
+
+
 def snapshot_source(vendor_dir: Path, role: str, status: str, notes: list[str], apply: bool) -> None:
     if not apply:
         return
@@ -246,11 +324,73 @@ def snapshot_source(vendor_dir: Path, role: str, status: str, notes: list[str],
     (vendor_dir / "SOURCE.md").write_text("\n".join(text))
 
 
+def clean_upstream_directories(selected: dict[str, dict], upstream_filter: set[str]) -> int:
+    rows: list[list[str]] = []
+    removed = 0
+    total = sum(len(filtered_repositories(config, upstream_filter)) for config in selected.values())
+    current = 0
+
+    for role, config in selected.items():
+        vendor_dir = ROOT / config["vendor_dir"]
+        repos = filtered_repositories(config, upstream_filter)
+        for repo in repos:
+            current += 1
+            purpose_dir = vendor_dir / repo["purpose"]
+            render_progress(current, total, f"clean {role}/{repo['purpose']}")
+            if purpose_dir.is_dir():
+                shutil.rmtree(purpose_dir)
+                removed += 1
+                result = "removed"
+            else:
+                result = "missing"
+            rows.append([role, repo["purpose"], str(purpose_dir), result])
+        rows.extend(clean_derived_templates(role))
+
+    clear_progress()
+    headers = ["Role", "Purpose", "Path", "Result"]
+    widths = [len(header) for header in headers]
+    for row in rows:
+        for index, value in enumerate(row):
+            widths[index] = max(widths[index], len(value))
+
+    def fmt(row: list[str]) -> str:
+        return "  ".join(value.ljust(widths[index]) for index, value in enumerate(row))
+
+    print(fmt(headers))
+    print("  ".join("-" * width for width in widths))
+    for row in rows:
+        print(fmt(row))
+    print(f"\nRemoved {removed} upstream purpose directories.")
+    return 0
+
+
 def main() -> int:
     parser = argparse.ArgumentParser(description="Test or vendor upstream README/Compose snapshots.")
     parser.add_argument("roles", nargs="*", help="Role names to process. Defaults to all roles.")
     parser.add_argument("--apply", action="store_true", help="Copy fetched files into role upstream directories.")
     parser.add_argument("--no-parse-readme", action="store_true", help="Do not extract Compose YAML from README blocks.")
+    parser.add_argument(
+        "--clean-upstreams",
+        action="store_true",
+        help="Delete upstream/<purpose> directories and marked derived templates for selected roles, then exit.",
+    )
+    parser.add_argument(
+        "--clean-purposes",
+        dest="clean_upstreams",
+        action="store_true",
+        help=argparse.SUPPRESS,
+    )
+    parser.add_argument(
+        "--upstream",
+        action="append",
+        default=[],
+        help="Limit processing to an upstream purpose name such as container or application. Repeatable.",
+    )
+    parser.add_argument(
+        "--upstreams",
+        default="",
+        help="Comma-separated upstream purpose names to process.",
+    )
     parser.add_argument("--list", action="store_true", help="List roles in the upstream inventory and exit.")
     parser.add_argument("--validate-inventory", action="store_true", help="Parse the inventory and exit without cloning.")
     args = parser.parse_args()
@@ -269,9 +409,22 @@ def main() -> int:
     if not selected:
         print(f"no matching roles found for: {', '.join(args.roles)}", file=sys.stderr)
         return 2
+    upstream_filter = split_filter_values(args.upstream, args.upstreams)
+    if upstream_filter:
+        known_upstreams = {repo["purpose"] for config in selected.values() for repo in config["repositories"]}
+        unknown_upstreams = sorted(upstream_filter - known_upstreams)
+        if unknown_upstreams:
+            print(f"no matching upstream purpose found for: {', '.join(unknown_upstreams)}", file=sys.stderr)
+            return 2
+    if args.clean_upstreams:
+        return clean_upstream_directories(selected, upstream_filter)
 
     rows: list[list[str]] = []
     failures: list[str] = []
+    total_steps = sum(len(filtered_repositories(config, upstream_filter)) for config in selected.values())
+    if not upstream_filter:
+        total_steps += sum(1 for config in selected.values() if config["local_compose_file"])
+    current_step = 0
 
     for role, config in selected.items():
         vendor_dir = ROOT / config["vendor_dir"]
@@ -285,20 +438,29 @@ def main() -> int:
         notes: list[str] = []
         snapshot_notes: list[str] = []
 
-        for repo in config["repositories"]:
+        for repo in filtered_repositories(config, upstream_filter):
+            current_step += 1
+            render_progress(current_step, total_steps, f"{role}/{repo['purpose']}")
             repo_count += 1
             with tempfile.TemporaryDirectory(prefix=f"nbops-upstream-{role}-") as tmp:
                 tmpdir = Path(tmp)
+                clone_dir = tmpdir / "repo"
+                purpose_dest = vendor_dir / repo["purpose"]
+                purpose_snapshot_dir = purpose_dest
+                if args.apply:
+                    purpose_snapshot_dir = tmpdir / "__snapshot__" / repo["purpose"]
+                    purpose_snapshot_dir.mkdir(parents=True, exist_ok=True)
+
                 ok = False
                 if "url" in repo:
-                    ok, output = run_capture(["git", "clone", "--depth", "1", "--branch", repo["ref"], repo["url"], str(tmpdir)])
+                    ok, output = run_capture(["git", "clone", "--depth", "1", "--branch", repo["ref"], repo["url"], str(clone_dir)])
                     if not ok:
                         notes.append(f"{repo['purpose']}: clone failed")
                         failures.append(f"{role}/{repo['purpose']}: {output.splitlines()[-1] if output.splitlines() else 'clone failed'}")
 
                 if ok:
                     for path in repo["readme_paths"]:
-                        result = copy_if_present(tmpdir, path, vendor_dir / repo["purpose"], args.apply)
+                        result = copy_if_present(clone_dir, path, purpose_snapshot_dir, args.apply)
                         if result == "missing":
                             readme_missing += 1
                         else:
@@ -307,30 +469,38 @@ def main() -> int:
 
                         if args.no_parse_readme:
                             continue
-                        candidates = readme_compose_candidates(tmpdir / path)
+                        candidates = readme_compose_candidates(clone_dir / path)
                         readme_candidates += len(candidates)
                         if len(candidates) > 1:
                             readme_ambiguous += 1
                         if args.apply and len(candidates) == 1:
-                            dest = vendor_dir / repo["purpose"] / "compose.from-readme.yaml"
+                            dest = purpose_snapshot_dir / NORMALIZED_COMPOSE_NAME
                             dest.parent.mkdir(parents=True, exist_ok=True)
                             dest.write_text(candidates[0]["body"])
 
                     for path in repo["compose_paths"]:
-                        result = copy_if_present(tmpdir, path, vendor_dir / repo["purpose"], args.apply)
+                        result = copy_compose_if_present(clone_dir, path, purpose_snapshot_dir, args.apply)
                         if result == "missing":
                             compose_missing += 1
+                        elif result == "invalid":
+                            compose_missing += 1
+                            notes.append(f"{repo['purpose']}: Compose `{path}` invalid")
                         else:
                             compose_found += 1
                         snapshot_notes.append(f"- {repo['purpose']} Compose `{path}`: {result}")
 
                 compose_urls = repo["compose_urls"]
                 for index, url in enumerate(compose_urls, start=1):
+                    if args.apply and (purpose_snapshot_dir / NORMALIZED_COMPOSE_NAME).is_file():
+                        snapshot_notes.append(
+                            f"- {repo['purpose']} Compose URL `{url}`: skipped; `{NORMALIZED_COMPOSE_NAME}` already staged"
+                        )
+                        continue
                     result, body, reason = fetch_compose_url(url)
                     if result == "found":
                         compose_found += 1
                         if args.apply:
-                            dest = vendor_dir / repo["purpose"] / compose_url_filename(index, len(compose_urls))
+                            dest = purpose_snapshot_dir / NORMALIZED_COMPOSE_NAME
                             dest.parent.mkdir(parents=True, exist_ok=True)
                             dest.write_text(body)
                             result = "copied"
@@ -340,7 +510,21 @@ def main() -> int:
                     suffix = f" ({reason})" if reason else ""
                     snapshot_notes.append(f"- {repo['purpose']} Compose URL `{url}`: {result}{suffix}")
 
-        if config["local_compose_file"]:
+                if args.apply and snapshot_has_content(purpose_snapshot_dir):
+                    if repo["purpose"] == "container" and preferred_override_exists(purpose_snapshot_dir):
+                        notes.append("container override preferred")
+                        snapshot_notes.append(
+                            "- container `docker-compose.override.yml`: prefer this when updating `templates/compose.yaml.j2`"
+                        )
+                    replace_directory(purpose_snapshot_dir, purpose_dest)
+                elif args.apply:
+                    snapshot_notes.append(f"- {repo['purpose']} snapshot: no files staged; existing purpose directory left unchanged")
+                elif repo["purpose"] == "container" and preferred_override_exists(purpose_dest):
+                    notes.append("container override preferred")
+
+        if config["local_compose_file"] and not upstream_filter:
+            current_step += 1
+            render_progress(current_step, total_steps, f"{role}/local")
             result = copy_local_file_if_present(config["local_compose_file"], vendor_dir, args.apply)
             if result in {"missing", "invalid"}:
                 compose_missing += 1
@@ -371,6 +555,7 @@ def main() -> int:
             ]
         )
 
+    clear_progress()
     print_table(rows)
 
     if failures:

templates/cloud-init-user-data.yml.j2

@@ -4,6 +4,9 @@ fqdn: {{ cloud_init_fqdn | default(nbops_fqdn) }}
 manage_etc_hosts: true
 timezone: {{ cloud_init_timezone | default(nbops_timezone) }}
 locale: {{ cloud_init_locale | default(nbops_locale) }}
+{% set serial_console_enabled = cloud_init_serial_console_enabled | default(true) %}
+{% set serial_console_tty = cloud_init_serial_console_tty | default('ttyS0') %}
+{% set serial_console_term = cloud_init_serial_console_term | default('xterm-256color') %}
 
 users:
   - default
@@ -72,6 +75,14 @@ write_files:
       net.ipv6.conf.all.forwarding = 1
       fs.inotify.max_user_watches = 1048576
       fs.inotify.max_user_instances = 1024
+{% if serial_console_enabled | bool %}
+  - path: /etc/systemd/system/serial-getty@{{ serial_console_tty }}.service.d/10-nbops-xterm.conf
+    owner: root:root
+    permissions: "0644"
+    content: |
+      [Service]
+      Environment=TERM={{ serial_console_term }}
+{% endif %}
 
 runcmd:
   - install -m 0755 -d /etc/apt/keyrings
@@ -87,9 +98,12 @@ runcmd:
   - ufw allow {{ cloud_init_ssh_port | default(nbops_ssh_port) }}/tcp
   - ufw --force enable
   - systemctl enable --now chrony qemu-guest-agent docker unattended-upgrades
+{% if serial_console_enabled | bool %}
+  - systemctl daemon-reload
+  - systemctl enable --now serial-getty@{{ serial_console_tty }}.service
+{% endif %}
   - sysctl --system
   - ansible-galaxy collection install -r {{ cloud_init_repo_path | default('/opt/nbops-ansible') }}/requirements.yml || true
   - ansible-pull -U {{ cloud_init_repo_url }} -i localhost, -c local {{ cloud_init_playbook | default('playbooks/pull.yml') }} --accept-host-key
 
 final_message: "nbops cloud-init bootstrap complete after $UPTIME seconds"
-