honestly its been months, i found unstaged changes and forgot
This commit is contained in:
boris committed
1 parent
b5d1ffda95
commit
ef8d5645e3
55 files changed
+1554
-180
No files matched your search
Executable
+425
@@ -0,0 +1,425 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
VMID=9000
|
||||
NAME=debian-13-genericcloud
|
||||
IMAGE_URL=https://cloud.debian.org/images/cloud/trixie/latest/debian-13-genericcloud-amd64.qcow2
|
||||
IMAGE_FILE=
|
||||
CACHE_DIR=/var/lib/vz/template/qcow2
|
||||
STORAGE=local-lvm
|
||||
CLOUDINIT_STORAGE=
|
||||
BRIDGE=vmbr0
|
||||
MEMORY=2048
|
||||
CORES=2
|
||||
CPU=host
|
||||
DISK_SIZE=32G
|
||||
MACHINE=q35
|
||||
BIOS=ovmf
|
||||
SCSI_HW=virtio-scsi-pci
|
||||
NET_MODEL=virtio
|
||||
PBS_STORAGE=
|
||||
DRY_RUN=0
|
||||
ASSUME_YES=0
|
||||
INTERACTIVE=1
|
||||
VMID_SET=0
|
||||
NAME_SET=0
|
||||
IMAGE_URL_SET=0
|
||||
IMAGE_FILE_SET=0
|
||||
CACHE_DIR_SET=0
|
||||
STORAGE_SET=0
|
||||
CLOUDINIT_STORAGE_SET=0
|
||||
BRIDGE_SET=0
|
||||
MEMORY_SET=0
|
||||
CORES_SET=0
|
||||
CPU_SET=0
|
||||
DISK_SIZE_SET=0
|
||||
PBS_STORAGE_SET=0
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Create a Proxmox VM template from the Debian 13 GenericCloud image.
|
||||
|
||||
Run this on a Proxmox node as root.
|
||||
|
||||
Usage:
|
||||
scripts/create-proxmox-template.sh [options]
|
||||
|
||||
Options:
|
||||
--vmid ID Template VMID. Default: 9000
|
||||
--name NAME Template name. Default: debian-13-genericcloud
|
||||
--storage STORAGE Disk storage. Default: local-lvm
|
||||
--cloudinit-storage NAME Cloud-Init storage. Default: same as --storage
|
||||
--bridge BRIDGE Network bridge. Default: vmbr0
|
||||
--memory MB Memory in MB. Default: 2048
|
||||
--cores N vCPU count. Default: 2
|
||||
--cpu TYPE CPU type. Default: host
|
||||
--disk-size SIZE Resize imported disk to this size. Default: 32G
|
||||
--image-url URL Debian GenericCloud qcow2 URL.
|
||||
--image-file PATH Use an existing local qcow2 instead of downloading.
|
||||
--cache-dir PATH Download cache directory. Default: /var/lib/vz/template/qcow2
|
||||
--pbs-storage NAME PBS storage/datastore ID. Enables main disk backup flag when set.
|
||||
--dry-run Print commands without executing them.
|
||||
--yes Accept defaults for any unanswered prompts.
|
||||
--non-interactive Do not prompt; use defaults and provided flags.
|
||||
-h, --help Show this help.
|
||||
|
||||
The template is configured with:
|
||||
- Q35 machine type
|
||||
- OVMF / UEFI
|
||||
- VirtIO SCSI
|
||||
- VirtIO network
|
||||
- QEMU guest agent
|
||||
- Cloud-Init drive
|
||||
- serial0 socket
|
||||
- vga serial0 for Proxmox xterm.js console support
|
||||
USAGE
|
||||
}
|
||||
|
||||
prompt_value() {
|
||||
local label=$1
|
||||
local default_value=$2
|
||||
local value
|
||||
|
||||
if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
|
||||
printf '%s\n' "$default_value"
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '%s [%s]: ' "$label" "$default_value" >&2
|
||||
read -r value
|
||||
if [ -z "$value" ]; then
|
||||
printf '%s\n' "$default_value"
|
||||
else
|
||||
printf '%s\n' "$value"
|
||||
fi
|
||||
}
|
||||
|
||||
prompt_optional_value() {
|
||||
local label=$1
|
||||
local value
|
||||
|
||||
if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
|
||||
printf '\n'
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '%s [leave empty]: ' "$label" >&2
|
||||
read -r value
|
||||
printf '%s\n' "$value"
|
||||
}
|
||||
|
||||
prompt_confirm() {
|
||||
local label=$1
|
||||
local value
|
||||
|
||||
if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '%s [Y/n]: ' "$label" >&2
|
||||
read -r value
|
||||
case "$value" in
|
||||
""|y|Y|yes|YES)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
log() {
|
||||
printf '%s\n' "$*"
|
||||
}
|
||||
|
||||
die() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
run() {
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
printf '+'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
return 0
|
||||
fi
|
||||
"$@"
|
||||
}
|
||||
|
||||
need_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
|
||||
}
|
||||
|
||||
download_image() {
|
||||
local url=$1
|
||||
local dest=$2
|
||||
|
||||
if [ -s "$dest" ]; then
|
||||
log "Using cached image: $dest"
|
||||
return 0
|
||||
fi
|
||||
|
||||
run mkdir -p "$(dirname "$dest")"
|
||||
if command -v curl >/dev/null 2>&1; then
|
||||
run curl -fL --retry 3 --retry-delay 2 -o "$dest" "$url"
|
||||
elif command -v wget >/dev/null 2>&1; then
|
||||
run wget -O "$dest" "$url"
|
||||
else
|
||||
die "curl or wget is required to download the image"
|
||||
fi
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--vmid)
|
||||
VMID=${2:?missing value for --vmid}
|
||||
VMID_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--name)
|
||||
NAME=${2:?missing value for --name}
|
||||
NAME_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--storage)
|
||||
STORAGE=${2:?missing value for --storage}
|
||||
STORAGE_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--cloudinit-storage)
|
||||
CLOUDINIT_STORAGE=${2:?missing value for --cloudinit-storage}
|
||||
CLOUDINIT_STORAGE_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--bridge)
|
||||
BRIDGE=${2:?missing value for --bridge}
|
||||
BRIDGE_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--memory)
|
||||
MEMORY=${2:?missing value for --memory}
|
||||
MEMORY_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--cores)
|
||||
CORES=${2:?missing value for --cores}
|
||||
CORES_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--cpu)
|
||||
CPU=${2:?missing value for --cpu}
|
||||
CPU_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--disk-size)
|
||||
DISK_SIZE=${2:?missing value for --disk-size}
|
||||
DISK_SIZE_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--image-url)
|
||||
IMAGE_URL=${2:?missing value for --image-url}
|
||||
IMAGE_URL_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--image-file)
|
||||
IMAGE_FILE=${2:?missing value for --image-file}
|
||||
IMAGE_FILE_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--cache-dir)
|
||||
CACHE_DIR=${2:?missing value for --cache-dir}
|
||||
CACHE_DIR_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--pbs-storage)
|
||||
PBS_STORAGE=${2:?missing value for --pbs-storage}
|
||||
PBS_STORAGE_SET=1
|
||||
shift 2
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
--yes)
|
||||
ASSUME_YES=1
|
||||
shift
|
||||
;;
|
||||
--non-interactive)
|
||||
INTERACTIVE=0
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
die "unknown option: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
collect_interactive_config() {
|
||||
if [ "$VMID_SET" -eq 0 ]; then
|
||||
VMID=$(prompt_value "Template VMID" "$VMID")
|
||||
fi
|
||||
if [ "$NAME_SET" -eq 0 ]; then
|
||||
NAME=$(prompt_value "Template name" "$NAME")
|
||||
fi
|
||||
if [ "$STORAGE_SET" -eq 0 ]; then
|
||||
STORAGE=$(prompt_value "Disk storage" "$STORAGE")
|
||||
fi
|
||||
if [ "$CLOUDINIT_STORAGE_SET" -eq 0 ]; then
|
||||
CLOUDINIT_STORAGE=$(prompt_value "Cloud-Init storage" "${CLOUDINIT_STORAGE:-$STORAGE}")
|
||||
fi
|
||||
if [ "$BRIDGE_SET" -eq 0 ]; then
|
||||
BRIDGE=$(prompt_value "Network bridge" "$BRIDGE")
|
||||
fi
|
||||
if [ "$MEMORY_SET" -eq 0 ]; then
|
||||
MEMORY=$(prompt_value "Memory MB" "$MEMORY")
|
||||
fi
|
||||
if [ "$CORES_SET" -eq 0 ]; then
|
||||
CORES=$(prompt_value "vCPU cores" "$CORES")
|
||||
fi
|
||||
if [ "$CPU_SET" -eq 0 ]; then
|
||||
CPU=$(prompt_value "CPU type" "$CPU")
|
||||
fi
|
||||
if [ "$DISK_SIZE_SET" -eq 0 ]; then
|
||||
DISK_SIZE=$(prompt_value "Disk size after resize" "$DISK_SIZE")
|
||||
fi
|
||||
if [ "$CACHE_DIR_SET" -eq 0 ]; then
|
||||
CACHE_DIR=$(prompt_value "Image cache directory" "$CACHE_DIR")
|
||||
fi
|
||||
if [ "$IMAGE_FILE_SET" -eq 0 ]; then
|
||||
IMAGE_FILE=$(prompt_optional_value "Local qcow2 image file")
|
||||
fi
|
||||
if [ -z "$IMAGE_FILE" ] && [ "$IMAGE_URL_SET" -eq 0 ]; then
|
||||
IMAGE_URL=$(prompt_value "Debian GenericCloud image URL" "$IMAGE_URL")
|
||||
fi
|
||||
if [ "$PBS_STORAGE_SET" -eq 0 ]; then
|
||||
PBS_STORAGE=$(prompt_optional_value "PBS storage/datastore ID for backup flag")
|
||||
fi
|
||||
}
|
||||
|
||||
print_summary() {
|
||||
cat <<SUMMARY
|
||||
|
||||
Template configuration:
|
||||
VMID: $VMID
|
||||
Name: $NAME
|
||||
Disk storage: $STORAGE
|
||||
Cloud-Init storage: $CLOUDINIT_STORAGE
|
||||
Bridge: $BRIDGE
|
||||
Memory: $MEMORY MB
|
||||
Cores: $CORES
|
||||
CPU: $CPU
|
||||
Disk size: $DISK_SIZE
|
||||
Machine: $MACHINE
|
||||
BIOS: $BIOS
|
||||
Serial console: serial0 socket, vga serial0
|
||||
SUMMARY
|
||||
|
||||
if [ -n "$IMAGE_FILE" ]; then
|
||||
printf ' Image file: %s\n' "$IMAGE_FILE"
|
||||
else
|
||||
printf ' Image URL: %s\n' "$IMAGE_URL"
|
||||
printf ' Cache directory: %s\n' "$CACHE_DIR"
|
||||
fi
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
printf ' Mode: dry-run\n'
|
||||
fi
|
||||
if [ -n "$PBS_STORAGE" ]; then
|
||||
printf ' Backup flag: enabled for main disk; PBS storage: %s\n' "$PBS_STORAGE"
|
||||
else
|
||||
printf ' Backup flag: disabled for main disk\n'
|
||||
fi
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
|
||||
collect_interactive_config
|
||||
|
||||
if [ -z "$CLOUDINIT_STORAGE" ]; then
|
||||
CLOUDINIT_STORAGE=$STORAGE
|
||||
fi
|
||||
|
||||
print_summary
|
||||
if ! prompt_confirm "Create this Proxmox template?"; then
|
||||
die "cancelled"
|
||||
fi
|
||||
|
||||
if [ "$DRY_RUN" -eq 0 ]; then
|
||||
need_command qm
|
||||
need_command awk
|
||||
fi
|
||||
|
||||
if [ "$(id -u)" -ne 0 ] && [ "$DRY_RUN" -eq 0 ]; then
|
||||
die "run this script as root on a Proxmox node"
|
||||
fi
|
||||
|
||||
if [ "$DRY_RUN" -eq 0 ] && qm status "$VMID" >/dev/null 2>&1; then
|
||||
die "VMID $VMID already exists; choose another --vmid or remove it manually"
|
||||
fi
|
||||
|
||||
local image_path
|
||||
if [ -n "$IMAGE_FILE" ]; then
|
||||
image_path=$IMAGE_FILE
|
||||
[ "$DRY_RUN" -eq 1 ] || [ -f "$image_path" ] || die "image file not found: $image_path"
|
||||
else
|
||||
image_path="$CACHE_DIR/$(basename "$IMAGE_URL")"
|
||||
download_image "$IMAGE_URL" "$image_path"
|
||||
fi
|
||||
|
||||
run qm create "$VMID" \
|
||||
--name "$NAME" \
|
||||
--ostype l26 \
|
||||
--machine "$MACHINE" \
|
||||
--bios "$BIOS" \
|
||||
--memory "$MEMORY" \
|
||||
--cores "$CORES" \
|
||||
--cpu "$CPU" \
|
||||
--agent enabled=1 \
|
||||
--scsihw "$SCSI_HW" \
|
||||
--net0 "$NET_MODEL,bridge=$BRIDGE" \
|
||||
--serial0 socket \
|
||||
--vga serial0
|
||||
|
||||
if [ "$BIOS" = "ovmf" ]; then
|
||||
run qm set "$VMID" --efidisk0 "$STORAGE:0,pre-enrolled-keys=0"
|
||||
fi
|
||||
|
||||
run qm importdisk "$VMID" "$image_path" "$STORAGE"
|
||||
|
||||
local imported_disk
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
imported_disk="$STORAGE:vm-$VMID-disk-0"
|
||||
log "Dry-run assumption: imported disk will be $imported_disk"
|
||||
else
|
||||
imported_disk=$(qm config "$VMID" | awk -F': ' '/^unused[0-9]+:/{print $2; exit}')
|
||||
[ -n "$imported_disk" ] || die "could not find imported unused disk in qm config for VMID $VMID"
|
||||
fi
|
||||
|
||||
local disk_backup=0
|
||||
if [ -n "$PBS_STORAGE" ]; then
|
||||
disk_backup=1
|
||||
fi
|
||||
|
||||
run qm set "$VMID" --scsi0 "$imported_disk,discard=on,ssd=1,backup=$disk_backup"
|
||||
run qm set "$VMID" --ide2 "$CLOUDINIT_STORAGE:cloudinit"
|
||||
run qm set "$VMID" --boot order=scsi0
|
||||
run qm set "$VMID" --ipconfig0 ip=dhcp
|
||||
|
||||
if [ -n "$DISK_SIZE" ]; then
|
||||
run qm resize "$VMID" scsi0 "$DISK_SIZE"
|
||||
fi
|
||||
|
||||
run qm template "$VMID"
|
||||
|
||||
log "Created Proxmox template $NAME ($VMID)"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Executable
+367
@@ -0,0 +1,367 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
HOST_VARS_DIR=host_vars
|
||||
TEMPLATE_VMID=9000
|
||||
TEMPLATE_STORAGE=local-lvm
|
||||
TEMPLATE_BRIDGE=vmbr0
|
||||
PBS_STORAGE=
|
||||
DRY_RUN=0
|
||||
ASSUME_YES=0
|
||||
INTERACTIVE=1
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Initialise Proxmox VMs from host_vars/*.yml.
|
||||
|
||||
Run this on a Proxmox node as root from the repository root.
|
||||
|
||||
Each host var file must set proxmox_vmid to be managed by this script.
|
||||
Files without proxmox_vmid are skipped.
|
||||
|
||||
Supported host vars:
|
||||
proxmox_vmid: 101
|
||||
proxmox_name: cloud
|
||||
proxmox_template_vmid: 9000
|
||||
proxmox_storage: local-lvm
|
||||
proxmox_bridge: vmbr0
|
||||
proxmox_memory: 4096
|
||||
proxmox_cores: 2
|
||||
proxmox_cpu: host
|
||||
proxmox_disk_size: 64G
|
||||
proxmox_backup_enabled: true
|
||||
proxmox_ipconfig0: ip=dhcp
|
||||
proxmox_onboot: true
|
||||
proxmox_start: false
|
||||
|
||||
Options:
|
||||
--host-vars-dir PATH Host vars directory. Default: host_vars
|
||||
--template-vmid ID Default template VMID. Default: 9000
|
||||
--template-storage NAME Storage used if template must be created. Default: local-lvm
|
||||
--template-bridge NAME Bridge used if template must be created. Default: vmbr0
|
||||
--pbs-storage NAME PBS storage/datastore ID. Passed to template script and used as default backup target note.
|
||||
--dry-run Print commands without executing them.
|
||||
--yes Accept defaults for prompts.
|
||||
--non-interactive Do not prompt.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
log() {
|
||||
printf '%s\n' "$*"
|
||||
}
|
||||
|
||||
die() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
run() {
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
printf '+'
|
||||
printf ' %q' "$@"
|
||||
printf '\n'
|
||||
return 0
|
||||
fi
|
||||
"$@"
|
||||
}
|
||||
|
||||
need_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
|
||||
}
|
||||
|
||||
prompt_value() {
|
||||
local label=$1
|
||||
local default_value=$2
|
||||
local value
|
||||
|
||||
if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
|
||||
printf '%s\n' "$default_value"
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '%s [%s]: ' "$label" "$default_value" >&2
|
||||
read -r value
|
||||
if [ -z "$value" ]; then
|
||||
printf '%s\n' "$default_value"
|
||||
else
|
||||
printf '%s\n' "$value"
|
||||
fi
|
||||
}
|
||||
|
||||
prompt_optional_value() {
|
||||
local label=$1
|
||||
local value
|
||||
|
||||
if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
|
||||
printf '\n'
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '%s [leave empty]: ' "$label" >&2
|
||||
read -r value
|
||||
printf '%s\n' "$value"
|
||||
}
|
||||
|
||||
prompt_confirm() {
|
||||
local label=$1
|
||||
local value
|
||||
|
||||
if [ "$ASSUME_YES" -eq 1 ] || [ "$INTERACTIVE" -eq 0 ] || [ ! -t 0 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '%s [Y/n]: ' "$label" >&2
|
||||
read -r value
|
||||
case "$value" in
|
||||
""|y|Y|yes|YES)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
yaml_scalar() {
|
||||
local file=$1
|
||||
local key=$2
|
||||
awk -v key="$key" '
|
||||
$0 ~ "^[[:space:]]*" key ":" {
|
||||
sub("^[[:space:]]*" key ":[[:space:]]*", "", $0)
|
||||
sub("[[:space:]]+#.*$", "", $0)
|
||||
gsub(/^["'\'']|["'\'']$/, "", $0)
|
||||
print $0
|
||||
exit
|
||||
}
|
||||
' "$file"
|
||||
}
|
||||
|
||||
bool_true() {
|
||||
case "${1:-}" in
|
||||
true|True|TRUE|yes|Yes|YES|1|on|On|ON)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--host-vars-dir)
|
||||
HOST_VARS_DIR=${2:?missing value for --host-vars-dir}
|
||||
shift 2
|
||||
;;
|
||||
--template-vmid)
|
||||
TEMPLATE_VMID=${2:?missing value for --template-vmid}
|
||||
shift 2
|
||||
;;
|
||||
--template-storage)
|
||||
TEMPLATE_STORAGE=${2:?missing value for --template-storage}
|
||||
shift 2
|
||||
;;
|
||||
--template-bridge)
|
||||
TEMPLATE_BRIDGE=${2:?missing value for --template-bridge}
|
||||
shift 2
|
||||
;;
|
||||
--pbs-storage)
|
||||
PBS_STORAGE=${2:?missing value for --pbs-storage}
|
||||
shift 2
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
--yes)
|
||||
ASSUME_YES=1
|
||||
shift
|
||||
;;
|
||||
--non-interactive)
|
||||
INTERACTIVE=0
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
die "unknown option: $1"
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
collect_config() {
|
||||
TEMPLATE_VMID=$(prompt_value "Default template VMID" "$TEMPLATE_VMID")
|
||||
TEMPLATE_STORAGE=$(prompt_value "Default template/storage name" "$TEMPLATE_STORAGE")
|
||||
TEMPLATE_BRIDGE=$(prompt_value "Default bridge" "$TEMPLATE_BRIDGE")
|
||||
if [ -z "$PBS_STORAGE" ]; then
|
||||
PBS_STORAGE=$(prompt_optional_value "PBS storage/datastore ID")
|
||||
fi
|
||||
}
|
||||
|
||||
template_exists() {
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
return 1
|
||||
fi
|
||||
qm status "$TEMPLATE_VMID" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
vm_exists() {
|
||||
local vmid=$1
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
return 1
|
||||
fi
|
||||
qm status "$vmid" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
ensure_template() {
|
||||
if template_exists; then
|
||||
log "Template VMID $TEMPLATE_VMID already exists."
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Template VMID $TEMPLATE_VMID not found; creating it first."
|
||||
local args=(
|
||||
scripts/create-proxmox-template.sh
|
||||
--vmid "$TEMPLATE_VMID"
|
||||
--storage "$TEMPLATE_STORAGE"
|
||||
--bridge "$TEMPLATE_BRIDGE"
|
||||
--yes
|
||||
--non-interactive
|
||||
)
|
||||
if [ -n "$PBS_STORAGE" ]; then
|
||||
args+=(--pbs-storage "$PBS_STORAGE")
|
||||
fi
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
args+=(--dry-run)
|
||||
fi
|
||||
run "${args[@]}"
|
||||
}
|
||||
|
||||
disk_volume() {
|
||||
local vmid=$1
|
||||
local storage=$2
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
printf '%s:vm-%s-disk-0\n' "$storage" "$vmid"
|
||||
return 0
|
||||
fi
|
||||
qm config "$vmid" | awk -F': ' '/^scsi0:/{split($2, parts, ","); print parts[1]; exit}'
|
||||
}
|
||||
|
||||
init_vm_from_host_vars() {
|
||||
local file=$1
|
||||
local stem
|
||||
stem=$(basename "$file" .yml)
|
||||
|
||||
local vmid
|
||||
vmid=$(yaml_scalar "$file" proxmox_vmid)
|
||||
if [ -z "$vmid" ]; then
|
||||
log "Skipping $file: proxmox_vmid not set"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local name template_vmid storage bridge memory cores cpu disk_size backup_enabled ipconfig0 onboot start_vm
|
||||
name=$(yaml_scalar "$file" proxmox_name)
|
||||
[ -n "$name" ] || name=$(yaml_scalar "$file" nbops_hostname)
|
||||
[ -n "$name" ] || name=$stem
|
||||
template_vmid=$(yaml_scalar "$file" proxmox_template_vmid)
|
||||
[ -n "$template_vmid" ] || template_vmid=$TEMPLATE_VMID
|
||||
storage=$(yaml_scalar "$file" proxmox_storage)
|
||||
[ -n "$storage" ] || storage=$TEMPLATE_STORAGE
|
||||
bridge=$(yaml_scalar "$file" proxmox_bridge)
|
||||
[ -n "$bridge" ] || bridge=$TEMPLATE_BRIDGE
|
||||
memory=$(yaml_scalar "$file" proxmox_memory)
|
||||
[ -n "$memory" ] || memory=2048
|
||||
cores=$(yaml_scalar "$file" proxmox_cores)
|
||||
[ -n "$cores" ] || cores=2
|
||||
cpu=$(yaml_scalar "$file" proxmox_cpu)
|
||||
[ -n "$cpu" ] || cpu=host
|
||||
disk_size=$(yaml_scalar "$file" proxmox_disk_size)
|
||||
[ -n "$disk_size" ] || disk_size=32G
|
||||
backup_enabled=$(yaml_scalar "$file" proxmox_backup_enabled)
|
||||
[ -n "$backup_enabled" ] || backup_enabled=false
|
||||
ipconfig0=$(yaml_scalar "$file" proxmox_ipconfig0)
|
||||
[ -n "$ipconfig0" ] || ipconfig0=ip=dhcp
|
||||
onboot=$(yaml_scalar "$file" proxmox_onboot)
|
||||
[ -n "$onboot" ] || onboot=true
|
||||
start_vm=$(yaml_scalar "$file" proxmox_start)
|
||||
[ -n "$start_vm" ] || start_vm=false
|
||||
|
||||
if vm_exists "$vmid"; then
|
||||
log "Skipping $name ($vmid): VM already exists"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Initialising $name ($vmid) from $file"
|
||||
run qm clone "$template_vmid" "$vmid" --name "$name" --full true --storage "$storage"
|
||||
run qm set "$vmid" --memory "$memory" --cores "$cores" --cpu "$cpu"
|
||||
run qm set "$vmid" --net0 "virtio,bridge=$bridge"
|
||||
run qm set "$vmid" --ipconfig0 "$ipconfig0"
|
||||
|
||||
if bool_true "$onboot"; then
|
||||
run qm set "$vmid" --onboot 1
|
||||
else
|
||||
run qm set "$vmid" --onboot 0
|
||||
fi
|
||||
|
||||
local volume backup_flag
|
||||
volume=$(disk_volume "$vmid" "$storage")
|
||||
backup_flag=0
|
||||
if bool_true "$backup_enabled"; then
|
||||
backup_flag=1
|
||||
fi
|
||||
run qm set "$vmid" --scsi0 "$volume,discard=on,ssd=1,backup=$backup_flag"
|
||||
|
||||
if [ -n "$disk_size" ]; then
|
||||
run qm resize "$vmid" scsi0 "$disk_size"
|
||||
fi
|
||||
|
||||
if bool_true "$start_vm"; then
|
||||
run qm start "$vmid"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
collect_config
|
||||
|
||||
if [ "$DRY_RUN" -eq 0 ]; then
|
||||
need_command qm
|
||||
need_command awk
|
||||
[ "$(id -u)" -eq 0 ] || die "run this script as root on a Proxmox node"
|
||||
fi
|
||||
|
||||
[ -d "$HOST_VARS_DIR" ] || die "host vars directory not found: $HOST_VARS_DIR"
|
||||
|
||||
cat <<SUMMARY
|
||||
|
||||
VM initialisation:
|
||||
Host vars: $HOST_VARS_DIR/*.yml
|
||||
Template VMID: $TEMPLATE_VMID
|
||||
Default storage: $TEMPLATE_STORAGE
|
||||
Default bridge: $TEMPLATE_BRIDGE
|
||||
PBS storage: ${PBS_STORAGE:-not set}
|
||||
Mode: $([ "$DRY_RUN" -eq 1 ] && printf 'dry-run' || printf 'apply')
|
||||
|
||||
SUMMARY
|
||||
|
||||
if ! prompt_confirm "Initialise VMs from host vars?"; then
|
||||
die "cancelled"
|
||||
fi
|
||||
|
||||
ensure_template
|
||||
|
||||
local found=0
|
||||
for file in "$HOST_VARS_DIR"/*.yml; do
|
||||
[ -e "$file" ] || continue
|
||||
found=1
|
||||
init_vm_from_host_vars "$file"
|
||||
done
|
||||
[ "$found" -eq 1 ] || die "no .yml files found in $HOST_VARS_DIR"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -22,6 +22,9 @@ from urllib.request import Request, urlopen
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
INVENTORY = ROOT / "docs" / "upstream-sources.yml"
|
||||
COMPOSE_BLOCK_LANGS = {"", "yaml", "yml", "docker-compose", "compose"}
|
||||
NORMALIZED_COMPOSE_NAME = "docker-compose.yml"
|
||||
DERIVED_TEMPLATE_MARKER = "nbops: derived-from-upstream"
|
||||
DERIVED_TEMPLATE_NAMES = ("compose.yaml.j2", "compose.override.yaml.j2")
|
||||
|
||||
|
||||
def parse_inline_list(value: str) -> list[str]:
|
||||
@@ -137,6 +140,21 @@ def copy_if_present(src_root: Path, relative_path: str, dest_root: Path, apply:
|
||||
return "copied"
|
||||
|
||||
|
||||
def copy_compose_if_present(src_root: Path, relative_path: str, dest_root: Path, apply: bool) -> str:
|
||||
src = src_root / relative_path
|
||||
if not src.is_file():
|
||||
return "missing"
|
||||
if not looks_like_compose(src.read_text(errors="replace")):
|
||||
return "invalid"
|
||||
if not apply:
|
||||
return "found"
|
||||
|
||||
dest = dest_root / NORMALIZED_COMPOSE_NAME
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src, dest)
|
||||
return "copied"
|
||||
|
||||
|
||||
def copy_local_file_if_present(relative_path: str, dest_root: Path, apply: bool) -> str:
|
||||
src = ROOT / relative_path
|
||||
if not src.is_file():
|
||||
@@ -152,6 +170,17 @@ def copy_local_file_if_present(relative_path: str, dest_root: Path, apply: bool)
|
||||
return "copied"
|
||||
|
||||
|
||||
def replace_directory(src: Path, dest: Path) -> None:
|
||||
if dest.exists():
|
||||
shutil.rmtree(dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.move(str(src), str(dest))
|
||||
|
||||
|
||||
def preferred_override_exists(snapshot_dir: Path) -> bool:
|
||||
return (snapshot_dir / "docker-compose.override.yml").is_file()
|
||||
|
||||
|
||||
def fetch_compose_url(url: str) -> tuple[str, str, str]:
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme not in {"http", "https"}:
|
||||
@@ -173,12 +202,6 @@ def fetch_compose_url(url: str) -> tuple[str, str, str]:
|
||||
return "found", body, ""
|
||||
|
||||
|
||||
def compose_url_filename(index: int, total: int) -> str:
|
||||
if total == 1:
|
||||
return "compose.from-url.yaml"
|
||||
return f"compose.from-url-{index}.yaml"
|
||||
|
||||
|
||||
def extract_fenced_blocks(markdown: str) -> list[tuple[str, str]]:
|
||||
pattern = re.compile(r"^```([^\n`]*)\n(.*?)^```", re.MULTILINE | re.DOTALL)
|
||||
blocks = []
|
||||
@@ -226,6 +249,61 @@ def print_table(rows: list[list[str]]) -> None:
|
||||
print(fmt(row))
|
||||
|
||||
|
||||
def progress_enabled() -> bool:
|
||||
return sys.stderr.isatty()
|
||||
|
||||
|
||||
def render_progress(current: int, total: int, label: str) -> None:
|
||||
if not progress_enabled() or total <= 0:
|
||||
return
|
||||
width = 28
|
||||
filled = min(width, int(width * current / total))
|
||||
bar = "#" * filled + "-" * (width - filled)
|
||||
sys.stderr.write(f"\r\033[K[{bar}] {current}/{total} {label}")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def clear_progress() -> None:
|
||||
if progress_enabled():
|
||||
sys.stderr.write("\r\033[K")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def snapshot_has_content(path: Path) -> bool:
|
||||
return path.exists() and any(path.iterdir())
|
||||
|
||||
|
||||
def split_filter_values(values: list[str], csv_value: str) -> set[str]:
|
||||
filters: set[str] = set()
|
||||
for value in values:
|
||||
filters.update(item.strip() for item in value.split(",") if item.strip())
|
||||
filters.update(item.strip() for item in csv_value.split(",") if item.strip())
|
||||
return filters
|
||||
|
||||
|
||||
def filtered_repositories(config: dict, upstream_filter: set[str]) -> list[dict]:
|
||||
repos = config["repositories"]
|
||||
if not upstream_filter:
|
||||
return repos
|
||||
return [repo for repo in repos if repo["purpose"] in upstream_filter]
|
||||
|
||||
|
||||
def clean_derived_templates(role: str) -> list[list[str]]:
|
||||
rows: list[list[str]] = []
|
||||
template_dir = ROOT / "roles" / role / "templates"
|
||||
for name in DERIVED_TEMPLATE_NAMES:
|
||||
path = template_dir / name
|
||||
if not path.is_file():
|
||||
continue
|
||||
text = path.read_text(errors="replace")
|
||||
if DERIVED_TEMPLATE_MARKER not in text:
|
||||
rows.append([role, "template", str(path), "kept: unmarked"])
|
||||
continue
|
||||
path.unlink()
|
||||
rows.append([role, "template", str(path), "removed"])
|
||||
return rows
|
||||
|
||||
|
||||
def snapshot_source(vendor_dir: Path, role: str, status: str, notes: list[str], apply: bool) -> None:
|
||||
if not apply:
|
||||
return
|
||||
@@ -246,11 +324,73 @@ def snapshot_source(vendor_dir: Path, role: str, status: str, notes: list[str],
|
||||
(vendor_dir / "SOURCE.md").write_text("\n".join(text))
|
||||
|
||||
|
||||
def clean_upstream_directories(selected: dict[str, dict], upstream_filter: set[str]) -> int:
|
||||
rows: list[list[str]] = []
|
||||
removed = 0
|
||||
total = sum(len(filtered_repositories(config, upstream_filter)) for config in selected.values())
|
||||
current = 0
|
||||
|
||||
for role, config in selected.items():
|
||||
vendor_dir = ROOT / config["vendor_dir"]
|
||||
repos = filtered_repositories(config, upstream_filter)
|
||||
for repo in repos:
|
||||
current += 1
|
||||
purpose_dir = vendor_dir / repo["purpose"]
|
||||
render_progress(current, total, f"clean {role}/{repo['purpose']}")
|
||||
if purpose_dir.is_dir():
|
||||
shutil.rmtree(purpose_dir)
|
||||
removed += 1
|
||||
result = "removed"
|
||||
else:
|
||||
result = "missing"
|
||||
rows.append([role, repo["purpose"], str(purpose_dir), result])
|
||||
rows.extend(clean_derived_templates(role))
|
||||
|
||||
clear_progress()
|
||||
headers = ["Role", "Purpose", "Path", "Result"]
|
||||
widths = [len(header) for header in headers]
|
||||
for row in rows:
|
||||
for index, value in enumerate(row):
|
||||
widths[index] = max(widths[index], len(value))
|
||||
|
||||
def fmt(row: list[str]) -> str:
|
||||
return " ".join(value.ljust(widths[index]) for index, value in enumerate(row))
|
||||
|
||||
print(fmt(headers))
|
||||
print(" ".join("-" * width for width in widths))
|
||||
for row in rows:
|
||||
print(fmt(row))
|
||||
print(f"\nRemoved {removed} upstream purpose directories.")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Test or vendor upstream README/Compose snapshots.")
|
||||
parser.add_argument("roles", nargs="*", help="Role names to process. Defaults to all roles.")
|
||||
parser.add_argument("--apply", action="store_true", help="Copy fetched files into role upstream directories.")
|
||||
parser.add_argument("--no-parse-readme", action="store_true", help="Do not extract Compose YAML from README blocks.")
|
||||
parser.add_argument(
|
||||
"--clean-upstreams",
|
||||
action="store_true",
|
||||
help="Delete upstream/<purpose> directories and marked derived templates for selected roles, then exit.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--clean-purposes",
|
||||
dest="clean_upstreams",
|
||||
action="store_true",
|
||||
help=argparse.SUPPRESS,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--upstream",
|
||||
action="append",
|
||||
default=[],
|
||||
help="Limit processing to an upstream purpose name such as container or application. Repeatable.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--upstreams",
|
||||
default="",
|
||||
help="Comma-separated upstream purpose names to process.",
|
||||
)
|
||||
parser.add_argument("--list", action="store_true", help="List roles in the upstream inventory and exit.")
|
||||
parser.add_argument("--validate-inventory", action="store_true", help="Parse the inventory and exit without cloning.")
|
||||
args = parser.parse_args()
|
||||
@@ -269,9 +409,22 @@ def main() -> int:
|
||||
if not selected:
|
||||
print(f"no matching roles found for: {', '.join(args.roles)}", file=sys.stderr)
|
||||
return 2
|
||||
upstream_filter = split_filter_values(args.upstream, args.upstreams)
|
||||
if upstream_filter:
|
||||
known_upstreams = {repo["purpose"] for config in selected.values() for repo in config["repositories"]}
|
||||
unknown_upstreams = sorted(upstream_filter - known_upstreams)
|
||||
if unknown_upstreams:
|
||||
print(f"no matching upstream purpose found for: {', '.join(unknown_upstreams)}", file=sys.stderr)
|
||||
return 2
|
||||
if args.clean_upstreams:
|
||||
return clean_upstream_directories(selected, upstream_filter)
|
||||
|
||||
rows: list[list[str]] = []
|
||||
failures: list[str] = []
|
||||
total_steps = sum(len(filtered_repositories(config, upstream_filter)) for config in selected.values())
|
||||
if not upstream_filter:
|
||||
total_steps += sum(1 for config in selected.values() if config["local_compose_file"])
|
||||
current_step = 0
|
||||
|
||||
for role, config in selected.items():
|
||||
vendor_dir = ROOT / config["vendor_dir"]
|
||||
@@ -285,20 +438,29 @@ def main() -> int:
|
||||
notes: list[str] = []
|
||||
snapshot_notes: list[str] = []
|
||||
|
||||
for repo in config["repositories"]:
|
||||
for repo in filtered_repositories(config, upstream_filter):
|
||||
current_step += 1
|
||||
render_progress(current_step, total_steps, f"{role}/{repo['purpose']}")
|
||||
repo_count += 1
|
||||
with tempfile.TemporaryDirectory(prefix=f"nbops-upstream-{role}-") as tmp:
|
||||
tmpdir = Path(tmp)
|
||||
clone_dir = tmpdir / "repo"
|
||||
purpose_dest = vendor_dir / repo["purpose"]
|
||||
purpose_snapshot_dir = purpose_dest
|
||||
if args.apply:
|
||||
purpose_snapshot_dir = tmpdir / "__snapshot__" / repo["purpose"]
|
||||
purpose_snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
ok = False
|
||||
if "url" in repo:
|
||||
ok, output = run_capture(["git", "clone", "--depth", "1", "--branch", repo["ref"], repo["url"], str(tmpdir)])
|
||||
ok, output = run_capture(["git", "clone", "--depth", "1", "--branch", repo["ref"], repo["url"], str(clone_dir)])
|
||||
if not ok:
|
||||
notes.append(f"{repo['purpose']}: clone failed")
|
||||
failures.append(f"{role}/{repo['purpose']}: {output.splitlines()[-1] if output.splitlines() else 'clone failed'}")
|
||||
|
||||
if ok:
|
||||
for path in repo["readme_paths"]:
|
||||
result = copy_if_present(tmpdir, path, vendor_dir / repo["purpose"], args.apply)
|
||||
result = copy_if_present(clone_dir, path, purpose_snapshot_dir, args.apply)
|
||||
if result == "missing":
|
||||
readme_missing += 1
|
||||
else:
|
||||
@@ -307,30 +469,38 @@ def main() -> int:
|
||||
|
||||
if args.no_parse_readme:
|
||||
continue
|
||||
candidates = readme_compose_candidates(tmpdir / path)
|
||||
candidates = readme_compose_candidates(clone_dir / path)
|
||||
readme_candidates += len(candidates)
|
||||
if len(candidates) > 1:
|
||||
readme_ambiguous += 1
|
||||
if args.apply and len(candidates) == 1:
|
||||
dest = vendor_dir / repo["purpose"] / "compose.from-readme.yaml"
|
||||
dest = purpose_snapshot_dir / NORMALIZED_COMPOSE_NAME
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(candidates[0]["body"])
|
||||
|
||||
for path in repo["compose_paths"]:
|
||||
result = copy_if_present(tmpdir, path, vendor_dir / repo["purpose"], args.apply)
|
||||
result = copy_compose_if_present(clone_dir, path, purpose_snapshot_dir, args.apply)
|
||||
if result == "missing":
|
||||
compose_missing += 1
|
||||
elif result == "invalid":
|
||||
compose_missing += 1
|
||||
notes.append(f"{repo['purpose']}: Compose `{path}` invalid")
|
||||
else:
|
||||
compose_found += 1
|
||||
snapshot_notes.append(f"- {repo['purpose']} Compose `{path}`: {result}")
|
||||
|
||||
compose_urls = repo["compose_urls"]
|
||||
for index, url in enumerate(compose_urls, start=1):
|
||||
if args.apply and (purpose_snapshot_dir / NORMALIZED_COMPOSE_NAME).is_file():
|
||||
snapshot_notes.append(
|
||||
f"- {repo['purpose']} Compose URL `{url}`: skipped; `{NORMALIZED_COMPOSE_NAME}` already staged"
|
||||
)
|
||||
continue
|
||||
result, body, reason = fetch_compose_url(url)
|
||||
if result == "found":
|
||||
compose_found += 1
|
||||
if args.apply:
|
||||
dest = vendor_dir / repo["purpose"] / compose_url_filename(index, len(compose_urls))
|
||||
dest = purpose_snapshot_dir / NORMALIZED_COMPOSE_NAME
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
dest.write_text(body)
|
||||
result = "copied"
|
||||
@@ -340,7 +510,21 @@ def main() -> int:
|
||||
suffix = f" ({reason})" if reason else ""
|
||||
snapshot_notes.append(f"- {repo['purpose']} Compose URL `{url}`: {result}{suffix}")
|
||||
|
||||
if config["local_compose_file"]:
|
||||
if args.apply and snapshot_has_content(purpose_snapshot_dir):
|
||||
if repo["purpose"] == "container" and preferred_override_exists(purpose_snapshot_dir):
|
||||
notes.append("container override preferred")
|
||||
snapshot_notes.append(
|
||||
"- container `docker-compose.override.yml`: prefer this when updating `templates/compose.yaml.j2`"
|
||||
)
|
||||
replace_directory(purpose_snapshot_dir, purpose_dest)
|
||||
elif args.apply:
|
||||
snapshot_notes.append(f"- {repo['purpose']} snapshot: no files staged; existing purpose directory left unchanged")
|
||||
elif repo["purpose"] == "container" and preferred_override_exists(purpose_dest):
|
||||
notes.append("container override preferred")
|
||||
|
||||
if config["local_compose_file"] and not upstream_filter:
|
||||
current_step += 1
|
||||
render_progress(current_step, total_steps, f"{role}/local")
|
||||
result = copy_local_file_if_present(config["local_compose_file"], vendor_dir, args.apply)
|
||||
if result in {"missing", "invalid"}:
|
||||
compose_missing += 1
|
||||
@@ -371,6 +555,7 @@ def main() -> int:
|
||||
]
|
||||
)
|
||||
|
||||
clear_progress()
|
||||
print_table(rows)
|
||||
|
||||
if failures:
|
||||
|
||||
Reference in new issue
Block a user