276 lines
7.0 KiB
Go
276 lines
7.0 KiB
Go
|
// proxmox-backup-watch
|
||
|
|
//
|
||
|
|
// Waits for the vzdump task of the given VMID(s) to finish, then restarts the
|
||
|
|
// guest (QEMU VM or LXC container). By default it restarts regardless of the
|
||
|
|
// backup result; use -only-failed to restart only when the backup did not
|
||
|
|
// finish OK.
|
||
|
|
//
|
||
|
|
// Usage: proxmox-backup-watch [options] VMID [VMID...]
|
||
|
|
package main
|
||
|
|
|
||
|
|
import (
|
||
|
|
"bytes"
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"flag"
|
||
|
|
"fmt"
|
||
|
|
"os"
|
||
|
|
"os/exec"
|
||
|
|
"os/signal"
|
||
|
|
"strings"
|
||
|
|
"syscall"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
// Set at build time with -ldflags (see Makefile).
|
||
|
|
var (
|
||
|
|
version = "dev"
|
||
|
|
commit = "none"
|
||
|
|
date = "unknown"
|
||
|
|
)
|
||
|
|
|
||
|
|
type task struct {
|
||
|
|
ID string `json:"id"`
|
||
|
|
StartTime int64 `json:"starttime"`
|
||
|
|
EndTime *int64 `json:"endtime"`
|
||
|
|
Status string `json:"status"`
|
||
|
|
UPID string `json:"upid"`
|
||
|
|
}
|
||
|
|
|
||
|
|
type config struct {
|
||
|
|
node string
|
||
|
|
dryRun bool
|
||
|
|
timeout time.Duration
|
||
|
|
interval time.Duration
|
||
|
|
onlyFailed bool
|
||
|
|
warnOK bool
|
||
|
|
startedAt int64
|
||
|
|
}
|
||
|
|
|
||
|
|
func main() {
|
||
|
|
os.Exit(run())
|
||
|
|
}
|
||
|
|
|
||
|
|
func run() int {
|
||
|
|
cfg := config{startedAt: time.Now().Unix()}
|
||
|
|
|
||
|
|
fs := flag.NewFlagSet("proxmox-backup-watch", flag.ContinueOnError)
|
||
|
|
fs.BoolVar(&cfg.dryRun, "dry-run", false, "show what would be done, don't restart anything")
|
||
|
|
fs.BoolVar(&cfg.dryRun, "n", false, "shorthand for -dry-run")
|
||
|
|
fs.DurationVar(&cfg.timeout, "timeout", 6*time.Hour, "max time to wait per VMID (0 = forever)")
|
||
|
|
fs.DurationVar(&cfg.timeout, "t", 6*time.Hour, "shorthand for -timeout")
|
||
|
|
fs.DurationVar(&cfg.interval, "interval", 30*time.Second, "poll interval")
|
||
|
|
fs.DurationVar(&cfg.interval, "i", 30*time.Second, "shorthand for -interval")
|
||
|
|
fs.BoolVar(&cfg.onlyFailed, "only-failed", false, "restart only if the backup did NOT finish OK")
|
||
|
|
fs.BoolVar(&cfg.onlyFailed, "f", false, "shorthand for -only-failed")
|
||
|
|
fs.BoolVar(&cfg.warnOK, "warn-ok", false, `with -only-failed, treat "WARNINGS: n" as success`)
|
||
|
|
fs.BoolVar(&cfg.warnOK, "w", false, "shorthand for -warn-ok")
|
||
|
|
fs.StringVar(&cfg.node, "node", "", "Proxmox node name (default: short hostname)")
|
||
|
|
var showVersion bool
|
||
|
|
fs.BoolVar(&showVersion, "version", false, "print version and exit")
|
||
|
|
fs.BoolVar(&showVersion, "v", false, "shorthand for -version")
|
||
|
|
fs.Usage = func() {
|
||
|
|
fmt.Fprintln(os.Stderr, "Usage: proxmox-backup-watch [options] VMID [VMID...]")
|
||
|
|
fs.PrintDefaults()
|
||
|
|
}
|
||
|
|
|
||
|
|
// Allow flags before, between and after VMIDs.
|
||
|
|
var vmids []string
|
||
|
|
args := os.Args[1:]
|
||
|
|
for {
|
||
|
|
if err := fs.Parse(args); err != nil {
|
||
|
|
if errors.Is(err, flag.ErrHelp) {
|
||
|
|
return 0
|
||
|
|
}
|
||
|
|
return 2
|
||
|
|
}
|
||
|
|
rest := fs.Args()
|
||
|
|
if len(rest) == 0 {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
vmids = append(vmids, rest[0])
|
||
|
|
args = rest[1:]
|
||
|
|
}
|
||
|
|
|
||
|
|
if showVersion {
|
||
|
|
fmt.Printf("proxmox-backup-watch %s (commit %s, built %s)\n", version, commit, date)
|
||
|
|
return 0
|
||
|
|
}
|
||
|
|
|
||
|
|
if len(vmids) == 0 {
|
||
|
|
fmt.Fprintln(os.Stderr, "Error: at least one VMID is required")
|
||
|
|
fs.Usage()
|
||
|
|
return 2
|
||
|
|
}
|
||
|
|
for _, id := range vmids {
|
||
|
|
if !isNumeric(id) {
|
||
|
|
fmt.Fprintf(os.Stderr, "Error: invalid VMID %q\n", id)
|
||
|
|
return 2
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if cfg.node == "" {
|
||
|
|
h, err := os.Hostname()
|
||
|
|
if err != nil {
|
||
|
|
fmt.Fprintf(os.Stderr, "Error: cannot determine hostname: %v\n", err)
|
||
|
|
return 1
|
||
|
|
}
|
||
|
|
cfg.node, _, _ = strings.Cut(h, ".")
|
||
|
|
}
|
||
|
|
|
||
|
|
fmt.Printf("proxmox-backup-watch %s\n", version)
|
||
|
|
|
||
|
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||
|
|
defer stop()
|
||
|
|
|
||
|
|
failed := false
|
||
|
|
for _, vmid := range vmids {
|
||
|
|
if err := handle(ctx, &cfg, vmid); err != nil {
|
||
|
|
fmt.Fprintf(os.Stderr, "VM %s: %v\n", vmid, err)
|
||
|
|
failed = true
|
||
|
|
if ctx.Err() != nil {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if failed {
|
||
|
|
return 1
|
||
|
|
}
|
||
|
|
return 0
|
||
|
|
}
|
||
|
|
|
||
|
|
func handle(ctx context.Context, cfg *config, vmid string) error {
|
||
|
|
fmt.Printf("VM %s: waiting for backup task to finish...\n", vmid)
|
||
|
|
|
||
|
|
t, err := waitForTask(ctx, cfg, vmid)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
|
||
|
|
if cfg.onlyFailed {
|
||
|
|
ok := t.Status == "OK" || (cfg.warnOK && strings.HasPrefix(t.Status, "WARNINGS"))
|
||
|
|
if ok {
|
||
|
|
fmt.Printf("VM %s: backup finished (%s), no restart needed\n", vmid, t.Status)
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
fmt.Printf("VM %s: backup finished (%s) -> restarting\n", vmid, t.Status)
|
||
|
|
return restartGuest(ctx, cfg, vmid)
|
||
|
|
}
|
||
|
|
|
||
|
|
// waitForTask polls until the newest vzdump task for vmid (running now, or
|
||
|
|
// finished after this program started) has an end time.
|
||
|
|
func waitForTask(ctx context.Context, cfg *config, vmid string) (*task, error) {
|
||
|
|
var deadline time.Time
|
||
|
|
if cfg.timeout > 0 {
|
||
|
|
deadline = time.Now().Add(cfg.timeout)
|
||
|
|
}
|
||
|
|
|
||
|
|
for {
|
||
|
|
t, err := latestTask(ctx, cfg, vmid)
|
||
|
|
if err != nil {
|
||
|
|
// Transient pvesh/JSON errors shouldn't abort a long wait.
|
||
|
|
fmt.Fprintf(os.Stderr, "VM %s: warning: %v\n", vmid, err)
|
||
|
|
} else if t != nil && t.EndTime != nil {
|
||
|
|
return t, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
if !deadline.IsZero() && time.Now().After(deadline) {
|
||
|
|
return nil, fmt.Errorf("timed out after %s, no restart", cfg.timeout)
|
||
|
|
}
|
||
|
|
|
||
|
|
select {
|
||
|
|
case <-ctx.Done():
|
||
|
|
return nil, ctx.Err()
|
||
|
|
case <-time.After(cfg.interval):
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
func latestTask(ctx context.Context, cfg *config, vmid string) (*task, error) {
|
||
|
|
out, err := output(ctx, "pvesh", "get", "/nodes/"+cfg.node+"/tasks",
|
||
|
|
"--typefilter", "vzdump", "--limit", "500", "--output-format", "json")
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
|
||
|
|
var tasks []task
|
||
|
|
if err := json.Unmarshal(out, &tasks); err != nil {
|
||
|
|
return nil, fmt.Errorf("parsing pvesh output: %w", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
var latest *task
|
||
|
|
for i := range tasks {
|
||
|
|
t := &tasks[i]
|
||
|
|
if t.ID != vmid {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
// Running (no endtime) or finished after we started.
|
||
|
|
if t.EndTime != nil && *t.EndTime < cfg.startedAt {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if latest == nil || t.StartTime > latest.StartTime {
|
||
|
|
latest = t
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return latest, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func restartGuest(ctx context.Context, cfg *config, vmid string) error {
|
||
|
|
var tool string
|
||
|
|
switch {
|
||
|
|
case exec.CommandContext(ctx, "qm", "status", vmid).Run() == nil:
|
||
|
|
tool = "qm"
|
||
|
|
case exec.CommandContext(ctx, "pct", "status", vmid).Run() == nil:
|
||
|
|
tool = "pct"
|
||
|
|
default:
|
||
|
|
return errors.New("not found on this node")
|
||
|
|
}
|
||
|
|
|
||
|
|
// Clear a leftover 'backup' lock; failure here is not fatal.
|
||
|
|
_ = runCmd(ctx, cfg, tool, "unlock", vmid)
|
||
|
|
|
||
|
|
if err := runCmd(ctx, cfg, tool, "shutdown", vmid, "--timeout", "120", "--forceStop", "1"); err != nil {
|
||
|
|
return fmt.Errorf("shutdown failed: %w", err)
|
||
|
|
}
|
||
|
|
if err := runCmd(ctx, cfg, tool, "start", vmid); err != nil {
|
||
|
|
return fmt.Errorf("start failed: %w", err)
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func runCmd(ctx context.Context, cfg *config, name string, args ...string) error {
|
||
|
|
if cfg.dryRun {
|
||
|
|
fmt.Printf("[dry-run] %s %s\n", name, strings.Join(args, " "))
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
cmd := exec.CommandContext(ctx, name, args...)
|
||
|
|
cmd.Stdout = os.Stdout
|
||
|
|
cmd.Stderr = os.Stderr
|
||
|
|
return cmd.Run()
|
||
|
|
}
|
||
|
|
|
||
|
|
func output(ctx context.Context, name string, args ...string) ([]byte, error) {
|
||
|
|
var stdout, stderr bytes.Buffer
|
||
|
|
cmd := exec.CommandContext(ctx, name, args...)
|
||
|
|
cmd.Stdout = &stdout
|
||
|
|
cmd.Stderr = &stderr
|
||
|
|
if err := cmd.Run(); err != nil {
|
||
|
|
return nil, fmt.Errorf("%s: %w: %s", name, err, strings.TrimSpace(stderr.String()))
|
||
|
|
}
|
||
|
|
return stdout.Bytes(), nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func isNumeric(s string) bool {
|
||
|
|
if s == "" {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
for _, r := range s {
|
||
|
|
if r < '0' || r > '9' {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return true
|
||
|
|
}
|