Home-Assistant-Watchdog/main.go

// ha-watchdog: list Home Assistant apps, pick which to watch, auto-restart them.
//
// Backend selection:
//   - SUPERVISOR_TOKEN set  -> Supervisor REST API (use when running as an app)
//   - otherwise             -> shells out to the `ha` CLI
//
// Usage:
//   ha-watchdog            interactive menu
//   ha-watchdog -daemon    headless: monitor the saved watchlist until killed
//   ha-watchdog -version   print version and build date
//
// Version info is injected at build time (see Makefile):
//   -ldflags "-X main.version=1.1.0 -X main.buildDate=2026-11-09T12:00:00Z"
package main

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"regexp"
	"runtime"
	"runtime/debug"
	"sort"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

// ---------- Version info ----------

var (
	version   = "1.1.0"
	buildDate = "unknown" // set via -ldflags; falls back to VCS time if available
)

func versionString() string {
	bd := buildDate
	if bd == "unknown" {
		if info, ok := debug.ReadBuildInfo(); ok {
			for _, s := range info.Settings {
				if s.Key == "vcs.time" && s.Value != "" {
					bd = s.Value
				}
			}
		}
	}
	return fmt.Sprintf("ha-watchdog %s (built %s, %s %s/%s)",
		version, bd, runtime.Version(), runtime.GOOS, runtime.GOARCH)
}

// ---------- Types ----------

type Addon struct {
	Slug            string `json:"slug"`
	Name            string `json:"name"`
	State           string `json:"state"`
	Version         string `json:"version"`
	VersionLatest   string `json:"version_latest"`
	UpdateAvailable bool   `json:"update_available"`
	Available       bool   `json:"available"`
	Detached        bool   `json:"detached"`
}

type envelope struct {
	Result  string `json:"result"`
	Message string `json:"message"`
	Data    struct {
		Addons []Addon `json:"addons"`
	} `json:"data"`
}

type Backend interface {
	List() ([]Addon, error)
	Start(slug string) error
	// Notify raises a Home Assistant persistent notification.
	// Backends that can't do this return an error; callers just log it.
	Notify(title, message string) error
}

// ---------- Supervisor API backend ----------

type apiBackend struct {
	base, token string
	client      *http.Client
}

func (a *apiBackend) do(method, path string, payload []byte) ([]byte, error) {
	var body io.Reader
	if payload != nil {
		body = bytes.NewReader(payload)
	}
	req, err := http.NewRequest(method, a.base+path, body)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Authorization", "Bearer "+a.token)
	if payload != nil {
		req.Header.Set("Content-Type", "application/json")
	}
	resp, err := a.client.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	out, _ := io.ReadAll(resp.Body)
	if resp.StatusCode >= 300 {
		return out, fmt.Errorf("supervisor returned %s: %s", resp.Status, strings.TrimSpace(string(out)))
	}
	return out, nil
}

func (a *apiBackend) List() ([]Addon, error) {
	out, err := a.do("GET", "/addons", nil)
	if err != nil {
		return nil, err
	}
	return parseList(out)
}

func (a *apiBackend) Start(slug string) error {
	_, err := a.do("POST", "/addons/"+slug+"/start", nil)
	return err
}

// Notify calls the HA Core API through the Supervisor proxy.
// Requires `homeassistant_api: true` in the app's config.yaml.
func (a *apiBackend) Notify(title, message string) error {
	payload, _ := json.Marshal(map[string]string{
		"title":           title,
		"message":         message,
		"notification_id": "ha_watchdog",
	})
	_, err := a.do("POST", "/core/api/services/persistent_notification/create", payload)
	return err
}

// ---------- `ha` CLI backend ----------

type cliBackend struct{}

func (cliBackend) List() ([]Addon, error) {
	out, err := exec.Command("ha", "addons", "--raw-json").Output()
	if err != nil {
		return nil, fmt.Errorf("ha addons: %w", err)
	}
	return parseList(out)
}

func (cliBackend) Start(slug string) error {
	out, err := exec.Command("ha", "addons", "start", slug).CombinedOutput()
	if err != nil {
		return fmt.Errorf("ha addons start %s: %w: %s", slug, err, strings.TrimSpace(string(out)))
	}
	return nil
}

func (cliBackend) Notify(title, message string) error {
	return errors.New("notifications need the Supervisor API (run as an HA app with homeassistant_api: true)")
}

func parseList(b []byte) ([]Addon, error) {
	var env envelope
	if err := json.Unmarshal(b, &env); err != nil {
		return nil, err
	}
	if env.Result != "ok" {
		return nil, errors.New("supervisor error: " + env.Message)
	}
	sort.Slice(env.Data.Addons, func(i, j int) bool {
		return strings.ToLower(env.Data.Addons[i].Name) < strings.ToLower(env.Data.Addons[j].Name)
	})
	return env.Data.Addons, nil
}

// ---------- SSH (remote) backend ----------
//
// Runs the `ha` CLI on the Home Assistant host over SSH, so the watchdog can
// live on a separate monitoring machine. Notifications (optional) go through
// the normal Home Assistant REST API using a long-lived access token.

var slugRe = regexp.MustCompile(`^[A-Za-z0-9_-]+$`)

type sshBackend struct {
	target  string // user@host
	port    int
	key     string // optional identity file
	haURL   string // optional, e.g. http://homeassistant.local:8123
	haToken string // optional long-lived access token
	client  *http.Client
}

func (s *sshBackend) run(remoteCmd string) ([]byte, error) {
	args := []string{"-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-p", strconv.Itoa(s.port)}
	if s.key != "" {
		args = append(args, "-i", s.key)
	}
	args = append(args, s.target, remoteCmd)

	ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
	defer cancel()
	cmd := exec.CommandContext(ctx, "ssh", args...)
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	out, err := cmd.Output()
	if err != nil {
		return nil, fmt.Errorf("ssh %s %q: %w: %s", s.target, remoteCmd, err, strings.TrimSpace(stderr.String()))
	}
	return out, nil
}

func (s *sshBackend) List() ([]Addon, error) {
	out, err := s.run("ha addons --raw-json")
	if err != nil {
		return nil, err
	}
	return parseList(out)
}

func (s *sshBackend) Start(slug string) error {
	if !slugRe.MatchString(slug) {
		return fmt.Errorf("refusing to run with unexpected slug %q", slug)
	}
	_, err := s.run("ha addons start " + slug)
	return err
}

func (s *sshBackend) Notify(title, message string) error {
	if s.haURL == "" || s.haToken == "" {
		return errors.New("remote notifications need -ha-url and -ha-token")
	}
	payload, _ := json.Marshal(map[string]string{
		"title":           title,
		"message":         message,
		"notification_id": "ha_watchdog",
	})
	req, err := http.NewRequest("POST", strings.TrimRight(s.haURL, "/")+"/api/services/persistent_notification/create", bytes.NewReader(payload))
	if err != nil {
		return err
	}
	req.Header.Set("Authorization", "Bearer "+s.haToken)
	req.Header.Set("Content-Type", "application/json")
	resp, err := s.client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 300 {
		body, _ := io.ReadAll(resp.Body)
		return fmt.Errorf("home assistant returned %s: %s", resp.Status, strings.TrimSpace(string(body)))
	}
	return nil
}

type backendConfig struct {
	sshTarget string
	sshPort   int
	sshKey    string
	haURL     string
	haToken   string
}

// newBackend picks a backend and returns it with a short description for logs.
func newBackend(c backendConfig) (Backend, string) {
	if c.sshTarget != "" {
		return &sshBackend{
			target: c.sshTarget, port: c.sshPort, key: c.sshKey,
			haURL: c.haURL, haToken: c.haToken,
			client: &http.Client{Timeout: 30 * time.Second},
		}, "remote over SSH (" + c.sshTarget + ")"
	}
	if tok := os.Getenv("SUPERVISOR_TOKEN"); tok != "" {
		return &apiBackend{base: "http://supervisor", token: tok, client: &http.Client{Timeout: 60 * time.Second}},
			"Supervisor API (local)"
	}
	return cliBackend{}, "ha CLI (local)"
}

// ---------- Watchlist (persisted) ----------

type Watchlist struct {
	mu    sync.Mutex
	path  string
	slugs map[string]bool
}

func loadWatchlist(path string) *Watchlist {
	w := &Watchlist{path: path, slugs: map[string]bool{}}
	data, err := os.ReadFile(path)
	if err != nil {
		return w
	}
	var list []string
	if json.Unmarshal(data, &list) == nil {
		for _, s := range list {
			w.slugs[s] = true
		}
	}
	return w
}

func (w *Watchlist) saveLocked() {
	list := make([]string, 0, len(w.slugs))
	for s := range w.slugs {
		list = append(list, s)
	}
	sort.Strings(list)
	data, _ := json.MarshalIndent(list, "", "  ")
	if err := os.WriteFile(w.path, data, 0o644); err != nil {
		log.Printf("warning: could not save watchlist: %v", err)
	}
}

func (w *Watchlist) Add(slug string) {
	w.mu.Lock()
	defer w.mu.Unlock()
	w.slugs[slug] = true
	w.saveLocked()
}

func (w *Watchlist) Remove(slug string) {
	w.mu.Lock()
	defer w.mu.Unlock()
	delete(w.slugs, slug)
	w.saveLocked()
}

func (w *Watchlist) Has(slug string) bool {
	w.mu.Lock()
	defer w.mu.Unlock()
	return w.slugs[slug]
}

func (w *Watchlist) Slugs() []string {
	w.mu.Lock()
	defer w.mu.Unlock()
	list := make([]string, 0, len(w.slugs))
	for s := range w.slugs {
		list = append(list, s)
	}
	sort.Strings(list)
	return list
}

// ---------- Monitor ----------

type Monitor struct {
	be         Backend
	wl         *Watchlist
	interval   time.Duration
	cooldown   time.Duration // minimum gap between start attempts for one app
	maxRetries int           // give up after this many attempts within retryWindow
	window     time.Duration

	mu       sync.Mutex
	attempts map[string][]time.Time // recent start attempts per slug
	gaveUp   map[string]bool        // apps we've stopped retrying
	skipped  map[string]bool        // apps we've already logged as unstartable
}

func newMonitor(be Backend, wl *Watchlist, interval, cooldown time.Duration, maxRetries int, window time.Duration) *Monitor {
	return &Monitor{
		be: be, wl: wl, interval: interval, cooldown: cooldown,
		maxRetries: maxRetries, window: window,
		attempts: map[string][]time.Time{},
		gaveUp:   map[string]bool{},
		skipped:  map[string]bool{},
	}
}

func (m *Monitor) notify(title, msg string) {
	if err := m.be.Notify(title, msg); err != nil {
		log.Printf("notify: %v", err)
	}
}

// recentAttempts drops expired attempts and returns the remaining list. Caller holds m.mu.
func (m *Monitor) recentAttempts(slug string) []time.Time {
	cutoff := time.Now().Add(-m.window)
	var keep []time.Time
	for _, t := range m.attempts[slug] {
		if t.After(cutoff) {
			keep = append(keep, t)
		}
	}
	m.attempts[slug] = keep
	return keep
}

func (m *Monitor) check() {
	addons, err := m.be.List()
	if err != nil {
		log.Printf("monitor: list failed: %v", err)
		return
	}
	state := map[string]Addon{}
	for _, a := range addons {
		state[a.Slug] = a
	}

	for _, slug := range m.wl.Slugs() {
		a, ok := state[slug]
		if !ok {
			m.mu.Lock()
			first := !m.skipped[slug]
			m.skipped[slug] = true
			m.mu.Unlock()
			if first {
				log.Printf("monitor: %s is on the watchlist but not installed, skipping", slug)
			}
			continue
		}

		// Can't start these, so don't try.
		if !a.Available || a.Detached {
			m.mu.Lock()
			first := !m.skipped[slug]
			m.skipped[slug] = true
			m.mu.Unlock()
			if first {
				log.Printf("monitor: %s (%s) is unavailable or detached, skipping", a.Name, slug)
			}
			continue
		}

		m.mu.Lock()
		delete(m.skipped, slug)

		if a.State == "started" {
			// Recovered. Allow a future give-up notification, but keep the attempt
			// history so a flapping app still trips the cap quickly.
			delete(m.gaveUp, slug)
			m.mu.Unlock()
			continue
		}
		if m.gaveUp[slug] {
			m.mu.Unlock()
			continue
		}

		recent := m.recentAttempts(slug)
		if len(recent) >= m.maxRetries {
			m.gaveUp[slug] = true
			m.mu.Unlock()
			msg := fmt.Sprintf("%s (%s) is %q and failed to stay up after %d start attempts in %s. "+
				"Watchdog has stopped retrying. Check the app's logs.",
				a.Name, slug, a.State, m.maxRetries, m.window)
			log.Printf("monitor: GIVING UP on %s: %s", slug, msg)
			m.notify("Watchdog gave up on "+a.Name, msg)
			continue
		}
		if len(recent) > 0 && time.Since(recent[len(recent)-1]) < m.cooldown {
			m.mu.Unlock()
			continue
		}
		m.attempts[slug] = append(recent, time.Now())
		n := len(m.attempts[slug])
		m.mu.Unlock()

		log.Printf("monitor: %s (%s) is %q, starting it (attempt %d/%d)", a.Name, slug, a.State, n, m.maxRetries)
		if err := m.be.Start(slug); err != nil {
			log.Printf("monitor: start %s failed: %v", slug, err)
		} else {
			m.notify("Watchdog restarted "+a.Name,
				fmt.Sprintf("%s was %q and has been started again (attempt %d/%d).", a.Name, a.State, n, m.maxRetries))
		}
	}
}

// Reset clears give-up state and attempt history for all apps.
func (m *Monitor) Reset() {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.attempts = map[string][]time.Time{}
	m.gaveUp = map[string]bool{}
}

// Status returns slugs the monitor has given up on.
func (m *Monitor) GaveUp() []string {
	m.mu.Lock()
	defer m.mu.Unlock()
	var out []string
	for s := range m.gaveUp {
		out = append(out, s)
	}
	sort.Strings(out)
	return out
}

func (m *Monitor) Run(ctx context.Context) {
	m.check()
	t := time.NewTicker(m.interval)
	defer t.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-t.C:
			m.check()
		}
	}
}

// ---------- Interactive UI ----------

func listApps(be Backend, wl *Watchlist) []Addon {
	addons, err := be.List()
	if err != nil {
		fmt.Println("Error:", err)
		return nil
	}
	fmt.Printf("\n%-4s %-30s %-9s %-16s %-16s %-7s %s\n", "#", "Name", "State", "Version", "Update", "Watched", "Notes")
	for i, a := range addons {
		upd := ""
		if a.UpdateAvailable {
			upd = "-> " + a.VersionLatest
		}
		watched := ""
		if wl.Has(a.Slug) {
			watched = "yes"
		}
		var notes []string
		if !a.Available {
			notes = append(notes, "unavailable")
		}
		if a.Detached {
			notes = append(notes, "detached")
		}
		fmt.Printf("%-4d %-30.30s %-9s %-16.16s %-16.16s %-7s %s\n",
			i+1, a.Name, a.State, a.Version, upd, watched, strings.Join(notes, ","))
	}
	return addons
}

func parseNumbers(s string, max int) []int {
	var out []int
	for _, f := range strings.FieldsFunc(s, func(r rune) bool { return r == ',' || r == ' ' }) {
		n, err := strconv.Atoi(f)
		if err != nil || n < 1 || n > max {
			fmt.Printf("Ignoring %q\n", f)
			continue
		}
		out = append(out, n-1)
	}
	return out
}

func interactive(be Backend, wl *Watchlist, m *Monitor) {
	in := bufio.NewScanner(os.Stdin)
	prompt := func(msg string) string {
		fmt.Print(msg)
		if !in.Scan() {
			os.Exit(0)
		}
		return strings.TrimSpace(in.Text())
	}

	var cancel context.CancelFunc
	running := func() bool { return cancel != nil }

	for {
		status := "OFF"
		if running() {
			status = "ON"
		}
		fmt.Printf("\n=== HA App Watchdog %s | monitor: %s | %d watched ===\n", version, status, len(wl.Slugs()))
		if g := m.GaveUp(); len(g) > 0 {
			fmt.Printf("!! Gave up on: %s (use option 6 to reset)\n", strings.Join(g, ", "))
		}
		fmt.Println("1) List installed apps")
		fmt.Println("2) Add apps to watchlist")
		fmt.Println("3) Remove apps from watchlist")
		fmt.Println("4) Show watchlist")
		fmt.Println("5) Start/stop monitoring")
		fmt.Println("6) Reset retry counters / give-up state")
		fmt.Println("v) Version info")
		fmt.Println("q) Quit")

		switch prompt("> ") {
		case "1":
			listApps(be, wl)

		case "2":
			addons := listApps(be, wl)
			if addons == nil {
				continue
			}
			for _, i := range parseNumbers(prompt("Numbers to add (e.g. 1,4,7): "), len(addons)) {
				wl.Add(addons[i].Slug)
				fmt.Println("Added", addons[i].Name)
			}

		case "3":
			slugs := wl.Slugs()
			if len(slugs) == 0 {
				fmt.Println("Watchlist is empty.")
				continue
			}
			for i, s := range slugs {
				fmt.Printf("%d) %s\n", i+1, s)
			}
			for _, i := range parseNumbers(prompt("Numbers to remove: "), len(slugs)) {
				wl.Remove(slugs[i])
				fmt.Println("Removed", slugs[i])
			}

		case "4":
			slugs := wl.Slugs()
			if len(slugs) == 0 {
				fmt.Println("Watchlist is empty.")
			}
			for i, s := range slugs {
				fmt.Printf("%d) %s\n", i+1, s)
			}

		case "5":
			if running() {
				cancel()
				cancel = nil
				fmt.Println("Monitoring stopped.")
			} else {
				var ctx context.Context
				ctx, cancel = context.WithCancel(context.Background())
				go m.Run(ctx)
				fmt.Printf("Monitoring started (every %s, max %d starts per %s). Logs print here.\n",
					m.interval, m.maxRetries, m.window)
			}

		case "6":
			m.Reset()
			fmt.Println("Retry counters and give-up state cleared.")

		case "v", "V":
			fmt.Println(versionString())

		case "q", "Q":
			if running() {
				cancel()
			}
			return
		}
	}
}

// ---------- main ----------

func defaultWatchlistPath() string {
	if p := os.Getenv("WATCHDOG_FILE"); p != "" {
		return p
	}
	if st, err := os.Stat("/data"); err == nil && st.IsDir() {
		return "/data/watchlist.json" // persistent storage inside an HA app
	}
	return "watchlist.json"
}

func main() {
	daemon := flag.Bool("daemon", false, "run headless, monitoring the saved watchlist")
	showVersion := flag.Bool("version", false, "print version and build date, then exit")
	interval := flag.Duration("interval", 30*time.Second, "how often to check app state")
	cooldown := flag.Duration("cooldown", 60*time.Second, "minimum time between start attempts for one app")
	maxRetries := flag.Int("max-retries", 5, "give up on an app after this many start attempts within -retry-window")
	window := flag.Duration("retry-window", 10*time.Minute, "time window for counting start attempts")
	file := flag.String("file", defaultWatchlistPath(), "watchlist file path")
	sshTarget := flag.String("ssh", "", "monitor a remote Home Assistant host over SSH, e.g. [email protected]")
	sshPort := flag.Int("ssh-port", 22, "SSH port on the Home Assistant host")
	sshKey := flag.String("ssh-key", "", "SSH identity file (private key) to use")
	haURL := flag.String("ha-url", "", "Home Assistant URL for remote notifications, e.g. http://homeassistant.local:8123")
	haToken := flag.String("ha-token", os.Getenv("HA_TOKEN"), "long-lived access token for remote notifications (or set HA_TOKEN)")
	flag.Parse()

	if *showVersion {
		fmt.Println(versionString())
		return
	}

	be, backendDesc := newBackend(backendConfig{
		sshTarget: *sshTarget, sshPort: *sshPort, sshKey: *sshKey,
		haURL: *haURL, haToken: *haToken,
	})
	wl := loadWatchlist(*file)
	m := newMonitor(be, wl, *interval, *cooldown, *maxRetries, *window)

	if *daemon {
		ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
		defer stop()
		log.Printf("%s", versionString())
		log.Printf("backend: %s", backendDesc)
		log.Printf("daemon mode: watching %v every %s (max %d starts per %s)",
			wl.Slugs(), *interval, *maxRetries, *window)
		m.Run(ctx)
		return
	}
	fmt.Println("Backend:", backendDesc)
	interactive(be, wl, m)
}