#!/bin/bash
#########################################################
# selfheal.sh – Autonomer Self-Healing-Reboot
#
# Überwacht Systemparameter und löst einen Reboot aus,
# wenn Schwellwerte über mehrere Checks hinweg verletzt
# werden. Reboot-Loop-Schutz: max. 3 Reboots pro Stunde.
#
# Cron: alle 5 Minuten
#
# Überwachte Parameter:
#   disk      – Füllstand /home/snowtv → aufräumen + API-Report (kein Reboot)
#   nwstate   – Kein Internet (persistent)
#   ram       – Freier RAM zu niedrig (persistent)
#   load      – System-Load zu hoch (persistent)
#   chromium  – Down während Sendezeit (persistent)
#   lte       – Signal zu schwach (persistent, Modem-Hang)
#########################################################

BIN=/home/snowtv/bin
LOG=/var/log/stv.log
STATE_FILE="$BIN/selfheal_state.json"
STATE_TMP="${STATE_FILE}.tmp"
REBOOT_LOG="$BIN/selfheal_reboots"
TS=$(date '+%Y-%m-%dT%H:%M:%S')

# ── Schwellwerte ──────────────────────────────────────────

THRESH_DISK_PCT=90        # Disk > 90% → aufräumen + API-Report (kein Reboot)
THRESH_NWSTATE=4          # nwstate ≥ 4 (kein Internet)
THRESH_NWSTATE_COUNT=3    # ... nach 3 × 5 Min = 15 Min
THRESH_RAM_MB=100         # Freier RAM < 100 MB
THRESH_RAM_COUNT=3        # ... nach 3 × 5 Min = 15 Min
THRESH_LOAD=6             # Load-Average > 6 (fitlet: 4 Kerne)
THRESH_LOAD_COUNT=3       # ... nach 3 × 5 Min = 15 Min
THRESH_CHROM_COUNT=3      # Chromium down 3 × 5 Min während Sendezeit
THRESH_LTE_SIGNAL=5       # LTE-Signal < 5% (Modem-Hang)
THRESH_LTE_COUNT=6        # ... nach 6 × 5 Min = 30 Min
MAX_REBOOTS_PER_HOUR=3

_log() { echo "$(date '+%Y-%m-%d %H:%M:%S') selfheal: $*" | tee -a "$LOG"; }

# ── State-Verwaltung ──────────────────────────────────────

_init_state() {
    jq -n '{
        nwstate_count:   0,
        ram_low_count:   0,
        load_high_count: 0,
        chrom_down_count: 0,
        lte_weak_count:  0
    }'
}

_load_state() {
    [ -f "$STATE_FILE" ] && cat "$STATE_FILE" || _init_state
}

_save_state() {
    echo "$1" > "$STATE_TMP" && mv "$STATE_TMP" "$STATE_FILE"
}

# ── Reboot-Loop-Schutz ────────────────────────────────────

_reboots_last_hour() {
    [ -f "$REBOOT_LOG" ] || { echo 0; return; }
    local cutoff count=0
    cutoff=$(date -d '1 hour ago' '+%s')
    while IFS= read -r line; do
        local ts
        ts=$(date -d "$line" '+%s' 2>/dev/null) || continue
        [ "$ts" -ge "$cutoff" ] && count=$(( count + 1 ))
    done < "$REBOOT_LOG"
    echo "$count"
}

_cleanup_reboot_log() {
    [ -f "$REBOOT_LOG" ] || return
    local cutoff
    cutoff=$(date -d '24 hours ago' '+%s')
    local tmp
    tmp=$(mktemp)
    while IFS= read -r line; do
        local ts
        ts=$(date -d "$line" '+%s' 2>/dev/null) || continue
        [ "$ts" -ge "$cutoff" ] && echo "$line"
    done < "$REBOOT_LOG" > "$tmp"
    mv "$tmp" "$REBOOT_LOG"
}

# ── API-Upload ────────────────────────────────────────────

_upload_event() {
    local reason="$1" value="$2" unit="$3" action="$4"
    local TOK ID DOMAIN
    TOK="$(cat $BIN/token 2>/dev/null)"
    ID="$($BIN/getid.sh)"
    DOMAIN="$($BIN/getdomain.sh | tr -d "[']")"
    [ -z "$TOK" ] && return

    local JSON
    JSON=$(jq -n \
        --arg ts     "$TS"     \
        --arg reason "$reason" \
        --arg value  "$value"  \
        --arg unit   "$unit"   \
        --arg action "$action" \
        '{"selfheal":{"ts":$ts,"reason":$reason,"value":$value,"unit":$unit,"action":$action}}')

    curl -k --connect-timeout 10 --max-time 20 -s -o /dev/null \
        --data "$JSON" \
        -H "APG-Authorization: ${TOK}" \
        "${DOMAIN}/apg-mp?device=${ID}" \
        --user-agent "apg-mp"
}

# ── Reboot auslösen ───────────────────────────────────────

_do_reboot() {
    local reason="$1" value="$2" unit="$3"
    local reboots
    reboots=$(_reboots_last_hour)

    if [ "$reboots" -ge "$MAX_REBOOTS_PER_HOUR" ]; then
        _log "BLOCKED [$reason: $value$unit] – Reboot-Loop-Schutz ($reboots/h)"
        _upload_event "$reason" "$value" "$unit" "blocked_loop"
        return
    fi

    _log "REBOOT [$reason: $value$unit]"
    echo "$TS" >> "$REBOOT_LOG"
    _cleanup_reboot_log

    # State zurücksetzen damit Counter nach Reboot bei 0 starten
    _save_state "$(_init_state)"

    _upload_event "$reason" "$value" "$unit" "reboot"
    sleep 2
    sudo /sbin/reboot
}

# ── Checks ────────────────────────────────────────────────

STATE=$(_load_state)
REBOOT_REASON="" REBOOT_VALUE="" REBOOT_UNIT=""

_trigger() {
    # Ersten Grund gewinnt
    [ -z "$REBOOT_REASON" ] && { REBOOT_REASON="$1"; REBOOT_VALUE="$2"; REBOOT_UNIT="$3"; }
}

# 1. Disk-Auslastung – aufräumen + reporten, kein Reboot
DISK_PCT=$(df /home/snowtv 2>/dev/null | awk 'NR==2 {gsub("%",""); print $5}')
if [ -n "$DISK_PCT" ] && [ "$DISK_PCT" -ge "$THRESH_DISK_PCT" ] 2>/dev/null; then
    _log "WARN: Disk ${DISK_PCT}% – Aufräumen wird eingeleitet"
    # Crashdumps, tmp-Zips, alte Logs bereinigen
    rm -f /home/snowtv/.config/chromium/Crash\ Reports/pending/*.dmp 2>/dev/null
    rm -f /home/snowtv/.config/chromium/Crash\ Reports/pending/*.meta 2>/dev/null
    rm -f /tmp/crash-*.zip 2>/dev/null
    find /tmp -maxdepth 1 -type f -mmin +60 -delete 2>/dev/null
    journalctl --vacuum-size=50M 2>/dev/null
    DISK_PCT_AFTER=$(df /home/snowtv 2>/dev/null | awk 'NR==2 {gsub("%",""); print $5}')
    _log "Disk nach Cleanup: ${DISK_PCT_AFTER}%"
    _upload_event "disk_full" "${DISK_PCT}→${DISK_PCT_AFTER}" "%" "cleanup"
fi

# 2. Kein Internet – nwstate (persistent)
NWSTATE="$(cat $BIN/nwstate 2>/dev/null)"
if [ -n "$NWSTATE" ] && [ "$NWSTATE" -ge "$THRESH_NWSTATE" ] 2>/dev/null; then
    STATE=$(echo "$STATE" | jq '.nwstate_count += 1')
    COUNT=$(echo "$STATE" | jq '.nwstate_count')
    _log "WARN: nwstate=${NWSTATE} – Counter ${COUNT}/${THRESH_NWSTATE_COUNT}"
    [ "$COUNT" -ge "$THRESH_NWSTATE_COUNT" ] && _trigger "no_internet" "$NWSTATE" "nwstate"
else
    STATE=$(echo "$STATE" | jq '.nwstate_count = 0')
fi

# 3. Freier RAM (persistent)
RAM_FREE=$(free -m 2>/dev/null | awk '/^Mem:/ {print $7}')
if [ -n "$RAM_FREE" ] && [ "$RAM_FREE" -lt "$THRESH_RAM_MB" ] 2>/dev/null; then
    STATE=$(echo "$STATE" | jq '.ram_low_count += 1')
    COUNT=$(echo "$STATE" | jq '.ram_low_count')
    _log "WARN: RAM frei ${RAM_FREE}MB – Counter ${COUNT}/${THRESH_RAM_COUNT}"
    [ "$COUNT" -ge "$THRESH_RAM_COUNT" ] && _trigger "ram_low" "$RAM_FREE" "MB"
else
    STATE=$(echo "$STATE" | jq '.ram_low_count = 0')
fi

# 4. System-Load (persistent)
LOAD=$(awk '{print $1}' /proc/loadavg 2>/dev/null)
LOAD_INT=$(echo "$LOAD" | awk -F. '{print $1}')
if [ -n "$LOAD_INT" ] && [ "$LOAD_INT" -ge "$THRESH_LOAD" ] 2>/dev/null; then
    STATE=$(echo "$STATE" | jq '.load_high_count += 1')
    COUNT=$(echo "$STATE" | jq '.load_high_count')
    _log "WARN: Load ${LOAD} – Counter ${COUNT}/${THRESH_LOAD_COUNT}"
    [ "$COUNT" -ge "$THRESH_LOAD_COUNT" ] && _trigger "load_high" "$LOAD" ""
else
    STATE=$(echo "$STATE" | jq '.load_high_count = 0')
fi

# 5. Chromium down während Sendezeit (persistent)
# chromon.sh handelt Crash-Recovery → selfheal greift nur bei anhaltend fehlenden Prozess
if [ -f "$BIN/mc.cfg" ]; then
    START=$(grep '^START=' "$BIN/mc.cfg" | cut -d= -f2 | tr -d '\r')
    END=$(grep '^END='   "$BIN/mc.cfg" | cut -d= -f2 | tr -d '\r')
    NOW_HHMM=$(date +%H%M)
    START_FMT=$(echo "$START" | tr -d ':')
    END_FMT=$(echo "$END"   | tr -d ':')
    if [ -n "$START_FMT" ] && [ -n "$END_FMT" ] && \
       [ "$NOW_HHMM" -ge "$START_FMT" ] && [ "$NOW_HHMM" -le "$END_FMT" ] 2>/dev/null; then
        if ! pgrep -x chromium &>/dev/null && ! pgrep -x chromium-browser &>/dev/null; then
            STATE=$(echo "$STATE" | jq '.chrom_down_count += 1')
            COUNT=$(echo "$STATE" | jq '.chrom_down_count')
            _log "WARN: Chromium down während Sendezeit – Counter ${COUNT}/${THRESH_CHROM_COUNT}"
            [ "$COUNT" -ge "$THRESH_CHROM_COUNT" ] && _trigger "chromium_down" "$COUNT" "checks"
        else
            STATE=$(echo "$STATE" | jq '.chrom_down_count = 0')
        fi
    else
        STATE=$(echo "$STATE" | jq '.chrom_down_count = 0')
    fi
fi

# 6. LTE-Signal zu schwach (persistent – Modem-Hang-Erkennung)
MODEM_IDX=$(mmcli -L 2>/dev/null | grep -oP 'Modem/\K[0-9]+' | head -1)
if [ -n "$MODEM_IDX" ]; then
    LTE_SIG=$(mmcli -m "$MODEM_IDX" 2>/dev/null | grep -oP 'signal quality:\s+\K[0-9]+')
    if [ -n "$LTE_SIG" ] && [ "$LTE_SIG" -lt "$THRESH_LTE_SIGNAL" ] 2>/dev/null; then
        STATE=$(echo "$STATE" | jq '.lte_weak_count += 1')
        COUNT=$(echo "$STATE" | jq '.lte_weak_count')
        _log "WARN: LTE-Signal ${LTE_SIG}% – Counter ${COUNT}/${THRESH_LTE_COUNT}"
        [ "$COUNT" -ge "$THRESH_LTE_COUNT" ] && _trigger "lte_weak" "$LTE_SIG" "%"
    else
        STATE=$(echo "$STATE" | jq '.lte_weak_count = 0')
    fi
fi

# ── State speichern + ggf. Reboot ────────────────────────
_save_state "$STATE"

[ -n "$REBOOT_REASON" ] && _do_reboot "$REBOOT_REASON" "$REBOOT_VALUE" "$REBOOT_UNIT"

exit 0