#!/usr/bin/env bash
#
# get-persistence-snapshot.sh — snapshot every common Linux persistence location
# into TSV/JSONL for triage and for diffing against a known-good or earlier snapshot.
#
# Linux counterpart of Get-PersistenceSnapshot.ps1. Read-only: nothing on the host
# is changed. Each record is tagged with its MITRE ATT&CK technique.
#
#   Scheduled      cron (system, cron.d, periodic dirs, user spools, anacron), at jobs,
#                  systemd timers
#   Services       systemd units + drop-ins (system and per-user, Exec* lines),
#                  generators, lingering users, SysV init.d, rc.local, inittab
#   Login hooks    /etc/profile(.d), bashrc/zshrc (system + every home), /etc/environment,
#                  XDG autostart, update-motd.d, NetworkManager dispatcher
#   Access         SSH authorized_keys (every home + AuthorizedKeysFile), ~/.ssh/rc,
#                  sshd AuthorizedKeysCommand, UID-0 accounts, empty passwords,
#                  sudoers, privileged groups (sudo/wheel/adm/docker/lxd/disk)
#   Hijacking      /etc/ld.so.preload, ld.so.conf.d, PAM modules (pam_exec, modules
#                  not from a package or modified), kernel modules + modprobe
#                  "install" hooks, udev RUN+= rules, APT/DNF hooks
#   Privilege      SUID/SGID files and file capabilities on local filesystems
#
# Every referenced executable, script and config file is SHA256-hashed and checked
# against the package manager (dpkg / rpm / apk): which package owns it and whether
# the file on disk still matches the package (dpkg --verify / rpm -V). Records get
# triage flags (NotFromPackage, PackageModified, TmpPath, HiddenPath, WorldWritable,
# SuspiciousCommand, SuspiciousContent, Uid0, NoPasswd, ...). Flags are leads, not
# verdicts — admin-made cron jobs are "NotFromPackage" too, which is why --compare-to
# exists.
#
# Output folder (default ./persistence-snapshot_<host>_<UTC>):
#   persistence.tsv     all records (header row; one record per line)
#   persistence.jsonl   same, one JSON object per line
#   flagged.tsv         records with at least one flag
#   diff.tsv            only with --compare-to: added / removed / changed
#   artifacts.tar.gz    copies of every config/script file referenced (paths kept)
#   errors.log          stderr of every collector
#   manifest.json       host, collector status, SHA256 of every output file
#
# Usage:
#   sudo ./get-persistence-snapshot.sh [options]
#     -o, --output DIR        output folder
#     -c, --compare-to FILE   persistence.tsv from an earlier run / clean host
#     -z, --zip               also write DIR.tar.gz and print its SHA256
#         --no-hash           skip SHA256 of referenced files
#         --no-pkg            skip package ownership / verification (much faster)
#         --no-suid           skip the SUID/SGID + capabilities filesystem walk
#         --recent-days N     flag non-package files modified in the last N days (default 30)
#
# Requires bash 4+, coreutils, find, awk. Run as root: as a normal user most of
# the other users' data, /etc/shadow, sudoers and spools are unreadable (the
# manifest records this).

set -uo pipefail
export LC_ALL=C
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin

OUT=""; COMPARE=""; ZIP=0; DO_HASH=1; DO_PKG=1; DO_SUID=1; RECENT_DAYS=30

while [ $# -gt 0 ]; do
    case "$1" in
        -o|--output)      OUT="$2"; shift 2 ;;
        -c|--compare-to)  COMPARE="$2"; shift 2 ;;
        -z|--zip)         ZIP=1; shift ;;
        --no-hash)        DO_HASH=0; shift ;;
        --no-pkg)         DO_PKG=0; shift ;;
        --no-suid)        DO_SUID=0; shift ;;
        --recent-days)    RECENT_DAYS="$2"; shift 2 ;;
        -h|--help)        sed -n '2,/^$/p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
        *) echo "unknown option: $1" >&2; exit 2 ;;
    esac
done

if [ "${BASH_VERSINFO[0]}" -lt 4 ]; then echo "bash 4+ required" >&2; exit 2; fi
for bin in find awk stat sha256sum; do
    command -v "$bin" >/dev/null || { echo "missing required tool: $bin" >&2; exit 2; }
done

IS_ROOT=0; [ "$(id -u)" -eq 0 ] && IS_ROOT=1
[ $IS_ROOT -eq 1 ] || echo "WARNING: not root — other users, shadow, sudoers and spools will be missing. See manifest.json." >&2

HOST=$(hostname 2>/dev/null || cat /etc/hostname)
STARTED=$(date -u +%Y-%m-%dT%H:%M:%SZ)
NOW=$(date +%s)
RECENT_CUTOFF=$(( NOW - RECENT_DAYS * 86400 ))
[ -n "$OUT" ] || OUT="./persistence-snapshot_${HOST}_$(date -u +%Y%m%dT%H%M%SZ)"
mkdir -p "$OUT" || exit 1
OUT=$(cd "$OUT" && pwd)
TSV="$OUT/persistence.tsv"; JSONL="$OUT/persistence.jsonl"; ERRLOG="$OUT/errors.log"
ARTLIST="$OUT/.artifact-list"
: > "$JSONL"; : > "$ERRLOG"; : > "$ARTLIST"
COLS="category technique scope location name command image image_sha256 payload payload_sha256 package package_state mode owner mtime_utc flags"
printf '%s\n' "$COLS" | tr ' ' '\t' > "$TSV"

PKGM=none
if command -v dpkg-query >/dev/null; then PKGM=dpkg
elif command -v rpm >/dev/null; then PKGM=rpm
elif command -v apk >/dev/null; then PKGM=apk; fi
[ $DO_PKG -eq 1 ] || PKGM=none

RECORDS=0
NOTES=()

#----------------------------------------------------------------------------#
#  Helpers                                                                   #
#----------------------------------------------------------------------------#

# Commands that download-and-run, open shells over the network, or decode payloads.
SUS_RE='(curl|wget)[^;|]*\|[[:space:]]*(sudo[[:space:]]+)?(ba|da|z)?sh|/dev/(tcp|udp)/|(^|[^a-z])(nc|ncat|netcat)[[:space:]][^;|]*-[a-z]*[ec]|socat[[:space:]]|base64[[:space:]]+(-d|--decode)|python[0-9.]*[[:space:]]+-c|perl[[:space:]]+-e|bash[[:space:]]+-i|mkfifo|chmod[[:space:]]+[ugo]*\+s|LD_PRELOAD=|xmrig|stratum\+tcp'

json_esc() {
    local s=$1
    s=${s//\\/\\\\}; s=${s//\"/\\\"}; s=${s//$'\t'/\\t}; s=${s//$'\n'/\\n}; s=${s//$'\r'/\\r}
    printf '%s' "$s" | tr -d '\000-\010\013\014\016-\037'
}

add_artifact() { [ -f "$1" ] && [ -r "$1" ] && printf '%s\n' "$1" >> "$ARTLIST"; }

# Package ownership + verification, cached per path / per package.
declare -A PKG_OF=() PKG_VERIFY=()
pkg_lookup() {  # sets P_PKG P_STATE for path $1
    local p=$1 alt="" out="" pkg=""
    P_PKG=""; P_STATE=""
    [ "$PKGM" = none ] && { P_STATE="unchecked"; return; }
    if [ -n "${PKG_OF[$p]+x}" ]; then pkg=${PKG_OF[$p]}
    else
        case "$PKGM" in
            dpkg)
                # usrmerge: dpkg may know the file as /bin/x while it lives at /usr/bin/x
                case "$p" in /usr/bin/*|/usr/sbin/*|/usr/lib/*|/usr/lib64/*) alt=${p#/usr} ;;
                             /bin/*|/sbin/*|/lib/*|/lib64/*) alt="/usr$p" ;; esac
                for cand in "$p" $alt; do
                    out=$(dpkg-query -S "$cand" 2>/dev/null | grep -v '^diversion' | head -n1)
                    [ -n "$out" ] && { pkg=${out%%:*}; pkg=${pkg%%,*}; break; }
                done ;;
            rpm) out=$(rpm -qf "$p" 2>/dev/null) && pkg=${out%%$'\n'*} ;;
            apk) out=$(apk info --who-owns "$p" 2>/dev/null) && pkg=${out##* } ;;
        esac
        PKG_OF[$p]=$pkg
    fi
    P_PKG=$pkg
    if [ -z "$pkg" ]; then P_STATE="unowned"; return; fi
    case "$PKGM" in
        dpkg)
            [ -n "${PKG_VERIFY[$pkg]+x}" ] || PKG_VERIFY[$pkg]=$(dpkg --verify "$pkg" 2>/dev/null)
            out=${PKG_VERIFY[$pkg]} ;;
        rpm)
            out=$(rpm -Vf "$p" 2>/dev/null) ;;
        *)  P_STATE="unverified"; return ;;
    esac
    # Third column '5' = digest mismatch, in both dpkg --verify and rpm -V output.
    if printf '%s\n' "$out" | awk -v f="$p" -v g="$alt" '($NF==f || (g!="" && $NF==g)) && substr($1,3,1)=="5" {m=1} END{exit !m}'; then
        P_STATE="modified"
    else
        P_STATE="ok"
    fi
}

declare -A FACTS=()
facts() {  # sets F_EXISTS F_SHA F_MODE F_OWNER F_MTIME F_MTIME_EPOCH F_PKG F_STATE
    local p=$1
    F_EXISTS=0; F_SHA=""; F_MODE=""; F_OWNER=""; F_MTIME=""; F_MTIME_EPOCH=0; F_PKG=""; F_STATE=""
    [ -n "$p" ] || return
    if [ -n "${FACTS[$p]+x}" ]; then
        IFS=$'\t' read -r F_EXISTS F_SHA F_MODE F_OWNER F_MTIME F_MTIME_EPOCH F_PKG F_STATE <<< "${FACTS[$p]}"
        return
    fi
    if [ -e "$p" ]; then
        F_EXISTS=1
        local st; st=$(stat -L -c '%a %U %Y' -- "$p" 2>/dev/null)
        F_MODE=${st%% *}; st=${st#* }; F_OWNER=${st%% *}; F_MTIME_EPOCH=${st##* }
        [ -n "$F_MTIME_EPOCH" ] && F_MTIME=$(date -u -d "@$F_MTIME_EPOCH" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)
        if [ $DO_HASH -eq 1 ] && [ -f "$p" ] && [ -r "$p" ]; then
            F_SHA=$(sha256sum -- "$p" 2>/dev/null | awk '{print $1}')
        fi
        pkg_lookup "$(readlink -f -- "$p" 2>/dev/null || printf '%s' "$p")"
        F_PKG=$P_PKG; F_STATE=$P_STATE
    fi
    FACTS[$p]=$(printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s' "$F_EXISTS" "${F_SHA:--}" "${F_MODE:--}" "${F_OWNER:--}" "${F_MTIME:--}" "${F_MTIME_EPOCH:-0}" "${F_PKG:--}" "${F_STATE:--}")
    # read back through the same path so cached and fresh values look identical
    IFS=$'\t' read -r F_EXISTS F_SHA F_MODE F_OWNER F_MTIME F_MTIME_EPOCH F_PKG F_STATE <<< "${FACTS[$p]}"
}

# The executable a command line runs, and the first existing file it hands a path to.
resolve_cmd() {  # sets R_IMAGE R_PAYLOAD
    local c=$1 t
    local -a toks=()
    R_IMAGE=""; R_PAYLOAD=""
    c="${c#"${c%%[![:space:]]*}"}"
    while [[ $c == [-@+!:]* ]]; do c=${c:1}; done          # systemd Exec prefixes
    read -r -a toks <<< "$c" || true
    local i=0 n=${#toks[@]}
    while [ "$i" -lt "$n" ]; do                             # skip wrappers and VAR=val
        t=${toks[$i]//[\"\']/}
        case "$t" in
            *=*|env|/usr/bin/env|/bin/env|nohup|nice|ionice|setsid|stdbuf|exec|command|-*) i=$((i+1)) ;;
            *) break ;;
        esac
    done
    [ "$i" -lt "$n" ] || return
    R_IMAGE=${toks[$i]//[\"\']/}
    R_IMAGE=${R_IMAGE%%[;&|]*}
    if [[ $R_IMAGE != /* ]]; then
        local found; found=$(command -v -- "$R_IMAGE" 2>/dev/null)
        [[ $found == /* ]] && R_IMAGE=$found
    fi
    for ((i=i+1; i<n; i++)); do
        t=${toks[$i]//[\"\'\(\)]/}; t=${t%%[;&|,]*}
        [[ $t == /* ]] || continue
        [ "$t" = "$R_IMAGE" ] && continue
        [ -f "$t" ] && { R_PAYLOAD=$t; return; }
    done
}

# emit KIND CATEGORY TECHNIQUE SCOPE LOCATION NAME COMMAND [EXTRA_FLAGS]
#   KIND=cmd : COMMAND is a command line; image/payload are resolved from it
#   KIND=file: LOCATION is the file itself (rc file, key file, rule file)
#   KIND=info: nothing to resolve (accounts, sudoers lines)
emit() {
    local kind=$1 cat=$2 tech=$3 scope=$4 loc=$5 name=$6 cmd=$7 xflags=${8:-}
    local image="" payload="" isha="" psha="" flags=() f
    [ -n "$xflags" ] && read -r -a flags <<< "$xflags"

    if [ "$kind" = cmd ] && [ -n "$cmd" ]; then
        resolve_cmd "$cmd"; image=$R_IMAGE; payload=$R_PAYLOAD
        [[ $cmd =~ $SUS_RE ]] && flags+=(SuspiciousCommand)
    elif [ "$kind" = file ]; then
        image=$loc
        add_artifact "$loc"
        if [ -f "$loc" ] && [ -r "$loc" ] && [ "$(stat -c %s -- "$loc" 2>/dev/null || echo 0)" -lt 1048576 ] \
           && grep -Eq -- "$SUS_RE" "$loc" 2>/dev/null; then
            flags+=(SuspiciousContent)
        fi
    fi

    local pkg="" pstate="" mode="" owner="" mtime=""
    if [ -n "$image" ]; then
        facts "$image"
        isha=$F_SHA; pkg=$F_PKG; pstate=$F_STATE; mode=$F_MODE; owner=$F_OWNER; mtime=$F_MTIME
        if [ "$F_EXISTS" = 0 ] && [[ $image == /* ]]; then flags+=(MissingFile); fi
        if [ "$F_EXISTS" = 1 ]; then
            [ "$F_STATE" = modified ] && flags+=(PackageModified)
            if [ "$kind" = cmd ] && [ "$F_STATE" = unowned ]; then flags+=(NotFromPackage); fi
            [[ ${F_MODE: -1} =~ [2367] ]] && flags+=(WorldWritable)
            if [ "$F_STATE" != ok ] && [ "${F_MTIME_EPOCH:-0}" -gt "$RECENT_CUTOFF" ]; then flags+=(RecentlyModified); fi
        fi
    fi
    if [ -n "$payload" ]; then
        facts "$payload"; psha=$F_SHA
        add_artifact "$payload"
        [ "$F_STATE" = modified ] && flags+=(PayloadPackageModified)
        [ "$F_STATE" = unowned ] && flags+=(PayloadNotFromPackage)
        if [ "$F_STATE" != ok ] && [ "${F_MTIME_EPOCH:-0}" -gt "$RECENT_CUTOFF" ]; then flags+=(PayloadRecentlyModified); fi
        [[ -n "$F_SHA" && -f "$payload" ]] && [ "$(stat -c %s -- "$payload" 2>/dev/null || echo 0)" -lt 1048576 ] \
            && grep -Eq -- "$SUS_RE" "$payload" 2>/dev/null && flags+=(SuspiciousContent)
    fi
    if [ "$kind" != info ]; then
        for f in "$image" "$payload"; do
            [ -n "$f" ] || continue
            [[ $f =~ ^/(tmp|var/tmp|dev/shm|run/user)/ ]] && { flags+=(TmpPath); break; }
        done
        # dot-directories are normal for ~/.bashrc and ~/.ssh; flag them only for executables
        [ "$kind" = cmd ] && [[ "$image$payload" =~ /\.[^/]+/ ]] && flags+=(HiddenPath)
    fi

    # No subshells from here on: emit runs thousands of times.
    local flagstr=""
    for f in "${flags[@]:-}"; do
        [ -n "$f" ] && [[ ";$flagstr;" != *";$f;"* ]] && flagstr+="${flagstr:+;}$f"
    done
    local vals=("$cat" "$tech" "$scope" "$loc" "$name" "$cmd" "$image" "$isha" "$payload" "$psha" "$pkg" "$pstate" "$mode" "$owner" "$mtime" "$flagstr")
    local line="" j="{" v k=0 keys; read -r -a keys <<< "$COLS"
    for v in "${vals[@]}"; do
        [ "$v" = "-" ] && v=""
        v=${v//$'\t'/ }; v=${v//$'\n'/ }; v=${v//$'\r'/ }
        [ $k -gt 0 ] && { line+=$'\t'; j+=","; }
        line+=$v
        v=${v//\\/\\\\}; v=${v//\"/\\\"}
        j+="\"${keys[$k]}\":\"$v\""
        k=$((k+1))
    done
    printf '%s\n' "$line" >> "$TSV"
    printf '%s}\n' "$j" >> "$JSONL"
    RECORDS=$((RECORDS+1))
}

# Non-comment, non-blank lines of a file, continuation-joined.
content_lines() { [ -r "$1" ] && sed -e 's/\r$//' -e ':a' -e '/\\$/N; s/\\\n//; ta' -- "$1" | grep -Ev '^[[:space:]]*(#|;|$)'; }

# Users whose home directory exists (service accounts included: a web server's
# ~/.bashrc is a persistence spot too).
declare -a U_NAME=() U_UID=() U_HOME=() U_SHELL=()
while IFS=: read -r n _ uid _ _ home shell; do
    [ -n "$home" ] && [ "$home" != / ] && [ -d "$home" ] || continue
    U_NAME+=("$n"); U_UID+=("$uid"); U_HOME+=("$home"); U_SHELL+=("$shell")
done < /etc/passwd

#----------------------------------------------------------------------------#
#  Collectors                                                                #
#----------------------------------------------------------------------------#

c_cron() {
    local f line
    # system crontabs: m h dom mon dow USER command
    for f in /etc/crontab /etc/cron.d/*; do
        [ -f "$f" ] || continue
        add_artifact "$f"
        while IFS= read -r line; do
            [[ $line =~ ^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*= ]] && continue    # VAR=value
            if [[ $line =~ ^[[:space:]]*(@[a-z]+)[[:space:]]+([^[:space:]]+)[[:space:]]+(.*)$ ]]; then
                emit cmd Cron T1053.003 "${BASH_REMATCH[2]}" "$f" "${BASH_REMATCH[1]}" "${BASH_REMATCH[3]}"
            elif [[ $line =~ ^[[:space:]]*([^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+(.*)$ ]]; then
                emit cmd Cron T1053.003 "${BASH_REMATCH[2]}" "$f" "${BASH_REMATCH[1]}" "${BASH_REMATCH[3]}"
            fi
        done < <(content_lines "$f")
    done
    # user crontabs: m h dom mon dow command (owner = file name)
    for f in /var/spool/cron/crontabs/* /var/spool/cron/*; do
        [ -f "$f" ] || continue
        add_artifact "$f"
        local u; u=$(basename "$f")
        while IFS= read -r line; do
            [[ $line =~ ^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*= ]] && continue
            if [[ $line =~ ^[[:space:]]*(@[a-z]+)[[:space:]]+(.*)$ ]]; then
                emit cmd Cron T1053.003 "$u" "$f" "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
            elif [[ $line =~ ^[[:space:]]*([^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+)[[:space:]]+(.*)$ ]]; then
                emit cmd Cron T1053.003 "$u" "$f" "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
            fi
        done < <(content_lines "$f")
    done
    # periodic directories: every file there is executed by run-parts
    for f in /etc/cron.hourly/* /etc/cron.daily/* /etc/cron.weekly/* /etc/cron.monthly/* /etc/cron.yearly/*; do
        [ -f "$f" ] || continue
        emit cmd CronPeriodic T1053.003 root "$(dirname "$f")" "$(basename "$f")" "$f"
        add_artifact "$f"
    done
    if [ -f /etc/anacrontab ]; then
        add_artifact /etc/anacrontab
        while IFS= read -r line; do
            [[ $line =~ ^[[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*= ]] && continue
            [[ $line =~ ^[[:space:]]*([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+([^[:space:]]+)[[:space:]]+(.*)$ ]] &&
                emit cmd Anacron T1053.003 root /etc/anacrontab "${BASH_REMATCH[3]}" "${BASH_REMATCH[4]}"
        done < <(content_lines /etc/anacrontab)
    fi
}

c_at() {
    local f
    for f in /var/spool/cron/atjobs/* /var/spool/at/*; do
        [ -f "$f" ] || continue
        [[ $(basename "$f") == .SEQ ]] && continue
        emit file AtJob T1053.002 "$(stat -c %U -- "$f" 2>/dev/null)" "$f" "$(basename "$f")" ""
    done
}

c_systemd() {
    command -v systemctl >/dev/null || [ -d /etc/systemd ] || { NOTES+=("systemd not present"); return 0; }
    declare -A STATE=()
    local u s
    if command -v systemctl >/dev/null; then
        while read -r u s _; do STATE[$u]=$s; done < <(systemctl list-unit-files --no-legend --no-pager 2>/dev/null)
    fi
    local dirs=(/etc/systemd/system /run/systemd/system /usr/local/lib/systemd/system /usr/lib/systemd/system /lib/systemd/system
                /etc/systemd/user /usr/lib/systemd/user /lib/systemd/user)
    local i
    for ((i=0; i<${#U_HOME[@]}; i++)); do dirs+=("${U_HOME[$i]}/.config/systemd/user"); done
    local f real scope unit key val line n
    declare -A SEEN_UNIT=()
    for d in "${dirs[@]}"; do
        [ -d "$d" ] || continue
        scope=system
        for ((i=0; i<${#U_HOME[@]}; i++)); do [[ $d == "${U_HOME[$i]}/"* ]] && scope=${U_NAME[$i]}; done
        [[ $d == */systemd/user ]] && [ "$scope" = system ] && scope=user-global
        while IFS= read -r -d '' f; do
            real=$(readlink -f -- "$f" 2>/dev/null)
            if [ "$real" = /dev/null ]; then
                emit info SystemdMasked T1543.002 "$scope" "$f" "$(basename "$f")" "masked"
                continue
            fi
            [ -f "$real" ] || continue
            [ -n "${SEEN_UNIT[$real]+x}" ] && continue
            SEEN_UNIT[$real]=1
            unit=$(basename "$f"); [[ $f == *.d/*.conf ]] && unit="$(basename "$(dirname "$f")")/$(basename "$f")"
            add_artifact "$real"
            local base=${unit%%.d/*} en
            en=${STATE[$base]:-?}
            n=0
            while IFS= read -r line; do
                [[ $line =~ ^[[:space:]]*(Exec[A-Za-z]*|OnCalendar|OnBootSec|OnUnitActiveSec|Unit)[[:space:]]*=[[:space:]]*(.*)$ ]] || continue
                key=${BASH_REMATCH[1]}; val=${BASH_REMATCH[2]}
                [ -n "$val" ] || continue
                case "$key" in
                    Exec*) n=$((n+1)); emit cmd SystemdUnit T1543.002 "$scope" "$real" "$unit:${key}[$n] ($en)" "$val" ;;
                    *)     [[ $base == *.timer ]] && emit info SystemdTimer T1053.006 "$scope" "$real" "$unit:$key ($en)" "$val" ;;
                esac
            done < <(content_lines "$real")
        done < <(find "$d" -maxdepth 2 \( -name '*.service' -o -name '*.timer' -o -name '*.socket' -o -name '*.path' -o -path '*.d/*.conf' \) -print0 2>/dev/null)
    done
    for d in /etc/systemd/system-generators /usr/local/lib/systemd/system-generators /usr/lib/systemd/system-generators /lib/systemd/system-generators \
             /etc/systemd/user-generators /usr/lib/systemd/user-generators; do
        [ -d "$d" ] || continue
        for f in "$d"/*; do [ -f "$f" ] && emit cmd SystemdGenerator T1543.002 system "$d" "$(basename "$f")" "$f"; done
    done
    for f in /var/lib/systemd/linger/*; do
        [ -e "$f" ] && emit info SystemdLinger T1543.002 "$(basename "$f")" /var/lib/systemd/linger "$(basename "$f")" "user services start at boot"
    done
}

c_sysv() {
    local f
    for f in /etc/init.d/*; do
        [ -f "$f" ] || continue
        emit cmd InitScript T1037.004 system /etc/init.d "$(basename "$f")" "$f"
    done
    for f in /etc/rc.local /etc/rc.d/rc.local /etc/inittab; do
        [ -f "$f" ] && emit file RcScript T1037.004 system "$f" "$(basename "$f")" ""
    done
}

c_shell_init() {
    local f i h
    for f in /etc/profile /etc/profile.d/* /etc/bash.bashrc /etc/bashrc /etc/bash.bash_logout /etc/environment \
             /etc/zshenv /etc/zprofile /etc/zshrc /etc/zlogin /etc/zsh/zshenv /etc/zsh/zprofile /etc/zsh/zshrc /etc/zsh/zlogin \
             /etc/csh.cshrc /etc/csh.login; do
        [ -f "$f" ] && emit file ShellInit T1546.004 system "$f" "$(basename "$f")" ""
    done
    for ((i=0; i<${#U_HOME[@]}; i++)); do
        h=${U_HOME[$i]}
        for f in .bashrc .bash_profile .bash_login .bash_logout .profile .zshrc .zprofile .zshenv .zlogin .cshrc .tcshrc .config/fish/config.fish; do
            [ -f "$h/$f" ] && emit file ShellInit T1546.004 "${U_NAME[$i]}" "$h/$f" "$f" ""
        done
    done
    if grep -qs 'LD_PRELOAD' /etc/environment; then
        emit info LdPreload T1574.006 system /etc/environment LD_PRELOAD "$(grep -h LD_PRELOAD /etc/environment)" RareLocation
    fi
}

c_autostart() {
    local f i line
    local dirs=(/etc/xdg/autostart)
    for ((i=0; i<${#U_HOME[@]}; i++)); do dirs+=("${U_HOME[$i]}/.config/autostart"); done
    for d in "${dirs[@]}"; do
        [ -d "$d" ] || continue
        for f in "$d"/*.desktop; do
            [ -f "$f" ] || continue
            add_artifact "$f"
            line=$(grep -m1 -E '^Exec=' "$f" 2>/dev/null); line=${line#Exec=}
            local scope=system
            for ((i=0; i<${#U_HOME[@]}; i++)); do [[ $d == "${U_HOME[$i]}/"* ]] && scope=${U_NAME[$i]}; done
            emit cmd XdgAutostart T1547.013 "$scope" "$f" "$(basename "$f")" "$line"
        done
    done
    for d in /etc/update-motd.d /etc/NetworkManager/dispatcher.d /etc/network/if-up.d /etc/network/if-pre-up.d /etc/ppp/ip-up.d; do
        [ -d "$d" ] || continue
        for f in "$d"/*; do [ -f "$f" ] && emit cmd EventScript T1037 system "$d" "$(basename "$f")" "$f"; done
    done
}

c_ssh() {
    local i h f line keyfile
    local files=()
    for ((i=0; i<${#U_HOME[@]}; i++)); do
        h=${U_HOME[$i]}
        for f in .ssh/authorized_keys .ssh/authorized_keys2; do [ -f "$h/$f" ] && files+=("${U_NAME[$i]}:$h/$f"); done
        [ -f "$h/.ssh/rc" ] && emit file SshRc T1098.004 "${U_NAME[$i]}" "$h/.ssh/rc" rc "" RareLocation
    done
    [ -f /etc/ssh/sshrc ] && emit file SshRc T1098.004 system /etc/ssh/sshrc sshrc "" RareLocation
    # Non-default AuthorizedKeysFile locations and AuthorizedKeysCommand
    local cfgs=(/etc/ssh/sshd_config /etc/ssh/sshd_config.d/*.conf)
    for f in "${cfgs[@]}"; do
        [ -f "$f" ] || continue
        add_artifact "$f"
        while IFS= read -r line; do
            if [[ $line =~ ^[[:space:]]*AuthorizedKeysCommand[[:space:]]+(.*)$ ]]; then
                emit cmd SshKeysCommand T1098.004 system "$f" AuthorizedKeysCommand "${BASH_REMATCH[1]}" RareLocation
            elif [[ $line =~ ^[[:space:]]*AuthorizedKeysFile[[:space:]]+(.*)$ ]]; then
                emit info SshConfig T1098.004 system "$f" AuthorizedKeysFile "${BASH_REMATCH[1]}"
                for keyfile in ${BASH_REMATCH[1]}; do
                    [[ $keyfile == /* && $keyfile != *%* ]] && [ -f "$keyfile" ] && files+=("config:$keyfile")
                done
            elif [[ $line =~ ^[[:space:]]*PermitRootLogin[[:space:]]+(.*)$ ]]; then
                emit info SshConfig T1098.004 system "$f" PermitRootLogin "${BASH_REMATCH[1]}"
            fi
        done < <(content_lines "$f")
    done
    for entry in "${files[@]:-}"; do
        [ -n "$entry" ] || continue
        local owner=${entry%%:*} path=${entry#*:}
        add_artifact "$path"
        while IFS= read -r line; do
            local comment; comment=$(awk '{ for (i=1;i<=NF;i++) if ($i ~ /^(ssh-|ecdsa-|sk-)/) { print (i<NF ? $NF : "(no comment)"); exit } }' <<< "$line")
            local opts=""; [[ $line =~ (command=|environment=|from=) ]] && opts="KeyOptions"
            emit info SshAuthorizedKey T1098.004 "$owner" "$path" "${comment:-(unparsed)}" "$line" "$opts"
        done < <(content_lines "$path")
    done
}

c_accounts() {
    local n uid shell pw
    # shellcheck disable=SC2094  # emit takes the path as a label only; nothing writes /etc/passwd
    while IFS=: read -r n _ uid _ _ _ shell; do
        if [ "$uid" = 0 ] && [ "$n" != root ]; then
            emit info Account T1136.001 system /etc/passwd "$n" "uid=0 shell=$shell" Uid0
        elif [[ $shell != */nologin && $shell != */false && $shell != /bin/sync && -n $shell ]]; then
            emit info Account T1136.001 system /etc/passwd "$n" "uid=$uid shell=$shell"
        fi
    done < /etc/passwd
    if [ -r /etc/shadow ]; then
        # shellcheck disable=SC2094  # same: /etc/shadow is only read
        while IFS=: read -r n pw _; do
            [ -z "$pw" ] && emit info Account T1136.001 system /etc/shadow "$n" "empty password field" EmptyPassword
        done < /etc/shadow
    else
        NOTES+=("/etc/shadow not readable: empty-password check skipped")
    fi
    local g members
    for g in root sudo wheel admin adm docker lxd disk shadow; do
        members=$(awk -F: -v g="$g" '$1==g {print $4}' /etc/group)
        [ -n "$members" ] && emit info PrivilegedGroup T1098 system /etc/group "$g" "$members"
    done
    local f line
    for f in /etc/sudoers /etc/sudoers.d/*; do
        [ -f "$f" ] || continue
        [ -r "$f" ] || { NOTES+=("$f not readable"); continue; }
        add_artifact "$f"
        while IFS= read -r line; do
            [[ $line =~ ^[[:space:]]*(Defaults|@include|#include) ]] && continue
            local fl=""; [[ $line == *NOPASSWD* ]] && fl="NoPasswd"
            emit info Sudoers T1548.003 system "$f" "${line%%[[:space:]]*}" "$line" "$fl"
        done < <(content_lines "$f")
    done
}

c_preload() {
    local f line
    if [ -f /etc/ld.so.preload ]; then
        add_artifact /etc/ld.so.preload
        while IFS= read -r line; do
            emit cmd LdPreload T1574.006 system /etc/ld.so.preload "$line" "$line" RareLocation
        done < <(content_lines /etc/ld.so.preload)
    fi
    for f in /etc/ld.so.conf /etc/ld.so.conf.d/*; do
        [ -f "$f" ] || continue
        add_artifact "$f"
        while IFS= read -r line; do
            [[ $line == include* ]] && continue
            emit info LdSoConf T1574.006 system "$f" "$line" "$line"
        done < <(content_lines "$f")
    done
}

c_pam() {
    [ -d /etc/pam.d ] || return 0
    local f line mod path d
    declare -A MODPATH=()
    local moddirs=(/lib/security /lib64/security /usr/lib/security /usr/lib64/security /lib/*-linux-gnu*/security /usr/lib/*-linux-gnu*/security)
    for f in /etc/pam.d/*; do
        [ -f "$f" ] || continue
        add_artifact "$f"
        while IFS= read -r line; do
            [[ $line =~ (pam_[A-Za-z0-9_]+\.so|/[^[:space:]]+\.so) ]] || continue
            mod=${BASH_REMATCH[1]}
            if [ -z "${MODPATH[$mod]+x}" ]; then
                path=""
                if [[ $mod == /* ]]; then path=$mod
                else for d in "${moddirs[@]}"; do [ -f "$d/$mod" ] && { path="$d/$mod"; break; }; done; fi
                MODPATH[$mod]=$path
            fi
            path=${MODPATH[$mod]}
            if [[ $line == *pam_exec.so* ]]; then
                emit cmd PamExec T1556.003 system "$f" "$mod" "${line#*pam_exec.so}" RareLocation
            elif [ -n "$path" ]; then
                facts "$path"
                # Only record modules that are not a clean package file; the rest is noise.
                [ "$F_STATE" != ok ] && emit cmd PamModule T1556.003 system "$f" "$mod" "$path"
            else
                emit info PamModule T1556.003 system "$f" "$mod" "$line" MissingFile
            fi
        done < <(content_lines "$f")
    done
}

c_kernel() {
    local m file f line
    if [ -r /proc/modules ] && ! command -v modinfo >/dev/null; then
        NOTES+=("modinfo not installed: loaded kernel modules not checked")
    elif [ -r /proc/modules ] && [ ! -d "/lib/modules/$(uname -r)" ]; then
        NOTES+=("no /lib/modules/$(uname -r) (container?): loaded kernel modules not checked")
    elif [ -r /proc/modules ]; then
        # shellcheck disable=SC2094  # same: /proc/modules is only read
        while read -r m _; do
            file=$(modinfo -n "$m" 2>/dev/null)
            if [ -z "$file" ] || [[ $file != /* ]]; then
                emit info KernelModule T1547.006 system /proc/modules "$m" "loaded, no module file ($file)" NoModuleFile
                continue
            fi
            facts "$file"
            [ "$F_STATE" = ok ] && continue
            emit cmd KernelModule T1547.006 system /proc/modules "$m" "$file"
        done < /proc/modules
    fi
    for f in /etc/modules /etc/modules-load.d/*.conf /usr/lib/modules-load.d/*.conf /run/modules-load.d/*.conf; do
        [ -f "$f" ] || continue
        add_artifact "$f"
        while IFS= read -r line; do emit info ModuleAutoload T1547.006 system "$f" "$line" "$line"; done < <(content_lines "$f")
    done
    # "install <module> <command>" runs an arbitrary command whenever the module loads
    for f in /etc/modprobe.d/*.conf /usr/lib/modprobe.d/*.conf /lib/modprobe.d/*.conf /run/modprobe.d/*.conf; do
        [ -f "$f" ] || continue
        add_artifact "$f"
        while IFS= read -r line; do
            [[ $line =~ ^[[:space:]]*install[[:space:]]+([^[:space:]]+)[[:space:]]+(.*)$ ]] || continue
            local cmd=${BASH_REMATCH[2]}
            [[ $cmd =~ ^(/bin/|/usr/bin/)?(true|false)$ ]] && continue      # the usual "disable this module" idiom
            emit cmd ModprobeInstall T1547.006 system "$f" "${BASH_REMATCH[1]}" "$cmd" RareLocation
        done < <(content_lines "$f")
    done
}

c_udev() {
    local f line cmd
    for f in /etc/udev/rules.d/*.rules /run/udev/rules.d/*.rules /usr/lib/udev/rules.d/*.rules /lib/udev/rules.d/*.rules; do
        [ -f "$f" ] || continue
        while IFS= read -r line; do
            [[ $line =~ RUN\{?[a-z]*\}?\+?=\"([^\"]+)\" ]] || continue
            cmd=${BASH_REMATCH[1]}
            add_artifact "$f"
            emit cmd UdevRun T1546.017 system "$f" "$(basename "$f")" "$cmd"
        done < <(content_lines "$f")
    done
}

c_pkg_hooks() {
    local f line
    for f in /etc/apt/apt.conf.d/*; do
        [ -f "$f" ] || continue
        while IFS= read -r line; do
            [[ $line =~ (Pre-Invoke|Post-Invoke|Post-Invoke-Success|Pre-Install-Pkgs)[^\"]*\"([^\"]+)\" ]] || continue
            add_artifact "$f"
            emit cmd PackageHook T1546.016 system "$f" "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}"
        done < <(content_lines "$f")
    done
    for d in /etc/yum/pluginconf.d /etc/dnf/plugins /usr/lib/python3*/site-packages/dnf-plugins /usr/lib/yum-plugins; do
        [ -d "$d" ] || continue
        for f in "$d"/*; do [ -f "$f" ] && emit file PackageHook T1546.016 system "$f" "$(basename "$f")" ""; done
    done
}

c_suid() {
    [ $DO_SUID -eq 1 ] || { NOTES+=("SUID/capabilities walk skipped (--no-suid)"); return 0; }
    local mnt fstype f
    # "/" always (it may be overlay/tmpfs), plus every other local disk filesystem.
    local mounts=(/)
    while read -r _ mnt fstype _; do
        [ "$mnt" != / ] && [ -d "$mnt" ] || continue
        case "$fstype" in ext2|ext3|ext4|xfs|btrfs|zfs|jfs|reiserfs|f2fs) mounts+=("$mnt") ;; esac
    done < /proc/mounts
    # All of them are recorded (a normal host has a few dozen): dpkg does not verify
    # modes, so "chmod u+s /bin/bash" on a package file is only visible this way.
    local shells_re='/(ba|da|z|k|c|tc|fi|a)?sh$|/(python|perl|ruby|php|lua|node|busybox|find|vim?|nano|less|more|env|cp|mv|tar|awk|gawk|mawk|sed|nmap|socat|nc|ncat|dd|tee|xargs|docker|systemctl)[0-9.]*$'
    for mnt in "${mounts[@]}"; do
        while IFS= read -r -d '' f; do
            local fl=""; [[ $f =~ $shells_re ]] && fl="SuidShellOrInterpreter"
            emit cmd SetuidFile T1548.001 system "$mnt" "$(stat -c %A -- "$f" 2>/dev/null)" "$f" "$fl"
        done < <(find "$mnt" -xdev -type f \( -perm -4000 -o -perm -2000 \) -print0 2>/dev/null)
    done
    if command -v getcap >/dev/null; then
        # getcap -r would cross into /proc and other mounts; walk each filesystem with -xdev instead.
        # Output is "path caps" (libcap >= 2.4x) or "path = caps" (older); both start with the path.
        for mnt in "${mounts[@]}"; do
            while IFS= read -r line; do
                f=${line%% *}
                local caps=${line#"$f"}; caps=${caps# = }; caps=${caps# }
                emit cmd FileCapability T1548 system "$mnt" "$caps" "$f"
            done < <(find "$mnt" -xdev -type f -print0 2>/dev/null | xargs -0 getcap 2>/dev/null | grep -v '^Failed')
        done
    else
        NOTES+=("getcap not installed: file capabilities not collected")
    fi
}

#----------------------------------------------------------------------------#
#  Run                                                                       #
#----------------------------------------------------------------------------#

COLLECTORS=(c_cron c_at c_systemd c_sysv c_shell_init c_autostart c_ssh c_accounts c_preload c_pam c_kernel c_udev c_pkg_hooks c_suid)
declare -A C_STATUS=() C_COUNT=() C_SECS=()
CERR="$OUT/.collector-stderr"
for c in "${COLLECTORS[@]}"; do
    before=$RECORDS; t0=$(date +%s)
    # Exit codes of these functions mean nothing (a final "[ -f x ] && ..." returns 1);
    # anything a collector printed to stderr is what counts.
    "$c" 2>"$CERR"
    nerr=$(grep -c . "$CERR" 2>/dev/null); nerr=${nerr:-0}
    if [ "$nerr" -gt 0 ]; then
        C_STATUS[$c]="$nerr stderr lines"
        { echo "### $c"; cat "$CERR"; } >> "$ERRLOG"
    else
        C_STATUS[$c]=ok
    fi
    C_COUNT[$c]=$((RECORDS - before)); C_SECS[$c]=$(( $(date +%s) - t0 ))
    printf '%-14s %-7s %5d records\n' "${c#c_}" "${C_STATUS[$c]}" "${C_COUNT[$c]}"
done

#----------------------------------------------------------------------------#
#  Write                                                                     #
#----------------------------------------------------------------------------#

awk -F'\t' 'NR==1 || $16!=""' "$TSV" > "$OUT/flagged.tsv"
FLAGGED=$(( $(wc -l < "$OUT/flagged.tsv") - 1 ))

if [ -s "$ARTLIST" ]; then
    sort -u "$ARTLIST" | tar -czf "$OUT/artifacts.tar.gz" -T - 2>>"$ERRLOG" || NOTES+=("artifacts.tar.gz incomplete, see errors.log")
fi
rm -f "$ARTLIST" "$CERR"

DIFF=""
if [ -n "$COMPARE" ]; then
    if [ -r "$COMPARE" ]; then
        # key: category|scope|location|name|command  — value: image + payload hashes
        awk -F'\t' -v OFS='\t' '
            FNR==1 { next }
            { k=$1 FS $3 FS $4 FS $5 FS $6; v=$8 "|" $10 }
            NR==FNR { old[k]=v; oldline[k]=$0; next }
            { new[k]=v; newline[k]=$0 }
            END {
                print "change", "category", "technique", "scope", "location", "name", "command", "image", "image_sha256", "payload", "payload_sha256", "package", "package_state", "mode", "owner", "mtime_utc", "flags"
                for (k in new) if (!(k in old)) print "added", newline[k]; else if (old[k] != new[k]) print "changed", newline[k]
                for (k in old) if (!(k in new)) print "removed", oldline[k]
            }' "$COMPARE" "$TSV" > "$OUT/diff.tsv"
        DIFF=$(awk -F'\t' 'NR>1 {c[$1]++} END {printf "+%d added, -%d removed, ~%d changed", c["added"], c["removed"], c["changed"]}' "$OUT/diff.tsv")
    else
        NOTES+=("--compare-to file not readable: $COMPARE")
    fi
fi

FINISHED=$(date -u +%Y-%m-%dT%H:%M:%SZ)
{
    printf '{\n'
    printf '  "tool": "get-persistence-snapshot.sh",\n'
    printf '  "tool_sha256": "%s",\n' "$(sha256sum -- "$0" 2>/dev/null | awk '{print $1}')"
    printf '  "host": "%s",\n' "$(json_esc "$HOST")"
    # shellcheck disable=SC1091  # os-release is data, not a script to follow
    printf '  "os": "%s",\n' "$(json_esc "$( . /etc/os-release 2>/dev/null; echo "${PRETTY_NAME:-unknown}")")"
    printf '  "kernel": "%s",\n' "$(json_esc "$(uname -r)")"
    printf '  "collected_by": "%s",\n' "$(json_esc "$(id -un) (uid $(id -u))")"
    printf '  "root": %s,\n' "$([ $IS_ROOT -eq 1 ] && echo true || echo false)"
    printf '  "package_manager": "%s",\n' "$PKGM"
    printf '  "started_utc": "%s",\n  "finished_utc": "%s",\n' "$STARTED" "$FINISHED"
    printf '  "options": {"hash": %s, "pkg": %s, "suid": %s, "recent_days": %s, "compare_to": "%s"},\n' \
        "$([ $DO_HASH -eq 1 ] && echo true || echo false)" "$([ $DO_PKG -eq 1 ] && echo true || echo false)" \
        "$([ $DO_SUID -eq 1 ] && echo true || echo false)" "$RECENT_DAYS" "$(json_esc "$COMPARE")"
    printf '  "total_records": %d,\n  "flagged_records": %d,\n' "$RECORDS" "$FLAGGED"
    printf '  "diff": "%s",\n' "$DIFF"
    printf '  "collectors": {'
    sep=""
    for c in "${COLLECTORS[@]}"; do
        printf '%s\n    "%s": {"status": "%s", "records": %d, "seconds": %d}' "$sep" "${c#c_}" "${C_STATUS[$c]}" "${C_COUNT[$c]}" "${C_SECS[$c]}"; sep=","
    done
    printf '\n  },\n  "notes": ['
    sep=""
    for n in "${NOTES[@]:-}"; do [ -n "$n" ] && { printf '%s\n    "%s"' "$sep" "$(json_esc "$n")"; sep=","; }; done
    printf '\n  ],\n  "files": ['
    sep=""
    while IFS= read -r f; do
        printf '%s\n    {"path": "%s", "sha256": "%s"}' "$sep" "$(json_esc "${f#"$OUT"/}")" "$(sha256sum -- "$f" | awk '{print $1}')"; sep=","
    done < <(find "$OUT" -type f ! -name manifest.json | sort)
    printf '\n  ]\n}\n'
} > "$OUT/manifest.json"

echo
echo "Records: $RECORDS   flagged: $FLAGGED   collector errors: see errors.log"
[ -n "$DIFF" ] && echo "Versus baseline: $DIFF (diff.tsv)"
[ ${#NOTES[@]} -gt 0 ] && echo "Notes: ${#NOTES[@]} (see manifest.json)"
echo "Output: $OUT"

if [ $ZIP -eq 1 ]; then
    tar -czf "$OUT.tar.gz" -C "$(dirname "$OUT")" "$(basename "$OUT")"
    echo "Archive: $OUT.tar.gz"
    echo "SHA256:  $(sha256sum -- "$OUT.tar.gz" | awk '{print $1}')"
fi