#!/bin/bash
# 
# Copyright (c) 2024 Red Hat.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.
#
# Pretty-print sosreport information using Performance Co-Pilot metrics
# from local host or an archive.
#

. $PCP_DIR/etc/pcp.env

sts=2
tmp=`mktemp -d "$PCP_TMPFILE_DIR/pcp-xsos.XXXXXXXXX"` || exit 1
trap "rm -rf $tmp; exit \$sts" 0 1 2 3 15

progname=`basename $0`

check_gawk()
{
    echo >&2 "$progname: this script needs gawk $@"
    exit
}
which gawk >/dev/null 2>&1 || check_gawk "but it is not installed"
gawk '@include "/dev/null"' {} 2>/dev/null || check_gawk "with @include syntax"

base_metrics=(
    kernel.all.boottime mem.physmem
)

ps_metrics=(
    proc.psinfo.utime proc.psinfo.stime
    proc.psinfo.sname proc.psinfo.psargs
    proc.psinfo.start_time proc.psinfo.threads
    proc.psinfo.rss proc.psinfo.vsize
    proc.id.uid_nm
)

disk_metrics=(
    disk.dev.capacity filesys.capacity
    filesys.used filesys.avail filesys.full filesys.mountdir
)

mem_metrics=(
    mem.util.used mem.util.dirty
    mem.util.shmem mem.util.bufmem mem.util.cached
    mem.util.swapFree mem.util.swapTotal
    mem.util.slab mem.util.pageTables mem.util.percpu
    mem.util.lowFree mem.util.lowTotal
    mem.util.anonhugepages mem.util.available
    mem.util.hugepagesFreeBytes mem.util.hugepagesTotalBytes
    mem.vmmemctl.current mem.vmmemctl.target
)

netdev_metrics=(
    network.interface.in.bytes network.interface.in.packets
    network.interface.in.errors network.interface.in.drops
    network.interface.in.fifo network.interface.in.frame
    network.interface.in.compressed network.interface.in.mcasts
    network.interface.out.bytes network.interface.out.packets
    network.interface.out.errors network.interface.out.drops
    network.interface.out.fifo network.interface.collisions
    network.interface.out.compressed network.interface.out.carrier
    network.sockstat.total network.sockstat.tcp.mem
    network.sockstat.tcp.inuse network.sockstat.tcp.orphan
    network.sockstat.tcp.tw network.sockstat.tcp.alloc
    network.sockstat.udp.inuse network.sockstat.frag.inuse
    network.sockstat.udp.mem network.sockstat.udplite.inuse
    network.sockstat.raw.inuse network.sockstat.raw6.inuse
    network.sockstat.frag.memory network.sockstat.tcp6.inuse
    network.sockstat.udp6.inuse network.sockstat.udplite6.inuse
    network.sockstat.frag6.inuse network.sockstat.frag6.memory
)

netstat_metrics=(
    network.icmp.inerrors network.icmp6.inerrors
    network.tcp.attemptfails network.tcp.estabresets
    network.tcp.inerrs network.tcp.outrsts
    network.tcp.delayedacklocked network.tcp.delayedacklost
    network.tcp.delayedacks network.tcp.pawsestabrejected
    network.tcp.abortontimeout
    network.tcp.lossproberecovery network.tcp.lossprobes
    network.tcp.timeouts network.tcp.tcptimeoutrehash
    network.ip.inaddrerrors network.ip6.inaddrerrors
)

os_metrics=(
    hinv.ncpu hinv.pagesize
    pmcd.hostname pmcd.timezone pmcd.zoneinfo
    kernel.uname.release kernel.uname.version
    kernel.uname.sysname kernel.uname.machine
    kernel.uname.nodename kernel.uname.distro
    kernel.all.boottime kernel.all.uptime
    kernel.all.nusers kernel.all.load kernel.all.hz
    kernel.all.runnable kernel.all.running
    kernel.all.blocked kernel.all.nprocs
    kernel.all.cpu.user kernel.all.cpu.nice
    kernel.all.cpu.sys kernel.all.cpu.idle
    kernel.all.cpu.wait.total kernel.all.cpu.steal
    kernel.all.cpu.irq.soft kernel.all.cpu.irq.hard
)

_usage()
{
    [ ! -z "$@" ] && echo $@ 1>&2
    pmgetopt --progname=$progname --usage --config=$tmp/usage
    exit
}

# usage spec for pmgetopt, note posix flag (commands mean no reordering)
cat > $tmp/usage << EOF
# getopts: a:h:O:S:?domnNpu:x
   --archive
   --host
   --origin
   --start
   --help
   --all           show everything
   -o,--os         show hostname, distro, kernel info, uptime, etc
   -d,--disks      show info from /proc/partitions, df
   -m,--mem        display memory summary
   -n,--netdev     display network interface summary
   -N,--netstat    display network statistics
   -p,--ps         inspect running processes, ps
   -u=P, --units=P change byte display where P is "b" for byte, "k", "m", "g", or "t"
   -x,--nocolor    disable output colorization
# end
EOF

color=true
osflag=false
memflag=false
diskflag=false
netdevflag=false
netstatflag=false
psflag=false
netunits='M' # options: B (byte), K, M, G or T
memunits='G' # options: B (byte), K, M, G or T
batch=''
args=`pmgetopt --progname=$progname --config=$tmp/usage -- "$@"`
[ $? != 0 ] && exit 1

eval set -- "$args"
while [ $# -gt 0 ]
do
    case "$1" in
      # pcp options
      -a)
        PCP_ARCHIVE="$2"; export PCP_ARCHIVE
        batch="-b 1"
        shift
        ;;
      -h)
        PCP_HOST="$2"; export PCP_HOST
        shift
        ;;
      -O)
        PCP_ORIGIN="$2"; export PCP_ORIGIN
        shift
        ;;
      -S)
        PCP_START_TIME="$2"; export PCP_START_TIME
        shift
        ;;
      # pcp-xsos options
      --all)
        osflag=true
        memflag=true
        diskflag=true
        netdevflag=true
        netstatflag=true
        psflag=true
        ;;
      -d)
        diskflag=true
        ;;
      -m)
        memflag=true
        ;;
      -n)
        netdevflag=true
        ;;
      -N)
        netstatflag=true
        ;;
      -o)
        osflag=true
        ;;
      -p)
        psflag=true
        ;;
      -u)
        memunits=`echo "$2" | tr '[:lower:]' '[:upper:]' | cut -c 1`
        netunits=${memunits}
        ;;
      -x)
        color=false
        ;;
      -\?)
        sts=0
        _usage ""
        ;;
      --)        # end of options, start of arguments
        sts=1
        _usage "Unknown argument: $2"
        ;;
    esac
    shift        # finished with this option, move to next
done

# accumulate an array of all metrics to be fetched
metrics=( ${base_metrics[*]} )
$osflag && metrics+=( ${os_metrics[*]} )
$memflag && metrics+=( ${mem_metrics[*]} )
$diskflag && metrics+=( ${disk_metrics[*]} )
$netdevflag && metrics+=( ${netdev_metrics[*]} )
$netstatflag && metrics+=( ${netstat_metrics[*]} )
$psflag && metrics+=( ${ps_metrics[*]} )

# default to OS metrics if nothing specified
if test ${#metrics[@]} -eq ${#base_metrics[@]}; then
    metrics+=( ${os_metrics[*]} )
    osflag=true
fi

# convert to printable form of unit string
unitstr()
{
    case "$1" in
      T) echo "TiB";;
      G) echo "GiB";;
      M) echo "MiB";;
      K) echo "KiB";;
      *) echo "B";;
    esac
}
memunits=`unitstr $memunits`
netunits=`unitstr $netunits`

if [ ! -z "$PCP_ARCHIVE" ]
then
    # extract pmcd values from log label in case metrics missing
    eval `pmdumplog -Ll 2>/dev/null | gawk '
/^Performance metrics from host/ { printf "pmcd_hostname_value=\"%s\"\n", $5 }
/^    commencing /               { $1 = ""; printf "sampletime=\"%s\"\n", $0 }
/^Archive timezone: /            { printf "pmcd_timezone_value=\"%s\"\n", $3 }
/^Archive zoneinfo: /            { printf "pmcd_zoneinfo_value=\"%s\"\n", $3 }
'`
else
    sampletime=$( date --iso-8601=ns )
fi
if [ ! -z "$PCP_START_TIME" ]
then
    sampletime=`echo ${PCP_START_TIME} | sed -e 's/^@//g'`
elif [ ! -z "$PCP_ORIGIN" ]
then
    sampletime=`echo ${PCP_ORIGIN} | sed -e 's/^@//g'`
fi
# Timestamp for sample in seconds since the epoch
timestamp=$( date --date "${sampletime}" +%s.%N 2>$tmp/error)
if test -s $tmp/error
then
    $PCP_ECHO_PROG $PCP_ECHO_N "$progname: ""$PCP_ECHO_C"
    sed < $tmp/error -e 's/^date: //g'
    sts=1
    exit
fi

# Extract values for all metrics at once.  Fetch using pminfo then
# cater for 3 cases: single-valued metrics, set-valued metrics and
# errors fetching individual metrics (see pminfo example below).
# It translates the pminfo output into a series of bash variables,
# including arrays (for set-valued metrics inst names and values).
# Note that this is both bash and awk syntax being generated here.
#
# Input:
# pminfo -f kernel.all.pswitch kernel.all.load kernel.cpu.util.user
#
# kernel.all.pswitch
#    value 730564942
#
# kernel.all.load
#    inst [1 or "1 minute"] value 0.02
#    inst [5 or "5 minute"] value 0.05
#    inst [15 or "15 minute"] value 0
#
# kernel.cpu.util.user
# No value(s) available!

# Output:
# kernel_all_pswitch_value=730564942
# kernel_all_load_inst[1]="1 minute"
# kernel_all_load_value[1]=0.19
# kernel_all_load_inst[5]="5 minute"
# kernel_all_load_value[5]=0.12
# kernel_all_load_inst[15]="15 minute"
# kernel_all_load_value[15]=0.06
# kernel_cpu_util_user_error="No value(s) available!"

PCP_SQUASH_NEWLINES=1; export PCP_SQUASH_NEWLINES
if ! pminfo $batch --fetch ${metrics[*]} > $tmp/metrics 2>$tmp/error
then
    if grep "^pminfo:" $tmp/error > /dev/null 2>&1
    then
        $PCP_ECHO_PROG $PCP_ECHO_N "$progname: ""$PCP_ECHO_C"
        sed < $tmp/error -e 's/^pminfo: //g'
        sts=1
        exit
    fi
fi
[ -s $tmp/error ] && sed -e '/Unknown metric name/d' <$tmp/error >&2

gawk < $tmp/metrics > $tmp/variables '
function filter(string) {
    gsub(/"/, "\\\"", string) # escape double quotes
    gsub(/\\u/, "\\\\u", string) # escape backslash-u
    # replace any characters with special shell meaning
    gsub("/\\(|\\$|\\*|)|\\{|\\}\\?|`|;|!/", "-", string)
    gsub(/%/, "%%", string) # percent sign in printf
    gsub(/^\\"|\\"$/, "\"", string) # except on ends
    return string
}
BEGIN { error = 0; count = 0; value = 0; metric = "" }
{
    if (NF == 0) {   # end previous metric (if any)
        metric = ""
    } else if ($1 == "Note:") {  # timezone message
        metric = ""
    } else if (metric == "") {   # new metric, name
        gsub("\\.", "_", $1)
        if (gsub(":$", "", $1) > 0) {
            printf("%s=%c%s%c\n", $1, "\"", $0, "\"")
            metric = ""
            error++
        } else {
            metric = $1
        }
        count++
    } else if ($1 == "value") {   # singleton metric
        printf("%s_value=%s\n", metric, substr($0,11))
        value++
    } else if ($1 == "inst") {   # set-valued metric
        sub("\\[", "")
        instid = $2
        instoff = index($0, " or \"") + 4
        instend = index($0, "\"]")
        instname = substr($0, instoff, instend-instoff+1)
        instname = filter(instname) # escape special chars
        valuestr = substr($0, index($0, "] value ") + 8)
        valuestr = filter(valuestr) # escape special chars
        printf("%s_inst[%s]=%s\n", metric, instid, instname)
        printf("%s_value[%s]=%s\n", metric, instid, valuestr)
        value++
    } else {    # set an error string for the metric
        printf("%s=%c%s%c\n", metric, "\"", $0, "\"")
        metric = ""
        error++
    }
}
END { printf "metrics=%d\nvalues=%d\nerrors=%d\n", count, value, error }'
eval `cat $tmp/variables`
#cat $tmp/variables

# check for a catastrophic failure (an error for every metric)
if test $errors -eq $metrics
then
    $PCP_ECHO_PROG $PCP_ECHO_N "$progname: ""$PCP_ECHO_C"
    $PCP_ECHO_PROG "failed to retrieve all $metrics metric values"
    sts=1
    exit
fi

# prepare an awk import with metric values and helper function(s)
cat > $tmp/metrics << EOF
function round(number, places) {
    places = 10 ^ places
    return int(number * places + .5) / places
}
BEGIN {
EOF
cat $tmp/variables >> $tmp/metrics
echo "}" >> $tmp/metrics
#cat $tmp/metrics

if $color; then
    RESET='\033[0m' # use defaults
    BLACK='\033[0;30m'
    RED='\033[0;31m'
    GREEN='\033[0;32m'
    ORANGE='\033[0;33m'
    BLUE='\033[0;34m'
    PURPLE='\033[0;35m'
    CYAN='\033[0;36m'
    WHITE='\033[0;37m'
    BACKBLUE='\033[44m' # background in blue
    BOLDWHITE='\033[1;37m'
    BOLDPURPLE='\033[1;35m'
    BOLDBLUE='\033[1;34m'
    BOLDRED='\033[1;31m'
fi

H0="${BOLDRED}"  # heading color
H1="  ${BOLDPURPLE}"  # next level down
H2="    ${BOLDBLUE}"  # second level heading

put_value()
{
    # $1 - mandatory metric name, e.g. kernel.all.pswitch
    # $2 - optional fallback value, used to override error string
    # $3 - optional formatting string for printf (allows alignment, etc)
    eval __value=`echo ${1} | sed -e 's/^/$/' -e 's/\./_/g' -e 's/$/_value/'`
    eval __error=`echo ${1} | sed -e 's/^/$/' -e 's/\./_/g' -e 's/$/_error/'`

    __format="%s"
    __fallback="${2}"
    test -n "${3}" && __format=${3}
    test -n "${__value}" && printf "${__format}" "${__value}" && return 0
    test -n "${__error}" -a -z "${__fallback}" && \
            printf "${__format}" "${RED}(missing)" && return 3
    test -n "${__fallback}" && printf "${__format}" "${ORANGE}${__fallback}" && echo 2
    printf "${__format}" "${ORANGE}unknown"
    return 1
}

get_value()
{
    # $1 - mandatory metric name, e.g. kernel.all.pswitch
    # $2 - mandatory fallback value, used when missing metric value
    eval __value=`echo ${1} | sed -e 's/^/$/' -e 's/\./_/g' -e 's/$/_value/'`
    test -n "${__value}" && echo "${__value}" && return 0
    echo "${2}"
    return 1
}

get_inst_value()
{
    # $1 - mandatory metric name, e.g. kernel.all.pswitch
    # $2 - mandatory numeric instance identifier, e.g. 1
    eval __value=`echo ${1} | sed -e 's/^/${/' -e 's/\./_/g' -e 's/$/_value['$2']}/'`
    test -n "${__value}" && echo "${__value}" && return 0
    echo "unknown"
    return 1
}

print_uptime()
{
    # Report on system up-time in days, hours and minutes
    # and number of users (given booted seconds + nusers)
    seconds=`echo "$1" | sed -e 's/\..*$//g'`
    users="$2"

    days=$((seconds / (60 * 60 * 24)))
    minutes=$((seconds / 60))
    hours=$((minutes / 60))
    hours=$((hours % 24))
    minutes=$((minutes % 60))
    if test $days -gt 1; then
        printf "$days days,"
    elif test $days -ne 0; then
        printf "1 day,"
    fi
    if test $hours -ne 0; then
        printf ' %2d:%02d,' $hours $minutes
    else
        printf ' %d min,' $minutes
    fi
    if test $users -eq 1; then
        printf '   1 user\n'
    else
        printf ' %2d users\n' $users
    fi
}

pcp_xsos_os()
{
    printf "${H0}OS\n"

    printf "${H1}Hostname:${RESET} "
    put_value pmcd.hostname; echo
    printf "${H1}Distro:${RESET}   "
    banner=`put_value kernel.uname.distro`
    printf "${BACKBLUE}${banner}${RESET}\n"
    printf "${H1}Arch:${RESET}     "
    platform=`put_value kernel.uname.machine`
    printf "platform=$platform\n"
    printf "${H1}Kernel:${RESET}\n"
    printf "${H2}Hertz:${RESET}         "
    put_value kernel.all.hz; echo
    printf "${H2}Pagesize:${RESET}      "
    put_value hinv.pagesize; echo
    printf "${H2}Build version:${RESET}\n"
    release=`put_value kernel.uname.release`
    release="version $release"
    sysname=`put_value kernel.uname.sysname`
    sysname="$sysname $release"
    version=`put_value kernel.uname.version "unknown build version"`
    printf "      ${ORANGE}${sysname}${RESET}\n"
    printf "      ${ORANGE}${version}${RESET}\n"
    printf "    - - - - - - - - - - - - - - - - - - -\n"

    printf "${H1}Boot time: ${RESET}"
    boottime=`get_value kernel.all.boottime 0`
    date --date="@$boottime" +"%a %b %d %I:%M:%S %P %Z %Y"
    printf "${H1}Time Zone: ${RESET}"
    timezone=`get_value pmcd.timezone unknown`
    zoneinfo=`get_value pmcd.zoneinfo unknown | sed -e 's/^://'`
    printf "${zoneinfo} [${timezone}]\n"
    printf "${H1}Uptime: ${RESET}  "
    uptime=`get_value kernel.all.uptime 0`
    nusers=`get_value kernel.all.nusers 0`
    print_uptime $uptime $nusers
    printf "${H1}LoadAvg:${RESET}  "
    ncpus=`get_value hinv.ncpu 0`
    printf "${BOLDWHITE}[$ncpus CPU]${RESET}"
    test $ncpus -lt 1 && ncpus=1  # safe division later
    for inst in 1 5 15; do
        load=`get_inst_value kernel.all.load $inst`
        percent=`gawk "BEGIN {print int(${load}*${ncpus}+.5)}"`
        test $inst -eq 1 || printf ","
        printf " %.2f (${GREEN}%d%%${RESET})" $load $percent
        test $inst -eq 15 && printf "\n"
    done

    printf "${H1}Processes: ${RESET}\n"
    put_value kernel.all.running "" "${H2}running:${RESET} %s"
    put_value kernel.all.runnable "" "${H2}runnable:${RESET} %s"
    put_value kernel.all.blocked "" "${H2}blocked:${RESET} %s"
    put_value kernel.all.nprocs "" "${H2}count:${RESET} %s\n"

    printf "${H1}Processors: ${RESET}\n"
    printf "${H2}cpu [Utilization since boot]: ${RESET}\n      "
    us=`get_value kernel.all.cpu.user 0`
    ni=`get_value kernel.all.cpu.nice 0`
    sy=`get_value kernel.all.cpu.sys 0`
    id=`get_value kernel.all.cpu.idle 0`
    wt=`get_value kernel.all.cpu.wa                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               