#!/bin/bash
# diagnostics.sh
# Script to gather information when things go wrong.
# Copyright (c) 2015 SoftNAS, Inc. All Rights Reserved

logit() {
	echo "`date +%Y%m%d%H%M%S` ${1}"
}

trap 'logit "Exiting..."' EXIT

logit "Starting..."

die() {
	logit "!!!!!!!!!!!!!!!!!!!!!!!"
	logit "!!!!!!!! ERROR !!!!!!!!"
	logit "!!!!!!!!!!!!!!!!!!!!!!!"
	logit "${1}"
	exit 1
}

logit "Checking permissions"
[ ${UID} -ne 0 ] && die "Must be root"

logit "Creating temporary path"
ZFS_INFO=$(mktemp -t zfs-info.XXXX)

logit "Gathering EBS data"
export EBSFILES=""
for i in /tmp/ebs* ; do
	export EBSFILES="${EBSFILES} $i"
done

logit "Looking at release information"
echo "System is $(cat /etc/redhat-release)" >> ${ZFS_INFO}

logit "Querying zdb"
echo "zdb output" >> ${ZFS_INFO}
zdb 2>&1 >> ${ZFS_INFO}

logit "obtain zpool import command output"
echo "obtain zpool import command output to ${ZFS_INFO}.history" >> ${ZFS_INFO}
zpool import -d /dev/ >> ${ZFS_INFO}.scan
logit "obtain zfs history"
echo "obtain zfs history to ${ZFS_INFO}.history" >> ${ZFS_INFO}
zpool history -i | gzip -5 > ${ZFS_INFO}.history.gz
logit "obtain zfs events"
echo "obtain zfs events to ${ZFS_INFO}.events" >> ${ZFS_INFO}
zpool events -v | gzip -5 > ${ZFS_INFO}.events.gz
logit "obtain zfs get"
echo "obtain zfs get all" >> ${ZFS_INFO}
zfs get all | gzip -5 > ${ZFS_INFO}.props.gz
logit "obtain zfs list"
echo "obtain zfs list" >> ${ZFS_INFO}
zfs list | gzip -5 > ${ZFS_INFO}.list.gz
logit "Get zpool status"
echo "zpool status -v" >> ${ZFS_INFO}
zpool status | gzip -5 > ${ZFS_INFO}.status.gz

logit "Looking at running processes"
echo "ps -auxw output (pre-stack dump)" >> ${ZFS_INFO}
ps -auxw >> ${ZFS_INFO}

logit "Viewing stack traces"
echo "/proc/*/stack output" >> ${ZFS_INFO}

for i in /proc/*/stack; do
	echo "Stack from ${i}" >> ${ZFS_INFO}
	if [[ -e  "${i}" ]]; then
		cat ${i} >> ${ZFS_INFO}
	fi
done

logit "Looking at running processes again"
echo "ps -ef output (pre-stack dump)" >> ${ZFS_INFO}
ps -ef >> ${ZFS_INFO}

logit "Reviewing ZFS ARC statistics"
echo "arcstats" >> ${ZFS_INFO}
cat /proc/spl/kstat/zfs/arcstats >> ${ZFS_INFO}

logit "Reviewing SPL SLABs"
echo "SPL SLABs" >> ${ZFS_INFO}
cat /proc/spl/kmem/slab >> ${ZFS_INFO}

logit "Reviewing kernel events"
echo "dmesg" >> ${ZFS_INFO}
dmesg >> ${ZFS_INFO}

logit "Reviewing ZFS events"
echo "ZFS Events" >> ${ZFS_INFO}
zpool events -v >> ${ZFS_INFO}

logit "Creating temporary directory"
STATS=$(mktemp -d stats.XXXX) 

logit "Copying softnas-cmd logs"
cp -R ~/softnas-cmd ${STATS}/softnas-cmd

logit "free memory"
free -m > ${STATS}/free
if [ -d /root/.azure ]; then
	logit "Azure logs"
	cp -Rv ~root/.azure ${STATS}/AzureLogs
else
	logit "No Azure Logs to collect, probably running on a different platform.."
fi
logit "NFS statistics"
nfsstat > ${STATS}/nfsstat
logit "lsof output"
lsof 2>&1 > ${STATS}/lsof
logit "IO statistics"
iostat > ${STATS}/iostat
logit "bash history"
cp ~root/.bash_history ${STATS}/history-root
if [ -d /home/ec2-user ]; then
	cp ~ec2-user/.bash_history ${STATS}/history-ec2user
fi
cp ~softnas/.bash_history ${STATS}/history-softnas
logit "Directory utilization"
df -h > ${STATS}/df-h
logit "Current mounts"
mount > ${STATS}/mount
logit "Block devices"
lsblk -l > ${STATS}/lsblk
logit "Installed packages"
yum list installed >> ${STATS}/yum_install
logit "Network routes"
netstat -r > ${STATS}/netroute
logit "Network connections"
netstat -an > ${STATS}/netstat
logit "Network configuration"
ifconfig -a > ${STATS}/ifconfig
logit "DNS configuration"
cat /etc/resolv.conf > ${STATS}/resolv.conf
logit "Mounts"
ls -lah /mnt/* > ${STATS}/mnts
logit "S3 Stats"
FILES=$(find /mnt -type f -name stats)
if [ -n "${FILES}" ]; then
	for n in ${FILES}; do
		mkdir -p $(dirname ${STATS}/${n})
		cp ${n} ${STATS}/${n}
	done
fi
logit "S3 Events"
FILES=$(find /mnt -type f -name events)
if [ -n "${FILES}" ]; then
	for n in ${FILES}; do
		mkdir -p $(dirname ${STATS}/${n})
		cat ${n} | gzip -9 > ${STATS}/${n}.gz
	done
fi
logit "MySQL statistics"
# mysql stats
if [ -f /root/sqlpass.conf ]; then
	source /root/sqlpass.conf
else
	source /var/www/softnas/config/sqlpass.conf
fi
if ! [ -z "$ROOTPASSWORD" ]; then
	mysql -uroot -p$ROOTPASSWORD -e 'SHOW FULL PROCESSLIST' > ${STATS}/mysql.processlist
	mysql -uroot -p$ROOTPASSWORD -e 'SHOW FULL TABLES' storagecenter > ${STATS}/mysql.tables
	mysql -uroot -p$ROOTPASSWORD -e 'SHOW STATUS' storagecenter > ${STATS}/mysql.status
	mysql -uroot -p$ROOTPASSWORD -e 'SHOW ENGINE InnoDB STATUS' storagecenter > ${STATS}/mysql.innodb.status
	mysql -uroot -p$ROOTPASSWORD -e 'SHOW ENGINE MyISAM STATUS' storagecenter > ${STATS}/mysql.myisam.status
	mysql -uroot -p$ROOTPASSWORD -e 'SELECT table_name AS "Table", round(((data_length + index_length) / 1024 / 1024), 2) as SIZE FROM information_schema.TABLES WHERE table_schema = "storagecenter" ORDER BY SIZE;' storagecenter > ${STATS}/mysql.tablesize
else
	logit "WARNING! MySQL credentials are unavailable! Can not gather SQL information."
fi
logit "Perf system analytics.."
# 'perf' stats output
perf stat -a sleep 5 1> ${STATS}/perf.stats
timeout 5s perf record -o ${STATS}/perf.data -ag

logit "Gathering UltraFast Files..."
ULTRAFAST_FILES=""
if [ -e /opt/ultrafast/log ]; then
	ULTRAFAST_FILES="/opt/ultrafast/log ${ULTRAFAST_FILES}"
fi
if [ -e /opt/ultrafast/conf ]; then
	ULTRAFAST_FILES="/opt/ultrafast/conf ${ULTRAFAST_FILES}"
fi
if [ -e /opt/ultrafast/data ]; then
	ULTRAFAST_FILES="/opt/ultrafast/data ${ULTRAFAST_FILES}"
fi
if [ -e /opt/ultrafast/run ]; then
	ULTRAFAST_FILES="/opt/ultrafast/run ${ULTRAFAST_FILES}"
fi

logit "Gathering Nifi Files..."
NIFI_HOME="/opt/nifidev-1.0/nifi-0.7.0"
if [ -f /etc/init.d/nifi ]; then
	NIFI_HOME=$(grep NIFI_HOME= /etc/init.d/nifi | awk -F "=" '{print $2}')
fi
NIFI_FILES=""
if [ -d $NIFI_HOME/bin ]; then
	NIFI_FILES="$NIFI_HOME/bin ${NIFI_FILES}"
fi
if [ -d $NIFI_HOME/conf ]; then
	NIFI_FILES="$NIFI_HOME/conf ${NIFI_FILES}"
fi
if [ -d $NIFI_HOME/logs ]; then
	NIFI_FILES="$NIFI_HOME/logs ${NIFI_FILES}"
fi
if [ -d $NIFI_HOME/ssl ]; then
	NIFI_FILES="$NIFI_HOME/ssl ${NIFI_FILES}"
fi
# Dump nifi processor states
if [ -f /var/www/softnas/snserver/LiftAndShift/nificmd.php ]; then
	php /var/www/softnas/snserver/LiftAndShift/nificmd.php --getprocstates >$NIFI_HOME/processor_states 2>&1
	if [ -f ${NIFI_HOME}/processor_states ]; then
		NIFI_FILES="$NIFI_HOME/processor_states ${NIFI_FILES}"
	fi
fi

logit "Gethering s3backer tunnel info..."
S3BACKER_FILES=""
if [ -e /tunnels ]; then
	S3BACKER_FILES=$(find /tunnels -name *.data)
	ls -laR /tunnels > /tunnels/file_list
	S3BACKER_FILES="$S3BACKER_FILES /tunnels/file_list"
fi

logit "Gathering LicenseManager files"
LICMGR_FILES=""
LICMGR_INSTALL_PATH="/opt/buurst/license_manager"
LICMGR_FOLDERS=( logs conf )
for folder in "${LICMGR_FOLDERS[@]}"; do
	if [ -d "$LICMGR_INSTALL_PATH/$folder" ]; then
		LICMGR_FILES="$LICMGR_FILES $LICMGR_INSTALL_PATH/$folder"
	fi
done

logit "Creating temporary file"
TARBALL=$(mktemp -t softnas-report.XXXX.tgz)

logit "Compressing into an archive file"
tar -czf ${TARBALL} \
	/tmp/zfs-info.* \
	/tmp/*.lock \
	/tmp/*.log \
	/tmp/ha* \
	${STATS}/* \
	/var/spool/abrt \
	/var/www/softnas/logs \
	/var/www/softnas/version \
	/var/www/softnas/config \
	/opt/buurst/softnas/*.log \
	/var/log \
	/etc \
	/var/spool/mail/root \
	/tmp/softnas-update.* \
	/tmp/*.sh \
	${EBSFILES} \
	${ULTRAFAST_FILES} \
	${NIFI_FILES} \
	${S3BACKER_FILES} \
	${LICMGR_FILES} 2> /dev/null

logit "Cleaning intermediate files"
rm -f ${ZFS_INFO} ${ZFS_INFO}.events ${ZFS_INFO}.history
rm -Rf ${STATS}

logit "Uploading to SoftNAS"
curl --upload-file "${TARBALL}" "${1}"
if [ $? -eq 0 ]; then
	logit "Successfully uploaded to SoftNAS. Removing archive from local filesystem: ${TARBALL}"
	rm -f ${TARBALL}
else
	die "Failed to upload to SoftNAS"
fi
logit "Cleaning archive"
rm -f ${TARBALL}
logit "**************************************************"
logit "******** OPERATION COMPLETED SUCCESSFULLY ********"
logit "**************************************************"
exit 0

