Initial commit
This commit is contained in:
commit
5826d924d6
|
|
@ -0,0 +1,4 @@
|
|||
USERNAME=username
|
||||
PASSWORD=password
|
||||
MAIN_DOMAIN="ip.mydomain.com"
|
||||
ADDITIONAL_DOMAINS="mydomain.com otherdomain.com www.otherdomain.com"
|
||||
|
|
@ -0,0 +1 @@
|
|||
FILES="/root/testfile.txt"
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
.*_env
|
||||
|
||||
storage/
|
||||
|
||||
# This file is unused atm
|
||||
docker_health_check.sh
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
SSH_MONITORING="192.168.1.1:22 example.com:2822"
|
||||
HTTP_MONITORING="http://wimi.meier.si https://google.com https://username:password@some-httpauth-protected.site"
|
||||
|
|
@ -0,0 +1 @@
|
|||
/home/robin/.post_startup_env
|
||||
|
|
@ -0,0 +1 @@
|
|||
.system_health_check_env
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
BOT_TOKEN=987654321:ABD-jTsfGp23cptjUsSv8md0sVjnFeCd8g
|
||||
CHAT_ID=12345678
|
||||
|
|
@ -0,0 +1 @@
|
|||
.zfs_health_check_env
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
#!/bin/bash
|
||||
|
||||
logfile=/root/logs/dyndns.log
|
||||
log_identifier="[DNS]"
|
||||
log() {
|
||||
echo -e $@ | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" >> $logfile
|
||||
}
|
||||
log_echo() {
|
||||
echo -e $@ | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" | tee -a $logfile
|
||||
}
|
||||
|
||||
set -o allexport
|
||||
source /root/scripts/.dyndns_env
|
||||
set +o allexport
|
||||
|
||||
url="https://${USERNAME}:${PASSWORD}@infomaniak.com/nic/update?hostname="
|
||||
|
||||
log "Updating DynDNS for ${MAIN_DOMAIN}"
|
||||
|
||||
response=$(curl -s -f "${url}${MAIN_DOMAIN}")
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log_echo "[ERROR] ${MAIN_DOMAIN} DynDNS Request Failed!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Response: ${response}"
|
||||
|
||||
if [[ "$response" =~ ^nochg ]]; then
|
||||
# IP has not changed
|
||||
# log_echo "IP has not changed, is still $(echo $response | awk '{print $2}')"
|
||||
exit 0
|
||||
|
||||
elif [[ "$response" =~ ^good ]]; then
|
||||
# IP has changed
|
||||
log_echo "IP HAS CHANGED TO $(echo $response | awk '{print $2}')"
|
||||
log_echo "${MAIN_DOMAIN} was updated successfully"
|
||||
|
||||
domains_error=0
|
||||
for $domain in $ADDITIONAL_DOMAINS
|
||||
do
|
||||
log "Updating DynDNS for ${domain}"
|
||||
additional_response=$(curl -s -f "${url}mneun.ch")
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log_echo "[ERROR] ${domain} DynDNS Request Failed!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "${domain} response: ${additional_response}"
|
||||
|
||||
if [[ "$additional_response" =~ ^good ]]; then
|
||||
# Change succeeded
|
||||
log_echo "${domain} was updated successfully"
|
||||
elif [[ "$additional_response" =~ ^nochg ]]; then
|
||||
log_echo "${domain} did not change"
|
||||
else
|
||||
log_echo "[ERROR] ${domain} DynDNS request response does not match expectations!"
|
||||
domains_error=1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ $domain_error -eq 1 ]; then
|
||||
exit 2
|
||||
fi
|
||||
else
|
||||
log_echo "[ERROR] ${MAIN_DOMAIN} DynDNS request respone does not match expectations!"
|
||||
exit 2
|
||||
fi
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
#!/bin/bash
|
||||
|
||||
################################################################################
|
||||
# FILE_MONITOR.SH
|
||||
# ---------------
|
||||
# This script checks the docker health and reports any problems on stdout
|
||||
#
|
||||
# Author: Robin Meier - robin@meier.si
|
||||
################################################################################
|
||||
|
||||
logfile=/root/logs/file_monitor.log
|
||||
log_identifier="[FILE]"
|
||||
log() {
|
||||
echo -e $@ | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" >> $logfile
|
||||
}
|
||||
log_echo() {
|
||||
echo -e $@ | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" | tee -a $logfile
|
||||
}
|
||||
|
||||
set -o allexport
|
||||
source /root/scripts/.file_monitor_env
|
||||
set +o allexport
|
||||
|
||||
mkdir -p /root/scripts/storage/file_monitor
|
||||
|
||||
for file in $FILES
|
||||
do
|
||||
# Touch storage file if not existing
|
||||
if [ ! -f /root/scripts/storage/file_monitor/${file//\//_} ]; then
|
||||
touch /root/scripts/storage/file_monitor/${file//\//_} ];
|
||||
fi
|
||||
|
||||
if [ "$file" -nt "/root/scripts/storage/file_monitor/${file//\//_}" ]; then
|
||||
log_echo "[CHANGE] $file"
|
||||
touch /root/scripts/storage/file_monitor/${file//\//_} ];
|
||||
fi
|
||||
done
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
#!/bin/bash
|
||||
|
||||
logfile=/root/logs/monitoring.log
|
||||
log_identifier="[MON]"
|
||||
log() {
|
||||
echo -e $@ | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" >> $logfile
|
||||
}
|
||||
log_echo() {
|
||||
echo -e $@ | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" | tee -a $logfile
|
||||
}
|
||||
|
||||
set -o allexport
|
||||
source /root/scripts/.monitoring_env
|
||||
set +o allexport
|
||||
|
||||
|
||||
problems=0
|
||||
|
||||
log "Starting monitoring run"
|
||||
|
||||
for ssh_host in $SSH_MONITORING
|
||||
do
|
||||
if [[ $(nc -w 2 ${ssh_host//:/ } <<< "\0" ) =~ "OpenSSH" ]] ; then
|
||||
log "[SSH] [OK] ${ssh_host} is reachable"
|
||||
else
|
||||
# TODO: Rate limit fail messages, also add is back up message
|
||||
log_echo "[SSH] [FAIL] ${ssh_host} not reachable"
|
||||
problems=1
|
||||
fi
|
||||
done
|
||||
|
||||
# TODO: HTTP Status Code 200 Monitoring
|
||||
for http_host in $HTTP_MONITORING
|
||||
do
|
||||
status_code=$(curl --write-out %{http_code} --silent --output /dev/null $http_host)
|
||||
if [[ "$status_code" -eq 200 ]] ; then
|
||||
log "[WEB] [OK] ${http_host}"
|
||||
else
|
||||
# TODO: Rate limit fail messages, also add is back up message
|
||||
log_echo "[WEB] [FAIL] ${http_host} status code is ${status_code}"
|
||||
problems=1
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
if [[ "$problems" -eq "0" ]]; then
|
||||
log "Monitoring Run Successful"
|
||||
else
|
||||
log_echo "Monitoring Run Failed"
|
||||
fi
|
||||
|
|
@ -0,0 +1 @@
|
|||
/home/robin/post_startup.sh
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
#!/bin/bash
|
||||
|
||||
################################################################################
|
||||
# SYSTEM_HEALTH_CHECK.SH
|
||||
# ----------------------
|
||||
# This script checks the system health and reports any problems on stdout
|
||||
#
|
||||
# Author: Robin Meier - robin@meier.si
|
||||
################################################################################
|
||||
|
||||
logfile=/root/logs/system_health_check.log
|
||||
log_identifier="[SYS]"
|
||||
log() {
|
||||
echo -e $@ | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" >> $logfile
|
||||
}
|
||||
log_echo() {
|
||||
echo -e $@ | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" | tee -a $logfile
|
||||
}
|
||||
|
||||
set -o allexport
|
||||
source /root/scripts/.system_health_check_env
|
||||
set +o allexport
|
||||
|
||||
|
||||
problems=0
|
||||
|
||||
log "Starting System Health Check"
|
||||
|
||||
# RAM usage percentage
|
||||
ram=$(free | awk '/Mem/{printf("%.2f"), $3/$2*100}')
|
||||
if [ $(echo "$ram > $RAM_LIMT" | bc -l) -eq 1 ]; then
|
||||
log_echo "[RAM] usage is ${ram}%! (Limit: $RAM_LIMIT)"
|
||||
problems=1
|
||||
else
|
||||
log "[RAM] usage is ${ram}%"
|
||||
fi
|
||||
|
||||
# CPU usage percentage
|
||||
cpu=$(top -bn1 | grep "Cpu(s)" | awk '{print $2 + $4}')
|
||||
if [ $(echo "$cpu > $CPU_LIMIT" | bc -l) -eq 1 ]; then
|
||||
log_echo "[CPU] load is ${cpu}%! (Limit: $CPU_LIMIT)"
|
||||
problems=1
|
||||
else
|
||||
log "[CPU] load is ${cpu}%"
|
||||
fi
|
||||
|
||||
# Load?
|
||||
# TODO: Maybe check `load`
|
||||
|
||||
# Temperature
|
||||
avg_cpu_temp=$(sensors | awk '/^Core /{++r; gsub(/[^[:digit:]]+/, "", $3); s+=$3} END{print s/(10*r)}')
|
||||
if [ $(echo "$avg_cpu_temp > $TEMP_LIMIT" | bc -l) -eq 1 ]; then
|
||||
log_echo "[TEMP] is ${avg_cpu_temp}°C! (Limit: $TEMP_LIMIT)"
|
||||
problems=1
|
||||
else
|
||||
log "[TEMP] is ${avg_cpu_temp}°C"
|
||||
fi
|
||||
|
||||
# Failed Services
|
||||
failed_services=$(systemctl status | grep -i 'Failed:\s.*\sunits' | awk '{print $2}')
|
||||
if [ $failed_services -eq 0 ]; then
|
||||
log "[SRV] No failed services"
|
||||
else
|
||||
log_echo "[SRV] [ERROR] Failed Services:"
|
||||
/bin/systemctl --failed | ts "[%Y-%m-%d %H:%M:%S] $log_identifier [SRV]" | tee -a $logfile
|
||||
log_echo "[SRV] You should run \"systemctl status/start <servicename>\" to find out more or start the unit."
|
||||
problems=1
|
||||
fi
|
||||
|
||||
# Finish
|
||||
if [ ${problems} -eq 0 ]; then
|
||||
log "System Health Check Successful"
|
||||
else
|
||||
log_echo "System Health Check Found Problems"
|
||||
fi
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
#!/bin/bash
|
||||
|
||||
################################################################################
|
||||
# TELEGRAM_NOTIFICATION.SH
|
||||
# ------------------------
|
||||
# This script takes input via stdin or parameters, removes timestamps from each
|
||||
# line and replaces newlines with telegram compatible ones and then sends the
|
||||
# message to a chat
|
||||
#
|
||||
# Author: Robin Meier - robin@meier.si
|
||||
################################################################################
|
||||
|
||||
set -o allexport
|
||||
source /root/scripts/.telegram_notification_env
|
||||
set +o allexport
|
||||
|
||||
BOT_API_URL=https://api.telegram.org/bot${BOT_TOKEN}
|
||||
|
||||
# Get input from standard input or via first parameter
|
||||
if [[ $# -eq 0 ]]; then
|
||||
MESSAGE=$(timeout 30 cat)
|
||||
elif [[ $# -eq 1 ]]; then
|
||||
MESSAGE=$1
|
||||
elif [[ $# -eq 2 ]]; then
|
||||
CHAT_ID=$1
|
||||
MESSAGE=$2
|
||||
else
|
||||
echo "[ERROR] Too many arguments!"
|
||||
fi
|
||||
|
||||
# Exit if input is empty
|
||||
if [[ -z "${MESSAGE}" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Strip timestamps from message
|
||||
if [ "${MESSAGE:0:12}" == "$(echo '' | ts "[%Y-%m-%d")" ]; then
|
||||
MESSAGE=$(echo -e "$MESSAGE" | cut -c 23-)
|
||||
fi
|
||||
|
||||
# Replace newlines in message for telegram
|
||||
TG_MESSAGE=${MESSAGE//$'\n'/\%0A}
|
||||
|
||||
# Send telegram to chat
|
||||
resp=$(curl -s -f -X POST ${BOT_API_URL}/sendMessage -d chat_id=$CHAT_ID -d text="⚠️ *$(hostname | tr . ' ')* ⚠️%0A\`\`\`%0A${TG_MESSAGE}%0A\`\`\`" -d parse_mode=markdown)
|
||||
|
||||
# Check if request succeeded
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "[ERROR] Telegram request failed!"
|
||||
else
|
||||
if [ "${resp:1:9}" == "\"ok\":true" ]; then
|
||||
# echo "Sent Telegram: \n${MESSAGE//$'\n'/\n}"
|
||||
else
|
||||
echo "[ERROR] Telegram sending did not succeed: $resp"
|
||||
echo "MESSAGE: \n${MESSAGE//$'\n'/\n}"
|
||||
fi
|
||||
fi
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
#!/bin/bash
|
||||
|
||||
################################################################################
|
||||
# ZFS_HEALTH_CHECK.SH
|
||||
# -------------------
|
||||
# This script checks the ZFS health and reports any problems on stdout
|
||||
#
|
||||
# Inspired by https://gist.github.com/petervanderdoes/bd6660302404ed5b094d
|
||||
#
|
||||
# Author: Robin Meier - robin@meier.si
|
||||
################################################################################
|
||||
|
||||
logfile=/root/logs/zfs_health_check.log
|
||||
log_identifier="[ZFS]"
|
||||
log() {
|
||||
echo -e $@ | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" >> $logfile
|
||||
}
|
||||
log_echo() {
|
||||
echo -e $@ | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" | tee -a $logfile
|
||||
}
|
||||
|
||||
problems=0
|
||||
|
||||
set -o allexport
|
||||
source /root/scripts/.zfs_health_check_env
|
||||
set +o allexport
|
||||
|
||||
log "Starting ZFS Health Check"
|
||||
|
||||
# Pool Status
|
||||
zpool_status=$(/sbin/zpool status | egrep -i '(DEGRADED|FAULTED|OFFLINE|UNAVAIL|REMOVED|FAIL|DESTROYED|corrupt|cannot|unrecover)')
|
||||
if [ "${zpool_status}" ]; then
|
||||
log_echo "[ERROR] !!! BAD ZFS HEALTH !!!"
|
||||
/sbin/zpool status | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" | tee -a $logfile
|
||||
problems=1
|
||||
fi
|
||||
|
||||
# Capacity
|
||||
maxCapacity=50
|
||||
if [ ${problems} -eq 0 ]; then
|
||||
capacity=$(/sbin/zpool list -H -o capacity)
|
||||
for line in ${capacity//%/}
|
||||
do
|
||||
if [ $line -ge $maxCapacity ]; then
|
||||
problems=1
|
||||
fi
|
||||
done
|
||||
if [ ${problems} -eq 1 ]; then
|
||||
log_echo "[ERROR] !!! BAD ZFS CAPACITY !!!"
|
||||
/sbin/zpool list -o name,cap,free | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" | tee -a $logfile
|
||||
fi
|
||||
fi
|
||||
|
||||
# Errors - Check the columns for READ, WRITE and CKSUM (checksum) drive errors
|
||||
# on all volumes and all drives using "zpool status". If any non-zero errors
|
||||
# are reported an email will be sent out. You should then look to replace the
|
||||
# faulty drive and run "zpool scrub" on the affected volume after resilvering.
|
||||
if [ ${problems} -eq 0 ]; then
|
||||
errors=$(/sbin/zpool status | grep ONLINE | grep -v state | awk '{print $3 $4 $5}' | grep -v 000)
|
||||
if [ "${errors}" ]; then
|
||||
log_echo "[ERROR] !!! ZFS ERRORS FOUND !!!"
|
||||
/sbin/zpool status | ts "[%Y-%m-%d %H:%M:%S] $log_identifier" | tee -a $logfile
|
||||
log_echo "[ERROR] You shoud replace the faulty drive and run \"zpool scrub\" after resilvering"
|
||||
problems=1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Scrub Expired - Check if all volumes have been scrubbed in at least the last
|
||||
# 8 days. The general guide is to scrub volumes on desktop quality drives once
|
||||
# a week and volumes on enterprise class drives once a month. You can always
|
||||
# use cron to schedule "zpool scrub" in off hours. We scrub our volumes every
|
||||
# Sunday morning for example.
|
||||
#
|
||||
# Scrubbing traverses all the data in the pool once and verifies all blocks can
|
||||
# be read. Scrubbing proceeds as fast as the devices allows, though the
|
||||
# priority of any I/O remains below that of normal calls. This operation might
|
||||
# negatively impact performance, but the file system will remain usable and
|
||||
# responsive while scrubbing occurs. To initiate an explicit scrub, use the
|
||||
# "zpool scrub" command.
|
||||
#
|
||||
# The scrubExpire variable is in seconds. So for 8 days we calculate 8 days
|
||||
# times 24 hours times 3600 seconds to equal 691200 seconds.
|
||||
# scrubExpire=691200
|
||||
scrubExpire=1382400
|
||||
if [ ${problems} -eq 0 ]; then
|
||||
currentDate=$(date +%s)
|
||||
zfsVolumes=$(/sbin/zpool list -H -o name)
|
||||
for volume in ${zfsVolumes}
|
||||
do
|
||||
if [ $(/sbin/zpool status $volume | egrep -c "none requested") -ge 1 ]; then
|
||||
log_echo "ERROR: You need to run \"zpool scrub $volume\" before this script can monitor the scrub expiration time."
|
||||
break
|
||||
fi
|
||||
if [ $(/sbin/zpool status $volume | egrep -c "scrub in progress|resilver") -ge 1 ]; then
|
||||
break
|
||||
fi
|
||||
scrubRawDate=$(/sbin/zpool status $volume | grep scrub | awk '{print $11" "$12" " $13" " $14" "$15}')
|
||||
scrubDate=$(date -d "$scrubRawDate" +%s)
|
||||
if [ $(($currentDate - $scrubDate)) -ge $scrubExpire ]; then
|
||||
problems=1
|
||||
log_echo "[ERROR] Pool: $volume needs scrub!"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
# Unmounted datasets
|
||||
unmounted=$(/sbin/zfs list -o mounted | grep no)
|
||||
if [ "${unmounted}" ]; then
|
||||
log "[WARN] THERE ARE UNMOUNTED DATASETS"
|
||||
fi
|
||||
|
||||
# Warnable unmounted datasets
|
||||
for dataset in $MONITORED_DATASETS
|
||||
do
|
||||
unmounted=$(/sbin/zfs list $dataset -o mounted | grep no)
|
||||
if [ "${unmounted}" ]; then
|
||||
log_echo "[MON] Monitored dataset $dataset is not mounted"
|
||||
problems=1
|
||||
fi
|
||||
done
|
||||
|
||||
# Finish
|
||||
if [ ${problems} -eq 0 ]; then
|
||||
log "ZFS Health Check Successful"
|
||||
else
|
||||
log_echo "ZFS Health Check Found Problems"
|
||||
fi
|
||||
Loading…
Reference in New Issue