Shell 腳本監控磁碟 I/O 與系統負載

需求背景

生產環境需定期蒐集磁碟 I/O 與系統負載指標，寫入資料庫供監控儀表板分析。以下提供三支協作腳本：collector.sh（單機蒐集）、dispatcher.sh（分發部署）、aggregator.sh（彙總入庫）。

1. 單機蒐集腳本 sarlog.sh

#!/bin/bash
# 部署於各受監控主機，每分鐘蒐集一次，持續 2 小時（可配合 cron 執行）

HOST=$(hostname)
CHANNEL=$(echo "$HOST" | sed 's/[0-9]//g')
RUNDAY=$(date +%Y-%m-%d)
IP=$(/sbin/ifconfig | awk '/inet addr:/ && !/127.0.0.1/ {gsub(/addr:/,"",$2); print $2; exit}')

LOADDIR="/tmp"
LOADFILE="${LOADDIR}/${HOST}-${RUNDAY}load.txt"
SARFILE="${LOADDIR}/${HOST}-${RUNDAY}sar.txt"

for i in {1..120}; do
    TS=$(date "+%Y-%m-%d %H:%M:%S")
    # loadavg: 1/5/15 min load, running/total tasks, last pid
    cat /proc/loadavg | awk -v ts="$TS" '{print ts","$1","$2","$3","$4","$5}' >> "$LOADFILE"
    # sar -u: CPU 使用率 %, %user, %nice, %system, %iowait, %steal, %idle
    sar -u 1 1 | awk -v ts="$TS" 'NR==4 {print ts","$4","$6","$7","$9}' >> "$SARFILE"
    sleep 59
done

# 後處理：加入主機資訊欄位，準備入庫
awk -F, -v h="$HOST" -v c="$CHANNEL" -v ip="$IP" \
    '{print h","c","ip","$0}' "$SARFILE" > "${SARFILE%.txt}.csv"
awk -F, -v h="$HOST" -v c="$CHANNEL" -v ip="$IP" \
    '{print h","c","ip","$0}' "$LOADFILE" > "${LOADFILE%.txt}.csv"

# 清理暫存
rm -f "$SARFILE" "$LOADFILE"

2. 分發部署腳本 dispatch.sh

#!/bin/bash
# 管理機執行：將 sarlog.sh 推送至所有受監控主機並背景執行

PDIR="/monitor/nagios/shell/sarlog"
MACHINES="${PDIR}/machine.lst"   # 每行一個 hostname/IP

for host in $(cat "$MACHINES"); do
    scp "${PDIR}/sarlog.sh" "${host}:/root/" && \
    ssh "${host}" "nohup /root/sarlog.sh >/dev/null 2>&1 &" &
done
wait
echo "所有節點蒐集已啟動"

3. 彙總入庫腳本 aggregate.sh

#!/bin/bash
# 管理機執行：收集各節點 CSV，計算 Busy%，載入 MySQL

PDIR="/monitor/nagios/shell/sarlog"
TMPDIR="${PDIR}/log"
MACHINES="${PDIR}/machine.lst"
RUNDAY=$(date +%Y-%m-%d)

mkdir -p "$TMPDIR"

# 1. 從各節點拉回 CSV
for host in $(cat "$MACHINES"); do
    scp "${host}:/tmp/*${RUNDAY}sar.csv" "$TMPDIR/"
    scp "${host}:/tmp/*${RUNDAY}load.csv" "$TMPDIR/"
done

# 2. 合併檔案
cat "$TMPDIR"/*${RUNDAY}sar.csv   > "$TMPDIR/sar_${RUNDAY}.csv"
cat "$TMPDIR"/*${RUNDAY}load.csv  > "$TMPDIR/load_${RUNDAY}.csv"

# 3. 計算 CPU Busy% = 100 - %idle
awk -F, '{
    idle=$8; busy=100-idle;
    print $0","busy
}' "$TMPDIR/sar_${RUNDAY}.csv" > "$TMPDIR/sar_busy_${RUNDAY}.csv"

# 4. 載入 MySQL (需預先建立 t_daily_stats, t_daily_load 表)
mysql -u root -D nagios -p'你的密碼' <

CREATE TABLE t_daily_stats (
    host VARCHAR(64), channel VARCHAR(64), ip VARCHAR(32),
    ts DATETIME, user_pct DECIMAL(5,2), nice_pct DECIMAL(5,2),
    sys_pct DECIMAL(5,2), idle_pct DECIMAL(5,2), busy_pct DECIMAL(5,2)
);

CREATE TABLE t_daily_load (
    host VARCHAR(64), channel VARCHAR(64), ip VARCHAR(32),
    ts DATETIME, load1 DECIMAL(5,2), load5 DECIMAL(5,2),
    load15 DECIMAL(5,2), tasks VARCHAR(32), last_pid INT
);