# === 下载安装 ===
cd /opt
wget https://github.com/prometheus/prometheus/releases/download/v2.52.0/prometheus-2.52.0.linux-amd64.tar.gz
tar xzf prometheus-2.52.0.linux-amd64.tar.gz
ln -s prometheus-2.52.0.linux-amd64 prometheus

# === 创建 systemd 服务 ===
cat > /etc/systemd/system/prometheus.service << 'EOF'
[Unit]
Description=Prometheus
After=network.target

[Service]
User=prometheus
Group=prometheus
Type=simple
ExecStart=/opt/prometheus/prometheus \
  --config.file=/opt/prometheus/prometheus.yml \
  --storage.tsdb.path=/data/prometheus \
  --storage.tsdb.retention.time=30d \
  --web.enable-lifecycle \
  --web.external-url=http://prometheus.example.com
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

useradd -r -s /sbin/nologin prometheus
mkdir -p /data/prometheus
chown -R prometheus:prometheus /opt/prometheus /data/prometheus
systemctl daemon-reload && systemctl enable --now prometheus

global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels:
    datacenter: 'bj-idc-01'
    env: 'production'

# 告警规则文件
rule_files:
  - 'rules/*.yml'

# 告警管理器
alerting:
  alertmanagers:
    - static_configs:
        - targets: ['localhost:9093']

# 采集目标
scrape_configs:
  # Prometheus 自身
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Node Exporter (系统指标)
  - job_name: 'node'
    scrape_interval: 30s
    static_configs:
      - targets:
          - '192.168.1.11:9100'
          - '192.168.1.12:9100'
          - '192.168.1.13:9100'
        labels:
          env: 'production'
    # 基于文件的动态发现
    file_sd_configs:
      - files:
          - '/opt/prometheus/targets/node/*.json'
        refresh_interval: 5m

  # MySQL Exporter
  - job_name: 'mysql'
    static_configs:
      - targets: ['192.168.1.21:9104']
        labels:
          instance: 'mysql-master'

  # Redis Exporter
  - job_name: 'redis'
    static_configs:
      - targets: ['192.168.1.31:9121']

  # Nginx Exporter (需 nginx-module-vts)
  - job_name: 'nginx'
    static_configs:
      - targets: ['192.168.1.41:9113']

# 安装 node_exporter
wget https://github.com/prometheus/node_exporter/releases/download/v1.8.0/node_exporter-1.8.0.linux-amd64.tar.gz
tar xzf node_exporter-1.8.0.linux-amd64.tar.gz
mv node_exporter-1.8.0.linux-amd64/node_exporter /usr/local/bin/

cat > /etc/systemd/system/node_exporter.service << 'EOF'
[Unit]
Description=Node Exporter
After=network.target

[Service]
User=prometheus
ExecStart=/usr/local/bin/node_exporter \
  --collector.systemd \
  --collector.processes \
  --collector.tcpstat \
  --collector.filesystem.mount-points-exclude='^/(dev|proc|sys|run|var/lib/docker/.+|var/lib/kubelet/.+)' \
  --web.listen-address=:9100

[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload && systemctl enable --now node_exporter

# Ubuntu/Debian
apt-get install -y software-properties-common
add-apt-repository "deb https://packages.grafana.com/oss/deb stable main"
wget -q -O - https://packages.grafana.com/gpg.key | apt-key add -
apt-get update && apt-get install -y grafana

# CentOS/RHEL 7/8
cat > /etc/yum.repos.d/grafana.repo << 'EOF'
[grafana]
name=grafana
baseurl=https://packages.grafana.com/oss/rpm
repo_gpgcheck=1
enabled=1
gpgcheck=1
gpgkey=https://packages.grafana.com/gpg.key
sslverify=1
sslcacert=/etc/pki/tls/certs/ca-bundle.crt
EOF
yum install -y grafana

systemctl enable --now grafana-server

# 重置管理员密码
grafana-cli admin reset-admin-password newpassword

# 安装常用插件
grafana-cli plugins install grafana-piechart-panel
grafana-cli plugins install grafana-clock-panel
grafana-cli plugins install vonage-status-panel
systemctl restart grafana-server

# /opt/prometheus/rules/node_alerts.yml
groups:
  - name: node_alerts
    interval: 30s
    rules:
      # 主机宕机
      - alert: NodeDown
        expr: up == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "主机 {{ $labels.instance }} 宕机"
          description: "主机 {{ $labels.instance }} 已超过 2 分钟不可达"

      # CPU 使用率过高
      - alert: HighCPUUsage
        expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "CPU 使用率 > 90%: {{ $labels.instance }}"
          description: "当前值: {{ $value | humanize }}%"

      # 内存使用率
      - alert: HighMemoryUsage
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 90
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "内存使用率 > 90%: {{ $labels.instance }}"

      # 磁盘使用率
      - alert: HighDiskUsage
        expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "磁盘使用率 > 85%: {{ $labels.instance }} /"

      # 磁盘预计填满
      - alert: DiskWillFillIn4Hours
        expr: predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[1h], 4*3600) < 0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "磁盘预计 4 小时内填满: {{ $labels.instance }}"

      # 系统负载过高
      - alert: HighSystemLoad
        expr: node_load15 / count without(cpu, mode)(node_cpu_seconds_total{mode="idle"}) > 1.5
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "负载过高 load15/cores > 1.5: {{ $labels.instance }}"

      # 磁盘 IO 饱和
      - alert: DiskIOSaturation
        expr: rate(node_disk_io_time_seconds_total{device=~"sd[a-z]+"}[5m]) > 0.9
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "磁盘 IO 饱和: {{ $labels.instance }} {{ $labels.device }}"

      # 网络错误
      - alert: NetworkErrors
        expr: rate(node_network_receive_errors_total[5m]) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "网络接口错误: {{ $labels.instance }} {{ $labels.device }}"

      # 内存即将耗尽
      - alert: OutOfMemorySoon
        expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "内存即将耗尽 (< 5%): {{ $labels.instance }}"

      # inode 使用率
      - alert: HighInodeUsage
        expr: (1 - node_filesystem_files_free{mountpoint="/"} / node_filesystem_files{mountpoint="/"}) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "inode 使用率 > 85%: {{ $labels.instance }}"

# /opt/alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m
  # 邮件配置
  smtp_smarthost: 'smtp.example.com:587'
  smtp_from: '[email protected]'
  smtp_auth_username: '[email protected]'
  smtp_auth_password: 'password'
  smtp_require_tls: true

# 告警路由
route:
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default'
  routes:
    - match:
        severity: critical
      receiver: 'critical'
      continue: true
    - match:
        severity: warning
      receiver: 'warning'

# 接收器
receivers:
  - name: 'default'
    email_configs:
      - to: '[email protected]'

  - name: 'critical'
    email_configs:
      - to: '[email protected]'
    webhook_configs:
      # 钉钉
      - url: 'https://oapi.dingtalk.com/robot/send?access_token=xxx'
        send_resolved: true
    # 企业微信
    wechat_configs:
      - corp_id: 'wwxxx'
        to_party: '1'
        agent_id: '1000001'
        api_secret: 'xxx'
        send_resolved: true

  - name: 'warning'
    email_configs:
      - to: '[email protected]'

# 抑制规则 (避免告警风暴)
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['instance']

# === CPU ===
# CPU 使用率 (%)
100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)

# 各 CPU 模式占比
avg by(cpu,mode)(rate(node_cpu_seconds_total[5m])) * 100

# === 内存 ===
# 可用内存百分比
(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# 内存使用率
(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100

# === 磁盘 ===
# 磁盘使用率
(1 - node_filesystem_avail_bytes / node_filesystem_size_bytes) * 100

# 磁盘读写速率 MB/s
rate(node_disk_read_bytes_total[5m]) / 1024 / 1024
rate(node_disk_written_bytes_total[5m]) / 1024 / 1024

# 磁盘 IOPS
rate(node_disk_reads_completed_total[5m])
rate(node_disk_writes_completed_total[5m])

# disk_io_time (IO 繁忙度)
rate(node_disk_io_time_seconds_total[5m]) * 100

# === 网络 ===
# 网络流量 bytes/sec
rate(node_network_receive_bytes_total[5m])
rate(node_network_transmit_bytes_total[5m])

# 网络连接状态
node_netstat_Tcp_CurrEstab

# TCP 重传率
rate(node_netstat_Tcp_RetransSegs[5m]) / rate(node_netstat_Tcp_OutSegs[5m]) * 100

# === 进程 ===
# 打开文件描述符数量
process_open_fds

# === 系统 ===
# 运行时间 (秒)
node_boot_time_seconds

# 预测磁盘空间
predict_linear(node_filesystem_avail_bytes[1h], 24*3600) < 0

# grafana.ini 配置
[smtp]
enabled = true
host = smtp.example.com:587
user = [email protected]
password = password
from_address = [email protected]

[alerting]
enabled = true
execute_alerts = true

# 一键安装
bash <(curl -Ss https://my-netdata.io/kickstart.sh)

# 仅本机访问
sed -i 's/bind to = \*/bind to = 127.0.0.1/g' /etc/netdata/netdata.conf
systemctl restart netdata

# 访问 http://localhost:19999
# 特点: 零配置、极低资源占用、1秒粒度、数千指标自动采集

#!/bin/bash
# /opt/scripts/monitor.sh
# crontab: */5 * * * * /opt/scripts/monitor.sh

HOSTNAME=$(hostname)
ALERT_WEBHOOK="https://hooks.slack.com/services/xxx"

# CPU
CPU_USAGE=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1)
if (( $(echo "$CPU_USAGE > 90" | bc -l) )); then
    curl -X POST -H 'Content-type: application/json' \
        --data "{\"text\":\"[${HOSTNAME}] CPU 使用率: ${CPU_USAGE}%\"}" \
        $ALERT_WEBHOOK
fi

# 内存
MEM_AVAIL=$(free -m | awk 'NR==2{print $7}')
if [ "$MEM_AVAIL" -lt 512 ]; then
    curl -X POST -H 'Content-type: application/json' \
        --data "{\"text\":\"[${HOSTNAME}] 可用内存不足: ${MEM_AVAIL}MB\"}" \
        $ALERT_WEBHOOK
fi

# 磁盘
DISK_USAGE=$(df / | awk 'NR==2{print $5}' | sed 's/%//')
if [ "$DISK_USAGE" -gt 85 ]; then
    curl -X POST -H 'Content-type: application/json' \
        --data "{\"text\":\"[${HOSTNAME}] 磁盘使用率: ${DISK_USAGE}%\"}" \
        $ALERT_WEBHOOK
fi

# 关键进程检查
for proc in nginx mysqld sshd; do
    if ! pgrep -x "$proc" > /dev/null; then
        curl -X POST -H 'Content-type: application/json' \
            --data "{\"text\":\"[${HOSTNAME}] 进程 $proc 未运行!\"}" \
            $ALERT_WEBHOOK
    fi
done

# /etc/rsyslog.conf

# === 模块加载 ===
module(load="imuxsock")     # 本地 socket
module(load="imklog")       # 内核日志
module(load="imtcp")        # TCP 接收
module(load="imudp")        # UDP 接收
module(load="impstats"
       interval="300"
       severity="7"
       log.file="/var/log/rsyslog-stats.log"
       Ruleset="stats")      # 性能统计

# === 全局配置 ===
global(
    workDirectory="/var/lib/rsyslog"
    maxMessageSize="64k"
    defaultTemplate="RSYSLOG_TraditionalFileFormat"
    privDropToUser="syslog"
    privDropToGroup="syslog"
)

# === 日志格式模板 ===
template(name="RemoteLogs" type="string"
    string="%TIMESTAMP% %HOSTNAME% %syslogtag% %msg%\n")

template(name="JsonFormat" type="list") {
    constant(value="{")
    constant(value="\"timestamp\":\"")      property(name="timereported" dateFormat="rfc3339")
    constant(value="\",\"host\":\"")        property(name="hostname")
    constant(value="\",\"severity\":\"")    property(name="syslogseverity")
    constant(value="\",\"facility\":\"")    property(name="syslogfacility")
    constant(value="\",\"tag\":\"")         property(name="syslogtag" format="json")
    constant(value="\",\"message\":\"")     property(name="msg" format="json")
    constant(value="\"}\n")
}

# === 日志规则 ===
# 认证日志
auth,authpriv.*                     /var/log/secure

# 系统日志
*.info;mail.none;authpriv.none     /var/log/messages

# Cron 日志
cron.*                             /var/log/cron

# 内核日志
kern.*                             /var/log/kern.log

# 邮件日志
mail.*                             /var/log/maillog

# Debug 日志 (丢弃)
*.debug                            stop

# 紧急日志发送给所有登录用户
*.emerg                            :omusrmsg:*

# === 转发到远程 ===
# TCP 转发
*.* @@192.168.1.100:514

# UDP 转发
# *.* @192.168.1.100:514

# 条件转发 (仅错误级别以上)
*.err @@192.168.1.100:514

# === 作为日志服务器接收 ===
input(type="imtcp" port="514" Ruleset="remote")
input(type="imudp" port="514" Ruleset="remote")

ruleset(name="remote") {
    # 按主机名分文件
    $template RemotePath,"/data/logs/%HOSTNAME%/%$YEAR%-%$MONTH%-%$DAY%.log"
    action(type="omfile" dynaFile="RemotePath")
    # 也可输出为 JSON
    # action(type="omfile" dynaFile="RemotePath" template="JsonFormat")
}

# Nginx rsyslog 配置
# /etc/rsyslog.d/nginx.conf
$ModLoad imfile
$InputFileName /var/log/nginx/access.log
$InputFileTag nginx-access:
$InputFileStateFile stat-nginx-access
$InputFileSeverity info
$InputFileFacility local7
$InputRunFileMonitor

$InputFileName /var/log/nginx/error.log
$InputFileTag nginx-error:
$InputFileStateFile stat-nginx-error
$InputFileSeverity error
$InputFileFacility local7
$InputRunFileMonitor

local7.* @@192.168.1.100:514

# === 查看日志 ===
journalctl                           # 所有日志
journalctl -n 100                    # 最近 100 行
journalctl -f                        # tail -f 模式
journalctl -k                        # 内核日志
journalctl -u nginx                  # 指定服务
journalctl -u nginx --since today    # 今天的日志
journalctl -u nginx --since "2024-01-01" --until "2024-01-02"
journalctl -p err                     # 仅错误级别以上
journalctl -p emerg..err              # emerg 到 err
journalctl _PID=1234                  # 按 PID
journalctl _UID=0                     # 按 UID (root)
journalctl -o json-pretty             # JSON 输出
journalctl --disk-usage               # 日志占用空间
journalctl -u sshd | grep "Failed"   # 配合 grep

# === journald 配置 ===
# /etc/systemd/journald.conf
[Journal]
Storage=persistent        # 持久化到磁盘
Compress=yes             # 压缩
Seal=yes                 # 防篡改密封
SystemMaxUse=4G          # 最多使用 4G
SystemMaxFileSize=100M   # 单文件最大
MaxRetentionSec=2week    # 最多保留 2 周
RuntimeMaxUse=1G         # /run 下最大使用
ForwardToSyslog=no       # 是否转发到 syslog
ForwardToConsole=no

systemctl restart systemd-journald

# /etc/logrotate.conf (全局)
weekly
rotate 12
create
dateext
compress
include /etc/logrotate.d

# /etc/logrotate.d/nginx (应用级)
/var/log/nginx/*.log {
    daily                       # 每天轮转
    missingok                   # 日志不存在不报错
    rotate 30                   # 保留 30 天
    compress                    # 压缩旧日志
    delaycompress               # 延迟一个周期压缩
    notifempty                  # 空文件不轮转
    create 640 nginx adm        # 创建新文件权限
    sharedscripts               # 轮转完后执行一次脚本
    postrotate
        [ -f /var/run/nginx.pid ] && kill -USR1 $(cat /var/run/nginx.pid)
    endscript
    dateext                     # 日期后缀
    dateformat -%Y%m%d
    maxsize 500M                # 超过 500M 强制轮转
}

# /etc/logrotate.d/syslog
/var/log/cron
/var/log/maillog
/var/log/messages
/var/log/secure
/var/log/spooler
{
    weekly
    rotate 12
    compress
    dateext
    missingok
    sharedscripts
    postrotate
        /bin/kill -HUP $(cat /var/run/syslogd.pid 2>/dev/null) 2>/dev/null || true
    endscript
}

# 手动执行轮转
logrotate -f /etc/logrotate.conf
logrotate -d /etc/logrotate.d/nginx   # 调试模式 (不实际轮转)

# Crontab 中每日执行
# 0 0 * * * /usr/sbin/logrotate /etc/logrotate.conf

# /etc/filebeat/filebeat.yml
filebeat.inputs:
  - type: log
    enabled: true
    paths:
      - /var/log/nginx/access.log
    fields:
      app: nginx
      type: access
    fields_under_root: true
    json.keys_under_root: true
    json.add_error_key: true

  - type: log
    enabled: true
    paths:
      - /var/log/nginx/error.log
    fields:
      app: nginx
      type: error
    fields_under_root: true
    multiline.pattern: '^\d{4}/\d{2}/\d{2}'
    multiline.negate: true
    multiline.match: after

  - type: log
    enabled: true
    paths:
      - /var/log/messages
    fields:
      app: system
      type: syslog

# === 输出到 Logstash ===
output.logstash:
  hosts: ["192.168.1.101:5044"]
  loadbalance: true
  compression_level: 3

# === 或直接输出到 Elasticsearch ===
# output.elasticsearch:
#   hosts: ["192.168.1.101:9200", "192.168.1.102:9200"]
#   index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}"

# === 处理器 ===
processors:
  - add_host_metadata: ~
  - add_cloud_metadata: ~
  - drop_fields:
      fields: ["agent.ephemeral_id", "agent.id"]

# === 日志本身 ===
logging.level: info
logging.to_files: true
logging.files:
  path: /var/log/filebeat
  name: filebeat.log
  keepfiles: 7

# /etc/logstash/conf.d/pipeline.conf
input {
  beats {
    port => 5044
    client_inactivity_timeout => 3600
  }
}

filter {
  if [type] == "access" {
    grok {
      match => {
        "message" => '%{IPORHOST:client_ip} - %{DATA:remote_user} \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{DATA:http_referer}" "%{DATA:http_user_agent}"'
      }
    }
    date {
      match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
      target => "@timestamp"
    }
    geoip {
      source => "client_ip"
    }
    useragent {
      source => "http_user_agent"
      target => "user_agent"
    }
  }

  if [type] == "syslog" {
    grok {
      match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" }
    }
  }
}

output {
  elasticsearch {
    hosts => ["http://192.168.1.101:9200", "http://192.168.1.102:9200"]
    index => "logstash-%{[app]}-%{+YYYY.MM.dd}"
    manage_template => false
  }
  # 如果 ES 不可用，存磁盘队列
  # dead_letter_queue {
  #   path => "/data/logstash/dead_letter_queue"
  #   max_bytes => "1gb"
  # }
}

# promtail (日志采集器) 配置
# /etc/promtail/config.yml
server:
  http_listen_port: 9080

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
  - job_name: system
    static_configs:
      - targets: [localhost]
        labels:
          job: varlogs
          host: ${HOSTNAME}
          __path__: /var/log/*.log

  - job_name: nginx
    static_configs:
      - targets: [localhost]
        labels:
          job: nginx
          host: ${HOSTNAME}
          __path__: /var/log/nginx/*.log
    pipeline_stages:
      - match:
          selector: '{job="nginx"} |= "/var/log/nginx/error.log"'
          stages:
            - regex:
                expression: '^(?P<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?P<level>\w+)\]'
            - labels:
                level:

# loki 配置 (单机)
# /etc/loki/loki-config.yaml
auth_enabled: false

server:
  http_listen_port: 3100

ingester:
  lifecycler:
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
  chunk_idle_period: 30m
  max_chunk_age: 1h
  chunk_target_size: 1536000
  chunk_retain_period: 30s

schema_config:
  configs:
    - from: 2024-01-01
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

storage_config:
  boltdb_shipper:
    active_index_directory: /data/loki/index
    cache_location: /data/loki/cache
  filesystem:
    directory: /data/loki/chunks

limits_config:
  enforce_metric_name: false
  max_entries_limit_per_query: 5000
  retention_period: 30d

compactor:
  working_directory: /data/loki/compactor
  shared_store: filesystem

# === 基础用法 ===
# 本地同步
rsync -avz --delete /data/ /backup/

# 远程同步 (SSH)
rsync -avz -e "ssh -p 22" /data/ [email protected]:/backup/

# 远程拉取
rsync -avz [email protected]:/data/ /backup/

# === 生产级备份脚本 ===
#!/bin/bash
# /opt/scripts/backup.sh

BACKUP_SRC="/data/app"
BACKUP_DST="/backup"
REMOTE_HOST="[email protected]"
REMOTE_PATH="/backup/$(hostname)"
LOG_FILE="/var/log/backup.log"
EXCLUDE_FILE="/opt/scripts/backup_exclude.txt"
LOCK_FILE="/var/run/backup.lock"

log() {
    echo "[$(date '+%F %T')] $*" | tee -a "$LOG_FILE"
}

# 防止并发执行
exec 200>"$LOCK_FILE"
flock -n 200 || { log "备份已在运行，退出"; exit 1; }

log "=== 开始备份 ==="

# 1. 本地每日快照 (保留 7 天)
rsync -avz \
    --delete \
    --exclude-from="$EXCLUDE_FILE" \
    --link-dest="$BACKUP_DST/latest" \
    "$BACKUP_SRC/" \
    "$BACKUP_DST/$(date +%Y%m%d)/"

# 2. 更新 latest 符号链接
rm -f "$BACKUP_DST/latest"
ln -s "$BACKUP_DST/$(date +%Y%m%d)" "$BACKUP_DST/latest"

# 3. 远程同步
rsync -avz --delete \
    -e "ssh -p 22 -i /root/.ssh/backup_key" \
    "$BACKUP_DST/" "$REMOTE_HOST:$REMOTE_PATH/"

# 4. 清理旧备份 (超过 30 天的远程备份)
ssh -i /root/.ssh/backup_key "$REMOTE_HOST" \
    "find $REMOTE_PATH -maxdepth 1 -type d -mtime +30 -exec rm -rf {} +"

log "=== 备份完成 ==="

# === 安装 ===
wget https://github.com/restic/restic/releases/download/v0.16.4/restic_0.16.4_linux_amd64.bz2
bunzip2 restic_0.16.4_linux_amd64.bz2
mv restic_0.16.4_linux_amd64 /usr/local/bin/restic
chmod +x /usr/local/bin/restic

# === 初始化仓库 ===
export RESTIC_REPOSITORY=/backup/restic
export RESTIC_PASSWORD=your_strong_password
restic init

# === 远程仓库 ===
export RESTIC_REPOSITORY=sftp:backup@storage:/backup/restic
export RESTIC_PASSWORD=your_strong_password
restic init

# S3 兼容 (MinIO / AWS S3)
export AWS_ACCESS_KEY_ID=xxx
export AWS_SECRET_ACCESS_KEY=yyy
export RESTIC_REPOSITORY=s3:s3.amazonaws.com/bucket-name/restic
restic init

# === 备份 ===
restic backup /data /etc /var/log --exclude "*.tmp" --exclude "*.log"

# === 快照管理 ===
restic snapshots                          # 列出快照
restic diff <snapshot1> <snapshot2>       # 比较快照差异
restic stats                              # 仓库统计

# === 恢复 ===
restic restore latest --target /restore/path/    # 恢复最新
restic restore <snapshot_id> --target /restore/  # 恢复指定快照
restic restore latest --target /restore/ --include "/data/app/*"

# === 清理 ===
restic forget --keep-daily 7 --keep-weekly 4 --keep-monthly 12 --keep-yearly 2
restic prune     # 删除未引用的数据块
restic check     # 验证仓库完整性

# === 自动备份脚本 ===
#!/bin/bash
export RESTIC_REPOSITORY=/backup/restic
export RESTIC_PASSWORD_FILE=/root/.restic-password
BACKUP_SRC="/data /etc /var/log"

restic backup $BACKUP_SRC \
    --exclude "*.tmp" \
    --exclude "*.bak" \
    --tag "$(date +%Y%m%d)" \
    --host "$(hostname)"

restic forget \
    --keep-daily 7 \
    --keep-weekly 4 \
    --keep-monthly 12 \
    --keep-yearly 2 \
    --prune

restic check --read-data-subset=2%

#!/bin/bash
# MySQL 全量 + binlog 备份

DB_USER="backup"
DB_PASS="password"
BACKUP_DIR="/backup/mysql"
RETENTION_DAYS=7

# 全量备份 (mysqldump)
mysqldump -u$DB_USER -p$DB_PASS --all-databases \
    --single-transaction \
    --routines --triggers --events \
    --master-data=2 \
    --set-gtid-purged=OFF \
    | gzip > "$BACKUP_DIR/full_$(date +%Y%m%d_%H%M).sql.gz"

# 或使用 xtrabackup (物理备份，大库推荐)
xtrabackup --backup \
    --user=$DB_USER --password=$DB_PASS \
    --target-dir="$BACKUP_DIR/xtra_$(date +%Y%m%d_%H%M)" \
    --compress --compress-threads=4

# 清理旧备份
find "$BACKUP_DIR" -mtime +$RETENTION_DAYS -delete

#!/bin/bash
# PostgreSQL 备份

BACKUP_DIR="/backup/postgres"
export PGPASSWORD="password"

# 逻辑备份
pg_dumpall -U postgres -h localhost | gzip > "$BACKUP_DIR/pg_all_$(date +%Y%m%d).sql.gz"

# 单库自定义格式 (支持并行恢复)
pg_dump -U postgres -h localhost -Fc -j 4 mydb > "$BACKUP_DIR/mydb_$(date +%Y%m%d).dump"

# WAL 归档配置 (postgresql.conf)
# wal_level = replica
# archive_mode = on
# archive_command = 'test ! -f /backup/pg_wal/%f && cp %p /backup/pg_wal/%f'

# === 分区镜像备份 (dd) ===
dd if=/dev/sda1 of=/backup/sda1_$(date +%Y%m%d).img bs=4M status=progress

# 压缩
dd if=/dev/sda1 bs=4M | gzip > /backup/sda1.img.gz

# 恢复
gunzip -c /backup/sda1.img.gz | dd of=/dev/sda1 bs=4M status=progress

# === tar 系统备份 ===
tar -cvpzf /backup/system_$(date +%Y%m%d).tar.gz \
    --exclude=/proc \
    --exclude=/tmp \
    --exclude=/sys \
    --exclude=/dev \
    --exclude=/run \
    --exclude=/mnt \
    --exclude=/media \
    --exclude=/backup \
    --exclude=/lost+found \
    /

# 恢复
tar -xvpzf /backup/system_20240101.tar.gz -C /

#!/bin/bash
# DR 演练检查清单脚本

echo "=== 灾难恢复检查清单 ==="
echo "日期: $(date)"

# 1. 验证备份完整性
echo -e "\n[1/6] 验证备份完整性..."
BACKUP_DIR="/backup"
LATEST=$(ls -t "$BACKUP_DIR" | head -1)
if [ -n "$LATEST" ]; then
    echo "  ✓ 最新备份: $LATEST"
else
    echo "  ✗ 未找到备份!"
fi

# 2. 验证备份可恢复性 (抽检)
echo -e "\n[2/6] 验证备份可恢复性..."
if restic check --read-data-subset=1%; then
    echo "  ✓ restic 仓库完整性验证通过"
fi

# 3. 验证数据库备份
echo -e "\n[3/6] 验证数据库备份..."
LATEST_SQL=$(ls -t /backup/mysql/*.sql.gz 2>/dev/null | head -1)
if gzip -t "$LATEST_SQL" 2>/dev/null; then
    echo "  ✓ MySQL 备份文件完整性验证通过"
fi

# 4. 验证远程同步
echo -e "\n[4/6] 验证远程备份..."
if rsync -azn --delete /backup/ [email protected]:/backup/; then
    echo "  ✓ 远程连接正常"
fi

# 5. 验证恢复文档
echo -e "\n[5/6] 验证恢复文档..."
if [ -f /opt/docs/recovery_procedure.md ]; then
    echo "  ✓ 恢复文档存在"
else
    echo "  ✗ 缺少恢复文档!"
fi

# 6. 验证恢复时间
echo -e "\n[6/6] 恢复时间估算..."
echo "  上次全量恢复耗时: ~30分钟 (记录于 2024-01-01)"
echo "  RTO 目标: 2小时"
echo "  RPO 目标: < 1小时 (binlog 实时同步)"

echo -e "\n=== 检查完成 ==="

# 安装
yum install -y ansible        # CentOS/RHEL
apt-get install -y ansible    # Ubuntu/Debian
pip install ansible            # pip (最新版)

# 验证
ansible --version

# === ansible.cfg 配置 ===
# /etc/ansible/ansible.cfg 或 ./ansible.cfg
[defaults]
inventory      = ./hosts
host_key_checking = False
remote_user    = root
private_key_file = /root/.ssh/id_rsa
forks          = 20
timeout        = 30
log_path       = /var/log/ansible.log
gathering      = smart
fact_caching   = jsonfile
fact_caching_connection = /tmp/ansible_cache
fact_caching_timeout = 3600
retry_files_enabled = False
callback_whitelist = timer, profile_tasks
stdout_callback = yaml

[privilege_escalation]
become         = True
become_method  = sudo
become_user    = root

[ssh_connection]
pipelining     = True
control_path   = /tmp/ansible-%%h-%%p-%%r

# hosts (静态清单)
[webservers]
web01 ansible_host=192.168.1.11
web02 ansible_host=192.168.1.12
web03 ansible_host=192.168.1.13

[dbservers]
db01 ansible_host=192.168.1.21 ansible_user=dbadmin
db02 ansible_host=192.168.1.22

[appservers]
app[01:05].example.com             # 范围: app01 ~ app05

[production:children]              # 分组嵌套
webservers
dbservers

[production:vars]                  # 组变量
ansible_user=root
ntp_server=ntp.prod.example.com

[all:vars]                         # 全局变量
ansible_port=22

# hosts.yml (YAML 清单)
all:
  hosts:
    bastion:
      ansible_host: 1.2.3.4
  children:
    production:
      hosts:
        web[01:03].example.com:
      vars:
        env: production
    staging:
      hosts:
        web-stg.example.com:
      vars:
        env: staging

# 基本语法: ansible <pattern> -m <module> -a "<arguments>"

# === 信息收集 ===
ansible all -m ping                                       # 存活检测
ansible all -m setup                                      # 收集 facts
ansible all -m setup -a "filter=ansible_memory_mb"        # 过滤 facts
ansible all -m shell -a "hostname; uptime"                # 执行 shell

# === 文件操作 ===
ansible all -m copy -a "src=/tmp/file dest=/tmp/file"     # 拷贝文件
ansible all -m fetch -a "src=/etc/hosts dest=/tmp/"       # 拉取文件
ansible all -m file -a "path=/data state=directory mode=0755"  # 创建目录
ansible all -m replace -a "path=/etc/nginx/nginx.conf regexp='worker_processes.*' replace='worker_processes 8;'"  # 文件内容替换

# === 包管理 ===
ansible all -m yum -a "name=nginx state=latest"           # 安装 (RHEL)
ansible all -m apt -a "name=nginx state=latest update_cache=yes"  # 安装 (Debian)

# === 服务管理 ===
ansible all -m systemd -a "name=nginx state=restarted"    # 重启服务
ansible all -m systemd -a "name=nginx enabled=yes"        # 开机启动

# === 用户管理 ===
ansible all -m user -a "name=app password={{ 'mypass' | password_hash('sha512') }} groups=wheel state=present"

# === 内核参数 ===
ansible all -m sysctl -a "name=net.ipv4.tcp_tw_reuse value=1 sysctl_set=yes reload=yes"

# === 计划任务 ===
ansible all -m cron -a "name='log cleanup' hour=2 job='/opt/scripts/cleanup.sh'"

# === 防火墙 ===
ansible all -m firewalld -a "port=80/tcp permanent=yes state=enabled immediate=yes"

# deploy_webapp.yml
---
- name: 部署 Web 应用
  hosts: webservers
  become: yes
  vars:
    app_name: myapp
    app_port: 8080
    app_version: "1.2.3"
    nginx_worker_processes: "{{ ansible_processor_vcpus }}"

  vars_files:
    - vars/secrets.yml   # 加密的敏感变量

  pre_tasks:
    - name: 更新 yum 缓存
      yum:
        update_cache: yes
        name: '*'
        state: latest
      when: ansible_os_family == "RedHat"
      tags: [update]

    - name: 检查磁盘空间
      shell: df -h /data | awk 'NR==2{print $5}' | sed 's/%//'
      register: disk_usage
      failed_when: disk_usage.stdout|int > 90

  tasks:
    - name: 安装基础包
      package:
        name: "{{ item }}"
        state: present
      loop:
        - nginx
        - supervisor
        - python3
      tags: [packages]

    - name: 创建应用用户
      user:
        name: "{{ app_name }}"
        system: yes
        shell: /sbin/nologin
        create_home: no
      tags: [user]

    - name: 创建目录结构
      file:
        path: "{{ item }}"
        state: directory
        owner: "{{ app_name }}"
        mode: '0755'
      loop:
        - /opt/{{ app_name }}
        - /opt/{{ app_name }}/config
        - /data/{{ app_name }}
        - /var/log/{{ app_name }}
      tags: [dirs]

    - name: 部署应用文件
      copy:
        src: files/{{ app_name }}-{{ app_version }}.jar
        dest: /opt/{{ app_name }}/{{ app_name }}.jar
        owner: "{{ app_name }}"
        mode: '0644'
      notify: restart app          # 触发 handler
      tags: [deploy]

    - name: 配置 Nginx 反向代理
      template:
        src: templates/nginx.conf.j2
        dest: /etc/nginx/conf.d/{{ app_name }}.conf
        validate: nginx -t -c %s
      notify: reload nginx
      tags: [nginx]

    - name: 配置 systemd 服务
      template:
        src: templates/app.service.j2
        dest: /etc/systemd/system/{{ app_name }}.service
      notify: restart app
      tags: [service]

    - name: 启动服务
      systemd:
        name: "{{ app_name }}"
        state: started
        enabled: yes
      tags: [service]

  handlers:
    - name: reload nginx
      systemd:
        name: nginx
        state: reloaded

    - name: restart app
      systemd:
        name: "{{ app_name }}"
        state: restarted

  post_tasks:
    - name: 健康检查
      uri:
        url: "http://localhost:{{ app_port }}/health"
        status_code: 200
      retries: 10
      delay: 3
      until: result.status == 200
      register: result
      tags: [verify]

    - name: 发送部署通知
      slack:
        token: "{{ slack_token }}"
        msg: "{{ app_name }} v{{ app_version }} 部署完成 [{{ ansible_hostname }}]"
      tags: [notify]

{# templates/nginx.conf.j2 #}
upstream {{ app_name }}_backend {
{% for host in groups['appservers'] %}
    server {{ hostvars[host]['ansible_host'] }}:{{ app_port }} weight=1 max_fails=3 fail_timeout=30s;
{% endfor %}
    keepalive 32;
}

server {
    listen 80;
    server_name {{ app_name }}.example.com;

    access_log /var/log/nginx/{{ app_name }}_access.log;
    error_log  /var/log/nginx/{{ app_name }}_error.log;

    location / {
        proxy_pass http://{{ app_name }}_backend;
        proxy_http_version 1.1;
        proxy_set_header Connection "";
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        # 基于环境控制超时
        {% if env == 'production' %}
        proxy_read_timeout 60s;
        {% else %}
        proxy_read_timeout 300s;
        {% endif %}
    }

    # 仅生产环境启用 SSL
    {% if env == 'production' %}
    listen 443 ssl;
    ssl_certificate     /etc/ssl/certs/{{ app_name }}.crt;
    ssl_certificate_key /etc/ssl/private/{{ app_name }}.key;
    if ($scheme != "https") {
        return 301 https://$host$request_uri;
    }
    {% endif %}
}

# 创建加密文件
ansible-vault create vars/secrets.yml

# 编辑加密文件
ansible-vault edit vars/secrets.yml

# 加密已有文件
ansible-vault encrypt vars/secrets.yml

# 使用密码文件 (生产环境)
echo "my_vault_password" > ~/.vault_pass
ansible-vault encrypt vars/secrets.yml --vault-password-file ~/.vault_pass

# 运行 playbook 时解密
ansible-playbook deploy.yml --vault-password-file ~/.vault_pass
ansible-playbook deploy.yml --ask-vault-pass   # 交互输入

# 多环境密码 (vault-id)
ansible-vault encrypt --vault-id prod@prompt vars/prod/secrets.yml
ansible-playbook deploy.yml --vault-id prod@prompt

# playbook 中使用 roles
---
- hosts: webservers
  roles:
    - role: nginx
      nginx_port: 8080          # 覆盖默认变量
      tags: [nginx]
    - role: app
      tags: [app]

# === 1. SSH 加固 ===
# /etc/ssh/sshd_config
Port 2222                              # 修改默认端口
Protocol 2                             # 仅 SSHv2
PermitRootLogin no                     # 禁止 root 登录
PasswordAuthentication no              # 禁用密码认证
PubkeyAuthentication yes               # 仅密钥认证
MaxAuthTries 3                         # 最大尝试次数
ClientAliveInterval 300                # 客户端心跳
ClientAliveCountMax 2                  # 推送周期
AllowUsers [email protected].*            # 限制用户和来源 IP
X11Forwarding no                       # 禁止 X11 转发
MaxSessions 5                          # 单连接最大会话数
LoginGraceTime 30                      # 认证超时
MaxStartups 10:30:60                   # 未认证连接限制

systemctl restart sshd

# === 2. 密码策略 ===
# /etc/login.defs
PASS_MAX_DAYS   90     # 密码 90 天过期
PASS_MIN_DAYS   7      # 修改后 7 天内不可再改
PASS_MIN_LEN    12     # 最小 12 位
PASS_WARN_AGE   14     # 过期前 14 天警告

# /etc/security/pwquality.conf
minlen = 12
dcredit = -1            # 至少 1 个数字
ucredit = -1            # 至少 1 个大写
lcredit = -1            # 至少 1 个小写
ocredit = -1            # 至少 1 个特殊字符
minclass = 4            # 至少 4 种字符类别
maxrepeat = 3           # 最多连续重复 3 次
maxclassrepeat = 3      # 同类字符最多连续 3 个
difok = 5               # 新密码与旧密码至少不同 5 个字符
enforce_for_root        # root 也适用

# === 3. 账户锁定 ===
# /etc/pam.d/sshd
# 连续 5 次失败后锁定 600 秒
auth required pam_tally2.so deny=5 unlock_time=600 onerr=fail audit

# === 4. 会话超时 ===
echo "TMOUT=600" >> /etc/profile
echo "readonly TMOUT" >> /etc/profile
echo "export TMOUT" >> /etc/profile

# === 5. 历史命令限制 ===
echo 'HISTSIZE=500' >> /etc/profile
echo 'HISTFILESIZE=500' >> /etc/profile
echo "readonly HISTSIZE HISTFILESIZE" >> /etc/profile
echo 'export HISTTIMEFORMAT="%F %T "' >> /etc/profile

# === 6. 限制 su/sudo ===
# 仅 wheel 组可 su
# /etc/pam.d/su
auth required pam_wheel.so use_uid

# sudo 日志审计
# /etc/sudoers
Defaults logfile=/var/log/sudo.log
Defaults log_input,log_output     # 记录输入输出 (需 sudo 1.9+)
Defaults requiretty               # 必须有 tty

# === 基础操作 ===
systemctl start firewalld
systemctl enable firewalld
firewall-cmd --state

# 查看
firewall-cmd --list-all                    # 当前区域详情
firewall-cmd --get-default-zone            # 默认区域
firewall-cmd --get-active-zones            # 活动区域
firewall-cmd --list-services              # 已允许服务
firewall-cmd --list-ports                 # 已允许端口

# 规则管理
firewall-cmd --add-port=8080/tcp --permanent    # 永久开放端口
firewall-cmd --add-service=http --permanent      # 开放服务
firewall-cmd --add-rich-rule='rule family="ipv4" source address="192.168.1.0/24" port port="22" protocol="tcp" accept' --permanent  # 仅允许特定 IP 段访问 SSH
firewall-cmd --remove-port=8080/tcp --permanent  # 删除

# 重载
firewall-cmd --reload

# 区域切换
firewall-cmd --set-default-zone=dmz
firewall-cmd --change-interface=ens33 --zone=trusted --permanent

# === 生产级规则示例 ===
# 默认拒绝并只开放必要端口
firewall-cmd --set-default-zone=drop

firewall-cmd --permanent --add-rich-rule='rule family="ipv4" source address="10.0.0.0/8" port port="22" protocol="tcp" accept'
firewall-cmd --permanent --add-service=http
firewall-cmd --permanent --add-service=https
firewall-cmd --permanent --add-port=9100/tcp    # node_exporter
firewall-cmd --reload

# === 端口转发 (NAT) ===
firewall-cmd --permanent --add-masquerade
firewall-cmd --permanent --add-forward-port=port=80:proto=tcp:toport=8080:toaddr=192.168.1.100
firewall-cmd --reload

# === 默认策略 ===
iptables -P INPUT DROP
iptables -P FORWARD DROP
iptables -P OUTPUT ACCEPT

# === 允许回环 ===
iptables -A INPUT -i lo -j ACCEPT

# === 允许已建立连接 ===
iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT

# === 允许 SSH ===
iptables -A INPUT -p tcp --dport 22 -s 10.0.0.0/8 -j ACCEPT

# === 允许 Web ===
iptables -A INPUT -p tcp --dport 80 -j ACCEPT
iptables -A INPUT -p tcp --dport 443 -j ACCEPT

# === 允许 ICMP (ping) ===
iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT

# === 防 DDoS ===
# 限制 SYN 包速率
iptables -A INPUT -p tcp --syn -m limit --limit 10/s --limit-burst 20 -j ACCEPT
iptables -A INPUT -p tcp --syn -j DROP

# 限制单个 IP 并发连接
iptables -A INPUT -p tcp --dport 80 -m connlimit --connlimit-above 50 -j DROP

# === 端口转发 ===
iptables -t nat -A PREROUTING -p tcp --dport 80 -j DNAT --to-destination 192.168.1.100:8080
iptables -t nat -A POSTROUTING -j MASQUERADE

# === 保存规则 ===
iptables-save > /etc/sysconfig/iptables       # CentOS 6
iptables-save > /etc/iptables/rules.v4        # Debian/Ubuntu
netfilter-persistent save                      # iptables-persistent

# === 状态查看 ===
getenforce                          # 查看模式
sestatus -v                         # 详细状态

# === 模式切换 ===
setenforce 0                        # 临时切换为 Permissive
setenforce 1                        # 启用 Enforcing

# 永久配置
# /etc/selinux/config
# SELINUX=enforcing|permissive|disabled

# === 上下文管理 ===
ls -Z /var/www/html/                # 查看文件上下文
ps -Z                               # 查看进程上下文

chcon -R -t httpd_sys_content_t /var/www/html/    # 修改上下文
restorecon -Rv /var/www/html/                      # 恢复默认上下文

semanage fcontext -a -t httpd_sys_content_t "/web(/.*)?"
restorecon -Rv /web

# === 布尔值管理 ===
getsebool -a                        # 列出所有布尔值
setsebool -P httpd_can_network_connect on   # 允许 Apache 网络连接
setsebool -P httpd_enable_homedirs on       # 允许用户目录

# === 端口管理 ===
semanage port -l                          # 列出所有端口
semanage port -a -t http_port_t -p tcp 8080  # 添加端口到类型

# === 审计日志排错 ===
ausearch -m avc -ts recent           # 查看最近的 AVC 拒绝
sealert -a /var/log/audit/audit.log  # 分析审计日志
audit2allow -a -M mypol              # 从审计日志生成策略模块
semodule -i mypol.pp                 # 安装自定义模块

# === SELinux 排错流程 ===
# 1. 查看审计日志
ausearch -m avc -ts today | grep denied
# 2. 分析并提供建议
audit2why < /var/log/audit/audit.log
# 3. 临时切换 permissive 排查
setenforce 0
# 4. 测试应用
# 5. 查看生成的 AVC
ausearch -m avc -ts recent
# 6. 创建自定义策略
grep denied /var/log/audit/audit.log | audit2allow -M custom_policy
semodule -i custom_policy.pp
# 7. 恢复 enforcing
setenforce 1

# 安装
yum install -y fail2ban      # CentOS
apt-get install -y fail2ban  # Ubuntu

# /etc/fail2ban/jail.local
[DEFAULT]
ignoreip = 127.0.0.1/8 10.0.0.0/8 192.168.0.0/16
bantime  = 3600                    # 封禁时间 (秒)
findtime = 600                     # 统计窗口 (秒)
maxretry = 5                       # 最大失败次数
destemail = [email protected]
action = %(action_mw)s             # 封禁 + whois + 邮件
banaction = iptables-multiport

[sshd]
enabled  = true
port     = ssh,2222
logpath  = %(sshd_log)s
maxretry = 3

[nginx-http-auth]
enabled  = true
port     = http,https
logpath  = /var/log/nginx/error.log
maxretry = 5

[nginx-botsearch]
enabled  = true
port     = http,https
logpath  = /var/log/nginx/access.log
maxretry = 3
findtime = 300

[mysqld-auth]
enabled  = true
port     = 3306
logpath  = /var/log/mysql/error.log
maxretry = 5

# 管理命令
fail2ban-client status                       # 查看状态
fail2ban-client status sshd                  # 查看 sshd jail
fail2ban-client set sshd unbanip 1.2.3.4     # 手动解封
fail2ban-client set sshd banip 1.2.3.4       # 手动封禁
fail2ban-client reload                       # 重载配置

# 查看封禁日志
grep "Ban" /var/log/fail2ban.log

# 安装
yum install -y audit             # CentOS
apt-get install -y auditd        # Ubuntu

systemctl enable --now auditd

# === 审计规则 ===
# /etc/audit/rules.d/audit.rules

# 监控关键文件
-w /etc/passwd -p wa -k identity_changes
-w /etc/shadow -p wa -k identity_changes
-w /etc/group -p wa -k identity_changes
-w /etc/sudoers -p wa -k sudo_changes
-w /etc/ssh/sshd_config -p wa -k sshd_config
-w /etc/crontab -p wa -k cron_changes

# 监控关键命令执行
-a always,exit -F path=/usr/bin/su -F perm=x -k su_exec
-a always,exit -F path=/usr/bin/sudo -F perm=x -k sudo_exec

# 监控系统调用
-a always,exit -F arch=b64 -S execve -k command_exec

# 监控网络配置修改
-a always,exit -F path=/sbin/ifconfig -F perm=x -k net_config

# 监控时间修改
-a always,exit -F arch=b64 -S adjtimex -S settimeofday -k time_change
-a always,exit -F arch=b64 -S clock_settime -k time_change

# === 审计查询 ===
ausearch -k identity_changes            # 按 key 查询
ausearch -f /etc/passwd                 # 按文件查询
ausearch -p 1234                        # 按 PID 查询
ausearch -ua root                       # 按用户查询
ausearch -ts today                      # 今天的审计记录
ausearch -m USER_LOGIN                  # 登录事件

# 生成报告
aureport -l                             # 登录报告
aureport -k                             # key 汇总
aureport -f                             # 文件审计报告
aureport --summary                      # 摘要

# 搜索失败事件
ausearch -m USER_LOGIN --success no

# === Lynis (系统安全审计) ===
# 安装
git clone https://github.com/CISOfy/lynis
cd lynis && ./lynis audit system

# 快速审计
lynis audit system --quick

# === ClamAV (病毒扫描) ===
# 安装
yum install -y clamav clamav-update    # CentOS
apt-get install -y clamav              # Ubuntu

freshclam                     # 更新病毒库
clamscan -r /data             # 递归扫描
clamscan -r --remove /tmp     # 扫描并删除

# 定期扫描 crontab
# 0 2 * * 0 clamscan -r /data --log=/var/log/clamav/scan.log

# === AIDE (文件完整性检查) ===
yum install -y aide
aide --init                   # 初始化数据库
cp /var/lib/aide/aide.db.new.gz /var/lib/aide/aide.db.gz
aide --check                  # 检查变更
aide --update                 # 更新基线数据库

# === OpenSCAP (合规检查) ===
yum install -y openscap-scanner scap-security-guide

# 检查系统合规性 (CIS 基线)
oscap xccdf eval \
    --profile xccdf_org.ssgproject.content_profile_cis \
    --results scan-results.xml \
    --report scan-report.html \
    /usr/share/xml/scap/ssg/content/ssg-rhel8-ds.xml

# === 核心内核参数 (/etc/sysctl.d/99-tuning.conf) ===

# ===== 网络调优 =====
# TCP 连接复用 (快速回收 TIME_WAIT)
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 0          # 4.12+ 已移除，设为 0

# TIME_WAIT 与端口范围
net.ipv4.tcp_fin_timeout = 30
net.ipv4.tcp_max_tw_buckets = 10000
net.ipv4.ip_local_port_range = 1024 65000

# TCP 缓冲区 (高吞吐场景)
net.core.rmem_max = 134217728        # 128MB
net.core.wmem_max = 134217728
net.ipv4.tcp_rmem = 4096 87380 134217728
net.ipv4.tcp_wmem = 4096 65536 134217728
net.ipv4.tcp_mem = 50576 64768 98152

# TCP Fast Open
net.ipv4.tcp_fastopen = 3            # 客户端 + 服务端

# BBR 拥塞控制 (4.9+)
net.core.default_qdisc = fq
net.ipv4.tcp_congestion_control = bbr

# 连接队列
net.core.somaxconn = 65535
net.ipv4.tcp_max_syn_backlog = 8192
net.core.netdev_max_backlog = 10000

# Keepalive
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_intvl = 60
net.ipv4.tcp_keepalive_probes = 3

# SYN Flood 防护
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_synack_retries = 3
net.ipv4.tcp_syn_retries = 3

# ===== 文件系统与 IO =====
# VM 参数
vm.swappiness = 1                    # 尽量不用 swap (SSD 推荐 1)
vm.dirty_ratio = 30
vm.dirty_background_ratio = 10
vm.dirty_expire_centisecs = 3000
vm.dirty_writeback_centisecs = 500
vm.vfs_cache_pressure = 50          # 保留更多 inode/dentry 缓存
vm.min_free_kbytes = 131072          # 128MB 最小空闲内存

# 文件描述符
fs.file-max = 655350
fs.nr_open = 1048576
fs.inotify.max_user_watches = 524288
fs.inotify.max_user_instances = 1024
fs.aio-max-nr = 1048576

# ===== 内核调度 =====
kernel.pid_max = 4194303
kernel.threads-max = 256000
kernel.msgmax = 65536
kernel.msgmnb = 65536

# 应用配置
sysctl -p /etc/sysctl.d/99-tuning.conf

# /etc/security/limits.conf
# <domain> <type> <item> <value>
*       soft    nofile          65535
*       hard    nofile          65535
*       soft    nproc           65535
*       hard    nproc           65535
root    soft    nofile          65535
root    hard    nofile          65535
nginx   soft    nofile          100000
nginx   hard    nofile          100000
mysql   soft    nofile          100000
mysql   hard    nofile          100000

# systemd 服务资源限制
# /etc/systemd/system/myservice.service.d/limits.conf
[Service]
LimitNOFILE=65535
LimitNPROC=65535
LimitCORE=infinity
MemoryLimit=2G
CPUQuota=200%
TasksMax=2048

# === CPU 调度策略 ===
# 查看当前策略
cat /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor

# 设为 performance (服务器推荐)
# CentOS/RHEL
echo "GOVERNOR=performance" >> /etc/sysconfig/cpufreq

# 或直接
for CPU in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
    echo performance > $CPU
done

# === CPU 亲和性 (IRQ 绑定) ===
# 查看中断
cat /proc/interrupts

# 将网卡中断绑定到特定 CPU
echo 2 > /proc/irq/89/smp_affinity    # 绑定到 CPU1

# 使用 irqbalance (自动平衡，一般开启即可)
systemctl enable --now irqbalance

# NUMA 感知
numactl --hardware                            # 查看 NUMA 拓扑
numactl --cpunodebind=0 --membind=0 nginx     # 绑定到 NUMA node 0

# === CPU 隔离 (实时/低延迟场景) ===
# /etc/default/grub
# GRUB_CMDLINE_LINUX="isolcpus=2,3 nohz_full=2,3 rcu_nocbs=2,3"
# 然后: grub2-mkconfig -o /boot/grub2/grub.cfg

# === 查看内存状况 ===
free -h
cat /proc/meminfo
vmstat 1 10

# === 查看进程内存详情 ===
# PSS (比例分摊共享内存)
smem -r -s pss

# 大页内存 (HugePages)
# 适合大内存数据库
echo "vm.nr_hugepages = 1024" >> /etc/sysctl.d/99-hugepages.conf
sysctl -p /etc/sysctl.d/99-hugepages.conf

# 查看大页使用
cat /proc/meminfo | grep Huge

# transparent hugepage (数据库通常建议关闭)
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag

# === OOM Killer 控制 ===
# 保护关键进程
echo -1000 > /proc/$(pgrep sshd)/oom_score_adj   # 永不 kill (范围 -1000 ~ 1000)

# /etc/systemd/system/mysqld.service.d/oom.conf
[Service]
OOMScoreAdjust=-800

# === 内存泄漏排查 ===
# 监控进程内存增长
while true; do
    ps -eo pid,ppid,cmd,%mem,%cpu,rss --sort=-rss | head -20
    sleep 10
done

# === IO 调度器 ===
# 查看当前调度器
cat /sys/block/sda/queue/scheduler
# [mq-deadline] kyber bfq none

# 设置 (SSD 推荐 none / mq-deadline)
echo none > /sys/block/sda/queue/scheduler
echo mq-deadline > /sys/block/sda/queue/scheduler

# 永久设置 (grub)
# GRUB_CMDLINE_LINUX="elevator=noop"

# === 磁盘队列深度 ===
cat /sys/block/sda/queue/nr_requests
echo 1024 > /sys/block/sda/queue/nr_requests

# === 预读大小 ===
blockdev --getra /dev/sda
blockdev --setra 8192 /dev/sda   # 设为 4MB (8192 个扇区)

# === 文件系统挂载选项 ===
# SSD 优化
# /etc/fstab
UUID=xxx  /data  ext4  defaults,noatime,nodiratime,discard  0  0

# noatime     : 不记录访问时间
# nodiratime  : 不记录目录访问时间
# discard     : 启用 TRIM (或使用 fstrim)
# nobarrier   : 关闭写屏障 (有电池的 RAID 卡)

# === fstrim (SSD TRIM) ===
fstrim -v /data               # 手动 TRIM
systemctl enable fstrim.timer # 启动定时 TRIM

# === IO 性能测试 ===
fio --name=randwrite --ioengine=libaio --iodepth=32 --rw=randwrite \
    --bs=4k --size=2G --numjobs=4 --runtime=60 --group_reporting \
    --filename=/data/test --direct=1

fio --name=randread --ioengine=libaio --iodepth=32 --rw=randread \
    --bs=4k --size=2G --numjobs=4 --runtime=60 --group_reporting \
    --filename=/data/test --direct=1

# === iotop (实时 IO 监控) ===
iotop -o           # 仅显示有 IO 的进程
iotop -oP          # 进程级别
iotop -b -n 3      # 批处理模式，3 次

yum install -y keepalived    # CentOS
apt-get install -y keepalived  # Ubuntu

# /etc/keepalived/keepalived.conf (Master)
global_defs {
    router_id web_lb_01
    # 通知脚本
    notification_email {
        [email protected]
    }
    notification_email_from [email protected]
    smtp_server smtp.example.com
    smtp_connect_timeout 30
}

# 健康检查脚本
vrrp_script chk_nginx {
    script "/usr/bin/killall -0 nginx"    # 检查 nginx 进程
    interval 2
    weight -20
    fall 3          # 连续 3 次失败触发切换
    rise 2          # 连续 2 次成功恢复
}

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    nopreempt          # 不抢占 (故障恢复后不自动切回)

    authentication {
        auth_type PASS
        auth_pass your_password
    }

    virtual_ipaddress {
        192.168.1.100/24 dev ens33
    }

    track_script {
        chk_nginx        # 关联健康检查
    }

    # 状态切换通知
    notify_master "/opt/scripts/notify.sh master"
    notify_backup "/opt/scripts/notify.sh backup"
    notify_fault  "/opt/scripts/notify.sh fault"
}

# /etc/keepalived/keepalived.conf (Backup)
global_defs {
    router_id web_lb_02
}

vrrp_script chk_nginx {
    script "/usr/bin/killall -0 nginx"
    interval 2
    weight -20
    fall 3
    rise 2
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 90
    advert_int 1

    authentication {
        auth_type PASS
        auth_pass your_password
    }

    virtual_ipaddress {
        192.168.1.100/24 dev ens33
    }

    track_script {
        chk_nginx
    }
}

# 允许非本地 IP 绑定 (使得 Backup 也能绑定 VIP)
# echo "net.ipv4.ip_nonlocal_bind = 1" >> /etc/sysctl.d/99-keepalived.conf
# sysctl -p /etc/sysctl.d/99-keepalived.conf

# 安装
yum install -y haproxy    # CentOS
apt-get install -y haproxy  # Ubuntu

# === 完整配置 ===
# /etc/haproxy/haproxy.cfg

global
    log /dev/log local0
    log /dev/log local1 notice
    chroot /var/lib/haproxy
    user haproxy
    group haproxy
    daemon
    maxconn 50000
    spread-checks 5
    stats socket /var/run/haproxy.sock mode 600 level admin
    stats timeout 2m
    tune.ssl.default-dh-param 2048

defaults
    log     global
    mode    http
    option  httplog
    option  dontlognull
    option  redispatch
    retries 3
    timeout connect 5s
    timeout client  50s
    timeout server  50s
    timeout http-request 10s
    timeout http-keep-alive 10s
    timeout check 5s
    maxconn 5000

# === 前端 ===
frontend web_frontend
    bind *:80
    bind *:443 ssl crt /etc/haproxy/certs/combined.pem alpn h2,http/1.1
    # HTTP 重定向到 HTTPS
    redirect scheme https if !{ ssl_fc }

    # ACL
    acl is_api path_beg /api
    acl is_admin path_beg /admin
    acl is_static path_end .jpg .png .css .js .woff2
    acl blocked_ua hdr_sub(User-Agent) -i curl wget

    # 按路径路由
    use_backend api_backend if is_api
    use_backend admin_backend if is_admin
    use_backend static_backend if is_static
    default_backend web_backend

    # 拒绝特定 User-Agent
    http-request deny if blocked_ua

    # 限速 (每 IP 每秒 100 请求)
    stick-table type ip size 1m expire 10s store http_req_rate(10s)
    http-request track-sc0 src
    http-request deny if { sc_http_req_rate(0) gt 100 }

# === 后端 ===
backend web_backend
    balance roundrobin
    option httpchk GET /health
    http-check expect status 200
    default-server inter 3s rise 2 fall 3 maxconn 1000

    server web01 192.168.1.11:8080 check weight 100
    server web02 192.168.1.12:8080 check weight 100
    server web03 192.168.1.13:8080 check weight 100 backup  # 备用节点

    # Cookie 会话保持
    cookie SERVERID insert indirect nocache

    # 长连接
    option http-keep-alive

api_backend
    balance leastconn
    option httpchk GET /api/health
    http-check expect status 200
    default-server inter 2s rise 2 fall 2

    server api01 192.168.1.11:8081 check
    server api02 192.168.1.12:8081 check

static_backend
    balance uri
    option httpchk HEAD /health

    server static01 192.168.1.11:8082 check
    server static02 192.168.1.12:8082 check

# === TCP 模式 (MySQL 代理) ===
listen mysql_proxy
    bind *:3307
    mode tcp
    balance leastconn
    option mysql-check user haproxy_check

    server db01 192.168.1.21:3306 check inter 3s
    server db02 192.168.1.22:3306 check inter 3s backup

# === 统计页面 ===
listen stats
    bind *:9000
    mode http
    stats enable
    stats uri /stats
    stats realm HAProxy\ Statistics
    stats auth admin:your_password
    stats refresh 10s
    stats admin if TRUE

# === DR 模式 (Direct Routing, 性能最高) ===
# Director 配置 (192.168.1.10)
ipvsadm -A -t 192.168.1.100:80 -s wrr
ipvsadm -a -t 192.168.1.100:80 -r 192.168.1.11 -g -w 100
ipvsadm -a -t 192.168.1.100:80 -r 192.168.1.12 -g -w 100
ipvsadm -a -t 192.168.1.100:80 -r 192.168.1.13 -g -w 80

ipvsadm -Ln    # 查看规则
ipvsadm -Sn    # 保存规则

# Real Server 配置 (每台)
ifconfig lo:0 192.168.1.100 netmask 255.255.255.255 up
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce

# === 调度算法 ===
# rr     : 轮询
# wrr    : 加权轮询
# lc     : 最少连接
# wlc    : 加权最少连接 (默认)
# lblc   : 基于局部性最少连接
# dh     : 目标哈希
# sh     : 源地址哈希

# === 基础抓包 ===
tcpdump -i any -nn                           # 所有接口，不解析主机名和端口
tcpdump -i ens33 -nn host 192.168.1.100      # 过滤主机
tcpdump -i ens33 -nn port 80                 # 过滤端口
tcpdump -i ens33 -nn src 192.168.1.100       # 源地址
tcpdump -i ens33 -nn dst port 443            # 目标端口
tcpdump -i ens33 -nn tcp                     # 仅 TCP

# === 组合过滤 ===
tcpdump -i ens33 -nn \
  '(host 192.168.1.100 and port 80) or (host 192.168.1.200 and port 443)'

# === 实战场景 ===
# 抓取 HTTP 请求
tcpdump -i ens33 -A -s 0 'tcp port 80 and (((ip[2:2] - ((ip[0]&0xf)<<2)) - ((tcp[12]&0xf0)>>2)) != 0)'

# 抓取特定 HTTP 方法
tcpdump -i ens33 -s 0 -A 'tcp[((tcp[12:1] & 0xf0) >> 2):4] = 0x47455420'   # GET
tcpdump -i ens33 -s 0 -A 'tcp[((tcp[12:1] & 0xf0) >> 2):4] = 0x504f5354'   # POST

# 抓取 DNS 查询
tcpdump -i ens33 -nn port 53

# 抓取特定标志位
tcpdump -i ens33 -nn 'tcp[tcpflags] & (tcp-syn|tcp-fin) != 0'  # SYN 或 FIN
tcpdump -i ens33 -nn 'tcp[tcpflags] & tcp-rst != 0'             # RST 包

# 保存到文件
tcpdump -i ens33 -w /tmp/capture.pcap -s 0 host 192.168.1.100
tcpdump -r /tmp/capture.pcap -nn          # 读取 pcap 文件

# 限制抓包数量
tcpdump -i ens33 -nn -c 100               # 抓 100 个包后停止

# 三次握手问题分析
tcpdump -i ens33 -nn 'tcp[tcpflags] & tcp-syn != 0 and tcp[tcpflags] & tcp-ack == 0'

# 大量 RST → 端口未监听 / 防火墙 REJECT
tcpdump -i ens33 -nn 'tcp[tcpflags] & tcp-rst != 0'

# 重传统计
tcpdump -i ens33 -nn 'tcp[tcpflags] & tcp-syn != 0 or tcp[tcpflags] & tcp-rst != 0'

# === 连通性测试 ===
ping -c 4 -i 0.2 192.168.1.1              # 快速 ping
ping -M do -s 1472 192.168.1.1             # 测试 MTU (禁止分片)
mtr -r -c 10 192.168.1.1                   # 路由追踪 + 统计

# === 路由诊断 ===
ip route get 8.8.8.8                       # 查看到目标的实际路由
traceroute -n 8.8.8.8                      # 路由追踪
tracepath 8.8.8.8                          # MTU 发现 + 路由追踪

# === DNS 诊断 ===
dig +short example.com                     # 简洁输出
dig example.com ANY                        # 所有记录
dig @8.8.8.8 example.com                  # 指定 DNS 服务器
dig -x 8.8.8.8                             # 反向解析
nslookup example.com                       # 交互式查询

# === 端口检测 ===
nc -zv 192.168.1.100 80                   # TCP 端口扫描
nc -zuv 192.168.1.100 53                  # UDP 端口扫描
timeout 3 bash -c '</dev/tcp/192.168.1.100/80 && echo OPEN || echo CLOSED'

# === 扫描工具 ===
nmap -sS 192.168.1.0/24                   # SYN 半连接扫描
nmap -sT -p 1-65535 192.168.1.100         # 全端口扫描
nmap -sV -p 80,443 192.168.1.100          # 服务版本探测
nmap -A 192.168.1.100                      # 综合扫描 (OS + 服务 + 脚本)

# === 连接状态分析 ===
ss -s                                       # 连接统计摘要
ss -tapn                                    # 所有 TCP 连接
ss -tlnp                                    # 监听端口
ss -tan state time-wait                     # TIME_WAIT 状态
ss -tan state established                    # 已建立连接

# 统计各状态连接数
ss -tan | awk 'NR>1{print $1}' | sort | uniq -c | sort -rn

# 统计每个 IP 的连接数
ss -tan | awk 'NR>1{print $5}' | cut -d: -f1 | sort | uniq -c | sort -rn | head -20

# === iperf3 带宽测试 ===
ipserf3 -s                                   # 服务端
iperf3 -c server_ip -t 30 -P 4              # 4 并发, 30 秒
iperf3 -c server_ip -R                        # 反向 (下载)
iperf3 -c server_ip -u -b 100M               # UDP 100Mbps

# === 网卡统计 ===
ethtool -S ens33                              # 网卡详细统计
ethtool ens33                                 # 网卡设置
ethtool -g ens33                              # Ring buffer 大小

# === 实时流量 ===
iftop -i ens33                                # 实时带宽
nload ens33                                   # 实时流量图

# === HTTP 延迟分析 ===
curl -w "time_namelookup: %{time_namelookup}\ntime_connect: %{time_connect}\ntime_starttransfer: %{time_starttransfer}\ntime_total: %{time_total}\n" -o /dev/null -s https://example.com

# 1. 确认高 CPU 进程
top -bn1 -o %CPU | head -20

# 2. 查看进程中的高 CPU 线程
top -H -p <PID>

# 3. 线程 ID 转十六进制 (用于 Java 线程 dump)
printf "%x\n" <TID>

# 4. 查看系统调用
strace -c -p <PID>                  # 统计系统调用耗时
strace -p <PID> -T                  # 显示每个调用耗时

# 5. perf 分析
perf top -g -p <PID>                # 实时采样
perf record -g -p <PID> -- sleep 30 # 记录 30 秒
perf report                          # 查看报告

# 6. Java 应用
jstack <PID>                        # 线程 dump
jstack <PID> | grep -A 20 "0x$(printf "%x" <TID>)"

# 1. 概览
free -h
cat /proc/meminfo

# 2. 进程内存排序
ps aux --sort=-%mem | head -20
ps -eo pid,ppid,cmd,%mem,%cpu,rss --sort=-rss | head -20

# 3. 进程内存详情
cat /proc/<PID>/smaps | grep -E "^(Rss|Pss|Swap):" | awk '{sum+=$2} END {print sum/1024" MB"}'
cat /proc/<PID>/status | grep -E "Vm|Threads"

# 4. 检查是否有内存泄漏
for i in {1..10}; do
    cat /proc/<PID>/status | grep VmRSS
    sleep 5
done

# 5. slab 内存 (内核)
slabtop -s c

# 6. 查看 OOM 历史
dmesg | grep -i "out of memory"
grep -i "killed process" /var/log/messages
journalctl -k | grep -i oom

# 7. OOM Killer 保护关键进程
echo -1000 > /proc/$(pgrep sshd)/oom_score_adj

# === 磁盘满排查流程 ===

# 1. 确认磁盘使用
df -h

# 2. 哪个目录占用大
du -sh /* 2>/dev/null | sort -rh | head -20
du -sh /var/* 2>/dev/null | sort -rh | head -10

# 3. 大文件查找
find / -type f -size +500M -exec ls -lh {} \; 2>/dev/null
find / -type f -size +1G 2>/dev/null

# 4. 已删除但未释放的文件 (进程仍持有)
lsof | grep deleted | awk '{print $1,$2,$7}' | sort -u
lsof +L1 | grep deleted

# 5. inode 耗尽检查
df -i
find / -xdev -type f | cut -d/ -f2 | sort | uniq -c | sort -rn | head -20

# 6. 快速清理
journalctl --vacuum-size=500M
find /var/log -type f -name "*.log" -mtime +30 -delete
yum clean all || apt-get clean
find /tmp -type f -mtime +7 -delete
docker system prune -af 2>/dev/null