🚀 Keepalived+Nginx双活实战配置【金融级】.md 💻

Nginx+keepalived 双机双活模式

1
2
3
4
5
6
7
$ yum -y install libnl libnl-devel libnfnetlink-devel
wget -P /tmp/keepalived  http://10.170.40.19/media/linux/soft/keepalived-2.2.7.tar.gz
cd /tmp/keepalived/
tar -zxvf keepalived-2.2.8.tar.gz
cd keepalived-2.2.8
./configure --prefix=/usr/local/keepalived
make && make install

主机1配置文件keepalived.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 $ vim /usr/local/keepalived/etc/keepalived/keepalived.conf
 ! Configuration File for keepalived
global_defs {
   router_id LVS_1
}

vrrp_script check_nginx {
        script "/etc/keepalived/check_nginx.sh"
        interval 2
}
vrrp_instance VI_1 {
    state MASTER    #表示主的nginx
    interface eth0
    virtual_router_id 31
    priority 100
    advert_int 1
    mcast_src_ip 192.168.0.214  #主nginx的ip地址
    authentication {
        auth_type PASS
        auth_pass ng111111
    }

    track_script {
        check_nginx
    }

    virtual_ipaddress {
        192.168.0.17  #主的虚拟ip地址
    }
    notify_master "/data/Keepalived-Slave/keepaliveds/Keepalived-Slave/etc/notify.sh master"
    notify_backup "/data/Keepalived-Slave/keepaliveds/Keepalived-Slave/etc/notify.sh backup"
    notify_fault "/data/Keepalived-Slave/keepaliveds/Keepalived-Slave/etc/notify.sh fault"
}

主机2配置文件keepalived.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
$ vim /usr/local/keepalived/etc/keepalived/keepalived.conf

! Configuration File for keepalived
global_defs {
   router_id LVS_1
}

vrrp_script check_nginx {
        script "/etc/keepalived/check_nginx.sh"
        interval 2
}
vrrp_instance VI_1 {
    state BACKUP    #表示主的nginx
    interface ens192
    virtual_router_id 1
    priority 90
    advert_int 1
    mcast_src_ip 10.170.20.16  #主nginx的ip地址
    authentication {
        auth_type PASS
        auth_pass ng111111
    }

    track_script {
        check_nginx
    }

    virtual_ipaddress {
        10.170.20.17  #主的虚拟ip地址
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

创建文件夹目录、日志和软链接

1
2
3
4
5
6
$ mkdir -p /etc/keepalived
ln -s /usr/local/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/
ln -s /usr/local/keepalived/sbin/keepalived /usr/sbin/
mkdir /var/log/keepalived
touch switch.log

创建主机脚本节点监控

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
$ vim /etc/keepalived/check_nginx.sh
#!/bin/bash

run=`ps -C nginx --no-header | wc -l`
export now=`/bin/date +%y%m%d'~'%H':'%M':'%S`
if [ $run -eq 0 ]
        then
                systemctl restart nginx
                echo $now'------nginx------10.170.20.20(master)，检测到无nginx进程， nginx服务重启。' >> /var/log/keepalived/switch.log
        sleep 5
        if [ `ps -C nginx --no-header | wc -l -eq 0` ]
        then
                systemctl stop keepalived.service
                export now1=`/bin/date +%y%m%d'~'%H':'%M':'%S`
                echo $now1'------nginx------10.170.20.20(master)，检测到重启后仍然无nginx进程， 关闭keepalive进程，主备切换。' >> /var/log/keepalived/switch.log
        fi
fi

• 创建主节点check_nginx.sh
• chmod 755 /etc/keepalived/check_nginx.sh #赋文件权限

创建备机脚本节点监控

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
$ vim /etc/keepalived/check_nginx.sh

#!/bin/bash

run=`ps -C nginx --no-header | wc -l`
export now=`/bin/date +%y%m%d'~'%H':'%M':'%S`
if [ $run -eq 0 ]
        then
                systemctl restart nginx
                echo $now'------nginx------10.170.20.21(backup)，检测到无nginx进程， nginx服务重启。' >> /var/log/keepalived/switch.log
        sleep 5
        if [ `ps -C nginx --no-header | wc -l -eq 0` ]
        then
                systemctl stop keepalived.service
                export now1=`/bin/date +%y%m%d'~'%H':'%M':'%S`
                echo $now1'------nginx------10.170.20.21(backup)，检测到重启后仍然无nginx进程， 关闭keepalive进程，主备切换。' >> /var/log/keepalived/switch.log
        fi
fi

• 创建备节点check_nginx.sh
• chmod 755 /etc/keepalived/check_nginx.sh #赋文件权限

创建节点notify脚本

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
$ #!/bin/bash

# Define variables for better readability and easier modification
readonly LOG_DIR="/var/log/keepalived"
readonly LOG_FILE="${LOG_DIR}/switch.log"
readonly SCRIPT_NAME=$(basename "$0")

# Ensure the log directory exists
mkdir -p "${LOG_DIR}"

# Get the current timestamp
now=$(/bin/date +%Y-%m-%d' '%H:%M:%S)

# Function to log messages with a clear format
log_message() {
    local message="$1"
    echo "${now} ------ SWITCH ------ ${message}" >> "${LOG_FILE}"
}

# The state argument is passed by Keepalived: master, backup, or fault
STATE=$1
# Keepalived also passes the VRRP instance type (e.g., INSTANCE) and name (e.g., VI_1)
# You can capture these if you need them for more complex logic or logging:
# TYPE=$2
# NAME=$3
# PRIORITY=$4

case "${STATE}" in
    "master")
        log_message "Local node became MASTER (Primary). VIP is now active here."
        # --- Add your MASTER-specific commands here ---
        # Example: Start a service that should only run on the master
        # /usr/bin/systemctl start my-application-service
        # Example: Announce the VIP using arp-scan or similar (often not needed with Keepalived, but good for quick failover)
        # /usr/sbin/arping -c 3 -U <YOUR_VIRTUAL_IP_ADDRESS> -I <YOUR_NETWORK_INTERFACE> > /dev/null 2>&1
        ;;
    "backup")
        log_message "Local node became BACKUP (Standby). Waiting for MASTER."
        # --- Add your BACKUP-specific commands here ---
        # Example: Stop a service that should only run on the master
        # /usr/bin/systemctl stop my-application-service
        ;;
    "fault")
        log_message "Local node entered FAULT state. Potential issue detected!"
        # --- Add your FAULT-specific commands here ---
        # Example: Send an email alert to administrators
        # echo "Keepalived alert: Node $(hostname) for VIP <YOUR_VIRTUAL_IP_ADDRESS> is in FAULT state!" | /usr/bin/mail -s "Keepalived FAULT Alert" your_email@example.com
        ;;
    *)
        log_message "Error: Unknown state '${STATE}' received for ${SCRIPT_NAME}. Usage: ${SCRIPT_NAME} {master|backup|fault}"
        exit 1
        ;;
esac

exit 0

• chmod 755 /etc/keepalived/notify.sh

配置负载均衡ginx.conf：client–>请求–>nginx–>proxy_pass–>upstream–>后端服务器

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
$ cat <<EOF > /usr/local/openresty/nginx/conf/nginx.conf

user  nginx;
worker_processes  4;
worker_cpu_affinity 00000001 00000010 00000100 00001000;
#worker_processes  8;
#worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
error_log  logs/error.log warn;
pid        logs/nginx.pid;
worker_rlimit_nofile 65535;

events {
    worker_connections  65535;
	use epoll;
}

http {
    include    mime.types;
    default_type  application/octet-stream;
	access_log  logs/access.log main;
	log_format  main  '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"';
    charset utf-8;
    sendfile  on;
    keepalive_timeout  300;
    client_header_timeout 80;
    client_body_timeout 80;
	client_max_body_size 1g;
    server_tokens off;
  #gzip  on;
  #gzip_http_version 1.1;
    include vhosts/*.conf;
}
EOF

创建Vhost虚拟子配置文件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
$ mkdir -p /data/nginx/conf/vhosts
vim /data/nginx/conf/vhosts/web.conf
upstream zxcx {
 ip_hash;
 server 10.170.16.27:80 weight=10;
 server 10.170.16.28:80 weight=10;
 }

server {
        listen 80;
         
        server_name 10.170.16.29;
        #server_name inqu-zsc.gwamcc.com;
        rewrite ^(.*)$ https://${server_name}$1 permanent;

        location / {
        proxy_pass   http://zxcx;
        #stub_status on;
        fastcgi_buffers  256 16K;
        fastcgi_buffer_size 128K;
        #access_log /usr/local/nginx/logs/nginxstatus.log;
        proxy_buffer_size 64K;
        proxy_buffers  256 64K;
        proxy_busy_buffers_size 128K;
        proxy_redirect off;
        proxy_set_header X-Real-IP $remote_addr;
        #后端的Web服务器可以通过X-Forwarded-For获取用户真实IP
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Host $host;
        client_max_body_size 50M; #允许客户端请求的最大单文件字节数
        client_body_buffer_size 20M; #缓冲区代理缓冲用户端请求的最大字节数，
        fastcgi_connect_timeout 60;
        fastcgi_send_timeout 180;  
        fastcgi_read_timeout 180;  
        proxy_connect_timeout 300; #nginx跟后端服务器连接超时时间(代理连接超时)
        proxy_send_timeout 300; #后端服务器数据回传时间(代理发送超时)
        proxy_read_timeout 300; #连接成功后，后端服务器响应时间(代理接收超时)
        }
}

 server {
        listen       443 ssl;
        server_name  10.170.16.29;
        #server_name  inqu-zsc.gwamcc.com;
		
        add_header X-Frame-Options SANEORIGIN;
        add_header Strict-Transport-Security max-age=63072000;
        add_header Referrer-Policy no-referrer-when-downgrade;
        add_header Content-Security-Policy "frame-ancestors 'self'";
        add_header X-Permitted-Cross-Domain-Policies "master-only";
        add_header X-XSS-Protection "1; mode=block;";
        add_header X-Download-Options SAMEORIGIN;
        add_header X-Content-Type-Options nosniff;

        ssl_certificate      /usr/local/nginx/ssl/server.cer;
        ssl_certificate_key  /usr/local/nginx/ssl/server.key;

        ssl_session_cache    shared:SSL:1m;
        ssl_session_timeout  5m;

        ssl_ciphers  HIGH:!aNULL:!MD5;
        ssl_prefer_server_ciphers  on;
	    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
        
		
	    location / {
        proxy_pass   http://zxcx;
        #stub_status on;
        fastcgi_buffers  256 16K;
        fastcgi_buffer_size 128K;
        #access_log /usr/local/nginx/logs/nginxstatus.log;
        proxy_buffer_size 64K;
        proxy_buffers  256 64K;
        proxy_busy_buffers_size 128K;
        proxy_redirect off;
        proxy_set_header X-Real-IP $remote_addr;
        #后端的Web服务器可以通过X-Forwarded-For获取用户真实IP
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header Host $host;
        client_max_body_size 50M; #允许客户端请求的最大单文件字节数
        client_body_buffer_size 20M; #缓冲区代理缓冲用户端请求的最大字节数，
        fastcgi_connect_timeout 60;
        fastcgi_send_timeout 180;  
        fastcgi_read_timeout 180;  
        proxy_connect_timeout 300; #nginx跟后端服务器连接超时时间(代理连接超时)
        proxy_send_timeout 300; #后端服务器数据回传时间(代理发送超时)
        proxy_read_timeout 300; #连接成功后，后端服务器响应时间(代理接收超时)
        }
}

配置同时支持ipv4和ipv6

1
2
3
$ listen 80;
listen [::]:80 ipv6only=on;

添加nginx.service服务文件

1
2
3
4
5
6
7
8
9
10
11
12
13
$ vim /lib/systemd/system/nginx.service 
[Unit]
Description=nginx service
After=network.target
[Service]
Type=forking
ExecStart=/data/nginx/sbin/nginx
ExecReload=/data/nginx/sbin/nginx -s reload
ExecStop=/data/nginx/sbin/nginx -s quit
PrivateTmp=true
[Install]
WantedBy=multi-user.target

服务的启动/停止/刷新配置文件/查看状态

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
$ systemctl start nginx.service　         启动nginx服务
systemctl stop nginx.service　          停止服务
systemctl restart nginx.service　       重新启动服务
systemctl status nginx.service           查看nginx服务当前状态

systemctl enable nginx.service          设置开机自启动
systemctl disable nginx.service          停止开机自启动
systemctl list-units --type=service     查看所有已启动的服务
systemctl is-active nginx.service	        查看服务启动启用状态
systemctl is-enabled nginx.service     查看服务启动启用状态

systemctl start keepalived.service　        启动keepalived服务
systemctl stop keepalived.service　        停止keepalived服务
systemctl restart keepalived.service　     重新启动keepalived服务
systemctl status keepalived.service         查看keepalived服务当前状态

systemctl enable keepalived.service        设置开机自启动
systemctl disable keepalived.service        停止开机自启动

tail -f /var/log/keepalived/switch.log
tail -f /usr/local/nginx/logs/access.log

创建软连接添加到sbin环境环境里

1
2
3
4
5
6
7
8
9
10
11
12
13
14
$ ln -s /data/nginx/sbin/nginx /usr/sbin/
nginx -V
systemctl daemon-reload
systemctl restart nginx
systemctl restart keepalived 
/usr/local/nginx/sbin/nginx -t 测试当前配置文件是否正确
/usr/local/nginx/sbin/nginx -s reload 重新加载配置文件
/usr/local/nginx/sbin/nginx -s reopen 重新打开日志文件

netstat -tlunp|grep nginx
ps -aux|grep nginx 
ps -ef|grep nginx
ps -ef|grep keepalived