作者 | 姜云鹏 黄俊楷

1 背景与问题场景

1.1 环境与故障现象

1.2 技术背景

2 部署与验证

2.1 部署步骤

2.1.1 在LVS端源码编译安装Keepalived

cd /opt/kyou
yum install -y libnl3-devel openssl-devel libnfnetlink-devel gcc make ipvsadm # 安装依赖
wget --no-check-certificate https://www.keepalived.org/software/keepalived-1.3.5.tar.gz # 下载源码包
tar -zxvf keepalived-1.3.5.tar.gz # 解压源码包
# 修改源码文件，避免安装时报错。vim到/opt/kyou/keepalived-1.3.5/keepalived/include/vrrp_if.h文件，将第130行的list garp_delay; 修改为 extern list garp_delay;
cd keepalived-1.3.5
./configure
make -j && make install

2.1.2 将Keepalived注册成服务

cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/keepalived
cp /usr/local/sbin/keepalived /usr/sbin/keepalived
mkdir /etc/keepalived
cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/

2.1.3 备份后修改LVS端Keepalived配置文件

! Configuration File for keepalived
global_defs {
    router_id VI_189
    script_user root
}

vrrp_instance VI_7 {
    state MASTER
    interface enahisic2i0  # 指定VRRP实例使用的网络接口
    virtual_router_id 7
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }

    virtual_ipaddress {
        192.168.1.9/24 dev enahisic2i0  # 将VIP绑定到指定的网络接口
    }
}

virtual_server 192.168.1.9 80 {  # 定义一个虚拟服务器，绑定在192.168.1.9的80端口
    delay_loop 7
    lb_algo wlc
    lb_kind DR  # 使用DR模式
    protocol TCP

    persistence_timeout 60

    real_server 192.168.1.204 80 {  # 真实服务器，IP为192.168.1.204，端口为80
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 5
            nb_get_retry 1
            delay_before_retry 1
        }
    }
    real_server 192.168.1.225 80 {  # 另一个真实服务器，IP为192.168.1.225，端口为80
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 5
            nb_get_retry 1
            delay_before_retry 1
        }
    }
}

2.1.4 安装Nginx

yum install -y nginx

2.1.5 备份后修改Nginx配置文件

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 4096;

    include             /etc/nginx/mime.types;
default_type        application/octet-stream;

    # Load modular configuration files from the /etc/nginx/conf.d directory.
    # See http://nginx.org/en/docs/ngx_core_module.html#include
    # for more information.
    include /etc/nginx/conf.d/*.conf;

    server {
        listen       80;
        listen       [::]:80;
        server_name  192.168.1.204;  # 这里是Nginx端的真实IP，如果是Nginx②端，需要改为192.168.1.225
        root         /usr/share/nginx/html;

        # Load configuration files for the default server block.
        include /etc/nginx/default.d/*.conf;

        error_page 404 /404.html;
            location = /40x.html {
        }

        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }
    }
}

2.1.6 配置Nginx端的回环IP和ARP抑制

ip addr add 192.168.1.9/32 dev lo label lo:0
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce

2.1.7 配置Nginx页面并启动Nginx

echo "A Web Server from Nginx/openEuler/204! Hello!" > /usr/share/nginx/html/index.html # 需要分别为两个Nginx端设置不同的内容
systemctl start nginx
systemctl enable nginx

2.1.8 启动Keepalived

systemctl enable keepalived
systemctl start keepalived

getenforce # 查看SELinux状态
sudo setenforce 0 # 将SELinux设置为宽容模式

echo 1 > /proc/sys/net/ipv4/conf/enp125s0f0/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/enp125s0f0/arp_announce

journalctl -u keepalived --since "5 minutes ago" | grep -i error

echo 0 > /proc/sys/net/ipv4/conf/enp125s0f0/rp_filter

uname -r # 显示当前运行的内核版本
grubby --info=ALL # 显示系统中所有可用的内核信息
grubby --set-default /boot/vmlinuz-5.10.0-60.18.0.50.oe2203.aarch64 # 设置默认启动的内核
reboot # 重启系统，使新的默认内核设置生效