使用 Keepalived 和 HAproxy 创建高可用集群

准备

IP hostname 操作系统 配置
30.0.1.157 LB1 centos7.6 2C4G80G
30.0.1.141 LB2 centos7.6 2C4G80G
30.0.1.220 node1 centos7.6 2C4G80G

安装

yum install keepalived haproxy psmisc -y

haproxy

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
# vi /etc/haproxy/haproxy.cfg
global
    log /dev/log  local0 warning
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
   
   stats socket /var/lib/haproxy/stats
   
defaults
  log global
  option  httplog
  option  dontlognull
        timeout connect 5000
        timeout client 50000
        timeout server 50000
   
frontend kube-apiserver
    bind *:6443
    mode tcp
    option tcplog
    default_backend kube-apiserver
   
backend kube-apiserver
    mode tcp
    option tcplog
    option tcp-check
    balance roundrobin
    default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
    server kube-apiserver-1 30.0.1.157:6443 check # Replace the IP address with your own.
    server kube-apiserver-2 30.0.1.141:6443 check
    server kube-apiserver-3 30.0.1.220:6443 check

systemctl restart haproxy
systemctl enable haproxy

Keepalived

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# vi /etc/keepalived/keepalived.conf
global_defs {
  notification_email {
  }
  router_id LVS_DEVEL
  vrrp_skip_check_adv_addr
  vrrp_garp_interval 0
  vrrp_gna_interval 0
}
   
vrrp_script chk_haproxy {
  script "killall -0 haproxy"
  interval 2
  weight 2
}
   
vrrp_instance haproxy-vip {
  state BACKUP
  priority 100
  interface eth0                       # Network card
  virtual_router_id 60
  advert_int 1
  authentication {
    auth_type PASS
    auth_pass 1111
  }
  unicast_src_ip 30.0.1.157      # The IP address of this machine
  unicast_peer {
    30.0.1.141                         # The IP address of peer machines
  }
   
  virtual_ipaddress {
    30.0.1.3/24                  # The VIP address
  }
   
  track_script {
    chk_haproxy
  }
}

systemctl restart keepalived
systemctl enable keepalived

验证

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# 在LB1 机器上执行ip a s命令查看
[root@openlab openlab]# ip a s
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:82:03:7d brd ff:ff:ff:ff:ff:ff
    inet 30.0.1.157/24 brd 30.0.1.255 scope global noprefixroute dynamic eth0
       valid_lft 82289sec preferred_lft 82289sec
    inet 30.0.1.3/24 scope global secondary eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe82:37d/64 scope link 
       valid_lft forever preferred_lft forever

在 node1 节点 Ping vip 看是否能正常访问

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
[root@openlab openlab]# ip a s
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:9b:31:d1 brd ff:ff:ff:ff:ff:ff
    inet 30.0.1.220/24 brd 30.0.1.255 scope global noprefixroute dynamic eth0
       valid_lft 81745sec preferred_lft 81745sec
    inet6 fe80::f816:3eff:fe9b:31d1/64 scope link 
       valid_lft forever preferred_lft forever
[root@openlab openlab]# ping 30.0.1.3
PING 30.0.1.3 (30.0.1.3) 56(84) bytes of data.
64 bytes from 30.0.1.3: icmp_seq=1 ttl=64 time=0.494 ms
64 bytes from 30.0.1.3: icmp_seq=2 ttl=64 time=0.486 ms
64 bytes from 30.0.1.3: icmp_seq=3 ttl=64 time=0.478 ms
64 bytes from 30.0.1.3: icmp_seq=4 ttl=64 time=0.420 ms
64 bytes from 30.0.1.3: icmp_seq=5 ttl=64 time=0.439 ms
64 bytes from 30.0.1.3: icmp_seq=6 ttl=64 time=0.477 ms

模拟LB1节点 haproxy 异常的情况

systemctl stop haproxy

再次检查浮动 IP 地址，可以看到该地址在 LB1 上消失

1
2
3
4
5
6
7
8
9
10
11
12
13
[root@openlab openlab]# ip a s
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:82:03:7d brd ff:ff:ff:ff:ff:ff
    inet 30.0.1.157/24 brd 30.0.1.255 scope global noprefixroute dynamic eth0
       valid_lft 81934sec preferred_lft 81934sec
    inet6 fe80::f816:3eff:fe82:37d/64 scope link 
       valid_lft forever preferred_lft forever

理论上讲，若配置成功，该虚拟 IP 会漂移到另一台机器 LB2 上。在 LB2 上运行以下命令，这是预期的输出

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[root@openlab openlab]# ip a s
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc pfifo_fast state UP group default qlen 1000
    link/ether fa:16:3e:66:84:a8 brd ff:ff:ff:ff:ff:ff
    inet 30.0.1.141/24 brd 30.0.1.255 scope global noprefixroute dynamic eth0
       valid_lft 81885sec preferred_lft 81885sec
    inet 30.0.1.3/24 scope global secondary eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::f816:3eff:fe66:84a8/64 scope link 
       valid_lft forever preferred_lft forever

参考

https://kubesphere.com.cn/docs/installing-on-linux/high-availability-configurations/set-up-ha-cluster-using-keepalived-haproxy/