目录
一.两台设备(2.130和2.133)作为调度器,前主后备
1.部署keepalived
2.修改配置文件准备启动
3.配置keepalived的系统日志并启动
二.模拟调度器掉点和web服务进程丢失
1.调度器掉点
2.当类似于httpd这种网站服务掉点
三.以三种健康检查方式引入演示LVS+keepalived
1.TCP_CHECK
2.HTTP_GET|SSL_GET
3.MISC
一.两台设备(2.130和2.133)作为调度器,前主后备
1.部署keepalived
链接:百度网盘 请输入提取码百度网盘为您提供文件的网络备份、同步和分享服务。空间大、速度快、安全稳固,支持教育网加速,支持手机端。注册使用百度网盘即可享受免费存储空间https://pan.baidu.com/s/1T0JmFUrKHe0I4htpniGYeg 提取码:dp1j
如下是两台设备都要做的,所以设备hostname有所不同但不影响
[root@localhost ~ ]# tar xvf keepalived-2.2.8.tar.gz -C /usr/local/src/
[root@localhost ~ ]# yum install -y openssl-devel
[root@localhost ~ ]# cd /usr/local/src/keepalived-2.2.8/
[root@localhost keepalived-2.2.8]# yum install -y gcc gcc-c++ make openssl-devel
[root@localhost keepalived-2.2.8]# ./configure --prefix=/usr/local/keepalived \
> --sysconfdir=/etc --sbindir=/usr/sbin --bindir=/usr/bin
#指定安装、系统配置目录等,有需要的可以自己修改
[root@localhost keepalived-2.2.8]# make && make install #编译安装
[root@main keepalived-2.2.8]# tree /etc/keepalived/
/etc/keepalived/
├── keepalived.conf.sample
└── samples
├── keepalived.conf.conditional_conf
├── keepalived.conf.fwmark
├── keepalived.conf.HTTP_GET.port
├── keepalived.conf.inhibit
├── keepalived.conf.IPv6
├── keepalived.conf.misc_check
├── keepalived.conf.misc_check_arg
├── keepalived.conf.PING_CHECK
├── keepalived.conf.quorum
├── keepalived.conf.sample
├── keepalived.conf.SMTP_CHECK
├── keepalived.conf.SSL_GET
├── keepalived.conf.status_code
├── keepalived.conf.track_interface
├── keepalived.conf.UDP_CHECK
├── keepalived.conf.virtualhost
├── keepalived.conf.virtual_server_group
├── keepalived.conf.vrrp
├── keepalived.conf.vrrp.localcheck
├── keepalived.conf.vrrp.lvs_syncd
├── keepalived.conf.vrrp.routes
├── keepalived.conf.vrrp.rules
├── keepalived.conf.vrrp.scripts
├── keepalived.conf.vrrp.static_ipaddress
├── keepalived.conf.vrrp.sync
├── sample.misccheck.smbcheck.sh
└── sample_notify_fifo.sh
1 directory, 28 files
[root@main keepalived-2.2.8]# tree /usr/local/keepalived/
/usr/local/keepalived/
└── share
├── doc
│ └── keepalived
│ └── README
├── man
│ ├── man1
│ │ └── genhash.1
│ ├── man5
│ │ └── keepalived.conf.5
│ └── man8
│ └── keepalived.8
└── snmp
└── mibs
9 directories, 4 files
[root@main keepalived-2.2.8]# cat /usr/lib/systemd/system/keepalived.service
[Unit]
Description=LVS and VRRP High Availability Monitor
After=network-online.target syslog.target
Wants=network-online.target
Documentation=man:keepalived(8)
Documentation=man:keepalived.conf(5)
Documentation=man:genhash(1)
Documentation=https://keepalived.org
[Service]
Type=forking
PIDFile=/run/keepalived.pid
KillMode=process
EnvironmentFile=-/etc/sysconfig/keepalived
ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS
ExecReload=/bin/kill -HUP $MAINPID
[Install]
WantedBy=multi-user.target
[root@main keepalived-2.2.8]# vim /etc/sysconfig/keepalived
[root@main keepalived-2.2.8]# tail -1 /etc/sysconfig/keepalived
KEEPALIVED_OPTIONS="-D -d -S 0"
[root@main keepalived-2.2.8]# vim /etc/rsyslog.conf
[root@main keepalived-2.2.8]# systemctl restart rsyslog.service
#服务脚本,但是启动还无法正常完成,继续往下看
2.修改配置文件准备启动
(1)这是主设备
[root@main keepalived]# pwd
/etc/keepalived
[root@main keepalived]# cp keepalived.conf.sample keepalived.conf #修改此文件,这里只放了修改了的部分
[root@main keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {
#notification_email {
# acassen@firewall.loc
# failover@firewall.loc
# sysadmin@firewall.loc #这些觉得暂时用不上可以先不管
#}
#notification_email_from Alexandre.Cassen@firewall.loc
#smtp_server 192.168.2.130
#smtp_connect_timeout 30
router_id 1 #router_id,主备两个双设备需要不一致
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER #设定类型为master
interface ens33 #通信网卡名称
virtual_router_id 1 #虚拟router组id,主备需要一致
priority 100 #优先级值,主高于备
advert_int 1
authentication {
auth_type PASS #身份验证,密码也需要主备一致
auth_pass 1111
}
virtual_ipaddress {
192.168.2.100 #要设定的VIP,主备一致
}
}
(2)备设备
[root@serverc keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {
#notification_email {
# acassen@firewall.loc
# failover@firewall.loc
# sysadmin@firewall.loc
#}
#notification_email_from Alexandre.Cassen@firewall.loc
#smtp_server 192.168.200.1
#smtp_connect_timeout 30
router_id 2 #主备不一致
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state BACKUP #设定为BACKUP
interface ens33 #通信网卡名称
virtual_router_id 1 #组id一致
priority 80 #优先级要低于主
advert_int 1
nopreempt #非抢占模式
authentication {
auth_type PASS #主备一致
auth_pass 1111
}
virtual_ipaddress {
192.168.2.100 #VIP,主备一致
}
}
3.配置keepalived的系统日志并启动
[root@main ~]# vim /etc/sysconfig/keepalived #此文件是自动生成的,修改内容如下
KEEPALIVED_OPTIONS="-D -d -S 0"
[root@main ~]# vim /etc/rsyslog.conf
.# Save boot messages also to boot.log
local7.* /var/log/boot.log
# Save keepalived messages also to keepalived.log
local0.* /var/log/keepalived.log
#找准位置添加local0这行
#将这几行取消注释
$ModLoad imudp
$UDPServerRun 514
$ModLoad imtcp
$InputTCPServerRun 514
[root@main ~]# systemctl restart rsyslog.service keepalived.service
[root@main ~]# tail -5 /var/log/keepalived.log #日志已经产生内容
Mar 27 20:01:48 main Keepalived_vrrp[33409]: Sending gratuitous ARP on ens33 for 192.168.2.100
Mar 27 20:01:48 main Keepalived_vrrp[33409]: Sending gratuitous ARP on ens33 for 192.168.2.100
Mar 27 20:01:48 main Keepalived_vrrp[33409]: Sending gratuitous ARP on ens33 for 192.168.2.100
Mar 27 20:01:48 main Keepalived_vrrp[33409]: Sending gratuitous ARP on ens33 for 192.168.2.100
Mar 27 20:01:48 main Keepalived_vrrp[33409]: Sending gratuitous ARP on ens33 for 192.168.2.100
[root@main ~]# ip a| grep ens33 -A3 #并且主设备上的VIP已经生成
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:5d:7f:b7 brd ff:ff:ff:ff:ff:ff
inet 192.168.2.130/24 brd 192.168.2.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.2.100/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::bf1e:b2a3:a943:8a6d/64 scope link noprefixroute
valid_lft forever preferred_lft forever
二.模拟调度器掉点和web服务进程丢失
1.调度器掉点
(1)VIP分配在主设备时,访问到hell
(2)主设备服务器断开、keepalived服务失效
此时VIP备绑定去备设备上了,访问到的内容也变为nihao
[root@main ~]# systemctl stop keepalived.service
[root@serverc keepalived]# ip a | grep ens33 -A1
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:2b:95:b3 brd ff:ff:ff:ff:ff:ff
inet 192.168.2.133/24 brd 192.168.2.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.2.100/32 scope global ens33
valid_lft forever preferred_lft forever
2.当类似于httpd这种网站服务掉点
使用脚本后台运行来保障httpd和keepalived持续运转
[root@main keepalived]# cat testhttpd.sh
#!/bin/bash
while true; do
httpdpid=$(ps -C httpd --no-header | wc -l)
if [ ${httpdpid} -eq 0 ]; then
systemctl start httpd
sleep 10
httpdpid=$(ps -C httpd --no-header | wc -l)
if [ ${httpdpid} -eq 0 ]; then
systemctl stop keepalived
fi
else
systemctl restart httpd
fi
sleep 10
done
[root@main keepalived]# nohup ./testhttpd.sh &
[2] 49373
[root@main keepalived]# nohup: ignoring input and appending output to ‘nohup.out’
^C
[root@main keepalived]# jobs
[1]+ Stopped (wd: ~)
[2]- Running nohup ./testhttpd.sh &
[root@main keepalived]# systemctl stop httpd #手动停掉后过几秒又将其启动起来了
[root@main keepalived]# ps -C httpd
PID TTY TIME CMD
[root@main keepalived]# ps -C httpd
PID TTY TIME CMD
51258 ? 00:00:00 httpd
51259 ? 00:00:00 httpd
51261 ? 00:00:00 httpd
51262 ? 00:00:00 httpd
51263 ? 00:00:00 httpd
51264 ? 00:00:00 httpd
51288 ? 00:00:00 httpd
三.以三种健康检查方式引入演示LVS+keepalived
主设备-192.168.2.130
备设备-192.168.2.133
VIP-192.168.2.100
RS1-192.168.2.131
RS2-192.168.2.132
1.TCP_CHECK
(1)主设备配置
[root@main keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {
#notification_email {
# acassen@firewall.loc
# failover@firewall.loc
# sysadmin@firewall.loc
#}
#notification_email_from Alexandre.Cassen@firewall.loc
#smtp_server 192.168.2.130
#smtp_connect_timeout 30
router_id 1
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER
interface ens33
virtual_router_id 1
mcast_src_ip 192.168.2.130
priority 100
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.2.100
}
}
virtual_server 192.168.2.100 80 { #VIP
delay_loop 6 #健康检查间隔时间
lb_algo rr #调度方式为rr
lb_kind DR #LVS模式为DR
protocol TCP #TCP协议
real_server 192.168.2.131 80 { #RIP1
weight 1 #权重值
TCP_CHECK { #TCP_CHECK方式
connect_timeout 3 #连接超时时间
nb_get_retry 3 #重连次数
connection_port 80 #检查端口
delay_before_retry 3 #重连间隔时间
}
}
real_server 192.168.2.132 80 { #RIP2
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
connection_port 80
delay_before_retry 3
}
}
}
[root@main keepalived]# systemctl restart keepalived.service
(2)备设备配置
[root@serverc keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {
#notification_email {
# acassen@firewall.loc
# failover@firewall.loc
# sysadmin@firewall.loc
#}
#notification_email_from Alexandre.Cassen@firewall.loc
#smtp_server 192.168.200.1
#smtp_connect_timeout 30
router_id 2
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
virtual_router_id 1
priority 80
advert_int 1
nopreempt
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.2.100
}
}
virtual_server 192.168.2.100 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.2.131 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
connection_port 80
delay_before_retry 3
}
}
real_server 192.168.2.132 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
connection_port 80
delay_before_retry 3
}
}
}
[root@serverc keepalived]# systemctl restart keepalived.service
(3)节点执行lvs-dr脚本服务,来进行绑定VIP和添加通信路由,这步可以手动做,参考前面lvs-dr集群的文章
[root@servera ~]# vim /etc/init.d/lvs-dr
[root@servera ~]# cat /etc/init.d/lvs-dr #VIP等需要自己更改
#!/bin/bash
LOCK=/var/lock/ipvsadm.lock
VIP=192.168.2.100
. /etc/rc.d/init.d/functions
start() {
PID=`ifconfig | grep lo:130 | wc -l`
if [ $PID -ne 0 ];
then
echo "The LVS-DR-RIP Server is already running !"
else
/sbin/ifconfig lo:130 $VIP netmask 255.255.255.255 broadcast $VIP up
/sbin/route add -host $VIP dev lo:130
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/ens33/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/ens33/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
/bin/touch $LOCK
echo "starting LVS-DR-RIP server is ok !"
fi
}
stop() {
/sbin/route del -host $VIP dev lo:130
/sbin/ifconfig lo:130 down >/dev/null
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/ens33/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/ens33/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
rm -rf $LOCK
echo "stopping LVS-DR-RIP server is ok !"
}
status() {
if [ -e $LOCK ];
then
echo "The LVS-DR-RIP Server is already running !"
else
echo "The LVS-DR-RIP Server is not running !"
fi
}
case "$1" in
start)
start
;;
stop)
stop
;;
restart)
stop
start
;;
status)
status
;;
*)
echo "Usage: $1 {start|stop|restart|status}"
exit 1
esac
exit 0
[root@servera ~]# systemctl daemon-reload
[root@servera ~]# service lvs-dr start
[root@servera ~]# route -n #通信路由添加成功
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.2.1 0.0.0.0 UG 100 0 0 ens33
192.168.2.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
192.168.2.100 0.0.0.0 255.255.255.255 UH 0 0 0 lo
[root@servera ~]# ip a| grep lo #环回创建成功
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet 192.168.2.100/32 brd 192.168.2.100 scope global lo:130
inet 192.168.2.131/24 brd 192.168.2.255 scope global noprefixroute ens33
(4)主设备上查看VIP是否创建成功
[root@main keepalived]# ip a | grep ens33 -A1
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:5d:7f:b7 brd ff:ff:ff:ff:ff:ff
inet 192.168.2.130/24 brd 192.168.2.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.2.100/32 scope global ens33
valid_lft forever preferred_lft forever
[root@main keepalived]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.2.100:80 rr
-> 192.168.2.131:80 Route 1 0 0
-> 192.168.2.132:80 Route 1 0 0
(5)进行测试
负载均衡测试
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
节点131上httpd服务掉点,具体可以通过watch ipvsadm -Ln来查看节点剔除和恢复过程
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
主调度器设备130掉点
[root@main keepalived]# systemctl stop keepalived.service
[root@serverc keepalived]# ip a | grep ens33 -A1
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:2b:95:b3 brd ff:ff:ff:ff:ff:ff
inet 192.168.2.133/24 brd 192.168.2.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.2.100/32 scope global ens33
valid_lft forever preferred_lft forever
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
#负载均衡正常
2.HTTP_GET|SSL_GET
以genhash来生成检查摘要信息
[root@main keepalived]# genhash -s 192.168.2.131 -p 80 -u /index.html
db1dd528b0e0c9a347eda778aec00559
[root@main keepalived]# genhash -s 192.168.2.132 -p 80 -u /index.html
27d4c8a485f28559e9b1737702b40225
#如下配置
virtual_server 192.168.2.100 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.2.131 80 {
weight 1
# TCP_CHECK {
# connect_timeout 3
# nb_get_retry 3
# connection_port 80
# delay_before_retry 3
# }
HTTP_GET {
url {
path /index.html
digset 2d4074c5771f087dd468d1960185f1f5
}
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.2.132 80 {
weight 1
# TCP_CHECK {
# connect_timeout 3
# nb_get_retry 3
# connection_port 80
# delay_before_retry 3
# }
HTTP_GET {
url {
path /index.html
digset 2d4074c5771f087dd468d1960185f1f5 #基于页面后端hash值
#status 200 #基于返回状态码
}
connect_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
#重启keepalived后测试
测试
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done #负载均衡
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done #节点1掉点
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
[root@main keepalived]# systemctl stop keepalived.service #主设备掉点,负载均衡正常
[root@serverc keepalived]# ip a |grep ens33 -A1
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:2b:95:b3 brd ff:ff:ff:ff:ff:ff
inet 192.168.2.133/24 brd 192.168.2.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.2.100/32 scope global ens33
valid_lft forever preferred_lft forever
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
3.MISC
利用健康测试脚本来测试httpd服务
real_server 192.168.2.131 80 {
weight 1
MISC_CHECK {
misc_path "/etc/keepalived/test.sh 192.168.2.131"
misc_timeout 3
}
# TCP_CHECK {
# connect_timeout 3
# nb_get_retry 3
# connection_port 80
# delay_before_retry 3
# }
# HTTP_GET {
# url {
# path /index.html
# digset 2d4074c5771f087dd468d1960185f1f5
# status 200
# }
# connect_port 80
# connect_timeout 3
# nb_get_retry 3
# delay_before_retry 3
# }
}
[root@main keepalived]# cat test.sh #只针对200状态码
#!/bin/bash
if [ $# -ne 1 ]
then
echo "You should supply an url parameter."
exit 1
else
n=`curl -I "$1" 2> /dev/null | grep "200 OK" | wc -l`
if [ $n -eq 1 ]
then
exit 0
else
exit 1
fi
fi
[root@main keepalived]# cat test.sh #可以使用nmap来做,yum install -y nmap
#!/bin/bash
if [ $# -ne 1 ]
then
echo "You should supply an url parameter."
exit 1
else
ip_and_path=$1
ip=$(echo "$ip_and_path" | sed 's/.*\/\/\([0-9.]*\).*/\1/')
n=$(nmap -p80 "$ip" | awk '/^80\/tcp/ {print $2}')
if [ $n == 'open' ]
then
exit 0
else
exit 1
fi
fi