###############################################
keepalived
keepalived+lvs实现高可用的负载均衡
测试
###############################################
keepalived
keepalived一款轻量级高可用软件,工作于layer3, 4 & 5,不同于前几篇博文中的Heartbeat、Corosync等软件的实现机制不同,它采用虚拟路由冗余协议(Virual Router Redundancy Protocal)来实现并且完美的与lvs结合,由于底层使用虚拟路由冗余协议,因此Keepalived具有切换速度快的特点,工作在layer3的keepalived定期向服务器群组中发送ICMP数据包宣告自己存活与否,工作在layer3的keepalived支持以检测TCP端口状态的方式来判定后台Realserver故障与否,自动并将那些判定为故障的后台Realserver从ipvs规则中踢出,工作在layer5可以支持用户自动以脚本来实现相应的智能操作。此lvs也可以结合ldirectord来实现对后台realserver的动态监测,相对于keepalived来说ldirectord属于重量级别的,部署和使用的灵活程度没有前者方便,本文将介绍keepalived。
keepalived+lvs实现高可用的负载均衡
架构图:
realserver端脚本
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
|
#!/bin/bash
#
# Script to start LVS DR real server.
# description: LVS DR real server
#
.
/etc/rc
.d
/init
.d
/functions
VIP=192.168.1.33
host=`
/bin/hostname
`
case
"$1"
in
start)
# Start LVS-DR real server on this machine.
/sbin/ifconfig
lo down
/sbin/ifconfig
lo up
echo
1 >
/proc/sys/net/ipv4/conf/lo/arp_ignore
echo
2 >
/proc/sys/net/ipv4/conf/lo/arp_announce
echo
1 >
/proc/sys/net/ipv4/conf/all/arp_ignore
echo
2 >
/proc/sys/net/ipv4/conf/all/arp_announce
/sbin/ifconfig
lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
/sbin/route
add -host $VIP dev lo:0
;;
stop)
# Stop LVS-DR real server loopback device(s).
/sbin/ifconfig
lo:0 down
echo
0 >
/proc/sys/net/ipv4/conf/lo/arp_ignore
echo
0 >
/proc/sys/net/ipv4/conf/lo/arp_announce
echo
0 >
/proc/sys/net/ipv4/conf/all/arp_ignore
echo
0 >
/proc/sys/net/ipv4/conf/all/arp_announce
;;
status)
# Status of LVS-DR real server.
islothere=`
/sbin/ifconfig
lo:0 |
grep
$VIP`
isrothere=`
netstat
-rn |
grep
"lo:0"
|
grep
$VIP`
if
[ !
"$islothere"
-o !
"isrothere"
];
then
# Either the route or the lo:0 device
# not found.
echo
"LVS-DR real server Stopped."
else
echo
"LVS-DR real server Running."
fi
;;
*)
# Invalid entry.
echo
"$0: Usage: $0 {start|status|stop}"
exit
1
;;
esac
|
安装httpd并建立测试页面如下:
安装ipvsadm和keepalived
|
1
2
|
yum
install
ipvsadm
rpm -ivh keepalived-1.2.7-5.el5.i386.rpm
|
director_master的配置vim /etc/keepalived/keepalived.conf
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
|
! Configuration File
for
keepalived
global_defs {
notification_email {
root@localhost
#报警收件人地址
}
notification_email_from root@localhost
#报警发件人地址
smtp_server 127.0.0.1
#设置smtp服务地址
smtp_connect_timeout 30
#设置连接smtp服务的超时时间
router_id LVS_DEVEL
#发送邮件的主体信息
}
vrrp_script chk_schedown {
#自定义脚本
script
"[ -e /etc/keepalived/down ] && exit 1 || exit 0"
interval 1
#重试时间间隔
weight -5
#减权重
fall 2
rise 1
}
vrrp_instance VI_1 {
state MASTER
#制定keepalived角色
interface eth0
#制定检测网络接口
virtual_router_id 54
#虚拟路由标示码
priority 100
#权重,1-255之间
advert_int 1
#设置同步检查的时间间隔,单位是秒
authentication {
auth_type PASS
#验证类型为PASS
auth_pass soulboy
#验证密码
}
virtual_ipaddress {
192.168.1.33
/24
dev eth0 label eth0:0
#设置虚拟IP
}
track_script {
chk_schedown
}
notify_master
"/etc/keepalived/notify.sh -n master -a 192.168.1.33"
notify_backup
"/etc/keepalived/notify.sh -n backup -a 192.168.1.33"
notify_fault
"/etc/keepalived/notify.sh -n fault -a 192.168.1.33"
}
virtual_server 192.168.1.33 80 {
#定义虚拟服务器
delay_loop 6
#设置健康检查时间
lb_algo wrr
#设置负载调度算法
lb_kind DR
#设置LVS工作模式
nat_mask 255.255.255.0
persistence_timeout 50
protocol TCP
#设置转发协议的类型
sorry_server 127.0.0.1 80
#设置紧急服务器
real_server 192.168.1.10 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 2
nb_get_retry 3
delay_before_retry 1
}
}
real_server 192.168.1.20 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 2
nb_get_retry 3
delay_before_retry 1
}
}
}
}
|
director_backup的配置vim /etc/keepalived/keepalived.conf
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
|
! Configuration File
for
keepalived
global_defs {
notification_email {
root@localhost
}
notification_email_from root@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_script chk_schedown {
script
"[ -e /etc/keepalived/down ] && exit 1 || exit 0"
interval 1
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 54
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass soulboy
}
virtual_ipaddress {
192.168.1.33
/24
dev eth0 label eth0:0
}
track_script {
chk_schedown
}
notify_master
"/etc/keepalived/notify.sh -n master -a 192.168.1.33"
notify_backup
"/etc/keepalived/notify.sh -n backup -a 192.168.1.33"
notify_fault
"/etc/keepalived/notify.sh -n fault -a 192.168.1.33"
}
virtual_server 192.168.1.33 80 {
delay_loop 6
lb_algo wrr
lb_kind DR
nat_mask 255.255.255.0
persistence_timeout 50
protocol TCP
sorry_server 127.0.0.1 80
real_server 192.168.1.10 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 2
nb_get_retry 3
delay_before_retry 1
}
}
real_server 192.168.1.20 80 {
weight 1
HTTP_GET {
url {
path /
status_code 200
}
connect_timeout 2
nb_get_retry 3
delay_before_retry 1
}
}
}
}
|
通知脚本vim /etc/keepalived/notify.sh
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
|
#!/bin/bash
#
ifalias=${2:-eth0:0}
interface=$(
echo
$ifalias |
awk
-F:
'{print $1}'
)
vip=$(ip addr show $interface |
grep
$ifalias |
awk
'{print $2}'
)
contact=
'root@localhost'
workspace=$(
dirname
$0)
notify() {
subject=
"$ip change to $1"
body=
"$ip change to $1 $(date '+%F %H:%M:%S')"
echo
$body | mail -s
"$1 transition"
$contact
}
case
"$1"
in
master)
notify master
exit
0
;;
backup)
notify backup
/etc/rc
.d
/init
.d
/httpd
restart
exit
0
;;
fault)
notify fault
exit
0
;;
*)
echo
'Usage: $(basename $0) {master|backup|fault}'
exit
1
;;
esac
|
测试
启动director_master的keepalive服务并查看ipvs规则
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
|
#####查看ipvs规则
[root@master ~]
# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.1.33:80 wrr
-> 192.168.1.20:80 Route 1 0 0
-> 192.168.1.10:80 Route 1 0 0
#####查看网络信息
[root@master ~]
# ifconfig
eth0 Link encap:Ethernet HWaddr 00:0C:29:C2:5E:01
inet addr:192.168.1.61 Bcast:192.168.1.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fec2:5e01
/64
Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:67996 errors:0 dropped:0 overruns:0 frame:0
TX packets:116217 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:15418633 (14.7 MiB) TX bytes:8387202 (7.9 MiB)
Interrupt:67 Base address:0x2024
eth0:0 Link encap:Ethernet HWaddr 00:0C:29:C2:5E:01
inet addr:192.168.1.33 Bcast:0.0.0.0 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
Interrupt:67 Base address:0x2024
|
启动director_backup的keepalive服务并查看ipvs规则
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
|
#####查看ipvs规则
[root@backup ~]
# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.1.33:80 wrr
-> 192.168.1.20:80 Route 1 0 0
-> 192.168.1.10:80 Route 1 0 0
#####查看网络信息
[root@backup ~]
# ifconfig
eth0 Link encap:Ethernet HWaddr 00:0C:29:FA:52:D6
inet addr:192.168.1.62 Bcast:192.168.1.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fefa:52d6
/64
Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:115068 errors:0 dropped:0 overruns:0 frame:0
TX packets:82940 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:19740061 (18.8 MiB) TX bytes:6476242 (6.1 MiB)
Interrupt:67 Base address:0x2024
|
使用客户端访问VIP
停止director_master的keepalived服务发现VIP消失
|
1
2
3
4
5
6
7
8
9
10
11
12
|
[root@master ~]
# service keepalived stop
Stopping keepalived: [ OK ]
[root@master ~]
# ifconfig
eth0 Link encap:Ethernet HWaddr 00:0C:29:C2:5E:01
inet addr:192.168.1.61 Bcast:192.168.1.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fec2:5e01
/64
Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:69371 errors:0 dropped:0 overruns:0 frame:0
TX packets:118587 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:15609985 (14.8 MiB) TX bytes:8588490 (8.1 MiB)
Interrupt:67 Base address:0x2024
|
在director_backup查看网络信息,发现VIP已成功转移
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
|
[root@backup ~]
# ifconfig
eth0 Link encap:Ethernet HWaddr 00:0C:29:FA:52:D6
inet addr:192.168.1.62 Bcast:192.168.1.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fefa:52d6
/64
Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:116816 errors:0 dropped:0 overruns:0 frame:0
TX packets:84293 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:19932196 (19.0 MiB) TX bytes:6597535 (6.2 MiB)
Interrupt:67 Base address:0x2024
eth0:0 Link encap:Ethernet HWaddr 00:0C:29:FA:52:D6
inet addr:192.168.1.33 Bcast:0.0.0.0 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
Interrupt:67 Base address:0x2024
|
停止realserver_one的httpd服务
|
1
2
|
[root@realserver_one ~]
# service httpd stop
Stopping httpd: [ OK ]
|
director_backup查看ipvs规则,发现realserver_one已经被踢出
|
1
2
3
4
5
6
|
[root@backup ~]
# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.1.33:80 wrr
-> 192.168.1.20:80 Route 1 0 0
|
客户端访问VIP发现页面恒为node2
|
1
2
|
[root@realserver_two ~]
# service httpd stop
Stopping httpd: [ OK ]
|
director_backup查看ipvs规则,发现紧急站点生效
|
1
2
3
4
5
6
|
[root@backup ~]
# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.1.33:80 wrr
-> 127.0.0.1:80 Local 1 0 0
|
客户端访问VIP发现页面为自定义警告页面
分别启动realserver_one和realserver_two的httpd服务
|
1
2
3
4
5
6
7
8
9
10
|
#####realserver_one
[root@realserver_one ~]
# service httpd start
Starting httpd: httpd: apr_sockaddr_info_get() failed
for
realserver_one
httpd: Could not reliably determine the server's fully qualified domain name, using 127.0.0.1
for
ServerName
[ OK ]
#####realserver_two
[root@realserver_two ~]
# service httpd start
Starting httpd: httpd: apr_sockaddr_info_get() failed
for
realserver_two
httpd: Could not reliably determine the server's fully qualified domain name, using 127.0.0.1
for
ServerName
[ OK ]
|
再次查看director_backup发现ipvs规则已经恢复
|
1
2
3
4
5
6
7
|
[root@backup ~]
# ipvsadm -L -n
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.1.33:80 wrr
-> 192.168.1.20:80 Route 1 0 0
-> 192.168.1.10:80 Route 1 0 0
|
客户端访问VIP发现负载正常
启动director_master的keepalived服务并查看网络信息发现VIP成功转移
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
|
[root@master ~]
# service keepalived start
Starting keepalived: [ OK ]
[root@master ~]
# ifconfig
eth0 Link encap:Ethernet HWaddr 00:0C:29:C2:5E:01
inet addr:192.168.1.61 Bcast:192.168.1.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fec2:5e01
/64
Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:70394 errors:0 dropped:0 overruns:0 frame:0
TX packets:118644 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:15679204 (14.9 MiB) TX bytes:8593207 (8.1 MiB)
Interrupt:67 Base address:0x2024
eth0:0 Link encap:Ethernet HWaddr 00:0C:29:C2:5E:01
inet addr:192.168.1.33 Bcast:0.0.0.0 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
Interrupt:67 Base address:0x2024
|
在director_backup查看网络信息发现VIP消失
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
|
[root@backup ~]
# ifconfig
eth0 Link encap:Ethernet HWaddr 00:0C:29:FA:52:D6
inet addr:192.168.1.62 Bcast:192.168.1.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fefa:52d6
/64
Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:118485 errors:0 dropped:0 overruns:0 frame:0
TX packets:87004 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:20112822 (19.1 MiB) TX bytes:6791097 (6.4 MiB)
Interrupt:67 Base address:0x2024
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1
/128
Scope:Host
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:6781 errors:0 dropped:0 overruns:0 frame:0
TX packets:6781 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:2122280 (2.0 MiB) TX bytes:2122280 (2.0 MiB
|