你的位置:首页 > 网海拾贝

corosync+pacemaker+http高可用操作手记

2017-01-17 浏览:(1080) 网海拾贝 评论(0)

实验规划:

node1:httpd
192.168.179.129 业务地址
192.168.107.128 心跳地址
192.168.8.111   访问地址
node2  httpd
192.168.179.130 业务地址
192.168.107.129 心跳地址
192.168.8.112 
node3  NFS-Server
192.168.179.131 业务地址
192.168.107.130 访问地址
fip(vip):192.168.8.140

准备工作:

node1 node2 node3:
yum install lrzsz wget -y
yum install ntpdate -y
[root@node1 ~]# crontab -l
*/1 * * * * /usr/sbin/ntpdate time.nist.gov >/dev/null 2>&1
[root@node2 ~]# crontab -l
*/1 * * * * /usr/sbin/ntpdate time.nist.gov >/dev/null 2>&1
[root@node3 ~]# crontab -l
*/1 * * * * /usr/sbin/ntpdate time.nist.gov >/dev/null 2>&1

hosts文件:

node1 node2 node3:
192.168.179.129 node1
192.168.179.130 node2
192.168.179.131 node3

主机互信:

node1 node2 node3:
[root@node1 ~]# ssh-keygen  -t rsa -f ~/.ssh/id_rsa  -P '' 
[root@node1 ~]# ssh-copy-id -i .ssh/id_rsa.pub root@node2
[root@node1 ~]# ssh-copy-id -i .ssh/id_rsa.pub root@node3
[root@node2 ~]# ssh-keygen  -t rsa -f ~/.ssh/id_rsa  -P ''
[root@node2 ~]# ssh-copy-id -i .ssh/id_rsa.pub root@node1
[root@node2 ~]# ssh-copy-id -i .ssh/id_rsa.pub root@node3
[root@node3 ~]# ssh-keygen  -t rsa -f ~/.ssh/id_rsa  -P ''
[root@node3 ~]# ssh-copy-id -i .ssh/id_rsa.pub root@node1
[root@node3 ~]# ssh-copy-id -i .ssh/id_rsa.pub root@node2

部署:

node1 node2:
yum install corosync* -y
yum install pacemaker* -y
yum install cluster-glue* -y

配置:

node1 node2:
[root@node1 ~]# cd /etc/corosync/
[root@node1 corosync]#  cp corosync.conf.example corosync.conf
[root@node1 corosync]# vim corosync.conf
[root@node1 ~]# cat /etc/corosync/corosync.conf
totem {
version: 2
crypto_cipher: none
crypto_hash: none
        interface {
          member {
            memberaddr: 192.168.107.128
          }
          member {
            memberaddr: 192.168.107.129
          }
          ringnumber:  0
          bindnetaddr: 192.168.107.0
          mcastport:   5405
        }
        transport: udpu
}
service {
        name: pacemaker
}
logging {
fileline: off
to_stderr: no
to_logfile: yes
logfile: /var/log/cluster/corosync.log
to_syslog: yes
debug: off
timestamp: on
logger_subsys {
subsys: QUORUM
debug: off
}
}
amf  {
        mode: disabled
}
aisexec {
         user:  root
         greoup:  root
}
quorum {
provider: corosync_votequorum
        expected_votes: 2
        two_nodes: 1
}
[root@node1 ~]#
[root@node1 ~]# scp /etc/corosync/corosync.conf root@node2:/etc/corosync/

为集群生成corosync keys并拷贝到其他节点:

node1:
[root@node1 ~]# mv /dev/{random,random.bak}
[root@node1 ~]# ln -s /dev/urandom /dev/random
[root@node1 ~]# corosync-keygen 
[root@node1 ~]# ll /etc/corosync/authkey 
[root@node1 ~]# chmod 0400 /etc/corosync/authkey 
[root@node1 ~]# scp  /etc/corosync/authkey root@node2:/etc/corosync/

启动服务:

node1 node2:
[root@node1 ~]# /etc/init.d/corosync start
Starting Corosync Cluster Engine (corosync):               [确定]
[root@node1 ~]# 
[root@node1 ~]# ssh node2 "/etc/init.d/corosync start"
Starting Corosync Cluster Engine (corosync): [确定]
[root@node1 ~]#

#crm编译安装:

[root@node1 ~]# tar xf crmsh-2.1.2.tar.gz -C /usr/local/
[root@node1 ~]# ln -s /usr/local/crmsh-2.1.2/ /usr/local/crmsh
[root@node1 crmsh]# cd /usr/local/crmsh
[root@node1 crmsh]# yum install autoconf automake -y
[root@node1 crmsh]# yum install autoconf automake -y
[root@node1 crmsh]# ./autogen.sh 
autoconf:autoconf (GNU Autoconf) 2.63
automake:automake (GNU automake) 1.11.1
aclocal
automake --add-missing --include-deps --copy
configure.ac:34: installing `./install-sh'
configure.ac:34: installing `./missing'
doc/Makefile.am:38: `%'-style pattern rules are a GNU make extension
doc/Makefile.am:40: `%'-style pattern rules are a GNU make extension
modules/Makefile.am:22: installing `./py-compile'
Makefile.am: installing `./INSTALL'
autoconf
Now run ./configure
[root@node1 crmsh]# 
[root@node1 crmsh]# yum install gcc  -y 
[root@node1 crmsh]# ./configure
[root@node1 crmsh]# make
[root@node1 crmsh]# make install

#yum 安装

[root@node1 ~]# yum install crmsh -y
[root@node1 ~]# yum install pssh -y
[root@node2~]# yum install crmsh -y
[root@node2~]# yum install pssh -y

#crm详解:

crm有两种工作方式

1,批处理模式,就是在shell命令行中直接输入命令

2,交互式模式(crm(live)#)进入到crmsh中交互执行

[root@node1 ~]# crm   
crm(live)#            # help #输入help查看一下,会出下很多子命令
--help      bye         configure   help        options     resource    up          
-h          cd          corosync    history     quit        script      
?           cib         end         ls          ra          site        
back        cluster     exit        node        report      status      
crm(live)#

一级子命令

[root@node1 corosync]# crm  
crm(live)# help  
This is crm shell, a Pacemaker command line interface.  
Available commands:  
    cib              manage shadow CIBs //cib沙盒  
    resource         resources management //所有的资源都在这个子命令后定义  
    configure        CRM cluster configuration //编辑集群配置信息  
    node             nodes management //集群节点管理子命令  
    options          user preferences //用户优先级  
    history          CRM cluster history //命令历史记录  
    site             Geo-cluster support  
    ra               resource agents information center //资源代理子命令(所有与资源代理相关的程都在此命令之下)  
    status           show cluster status //显示当前集群的状态信息  
    help,?           show help (help topics for list of topics)//查看当前区域可能的命令  
    end,cd,up        go back one level //返回第一级crm(live)  
    quit,bye,exit    exit the program //退出crm(live)交互模式

resource资源状态控制子命令

crm(live)resource# help  
vailable commands:  
        status           show status of resources //显示资源状态信息  
        start            start a resource //启动一个资源  
        stop             stop a resource //停止一个资源  
        restart          restart a resource //重启一个资源  
        promote          promote a master-slave resource //提升一个主从资源  
        demote           demote a master-slave resource //降级一个主从资源  
        manage           put a resource into managed mode  //将一个资源加入到管理模式下  
        unmanage         put a resource into unmanaged mode  //从管理模式下去除一个资源  
        migrate          migrate a resource to another node //将资源迁移到另一个节点上  
        unmigrate        unmigrate a resource to another node  
        param            manage a parameter of a resource //管理资源的参数  
        secret           manage sensitive parameters //管理敏感参数  
        meta             manage a meta attribute //管理源属性  
        utilization      manage a utilization attribute  
        failcount        manage failcounts //管理失效计数器  
        cleanup          cleanup resource status //清理资源状态  
        refresh          refresh CIB from the LRM status //从LRM(LRM本地资源管理)更新CIB(集群信息库),在  
        reprobe          probe for resources not started by the CRM //探测在CRM中没有启动的资源  
        trace            start RA tracing //启用资源代理(RA)追踪  
        untrace          stop RA tracing //禁用资源代理(RA)追踪  
        help             show help (help topics for list of topics) //显示帮助  
        end              go back one level //返回一级(crm(live)#)  
        quit             exit the program //退出交互式程序

configure资源定义子命令

crm(live)configure# help    #两次table显示子命令,输入help加你想了解的任意命令,就会显示该命令的使用帮助与案例
Available commands:  
        node             define a cluster node //定义一个集群节点  
        primitive        define a resource //定义资源  
        monitor          add monitor operation to a primitive //对一个资源添加监控选项(如超时时间,启动失败后的操作)  
        group            define a group //定义一个组类型(将多个资源整合在一起)  
        clone            define a clone //定义一个克隆类型(可以设置总的克隆数,每一个节点上可以运行几个克隆)  
        ms               define a master-slave resource //定义一个主从类型(集群内的节点只能有一个运行主资源,其它从的做备用)  
        rsc_template     define a resource template //定义一个资源模板  
        location         a location preference //定义位置约束优先级(默认运行于那一个节点(如果位置约束的值相同,默认倾向性那一个高,就在那一个节点上运行))  
        colocation       colocate resources //排列约束资源(多个资源在一起的可能性)  
        order            order resources //资源的启动的先后顺序  
        rsc_ticket       resources ticket dependency  
        property         set a cluster property //设置集群属性  
        rsc_defaults     set resource defaults //设置资源默认属性(粘性)  
        fencing_topology node fencing order //隔离节点顺序  
        role             define role access rights //定义角色的访问权限  
        user             define user access rights //定义用用户访问权限  
        op_defaults      set resource operations defaults //设置资源默认选项  
        schema           set or display current CIB RNG schema  
        show             display CIB objects //显示集群信息库对  
        edit             edit CIB objects //编辑集群信息库对象(vim模式下编辑)  
        filter           filter CIB objects //过滤CIB对象  
        delete           delete CIB objects //删除CIB对象  
        default-timeouts set timeouts for operations to minimums from the meta-data  
        rename           rename a CIB object //重命名CIB对象  
        modgroup         modify group //改变资源组  
        refresh          refresh from CIB //重新读取CIB信息  
        erase            erase the CIB //清除CIB信息  
        ptest            show cluster actions if changes were committed  
        rsctest          test resources as currently configured  
        cib              CIB shadow management  
        cibstatus        CIB status management and editing    //cib状态管理和编辑  
        template         edit and import a configuration from a template   //编辑或导入配置模板  
        commit           commit the changes to the CIB //将更改后的信息提交写入CIB  
        verify           verify the CIB with crm_verify //CIB语法验证  
        upgrade          upgrade the CIB to version 1.0 //升级CIB到1.0  
        save             save the CIB to a file //将当前CIB导出到一个文件中(导出的文件存于切换crm 之前的目录)  
        load             import the CIB from a file //从文件内容载入CIB  
        graph            generate a directed graph  
        xml              raw xml  
        help             show help (help topics for list of topics) //显示帮助信息  
        end              go back one level //回到第一级(crm(live)#)  
        quit             exit the program  //退出crm交互模式

对配置进行操作

verify、show、edit、delete、save、load、commite都是对已有的配置进行的操作。

(1)verify

用于验证刚刚完成的配置是否有效。如果出现错误,verify会给出一些有用的提示;

(2)show

格式为:

show [xml] [<id> ...]
show [xml] changed

打印出当前已有的配置。不加参数,会以与命令行相同格式的文本输出配置。加上xml参数,会以xml的格式输出配置。如果指定了资源或者属性的id,show可以输出指定部分的配置;

另外,show [xml] changed可以显示配置文件发生了那些变化。

(3)edit、delete

edit的格式为:

edit [xml] [<id> ...]
edit [xml] changed

edit可以修改已有的配置,不带参数,则是修改命令行相同格式的文本。加上xml参数,则需要修改xml格式的文本。如果指定了资源或者属性的id,就可以修改指定部分的配置。

edit [xml] changed可以对发生的变动进行编辑。

delete的格式为:

delete <id> [<id> ...]

delete用于删除指定id的资源或者属性,可以同时删除多个,id之间使用空格分隔;

(4)save、load

save的格式为:

save [xml] <file>

save可以保存当前已有的配置。

不加xml参数,save会以于命令行相同的格式将配置保存为纯文本文件;加上xml参数,则save会将配置保存为xml格式的文件。

load命令的格式为:

load [xml] method URL
method :: replace | update

load可以加载之前保存过的配置文件。提供了2种加载方法;

replace,会清空原来的配置,使用加载的配置文件;

update,会保留原来的配置,尝试将加载的配置加入当前的配置中。

同样的,不加xml参数,load会认为加载的文件使用的是命令行格式;加上xml参数,则load会以xml的方式解析加载的文件。

另外,load可以直接加载通过url地址加载远程的配置文件,如http://storage.big.com/cibs/bigcib.xml。

(5)commit

对于集群配置的修改并不是立即生效的,需要执行commite之后才能真正产生作用。某些情况下,commite时可能会提示拒绝提交刚刚修改的配置,那么肯定是配置的变化存在一些问题。如果能够确定自己的配置没有问题,可以使用commite force强制提交配置。

node节点管理子命令

crm(live)# node  
crm(live)node# help  
Node management and status commands.  
Available commands:  
    status           show nodes status as XML //以xml格式显示节点状态信息  
    show             show node //命令行格式显示节点状态信息  
    standby          put node into standby //模拟指定节点离线(standby在后面必须的FQDN)  
    online           set node online // 节点重新上线  
    maintenance      put node into maintenance mode  //将一个节点状态改为maintenance  
    ready            put node into ready mode   //将一个节点状态改为ready  
    fence            fence node //隔离节点  
    clearstate       Clear node state //清理节点状态信息  
    delete           delete node //删除 一个节点  
    attribute        manage attributes  
    utilization      manage utilization attributes  
    status-attr      manage status attributes  
    help             show help (help topics for list of topics)  
    end              go back one level  //回到上一次  
    quit             exit the program    //退出

ra资源代理子命令

crm(live)# ra  
crm(live)ra# help  
Available commands:  
        classes          list classes and providers //为资源代理分类  
        list             list RA for a class (and provider)//显示一个类别中的提供的资源  
        meta             show meta data for a RA //显示一个资源代理序的可用参数(如meta ocf:heartbeat:IPaddr2)  
        providers        show providers for a RA and a class  
        help             show help (help topics for list of topics)  
        end              go back one level  
        quit             exit the program

查看当前集群系统所支持的类型

[root@node1 ~]# crm
crm(live)# ra
crm(live)ra# classes
lsb
ocf / .isolation heartbeat pacemaker
service
stonith
crm(live)ra#

查看某种类别下的所用资源代理的列表

crm(live)ra# list lsb
auditd                  blk-availability        bmc-snmp-proxy          cman                    corosync
corosync-notifyd        crond                   exchange-bmc-os-info    haldaemon               halt
htcacheclean            httpd                   ip6tables               ipmievd                 iptables
iscsi                   iscsid                  killall                 libvirt-guests          lvm2-lvmetad
lvm2-monitor            mdmonitor               messagebus              modclusterd             multipathd
netconsole              netfs                   network                 nfs                     nfs-rdma
nfslock                 ntpdate                 oddjobd                 pacemaker               pacemaker_remote
postfix                 quota_nld               rdisc                   rdma                    restorecond
ricci                   rpcbind                 rpcgssd                 rpcidmapd               rpcsvcgssd
rsyslog                 sandbox                 saslauthd               single                  sshd
udev-post               winbind                 
crm(live)ra#

查看某个资源代理的配置方法

crm(live)ra# info ocf:heartbeat:IPaddr
crm(live)ra# list ocf heartbeat
CTDB                Delay               Dummy               Filesystem          IPaddr              IPaddr2
IPsrcaddr           LVM                 MailTo              Route               SendArp             Squid
VirtualDomain       Xinetd              apache              conntrackd          db2                 dhcpd
ethmonitor          exportfs            iSCSILogicalUnit    mysql               named               nfsnotify
nfsserver           nginx               oracle              oralsnr             pgsql               portblock
postfix             rsyncd              symlink             tomcat              
crm(live)ra#

如果想要查看某种类别下的所用资源代理的列表,可以使用类似如下命令实现:

# crm ra list lsb
# crm ra list ocf heartbeat
# crm ra list ocf pacemaker
# crm ra list stonith

eg:示例

crm(live)configure# show
node node1
node node2
property cib-bootstrap-options: \
have-watchdog=false \
dc-version=1.1.15-5.el6-e174ec8 \
cluster-infrastructure="classic openais (with plugin)" \
expected-quorum-votes=2
crm(live)configure

全局配置:

crm(live)configure# property stonith-enabled="false"
crm(live)configure# property no-quorum-policy="ignore"
crm(live)configure# rsc_defaults resource-stickiness=100

配置fip资源

crm(live)configure# primitive fip ocf:heartbeat:IPaddr params ip="192.168.179.132" cidr_netmask="255.255.255.0" nic="eth0" op monitor interval="10"
crm(live)configure# 
crm(live)configure# show  #查看已增加好的VIP
node node1
node node2
primitive fip IPaddr \
params ip=192.168.179.132 cidr_netmask=255.255.255.0 nic=eth0 \
op monitor interval=10
property cib-bootstrap-options: \
have-watchdog=false \
dc-version=1.1.15-5.el6-e174ec8 \
cluster-infrastructure="classic openais (with plugin)" \
expected-quorum-votes=2 \
stonith-enabled=false \
no-quorum-policy=ignore
rsc_defaults rsc-options: \
resource-stickiness=100
crm(live)configure# 
crm(live)configure# verify #检查一下配置文件有没有错误 
crm(live)configure# commit #提交配置的资源,在命令行配置资源时,只要不用commit提交配置好资源,就不会生效。
[root@node1 ~]# ping 192.168.8.140
PING 192.168.8.140 (192.168.8.140) 56(84) bytes of data.
64 bytes from 192.168.8.140: icmp_seq=1 ttl=64 time=0.027 ms
64 bytes from 192.168.8.140: icmp_seq=2 ttl=64 time=0.046 ms
^C
--- 192.168.8.140 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1172ms
rtt min/avg/max/mdev = 0.027/0.036/0.046/0.011 ms
[root@node1 ~]#

将此集群配置成为一个active/passive模型的web(httpd)服务集群

[root@node1 ~]# yum install httpd -y
[root@node1 ~]# echo "node1 by renzhiyuan" >/var/www/html/index.html
[root@node1 ~]# /etc/init.d/httpd start
[root@node2 ~]# yum install httpd -y
[root@node2 ~]# echo "node2 by renzhiyuan" >/var/www/html/index.html
[root@node2 ~]# /etc/init.d/httpd start
测试确保可正常访问:
http://192.168.8.111/
http://192.168.8.112/

接下来我们将此httpd服务添加为集群资源。将httpd添加为集群资源有两处资源代理可用:lsb和ocf:heartbeat,为了简单起见,我们这里使用lsb类型:

首先可以使用如下命令查看lsb类型的httpd资源的语法格式:

crm(live)ra# info lsb:httpd
start and stop Apache HTTP Server (lsb:httpd)
server implementing the current HTTP standards.
Operations' defaults (advisory minimum):
    start         timeout=15
    stop          timeout=15
    status        timeout=15
    restart       timeout=15
    force-reload  timeout=15
    monitor       timeout=15 interval=15
crm(live)ra#

接下来新建资源httpd:   

crm(live)# configure
crm(live)configure# primitive httpd lsb:httpd
crm(live)configure# show
node node1
node node2
primitive httpd lsb:httpd
primitive vip IPaddr \
params ip=192.168.8.140 nic=eth2 cidr_netmask=24
property cib-bootstrap-options: \
expected-quorum-votes=2 \
have-watchdog=false \
dc-version=1.1.15-5.el6-e174ec8 \
cluster-infrastructure="classic openais (with plugin)" \
stonith-enabled=false \
no-quorum-policy=ignore
rsc_defaults rsc-options: \
resource-stickiness=100
crm(live)configure# verify
crm(live)configure# commit
crm(live)configure#

来查看一下资源状态

crm(live)# status
Stack: classic openais (with plugin)
Current DC: node1 (version 1.1.15-5.el6-e174ec8) - partition with quorum
Last updated: Wed Jun 28 12:08:55 2017Last change: Wed Jun 28 12:08:28 2017 by root via cibadmin on node1
, 2 expected votes
2 nodes and 2 resources configured
Online: [ node1 node2 ]
Full list of resources:
 vip(ocf::heartbeat:IPaddr):Started node1
 httpd(lsb:httpd):Started node2
crm(live)#

定义组资源

crm(live)# configure
crm(live)configure# group webservice vip httpd
crm(live)configure# show
node node1
node node2
primitive httpd lsb:httpd
primitive vip IPaddr \
params ip=192.168.8.140 nic=eth2 cidr_netmask=24
group webservice vip httpd
property cib-bootstrap-options: \
expected-quorum-votes=2 \
have-watchdog=false \
dc-version=1.1.15-5.el6-e174ec8 \
cluster-infrastructure="classic openais (with plugin)" \
stonith-enabled=false \
no-quorum-policy=ignore
rsc_defaults rsc-options: \
resource-stickiness=100
crm(live)configure# verify
crm(live)configure# commit
crm(live)configure#

再次来查看一下资源状态

crm(live)# status
Stack: classic openais (with plugin)
Current DC: node1 (version 1.1.15-5.el6-e174ec8) - partition with quorum
Last updated: Wed Jun 28 12:13:16 2017Last change: Wed Jun 28 12:11:45 2017 by root via cibadmin on node1
, 2 expected votes
2 nodes and 2 resources configured
Online: [ node1 node2 ]
Full list of resources:
 Resource Group: webservice
     vip(ocf::heartbeat:IPaddr):Started node1
     httpd(lsb:httpd):Started node1
crm(live)#

访问:

http://192.168.8.140/   node1 by renzhiyuan 

故障模拟:

[root@node1 ~]# /etc/init.d/corosync stop
[root@node2 ~]# crm
crm(live)# node
crm(live)node# standby
crm(live)node# cd ..
crm(live)# cd node
crm(live)node# standby
crm(live)node# cd ..
crm(live)# status
Stack: classic openais (with plugin)
Current DC: node2 (version 1.1.15-5.el6-e174ec8) - partition with quorum
Last updated: Wed Jun 28 12:55:16 2017Last change: Wed Jun 28 12:55:12 2017 by root via crm_attribute on node1
, 2 expected votes
2 nodes and 2 resources configured
Node node1: standby
Online: [ node2 ]
Full list of resources:
 Resource Group: webservice
     vip(ocf::heartbeat:IPaddr):Started node2
     httpd(lsb:httpd):Started node2
crm(live)#

访问:

http://192.168.8.140/  node2 by renzhiyuan

文章来源:http://blog.51cto.com/renzhiyuan/1942676

  • 发表评论
  • 查看评论
【暂无评论!】

发表评论:

◎欢迎参与讨论,请在这里发表您的看法、交流您的观点。