rsync3.1.3的编译安装和常用操作

1.rsync的编译安装

tar -zxf rsync-3.1.3.tar.gz
cd rsync-3.1.3
./configure --prefix=/usr/local/rsync-3.1.3 --disable-ipv6

2.rsync的配置文件：
[root@web3:~]# more /usr/local/rsync/etc/rsyncd.conf
uid = apache
gid = users
port = 873
use chroot = no
#hosts allow = 192.168.9.3
#hosts deny =  192.168.10.0/24
pid file = /usr/local/rsync/rsyncd.pid
lock file = /usr/local/rsync/rsync.lock
#log file = /usr/local/rsync/logs/rsyncd.log

[apache]
comment=all web
path=/
read only=no

secrets file=/usr/local/rsync/etc/rsyncd.passwd
auth users=apache

密码文件：
[root@web3:~]# cat /usr/local/rsync/etc/rsyncd.passwd
apache:apachepass

3.rsync的启动脚本
[root@web3:~]# cat /etc/init.d/rsync 
#!/bin/bash 
 
#this script for start|stop rsync daemon service 
#date:2012/2/13 
 
status1=$(ps -ef | egrep "rsync --daemon.*rsyncd.conf" | grep -v 'grep') 
#pidfile="/etc/rsync/rsyncd.pid"
pidfile="/usr/local/rsync/rsyncd.pid"
#start_rsync="rsync --daemon --config=/etc/rsync/rsyncd.conf" 
 start_rsync="rsync --daemon --config=/usr/local/rsync/etc/rsyncd.conf" 
function rsyncstart() { 
 
    if [ "${status1}X" == "X" ];then 
 
        rm -f $pidfile       
 
        ${start_rsync}   
 
        status2=$(ps -ef | egrep "rsync --daemon.*rsyncd.conf" | grep -v 'grep') 
         
        if [  "${status2}X" != "X"  ];then 
             
            echo "rsync service start.......OK" 
             
        fi 
 
    else 
 
        echo "rsync service is running !"    
 
    fi 
} 
 
function rsyncstop() { 
 
    if [ "${status1}X" != "X" ];then 
     
        kill -9 $(cat $pidfile) 
 
        status2=$(ps -ef | egrep "rsync --daemon.*rsyncd.conf" | grep -v 'grep') 
 
        if [ "${statusw2}X" == "X" ];then 
             
            echo "rsync service stop.......OK" 
        fi 
    else 
 
        echo "rsync service is not running !"    
 
    fi 
} 
 
 
function rsyncstatus() { 
 
 
    if [ "${status1}X" != "X" ];then 
 
        echo "rsync service is running !"   
     
    else 
 
         echo "rsync service is not running !"  
 
    fi 
 
} 
 
function rsyncrestart() { 
 
    if [ "${status1}X" == "X" ];then 
 
               echo "rsync service is not running..." 
 
               rsyncstart 
        else 
 
               rsyncstop 
 
               rsyncstart    
 
        fi       
}  
 
case $1 in 
 
        "start") 
               rsyncstart 
                ;; 
 
        "stop") 
               rsyncstop 
                ;; 
 
        "status") 
               rsyncstatus 
               ;; 
 
        "restart") 
               rsyncrestart 
               ;; 
 
        *) 
          echo 
                echo  "Usage: $0 start|stop|restart|status" 
          echo 
esac


# 自动启动rsync服务脚本
[root@web3:~]# cat /usr/local/zabbix_agents_3.2.0/scripts/start_rsy.sh 
#!/bin/bash
count_num=`ps -ef|grep 'rsync --daemon'|grep -v grep|wc -l`
echo $count_num
rm -f /usr/local/rsync/rsyncd.pid
if [ $count_num -eq 0 ];then
/etc/init.d/rsync start
fi

###########使用rsync同步脚本的示例

# vim /usr/local/worksh/jenkins_task/rsync_cbs_chinasoft_com.sh

#!/bin/bash

##############jenkins拉取git代码同步到web3上，然后触发web3上的该脚本进行 ws/eus/eop 三个机房同步####################
###### rsync cbs.chinasoft.com data ######################################
passwd="/data/www/.rsync/rsyncd.pass"

# ws_cbs_ip
ws_cbs_ip_list="192.168.254.1 192.168.254.2"

# eus_cbs_ip
eus_cbs_ip_list="1.1.1.1"

# eop_cbs_ip
eop_cbs_ip_list="2.2.2.2"

#############################################################

# 同步所有机房
function cbs_rsync_all()
{
# rsync ws_cbs
for ws_cbs_ip in $ws_cbs_ip_list
do    
    echo "####################rsync ws_cbs start################################"
        echo $ws_cbs_ip
    rsync -zavP --delete --exclude=logs/ --exclude=cache/ --password-file=$passwd /data/cbs.chinasoft.com/ apache@$ws_cbs_ip::apache/data/cbs.chinasoft.com/
　　  # 添加如下对执行结果的判断，如果执行失败就退出程序，这样可以避免当代码要求强一致的时候，有些没有同步到也报success

　　　　if [[ $? == 0 || $? == 23 ]];then
　　　　　　rsync_edit=1
　　　  else
　　　　　　rsync_edit=0
　　　　　　echo "`date` rsync发布失败! -> editUrls.txt"
　　　　　　exit 1
　　　　fi

    echo "################### rsync ws_cbs end #######################"
done


# rsync eus_cbs
for eus_cbs_ip in $eus_cbs_ip_list
do    
    echo "####################rsync eus_cbs start################################"
        echo $eus_cbs_ip
    rsync -zavP --delete --exclude=logs/ --exclude=cache/ --password-file=$passwd /data/cbs.chinasoft.com/ apache@$eus_cbs_ip::apache/data/cbs.chinasoft.com/
    echo "################### rsync eus_cbs end #######################"
done

# rsync eop_cbs
for eop_cbs_ip in $eop_cbs_ip_list
do    
    echo "####################rsync eop_cbs start################################"
        echo $eop_cbs_ip
    rsync -zavP --delete --exclude=logs/ --exclude=cache/ --password-file=$passwd /data/cbs.chinasoft.com/ apache@$eop_cbs_ip::apache/data/cbs.chinasoft.com/
    echo "################### rsync eop_cbs end #######################"
done

}

# 只同步网宿机房
function cbs_rsync_ws()
{
# rsync ws_cbs
for ws_cbs_ip in $ws_cbs_ip_list
do    
    echo "####################rsync ws_cbs start################################"
        echo $ws_cbs_ip
    rsync -zavP --delete --exclude=logs/ --exclude=cache/ --password-file=$passwd /data/cbs.chinasoft.com/ apache@$ws_cbs_ip::apache/data/cbs.chinasoft.com/
    echo "################### rsync ws_cbs end #######################"
done
}


# 只同步美西机房
function cbs_rsync_eus()
{
# rsync eus_cbs
for eus_cbs_ip in $eus_cbs_ip_list
do    
    echo "####################rsync eus_cbs start################################"
        echo $eus_cbs_ip
    rsync -zavP --delete --exclude=logs/ --exclude=cache/ --password-file=$passwd /data/cbs.chinasoft.com/ apache@$eus_cbs_ip::apache/data/cbs.chinasoft.com/
    echo "################### rsync eus_cbs end #######################"
done
}


# 只同步欧洲德国机房
function cbs_rsync_eop()
{
# rsync eop_cbs
for eop_cbs_ip in $eop_cbs_ip_list
do    
    echo "####################rsync eop_cbs start################################"
        echo $eop_cbs_ip
    rsync -zavP --delete --exclude=logs/ --exclude=cache/ --password-file=$passwd /data/cbs.chinasoft.com/ apache@$eop_cbs_ip::apache/data/cbs.chinasoft.com/
    echo "################### rsync eop_cbs end #######################"
done
}


#####################  MAIN  ###############################
usage () {
        echo ""
        echo "  Please Input server infomation!"
        echo ""
        echo "  USAGE: `basename $0` [all|cbs_ws|cbs_eus|cbs_eop]"
        echo ""
}
    
if [ $# != 1 ]
then
        usage >&2
        exit 1
fi
OPT=$1
case $OPT in
all)
        echo "start rsync `basename $0` to all cbs servers"
        cbs_rsync_all
        ;;
cbs_ws)
        echo "start rsync `basename $0` to ws_cbs_servers"
        cbs_rsync_ws
    ;;
cbs_eus)
        echo "start rsync `basename $0` to eus_cbs_servers"
    cbs_rsync_eus
        ;;
cbs_eop)
        echo "start rsync `basename $0` to eop_cbs_servers"
        cbs_rsync_eop
        ;;

*)
        echo "Usage:`basename $0` [all|cbs_ws|cbs_eus|cbs_eop]"
        ;;
esac

#########脚本使用示例
./rsync_cbs_chinasoft_com.sh all # 同步所有机房
./rsync_cbs_chinasoft_com.sh cbs_eus # 同步美西机房


常用示例：
1.推送本地文件目录 /data/cbs.chinasoft.com 到远程服务器的 /data/cbs.chinasoft.com/ 目录：

rsync -zavP --delete --exclude=logs/ --exclude=cache/ --password-file=$passwd /data/cbs.chinasoft.com/ apache@192.168.3.10::apache/data/cbs.chinasoft.com/

2.主动拉取远程服务器的目录
[root@localhost 2018-11]# cat /opt/platform/rsync_datacollection_new_wuchao.sh
#!/bin/bash
SIP="192.168.10.207"

ps -ef | grep "${SIP}::apache/opt/platform/downloader/all" | grep -v grep | awk '{print $2}' | xargs kill -9 
ps -ef | grep "${SIP}::apache/opt/platform/datacollection/all" | grep -v grep | awk '{print $2}' | xargs kill -9 

DATE_1="`date -d '-1 day' +%Y-%m`"
DATE0="`date -d '0 day' +%Y-%m`"

for a in `seq -2 0`
do

DATE="`date -d "${a} day" +%Y-%m`"
# 拉取远程服务器 192.168.10.207 上 的 /opt_log/platform_logs/datacollection/all/ 目录到本地的 /opt/platform/datacollection/all/ 目录
/usr/bin/rsync -avrp -P --password-file=/etc/rsnyc_for_logs_by_Wanjie.scrt apache@${SIP}::apache/opt_log/platform_logs/datacollection/all/$DATE /opt/platform/datacollection/all/
/usr/bin/rsync -avrp -P --password-file=/etc/rsnyc_for_logs_by_Wanjie.scrt apache@${SIP}::apache/opt_log/platform_logs/downloader/all/$DATE /opt/platform/downloader/all/



#########################




cms_back主动拉取cms远程服务器的db数据进行备份
[root@localhost 2018-11]# cat /opt/cms_dbback.sh
#!/bin/bash
SIP="192.168.1.3"

ps -ef | grep "${SIP}::apache/data/MySQL_Data_Backup" | grep -v grep | awk '{print $2}' | xargs kill -9 

# 拉取远程服务器cms 192.168.1.3 上 的mysql数据库备份 /data/MySQL_Data_Backup 目录到本地的 /data/cmsdb_back/ 目录
/usr/local/rsync/bin/rsync -avrp -P --password-file=/opt/cmsdbback.pass apache@${SIP}::apache/data/MySQL_Data_Backup /data/cmsdb_back/

问题分析：

同步多次都不能把本地文件和远端文件保持一致

#!/bin/bash
#############################################

# uat和pro获取代码源头都是一个相同目录:  /data/www/vhosts/u.chinasoft.com/httpdocs/
# uat只发布一台服务器进行验证，生产环境发布两台

chinasoft_iplist="1.1.1.1 1.1.1.2"

function chinasoft_eus_rsync()
{
for ip in $chinasoft_iplist
do
        echo "-- start pub --- 预发布到外网 ${ip} ----------------------------------------"
    #/usr/local/bin/rsync -vaRzP --bwlimit=1000 --exclude='.env' --exclude='runtime/' --exclude='config.py' --exclude='docs/' --password-file=/data/www/.rsync/rsyncd.pass /data/www/vhosts/u.chinasoft.com/httpdocs/ apache@$ip::apache/data/www/vhosts/u.chinasoft.com/httpdocs/
    /usr/local/bin/rsync -zavP --bwlimit=1000 --exclude='.env' --exclude='runtime/' --exclude='config.py' --exclude='docs/' --password-file=/data/www/.rsync/rsyncd.pass /data/www/vhosts/u.chinasoft.com/httpdocs/ apache@$ip::apache/data/www/vhosts/u.chinasoft.com/httpdocs/
    if [[ $? == 0 || $? == 23 ]];then
        rsync_edit=1
    else
        rsync_edit=0
        echo "`date` rsync发布失败! -> editUrls.txt"
        exit 1
    fi
        echo -e "-- end pub rsync ${ip} ----------------------------------------------------------\n\n"

    done


echo "-- end pub all rsync success----------------------------------------------------------"
exit 0

}

# 执行同步
chinasoft_eus_rsync


# 现象：同步后始终有文件不同，多次同步亦然


# rsync -vaRzP

更换为如下即可

rsync -vazP

 

 

限制根目录的配置示例：

[root@sz_publish01:~]# cat /usr/local/rsync/etc/rsyncd.conf
uid = apache
gid = users
port = 873
use chroot = no
#hosts allow = 192.168.9.3
#hosts deny =  192.168.10.0/24
pid file = /usr/local/rsync/rsyncd.pid
lock file = /usr/local/rsync/rsync.lock
#log file = /usr/local/rsync/logs/rsyncd.log
reverse lookup = no

[apache_project]
comment=all web
path=/data/www/vhosts/
read only=no
secrets file=/usr/local/rsync/etc/rsyncd.passwd
auth users=apache

[apache]
comment=all web
path=/
read only=no
secrets file=/usr/local/rsync/etc/rsyncd.haiwai
auth users=apache


# 同步示例
function mms_rsync_all()
{
# rsync huadong
for huadong_ip in $huadong_mms_web_ip_list
do    
    echo "####################rsync huadong start################################"
        echo $huadong_ip
        rsync -zavP --delete $exclude_list  --password-file=$passwd /data/www/vhosts/mms_task/ apache@${huadong_ip}::apache_project/mms_task/
        #调用初始化脚本
        run_init_sh ${huadong_ip} 2025
    echo "################### rsync huadong end #######################"
done
}

 按照模块授权，每个模块一个密码，这样可以避免因误操作删除整个根目录文件

# 新的推送规则
/usr/local/rsync/bin/rsync -zavP --password-file="/data/www/.rsync/rsyncd.szpublish" /data/www/vhosts/apiservice.chinasoft.com/httpdocs/ apache@172.18.54.140::apache_apiservice_chinasoft_com
# 注意密码文件格式

# cat /usr/local/rsync/etc/rsyncd.apiservice.chinasoft.com

apache:pass
# 一定要修改客户端和服务端密码文件权限，否则无法正常同步
# chmod 600 /usr/local/rsync/etc/rsyncd.apiservice.chinasoft.com


[root@sz_publish01:/data/www/vhosts/apiservice.chinasoft.com]# more /usr/local/rsync/etc/rsyncd.conf 

uid = apache
gid = users
port = 873
use chroot = no
#hosts allow = 192.168.9.3
#hosts deny =  192.168.10.0/24
pid file = /usr/local/rsync/rsyncd.pid
lock file = /usr/local/rsync/rsync.lock
log file = /usr/local/rsync/logs/rsyncd.log
reverse lookup = no

# 后续拿掉
[apache_project]
comment=all web
path=/data/www/vhosts/
read only=no
secrets file=/usr/local/rsync/etc/rsyncd.passwd
auth users=apache


# 1.购物车中文服务项目
[apache_www-api_chinasoft_com]
comment=www-api.chinasoft.com
path=/data/www/vhosts/www-api.chinasoft.com/httpdocs/
read only=no
#log file = /usr/local/rsync/logs/rsyncd_www-api.chinasoft.com.log
secrets file=/usr/local/rsync/etc/rsyncd.www-api.chinasoft.com
auth users=apache


# 2.购物车中文服务项目
[apache_product-api_chinasoft_com]
comment=www-product-api.chinasoft.com
path=/data/www/vhosts/product-api.chinasoft.com/httpdocs/
read only=no
secrets file=/usr/local/rsync/etc/rsyncd.product-api.chinasoft.com
auth users=apache

# 3.购物车中文服务项目
[apache_apiservice_chinasoft_com]
comment=apiservice.chinasoft.com
path=/data/www/vhosts/apiservice.chinasoft.com/httpdocs/
read only=no
secrets file=/usr/local/rsync/etc/rsyncd.apiservice.chinasoft.com
auth users=apache

# 4.购物车中文服务项目
[apache_product-service_chinasoft_com]
comment=product-service.chinasoft.com
path=/data/www/vhosts/product-service.chinasoft.com/httpdocs/
read only=no
secrets file=/usr/local/rsync/etc/rsyncd.product-service.chinasoft.com
auth users=apache


# N.此处为注释
[apache]
comment=all web
path=/
read only=no
secrets file=/usr/local/rsync/etc/rsyncd.haiwai
auth users=apache




[apache@sz_publish01:/usr/local/worksh/jenkins_rsync]$ more alisz_rsync_apiservice.chinasoft.com.sh 
#!/bin/bash

##############公司jenkins拉取git代码同步到 sz_publish01(1.1.1.1)  上，然后触发 sz_publish01 上的该脚本同步到web服务器####################
#1. rsync apiservice.chinasoft.com 
#2. 初始化脚本  by pengll  2020.04.03

## 定义排除的目录，线上一般排除logs文件夹
passwd="/data/www/.rsync/rsyncd.pass.apiservice.chinasoft.com"

exclude_list="--exclude=logs/"


# 尽量使用IP地址，多个IP用空格隔开
hnsz_ip_list="192.168.1.10 192.168.1.11"


# 发布单台机器
hnsz_ip_list_web01="192.168.1.10"
hnsz_ip_list_web02="192.168.1.11"


# 代码发布目录，一般为/data/www/vhosts/${project_dir}
project_dir="apiservice.chinasoft.com/httpdocs"

# 同步所有机房,目前只有华南深圳一个机房
function rsync_hnsz_all()
{

for ip in ${hnsz_ip_list}
do      
        echo "#################### Run rsync ${ip}  start ################################"
        rsync -zavP --delete $exclude_list  --password-file=$passwd /data/www/vhosts/${project_dir}/ apache@${ip}::apache/data/www/vhosts/${project_dir}/
        echo "################### Run rsync ${ip} end #######################"
done
}


# 同步华南深圳机房的第1台web机器
function rsync_hnsz_web1()
{

for ip in ${hnsz_ip_list_web01}
do
        echo "#################### Run rsync ${ip}  start ################################"
        rsync -zavP --delete $exclude_list  --password-file=$passwd /data/www/vhosts/${project_dir}/ apache@${ip}::apache/data/www/vhosts/${project_dir}/
        echo "################### Run rsync ${ip} end #######################"
done
}


# 同步华南深圳机房的第2台web机器
function rsync_hnsz_web2()
{

for ip in ${hnsz_ip_list_web02}
do
        echo "#################### Run rsync ${ip}  start ################################"
        rsync -zavP --delete $exclude_list  --password-file=$passwd /data/www/vhosts/${project_dir}/ apache@${ip}::apache/data/www/vhosts/${project_dir}/
        echo "################### Run rsync ${ip} end #######################"
done
}



#####################  MAIN  ###############################
usage () {
        echo ""
        echo "  Please Input server infomation!"
        echo ""
        echo "  USAGE: `basename $0` [all|apiservice_web01|apiservice_web02]"
        echo ""
}
        
if [ $# != 1 ]
then
        usage >&2
        exit 1
fi
OPT=$1
case $OPT in
all)
        echo "start rsync ${project_dir}  to ${hnsz_ip_list} servers"
        rsync_hnsz_all
        echo "rsync ${project_dir} to ${hnsz_ip_list} Done"
        ;;

apiservice_web01)
        echo "start rsync ${project_dir}  to ${hnsz_ip_list_web01} servers"
        rsync_hnsz_web1
        ;;
apiservice_web02)
        echo "start rsync ${project_dir}  to ${hnsz_ip_list_web02} servers"
        rsync_hnsz_web2
        ;;

*)
        echo "Usage:`basename $0` [all|apiservice_web01|apiservice_web02]"
        ;;
esac

 

rsync同步时报错：

------------ 域名:images.chinasoft.fr 服务器:192.168.0.1 动作:同步 ------------
sending incremental file list
cannot delete non-empty directory: images.chinasoft.fr/httpdocs/images/multimedia/dvd-ripper-platinum
cannot delete non-empty directory: images.chinasoft.fr/httpdocs/images/multimedia/dvd-ripper-platinum
cannot delete non-empty directory: images.chinasoft.fr/httpdocs/images/mobile/iphone-ringtone-maker
cannot delete non-empty directory: images.chinasoft.fr/httpdocs/images/mobile/iphone-ringtone-maker
cannot delete non-empty directory: images.chinasoft.fr/httpdocs/images/mobile

/usr/bin/rsync -avzP --password-file=/data/www/.rsync/rsyncd.pass --delete $exclude --exclude=logs/* --exclude=cache/* --exclude=.svn /data/www/vhosts/${domain} apache@$ip::apache/data/www/vhosts/

# 分析：发现是部分目录比如.svn 被排除了，实际上很多目录下是有这个文件的，需要加上这个参数即可 --delete-excluded
/usr/bin/rsync -avzP --password-file=/data/www/.rsync/rsyncd.pass --delete --delete-excluded $exclude --exclude=resource.chinasoft.com --exclude=logs/* --exclude=cache/* --exclude=.svn /data/www/vhosts/${domain} apache@$ip::apache/data/www/vhosts/