FastDFS 双tracker负载均衡 及多组存储配置

http://www.cnblogs.com/cpy-devops/p/6105845.html

应用场景: 三台服务器分别做三组存储，并且需要两台tracker地址做主备关系，当一台down机后需要另外一台可以提供正常的访问连接

#注:此处不考虑存储数据安全性，如果需要保障数据不丢失，可以分别每台存储配置两组group(相同group之间数据会自动备份) ，三台服务器group分别为:

      第一台服务器:group1  group2

      第二台服务器:group1 group3

      第三台服务器:group2 group3

 

需求图如下:

 

一: 为三台服务器分别部署FASTDFS + Nginx-fastdfs模块

    1. 安装所需依赖：

 1 1. 安装libfastcommon
 2 cd /opt
 3 yum install git xz gcc g++ gcc-c++ autoconf automake make unzip  -y
 4 git clone https://github.com/happyfish100/libfastcommon.git
 5 cd libfastcommon/
 6 ./make.sh
 7 ./make.sh install
 8 
 9 #设置环境变量或创建软链接
10 export LD_LIBRARY_PATH=/usr/lib64/
11 ln -s /usr/lib64/libfastcommon.so /usr/local/lib/libfastcommon.so
12 
13 2.  安装libevent
14 cd /opt
15 wget https://github.com/downloads/libevent/libevent/libevent-2.0.21-stable.tar.gz
16 tar zxvf libevent-2.0.21-stable.tar.gz
17 cd libevent-2.0.21-stable
18 ./configure --prefix=/usr/local/
19 make clean
20 make && make install
21 ls –al /usr/lib | grep libevent(或ls –al /usr/local/lib | grep libevent)
22 #如果有返回结果则安装正常
23 
24 ln -s /usr/local/lib/libevent-2.0.so.5 /usr/lib/libevent-2.0.so.5
25 ln -s /usr/local/lib/libevent-2.0.so.5 /usr/lib64/libevent-2.0.so.5

    2. 安装配置FastDFS

 1 1.下载安装fastdfs
 2 https://sourceforge.net/projects/fastdfs/files/?source=navbar（该网址下载所需版本FastDFS 此处使用4.0.6为例）
 3 cd /opt
 4 tar -zxvf FastDFS_4.06.tar.gz
 5 cd FastDFS
 6 ./make.sh
 7 ./make.sh install
 8 
 9 2. 创建文件存放路径
10 mkdir -p /home/fastdfs/tracker   #创建tracker文件存放路径
11 mkdir -p /home/fastdfs/storage    #创建storage 文件存放路径
12 mkdir -p /home/fastdfs/client    #创建client 文件存放路径
13 
14 cd /etc/fdfs/    #查看目录下文件是否齐全，应该有client.conf http.conf mime.types     storage.conf tracker.conf  5个文件，如果没有，libevent 安装有问题
15 
16 3. 修改配置文件（172.18.4.217-218 由于每台均作为tracker 所以需要全部配置，而172.18.4.219只作为存储服务器所以不需要配置tracker.conf ；每台服务器配置内容如组名，存放路径等按真实情况填入 ）
17 
18 vim /etc/fdfs/tracker.conf #编辑tracker配置文件
19 disabled=false #启用配置文件
20 bind_addr=     #解析主机地址，空则解析所有主机
21 port=22122     #tracker服务端口
22 connect_timeout=30   #连接超时时间30S
23 network_timeout=60   #网络超时时间60S    
24 base_path=/usr/share/fastdfs/tracker   #tracker基础数据存储路径及日志存放路径
25 max_connections=256  #最大并发连接数
26 work_threads=4         #工作线程数，最好和cpu核数保持一致
27 store_lookup=0         #选择上传文件模式 0代表group轮询 1指定特定group 2选择空间最大的group
28 #store_group=        #上传文件组，如果模式为1，则必须设置成核特定group一致的组名
29 store_server=0        #选择存储服务器上传文件 0代表轮询，1根据通过IP第的顺序 2通过优先级
30 store_path=0        #选择哪块存储盘上传文件 0代表轮询，2代表优先最大存储空间盘(路径)
31 download_server=0    #选择哪台存储服务器下载文件0代表轮询，1代表当前文件上传的源服务器
32 reserved_storage_space = 10% #系统保留存储空间10%
33 ######其余都默认就好
34 
35 vim /etc/fdfs/storage.conf #编辑存储配置文件
36 group_name=group1   #存储组名
37 client_bind=true          #当连接其他服务器时解析该主机地址
38 port=23000                #storage端口 23000
39 base_path=/usr/share/fastdfs/storage  #基础存储数据和日志文件
40 store_path0=/usr/share/fastdfs/storage #group 所占用的目录或硬盘，有几个写几个
41 tracker_server=172.18.4.217:22122  #指定tracker1服务器
42 tracker_server=172.18.4.218:22122  #指定tracker2服务器
43 
44 
45 vim /etc/fdfs/client.conf
46 base_path=/usr/share/fastdfs/client   #基础数据和日志文件
47 tracker_server=172.18.4.217:22122   #tracker1服务器
48 tracker_server=172.18.4.218:22122   #tracker2服务器    

 

   

　　3. 下载安装nginx 及 nginx-fastdfs模块

 1 1. 下载安装配置fastdfs-nginx   
 2 https://sourceforge.net/projects/fastdfs/files/?source=navbar 下载nginx-fastdfs模块压缩包
 3 cd /opt/
 4 tar zxvf fastdfs-nginx-module-v1.16.tar.gz
 5 cd fastdfs-nginx-module/src
 6 cp mod_fastdfs.conf /etc/fdfs/ #将mod_fastdfs.conf 拷贝到fdfs下进行配置
 7 vim  /etc/fdfs/mod_fastdfs.conf
 8 base_path=/home/fastdfs/storage  #修改成和storage 存放路径一致
 9 tracker_server=172.12.4.201:22122    #配置成tracker server 地址和端口
10 url_have_group_name = true #url 中是否包组名
11 store_path0=/home/fastdfs/storage #文件存放路径，与storage 一致
12 group_count = 0 #0代表单组，非零代表多组，一般设置几就为几组
13 group_count = 1
14 [group1]
15 group_name=group1
16 storage_server_port=23000
17 store_path_count=1
18 store_path0=/usr/share/fastdfs/storage
19 
20 2. 下载所需依赖库
21 https://sourceforge.net/projects/pcre/files/ #下载所需PCRE
22 cd /opt
23 tar xf pcre-8.12.tar.gz
24 https://sourceforge.net/projects/zlib/ #下载zlib
25 tar xf zlib-1.2.8.tar.xz
26 
27 
28 3. 下载安装nginx
29 wget http://nginx.org/download/nginx-1.8.1.tar.gz  #下载nginx
30 tar -zxvf ngin-1.8.1.tar.gz
31 cd nginx-1.8.1
32 
33 ./configure --prefix=/usr/local/nginx --with-zlib=/opt/zlib-1.2.8     --with-pcre=/opt/pcre-8.12 --add-module=/opt/fastdfs-nginx-module/src
34 make && make install

　　4. 配置nginx

　　172.18.4.217　　

 1 172.18.4.217 nginx配置
 2 
 3 vim /usr/local/nginx/conf/nginx.conf
 4 user root;  
 5 worker_processes 4;  
 6 events {  
 7     worker_connections 40960;
 8     use epoll;
 9 }  
10 error_log /usr/local/nginx/logs/error.log;  
11 pid /usr/local/nginx/logs/nginx.pid;  
12   
13 http {  
14     server_names_hash_bucket_size 128;  
15     client_header_buffer_size 32k;  
16     large_client_header_buffers 4 32k;  
17     client_max_body_size 300m;  
18     sendfile on;  
19     tcp_nopush on;  
20     proxy_redirect off;  
21     proxy_set_header Host $http_host;  
22     proxy_set_header X-Real-IP $remote_addr;  
23     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;  
24     proxy_connect_timeout 90;  
25     proxy_send_timeout 90;  
26     proxy_read_timeout 90;  
27     proxy_buffer_size 16k;  
28     proxy_buffers 4 64k;  
29     proxy_busy_buffers_size 128k;  
30     proxy_temp_file_write_size 128k;  
31     
32     log_format main '$remote_addr - $remote_user [$time_local] "$request" '  
33                     '$status $body_bytes_sent "$http_referer" '  
34                     '"$http_user_agent" "$http_x_forwarded_for"';  
35     
36     access_log /usr/local/nginx/logs/access.log main;  
37     
38 
39     upstream fdfs_group1 {
40         server 172.18.4.217:8080 weight=1 max_fails=2 fail_timeout=30s;
41     }  
42     
43     upstream fdfs_group2 {
44         server 172.18.4.218:8080 weight=1 max_fails=2 fail_timeout=30s;
45     }  
46     
47     upstream fdfs_group3 {
48         server 172.18.4.219:8080 weight=1 max_fails=2 fail_timeout=30s;
49     
50     }
51     include /usr/local/nginx/conf.d/*.conf;  
52 }  

 1 172.18.4.217 tracker.conf 配置
 2 vim /usr/local/nginx/conf.d/tracker.conf
 3 
 4 server {  
 5     listen 80;
 6     server_name 172.18.4.217;  
 7     
 8     location ~ /group1/M00 {
 9     proxy_next_upstream http_502 http_504 error timeout invalid_header;  
10     proxy_pass http://fdfs_group1;  
11     expires 30d;  
12     }  
13     
14     location ~ /group2/M00 {
15     proxy_next_upstream http_502 http_504 error timeout invalid_header;  
16     proxy_pass http://fdfs_group2;  
17     expires 30d;  
18     }  
19     
20     location ~ /group3/M00 {
21     proxy_next_upstream http_502 http_504 error timeout invalid_header;  
22     proxy_pass http://fdfs_group3;  
23     expires 30d;  
24     }  
25 } 

 1 172.18.4.217 storage.conf配置
 2 server {
 3     listen    8080;
 4     server_name    172.18.4.217;
 5     
 6     location ~ /group1/M00 {
 7         root    /usr/share/fastdfs/storage/data;
 8         index    index.html index.htm;
 9         ngx_fastdfs_module;
10     }
11 }

 

　　　　172.18.4.218

 1 vim /usr/local/nginx/conf/nginx.conf
 2 
 3 user root;  
 4 worker_processes 4;  
 5 events {  
 6     worker_connections 40960;
 7     use epoll;
 8 }  
 9 error_log /usr/local/nginx/logs/error.log;  
10 pid /usr/local/nginx/logs/nginx.pid;  
11   
12 http {  
13     server_names_hash_bucket_size 128;  
14     client_header_buffer_size 32k;  
15     large_client_header_buffers 4 32k;  
16     client_max_body_size 300m;  
17     sendfile on;  
18     tcp_nopush on;  
19     proxy_redirect off;  
20     proxy_set_header Host $http_host;  
21     proxy_set_header X-Real-IP $remote_addr;  
22     proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;  
23     proxy_connect_timeout 90;  
24     proxy_send_timeout 90;  
25     proxy_read_timeout 90;  
26     proxy_buffer_size 16k;  
27     proxy_buffers 4 64k;  
28     proxy_busy_buffers_size 128k;  
29     proxy_temp_file_write_size 128k;  
30     
31     log_format main '$remote_addr - $remote_user [$time_local] "$request" '  
32                     '$status $body_bytes_sent "$http_referer" '  
33                     '"$http_user_agent" "$http_x_forwarded_for"';  
34     
35     access_log /usr/local/nginx/logs/access.log main;  
36     
37 
38     upstream fdfs_group1 {
39         server 172.18.4.217:8080 weight=1 max_fails=2 fail_timeout=30s;
40     }  
41     
42     upstream fdfs_group2 {
43         server 172.18.4.218:8080 weight=1 max_fails=2 fail_timeout=30s;
44     }  
45     
46     upstream fdfs_group3 {
47         server 172.18.4.219:8080 weight=1 max_fails=2 fail_timeout=30s;
48     
49     }
50     include /usr/local/nginx/conf.d/*.conf;  
51 }

 1 vim /usr/local/nginx/conf.d/tracker.conf
 2 
 3 server {
 4         listen 80;
 5         server_name 172.18.4.218;
 6 
 7         location ~ /group1/M00 {
 8         proxy_next_upstream http_502 http_504 error timeout invalid_header;
 9         proxy_pass http://fdfs_group1;
10         expires 30d;
11         }
12 
13         location ~ /group2/M00 {
14         proxy_next_upstream http_502 http_504 error timeout invalid_header;
15         proxy_pass http://fdfs_group2;
16         expires 30d;
17         }
18 
19         location ~ /group3/M00 {
20         proxy_next_upstream http_502 http_504 error timeout invalid_header;
21         proxy_pass http://fdfs_group3;
22         expires 30d;
23         }
24 }

 1 vim /usr/local/nginx/conf.d/storage.conf
 2 
 3 server {
 4         listen  8080;
 5         server_name     172.18.4.218;
 6 
 7         location ~ /group2/M00 {
 8                 root    /usr/share/fastdfs/storage/data;
 9                 index   index.html index.htm;
10                 ngx_fastdfs_module;
11         }
12 }

　　　　172.18.4.219

 1 vim /usr/local/nginx.conf
 2 user  root;
 3 worker_processes  4;
 4 
 5 events {
 6     worker_connections  40960;
 7 }
 8 
 9 
10 http {
11     include     mime.types;
12     default_type        application/octet-stream;
13 
14 
15     sendfile    on;
16     keepalive_timeout   65;
17 
18     server {
19         listen  8080;
20         server_name     172.18.4.219;
21 
22         location / {
23             root        html;
24             index       index.html index.htm;
25         }
26 
27         location ~ /group3/M00 {
28             root        /usr/share/fastdfs/storage/data;
29             index       index.html index.htm;
30             ngx_fastdfs_module;
31         }
32         error_page   500 502 503 504  /50x.html;
33         location = /50x.html {
34             root        html;
35         }
36     }
37 }

#配置完成，接下来启动服务并测试：

172.18.4.217-218 三个进程全启动，172.18.4.219 只需要启动storage 和nginx 即可

usr/local/bin/fdfs_trackerd /etc/fdfs/tracker.conf

/usr/local/bin/fdfs_storaged /etc/fdfs/storage.conf

/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf

检查是否启动成功：

ps aux |grep fdfs

netstat -ntpl |grep fdfs

ps aux |grep nginx

netstat -ngpl |grep nginx

 

fdfs_monitor /etc/fdfs/client.conf 查看状态是否正常

1. 编辑3个文件: echo “1111” >>/opt/1.txt && echo “2222” >>/opt/2.txt && echo “3333” >>/opt/3.txt
2. 上传文件:

　　　　　　/usr/local/bin/fdfs_test /etc/fdfs/client.conf upload 1.txt 
　　　　　　/usr/local/bin/fdfs_test /etc/fdfs/client.conf upload 2.txt 
　　　　　　/usr/local/bin/fdfs_test /etc/fdfs/client.conf upload 3.txt

　　　　　　#返回结果中会把上传完成的url 打印出来，看三个上传的url group是否轮询变化，如果停掉172.18.4.217 的tracker 返回url是不是地址变成172.18.4.218，

如都成功则 复制URL在浏览器中打开看是否正常