FastDFS+Nginx+fastdfs-nginx-module集群搭建

一、實驗環境說明 

　　操作系統： Centos 6.6 x64

　　FastDFS 相關版本： fastdfs-5.05 fastdfs-nginx-module-v1.16 libfastcommon-v1.0.7

　　web 服務器軟件： nginx-1.7.8

　　角色分配： 2 個 tracker， 地址分別為:10.1.1.243 10.1.1.244 兩塊磁盤

　　2個 group：

　　G1:10.1.1.245 10.1.1.246 三塊磁盤

　　G2:10.1.1.247 10.1.1.248 三塊磁盤

　　Client：10.1.1.249

目錄規划：

　　/dev/sdb1 /data1 日志元數據目錄

　　/dev/sdc1 /data2 數據目錄

　　關閉selinux和iptables

　　做時間同步，最好添加定時任務

　　配置好yum源，推薦國內源

　　分區格式化磁盤並創建掛載目錄進行掛載

 

 

二、安裝 libfastcommon 和 fastdfs(所有機器均安裝)

1.下載所需軟件（鏈接: https://pan.baidu.com/s/1eSvSRPg 密碼: 7k66）

 

unzip libfastcommon-1.0.7.zip cd libfastcommon-1.0.7 ./make.sh ./make.sh install cd .. tar xf FastDFS_v5.05.tar.gz cd FastDFS sed -i 's#/usr/local/bin/#/usr/bin/#g' init.d/fdfs_storaged sed -i 's#/usr/local/bin/#/usr/bin/#g' init.d/fdfs_trackerd ./make.sh ./make.sh install

三、配置 tracker 10.1.1.243 10.1.1.244 (配置相同)

cp /etc/fdfs/tracker.conf.sample /etc/fdfs/tracker.conf

vim /etc/fdfs/tracker.conf

disabled=false bind_addr= port=22122 connect_timeout=30 network_timeout=60 base_path=/data1/ max_connections=256 accept_threads=1 work_threads=4 store_lookup=2 store_group=group2 store_server=0 store_path=0 download_server=0 reserved_storage_space = 10% log_level=info run_by_group= run_by_user= allow_hosts=* sync_log_buff_interval = 10 check_active_interval = 120 thread_stack_size = 64KB storage_ip_changed_auto_adjust = true storage_sync_file_max_delay = 86400 storage_sync_file_max_time = 300 use_trunk_file = false slot_min_size = 256 slot_max_size = 16MB trunk_file_size = 64MB trunk_create_file_advance = false trunk_create_file_time_base = 02:00 trunk_create_file_interval = 86400 trunk_create_file_space_threshold = 20G trunk_init_check_occupying = false trunk_init_reload_from_binlog = false trunk_compress_binlog_min_interval = 0 use_storage_id = false storage_ids_filename = storage_ids.conf id_type_in_filename = ip store_slave_file_use_link = false rotate_error_log = false error_log_rotate_time=00:00 rotate_error_log_size = 0 log_file_keep_days = 0 use_connection_pool = false connection_pool_max_idle_time = 3600 http.server_port=8080 http.check_alive_interval=30 http.check_alive_type=tcp http.check_alive_uri=/status.html

啟動 tracker 服務：

　　service fdfs_trackerd start

　　chkconfig fdfs_trackerd on

四、配置組G1 10.1.1.245 10.1.1.246 (配置相同)

cp /etc/fdfs/storage.conf.sample /etc/fdfs/storage.conf

vim /etc/fdfs/storage.conf

disabled=false group_name=G1 bind_addr= client_bind=true port=23000 connect_timeout=30 network_timeout=60 heart_beat_interval=30 stat_report_interval=60 base_path=/data1/ max_connections=256 buff_size = 256KB accept_threads=1 work_threads=4 disk_rw_separated = true disk_reader_threads = 1 disk_writer_threads = 1 sync_wait_msec=50 sync_interval=0 sync_start_time=00:00 sync_end_time=23:59 write_mark_file_freq=500 store_path_count=1 store_path0=/data2 subdir_count_per_path=256 tracker_server=10.1.1.243:22122 tracker_server=10.1.1.244:22122 log_level=info run_by_group= run_by_user= allow_hosts=* file_distribute_path_mode=0 file_distribute_rotate_count=100 fsync_after_written_bytes=0 sync_log_buff_interval=10 sync_binlog_buff_interval=10 sync_stat_file_interval=300 thread_stack_size=512KB upload_priority=10 if_alias_prefix= check_file_duplicate=0 file_signature_method=hash key_namespace=FastDFS keep_alive=0 use_access_log = false rotate_access_log = false access_log_rotate_time=00:00 rotate_error_log = false error_log_rotate_time=00:00 rotate_access_log_size = 0 rotate_error_log_size = 0 log_file_keep_days = 0 file_sync_skip_invalid_record=false use_connection_pool = false connection_pool_max_idle_time = 3600 http.domain_name= http.server_port=8888

配置組G2 10.1.1.247 10.1.1.248 配置相同

cp /etc/fdfs/storage.conf.sample /etc/fdfs/storage.conf

vim /etc/fdfs/storage.conf

disabled=false group_name=G2 bind_addr= client_bind=true port=23000 connect_timeout=30 network_timeout=60 heart_beat_interval=30 stat_report_interval=60 base_path=/data1 max_connections=256 buff_size = 256KB accept_threads=1 work_threads=4 disk_rw_separated = true disk_reader_threads = 1 disk_writer_threads = 1 sync_wait_msec=50 sync_interval=0 sync_start_time=00:00 sync_end_time=23:59 write_mark_file_freq=500 store_path_count=1 store_path0=/data2 subdir_count_per_path=256 tracker_server=10.1.1.243:22122 tracker_server=10.1.1.244:22122 log_level=info run_by_group= run_by_user= allow_hosts=* file_distribute_path_mode=0 file_distribute_rotate_count=100 fsync_after_written_bytes=0 sync_log_buff_interval=10 sync_binlog_buff_interval=10 sync_stat_file_interval=300 thread_stack_size=512KB upload_priority=10 if_alias_prefix= check_file_duplicate=0 file_signature_method=hash key_namespace=FastDFS keep_alive=0 use_access_log = false rotate_access_log = false access_log_rotate_time=00:00 rotate_error_log = false error_log_rotate_time=00:00 rotate_access_log_size = 0 rotate_error_log_size = 0 log_file_keep_days = 0 file_sync_skip_invalid_record=false use_connection_pool = false connection_pool_max_idle_time = 3600 http.domain_name= http.server_port=8888

啟動 storage 服務：(四台)

service fdfs_storaged start

chkconfig fdfs_storaged on

五、storage server 安裝 nginx

編譯安裝nginx，添加fastdfs nginx 模塊

tar xf fastdfs-nginx-module_v1.16.tar.gz vim fastdfs-nginx-module/src/config CORE_INCS="$CORE_INCS /usr/include/fastdfs /usr/include/fastcommon/" #去掉local 添加軟連接 ln -s /usr/lib64/libfastcommon.so /usr/local/lib/libfastcommon.so ln -s /usr/lib64/libfastcommon.so /usr/lib/libfastcommon.so ln -s /usr/lib64/libfdfsclient.so /usr/local/lib/libfdfsclient.so ln -s /usr/lib64/libfdfsclient.so /usr/lib/libfdfsclient.so tar xf nginx-1.7.8.tar.gz cd nginx-1.7.8 ./configure --add-module=../fastdfs-nginx-module/src/ --prefix=/usr/local/nginx --user=nobody --group=nobody  --with-http_gzip_static_module --with-http_gunzip_module make make install

nginx 啟動腳本

#!/bin/bash # # nginx - this script starts and stops the nginx daemon # # chkconfig: - 85 15 # description: Nginx is an HTTP(S) server, HTTP(S) reverse # proxy and IMAP/POP3 proxy server # processname: nginx # config: /usr/local/nginx/conf/nginx.conf # Source function library. . /etc/rc.d/init.d/functions # Source networking configuration. . /etc/sysconfig/network # Check that networking is up. [ "$NETWORKING" = "no" ] && exit 0 NGINX_HOME="/usr/local/nginx/" nginx=$NGINX_HOME"sbin/nginx" prog=$(basename $nginx) NGINX_CONF_FILE=$NGINX_HOME"conf/nginx.conf" [ -f /etc/sysconfig/nginx ] && /etc/sysconfig/nginx lockfile=/var/lock/subsys/nginx start() { [ -x $nginx ] || exit 5 [ -f $NGINX_CONF_FILE ] || exit 6 echo -n $"Starting $prog: " daemon $nginx -c $NGINX_CONF_FILE retval=$? echo [ $retval -eq 0 ] && touch $lockfile return $retval } stop() { echo -n $"Stopping $prog: " killproc $prog -QUIT retval=$? echo [ $retval -eq 0 ] && rm -f $lockfile return $retval killall -9 nginx } restart() { configtest || return $? stop sleep 1 start } reload() { configtest || return $? echo -n $"Reloading $prog: " killproc $nginx -HUP RETVAL=$? echo } force_reload() { restart } configtest() { $nginx -t -c $NGINX_CONF_FILE } rh_status() { status $prog } rh_status_q() { rh_status >/dev/null 2>&1 } case "$1" in start) rh_status_q && exit 0 $1 ;; stop) rh_status_q || exit 0 $1 ;; restart|configtest) $1 ;; reload) rh_status_q || exit 7 $1 ;; force-reload) force_reload ;; status) rh_status ;; condrestart|try-restart) rh_status_q || exit 0 ;; *) echo $"Usage: $0
{start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}" exit 2 esac

配置 nginx(添加一個server)

cp fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/ cp fastdfs-5.05/conf/{anti-steal.jpg,http.conf,mime.types} /etc/fdfs/ touch /var/log/mod_fastdfs.log chown nobody.nobody /var/log/mod_fastdfs.log server { listen 80; server_name 本機IP地址; location /G1/M00{ root /data2/; ngx_fastdfs_module; } }

  配置 模塊配置文件 mod_fastdfs.conf

G1 組配置： vim /etc/fdfs/mod_fastdfs.conf connect_timeout=2 network_timeout=30 base_path=/tmp load_fdfs_parameters_from_tracker=true storage_sync_file_max_delay = 86400 use_storage_id = false storage_ids_filename = storage_ids.conf tracker_server=10.1.1.243:22122 tracker_server=10.1.1.244:22122 storage_server_port=23000 group_name=G1 url_have_group_name = true store_path_count=1 store_path0=/data2/ log_level=info log_filename=/var/log/mod_fastdfs.log response_mode=proxy if_alias_prefix= #include http.conf flv_support = true flv_extension = flv group_count = 2 [group1] group_name=G1 storage_server_port=23000 store_path_count=1 store_path0=/data2/ [group2] group_name=G2 storage_server_port=23000 store_path_count=1 store_path0=/data2/ G2 組配置： vim /etc/fdfs/mod_fastdfs.conf connect_timeout=2 network_timeout=30 base_path=/tmp load_fdfs_parameters_from_tracker=true storage_sync_file_max_delay = 86400 use_storage_id = false storage_ids_filename = storage_ids.conf tracker_server=10.1.1.243:22122 tracker_server=10.1.1.244:22122 storage_server_port=23000 group_name=G2 url_have_group_name = true store_path_count=1 store_path0=/data2/ log_level=info log_filename=/var/log/mod_fastdfs.log response_mode=proxy if_alias_prefix= #include http.conf flv_support = true flv_extension = flv group_count = 2 [group1] group_name=G1 storage_server_port=23000 store_path_count=1 store_path0=/data2/ [group2] group_name=G2 storage_server_port=23000 store_path_count=1 store_path0=/data2/ service nginx reload chkconfig nginx on

六、測試

客戶端配置/etc/fdfs/client.conf

connect_timeout=30 network_timeout=60 base_path=/tmp tracker_server=10.1.1.243:22122 tracker_server=10.1.1.244:22122 log_level=info use_connection_pool = false connection_pool_max_idle_time = 3600 load_fdfs_parameters_from_tracker=false use_storage_id = false storage_ids_filename = storage_ids.conf http.tracker_server_port=80 客戶端測試上傳下載(其他命令暫時不測試，這里僅為測試服務是否正常)： fdfs_upload_file /etc/fdfs/client.conf /etc/passwd #上傳文件 G1/M00/00/00/CgEB9Vnwv1SAXm1HAAAEeivAsqE3998928 md5sum /etc/passwd 6923374d09e7f8c9a296fdf2e84764a1 /etc/passwd fdfs_download_file /etc/fdfs/client.conf   G1/M00/00/00/CgEB9Vnwv1SAXm1HAAAEeivAsqE3998928 #下載文件 md5sum CgEB9Vnwv1SAXm1HAAAEeivAsqE3998928 6923374d09e7f8c9a296fdf2e84764a1 CgEB9Vnwv1SAXm1HAAAEeivAsqE3998928 上傳下載文件內容一致，未發生改變

上傳一張圖片自定義meizhi.jpg

 

fdfs_upload_file /etc/fdfs/client.conf meizhi.jpg G1/M00/00/00/CgEB9lnwwTaAFqHZAAC9pQqjlp0604.jpg

訪問效果：

 

#前端可以設置nginx服務器做反向代理，對后端做負載均衡，這里附上一份參考示例：

http { include mime.types; default_type application/octet-stream; #設置主機名哈希表的表項長度默認 32|64|128 server_names_hash_bucket_size 128; #設置請求緩沖區大小， 默認使用 client_header_buffer_size 來讀取 header 值， #如果 header 過大，會使用 large_client_header_buffers 來讀取 client_header_buffer_size 32k; large_client_header_buffers 4 32k; #定義最大允許上傳的文件大小 client_max_body_size 300m; #開啟高效文件傳輸模式， sendfile 指令指定 nginx 是否調用 sendfile 函數來輸出文件，對於普通應用 設為 on，如果用來進行下載等應用磁盤 IO 重負載應用，可設置為 off，以平衡磁盤與網絡 I/O 處理速度， 降低系統的負載。注意：如果圖片顯示不正常把這個改成 off sendfile on; # 防止網絡堵塞 tcp_nopush on; #長連接超時時間 keepalive_timeout 120; #gzip 模塊設置 gzip on; #開啟 gzip 壓縮輸出 gzip_min_length 1k; #最小壓縮文件大小 gzip_buffers 4 16k; #壓縮緩沖區 gzip_http_version 1.0; #壓縮版本（默認 1.1，前端如果是 squid2.5 請使用 1.0） gzip_comp_level 2; #壓縮等級(1~9) gzip_types text/plain application/x-javascript text/css application/xml; #壓縮類型，默認就已經包含 text/html gzip_vary on; #根據 HTTP 頭來判斷是否需要壓縮 gzip_disable “MSIE[1-6].”#IE6 以下對 gzip 的支持不好， 在 IE6 下禁用 gzip proxy_redirect off; #當后端單台 web 服務器上也配置有多個虛擬主機時，需要使用該 Header 來區分反向代理哪個主 機名 proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; #后端的 Web 服務器可以通過 X-Real-IP/X-Forwarded-For 獲取用戶真實 IP proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; #后端服務器連接的超時時間(發起握手等候響應的超時時間) proxy_connect_timeout 90; #后端服務器數據回傳時間(規定此時間內后端服務器必須傳完所有數據) proxy_send_timeout 90; #等待后端服務器處理請求的時間 proxy_read_timeout 90; #nginx 從被代理的服務器讀取響應時，使用該緩沖區保存響應的開始部分，該值默認等於 proxy_buffers 設置的一塊緩沖區大小，但可以被設置更小 proxy_buffer_size 16k; #為每個連接設置緩沖區的數量為 number，每塊緩沖區的大小為 size，用於保存從被 代理服務器讀取的響應 proxy_buffers 4 64k; proxy_busy_buffers_size 128k; #在開啟緩沖后端服務器響應到臨時文件的功能后，設置 nginx 每次寫數據到臨時文件的 size(大 小)限制 proxy_temp_file_write_size 128k; log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"'; access_log /usr/local/nginx/access.log main; #設置緩存存儲路徑、存儲方式、分配內存大小、磁盤最大空間、緩存期限 proxy_cache_path /var/cache/nginx/proxy_cache levels=1:2 keys_zone=http-cache:500m max_size=10g inactive=30d; proxy_temp_path /var/cache/nginx/proxy_cache/tmp; upstream group1 { server 10.1.1.245:80; server 10.1.1.246:80; } upstream group2 { server 10.1.1.247:80; server10.1.1.248:80; } server { listen 80; server_name 10.1.1.249; location /G1/M00 { #設置 group1 的負載均衡參數 #如果后端的服務器返回 502、 504、執行超時等錯誤，自動將請求轉發到 upstream 負載均衡池中 的另一台服務器，實現故障轉移 proxy_next_upstream http_502 http_504 error timeout invalid_header; proxy_cache http-cache; proxy_cache_valid 200 304 12h; proxy_cache_key $uri$is_args$args; proxy_pass http://group1;
expires 30d; } location ~* /G2/(M00|M01) { #設置 group2 的負載均衡參數 proxy_next_upstream http_502 http_504 error timeout invalid_header; proxy_cache http-cache; proxy_cache_valid 200 304 12h; proxy_cache_key $uri$is_args$args; proxy_pass http://group2;
expires 30d; } }