fastdfs nginx 集群搭建参考一

FastDFS是一个开源的轻量级分布式文件系统，由跟踪服务器（tracker server）、存储服务器（storage server）和客户端（client）三个部分组成，主要解决了海量数据存储问题，特别适合以中小文件（建议范围：4KB < file_size <500MB）为载体的在线服务。在生成环境FastDFS一般都是用集群配置，以提高FastDFS的可用性，并发能力。

部署架构：

环境IP地址(关闭所有环境的防火墙)：

Tracker 192.168.18.178

Group 1：

  S1：192.168.110.71

  S2：192.168.110.91

Group 2：

  S3：192.168.100.90

  S4：192.168.100.194

注：Tracker可以部署多台，提供负载，这里资源有限，就部署一台。

由于需要安装nginx，每台机器都安装依赖：

yum -y install  zlib pcre pcre-devel zlib-devel

所用的安装软件下载：http://download.csdn.net/detail/tianwei7518/8745279

一、安装tracker

1.安装依赖libfastcommon

unzip libfastcommon-master.zip

cd libfastcommon

./make.sh 
./make.sh  install

2.安装FastDFS

unzip fastdfs.zip

cd fastdfs

./make.sh 

./make.sh install

默认安装目录：/usr/bin

将原安装文件夹下的配置文件复制到/etc/fdfs目下：cp ./conf/*  /etc/fdfs/

3.配置

编辑配置文件目录下的tracker.conf
一般只需改动以下几个参数即可：
disabled=false            #启用配置文件
port=22122                #设置tracker的端口号
base_path=/home/fastdfs   #设置tracker的数据文件和日志目录（需预先创建）
http.server_port=8080     #设置http端口号
4.启动
/usr/bin/fdfs_trackerd /etc/fdfs/tracker.conf start

二、安装tracker代理nginx

在tracker上安装的nginx主要为了提供http访问的反向代理、负载均衡以及缓存服务。

1.安装nginx

tar -zxvf nginx-1.8.0.tar.gz 
tar -zxvf ngx_cache_purge-2.3.tar.gz 
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx --add-module=/root/ngx_cache_purge-2.3

make 
make install

如果提示错误，可能缺少依赖的软件包，需先安装依赖包，再次运行./configure
nginx以及nginx cache purge插件模块安装完成，安装目录/usr/local/nginx

2.配置nginx

user  root;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;
    #设置缓存参数
    server_names_hash_bucket_size 128;
    client_header_buffer_size 32k;
    large_client_header_buffers 4 32k;
    client_max_body_size 300m;
    sendfile        on;
    tcp_nopush     on;
    proxy_redirect off;
    proxy_set_header Host $http_host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
    proxy_connect_timeout 90;
    proxy_send_timeout 90;
    proxy_read_timeout 90;
    proxy_buffer_size 16k;
    proxy_buffers 4 64k;
    proxy_busy_buffers_size 128k;
    proxy_temp_file_write_size 128k;
    #设置缓存存储路径、存储方式、分配内存大小、磁盘最大空间、缓存期限
    proxy_cache_path /var/cache/nginx/proxy_cache levels=1:2 keys_zone=http-cache:500m max_size=10g inactive=30d;

    proxy_temp_path /var/cache/nginx/proxy_cache/tmp;

    keepalive_timeout  65;

    #设置group服务器
    upstream fdfs_group1 {
        server 192.168.110.71:8090 weight=1 max_fails=2 fail_timeout=30s;
        server 192.168.110.91:8090 weight=1 max_fails=2 fail_timeout=30s;
    }
    upstream fdfs_group2 {
        server 192.168.100.90:8090 weight=1 max_fails=2 fail_timeout=30s;
        server 192.168.100.194:8090 weight=1 max_fails=2 fail_timeout=30s;
    }

    server {
        listen       80;
        server_name  localhost;
        charset utf-8;
        #access_log  /usr/local/nginx/logs/localhost.access.log  main;

        location /group1/M00 {
                proxy_next_upstream http_502 http_504 error timeout invalid_header;
                proxy_cache http-cache;
                proxy_cache_valid  200 304 12h;
                proxy_cache_key $uri$is_args$args;
                proxy_pass http://fdfs_group1;
                expires 30d;
        }
        location /group2/M00 {
                proxy_next_upstream http_502 http_504 error timeout invalid_header;
                proxy_cache http-cache;
                proxy_cache_valid  200 304 12h;
                proxy_cache_key $uri$is_args$args;
                proxy_pass http://fdfs_group2;
                expires 30d;
        }
        #设置清除缓存的访问权限
        location ~ /purge(/.*) {
                allow 127.0.0.1;
                allow 172.16.1.0/24;
                deny all;
                proxy_cache_purge http-cache  $1$is_args$args;
        }
    }

}

创建缓存目录：/var/cache/nginx/proxy_cache/tmp

3.启动

/usr/local/nginx/sbin/nginx 

三、安装storage
1.安装

参考安装tracker前2步骤。

2.配置

编辑配置文件目录下的storage.conf

 

只需改动以下几个参数即可：

disabled=false     #启用配置文件

group_name=group1 #组名，根据实际情况修改

port=23000 #设置storage的端口号

base_path=/home/fastdfs #设置storage的日志目录（需预先创建）

store_path_count=1 #存储路径个数，需要和store_path个数匹配

store_path0=/home/fastdfs#存储路径

tracker_server=192.168.17.43:22122#tracker服务器的IP地址和端口号

http.server_port=8080   #设置http端口号

创建目录mkdir /home/fastdfs

3.运行

/usr/bin/fdfs_storaged /etc/fdfs/storage.conf start

另外：

分别在其他机器上全部安装storage并确认运行正常。注意配置文件中group名参数需要根据实际情况调整：
group1：192.168.110.71，192.168.110.91
group2：192.168.100.90，192.168.100.194
另外每个group中所有storage的端口号必须一致。

四、在storage上安装nginx
在storage上安装的nginx主要为了提供http的访问服务，同时解决group中storage服务器的同步延迟问题。
1.解压fastdfs-nginx-module插件
unzip  fastdfs-nginx-module.zip 
2.安装nginx
tar -zxvf nginx-1.8.0.tar.gz 
cd nginx-1.8.0
./configure --prefix=/usr/local/nginx --add-module=../fastdfs-nginx-module/src
make 
make install
安装目录：/usr/local/nginx

若安装报错：[emerg] 13513#0: eventfd() failed (38: Function not implemented)
原因是：编译时带了--with-file-aio模块，这个要Linux 2.6.22以后内核才支持．服务器是2.6.18。也可以下载低版本的nginx版本
3.配置
1）配置FastDFS的nginx插件
将FastDFS的nginx插件模块的配置文件copy到FastDFS配置文件目录
fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs/
编辑/etc/fdfs配置文件目录下的mod_fastdfs.conf，设置storage信息并保存。
一般只需改动以下几个参数即可：
base_path=/home/fastdfs #保存日志目录
tracker_server=192.168.18.43:22122 #tracker服务器的IP地址以及端口号
storage_server_port=23000#storage服务器的端口号
group_name=group1#当前服务器的group名
url_have_group_name = true        #文件url中是否有group名
store_path_count=1                #存储路径个数，需要和store_path个数匹配
store_path0=/home/fastdfs         #存储路径
http.need_find_content_type=true#从文件扩展名查找文件类型（nginx时为true）
group_count = 2                   #设置组的个数
在末尾增加2个组的具体信息：
[group1]
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/home/fastdfs

[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/home/fastdfs
建立M00至存储目录的符号连接：
ln -s /home/fastdfs/data /home/fastdfs/data/M00
2）配置nginx
vi nginx.conf
user root; 


location ~/group[1-3]/M00 {
    root /home/fastdfs/data;
    ngx_fastdfs_module;
}
4.启动
/usr/local/nginx/sbin/nginx

另外：
分别在其他机器storage上全部安装nginx并确认运行正常。注意配置文件中group名参数需要根据实际情况调整：
group1：192.168.110.71，192.168.110.91
group2：192.168.100.90，192.168.100.194
另外nginx的端口号8090。

至此所有配置完毕。

四、测试

配置/etc/fdfs/client.conf

base_path=/home/fastdfs #日志存放路径

tracker_server=192.168.18.43:22122 #tracker服务器IP地址和端口号

http.tracker_server_port=8080   #tracker服务器的http端口号

通过fdfs_upload_file上传一个文件到FastDFS，程序会自动返回文件的URL，
#fdfs_upload_file /etc/fdfs/client.conf 40-15052PZK5.jpg 
group1/M00/00/00/wKhuR1Vmh_2ADmdfAAF1dmVtk4w934.jpg
然后使用浏览器访问，访问正常
http://192.168.18.43/group1/M00/00/00/wKhuR1Vmh_2ADmdfAAF1dmVtk4w934.jpg

注：可以使用fdfs_monitor查看tracker和所有group的运行情况

 

[plain] view plain copy print?

1. # fdfs_monitor /etc/fdfs/client.conf   
2. [2015-05-27 20:19:59] DEBUG - base_path=/home/fastdfs, connect_timeout=30, network_timeout=60, tracker_server_count=1, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0  
3.   
4. server_count=1, server_index=0  
5.   
6. tracker server is 192.168.18.43:22122  
7.   
8. group count: 2  
9.   
10. Group 1:  
11. group name = group1  
12. disk total space = 45438 MB  
13. disk free space = 33920 MB  
14. trunk free space = 0 MB  
15. storage server count = 2  
16. active server count = 2  
17. storage server port = 23000  
18. storage HTTP port = 8080  
19. store path count = 1  
20. subdir count per path = 256  
21. current write server index = 1  
22. current trunk file id = 0  
23.   
24.         Storage 1:  
25.                 id = 192.168.110.71  
26.                 ip_addr = 192.168.110.71 (localhost)  ACTIVE  
27.                 http domain =   
28.                 version = 5.06  
29.                 join time = 2015-05-27 01:37:04  
30.                 up time =   
31.                 total storage = 95217 MB  
32.                 free storage = 47563 MB  
33.                 upload priority = 10  
34.                 store_path_count = 1  
35.                 subdir_count_per_path = 256  
36.                 storage_port = 23000  
37.                 storage_http_port = 8080  
38.                 current_write_path = 0  
39.                 source storage id =   
40.                 if_trunk_server = 0  
41.                 connection.alloc_count = 256  
42.                 connection.current_count = 1  
43.                 connection.max_count = 2  
44.                 total_upload_count = 1  
45.                 success_upload_count = 1  
46.                 total_append_count = 0  
47.                 success_append_count = 0  
48.                 total_modify_count = 0  
49.                 success_modify_count = 0  
50.                 total_truncate_count = 0  
51.                 success_truncate_count = 0  
52.                 total_set_meta_count = 0  
53.                 success_set_meta_count = 0  
54.                 total_delete_count = 0  
55.                 success_delete_count = 0  
56.                 total_download_count = 0  
57.                 success_download_count = 0  
58.                 total_get_meta_count = 0  
59.                 success_get_meta_count = 0  
60.                 total_create_link_count = 0  
61.                 success_create_link_count = 0  
62.                 total_delete_link_count = 0  
63.                 success_delete_link_count = 0  
64.                 total_upload_bytes = 95606  
65.                 success_upload_bytes = 95606  
66.                 total_append_bytes = 0  
67.                 success_append_bytes = 0  
68.                 total_modify_bytes = 0  
69.                 success_modify_bytes = 0  
70.                 stotal_download_bytes = 0  
71.                 success_download_bytes = 0  
72.                 total_sync_in_bytes = 0  
73.                 success_sync_in_bytes = 0  
74.                 total_sync_out_bytes = 0  
75.                 success_sync_out_bytes = 0  
76.                 total_file_open_count = 1  
77.                 success_file_open_count = 1  
78.                 total_file_read_count = 0  
79.                 success_file_read_count = 0  
80.                 total_file_write_count = 1  
81.                 success_file_write_count = 1  
82.                 last_heart_beat_time = 2015-05-27 20:19:49  
83.                 last_source_update = 2015-05-27 20:14:04  
84.                 last_sync_update = 1969-12-31 16:00:00  
85.                 last_synced_timestamp = 1969-12-31 16:00:00   
86.         Storage 2:  
87.                 id = 192.168.110.91  
88.                 ip_addr = 192.168.110.91 (localhost)  ACTIVE  
89.                 http domain =   
90.                 version = 5.06  
91.                 join time = 2015-05-27 19:35:36  
92.                 up time = 2015-05-27 19:35:36  
93.                 total storage = 45438 MB  
94.                 free storage = 33920 MB  
95.                 upload priority = 10  
96.                 store_path_count = 1  
97.                 subdir_count_per_path = 256  
98.                 storage_port = 23000  
99.                 storage_http_port = 8080  
100.                 current_write_path = 0  
101.                 source storage id = 192.168.110.71  
102.                 if_trunk_server = 0  
103.                 connection.alloc_count = 256  
104.                 connection.current_count = 1  
105.                 connection.max_count = 1  
106.                 total_upload_count = 0  
107.                 success_upload_count = 0  
108.                 total_append_count = 0  
109.                 success_append_count = 0  
110.                 total_modify_count = 0  
111.                 success_modify_count = 0  
112.                 total_truncate_count = 0  
113.                 success_truncate_count = 0  
114.                 total_set_meta_count = 0  
115.                 success_set_meta_count = 0  
116.                 total_delete_count = 0  
117.                 success_delete_count = 0  
118.                 total_download_count = 0  
119.                 success_download_count = 0  
120.                 total_get_meta_count = 0  
121.                 success_get_meta_count = 0  
122.                 total_create_link_count = 0  
123.                 success_create_link_count = 0  
124.                 total_delete_link_count = 0  
125.                 success_delete_link_count = 0  
126.                 total_upload_bytes = 0  
127.                 success_upload_bytes = 0  
128.                 total_append_bytes = 0  
129.                 success_append_bytes = 0  
130.                 total_modify_bytes = 0  
131.                 success_modify_bytes = 0  
132.                 stotal_download_bytes = 0  
133.                 success_download_bytes = 0  
134.                 total_sync_in_bytes = 95606  
135.                 success_sync_in_bytes = 95606  
136.                 total_sync_out_bytes = 0  
137.                 success_sync_out_bytes = 0  
138.                 total_file_open_count = 1  
139.                 success_file_open_count = 1  
140.                 total_file_read_count = 0  
141.                 success_file_read_count = 0  
142.                 total_file_write_count = 1  
143.                 success_file_write_count = 1  
144.                 last_heart_beat_time = 2015-05-27 20:19:51  
145.                 last_source_update = 1969-12-31 16:00:00  
146.                 last_sync_update = 2015-05-27 20:14:07  
147.                 last_synced_timestamp = 2015-05-27 20:14:05 (-1s delay)  
148.   
149. Group 2:  
150. group name = group2  
151. disk total space = 9916 MB  
152. disk free space = 7434 MB  
153. trunk free space = 0 MB  
154. storage server count = 2  
155. active server count = 2  
156. storage server port = 23000  
157. storage HTTP port = 8080  
158. store path count = 1  
159. subdir count per path = 256  
160. current write server index = 0  
161. current trunk file id = 0  
162.   
163.         Storage 1:  
164.                 id = 192.168.100.194  
165.                 ip_addr = 192.168.100.194 (localhost)  ACTIVE  
166.                 http domain =   
167.                 version = 5.06  
168.                 join time = 2015-05-27 20:03:37  
169.                 up time = 2015-05-27 20:03:37  
170.                 total storage = 47368 MB  
171.                 free storage = 37371 MB  
172.                 upload priority = 10  
173.                 store_path_count = 1  
174.                 subdir_count_per_path = 256  
175.                 storage_port = 23000  
176.                 storage_http_port = 8080  
177.                 current_write_path = 0  
178.                 source storage id = 192.168.100.90  
179.                 if_trunk_server = 0  
180.                 connection.alloc_count = 256  
181.                 connection.current_count = 1  
182.                 connection.max_count = 1  
183.                 total_upload_count = 0  
184.                 success_upload_count = 0  
185.                 total_append_count = 0  
186.                 success_append_count = 0  
187.                 total_modify_count = 0  
188.                 success_modify_count = 0  
189.                 total_truncate_count = 0  
190.                 success_truncate_count = 0  
191.                 total_set_meta_count = 0  
192.                 success_set_meta_count = 0  
193.                 total_delete_count = 0  
194.                 success_delete_count = 0  
195.                 total_download_count = 0  
196.                 success_download_count = 0  
197.                 total_get_meta_count = 0  
198.                 success_get_meta_count = 0  
199.                 total_create_link_count = 0  
200.                 success_create_link_count = 0  
201.                 total_delete_link_count = 0  
202.                 success_delete_link_count = 0  
203.                 total_upload_bytes = 0  
204.                 success_upload_bytes = 0  
205.                 total_append_bytes = 0  
206.                 success_append_bytes = 0  
207.                 total_modify_bytes = 0  
208.                 success_modify_bytes = 0  
209.                 stotal_download_bytes = 0  
210.                 success_download_bytes = 0  
211.                 total_sync_in_bytes = 0  
212.                 success_sync_in_bytes = 0  
213.                 total_sync_out_bytes = 0  
214.                 success_sync_out_bytes = 0  
215.                 total_file_open_count = 0  
216.                 success_file_open_count = 0  
217.                 total_file_read_count = 0  
218.                 success_file_read_count = 0  
219.                 total_file_write_count = 0  
220.                 success_file_write_count = 0  
221.                 last_heart_beat_time = 2015-05-27 20:19:44  
222.                 last_source_update = 1969-12-31 16:00:00  
223.                 last_sync_update = 1969-12-31 16:00:00  
224.                 last_synced_timestamp = 1969-12-31 16:00:00   
225.         Storage 2:  
226.                 id = 192.168.100.90  
227.                 ip_addr = 192.168.100.90 (localhost)  ACTIVE  
228.                 http domain =   
229.                 version = 5.06  
230.                 join time = 2015-05-27 19:50:27  
231.                 up time = 2015-05-27 19:50:27  
232.                 total storage = 9916 MB  
233.                 free storage = 7434 MB  
234.                 upload priority = 10  
235.                 store_path_count = 1  
236.                 subdir_count_per_path = 256  
237.                 storage_port = 23000  
238.                 storage_http_port = 8080  
239.                 current_write_path = 0  
240.                 source storage id =   
241.                 if_trunk_server = 0  
242.                 connection.alloc_count = 256  
243.                 connection.current_count = 1  
244.                 connection.max_count = 1  
245.                 total_upload_count = 0  
246.                 success_upload_count = 0  
247.                 total_append_count = 0  
248.                 success_append_count = 0  
249.                 total_modify_count = 0  
250.                 success_modify_count = 0  
251.                 total_truncate_count = 0  
252.                 success_truncate_count = 0  
253.                 total_set_meta_count = 0  
254.                 success_set_meta_count = 0  
255.                 total_delete_count = 0  
256.                 success_delete_count = 0  
257.                 total_download_count = 0  
258.                 success_download_count = 0  
259.                 total_get_meta_count = 0  
260.                 success_get_meta_count = 0  
261.                 total_create_link_count = 0  
262.                 success_create_link_count = 0  
263.                 total_delete_link_count = 0  
264.                 success_delete_link_count = 0  
265.                 total_upload_bytes = 0  
266.                 success_upload_bytes = 0  
267.                 total_append_bytes = 0  
268.                 success_append_bytes = 0  
269.                 total_modify_bytes = 0  
270.                 success_modify_bytes = 0  
271.                 stotal_download_bytes = 0  
272.                 success_download_bytes = 0  
273.                 total_sync_in_bytes = 0  
274.                 success_sync_in_bytes = 0  
275.                 total_sync_out_bytes = 0  
276.                 success_sync_out_bytes = 0  
277.                 total_file_open_count = 0  
278.                 success_file_open_count = 0  
279.                 total_file_read_count = 0  
280.                 success_file_read_count = 0  
281.                 total_file_write_count = 0  
282.                 success_file_write_count = 0  
283.                 last_heart_beat_time = 2015-05-27 20:19:48  
284.                 last_source_update = 1969-12-31 16:00:00  
285.                 last_sync_update = 1969-12-31 16:00:00  
286.                 last_synced_timestamp = 1969-12-31 16:00:00