FastDFS && Nginx安装及使用

由于公司最近需要进行图片的集中存储，现在互联网行业主要有两种大的实现思路，基于简单hash实现，基于分布式文件系统实现（比如淘宝的TFS，Amozon的S3，google的GFS等等），下面主要讲解如何使用FastDFS来实现，架构图如下：

 

>> 安装FastDFS
1) 安装libevent
# rpm -qa | grep libevent
# rpm -qa | grep libevent | xargs rpm -e --nodeps
# tar -zxvf libevent-2.0.20-stable.tar.gz
# cd libevent-2.0.20-stable
# ./configure
# make
# make install
# ln -s /usr/local/lib/libevent* /lib/
# ln -s /usr/local/lib/libevent* /lib64/

2) 安装FastDFS
# tar -zxvf FastDFS_v3.11.tar.gz
# cd FastDFS
# vi make.sh
WITH_LINUX_SERVICE=1
# ./make.sh
# ./make.sh install (如果需要修改为开机自启动，则去掉make.sh里的行注释WITH_LINUX_SERVICE=1)

3) 配置及启动Tracker(IP: 10.1.1.121, 10.1.1.122，形成Cluster)
# vi /etc/fdfs/tracker.conf
http.disabled=true

# mkdir -p /home/yuqing/fastdfs
# service fdfs_trackerd start (fdfs_trackerd /etc/fdfs/tracker.conf)

4) 配置及启动Storage(IP: 10.1.1.121, 10.1.1.122，形成Group1，可以水平线性扩展，GroupN)
# vi /etc/fdfs/storage.conf
tracker_server=10.1.1.121:22122
tracker_server=10.1.1.122:22122
group_name=group1
http.disabled=true
http.server_port=80
# mkdir -p /home/yuqing/fastdfs
# service fdfs_storaged start (fdfs_storaged /etc/fdfs/storaged.conf)

>> 安装nginx及fastdfs-nginx-module
1) 安装
# tar -zxvf fastdfs-nginx-module_v1.11.tar.gz
# tar -zxvf nginx-1.2.4.tar.gz
# cd nginx-1.2.4
# yum -y install pcre-devel (如果是RHEL系统确保yum能正常使用，请参考http://www.cnblogs.com/javapro/archive/2012/09/25/2701462.html)
# ./configure --prefix=/usr/local/nginx-1.2.4 --add-module=/software/fastdfs-nginx-module/src
# make
# make install
# cp /software/fastdfs-nginx-module/src/mod_fastdfs.conf /etc/fdfs
# vi /etc/fdfs/mod_fastdfs.conf
tracker_server=10.1.1.121:22122
tracker_server=10.1.1.122:22122
group_name=group1
response_mode=proxy

2) 配置
# vi /usr/local/nginx-1.2.4/conf/nginx.conf

worker_processes 2;
worker_cpu_affinity 0101 1010;
error_log /usr/local/nginx-1.2.4/logs/error.log crit;
worker_rlimit_nofile 65536;

events {
use epoll;
worker_connections 65536;
}

location /M00 {
root /home/yuqing/fastdfs/data;
ngx_fastdfs_module;
}

# ln -s /home/yuqing/fastdfs/data /home/yuqing/fastdfs/data/M00

# vi /etc/sysctl.conf
net.ipv4.tcp_max_tw_buckets = 65536
net.ipv4.tcp_sack = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_rmem = 4096 87380 4194304
net.ipv4.tcp_wmem = 4096 16384 4194304
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.core.netdev_max_backlog = 262144
net.core.somaxconn = 262144
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_syn_backlog = 262144
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_time = 30
net.ipv4.ip_local_port_range = 1024 65000

# /sbin/sysctl -p

# ulimit -SHn 65536

>> 安装Varnish
# tar -zxvf varnish-3.0.3.tar.gz
# cd varnish-3.0.3
# ./configure --prefix=/usr/local/varnish-3.0.3
# make
# make install
# vi /usr/local/varnish-3.0.3/etc/varnish/lefeng.vcl
# varnishd -f /usr/local/varnish-3.0.3/etc/varnish/lefeng.vcl -s malloc,2G -T 0.0.0.0:2000

>> 管理
# mkdir /var/log/varnish
# /usr/local/varnish-3.0.3/bin/varnishncsa -w /var/log/varnish/varnish.log &

# /usr/local/varnish-3.0.3/bin/varnishstat /var/log/varnish

# /usr/local/varnish-3.0.3/bin/varnishadm -T 10.1.1.121:2000 help
# /usr/local/varnish-3.0.3/bin/varnishadm -T 10.1.1.121:2000 ban.url /M00/00/00/CgEBelCFJ9rRa1xzAAIC6op8sr8381.png

# pkill varnishd

PS: 测试时注意观察Http状态码，第一次请求时应该都是200，而后请求应该都是304，如果一直为200(很可能是由于服务器时间问题)

>> 测试
# curl -I http://10.1.1.121/M00/00/00/CgEBelCBhxGQ5nfAAAIC6op8sr8593.png
# ab -c 1000 -n 10000 http://10.1.1.121/M00/00/00/CgEBelCBhxGQ5nfAAAIC6op8sr8593.png

常用维护命令：
>> fdfs启动
service fdfs_trackerd start
service fdfs_storaged start

>> fdfs停止
killall fdfs_trackerd
killall fdfs_storaged

>> nginx启动
/usr/local/nginx-1.2.4/sbin/nginx -c /usr/local/nginx-1.2.4/conf/nginx.conf

/usr/local/nginx-1.2.4/sbin/nginx -s reload

kill -QUIT PID

TODO:
>> 图片管理(查看，删除)