CentOS7下搭建FastDfs（V5.11）+Keepalived分布式集群部署

FastDfs介绍

http://kb.cnblogs.com/page/82280/

1、准备

---

系统

CentOS7 最小化安装。

#VIP虚拟IP 
10.1.6.218
#Keepalived
10.1.6.216  #Keepalived1+Nginx
10.1.6.217  #Keepalived2+Nginx
#Tracke
10.1.6.210  #Tracke1+Nginx
10.1.6.211  #Tracke2+Nginx
#Group1
10.1.6.212  #Storage1+Nginx
10.1.6.213  #Storage2+Nginx
#Group2
10.1.6.214  #Storage3+Nginx
10.1.6.215  #Storage4+Nginx
#部署结构示意
                       +-----+
                       | VIP |
                       +-----+
                          |
        +------------+    |    +------------+
        | Keepalived |----+----| Keepalived |
        | Nginx      |         | Nginx      |
        +------------+         +------------+
                |←-----------------→|
                |                   |
                ↓                   ↓
           +---------+         +---------+
           | Tracke1 |         | Tracke2 |
           | Nginx   |         | Nginx   |
           +---------+         +---------+
                |←-----------------→|
                |                   |
                ↓                   ↓
 +-------------------+         +-------------------+
 | Group1            |         | Group2            |
 | Storage+Nginx     |         | Storage+Nginx     |
 | +----+     +----+ |         | +----+     +----+ |
 | | S1 |--+--| S2 | |         | | S3 |--+--| S4 | |
 | | Ng |     | Ng | |         | | Ng |     | Ng | |
 | +----+     +----+ |         | +----+     +----+ |
 +-------------------+         +-------------------+

#测试环境，禁用防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service

下载

文件	下载地址	介绍
FastDFS	https://github.com/happyfish100/fastdfs	（Version 5.11 2017-05-26 fastdfs-master.zip）
libfastcommon	https://github.com/happyfish100/libfastcommon	FastDFS 运行所需要的基础库（Version 1.36 2017-03-08 libfastcommon-master.zip）
fastdfs-nginx-module	https://github.com/happyfish100/fastdfs-nginx-module	storage节点http服务nginx模块(Version 1.20 2017-05-26 fastdfs-nginx-module-master.zip)
Nginx	http://nginx.org/	(nginx-1.12.0)
ngx_cache_purge	https://github.com/FRiCKLE/ngx_cache_purge/tree/2.3	Nginx图片缓存清除模块

将文件下载到 /usr/local/src 文件夹。

2、安装

---

依赖安装

#安装 gcc、libevent、perl
yum -y install gcc-c++
yum -y install libevent
yum -y install perl
#安装zip解压
yum -y install unzip

libfastcommon安装

#libfastcommon解压、安装
unzip libfastcommon-master.zip
ln -s libfastcommon-master libfastcommon
cd libfastcommon
./make.sh
./make.sh install

安装完成后/usr/lib64、/usr/lib文件夹内会有libfastcommon.so、libfdfsclient.so文件。

#V5.11可忽略
#FastDFS主程序设置的lib目录是/usr/local/lib，所以此处需要重新设置软链接
ln -s /usr/lib64/libfastcommon.so /usr/local/lib/libfastcommon.so
ln -s /usr/lib64/libfastcommon.so /usr/lib/libfastcommon.so
ln -s /usr/lib64/libfdfsclient.so /usr/local/lib/libfdfsclient.so
ln -s /usr/lib64/libfdfsclient.so /usr/lib/libfdfsclient.so

fastdfs安装

#fastdfs解压、安装
unzip fastdfs-master.zip
cd fastdfs-master
./make.sh
./make.sh install

安装完成后/etc/fdfs文件夹内会有client.conf.sample、storage.conf.sample、storage_ids.conf.sample、tracker.conf.sample文件。

#将示例配置文件拷贝一份
cd /etc/fdfs/
cp client.conf.sample client.conf
cp storage.conf.sample storage.conf
cp tracker.conf.sample tracker.conf

3、启动服务、测试

---

#创建存储数据、日志目录
mkdir -p /usr/local/FastDFS/Tracker
mkdir -p /usr/local/FastDFS/Storage
mkdir -p /usr/local/FastDFS/FileData
mkdir -p /usr/local/FastDFS/Client

Tracker配置

#修改tracker.conf配置
vi tracker.conf
#设置Tracker的数据文件和日志目录
base_path=/usr/local/FastDFS/Tracker
#0轮询策略方便测试，默认是2（负载均衡策略）
store_lookup=0
#默认
port=22122
http.server_port=8080
#启动Tracker服务
service fdfs_trackerd start
ps -ef|grep fdfs
netstat -unltp|grep fdfs

Storage配置

vi storage.conf
#设置Storage的数据文件和日志目录
base_path=/usr/local/FastDFS/Storage
#实际文件存储路径
store_path0=/usr/local/FastDFS/FileData
#tracker 服务器的 IP地址和端口号
tracker_server=10.1.6.210:22122
tracker_server=10.1.6.211:22122
#默认
group_name=group1
port=23000
store_path_count=1
#启动Storage服务
service fdfs_storaged start
ps -ef|grep fdfs
netstat -unltp|grep fdfs

#查看storage服务器是否已经登记到tracker服务器
fdfs_monitor /etc/fdfs/storage.conf

[root@localhost logs]# fdfs_monitor /etc/fdfs/storage.conf
[2017-07-01 19:36:50] DEBUG - base_path=/usr/local/FastDFS/Storage, connect_timeout=30, network_timeout=60, tracker_server_count=2, anti_steal_token=0, anti_steal_secret_key length=0, use_connection_pool=0, g_connection_pool_max_idle_time=3600s, use_storage_id=0, storage server id count: 0

server_count=2, server_index=1

tracker server is 10.1.6.211:22122

group count: 2

Group 1:
group name = group1
...
current trunk file id = 0

        Storage 1:
                id = 10.1.6.212
                ip_addr = 10.1.6.212  ACTIVE
                http domain = 
                version = 5.11
                ...
                last_synced_timestamp = 2017-07-01 19:36:50 (0s delay)
        Storage 2:
                id = 10.1.6.213
                ip_addr = 10.1.6.213  ACTIVE
                http domain = 
                version = 5.11
                ...
                last_synced_timestamp = 2017-07-01 19:36:50 (-1s delay)

Group 2:
group name = group2
...
current trunk file id = 0

        Storage 1:
                id = 10.1.6.214
                ip_addr = 10.1.6.214  ACTIVE
                http domain = 
                version = 5.11
                ...
                last_synced_timestamp = 2017-07-01 19:36:50 (0s delay)
        Storage 2:
                id = 10.1.6.215
                ip_addr = 10.1.6.215 (localhost.localdomain)  ACTIVE
                http domain = 
                version = 5.11
                ...
                last_synced_timestamp = 2017-07-01 19:36:50 (0s delay)

测试（上传、删除）

#配置client.conf
vi /etc/fdfs/client.conf
base_path=/usr/local/FastDFS/Client
#tracker服务器IP地址和端口号
tracker_server=10.1.6.210:22122
tracker_server=10.1.6.211:22122
# tracker 服务器的 http 端口号，必须和tracker的设置对应起来
http.tracker_server_port=8080
#上传
fdfs_upload_file /etc/fdfs/client.conf /usr/local/src/fastdfs-master/conf/anti-steal.jpg
#删除
fdfs_delete_file /etc/fdfs/client.conf group1/M00/00/00/CgEG5llJQGGA01_1AABdrzgsqUU484.jpg

#在/usr/local/FastDFS/FileData/data/00/00下查看上传文件。
[root@localhost 00]# pwd
/usr/local/FastDFS/FileData/data/00/00
[root@localhost 00]# ll
总用量 24
-rw-r--r--. 1 root root 23981 6月  20 23:33 CgEG5llJQGGAO1_1AABdrZgsqUU484.jpg

4、Storage存储节点（10.1.6.212、10.1.6.213、10.1.6.214、10.1.6.215）安装Nginx和fastdfs-nginx-module模块

---

#安装nginx需要的一些模块依赖的lib库
yum -y install pcre pcre-devel
yum -y install zlib zlib-devel
yum -y install openssl openssl-devel
#解压nginx
tar -zxvf nginx-1.12.0.tar.gz
#解压fastdfs-nginx-module-master
yum -y install unzip
unzip fastdfs-nginx-module-master.zip
#安装nginx
cd nginx-1.12.0
#配置参考（Nginx安装.md）
./configure --prefix=/usr/local/Nginx --add-module=/usr/local/src/fastdfs-nginx-module-master/src
make
make install

./configure配置成功会看到如下信息：

将http.conf和mime.types拷贝到/etc/fdfs目录下

cp /usr/local/src/fastdfs-master/conf/http.conf /etc/fdfs/
cp /usr/local/src/fastdfs-master/conf/mime.types /etc/fdfs/

复制 fastdfs-nginx-module 源码中的配置文件到/etc/fdfs 目录，并修改

cp /usr/local/src/fastdfs-nginx-module-master/src/mod_fastdfs.conf /etc/fdfs/
#修改mod_fastdfs.conf
vi /etc/fdfs/mod_fastdfs.conf
#与当前Storage服务器storage.conf中的group_name一致
group_name=group1
#保存日志目录
base_path=/usr/local/FastDFS/Storage
#tracker服务器的IP地址以及端口号
tracker_server=10.1.6.210:22122
tracker_server=10.1.6.211:22122
#文件 url 中是否有 group 名
url_have_group_name = true
#文件实际存储路径
store_path0=/usr/local/FastDFS/FileData
group_count = 2
[group1]
group_name=group1
storage_server_port=23000
store_path_count=1
store_path0=/usr/local/FastDFS/FileData

[group2]
group_name=group2
storage_server_port=23000
store_path_count=1
store_path0=/usr/local/FastDFS/FileData

建立M00至存储目录的符号连接

ln -s /usr/local/FastDFS/FileData/data /usr/local/FastDFS/FileData/data/M00

配置nginx.conf

vi /usr/local/Nginx/conf/nginx.conf
#添加server
    server {
        listen       8888; #与/etc/fdfs/storage.conf 中的 http.server_port=8888 相对应

        # FastDFS 文件访问配置(fastdfs-nginx-module模块)
        location ~/group([0-9])/M00 {
            ngx_fastdfs_module;
        }
    }

每台Storage都需要安装，配置的listen端口保持一致（不一致会出现一个组内的nginx访问另外一个组的无法访问。当天上传跨组访问正常，周末过后周一在尝试无法跨组访问不知道为什么，难道是mod_fastdfs.conf不路由非当天上传文件。）。
group1的Storage的Nginx访问
http://10.1.6.212:9999/group1/M00/00/00/CgEG1FlLMfaAIkfuAACTpuD4VrU630.jpg
http://10.1.6.212:9999/group2/M00/00/00/CgEG1llLMfKAWqrmAAHTvZk6Ln4301.jpg
http://10.1.6.213:9999/group1/M00/00/00/CgEG1FlLMfaAIkfuAACTpuD4VrU630.jpg
http://10.1.6.213:9999/group2/M00/00/00/CgEG1llLMfKAWqrmAAHTvZk6Ln4301.jpg

group2的Storage的Nginx访问
http://10.1.6.214:9999/group1/M00/00/00/CgEG1FlLMfaAIkfuAACTpuD4VrU630.jpg
http://10.1.6.214:9999/group2/M00/00/00/CgEG1llLMfKAWqrmAAHTvZk6Ln4301.jpg
http://10.1.6.215:9999/group1/M00/00/00/CgEG1FlLMfaAIkfuAACTpuD4VrU630.jpg
http://10.1.6.215:9999/group2/M00/00/00/CgEG1llLMfKAWqrmAAHTvZk6Ln4301.jpg

10.1.6.212与10.1.6.213是group1

10.1.6.214与10.1.6.215是group2

5、Tracker在跟踪器节点（10.1.6.210、10.1.6.211）上 安装 Nginx

---

#安装nginx需要的一些模块依赖的lib库
yum -y install pcre pcre-devel
yum -y install zlib zlib-devel
yum -y install openssl openssl-devel
#解压ngx_cache_purge
cd /usr/local/src
tar -zxvf ngx_cache_purge-2.3.tar.gz
#解压nginx
tar -zxvf nginx-1.12.0.tar.gz
#编译安装nginx（添加ngx_cache_purge模块）
cd nginx-1.12.0
#配置参考（Nginx安装.md）
./configure --prefix=/usr/local/nginx --add-module=/usr/local/src/ngx_cache_purge-2.3
make && make install

配置nginx.conf

vi /etc/nginx/nginx.conf
#配置如下
user  root; #root防止404错误
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    #设置缓存
    server_names_hash_bucket_size 128;
    client_header_buffer_size 32k;
    large_client_header_buffers 4 32k;
    client_max_body_size 300m;
    proxy_redirect off;
    proxy_set_header Host $http_host;
    proxy_set_header X-Real-IP $remote_addr;
    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_connect_timeout 90;
    proxy_send_timeout 90;
    proxy_read_timeout 90;
    proxy_buffer_size 16k;
    proxy_buffers 4 64k;
    proxy_busy_buffers_size 128k;
    proxy_temp_file_write_size 128k; #设置缓存存储路径、存储方式、分配内存大小、磁盘最大空间、缓存期限 
    proxy_temp_path /var/cache/nginx/proxy_temp/tmp;
    proxy_cache_path /var/cache/nginx/proxy_temp levels=1:2 keys_zone=http-cache:200m inactive=1d max_size=30g;

    #设置 group1 的服务器
    upstream fdfs_group1 {
         server 10.1.6.212:9999 weight=1 max_fails=2 fail_timeout=30s;
         server 10.1.6.213:9999 weight=1 max_fails=2 fail_timeout=30s;
    }

    #设置 group2 的服务器
    upstream fdfs_group2 {
         server 10.1.6.214:9999 weight=1 max_fails=2 fail_timeout=30s;
         server 10.1.6.215:9999 weight=1 max_fails=2 fail_timeout=30s;
    }

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;

        location / {
            root   html;
            index  index.html index.htm;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        #设置 group 的负载均衡参数
        location /group1/M00 {
            proxy_next_upstream http_502 http_504 error timeout invalid_header;
            proxy_cache http-cache;
            proxy_cache_valid  200 304 12h;
            proxy_cache_key $uri$is_args$args;
            proxy_pass http://fdfs_group1;
            expires 30d;
        }

        location /group2/M00 {
            proxy_next_upstream http_502 http_504 error timeout invalid_header; proxy_cache http-cache;
            proxy_cache_valid 200 304 12h;
            proxy_cache_key $uri$is_args$args;
            proxy_pass http://fdfs_group2;
            expires 30d;
        }

        #设置清除缓存的访问权限
        location ~/purge(/.*) {
            #allow 127.0.0.1;
            allow all;
            deny all;
            proxy_cache_purge http-cache $1$is_args$args;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }


    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}


    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}

#按配置要求，创建对应缓存目录
mkdir -p /var/cache/nginx/proxy_temp
mkdir -p /var/cache/nginx/proxy_temp/tmp

6、配置Keepalived + Nginx高可用

---

10.1.6.216、10.1.6.217

keepalived安装：http://www.cnblogs.com/ddrsql/p/7075414.html
nginx安装：http://www.cnblogs.com/ddrsql/p/7077622.html、

nginx.conf配置

vi /etc/nginx/nginx.conf
#配置如下
user  root;
worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  65;

    ## FastDFS Tracker Proxy
    upstream fastdfs_tracker {
         server 10.1.6.210:80 weight=1 max_fails=2 fail_timeout=30s;
         server 10.1.6.211:80 weight=1 max_fails=2 fail_timeout=30s;
    }

    server {
        listen       80;
        server_name  localhost;
        location / {
            root html;
            index  index.html index.htm;
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root html;
        }

        ## FastDFS Proxy
        location /dfs {
            root   html;
            index  index.html index.htm;
            proxy_pass  http://fastdfs_tracker/;
            proxy_set_header Host  $http_host;
            proxy_set_header Cookie $http_cookie;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            client_max_body_size  300m;
        }
    }
}

keepalived.conf配置

VI_1与VI_2的"state"值在另外一台机器互换。
10.1.6.216(VI_1：state MASTER； VI_2：state BACKUP)
10.1.6.217(VI_1：state BACKUP； VI_2：state MASTER)

vi /etc/keepalived/keepalived.conf
#配置如下
! Configuration File for keepalived

vrrp_script nginx {
    script "/etc/keepalived/nginx.sh"
    interval 2 # 检测间隔2s
    weight -5 # 若检测失败权重减低5
    fall 3 # 检测失败3次就定义为down状态
    rise 2 # 检测失败后,检测成功超过2次就定义为up状态
}

vrrp_instance VI_1 {
    state MASTER # master_server
    interface eth0
    virtual_router_id 51
    priority 100 # 权重值，值大的优先级高
    advert_int 2 # 检测时间间隔2s
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.1.6.218 # VIP
    }
    track_script {
       nginx # 检测脚本
    }
}

vrrp_instance VI_2 {
    state BACKUP # backup_server
    interface eth0
    virtual_router_id 52
    priority 98
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.1.6.219
    }
    track_script {
       nginx
    }
}

nginx.sh

vi /etc/keepalived/nginx.sh
#添加执行权限
chmod +x /etc/keepalived/nginx.sh
#配置如下
#!/bin/bash
status=$(ss -lnp | grep -c 'nginx')
if [ ${status} == 0 ]; then
    service nginx start
    sleep 2
    status=$(ss -lnp | grep -c 'nginx')
    if [ ${status} == 0 ]; then
        service keepalived stop
    fi
fi

通过虚拟ip访问
http://10.1.6.218/dfs/group2/M00/00/00/CgEG11la-OqEBSdfAAAAAJk6Ln4052.jpg