openresty ngx_http_upstream_check_module 模块安装
实际上测试了tengine以及其他的ngx_http_upstream_check_module发现都不好使，反而发现ngx_healthcheck_module 值得使用
构建
基于docker，复用了openresty 官方的dockerfile 构建模式，核心流程如下
clone ngx_healthcheck_module 模块，patch 布丁到openresty，执行构建
- dockerfile
集成了部分tengine的模块
# Dockerfile - alpine
# https://github.com/openresty/docker-openresty
ARG RESTY_IMAGE_BASE="alpine"
ARG RESTY_IMAGE_TAG="3.15"
FROM ${RESTY_IMAGE_BASE}:${RESTY_IMAGE_TAG}
LABEL maintainer="Evan Wies <evan@neomantra.net>"
# Docker Build Arguments
ARG RESTY_IMAGE_BASE="alpine"
ARG RESTY_IMAGE_TAG="3.15"
ARG RESTY_VERSION="1.19.9.1"
ARG RESTY_OPENSSL_VERSION="1.1.1n"
ARG RESTY_OPENSSL_PATCH_VERSION="1.1.1f"
ARG RESTY_OPENSSL_URL_BASE="https://www.openssl.org/source"
ARG RESTY_PCRE_VERSION="8.44"
ARG RESTY_PCRE_SHA256="aecafd4af3bd0f3935721af77b889d9024b2e01d96b58471bd91a3063fb47728"
ARG RESTY_J="1"
ARG RESTY_CONFIG_OPTIONS="\
--with-compat \
--with-file-aio \
--with-http_addition_module \
--with-http_auth_request_module \
--with-http_dav_module \
--with-http_flv_module \
--with-http_geoip_module=dynamic \
--with-http_gunzip_module \
--with-http_gzip_static_module \
--with-http_image_filter_module=dynamic \
--with-http_mp4_module \
--with-http_random_index_module \
--with-http_realip_module \
--with-http_secure_link_module \
--with-http_slice_module \
--with-http_ssl_module \
--with-http_stub_status_module \
--with-http_sub_module \
--with-http_v2_module \
--with-http_xslt_module=dynamic \
--with-ipv6 \
--with-mail \
--with-mail_ssl_module \
--with-md5-asm \
--with-pcre-jit \
--with-sha1-asm \
--with-stream \
--with-stream_ssl_module \
--add-module=/tmp/ngx_healthcheck_module-master \
--add-module=/tmp/tengine-2.3.3/modules/ngx_http_concat_module \
--add-module=/tmp/tengine-2.3.3/modules/ngx_http_trim_filter_module \
--add-module=/tmp/tengine-2.3.3/modules/ngx_http_footer_filter_module \
--with-threads \
"
ARG RESTY_CONFIG_OPTIONS_MORE=""
ARG RESTY_LUAJIT_OPTIONS="--with-luajit-xcflags='-DLUAJIT_NUMMODE=2 -DLUAJIT_ENABLE_LUA52COMPAT'"
ARG RESTY_ADD_PACKAGE_BUILDDEPS=""
ARG RESTY_ADD_PACKAGE_RUNDEPS=""
ARG RESTY_EVAL_PRE_CONFIGURE=""
ARG RESTY_EVAL_POST_MAKE=""
# These are not intended to be user-specified
ARG _RESTY_CONFIG_DEPS="--with-pcre \
--with-cc-opt='-DNGX_LUA_ABORT_AT_PANIC -I/usr/local/openresty/pcre/include -I/usr/local/openresty/openssl/include' \
--with-ld-opt='-L/usr/local/openresty/pcre/lib -L/usr/local/openresty/openssl/lib -Wl,-rpath,/usr/local/openresty/pcre/lib:/usr/local/openresty/openssl/lib' \
"
LABEL resty_image_base="${RESTY_IMAGE_BASE}"
LABEL resty_image_tag="${RESTY_IMAGE_TAG}"
LABEL resty_version="${RESTY_VERSION}"
LABEL resty_openssl_version="${RESTY_OPENSSL_VERSION}"
LABEL resty_openssl_patch_version="${RESTY_OPENSSL_PATCH_VERSION}"
LABEL resty_openssl_url_base="${RESTY_OPENSSL_URL_BASE}"
LABEL resty_pcre_version="${RESTY_PCRE_VERSION}"
LABEL resty_pcre_sha256="${RESTY_PCRE_SHA256}"
LABEL resty_config_options="${RESTY_CONFIG_OPTIONS}"
LABEL resty_config_options_more="${RESTY_CONFIG_OPTIONS_MORE}"
LABEL resty_config_deps="${_RESTY_CONFIG_DEPS}"
LABEL resty_add_package_builddeps="${RESTY_ADD_PACKAGE_BUILDDEPS}"
LABEL resty_add_package_rundeps="${RESTY_ADD_PACKAGE_RUNDEPS}"
LABEL resty_eval_pre_configure="${RESTY_EVAL_PRE_CONFIGURE}"
LABEL resty_eval_post_make="${RESTY_EVAL_POST_MAKE}"
COPY ngx_healthcheck_module-master.tar.gz /tmp/ngx_healthcheck_module-master.tar.gz
RUN apk add --no-cache --virtual .build-deps \
build-base \
coreutils \
curl \
gd-dev \
geoip-dev \
libxslt-dev \
linux-headers \
make \
perl-dev \
readline-dev \
zlib-dev \
${RESTY_ADD_PACKAGE_BUILDDEPS} \
&& apk add --no-cache \
gd \
geoip \
libgcc \
libxslt \
zlib \
${RESTY_ADD_PACKAGE_RUNDEPS} \
&& cd /tmp \
&& if [ -n "${RESTY_EVAL_PRE_CONFIGURE}" ]; then eval $(echo ${RESTY_EVAL_PRE_CONFIGURE}); fi \
&& cd /tmp \
&& curl -fSL "${RESTY_OPENSSL_URL_BASE}/openssl-${RESTY_OPENSSL_VERSION}.tar.gz" -o openssl-${RESTY_OPENSSL_VERSION}.tar.gz \
&& tar xzf openssl-${RESTY_OPENSSL_VERSION}.tar.gz \
&& cd openssl-${RESTY_OPENSSL_VERSION} \
&& if [ $(echo ${RESTY_OPENSSL_VERSION} | cut -c 1-5) = "1.1.1" ] ; then \
echo 'patching OpenSSL 1.1.1 for OpenResty' \
&& curl -s https://raw.githubusercontent.com/openresty/openresty/master/patches/openssl-${RESTY_OPENSSL_PATCH_VERSION}-sess_set_get_cb_yield.patch | patch -p1 ; \
fi \
&& if [ $(echo ${RESTY_OPENSSL_VERSION} | cut -c 1-5) = "1.1.0" ] ; then \
echo 'patching OpenSSL 1.1.0 for OpenResty' \
&& curl -s https://raw.githubusercontent.com/openresty/openresty/ed328977028c3ec3033bc25873ee360056e247cd/patches/openssl-1.1.0j-parallel_build_fix.patch | patch -p1 \
&& curl -s https://raw.githubusercontent.com/openresty/openresty/master/patches/openssl-${RESTY_OPENSSL_PATCH_VERSION}-sess_set_get_cb_yield.patch | patch -p1 ; \
fi \
&& ./config \
no-threads shared zlib -g \
enable-ssl3 enable-ssl3-method \
--prefix=/usr/local/openresty/openssl \
--libdir=lib \
-Wl,-rpath,/usr/local/openresty/openssl/lib \
&& make -j${RESTY_J} \
&& make -j${RESTY_J} install_sw \
&& cd /tmp \
&& curl -fSL https://downloads.sourceforge.net/project/pcre/pcre/${RESTY_PCRE_VERSION}/pcre-${RESTY_PCRE_VERSION}.tar.gz -o pcre-${RESTY_PCRE_VERSION}.tar.gz \
&& echo "${RESTY_PCRE_SHA256} pcre-${RESTY_PCRE_VERSION}.tar.gz" | shasum -a 256 --check \
&& tar xzf pcre-${RESTY_PCRE_VERSION}.tar.gz \
&& cd /tmp/pcre-${RESTY_PCRE_VERSION} \
&& ./configure \
--prefix=/usr/local/openresty/pcre \
--disable-cpp \
--enable-jit \
--enable-utf \
--enable-unicode-properties \
&& make -j${RESTY_J} \
&& make -j${RESTY_J} install \
&& cd /tmp \
&& curl -fSL https://openresty.org/download/openresty-${RESTY_VERSION}.tar.gz -o openresty-${RESTY_VERSION}.tar.gz \
&& curl -fSL https://github.com/alibaba/tengine/archive/refs/tags/2.3.3.tar.gz -o tengine-2.3.3.tar.gz \
&& tar xzf openresty-${RESTY_VERSION}.tar.gz \
&& tar xzf tengine-2.3.3.tar.gz \
&& tar xzf ngx_healthcheck_module-master.tar.gz\
&& cd /tmp/openresty-${RESTY_VERSION}/bundle/nginx-1.19.9 \
&& patch -p1 < /tmp/ngx_healthcheck_module-master/nginx_healthcheck_for_nginx_1.19+.patch \
&& cd /tmp/openresty-${RESTY_VERSION} \
&& eval ./configure -j${RESTY_J} ${_RESTY_CONFIG_DEPS} ${RESTY_CONFIG_OPTIONS} ${RESTY_CONFIG_OPTIONS_MORE} ${RESTY_LUAJIT_OPTIONS} \
&& make -j${RESTY_J} \
&& make -j${RESTY_J} install \
&& cd /tmp \
&& if [ -n "${RESTY_EVAL_POST_MAKE}" ]; then eval $(echo ${RESTY_EVAL_POST_MAKE}); fi \
&& rm -rf \
openssl-${RESTY_OPENSSL_VERSION}.tar.gz openssl-${RESTY_OPENSSL_VERSION} \
pcre-${RESTY_PCRE_VERSION}.tar.gz pcre-${RESTY_PCRE_VERSION} \
openresty-${RESTY_VERSION}.tar.gz openresty-${RESTY_VERSION} \
ngx_healthcheck_module-master.tar.gz ngx_healthcheck_module-master \
tengine-2.3.3.tar.gz tengine-2.3.3 \
&& apk del .build-deps \
&& mkdir -p /var/run/openresty \
&& ln -sf /dev/stdout /usr/local/openresty/nginx/logs/access.log \
&& ln -sf /dev/stderr /usr/local/openresty/nginx/logs/error.log
# Add additional binaries into PATH for convenience
ENV PATH=$PATH:/usr/local/openresty/luajit/bin:/usr/local/openresty/nginx/sbin:/usr/local/openresty/bin
# Copy nginx configuration files
COPY nginx.conf /usr/local/openresty/nginx/conf/nginx.conf
COPY nginx.vh.default.conf /etc/nginx/conf.d/default.conf
CMD ["/usr/local/openresty/bin/openresty", "-g", "daemon off;"]
# Use SIGQUIT instead of default SIGTERM to cleanly drain requests
# See https://github.com/openresty/docker-openresty/blob/master/README.md#tips--pitfalls
STOPSIGNAL SIGQUIT
参考使用
- docker-compose
version: '3'
services:
app:
image: dalongrong/openresty-tengine:latest
volumes:
- "./nginx.conf:/usr/local/openresty/nginx/conf/nginx.conf"
privileged: true
cap_add:
- ALL
ports:
- "80:80"
app2:
image: dalongrong/openresty-tengine:latest
app3:
image: dalongrong/openresty-tengine:latest
- nginx.conf
worker_processes 1;
user root;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
gzip on;
upstream cluster1 {
# simple round-robin
server app2:80;
server app3:80;
check interval=1000 rise=2 fall=5 timeout=1000 type=http;
check_http_send "HEAD / HTTP/1.0\r\n\r\n";
check_http_expect_alive http_2xx http_3xx;
}
server {
listen 80;
server_name localhost;
charset utf-8;
default_type text/html;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
client_body_buffer_size 10M;
client_max_body_size 10G;
proxy_buffers 1024 4k;
proxy_read_timeout 300;
proxy_connect_timeout 80;
proxy_pass http://cluster1;
}
location /status {
healthcheck_status;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
}
- 效果
当然直接prometheus metrics 可以直接使用
说明
我已经构建好了一个镜像，可以直接使用dalongrong/openresty-tengine:latest，还是得吐槽下tengine了问题不少，而且基本不维护了（好像今年又要开始维护了。。。。）,如果为了方便监控，推荐也集成上nginx 的vts模块,真得好好学学nginx 模块开发了，不然解决问题就有点费事了
参考资料
https://github.com/zhouchangxun/ngx_healthcheck_module
https://github.com/alibaba/tengine/tree/master/modules
https://github.com/vozlt/nginx-module-vts
