监控系统搭建
环境介绍
OS: CentOS Linux release 7.2.1511 (Core) kernel: 3.10.0-327.el7.x86_64 host: 10.16.31.41 hostname: localhost
技术选型
流程概要
数据源: 数据来源为运维人员手动编写脚本 或通过某种方式从其他渠道进行数据查询
数据存储: 将数据存储于influxdb数据库中(时序型数据库, X 轴为时间, Y 轴为个人业务数据, 该库数据存在过期策略)
图表展示: grafana读取influxdb等数据源(mysql/es/open-falcon etc.), 通过配置查询语句进行展示
influxdb version: InfluxDB v1.2.0 DOCS: https://docs.influxdata.com/influxdb/v1.2/introduction/getting_started/ grafana version: 5.0.0 DOCS: http://docs.grafana.org/
数据传输
influxdb 概要
database: 数据库
measurement: 数据库中的表
points: 表里面的一行数据
influxDB中独有的一些概念
Point由时间戳(time)、数据(field)和标签(tags)组成
time: 每条数据记录的时间, 为数据插入时间
fields: 各种记录的值, 数据类型类似mysql中 int
tags: 各种有索引的属性, 数据类型类型mysql中 varchar
其中, 至少存在一个tags, 至少存在一个fields
插入数据
插入数据支持多种途径,如 shell(curl 发送post请求)/ python (requests.post方法) / java (httpclinet 包)/ etc.
将数据通过post形式即可存储, 其中, 数据插入成功返回状态码为204
数据查询
通过http://${GRAFANA_SERVER_IP}:3000 即可查看图表展示 或修改查询条件
SHELL DEMO
格式: # 不需要指定时间, time 为数据插入时间 # curl -i -XPOST 'http://${INFLUXDB_SERVER_IP}:8086/write?db=${DATABASE}' --data-binary '${MEASUREMENTS_NAME},${TAG_NAME_1}=${TAG_VALUE_1},${TAG_NAME_N}=${TAG_VALUE_N} ${FIELDS_NAME_1}=${FIELDS_VALUE_1},${FIELDS_NAME_N}=${FIELDS_VALUE_N}' # 需要指定时间, time 为指定插入时间 # curl -i -XPOST 'http://${INFLUXDB_SERVER_IP}:8086/write?db=${DATABASE}' --data-binary '${MEASUREMENTS_NAME},${TAG_NAME_1}=${TAG_VALUE_1},${TAG_NAME_N}=${TAG_VALUE_N} ${FIELDS_NAME_1}=${FIELDS_VALUE_1},${FIELDS_NAME_N}=${FIELDS_VALUE_N} '${TIMESTAMP} ${TIMESTAMP} -> `date +%s`000000000 # 注意事项: fields 与 tags 之间需要空格分割, 多个fields 以"," 分割;多个tags 以"," 分割 示例: # curl -i -XPOST 'http://10.6.31.41:8086/write?db=ops' --data-binary 'test,type=parse useTime=80,isFlinsh=0' # curl -i -XPOST 'http://10.6.31.41:8086/write?db=ops' --data-binary 'test,type=parse useTime=60,isFlinsh=1 '1522047676000000000 HTTP/1.1 204 No Content Content-Type: application/json Request-Id: 8063c15d-30c3-11e8-a158-000000000000 X-Influxdb-Version: 1.2.0 Date: Mon, 26 Mar 2018 07:01:26 GMT
PYTHON DEMO
${TIMESTAMP} -> str(int(time.time())) + "000000000" for row in result: table=row["inst_code"]+"_"+row["trans_code"] sums=str(row["总笔数"]) success=str(row["成功数"]) fails=str(row["失败数"]) Psuccess=str(float(row["成功率"])*100)+"%" rows=[paytable,sums,success,fails,Psuccess] ### ----- 百分数转换 ------------ ## 这里将百分比以int 形式存储,grafana设置百分比类型展示即可 psuccess = rows[4].split("%")[0] ## 定义database influxdb_url = 'http://10.6.31.41:8086/write?db=ops' ## 组装查询插入数据 (注1) idata = rows[0]+",table="+(rows[0])+" sums=" + (rows[1])+",success="+(rows[2])+",Psuccess=" + psuccess +",fails="+(rows[3]) #print(idata) # 发送post 请求 response = requests.post(influxdb_url, data=idata) print response 注1: fields 与 tags 之间需要空格分割, 多个fields 以"," 分割;多个tags 以"," 分割
influxdb启停脚本
#!/bin/bash #-------------CopyRight------------- # Name:Influxdb start \ stop script # Version Number:V1.0 # Type:online # Language:bash shell # Date:2018-03-14 # Author: quanzhang #------------Environment------------ # Terminal: column 80 line 24 # Linux 2.6.32-431.el6.x86_64 # GNU Bash 4.1.2 #----------------------------------- # The working directory INFLUXDB_HOME=$(cd `dirname $0`; pwd) sed -i "s#INFLUXDB_HOME#$INFLUXDB_HOME#g" "$INFLUXDB_HOME"/conf/influxdb.conf # Daemon name, where is the actual executable DAEMON=${INFLUXDB_HOME}/bin/influxd # Configuration file CONFIG=${INFLUXDB_HOME}/conf/influxdb.conf # Log path LOG_PATH=${INFLUXDB_HOME}/logs/influxd.log # PID file for the daemon PIDFILE=${INFLUXDB_HOME}/var/influxd.pid # Process name ( For display ) NAME=influxdb # Daemon options INFLUXD_OPTS= FAIL="\e[1;31m" SUCC="\e[1;32m" END="\033[0m" TRUE=/bin/true FALSE=/bin/false chmod +x ${INFLUXDB_HOME}/bin/influx* # # Source function library. # if [ -f /etc/init.d/functions ]; then . /etc/init.d/functions fi PIDDIR=`dirname $PIDFILE` if [ ! -d "$PIDDIR" ]; then mkdir -p $PIDDIR fi # Max open files OPEN_FILE_LIMIT=65536 # Logging if [ -z "$STDOUT" ]; then STDOUT=/dev/null fi if [ ! -f "$STDOUT" ]; then mkdir -p $(dirname $STDOUT) fi if [ -z "$STDERR" ]; then STDERR=${LOG_PATH} fi if [ ! -f "$STDERR" ]; then mkdir -p $(dirname $STDERR) fi function pidofproc() { if [ $# -ne 3 ]; then action "Expected three arguments, e.g. $0 -p pidfile daemon-name" $FALSE fi PID=`pgrep -f $3` local PIDFILE=`cat $2` if [ "x$PIDFILE" == "x" ]; then return 1 fi if [ "x$PID" != "x" -a "$PIDFILE" == "$PID" ]; then return 0 fi return 1 } function killproc() { if [ $# -ne 3 ]; then action "Expected three arguments, e.g. $0 -p pidfile signal" $FALSE fi PID=`cat $2` /bin/kill -s $3 $PID while true; do pidof `basename $DAEMON` >/dev/null if [ $? -ne 0 ]; then return 0 fi sleep 1 n=$(expr $n + 1) if [ $n -eq 30 ]; then /bin/kill -s SIGKILL $PID return 0 fi done } function log_failure_msg() { action "$@" $FALSE } function log_success_msg() { action "$@" $TRUE } function start() { # Check if config file exist if [ ! -r $CONFIG ]; then log_failure_msg "config file doesn't exist (or you don't have permission to view)" exit 4 fi # Checked the PID file exists and check the actual status of process if [ -e $PIDFILE ]; then PID="$(pgrep -f $PIDFILE)" if test -n "$PID" && kill -0 "$PID" &>/dev/null; then # If the status is SUCCESS then don't need to start again. log_failure_msg "$NAME process is running" exit 0 # Exit fi # if PID file does not exist, check if writable else touch $PIDFILE > /dev/null 2>&1 if [ $? -ne 0 ]; then log_failure_msg "$PIDFILE not writable, check permissions" exit 5 fi fi # Bump the file limits, before launching the daemon. These will carry over to # launched processes. ulimit -n $OPEN_FILE_LIMIT if [ $? -ne 0 ]; then log_failure_msg "set open file limit to $OPEN_FILE_LIMIT" fi log_success_msg "Starting the process $NAME" if which start-stop-daemon > /dev/null 2>&1; then start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $DAEMON -- -pidfile $PIDFILE -config $CONFIG $INFLUXD_OPTS >>$STDOUT 2>>$STDERR & else nohup $DAEMON -pidfile $PIDFILE -config $CONFIG $INFLUXD_OPTS >>$STDOUT 2>>$STDERR & fi log_success_msg "$NAME process was started" } function stop() { # Stop the daemon. if [ -e $PIDFILE ]; then PID="$(pgrep -f $PIDFILE)" if test -n "$PID" && kill -0 "$PID" &>/dev/null; then if killproc -p $PIDFILE SIGTERM && /bin/rm -rf $PIDFILE; then log_success_msg "$NAME process was stopped" else log_failure_msg "$NAME failed to stop service" fi fi else log_failure_msg "$NAME process is not running" fi } function restart() { # Restart the daemon. PID="$(pgrep -f $PIDFILE)" stop while test -n "$PID" && test -d "/proc/$PID" &>/dev/null do echo "Process $PID is still running..." sleep 1 done start } function status() { # Check the status of the process. if [ -e $PIDFILE ]; then PID="$(pgrep -f $PIDFILE)" if test -n "$PID" && test -d "/proc/$PID" &>/dev/null; then log_success_msg "$NAME Process is running" exit 0 else log_failure_msg "$NAME Process is not running" exit 1 fi else log_failure_msg "$NAME Process is not running" exit 3 fi } case $1 in start) start ;; stop) stop ;; restart) restart ;; status) status ;; version) $DAEMON version ;; *) # For invalid arguments, print the usage message. echo "Usage: $0 {start|stop|restart|status|version}" exit 2 ;; esac
grafana启停脚本
#!/bin/bash #-------------CopyRight------------- # Name:Grafana start \ stop script # Version Number:V1.0 # Type:online # Language:bash shell # Date:2018-03-14 # Author:quanzhang #------------Environment------------ # Terminal: column 80 line 24 # Linux 2.6.32-431.el6.x86_64 # GNU Bash 4.1.2 #----------------------------------- # The working directory # GRAFANA_HOME=$(cd `dirname $0`; pwd) GRAFANA_HOME=/opt/grafana PATH=/bin:/usr/bin:/sbin:/usr/sbin NAME=grafana-server DESC="Grafana Server" CONF_DIR=${GRAFANA_HOME}/conf WORK_DIR=${GRAFANA_HOME} DATA_DIR=${GRAFANA_HOME}/data PLUGINS_DIR=${GRAFANA_HOME}/plugins LOG_DIR=${GRAFANA_HOME}/log CONF_FILE=${CONF_DIR}/grafana.ini MAX_OPEN_FILES=10000 PID_FILE_DIR=${GRAFANA_HOME}/var PID_FILE=${PID_FILE_DIR}/$NAME.pid DAEMON=${GRAFANA_HOME}/bin/$NAME PID="" FAIL="\e[1;31m" SUCC="\e[1;32m" END="\033[0m" TRUE=/bin/true FALSE=/bin/false if [ ! -x $DAEMON ]; then echo "Program not installed or not executable" exit 5 fi # # Source function library. # if [ -f /etc/init.d/functions ]; then . /etc/init.d/functions fi DAEMON_OPTS="--pidfile=${PID_FILE} --config=${CONF_FILE} cfg:default.paths.data=${DATA_DIR} cfg:default.paths.logs=${LOG_DIR} cfg:default.paths.plugins=${PLUGINS_DIR}" function isRunning() { status -p $PID_FILE $NAME > /dev/null 2>&1 } case "$1" in start) #action $"Starting $DESC: .... " $TRUE isRunning if [ $? -eq 0 ]; then action "$DESC Already running." $FALSE exit 0 fi # Prepare environment mkdir -p "$LOG_DIR" mkdir -p "$DATA_DIR" mkdir -p "$PID_FILE_DIR" touch "$PID_FILE" if [ -n "$MAX_OPEN_FILES" ]; then ulimit -n $MAX_OPEN_FILES fi # Start Daemon cd $GRAFANA_HOME nohup ${DAEMON} ${DAEMON_OPTS} >> /dev/null 2>&1 & return=$? if [ $return -eq 0 ] then sleep 1 # check if pid file has been written two if ! [[ -s $PID_FILE ]]; then action "FAILED" $FAILED exit 1 fi i=0 timeout=10 # Wait for the process to be properly started before exiting until { cat "$PID_FILE" | xargs kill -0; } >/dev/null 2>&1 do sleep 1 i=$(($i + 1)) if [ $i -gt $timeout ]; then action "FAILED" $FAILED exit 1 fi done fi action $"Starting $DESC: .... " $TRUE exit $return ;; stop) #echo -n "Stopping $DESC ..." if [ -f "$PID_FILE" ]; then PID=`cat $PID_FILE` kill -9 $PID if [ $? -eq 0 ]; then action "Stopping $DESC ..." $TRUE else action "Failed, pls check it." $FAILED exit 1 fi rm -f "$PID_FILE" exit 0 else action "$DESC not running" $FAILED fi exit 0 ;; status) status -p $PID_FILE $NAME exit $? ;; restart|force-reload) if [ -f "$PID_FILE" ]; then $0 stop sleep 1 fi $0 start ;; *) echo "Usage: $0 {start|stop|restart|force-reload|status}" exit 3 ;; esac