制作Elasticsearch索引初始化docker镜像
背景
由于公司使用自己的编排系统（基于docker），所以在业务服务启动时，需要对某些服务对象进行初始化，本文就对es索引的初始做一个小的分享；
大致流程
初始化步骤主要是基于docker启动一个容器执行自己编写好的sh脚本
基础镜像选择
由于es大部分操作都是基于es的rest api进行，所以，选择alpine:latest这个轻量级操作系统进行操作
dockerfile如下：
FROM alpine:latest
RUN apk add --update curl && rm -rf /var/cache/apk/*
ADD ./*.sh /scripts/
ADD ./*.json /scripts/
ADD ./template/ /scripts/template/
RUN mkdir -p /devops_runonce
CMD sh -c 'echo start > /devops_runonce/status; /scripts/init-index.sh; echo ok > /devops_runonce/status'
镜像中将后缀为sh、json,template目录所有文件拷贝到对应目录，然后执行init-index.sh脚本，主要操作就是init-index.sh，你的任何初始化逻辑都在这里；
初始化脚本
#!/bin/sh
init_elastic_address() {
if [ ${MEGCITY_ELASTIC_ADDRESS} ];
then
echo "MEGCITY_ELASTIC_ADDRESS = $MEGCITY_ELASTIC_ADDRESS"
else
echo "no env: MEGCITY_ELASTIC_ADDRESS"
exit 1
fi
if [ ${MEGCITY_ELASTIC_USER} ];
then
echo "MEGCITY_ELASTIC_USER = $MEGCITY_ELASTIC_USER"
else
MEGCITY_ELASTIC_USER=elastic
fi
}
init_elastic_setting() {
if [ ${NUMBER_OF_SHARDS} ];
then
echo "NUMBER_OF_SHARDS = $NUMBER_OF_SHARDS"
else
NUMBER_OF_SHARDS=1
fi
if [ ${NUMBER_OF_REPLICAS} ];
then
echo "NUMBER_OF_REPLICAS = $NUMBER_OF_REPLICAS"
else
NUMBER_OF_REPLICAS=0
fi
if [ ${MAX_RESULT_WINDOW} ];
then
echo "MAX_RESULT_WINDOW = $MAX_RESULT_WINDOW"
else
MAX_RESULT_WINDOW=5000000
fi
}
# 初始化elastic地址配置
init_elastic_address
# 初始化elastic配置配置
init_elastic_setting
#根据.json后缀文件数据创建索引
# 参数：索引名称、分片数量、副本数量、最大返回结果数
create_index(){
echo "create index:$1 with shards: $2 replicas: $3! window: $4!"
mappings=`cat /scripts/"$1"'.json'`
curl --insecure -u ${MEGCITY_ELASTIC_USER}:${DEVOPS_INFRA_PASSWORD} -XPUT $MEGCITY_ELASTIC_ADDRESS'/'"$1"'?pretty' -H 'Content-Type: application/json' -d'
{
"settings" : {
"index" : {
"number_of_shards" : '"$2"',
"number_of_replicas" : '"$3"',
"max_result_window": '"$4"'
}
},
"mappings" : '"$mappings"'
}
'
curl --insecure -u ${MEGCITY_ELASTIC_USER}:${DEVOPS_INFRA_PASSWORD} -XPOST $MEGCITY_ELASTIC_ADDRESS'/_aliases?pretty' -H 'Content-Type: application/json' -d'
{
"actions" : [
{ "add" : { "index" : "'"$1"'", "alias" : "alias_'"$1"'" } }
]
}
'
}
# 初始化索引：在执行create_field_template函数后执行
# 因为create_field_template只是创建一个索引模板，索引并没有真正创建
init_index(){
echo "create index:$2"
mappings=`cat /scripts/"$1"'.json'`
echo "init $2"
curl --insecure -u ${MEGCITY_ELASTIC_USER}:${DEVOPS_INFRA_PASSWORD} -XPUT $MEGCITY_ELASTIC_ADDRESS'/'"$2"'?pretty' -H 'Content-Type: application/json' -d'
{
"mappings" : '"$mappings"'
}
'
}
# 根据模板文件创建索引
#配置字段的模板alarm_template.json
create_field_template(){
echo "create template:/scripts/template/$1_template.json"
echo "with shards: $2 replicas: $3! window: $4!"
mappings=`cat /scripts/template/"$1"_template.json`
mappings=`cat /scripts/template/"$1"_template.json \
| sed 's/SHARDS_NUM/'"$2"'/g' | sed 's/REPLICAS_NUM/'"$3"'/g' | sed 's/WINDOW_NUM/'"$4"'/g' \
`
curl --insecure -u ${MEGCITY_ELASTIC_USER}:${DEVOPS_INFRA_PASSWORD} -XPUT ${MEGCITY_ELASTIC_ADDRESS}'/_template/'"$1"'?pretty' -H 'Content-Type: application/json' -d'
'"$mappings"'
'
}
# 执行创建索引操作：基于.json
create_index megcity-event ${NUMBER_OF_SHARDS} ${NUMBER_OF_REPLICAS} ${MAX_RESULT_WINDOW}
# 执行创建索引操作：基于template
create_field_template alarm ${NUMBER_OF_SHARDS} ${NUMBER_OF_REPLICAS} ${MAX_RESULT_WINDOW}
#
init_index megcity-alarm megcity-alarm
以上shell脚本中有一定的注释说明
es索引模板
es索引模板：
就是把已经创建好的某个索引的参数设置(settings)和索引映射(mapping)保存下来作为模板, 在创建新索引时, 指定要使用的模板名, 就可以直接重用已经定义好的模板中的设置和映射.
索引模板一般用在时间序列相关的索引中.
—— 也就是说, 如果你需要每间隔一定的时间就建立一次索引, 你只需要配置好索引模板, 以后就可以直接使用这个模板中的设置, 不用每次都设置settings和mappings.
{
"index_patterns": [
"megcity-alarm"
],
"order": 0,
"mappings": {
"properties": {
"causeId": {
"type": "keyword"
},
"caseNumber": {
"type": "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"videoId": {
"type": "keyword"
},
"causeArea": {
"properties":{
"x":{
"type":"float"
},
"y":{
"type":"float"
}
}
},
"createTime": {
"type": "date",
"format": "epoch_millis"
},
"location": {
"type": "geo_point"
},
"state": {
"type": "integer"
}
}
},
"settings" : {
"index" : {
"number_of_shards": "SHARDS_NUM",
"number_of_replicas": "REPLICAS_NUM",
"max_result_window": "WINDOW_NUM",
"sort": {
"field": [
"createTime",
"caseNumber.keyword"
],
"order": [
"desc",
"desc"
]
}
}
},
"aliases": {
"alias_alarm": {}
}
}
json
这个其实就是索引的mapping
{
"properties": {
"causeId": {
"type": "keyword"
},
"caseNumber": {
"type": "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"videoId": {
"type": "keyword"
},
"causeArea": {
"properties":{
"x":{
"type":"float"
},
"y":{
"type":"float"
}
}
},
"createTime": {
"type": "date",
"format": "epoch_millis"
},
"location": {
"type": "geo_point"
},
"state": {
"type": "integer"
}
}
}
目录结构
