日志
elasticsearch自动按天创建索引脚本
|
elasticsearch保存在一个索引中数据量太大无法查询，现在需要将索引按照天来建，查询的时候关联查询即可
有时候es集群创建了很多索引，删不掉，如果是测试环境或者初始化es集群(清空所有数据)，可以直接关掉elastic进程，然后删除nodes下面的所有数据，再次启动集群即可，记录一下避免忘记
导出mapping信息放到/root/index_mapping目录下
1.导出的语句
yum install epel-release -y
yum install nodejs -y
yum install nodejs npm -y
npm install elasticdump -y
/root/node_modules/elasticdump/bin/elasticdump --ignore-errors=true --scrollTime=120m --bulk=true --input=http://10.30.138.62:9200/.kibana --output=mapping.json --type=mapping
2.创建索引的脚本如下
#!/bin/bash
# 1.创建今天和明天的索引
# 2.删除3天以前的索引
today_date=`date '+%Y%m%d'`
tomorrow_date=`date -d tomorrow +%Y%m%d`
# 需要导入索引的es集群服务器ip
es_ip=10.10.33.84
# 找到具体的index目录和index，对这个index进行处理
# start log
echo "${today_date} create index start">> /data/scripts/create_index.log
for FULLPATH in `ls /root/index_mapping/*.json`;
do
# 文件名
FILE=${FULLPATH##*/}
# 去掉文件名后缀
FILE_NAME=${FILE%%.*}
# 找到类似push:user:req的索引名称
FILE_INDEX=`echo $FILE_NAME|sed 's/_/:/g'`
FILE_INDEX_NAME=${FILE_INDEX%:*}
echo "create index ${FILE_INDEX_NAME} start" >> /data/scripts/create_index.log
/root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:${today_date} --type=mapping
/root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:${tomorrow_date} --type=mapping
# /root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:20170905 --type=mapping
# /root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:20170904 --type=mapping
# /root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:20170903 --type=mapping
# /root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:20170906 --type=mapping
done
# end log
echo "${today_date} create index end">> /data/scripts/create_index.log
three_daysago=`date -d '3 days ago' +%Y%m%d`
echo $three_daysago
echo "delete index ${three_daysago} start" >> /data/scripts/create_index.log
curl -XDELETE "http://10.10.33.84:9200/*${three_daysago}*"
echo "delete index ${three_daysago} end" >> /data/scripts/create_index.log
某些索引特殊处理，改进后的脚本
#!/bin/bash
# 1.创建今天和明天的索引
# 2.删除3天以前的索引
today_date=`date '+%Y%m%d'`
tomorrow_date=`date -d tomorrow +%Y%m%d`
today_month=`date '+%Y%m'`
# 需要导入索引的es集群服务器ip
es_ip=10.10.33.84
# 找到具体的index目录和index，对这个index进行处理
# start log
echo "${today_date} create index start">> /data/scripts/create_index.log
for FULLPATH in `ls /root/index_mapping/*.json`;
do
# 文件名
FILE=${FULLPATH##*/}
# 去掉文件名后缀
FILE_NAME=${FILE%%.*}
# 找到类似push:user:req的索引名称
FILE_INDEX=`echo $FILE_NAME|sed 's/_/:/g'`
FILE_INDEX_NAME=${FILE_INDEX%:*}
echo "create index ${FILE_INDEX_NAME} start" >> /data/scripts/create_index.log
/root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:${today_date} --type=mapping
/root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:${tomorrow_date} --type=mapping
# /root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:20170905 --type=mapping
# /root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:20170904 --type=mapping
# /root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:20170903 --type=mapping
# /root/node_modules/elasticdump/bin/elasticdump --input=${FULLPATH} --output=http://${es_ip}:9200/${FILE_INDEX_NAME}:20170906 --type=mapping
done
# end log
echo "${today_date} create index end">> /data/scripts/create_index.log
three_daysago=`date -d '3 days ago' +%Y%m%d`
echo $three_daysago
echo "delete index ${three_daysago} start" >> /data/scripts/create_index.log
# 索引保留3天
curl -XDELETE "http://${es_ip}:9200/voice*${three_daysago}*"
curl -XDELETE "http://${es_ip}:9200/script*${three_daysago}*"
curl -XDELETE "http://${es_ip}:9200/push*${three_daysago}*"
curl -XDELETE "http://${es_ip}:9200/advert*${three_daysago}*"
curl -XDELETE "http://${es_ip}:9200/user*${three_daysago}*"
curl -XDELETE "http://${es_ip}:9200/speech*${three_daysago}*"
curl -XDELETE "http://${es_ip}:9200/user*${three_daysago}*"
echo "delete index ${three_daysago} end" >> /data/scripts/create_index.log
# bin开头的索引保留7天
seven_daysago=`date -d '7 days ago' +%Y%m%d`
echo $seven_daysago
echo "delete index ${seven_daysago} start" >> /data/scripts/create_index.log
curl -XDELETE "http://${es_ip}:9200/bin*${seven_daysago}*"
echo "delete index ${seven_daysago} end" >> /data/scripts/create_index.log
curl -XPUT "http://${es_ip}:9200/*${today_month}*/_settings" -d '{"number_of_replicas": 0}'
echo "setting replication 0 ${today_date}" >> /data/scripts/create_index.log
查看索引结构的命令：
[root@u04es01 ~]# curl 10.19.142.99:9200/bin:user:task:20171230/_mapping?pretty
{
"bin:user:task:20171230" : {
"mappings" : {
"_default_" : {
"properties" : {
"appId" : {
"type" : "keyword",
"store" : true
},
"taskFlag" : {
"type" : "integer",
"store" : true
},
"taskId" : {
"type" : "keyword",
"store" : true
},
"time" : {
"type" : "date",
"store" : true,
"format" : "yyyy-MM-dd HH:mm:ss.SSS.Z"
},
"uuid" : {
"type" : "keyword",
"store" : true
}
}
}
}
}
}
---------------------
作者：郑子明
来源：CSDN
原文：https://blog.csdn.net/reblue520/article/details/80553317
版权声明：本文为博主原创文章，转载请附上博文链接！
/2 
