一、用到工具elasticsearch-curator
二、配置ES连接信息
[root@iimesnode01 ~]# cat /opt/elasticsearch-curator/config.yaml
client:
hosts:
- 127.0.0.1
port: 9200
url_prefix:
use_ssl: False
certificate:
client_cert:
client_key:
ssl_no_validate: False
http_auth:
timeout: 30
master_only: False
logging:
loglevel: INFO
logfile: /var/log/elasticsearch-curator.log
logformat: default
blacklist: ['elasticsearch', 'urllib3']
三、配置动作文件
1、删除未来的数据 我只保留未来1天的索引
[root@iimesnode01 ~]# cat /opt/elasticsearch-curator/active-younger.yaml
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: dbl-
- filtertype: age
source: name
direction: younger
timestring: '%Y-%m-%d'
unit: days
unit_count: 1
2:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: log-
- filtertype: age
source: name
direction: younger
timestring: '%Y-%m-%d'
unit: days
unit_count: 1
3:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: text-
- filtertype: age
source: name
direction: younger
timestring: '%Y-%m-%d'
unit: days
unit_count: 1
4:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: uint-
- filtertype: age
source: name
direction: younger
timestring: '%Y-%m-%d'
unit: days
unit_count: 1
5:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: str-
- filtertype: age
source: name
direction: younger
timestring: '%Y-%m-%d'
unit: days
unit_count: 1
2、删除历史索引 我只保留最近3天的数据
[root@iimesnode01 ~]# cat /opt/elasticsearch-curator/active-older.yaml
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: dbl-
- filtertype: age
source: name
direction: older
timestring: '%Y-%m-%d'
unit: days
unit_count: 3
2:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: log-
- filtertype: age
source: name
direction: older
timestring: '%Y-%m-%d'
unit: days
unit_count: 3
3:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: text-
- filtertype: age
source: name
direction: older
timestring: '%Y-%m-%d'
unit: days
unit_count: 3
4:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: uint-
- filtertype: age
source: name
direction: older
timestring: '%Y-%m-%d'
unit: days
unit_count: 3
5:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: str-
- filtertype: age
source: name
direction: older
timestring: '%Y-%m-%d'
unit: days
unit_count: 3
三、添加计划任务定时清理
#每周六凌晨3点清理未来的数据以及每天凌晨清理历史索引只保留最近3天
[root@iimesnode01 ~]# crontab -l
0 3 * * 6 /usr/bin/curator --config /opt/elasticsearch-curator/config.yaml /opt/elasticsearch-curator/active-younger.yaml
0 2 * * * /usr/bin/curator --config /opt/elasticsearch-curator/config.yaml /opt/elasticsearch-curator/active-older.yaml