一、用到工具elasticsearch-curator
二、配置ES連接信息
[root@iimesnode01 ~]# cat /opt/elasticsearch-curator/config.yaml
client:
hosts:
- 127.0.0.1
port: 9200
url_prefix:
use_ssl: False
certificate:
client_cert:
client_key:
ssl_no_validate: False
http_auth:
timeout: 30
master_only: False
logging:
loglevel: INFO
logfile: /var/log/elasticsearch-curator.log
logformat: default
blacklist: ['elasticsearch', 'urllib3']三、配置動作文件
1、刪除未來的數據 我只保留未來1天的索引
[root@iimesnode01 ~]# cat /opt/elasticsearch-curator/active-younger.yaml
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: dbl-
- filtertype: age
source: name
direction: younger
timestring: '%Y-%m-%d'
unit: days
unit_count: 1
2:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: log-
- filtertype: age
source: name
direction: younger
timestring: '%Y-%m-%d'
unit: days
unit_count: 1
3:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: text-
- filtertype: age
source: name
direction: younger
timestring: '%Y-%m-%d'
unit: days
unit_count: 1
4:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: uint-
- filtertype: age
source: name
direction: younger
timestring: '%Y-%m-%d'
unit: days
unit_count: 1
5:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: str-
- filtertype: age
source: name
direction: younger
timestring: '%Y-%m-%d'
unit: days
unit_count: 12、刪除歷史索引 我只保留最近3天的數據
[root@iimesnode01 ~]# cat /opt/elasticsearch-curator/active-older.yaml
---
# Remember, leave a key empty if there is no value. None will be a string,
# not a Python "NoneType"
#
# Also remember that all examples have 'disable_action' set to True. If you
# want to use this action as a template, be sure to set this to False after
# copying it.
actions:
1:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: dbl-
- filtertype: age
source: name
direction: older
timestring: '%Y-%m-%d'
unit: days
unit_count: 3
2:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: log-
- filtertype: age
source: name
direction: older
timestring: '%Y-%m-%d'
unit: days
unit_count: 3
3:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: text-
- filtertype: age
source: name
direction: older
timestring: '%Y-%m-%d'
unit: days
unit_count: 3
4:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: uint-
- filtertype: age
source: name
direction: older
timestring: '%Y-%m-%d'
unit: days
unit_count: 3
5:
action: delete_indices
description: >-
Delete indices older than 120 days (based on index name), for logstash-
prefixed indices. Ignore the error if the filter does not result in an
actionable list of indices (ignore_empty_list) and exit cleanly.
options:
ignore_empty_list: True
disable_action: False
filters:
- filtertype: pattern
kind: prefix
value: str-
- filtertype: age
source: name
direction: older
timestring: '%Y-%m-%d'
unit: days
unit_count: 3三、添加計劃任務定時清理
#每週六凌晨3點清理未來的數據以及每天凌晨清理歷史索引只保留最近3天
[root@iimesnode01 ~]# crontab -l
0 3 * * 6 /usr/bin/curator --config /opt/elasticsearch-curator/config.yaml /opt/elasticsearch-curator/active-younger.yaml
0 2 * * * /usr/bin/curator --config /opt/elasticsearch-curator/config.yaml /opt/elasticsearch-curator/active-older.yaml