ELK使用記錄-Kibana操作

自己在項目中用到的一些東西。分享一下

查詢所有

GET index_search_model/_search
{
  "query": {
    "match_all": {}
  }
}

基本條件查詢

GET index_search_model/_search
{
  "query": {
    "match": {
      "search_type": 1
    }
  },
  # 取消最大10000條限制
  "track_total_hits": true
}

分頁+區間+條件

GET index_search_model/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "match": {
            "content_name": {
              "query": "一切"
            }
          }
        }
      ]
    },
    "range": {
      "search_type": {
        "gte": 3,
        "lte": 4
      }
    }
  },
  "_source": ["content_id","content_name","search_type"],
  "from": 0,
  "size": 2
}

in操作

{
  terms: {
    user_id: user_id1,user_id2
  }
}

分組聚合

GET index_xxx/_search
{
  "size": 0,
  "query": {
      "bool": {
          "must": [{
              "range": {
                  "create_time": {
                      "gte": 1602724120662,
                      "lte": 1602866828039
                  }
              }
          }]
      }
  },
  "aggs": {
    "group_by_type": {
      "terms": {
        "field": "ad_type"
      },
      "aggs": {
        "group_by_position": {
          "terms": {
            "field": "ad_position"
          },
          "aggs": {
            "group_by_pv_uv_type": {
              "terms": {
                "field": "pv_uv_page_type"
              },
              "aggs":{
                "group_by_time":{
                  "date_histogram":{
                    "field": "create_time",
                    "interval": "1h", 
                    "min_doc_count": 0
                  },
                  "aggs": {
                    "group_by_pv_uv_type2": {
                      "cardinality": {
                        "field": "user_id"
                      }
                    },
                    "pvCount":{
                      "value_count": {
                        "field": "pv_uv_page_type"
                      }
                    },
                    "times":{
                        "field": "create_time"
                    },
                    "sales_bucket_sort": {
                      "bucket_sort": {
                          "sort": [
                            {"pvCount": {"order": "desc"}}
                          ],
                          "from": 0, 
                          "size":4
                      }
                    }
                  }
                }
              }
            }
          }
        }  
      }
    }
  }
}

設置操作

#查看索引設置
GET adanalysis/_settings 

#桶聚合後的hits返回數據最大數
PUT bookanalysis/_settings
{
  "index.max_inner_result_window" : "9999"
} 

#查詢結果的最大返回數
PUT bookdetailanalysis/_settings
{
  "index.max_result_window" : "2147483647"
}


#查詢桶的最大返回數
PUT /_cluster/settings
{
  "persistent":{
    "search.max_buckets": 100000
  }
}

#fielddata 斷路器默認設置堆的 60% 作爲 fielddata 大小的上限。
PUT /_cluster/settings
{
  "persistent": {
    "indices.breaker.fielddata.limit": "60%"
  }
} 

#request 斷路器估算需要完成其他請求部分的結構大小，例如創建一個聚合桶，默認限制是堆內存的 40%。
PUT /_cluster/settings
{
  "persistent": {
    "indices.breaker.request.limit": "40%"
  }
} 

#total 揉合 request 和 fielddata 斷路器保證兩者組合起來不會使用超過堆內存的 70%。
PUT /_cluster/settings
{
  "persistent": {
    "indices.breaker.total.limit": "70%"
  }
}  

indices.breaker.fielddata.limit 默認值是JVM堆內存的60%,
  注意爲了讓設置正常生效，一定要確保 indices.breaker.fielddata.limit 的值
  大於 indices.fielddata.cache.size 的值。
  否則的話，fielddata 大小一到 limit 閾值就報錯，就永遠道不了 size 閾值，
  無法觸發對舊數據的交換任務了。 
  
 
# 索引刷新時間,-1取消刷新時間。 例如：5s
PUT /adanalysisnew/_settings
{
  "refresh_interval": -1
} 


# 修改索引副本數
PUT /adanalysisdata/_settings
{
  "number_of_replicas":1
} 

刪除操作

# 參數解釋如下:
# wait_for_completion=false : 開啓異步
# scroll_size=2000 : 刪除文檔數
# slices=2 : 線程數量
# conflicts=proceed : 衝突繼續???

POST adanalysis/_delete_by_query?wait_for_completion=false&conflicts=proceed&scroll_size=2000&slices=2
{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "create_time": {
              "gte": 1607529600000,
              "lte": 1607702400000
            }
          }
        }
      ]
    }
  }
} 

其它操作

索引別名

# 單操作
PUT index/_alias/indexbieming

# 多操作
POST /_aliases
{
    "actions": [
        { "remove": { "index": "adanalysisnew", "alias": "adanalysisdata" }},
        { "add":    { "index": "adanalysisnews", "alias": "adanalysisdata" }}
    ]
} 

創建索引

# number_of_shards 創建分片的數量，在網上查詢過分片的大小最好是維持在20G-50G左右

PUT /index/
{
  "settings" : {
    "index":{
      "number_of_shards" : 6, # 分片
      "number_of_replicas": 0 # 副本分片
    }
  },
  "properties": {
      "create_time": {
        "type": "long"
      },
      "device_id": {
        "type": "keyword"
      },
      "user_id": {
        "type": "long"
      }
  }
}

數據遷移

POST /_reindex?slices=5&wait_for_completion=false
{
  "source": {
    "index": "source_index",
    "size" : 10000
  },
  "dest": {
    "index": "new_index"
  }
}

任務操作

# 查看所有的進行中的刪除任務
GET _tasks?detailed=true&actions=*/delete/byquery

# 查看所有的進行中的任務
GET _tasks?detailed=true&actions=*

# 查看某個父任務(多線程下使用)
GET /_tasks?parent_task_id=Ljd1T6xxTpG0XEZcsGeGrA:85553680 

# 查詢某個任務
GET _tasks/N1NRSrCYQzCXE1d4rb0IEQ:444398383

# 取消任務
POST  _tasks/N1NRSrCYQzCXE1d4rb0IEQ:444398383/_cancel 
 
 

刪除scroll深分頁id

DELETE /_search/scroll/_all

其它優秀文檔

Elasticsearch數據類型及其屬性: https://www.jianshu.com/p/01f489c46c38

reindex 重建索引：https://blog.csdn.net/winterking3/article/details/108242124