Filebeat의 processors - 2nd

로그스태시에서 원본 데이터 발생 시간을 기록하려면

filter {
 mutate {
  remove_field => ["host", "path", "@version"]
 }

 dissect {
  mapping => { "message" => '%{} [%{timestamp}] %{}' }
 }
}

[2023-04-09T16:43:37,259][INFO ][logstash.agent           ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
{
    "@timestamp" => 2023-04-09T07:43:37.271785800Z,
       "message" => "37.115.12.63 - - [12/May/2022:21:56:33 +0000] \"GET /presentations/vim/result.php?a=b; HTTP/1.0\" 404 414 \"http://www.semicomplete.com/presentations/Result:+chosen+nickname+%22BeauttWantmes%22;+registered+%28registering+only+mode+is+ON%29;\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10\"\r",
     "timestamp" => "12/May/2022:21:56:33 +0000"
}

date 필터를 사용하면 된다.

filter {
 mutate {
  remove_field => ["host", "path", "@version"]
 }

 dissect {
  mapping => { "message" => '%{} [%{timestamp}] %{}' }
 }

 date {
  match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
 }
}

[2023-04-09T16:47:31,457][INFO ][logstash.agent           ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
{
    "@timestamp" => 2022-05-12T21:56:33.000Z,
       "message" => "37.115.12.63 - - [12/May/2022:21:56:33 +0000] \"GET /presentations/vim/result.php?a=b; HTTP/1.0\" 404 414 \"http://www.semicomplete.com/presentations/Result:+chosen+nickname+%22BeauttWantmes%22;+registered+%28registering+only+mode+is+ON%29;\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10\"\r",
     "timestamp" => "12/May/2022:21:56:33 +0000"
}

beats에선 timestamp 프로세서를 사용하면 됨

processors: 
  - dissect:
      field: "message"
      tokenizer : '%{} [%{timestamp}] %{}'
      target_prefix: ""
  - timestamp:
      field: timestamp
      layouts:
        - '2/Jan/2006:15:04:05 Z'

그런데 바뀌지 않네?

{
  "@timestamp": "2023-04-09T07:50:29.377Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "8.7.0"
  },
  "message": "37.115.12.63 - - [12/May/2022:21:56:33 +0000] \"GET /presentations/vim/result.php?a=b; HTTP/1.0\" 404 414 \"http://www.semicomplete.com/presentations/Result:+chosen+nickname+%22BeauttWantmes%22;+registered+%28registering+only+mode+is+ON%29;\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10\"",
  "timestamp": "12/May/2022:21:56:33 +0000"
}

timezone 포맷을 원본 데이터와 같은 형식으로 바꿔봤다.

processors: 
  - dissect:
      field: "message"
      tokenizer : '%{} [%{timestamp}] %{}'
      target_prefix: ""
  - timestamp:
      field: timestamp
      layouts:
        - '2/Jan/2006:15:04:05 +0000'

{
  "@timestamp": "2022-05-12T21:56:33.000Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "_doc",
    "version": "8.7.0"
  },
  "message": "37.115.12.63 - - [12/May/2022:21:56:33 +0000] \"GET /presentations/vim/result.php?a=b; HTTP/1.0\" 404 414 \"http://www.semicomplete.com/presentations/Result:+chosen+nickname+%22BeauttWantmes%22;+registered+%28registering+only+mode+is+ON%29;\" \"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.10 (KHTML, like Gecko) Chrome/8.0.552.224 Safari/534.10\"",
  "timestamp": "12/May/2022:21:56:33 +0000"
}

관련 글

• Filebeat의 processors - 3rd
• Filebeat의 processors