filebeat의 xml 데이터 처리. 깔끔.
processors: - include_fields: fields: "message" - decode_xml: field: message target_field: ""{ "@timestamp": "2026-01-06T06:43:46.901Z", "@metadata": { "beat": "filebeat", "type": "_doc", "version": "9.2.1" }, "message": "Jan 5 03:12:34 rocky sysmon[947]: <Event><System><Provider Name=\"Linux-Sysmon\" Guid=\"{ff032593-a8d3-4f13-b0d6-01fc615a0f97}\"/><EventID>5</EventID><Version>3</Version><Level>4</Level><Task>5</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated SystemTime=\"2026-01-04T18:12:34.451516000Z\"/><EventRecordID>8660</EventRecordID><Correlation/><Execution ProcessID=\"947\" ThreadID=\"947\"/><Channel>Linux-Sysmon/Operational</Channel><Computer>rocky</Computer><Security UserId=\"0\"/></System><EventData><Data Name=\"RuleName\">-</Data><Data Name=\"UtcTime\">2026-01-04 18:12:34.454</Data><Data Name=\"ProcessGuid\">{00000000-0000-0000-0000-000000000000}</Data><Data Name=\"ProcessId\">2481</Data><Data Name=\"Image\"><unknown process></Data><Data Name=\"User\">root</Data></EventData></Event>", "event": { "system": { "channel": "Linux-Sysmon/Operational", "level": "4", "task": "5", "opcode": "0", "correlation": "", "computer": "rocky", "security": { "userid": "0" }, "provider": { "guid": "{ff032593-a8d3-4f13-b0d6-01fc615a0f97}", "name": "Linux-Sysmon" }, "eventid": "5", "version": "3", "keywords": "0x8000000000000000", "timecreated": { "systemtime": "2026-01-04T18:12:34.451516000Z" }, "eventrecordid": "8660", "execution": { "processid": "947", "threadid": "947" } }, "eventdata": { "data": [ { "name": "RuleName", "#text": "-" }, { "name": "UtcTime", "#text": "2026-01-04 18:12:34.454" }, { "name": "ProcessGuid", "#text": "{00000000-0000-0000-0000-000000000000}" }, { "name": "ProcessId", "#text": "2481" }, { "name": "Image", "#text": "<unknown process>" }, { "name": "User", "#text": "root" } ] } }}로그스태시로 같은 xml 데이터 처리.
filter { mutate { remove_field => ["@timestamp", "@version", "path", "host"] } xml { force_array => false source => "message" target => "event" }}[2026-01-06T15:53:26,650][WARN ][logstash.filters.xml ][main][f625819b8f2d9177fe8c582c21ef75942b54954d87974d0df6db584ad56a228f] Error parsing xml with XmlSimple {:source=>"message", :value=>"Jan 5 03:12:34 rocky sysmon[947]: <Event><System><Provider Name=\"Linux-Sysmon\" Guid=\"{ff032593-a8d3-4f13-b0d6-01fc615a0f97}\"/><EventID>5</EventID><Version>3</Version><Level>4</Level><Task>5</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated SystemTime=\"2026-01-04T18:12:34.451516000Z\"/><EventRecordID>8660</EventRecordID><Correlation/><Execution ProcessID=\"947\" ThreadID=\"947\"/><Channel>Linux-Sysmon/Operational</Channel><Computer>rocky</Computer><Security UserId=\"0\"/></System><EventData><Data Name=\"RuleName\">-</Data><Data Name=\"UtcTime\">2026-01-04 18:12:34.454</Data><Data Name=\"ProcessGuid\">{00000000-0000-0000-0000-000000000000}</Data><Data Name=\"ProcessId\">2481</Data><Data Name=\"Image\"><unknown process></Data><Data Name=\"User\">root</Data></EventData></Event>\r", :exception=>#<REXML::ParseException: Malformed XML: Content at the start of the document (got 'Jan 5 03:12:34 rocky sysmon[947]: ')Line: 1Position: 42Last 80 unconsumed characters:<Event>>, :backtrace=>["D:/ELK/v9/logstash-9.2.0/vendor/bundle/jruby/3.1.0/gems/rexml-3.4.4/lib/rexml/parsers/baseparser.rb:517:in `pull_event'", "D:/ELK/v9/logstash-9.2.0/vendor/bundle/jruby/3.1.0/gems/rexml-3.4.4/lib/rexml/parsers/baseparser.rb:249:in `pull'", "D:/ELK/v9/logstash-9.2.0/vendor/bundle/jruby/3.1.0/gems/rexml-3.4.4/lib/rexml/parsers/treeparser.rb:21:in `parse'", "D:/ELK/v9/logstash-9.2.0/vendor/bundle/jruby/3.1.0/gems/rexml-3.4.4/lib/rexml/document.rb:468:in `build'", "D:/ELK/v9/logstash-9.2.0/vendor/bundle/jruby/3.1.0/gems/rexml-3.4.4/lib/rexml/document.rb:105:in `initialize'", "org/jruby/RubyClass.java:923:in `new'", "D:/ELK/v9/logstash-9.2.0/vendor/bundle/jruby/3.1.0/gems/xml-simple-1.1.9/lib/xmlsimple.rb:979:in `parse'", "D:/ELK/v9/logstash-9.2.0/vendor/bundle/jruby/3.1.0/gems/xml-simple-1.1.9/lib/xmlsimple.rb:164:in `xml_in'", "D:/ELK/v9/logstash-9.2.0/vendor/bundle/jruby/3.1.0/gems/xml-simple-1.1.9/lib/xmlsimple.rb:203:in `xml_in'", "D:/ELK/v9/logstash-9.2.0/vendor/bundle/jruby/3.1.0/gems/logstash-filter-xml-4.3.2/lib/logstash/filters/xml.rb:196:in `filter'", "D:/ELK/v9/logstash-9.2.0/logstash-core/lib/logstash/filters/base.rb:158:in `do_filter'", "D:/ELK/v9/logstash-9.2.0/logstash-core/lib/logstash/filters/base.rb:176:in `block in multi_filter'", "org/jruby/RubyArray.java:2009:in `each'", "D:/ELK/v9/logstash-9.2.0/logstash-core/lib/logstash/filters/base.rb:173:in `multi_filter'", "org/logstash/config/ir/compiler/AbstractFilterDelegatorExt.java:133:in `multi_filter'", "D:/ELK/v9/logstash-9.2.0/logstash-core/lib/logstash/java_pipeline.rb:317:in `block in start_workers'"]}{ "message" => "Jan 5 03:12:34 rocky sysmon[947]: <Event><System><Provider Name=\"Linux-Sysmon\" Guid=\"{ff032593-a8d3-4f13-b0d6-01fc615a0f97}\"/><EventID>5</EventID><Version>3</Version><Level>4</Level><Task>5</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated SystemTime=\"2026-01-04T18:12:34.451516000Z\"/><EventRecordID>8660</EventRecordID><Correlation/><Execution ProcessID=\"947\" ThreadID=\"947\"/><Channel>Linux-Sysmon/Operational</Channel><Computer>rocky</Computer><Security UserId=\"0\"/></System><EventData><Data Name=\"RuleName\">-</Data><Data Name=\"UtcTime\">2026-01-04 18:12:34.454</Data><Data Name=\"ProcessGuid\">{00000000-0000-0000-0000-000000000000}</Data><Data Name=\"ProcessId\">2481</Data><Data Name=\"Image\"><unknown process></Data><Data Name=\"User\">root</Data></EventData></Event>\r", "tags" => [ [0] "_xmlparsefailure" ]}처리를 못한다. 에러 메시지를 보면 xml 태그 시작 전 문자열 때문에 정상 포맷이 아니라 판단하는 듯. 해당 구간 삭제.
filter { mutate { remove_field => ["@timestamp", "@version", "path", "host"] } mutate { gsub => ["message", "^[^<]+", ""] }
xml { force_array => false source => "message" target => "event" }}[2026-01-06T15:55:20,515][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}{ "event" => { "EventData" => { "Data" => [ [0] { "content" => "-", "Name" => "RuleName" }, [1] { "content" => "2026-01-04 18:12:34.454", "Name" => "UtcTime" }, [2] { "content" => "{00000000-0000-0000-0000-000000000000}", "Name" => "ProcessGuid" }, [3] { "content" => "2481", "Name" => "ProcessId" }, [4] { "content" => "<unknown process>", "Name" => "Image" }, [5] { "content" => "root", "Name" => "User" } ] }, "System" => { "Opcode" => "0", "EventID" => "5", "EventRecordID" => "8660", "Channel" => "Linux-Sysmon/Operational", "Version" => "3", "TimeCreated" => { "SystemTime" => "2026-01-04T18:12:34.451516000Z" }, "Computer" => "rocky", "Level" => "4", "Task" => "5", "Execution" => { "ProcessID" => "947", "ThreadID" => "947" }, "Provider" => { "Guid" => "{ff032593-a8d3-4f13-b0d6-01fc615a0f97}", "Name" => "Linux-Sysmon" }, "Keywords" => "0x8000000000000000", "Security" => { "UserId" => "0" } } }, "message" => "<Event><System><Provider Name=\"Linux-Sysmon\" Guid=\"{ff032593-a8d3-4f13-b0d6-01fc615a0f97}\"/><EventID>5</EventID><Version>3</Version><Level>4</Level><Task>5</Task><Opcode>0</Opcode><Keywords>0x8000000000000000</Keywords><TimeCreated SystemTime=\"2026-01-04T18:12:34.451516000Z\"/><EventRecordID>8660</EventRecordID><Correlation/><Execution ProcessID=\"947\" ThreadID=\"947\"/><Channel>Linux-Sysmon/Operational</Channel><Computer>rocky</Computer><Security UserId=\"0\"/></System><EventData><Data Name=\"RuleName\">-</Data><Data Name=\"UtcTime\">2026-01-04 18:12:34.454</Data><Data Name=\"ProcessGuid\">{00000000-0000-0000-0000-000000000000}</Data><Data Name=\"ProcessId\">2481</Data><Data Name=\"Image\"><unknown process></Data><Data Name=\"User\">root</Data></EventData></Event>\r"}관련 글
댓글 없음:
댓글 쓰기