Вот итоговый pipeline который я использую:
"onec_techlog_pipeline": {
"description": "onec tech log pipeline",
"processors": [
{
"set": {
"field": "LogRowsID",
"value": [
"{{source}}",
"{{_source.offset}}"
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"%{INT:_ingest.tempmm}:%{BASE10NUM:_ingest.tempss}-%{INT:duration},(%{WORD:event}|%{SPACE:event}),%{INT:level}"
]
}
},
{
"grok": {
"field": "source",
"patterns": [
"%{INT:_ingest.tempyymmddhh}.log"
]
}
},
{
"set": {
"field": "_ingest.tempdate",
"value": "{{_ingest.tempyymmddhh}}{{_ingest.tempmm}}{{_ingest.tempss}}"
}
},
{
"date": {
"field": "_ingest.tempdate",
"target_field": "@timestamp",
"formats": [
"yyMMddHHmmss.SSSSSS"
],
"timezone": "Europe/Moscow"
}
},
{
"set": {
"field": "_id",
"value": "{{_ingest.tempdate}}-{{_source.offset}}"
}
},
{
"grok": {
"field": "message",
"patterns": [
"process=(%{DATA:process},|%{GREEDYDATA:process})"
],
"on_failure": [
{
"set": {
"field": "process",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"p:processName=(%{DATA:processName},|%{GREEDYDATA:processName})",
"ProcessName=(%{DATA:processName},|%{GREEDYDATA:processName})"
],
"on_failure": [
{
"set": {
"field": "processName",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"t:clientID=(%{DATA:clientID},|%{GREEDYDATA:clientID})",
"ClientID=(%{DATA:clientID},|%{GREEDYDATA:clientID})"
],
"on_failure": [
{
"set": {
"field": "clientID",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"t:applicationName=(%{DATA:applicationName},|%{GREEDYDATA:applicationName})"
],
"on_failure": [
{
"set": {
"field": "applicationName",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"t:computerName=(%{DATA:computerName},|%{GREEDYDATA:computerName})"
],
"on_failure": [
{
"set": {
"field": "computerName",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"t:connectID=(%{DATA:connectID},|%{GREEDYDATA:connectID})"
],
"on_failure": [
{
"set": {
"field": "connectID",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"SessionID=(%{DATA:SessionID},|%{GREEDYDATA:SessionID})"
],
"on_failure": [
{
"set": {
"field": "SessionID",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Usr=(%{DATA:Usr},|%{GREEDYDATA:Usr})"
],
"on_failure": [
{
"set": {
"field": "Usr",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Descr=(%{QS:Descr}|%{DATA:Descr},|%{GREEDYDATA:Descr})"
],
"on_failure": [
{
"set": {
"field": "Descr",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"AppID=(%{DATA:AppID},|%{GREEDYDATA:AppID})"
],
"on_failure": [
{
"set": {
"field": "AppID",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Trans=(%{DATA:Trans},|%{GREEDYDATA:Trans})"
],
"on_failure": [
{
"set": {
"field": "Trans",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"WaitConnections=(%{DATA:WaitConnections},|%{GREEDYDATA:WaitConnections})"
],
"on_failure": [
{
"set": {
"field": "WaitConnections",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Regions=(%{DATA:Regions},|%{GREEDYDATA:Regions})"
],
"on_failure": [
{
"set": {
"field": "Regions",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Locks=(%{DATA:Locks},|%{GREEDYDATA:Locks})"
],
"on_failure": [
{
"set": {
"field": "Locks",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"DeadlockConnectionIntersections=(%{DATA:DeadlockConnectionIntersections},|%{GREEDYDATA:DeadlockConnectionIntersections})"
],
"on_failure": [
{
"set": {
"field": "DeadlockConnectionIntersections",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Exception=(%{DATA:Exception},|%{GREEDYDATA:Exception})"
],
"on_failure": [
{
"set": {
"field": "Exception",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Sql=(%{QS:Sql}|%{DATA:Sql},|%{GREEDYDATA:Sql})"
],
"on_failure": [
{
"set": {
"field": "Sql",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"planSQLText=(%{QS:planSQLText}|%{DATA:planSQLText},|%{GREEDYDATA:planSQLText})"
],
"on_failure": [
{
"set": {
"field": "planSQLText",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Rows=(%{DATA:Rows},|%{GREEDYDATA:Rows})"
],
"on_failure": [
{
"set": {
"field": "Rows",
"value": 0
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"RowsAffected=(%{DATA:RowsAffected},|%{GREEDYDATA:RowsAffected})"
],
"on_failure": [
{
"set": {
"field": "RowsAffected",
"value": 0
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"dbpid=(%{DATA:dbpid},|%{GREEDYDATA:dbpid})"
],
"on_failure": [
{
"set": {
"field": "dbpid",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Func=(%{QS:Func}|%{DATA:Func},|%{GREEDYDATA:Func})"
],
"on_failure": [
{
"set": {
"field": "Func",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Sdbl=(%{QS:Sdbl}|%{DATA:Sdbl},|%{GREEDYDATA:Sdbl})"
],
"on_failure": [
{
"set": {
"field": "Sdbl",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"SrcName=(%{DATA:SrcName},|%{GREEDYDATA:SrcName})"
],
"on_failure": [
{
"set": {
"field": "SrcName",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"OSThread=(%{DATA:OSThread},|%{GREEDYDATA:OSThread})"
],
"on_failure": [
{
"set": {
"field": "OSThread",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Prm=(%{QS:Prm}|%{DATA:Prm},|%{GREEDYDATA:Prm})"
],
"on_failure": [
{
"set": {
"field": "Prm",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Context=(%{QS:Context}|%{DATA:Context},|%{GREEDYDATA:Context})"
],
"on_failure": [
{
"set": {
"field": "Context",
"value": ""
}
}
]
}
},
{
"grok": {
"field": "message",
"patterns": [
"Txt=(%{QS:Txt}|%{DATA:Txt},|%{GREEDYDATA:Txt})"
],
"on_failure": [
{
"set": {
"field": "Txt",
"value": ""
}
}
]
}
}
],
"on_failure": [
{
"set": {
"field": "error",
"value": "{{_ingest.on_failure_message}}"
}
}
]
}