I am using the Serilog elasticsearch sink (version 8.4.1, elastic 7.8.0) within ASP.NET Core 3.1 with the following configuration:
{
"Serilog": {
"MinimumLevel": {
"Default": "Information",
"Override": {
"Serilog.AspNetCore": "Information"
}
},
"Enrich": [ "FromLogContext" ],
"WriteTo": [
{
"Name": "Logger",
"Args": {
"configureLogger": {
"Filter": [
{
"Name": "ByIncludingOnly",
"Args": {
"expression": "SourceContext = 'Serilog.AspNetCore.RequestLoggingMiddleware'"
}
}
],
"WriteTo": [
{
"Name": "Elasticsearch",
"Args": {
"nodeUris": "http://localhost:9200",
"indexFormat": "request-logs-{0:yyyy.MM.dd}",
"period": 1,
"connectionTimeout": 5,
"typeName": "_doc",
"inlineFields": true,
"restrictToMinimumLevel": "Information"
}
}
]
}
}
}
]
}
}
I'm using the RequestLogging by Serilog.AspNetCore
app.UseSerilogRequestLogging();
and enrich the IDiagnosticContext within a custom middleware with an object that requires special mapping of GeoSpatial and IP properties.
HttpDiagnostics diagnostics = new HttpDiagnostics
{
Host = host.ToString(),
IsHttps = isHttps,
LocalIp = localIpAddress,
LocalPort = localPort,
Protocol = protocol,
RemoteIp = remoteIpAddress,
RemotePort = remotePort,
RequestContentLength = requestContentLength,
RequestContentType = requestContentType,
Scheme = scheme,
UserAgent = userAgent,
ResponseContentLength = responseContentLength,
ResponseContentType = responseContentType
};
this.diagnosticContext.Set("Http", diagnostics, true);
My approach to ending up with a correct mapping of the fields in the logevent was: remodelling the LogEvent type with my custom Object and creating a mapping for an index template with a NEST client.
ElasticClient client = new ElasticClient(new Uri(settings.Uri));
PutIndexTemplateResponse response = client.Indices.PutTemplate(
settings.Name,
p => p.IndexPatterns(settings.IndexPattern)
.Settings(s => s.DefaultPipeline("geoip"))
.Map<SerilogDiagnosticsLogEvent>(m => m.AutoMap()));
The index template mapping is created correctly
{
"_doc": {
"properties": {
"traceId": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"level": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"requestMethod": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"message": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"sourceContext": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"parentId": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"elapsed": {
"type": "double"
},
"spanId": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"@timestamp": {
"type": "date"
},
"requestId": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"http": {
"type": "object",
"properties": {
"responseContentLength": {
"type": "long"
},
"requestContentLength": {
"type": "long"
},
"geoIp": {
"type": "object",
"properties": {
"cityName": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"countryIsoCode": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"regionName": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"location": {
"type": "geo_point"
},
"continentName": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
},
"remoteIp": {
"type": "ip"
},
"localPort": {
"type": "integer"
},
"scheme": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"remotePort": {
"type": "integer"
},
"userAgent": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"protocol": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"responseContentType": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"host": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"isHttps": {
"type": "boolean"
},
"localIp": {
"type": "ip"
},
"requestContentType": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
},
"connectionId": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"messageTemplate": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"requestPath": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"statusCode": {
"type": "integer"
}
}
}
}
So now I am expecting the documents indexed by Serilog to be using the mapping of the index template I have created. But what actually happens is, the indices created are NOT using the correct mapping from the template. So the IP fields are indexed as strings, which results into the GeoIp pipeline not working on the fields.
Now I am asking myself: Is my configuration faulty? Is Serilog always indexing the logevents with THEIR mapping? How do I apply the correct mapping to custom properties within a logevent?
