Read stream of json log objects using c#

Question

I'm using ModSecurity and my Audit Log logs stream of json objects like the ones below:

{"transaction":{"time":"28/Mar/2017:15:39:04 +0200","transaction_id":"18158513699705323558","remote_address":"","remote_port":80,"local_address":"127.0.0.1","local_port":80},"request":{"request_line":"GET /iisstart.htm HTTP/1.1","headers":{"Connection":"keep-alive","Content-Length":"0","Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Accept-Encoding":"gzip, deflate, sdch, br","Accept-Language":"sv-SE,sv;q=0.8,en-US;q=0.6,en;q=0.4","Cookie":"__RequestVerificationToken_L1RyaWdnZXJmaXNoQ2hlY2tlcg2=5nsH5sCVPvlJkp2YTy6WfYQZaKVxA29eUNBnNIc_c_MvRN2mcbMzidOcQ08ZiVIzUSi66El47gpRMhUGSXQp80iesDfwrQBs9sHLf8fjIA01; .AspNet.ApplicationCookie=rURcshk7kll_zQlPMEBpFjDu3Pah-k__4WpYefzrps_Fe6IDVSzZwp2mRzhlYbSwcGv0f8mITnGmKm6bHcif1G1hHJcOm-SRYIK6_f4jiAFRH4Bw95dcbErunAJsxhI72jLEuGm9cifuIyxRWFjDcDDq5KS6Qvs8I359H_gXYjYUyTFAkTP90mgpNHVV8Z3jrIHCGGIWvB0Un7qC0mXt_09fuX7YA2PZXN5qeVfAhyOhEB1buIIEaRfTlzqIdECW_09bQXoCDO6srg3nzhiQ_UdGUveiBlG06VfVV6RgpMix_T7dBQIUKbD3xRk-hacWrpWfgMkE6hAi1DDA8Y3dFLJof4bX_gfAt4293u7EtEXN1SiiA0Y120IuwuG8Eo3DX0moFM292XtVE_9ZCgdesTvjseuk6yncjrKuvdpfDzh8BnT_oyQWRURv_WMp-KC7ju_4RxnMa3yx1K2pSC5Yn4aSMYCtihrzRRxd50AhVNJezn3YsOzzWJp9HKDYTV4r","Host":"localhost","User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36","Upgrade-Insecure-Requests":"1"},"body":[]},"response":{"protocol":"HTTP/1.1","status":0,"headers":{}},"audit_data":{"messages":["collections_remove_stale: Failed to access DBM file \"C:/inetpub/temp/global\": Access is denied.  ","collections_remove_stale: Failed to access DBM file \"C:/inetpub/temp/ip\": Access is denied.  "],"handler":"IIS","stopwatch":{"p1":0,"p2":10052,"p3":0,"p4":0,"p5":501,"sr":0,"sw":0,"l":0,"gc":501},"producer":["ModSecurity for IIS (STABLE)/2.9.1 (http://www.modsecurity.org/)","OWASP_CRS/2.2.9","OWASP_CRS/3.0.0"],"server":"ModSecurity Standalone","engine_mode":"DETECTION_ONLY"}}
{"transaction":{"time":"28/Mar/2017:15:39:04 +0200","transaction_id":"18158513699705323558","remote_address":"","remote_port":80,"local_address":"127.0.0.1","local_port":80},"request":{"request_line":"GET / HTTP/1.1","headers":{"Connection":"keep-alive","Content-Length":"0","Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Accept-Encoding":"gzip, deflate, sdch, br","Accept-Language":"sv-SE,sv;q=0.8,en-US;q=0.6,en;q=0.4","Cookie":"__RequestVerificationToken_L1RyaWdnZXJmaXNoQ2hlY2tlcg2=5nsH5sCVPvlJkp2YTy6WfYQZaKVxA29eUNBnNIc_c_MvRN2mcbMzidOcQ08ZiVIzUSi66El47gpRMhUGSXQp80iesDfwrQBs9sHLf8fjIA01; .AspNet.ApplicationCookie=rURcshk7kll_zQlPMEBpFjDu3Pah-k__4WpYefzrps_Fe6IDVSzZwp2mRzhlYbSwcGv0f8mITnGmKm6bHcif1G1hHJcOm-SRYIK6_f4jiAFRH4Bw95dcbErunAJsxhI72jLEuGm9cifuIyxRWFjDcDDq5KS6Qvs8I359H_gXYjYUyTFAkTP90mgpNHVV8Z3jrIHCGGIWvB0Un7qC0mXt_09fuX7YA2PZXN5qeVfAhyOhEB1buIIEaRfTlzqIdECW_09bQXoCDO6srg3nzhiQ_UdGUveiBlG06VfVV6RgpMix_T7dBQIUKbD3xRk-hacWrpWfgMkE6hAi1DDA8Y3dFLJof4bX_gfAt4293u7EtEXN1SiiA0Y120IuwuG8Eo3DX0moFM292XtVE_9ZCgdesTvjseuk6yncjrKuvdpfDzh8BnT_oyQWRURv_WMp-KC7ju_4RxnMa3yx1K2pSC5Yn4aSMYCtihrzRRxd50AhVNJezn3YsOzzWJp9HKDYTV4r","Host":"localhost","User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36","Upgrade-Insecure-Requests":"1"},"body":[]},"response":{"protocol":"HTTP/1.1","status":0,"headers":{}},"audit_data":{"messages":["IPmatch: bad IPv4 specification \"\".","Rule processing failed."],"handler":"IIS","stopwatch":{"p1":499,"p2":12501,"p3":0,"p4":0,"p5":0,"sr":0,"sw":0,"l":0,"gc":0},"producer":["ModSecurity for IIS (STABLE)/2.9.1 (http://www.modsecurity.org/)","OWASP_CRS/2.2.9","OWASP_CRS/3.0.0"],"server":"ModSecurity Standalone","engine_mode":"DETECTION_ONLY"}}
{"transaction":{"time":"28/Mar/2017:15:39:04 +0200","transaction_id":"18158513699705323558","remote_address":"","remote_port":80,"local_address":"127.0.0.1","local_port":80},"request":{"request_line":"GET / HTTP/1.1","headers":{"Connection":"keep-alive","Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Accept-Encoding":"gzip, deflate, sdch, br","Accept-Language":"sv-SE,sv;q=0.8,en-US;q=0.6,en;q=0.4","Cookie":"__RequestVerificationToken_L1RyaWdnZXJmaXNoQ2hlY2tlcg2=5nsH5sCVPvlJkp2YTy6WfYQZaKVxA29eUNBnNIc_c_MvRN2mcbMzidOcQ08ZiVIzUSi66El47gpRMhUGSXQp80iesDfwrQBs9sHLf8fjIA01; .AspNet.ApplicationCookie=rURcshk7kll_zQlPMEBpFjDu3Pah-k__4WpYefzrps_Fe6IDVSzZwp2mRzhlYbSwcGv0f8mITnGmKm6bHcif1G1hHJcOm-SRYIK6_f4jiAFRH4Bw95dcbErunAJsxhI72jLEuGm9cifuIyxRWFjDcDDq5KS6Qvs8I359H_gXYjYUyTFAkTP90mgpNHVV8Z3jrIHCGGIWvB0Un7qC0mXt_09fuX7YA2PZXN5qeVfAhyOhEB1buIIEaRfTlzqIdECW_09bQXoCDO6srg3nzhiQ_UdGUveiBlG06VfVV6RgpMix_T7dBQIUKbD3xRk-hacWrpWfgMkE6hAi1DDA8Y3dFLJof4bX_gfAt4293u7EtEXN1SiiA0Y120IuwuG8Eo3DX0moFM292XtVE_9ZCgdesTvjseuk6yncjrKuvdpfDzh8BnT_oyQWRURv_WMp-KC7ju_4RxnMa3yx1K2pSC5Yn4aSMYCtihrzRRxd50AhVNJezn3YsOzzWJp9HKDYTV4r","Host":"localhost","User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36","Upgrade-Insecure-Requests":"1"}},"response":{"protocol":"HTTP/1.1","status":0,"headers":{}},"audit_data":{"messages":["IPmatch: bad IPv4 specification \"\".","Rule processing failed."],"handler":"IIS","stopwatch":{"p1":1003,"p2":20520,"p3":0,"p4":0,"p5":0,"sr":0,"sw":0,"l":0,"gc":0},"producer":["ModSecurity for IIS (STABLE)/2.9.1 (http://www.modsecurity.org/)","OWASP_CRS/2.2.9","OWASP_CRS/3.0.0"],"server":"ModSecurity Standalone","engine_mode":"DETECTION_ONLY"}}

They are not in a list and they are not comma separated.

The only way I got it working now is using the method below. However this method requires that my stream is open when I'm using the result from this metod and I think this may be causing some trouble in the application due to closed streams. Is there any better way to read a stream of json objects from a file?

public IEnumerable<ModsecurityLogEntry> ReadAuditLog()
{
    string path = "C:\\inetpub\\logs\\modsec_audit.log";

    using (FileStream fileStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
    {
        using (StreamReader streamReader = new StreamReader(fileStream))
        {
            var serializer = new JsonSerializer();
            using (var jsonTextReader = new JsonTextReader(streamReader))
            {
                jsonTextReader.SupportMultipleContent = true;

                while (jsonTextReader.Read())
                {
                    yield return serializer.Deserialize<ModsecurityLogEntry>(jsonTextReader);
                }
            }
        }
    }
}

score 0 · Accepted Answer · answered Mar 28 '17 at 14:24

Solved it like this, not the prettiest solution but now I don't have to worry about closed streams. There could be problems if the log file gets to big but it will be handled separately.

public IEnumerable<ModsecurityLogEntry> ReadAuditLog()
{
    var path = "C:\\inetpub\\logs\\modsec_audit.log";

    var list = new List<ModsecurityLogEntry>();

    using (FileStream fileStream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
    {
        using (StreamReader streamReader = new StreamReader(fileStream))
        {
            var serializer = new JsonSerializer();
            using (var jsonTextReader = new JsonTextReader(streamReader))
            {
                jsonTextReader.SupportMultipleContent = true;

                while (jsonTextReader.Read())
                {
                    JObject obj = JObject.Load(jsonTextReader);
                    var logEntry = JsonConvert.DeserializeObject<ModsecurityLogEntry>(obj.ToString());
                    list.Add(logEntry);
                }
            }
        }
    }

    return list;

}

Read stream of json log objects using c#

1 Answers1