syslog-ng json parser does nothing

Question

My syslog-ng config is driving me insane. I have an app, that puts out simple json log messages like:

{"level":"error","message":"connection ended without disconnect receipt","timestamp":"2018-10-12T17:49:08.650Z"}

All I want to do, is parse these 3 values and send them to a hosted Graylog cluster. Sending works, but the message gets inserted as

application name: {"level"
message: "error","message":"connection ended without disconnect receipt","timestamp":"2018-10-12T17:49:08.650Z"}

it's almost like syslog-ng doesn't even interpret the file as json. I tried other variants, read the docs but I am at my wits end now...

This is my config (on the application host; it should send the logs directly to the logging cluster)

@version: 3.5
@include "scl.conf"
@include "`scl-root`/system/tty10.conf"

options { chain_hostnames(off); flush_lines(0); use_dns(no); use_fqdn(no);
       owner("root"); group("adm"); perm(0640); stats_freq(0);
       bad_hostname("^gconfd$");
};

source s_src { 
    file(
        "{{syslog_ng_src}}"
        flags(no-parse)
        );
};

template unitManagerTemplate {
    template("$(format-json --scope dot-nv-pairs) [sdid@123456 X-OVH-TOKEN=\"XXXXXXXXXXXXXXXXXXXXXXXXXX\"\n");
};

destination ovhPaaSLogs {
    tcp("gra2.logs.ovh.com"
        port(6514),
        template(unitManagerTemplate),
        ts_format("iso"),
        tls(peer-verify("require-trusted") ca_dir("/etc/ssl/certs/")),
        keep-alive(yes),
        so_keepalive(yes),
    );
};

parser p_json { 
    json-parser(prefix(".json.")); 
};

log {
    source(s_src);
    parser(p_json);
    destination(ovhPaaSLogs);
};

@include "/etc/syslog-ng/conf.d/"

I tried a different a template variant like this:

template("${.json.level} ${.json.message} ${.json.timestamp} [sdid@123456 X-OVH-TOKEN=\"XXXXXXXXXXXXXXXXXXXXXXXXXX\"\n");

Results where absolutely identical. I'd appreciate any help!

score 2 · Answer 1 · answered Oct 18 '18 at 10:50

I updated to the latest version of syslog-ng and got it running with minor tweaks to the config:

@version: 3.16
@include "scl.conf"
@include "`scl-root`/system/tty10.conf"

options { chain_hostnames(off); flush_lines(0); use_dns(no); use_fqdn(no);
       owner("root"); group("adm"); perm(0640); stats_freq(0);
       bad_hostname("^gconfd$");
};

source s_src {
    wildcard-file(
        base-dir("/var/log/worker/")
        filename-pattern("error*.log")
        flags(no-parse)
    );


};

template unitManagerTemplate {
    template("<${LEVEL_NUM}>1 ${.json.timestamp} ${HOST} worker ${PID} - [sdid@32473 X-OVH-TOKEN=\"XXXXXXXXXXXXXXXXXXXXXXXXXXX\" pid=\"${PID}\" facility=\"${FACILITY}\" priority=\"${.json.level}\"] ${.json.message}\n");
    template_escape(no);
};


destination ovhPaaSLogs {
    network("gra2.logs.ovh.com"
        port(6514),
        transport("tls")
        tls(
            ca-dir("/etc/ssl/certs")
            peer-verify("required-trusted")
        )
        template(unitManagerTemplate),
        ts_format("iso"),
        keep-alive(yes),
        so_keepalive(yes),
    );
};

parser p_json {
    json-parser(prefix(".json."));
};

log {
    source(s_src);
    parser(p_json);
    destination(ovhPaaSLogs);
};

@include "/etc/syslog-ng/conf.d/"

syslog-ng json parser does nothing

1 Answers1