Parse Deep Security Logs - AWS Lambda 'splunk-logger' node.js

Question

I am trying to modify a Node.js function called 'splunk-logger'. The problem is that when the SNS Message comes into the function, the events from the Anti-Virus (Trend Micro DeepSecurity) console are grouped together. I already contacted their support and they said this is just the way events are sent and they can't help.

Example: {Message {Event_1} {Event_2} {Event_3}}

Now the JavaScript function works great and the events are forwarded to Splunk. However, since they are grouped together BEFORE they even hit the Lambda function, Splunk sees them as 1 single event instead of 3.

My thought is to take the 'event' variable (since it contains the sns 'message') and parse through that to separate each event (probably using regex or something). Then, I can either create another function to send each event immediately or simply call the "logger.flushAsync" function to send them.

Link to splunk-dev explaining the funciton: http://dev.splunk.com/view/event-collector/SP-CAAAE6Y#create.

Here is the code from the index.js:

const loggerConfig = {
    url: process.env.SPLUNK_HEC_URL,
    token: process.env.SPLUNK_HEC_TOKEN,
};
const SplunkLogger = require('./lib/mysplunklogger');
const logger = new SplunkLogger(loggerConfig);
exports.handler = (event, context, callback) => {
    console.log('Received event:', JSON.stringify(event, null, 2));
// Log JSON objects to Splunk
    logger.log(event);
// Send all the events in a single batch to Splunk
    logger.flushAsync((error, response) => {
        if (error) {
            callback(error);
        } else {
            console.log(`Response from Splunk:\n${response}`);
            callback(null, event.key1); // Echo back the first key value
        }
    });
};

Here is the code from the mysplunklogger.js file.

'use strict';

const url = require('url');

const Logger = function Logger(config) {
    this.url = config.url;
    this.token = config.token;

    this.addMetadata = true;
    this.setSource = true;

    this.parsedUrl = url.parse(this.url);
    // eslint-disable-next-line import/no-dynamic-require
    this.requester = require(this.parsedUrl.protocol.substring(0, this.parsedUrl.protocol.length - 1));
    // Initialize request options which can be overridden & extended by consumer as needed
    this.requestOptions = {
        hostname: this.parsedUrl.hostname,
        path: this.parsedUrl.path,
        port: this.parsedUrl.port,
        method: 'POST',
        headers: {
            Authorization: `Splunk ${this.token}`,
        },
        rejectUnauthorized: false,
    };

    this.payloads = [];
};

// Simple logging API for Lambda functions
Logger.prototype.log = function log(message, context) {
    this.logWithTime(Date.now(), message, context);
};

Logger.prototype.logWithTime = function logWithTime(time, message, context) {
    const payload = {};

    if (Object.prototype.toString.call(message) === '[object Array]') {
        throw new Error('message argument must be a string or a JSON object.');
    }
    payload.event = message;

    // Add Lambda metadata
    if (typeof context !== 'undefined') {
        if (this.addMetadata) {
            // Enrich event only if it is an object
            if (message === Object(message)) {
                payload.event = JSON.parse(JSON.stringify(message)); // deep copy
                payload.event.awsRequestId = context.awsRequestId;
            }
        }
        if (this.setSource) {
            payload.source = `lambda:${context.functionName}`;
        }
    }

    payload.time = new Date(time).getTime() / 1000;

    this.logEvent(payload);
};

Logger.prototype.logEvent = function logEvent(payload) {
    this.payloads.push(JSON.stringify(payload));
};

Logger.prototype.flushAsync = function flushAsync(callback) {
    callback = callback || (() => {}); // eslint-disable-line no-param-reassign

    console.log('Sending event(s)');
    const req = this.requester.request(this.requestOptions, (res) => {
        res.setEncoding('utf8');

        console.log('Response received');
        res.on('data', (data) => {
            let error = null;
            if (res.statusCode !== 200) {
                error = new Error(`error: statusCode=${res.statusCode}\n\n${data}`);
                console.error(error);
            }
            this.payloads.length = 0;
            callback(error, data);
        });
    });

    req.on('error', (error) => {
        callback(error);
    });

    req.end(this.payloads.join(''), 'utf8');
};

module.exports = Logger;

Answer 1

import requests
import re
import json
import os

def lambda_handler(event, context):

    data = json.dumps(event)

    EventIds = re.findall(r'{\\\".+?\\\"}', data)
    EventLength = len(EventIds)

    headers = {'Authorization': 'Splunk ' + os.environ['SPLUNK_HEC_TOKEN']}

    i = 0
    while i < EventLength:
        response = requests.post(os.environ['SPLUNK_HEC_URL'], headers=headers, json={"event":EventIds[i]}, verify=True)
    i+=1

Answer 2

Arrays are the data type used when Deep Security 10.0 or newer sends events to Amazon SNS. But Splunk wants one event per message. So don't send the array directly.

Instead, use the Splunk logger or Lambda to iterate through the array, sending each item as an individual message. You can modify this sample Lambda script for Node.js:

https://github.com/deep-security/amazon-sns/blob/master/lambda-save-ds-event-to-s3.js

It sends events to S3 individually (which is what you need). Just change it to send to Splunk instead.

Disclosure: I work for Trend Micro.