3

I am creating a log receiver, which will receive the logging output of another go program, do some filtering and then directly POST to GCP logging. The logging received are pure JSON entries. I'd rather avoid a JSON marshall if I can. Just adding the JSON as JSON payload to the log entry would not pick up the level --> Severity or the timestamp. I can force feed the Severity, but my attempt at the timestamp is failing. Best case, however, would be to have a JSON string and a tiny bit of configuration that says find Severity in level and use the timestamp field. Here is my minimal example:

package main

import (
    "context"
    "encoding/json"
    "fmt"
    "log"
    "time"

    "cloud.google.com/go/logging"
)

func main() {
    ctx := context.Background()
    projectID := "redacted1"
    client, err := logging.NewClient(ctx, projectID)
    if err != nil {
        log.Fatalf("Failed to create client: %v", err)
    }
    logName := "redacted2"
    logger := client.Logger(logName)
    // JSON string
    text := "{\"decision_id\":\"21986d12-e3b2-47f3-b3c7-9a8d77bd048d\",\"input\":{\"networks\":[{\"id\":\"net1\",\"public\":false},{\"id\":\"net2\",\"public\":false},{\"id\":\"net3\",\"public\":true},{\"id\":\"net4\",\"public\":true}],\"ports\":[{\"id\":\"p1\",\"network\":\"net1\"},{\"id\":\"p2\",\"network\":\"net3\"},{\"id\":\"p3\",\"network\":\"net2\"}],\"servers\":[{\"id\":\"app\",\"ports\":[\"p1\",\"p2\",\"p3\"],\"protocols\":[\"https\",\"ssh\"]},{\"id\":\"db\",\"ports\":[\"p3\"],\"protocols\":[\"mysql\"]},{\"id\":\"cache\",\"ports\":[\"p3\"],\"protocols\":[\"memcache\"]},{\"id\":\"ci\",\"ports\":[\"p1\",\"p2\"],\"protocols\":[\"http\"]},{\"id\":\"busybox\",\"ports\":[\"p1\"],\"protocols\":[\"telnet\"]}]},\"labels\":{\"id\":\"d29ba0a9-75d4-4d74-9d03-7bf0399a47c3\",\"version\":\"0.23.2\"},\"level\":\"info\",\"metrics\":{\"counter_server_query_cache_hit\":0,\"timer_rego_input_parse_ns\":554348,\"timer_rego_query_compile_ns\":484825,\"timer_rego_query_eval_ns\":1002441,\"timer_rego_query_parse_ns\":46624,\"timer_server_handler_ns\":2290410},\"msg\":\"Decision Log\",\"path\":\"example/violation\",\"requested_by\":\"127.0.0.1:44934\",\"result\":[\"ci\",\"busybox\"],\"time\":\"2021-01-04T08:44:24-06:00\",\"timestamp\":\"2021-01-04T14:44:24.215618442Z\",\"type\":\"openpolicyagent.org/decision_logs\"}"
    sev := logging.Info
    tim, err := time.Parse(time.RFC3339Nano, "2021-01-04T14:44:24.215618442Z")
    if err != nil {
        log.Fatalf("Failed to parse time: %v", err)
    }
    entry := logging.Entry{Payload: json.RawMessage([]byte(text)), Severity: sev,
        Timestamp: tim}
    err = logger.LogSync(ctx, entry)
    if err != nil {
        log.Fatalf("Failed to flush client: %v", err)
    }
    err = client.Close()
    if err != nil {
        log.Fatalf("Failed to close client: %v", err)
    }
}
Jonathan Hall
  • 75,165
  • 16
  • 143
  • 189
Kevin Buchs
  • 2,520
  • 4
  • 36
  • 55

1 Answers1

1

how about:

sev := logging.Info
if strings.Contains("\"level\":\"info\"", text) {
  sev = logging.Info
} else if strings.Contains("\"level\":\"debug\"", text) {
  sev = logging.Debug
}

but you might end up wanting to run

tokens := strings.Split(text, ":")

on text first just once, and then search the array? Anyway you slice it though, you want data from that string but you don't want to unmarshal it.

Andrew Arrow
  • 4,248
  • 9
  • 53
  • 80
  • Thank @Andrew. That was what I was settling on, as the best I could do for this. – Kevin Buchs Jan 08 '21 at 20:17
  • my only other idea was, what if you run N of these log collectors on different ports and say port 3001 is INFO and 3002 is DEBUG etc and then do the check for the level in the thing that's sending the json. (Could also add the timestamp to front of string with known number of chars and just assume first N chars are not json but this timestamp for you.) – Andrew Arrow Jan 09 '21 at 15:51