Golang parse JSON with unknown keys

Question

I'm trying to unmarshal json with dynamic field keys to the struct That json returned from storcli utility for linux. One part of the code work well, but if json data contains many structs I can't unmarshal it. I think that is because DriveDetailedInformation struct do not have all json tags. Can anybody help me?

package main

import (
    "fmt"
    "encoding/json"
)

type jsonStruct struct {
    Controllers []struct {
        CommandStatus struct {
            Controller  int    `json:"Controller"`
            Status      string `json:"Status"`
            Description string `json:"Description"`
        } `json:"Command Status"`
        ResponseData map[string]*json.RawMessage `json:"Response Data"`
    } `json:"Controllers"`
}

type DriveStruct struct {
    EIDSlt string `json:"EID:Slt"`
    DID    int    `json:"DID"`
    State  string `json:"State"`
    DG     int    `json:"DG"`
    Size   string `json:"Size"`
    Intf   string `json:"Intf"`
    Med    string `json:"Med"`
    SED    string `json:"SED"`
    PI     string `json:"PI"`
    SeSz   string `json:"SeSz"`
    Model  string `json:"Model"`
    Sp     string `json:"Sp"`
}

type DriveDetailedInformation struct {
    DriveState            map[string]DriveStateStruct
    DriveDeviceAttributes map[string]DriveDeviceAttributesStruct
    DrivePoliciesSettings map[string]DrivePoliciesSettingsStruct
    InquiryData           string `json:"Inquiry Data"`
}

type DriveStateStruct struct {
    ShieldCounter            int    `json:"Shield Counter"`
    MediaErrorCount          int    `json:"Media Error Count"`
    OtherErrorCount          int    `json:"Other Error Count"`
    BBMErrorCount            int    `json:"BBM Error Count"`
    DriveTemperature         string `json:"Drive Temperature"`
    PredictiveFailureCount   int    `json:"Predictive Failure Count"`
    SMARTAlertFlaggedByDrive string `json:"S.M.A.R.T alert flagged by drive"`
}

type DriveDeviceAttributesStruct struct {
    SN               string `json:"SN"`
    ManufacturerID   string `json:"Manufacturer Id"`
    ModelNumber      string `json:"Model Number"`
    NANDVendor       string `json:"NAND Vendor"`
    WWN              string `json:"WWN"`
    FirmwareRevision string `json:"Firmware Revision"`
    RawSize          string `json:"Raw size"`
    CoercedSize      string `json:"Coerced size"`
    NonCoercedSize   string `json:"Non Coerced size"`
    DeviceSpeed      string `json:"Device Speed"`
    LinkSpeed        string `json:"Link Speed"`
    NCQSetting       string `json:"NCQ setting"`
    WriteCache       string `json:"Write cache"`
    SectorSize       string `json:"Sector Size"`
    ConnectorName    string `json:"Connector Name"`
}

type DrivePoliciesSettingsStruct struct {
    DrivePosition                            string `json:"Drive position"`
    EnclosurePosition                        int    `json:"Enclosure position"`
    ConnectedPortNumber                      string `json:"Connected Port Number"`
    SequenceNumber                           int    `json:"Sequence Number"`
    CommissionedSpare                        string `json:"Commissioned Spare"`
    EmergencySpare                           string `json:"Emergency Spare"`
    LastPredictiveFailureEventSequenceNumber int    `json:"Last Predictive Failure Event Sequence Number"`
    SuccessfulDiagnosticsCompletionOn        string `json:"Successful diagnostics completion on"`
    SEDCapable                               string `json:"SED Capable"`
    SEDEnabled                               string `json:"SED Enabled"`
    Secured                                  string `json:"Secured"`
    Locked                                   string `json:"Locked"`
    NeedsEKMAttention                        string `json:"Needs EKM Attention"`
    PIEligible                               string `json:"PI Eligible"`
    Certified                                string `json:"Certified"`
    WidePortCapable                          string `json:"Wide Port Capable"`
    PortInformation                          []struct {
        Port       int    `json:"Port"`
        Status     string `json:"Status"`
        Linkspeed  string `json:"Linkspeed"`
        SASAddress string `json:"SAS address"`
    } `json:"Port Information"`
}

var jsonData = `
{
   "Controllers":[
      {
         "Command Status":{
            "Controller":0,
            "Status":"Success",
            "Description":"Show Drive Information Succeeded."
         },
         "Response Data":{
            "Drive /c0/e31/s0":[
               {
                  "EID:Slt":"31:0",
                  "DID":19,
                  "State":"Onln",
                  "DG":0,
                  "Size":"9.094 TB",
                  "Intf":"SATA",
                  "Med":"HDD",
                  "SED":"N",
                  "PI":"N",
                  "SeSz":"512B",
                  "Model":"ST10000DM0004-1ZC101",
                  "Sp":"U"
               }
            ],
            "Drive /c0/e31/s0 - Detailed Information":{
               "Drive /c0/e31/s0 State":{
                  "Shield Counter":0,
                  "Media Error Count":0,
                  "Other Error Count":0,
                  "BBM Error Count":0,
                  "Drive Temperature":" 25C (77.00 F)",
                  "Predictive Failure Count":0,
                  "S.M.A.R.T alert flagged by drive":"No"
               },
               "Drive /c0/e31/s0 Device attributes":{
                  "SN":"            ZA23V0DH",
                  "Manufacturer Id":"ATA     ",
                  "Model Number":"ST10000DM0004-1ZC101",
                  "NAND Vendor":"NA",
                  "WWN":"5000c500a5ad06b6",
                  "Firmware Revision":"DN01    ",
                  "Raw size":"9.095 TB [0x48c400000 Sectors]",
                  "Coerced size":"9.094 TB [0x48c300000 Sectors]",
                  "Non Coerced size":"9.094 TB [0x48c300000 Sectors]",
                  "Device Speed":"6.0Gb/s",
                  "Link Speed":"6.0Gb/s",
                  "NCQ setting":"N/A",
                  "Write cache":"N/A",
                  "Sector Size":"512B",
                  "Connector Name":""
               },
               "Drive /c0/e31/s0 Policies/Settings":{
                  "Drive position":"DriveGroup:0, Span:0, Row:0",
                  "Enclosure position":0,
                  "Connected Port Number":"0(path0) ",
                  "Sequence Number":2,
                  "Commissioned Spare":"No",
                  "Emergency Spare":"No",
                  "Last Predictive Failure Event Sequence Number":0,
                  "Successful diagnostics completion on":"N/A",
                  "SED Capable":"No",
                  "SED Enabled":"No",
                  "Secured":"No",
                  "Locked":"No",
                  "Needs EKM Attention":"No",
                  "PI Eligible":"No",
                  "Certified":"No",
                  "Wide Port Capable":"No",
                  "Port Information":[
                     {
                        "Port":0,
                        "Status":"Active",
                        "Linkspeed":"6.0Gb/s",
                        "SAS address":"0x5003048001927c6c"
                     }
                  ]
               },
               "Inquiry Data":""
            },
            "Drive /c0/e31/s1":[
               {
                  "EID:Slt":"31:1",
                  "DID":20,
                  "State":"Onln",
                  "DG":0,
                  "Size":"9.094 TB",
                  "Intf":"SATA",
                  "Med":"HDD",
                  "SED":"N",
                  "PI":"N",
                  "SeSz":"512B",
                  "Model":"ST10000DM0004-1ZC101",
                  "Sp":"U"
               }
            ],
            "Drive /c0/e31/s1 - Detailed Information":{
               "Drive /c0/e31/s1 State":{
                  "Shield Counter":0,
                  "Media Error Count":0,
                  "Other Error Count":0,
                  "BBM Error Count":0,
                  "Drive Temperature":" 25C (77.00 F)",
                  "Predictive Failure Count":0,
                  "S.M.A.R.T alert flagged by drive":"No"
               },
               "Drive /c0/e31/s1 Device attributes":{
                  "SN":"            ZA23MCVS",
                  "Manufacturer Id":"ATA     ",
                  "Model Number":"ST10000DM0004-1ZC101",
                  "NAND Vendor":"NA",
                  "WWN":"5000c500a5acc582",
                  "Firmware Revision":"DN01    ",
                  "Raw size":"9.095 TB [0x48c400000 Sectors]",
                  "Coerced size":"9.094 TB [0x48c300000 Sectors]",
                  "Non Coerced size":"9.094 TB [0x48c300000 Sectors]",
                  "Device Speed":"6.0Gb/s",
                  "Link Speed":"6.0Gb/s",
                  "NCQ setting":"N/A",
                  "Write cache":"N/A",
                  "Sector Size":"512B",
                  "Connector Name":""
               },
               "Drive /c0/e31/s1 Policies/Settings":{
                  "Drive position":"DriveGroup:0, Span:0, Row:1",
                  "Enclosure position":0,
                  "Connected Port Number":"0(path0) ",
                  "Sequence Number":2,
                  "Commissioned Spare":"No",
                  "Emergency Spare":"No",
                  "Last Predictive Failure Event Sequence Number":0,
                  "Successful diagnostics completion on":"N/A",
                  "SED Capable":"No",
                  "SED Enabled":"No",
                  "Secured":"No",
                  "Locked":"No",
                  "Needs EKM Attention":"No",
                  "PI Eligible":"No",
                  "Certified":"No",
                  "Wide Port Capable":"No",
                  "Port Information":[
                     {
                        "Port":0,
                        "Status":"Active",
                        "Linkspeed":"6.0Gb/s",
                        "SAS address":"0x5003048001927c6d"
                     }
                  ]
               },
               "Inquiry Data":""
            }
         }
      }
   ]
}
`

func main() {
    var f jsonStruct
    err := json.Unmarshal([]byte(jsonData), &f)
    if err != nil {
        fmt.Println("Error parsing JSON: ", err)
    }

    for _, controller := range f.Controllers {
        for k, v := range controller.ResponseData {

            ///THAT IS WORK
            var ds []DriveStruct
            if err := json.Unmarshal(*v, &ds); err == nil {
                fmt.Println(k, ds)

            }

            ///THAT IS NOT WORK WHY?
            var dds DriveDetailedInformation
            if err := json.Unmarshal(*v, &dds); err == nil {
                fmt.Println(k, dds)
            }
        }
    }
}

What do you mean you "cannot unmarshal it"? Why not? What is the problem you're facing? — Jonathan Hall, Sep 16 '18 at 16:49
Unmarshal does not return any error, but all maps in the DriveDetailedInformation struct is empty. — CRASH, Sep 16 '18 at 17:20

score 0 · Answer 1 · edited Sep 24 '19 at 05:30

Because your key value is not correct. Below is the code I modified.

type DriveDetailedInformation struct {
DriveState            DriveStateStruct `json:"Drive /c0/e31/s0 State"`
DriveDeviceAttributes DriveDeviceAttributesStruct `json:"Drive /c0/e31/s0 Device attributes"`
DrivePoliciesSettings DrivePoliciesSettingsStruct `json:"Drive /c0/e31/s0 Policies/Settings"`
InquiryData           string `json:"Inquiry Data"`
}

type DriveDetailedInformation1 struct {
    DriveState            DriveStateStruct `json:"Drive /c0/e31/s1 State"`
    DriveDeviceAttributes DriveDeviceAttributesStruct `json:"Drive /c0/e31/s1 Device attributes"`
    DrivePoliciesSettings DrivePoliciesSettingsStruct `json:"Drive /c0/e31/s1 Policies/Settings"`
    InquiryData           string `json:"Inquiry Data"`
}
func main() {
    var f jsonStruct
    err := json.Unmarshal([]byte(jsonData), &f)
    if err != nil {
        fmt.Println("Error parsing JSON: ", err)
    }

    for _, controller := range f.Controllers {
        for k, v := range controller.ResponseData {
            switch k {
            case "Drive /c0/e31/s0","Drive /c0/e31/s1":
                var ds []DriveStruct
                if err := json.Unmarshal(*v, &ds); err == nil {
                    fmt.Println(k, ds)
                }
            case "Drive /c0/e31/s1 - Detailed Information":
                var dds1 DriveDetailedInformation1
                if err := json.Unmarshal(*v, &dds1); err == nil {
                    fmt.Println(k, dds1)
                }
            case "Drive /c0/e31/s0 - Detailed Information":
                var dds DriveDetailedInformation
                if err := json.Unmarshal(*v, &dds); err == nil {
                    fmt.Println(k, dds)
                }
            }
        }
    }
}

I think Response Data can be parsed in a structure.

`json:"Drive /c0/e31/s1 Device attributes"` - is a dynamic key, I can't hardcode it. — CRASH, Sep 17 '18 at 07:06
So you only have to nest a few more layers of map [string] * json.RawMessage, which I think is very tedious, so you might as well modify JSON and add a name field.like this:{"Drive":[{"name":"Drive /c0/e31/s0",...}],"Drive Detailed Information":[{"driveName":"Drive /c0/e31/s0","name":"Drive /c0/e31/s0 - Detailed Information",...}]} — JokerSTACKOVERFLOW, Sep 17 '18 at 07:28

Golang parse JSON with unknown keys

1 Answers1