0

After several days of trying, I'm posting for help. I know this question has been asked many times. I tried several solutions but I couldn't solve my problem.

I have a large nested JSON file (1GB) and I would like to make it flat and then convert it to a CSV file.

The JSON file is:

`

[{
        "id": {
            "$oooled": "hfhfhgfhgfg"
        },
        "id_52": "525252525252525252",
        "__v": {
            "$numberInt": "52"
        },
        "columnA": {
            "$date": {
                "$numberLong": "525252525252525252"
            }
        },
        "columnB": {
            "Start": "52525252-5252-5252",
            "End": "52525252-5252-5252"
        },
        "columnC": "52525252-5252-5252",
        "columnD": {
            "$date": {
                "$numberLong": "52525252525252525252525252"
            }
        },
        "ComeAt": {
            "$date": {
                "$numberLong": "52525252525252525252525252"
            }
        },
        "columnE": "ROACH LM",
        "columnF": [{
                "key": {
                    "$numberInt": "525252525252525252"
                },
                "Tr": null,
                "ke": "525252525252525252-OFF-CCK-5252525252525252",
                "sky": null,
                "Code": "ACY",
                "dutyStartUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyStartLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyEndUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyEndLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyPeriod": {
                    "$numberDouble": "5252.525252525252"
                },
                "by": false,
                "type": "OFF",
                "dutyDesc": "G Day in CCK",
                "asDutyIndicator": "N",
                "patternCode": "G  5252",
                "not": "52525252-5252-5252 5252:5252:5252",
                "ack": null,
                "StaUtc": "52525252-5252-5252 5252:5252:5252",
                "tartLll": "52525252-5252-5252 5252:5252:5252",
                "patternEndUtc": "52525252-5252-5252 5252:5252:5252",
                "patternEndLocal": "52525252-5252-5252 5252:5252:5252",
                "Effective": "52525252-5252-5252 5252:5252:5252",
                "dutyPort": "CCK",
                "dutyEndPort": "CCK",
                "creditInfo": {
                    "rosterPeriod": {
                        "$numberInt": "525252525252"
                    },
                    "indicator": null,
                    "factor": null,
                    "hours": null
                },
                "creditHours": {
                    "$numberInt": "52"
                },
                "specialCode": [],
                "md52": "5252db52525252525252d52525252a5252c52525252b5252e52ecb",
                "_displayOnly": true
            }, {
                "key": {
                    "$numberInt": "525252525252525252"
                },
                "Tr": null,
                "ke": "525252525252525252-OFF-CCK-5252525252525252",
                "sky": null,
                "Code": "ACY",
                "dutyStartUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyStartLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyEndUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyEndLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyPeriod": {
                    "$numberDouble": "5252.525252525252"
                },
                "by": false,
                "type": "OFF",
                "dutyDesc": "G Day in CCK",
                "asDutyIndicator": "N",
                "patternCode": "G  5252",
                "not": "52525252-5252-5252 5252:5252:5252",
                "ack": null,
                "StaUtc": "52525252-5252-5252 5252:5252:5252",
                "tartLll": "52525252-5252-5252 5252:5252:5252",
                "patternEndUtc": "52525252-5252-5252 5252:5252:5252",
                "patternEndLocal": "52525252-5252-5252 5252:5252:5252",
                "Effective": "52525252-5252-5252 5252:5252:5252",
                "dutyPort": "CCK",
                "dutyEndPort": "CCK",
                "creditInfo": {
                    "rosterPeriod": {
                        "$numberInt": "525252525252"
                    },
                    "indicator": null,
                    "factor": null,
                    "hours": null
                },
                "creditHours": {
                    "$numberInt": "52"
                },
                "specialCode": [],
                "md52": "525252525252f5252d52f52dfa52a5252525252b52525252f52f52",
                "_displayOnly": true
            }, {
                "key": {
                    "$numberInt": "525252525252525252"
                },
                "Tr": null,
                "ke": "525252525252525252-OFF-CCK-5252525252525252",
                "sky": null,
                "Code": "ACY",
                "dutyStartUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyStartLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyEndUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyEndLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyPeriod": {
                    "$numberDouble": "5252.525252525252"
                },
                "by": false,
                "type": "OFF",
                "dutyDesc": "G Day in CCK",
                "asDutyIndicator": "N",
                "patternCode": "G  5252",
                "not": "52525252-5252-5252 5252:5252:5252",
                "ack": null,
                "StaUtc": "52525252-5252-5252 5252:5252:5252",
                "tartLll": "52525252-5252-5252 5252:5252:5252",
                "patternEndUtc": "52525252-5252-5252 5252:5252:5252",
                "patternEndLocal": "52525252-5252-5252 5252:5252:5252",
                "Effective": "52525252-5252-5252 5252:5252:5252",
                "dutyPort": "CCK",
                "dutyEndPort": "CCK",
                "creditInfo": {
                    "rosterPeriod": {
                        "$numberInt": "525252525252"
                    },
                    "indicator": null,
                    "factor": null,
                    "hours": null
                },
                "creditHours": {
                    "$numberInt": "52"
                },
                "specialCode": [],
                "md52": "5252a52c52b52a5252525252bbe52525252bd525252a52a525252",
                "_displayOnly": true
            }, {
                "key": {
                    "$numberInt": "525252525252525252"
                },
                "Tr": null,
                "ke": "525252525252525252-OFF-CCK-5252525252525252",
                "sky": null,
                "Code": "ACY",
                "dutyStartUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyStartLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyEndUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyEndLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyPeriod": {
                    "$numberDouble": "5252.525252525252"
                },
                "by": false,
                "type": "OFF",
                "dutyDesc": "G Day in CCK",
                "asDutyIndicator": "N",
                "patternCode": "G  5252",
                "not": "52525252-5252-5252 5252:5252:5252",
                "ack": null,
                "StaUtc": "52525252-5252-5252 5252:5252:5252",
                "tartLll": "52525252-5252-5252 5252:5252:5252",
                "patternEndUtc": "52525252-5252-5252 5252:5252:5252",
                "patternEndLocal": "52525252-5252-5252 5252:5252:5252",
                "Effective": "52525252-5252-5252 5252:5252:5252",
                "dutyPort": "CCK",
                "dutyEndPort": "CCK",
                "creditInfo": {
                    "rosterPeriod": {
                        "$numberInt": "525252525252"
                    },
                    "indicator": null,
                    "factor": null,
                    "hours": null
                },
                "creditHours": {
                    "$numberInt": "52"
                },
                "specialCode": [],
                "md52": "d52fe5252525252d52c525252cc5252f5252b52cff52bcf52",
                "_displayOnly": true
            }, {
                "key": {
                    "$numberInt": "525252525252525252"
                },
                "Tr": null,
                "ke": "525252525252525252-OFF-CCK-5252525252525252",
                "sky": null,
                "Code": "ACY",
                "dutyStartUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyStartLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyEndUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyEndLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyPeriod": {
                    "$numberDouble": "5252.525252525252"
                },
                "by": false,
                "type": "OFF",
                "dutyDesc": "G Day in CCK",
                "asDutyIndicator": "N",
                "patternCode": "G  5252",
                "not": "52525252-5252-5252 5252:5252:5252",
                "ack": null,
                "StaUtc": "52525252-5252-5252 5252:5252:5252",
                "tartLll": "52525252-5252-5252 5252:5252:5252",
                "patternEndUtc": "52525252-5252-5252 5252:5252:5252",
                "patternEndLocal": "52525252-5252-5252 5252:5252:5252",
                "Effective": "52525252-5252-5252 5252:5252:5252",
                "dutyPort": "CCK",
                "dutyEndPort": "CCK",
                "creditInfo": {
                    "rosterPeriod": {
                        "$numberInt": "525252525252"
                    },
                    "indicator": null,
                    "factor": null,
                    "hours": null
                },
                "creditHours": {
                    "$numberInt": "52"
                },
                "specialCode": [],
                "md52": "adb525252ac52525252a5252be52525252e5252b5252525252ca",
                "_displayOnly": true
            }, {
                "key": {
                    "$numberInt": "525252525252525252"
                },
                "Tr": null,
                "ke": "525252525252525252-OFF-CCK-5252525252525252",
                "sky": null,
                "Code": "ACY",
                "dutyStartUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyStartLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyEndUTC": "52525252-5252-5252 5252:5252:5252",
                "dutyEndLocal": "52525252-5252-5252 5252:5252:5252",
                "dutyPeriod": {
                    "$numberDouble": "5252.525252525252"
                },
                "by": false,
                "type": "OFF",
                "dutyDesc": "G Day in CCK",
                "asDutyIndicator": "N",
                "patternCode": "G  5252",
                "not": "52525252-5252-5252 5252:5252:5252",
                "ack": null,
                "StaUtc": "52525252-5252-5252 5252:5252:5252",
                "tartLll": "52525252-5252-5252 5252:5252:5252",
                "patternEndUtc": "52525252-5252-5252 5252:5252:5252",
                "patternEndLocal": "52525252-5252-5252 5252:5252:5252",
                "Effective": "52525252-5252-5252 5252:5252:5252",
                "dutyPort": "CCK",
                "dutyEndPort": "CCK",
                "creditInfo": {
                    "rosterPeriod": {
                        "$numberInt": "525252525252"
                    },
                    "indicator": null,
                    "factor": null,
                    "hours": null
                },
                "creditHours": {
                    "$numberInt": "52"
                },
                "specialCode": [],
                "md52": "52525252f5252f52fc525252eec5252ad525252525252c52c5252",
                "_displayOnly": true
            }
        ],
        "requestorErn": "525252525252C",
        "requestorFirstErn": "525252525252C",
        "requestorNotes": "",
        "requestorRead": true,
        "startDateL": "52525252-5252-5252",
        "status": "APPROVED",
        "updatedAt": {
            "$date": {
                "$numberLong": "52525252525252525252525252"
            }
        },
        "aOrN": false,
        "isComplex": true,
        "shortNoticeInd": false,
        "submitRetry": {
            "$numberInt": "52"
        },
        "cCList": ["52525252525252525252525252525252525252525252525252", "52525252525252525252525252525252525252525252525252"],
        "submitAt": {
            "$date": {
                "$numberLong": "52525252525252525252525252"
            }
        },
        "mqResult": {
            "swapId": "52525252525252525252525252525252525252525252525252",
            "enquiryStatus": "",
            "processBy": "FOPWHH",
            "processTimestamp": "525252525252525252525252525252",
            "status": "A",
            "seqStart": {
                "$numberInt": "525252"
            },
            "seqEnd": {
                "$numberInt": "525252"
            },
            "remarks": "",
            "checkDate": "52525252-5252-5252T5252:5252:5252Z"
        }
    }
    
]

`

I tried to use json_normalize() but I couldn't managed to do it.

Anyone have ideas? really appreciated for the help.

CCS22
  • 11
  • 1

1 Answers1

0

Not sure if I'm missing something but it seems pretty straight foreward.

df = pd.json_normalize(data=data, record_path="columnF")
Jason Baker
  • 3,170
  • 2
  • 12
  • 15