How to convert txt file to json using leading spaces?

Question

I have formatted txt file looks like this:

Hostinfo Start
  DATE 190819 1522
  HOST midas
  DOMAIN test.de
  HW_PLATFORM x86_64
  SERVER_TYPE virtual
  CPU_INFO
    CPU_TYPE  Intel(R) Xeon(R) CPU E7-8867 v4 @ 2.40GHz
    CPU_COUNT 2
    CORE_COUNT 2
  THREAD_COUNT 8
  MEMORY       32951312 kB
  OS Start
    OS Linux
    OS_VERSION 4.9.0-6-amd64
    OS_UPTIME 536 days 21:08
    OS End
  RELEASE Debian GNU/Linux 9 (stretch)
  RELEASE_VERSION 9
  RELEASE_PATCHLEVEL
Hostinfo End

Using the count of leading spaces need to convert it to json format looking similar to this:

"Hostinfo": [
{
  "DATE": "190819 1522"
  "HOST": "midas"
  "DOMAIN": "test.de"
  "HW_PLATFORM": "x86_64"
  "SERVER_TYPE": "virtual"
  "CPU_INFO": {
    "CPU_TYPE": "Intel(R) Xeon(R) CPU E7-8867 v4 @ 2.40GHz"
    "CPU_COUNT": "2"
    "CORE_COUNT": "2"
    }
  "THREAD_COUNT": "8"
  "MEMORY": "32951312 kB"
  "OS": [
      {
      "OS": "Linux"
      "OS_VERSION": "4.9.0-6-amd64"
      "OS_UPTIME": "536 days 21:08"
      }
    ]
  "RELEASE": "Debian GNU/Linux 9 (stretch)"
  "RELEASE_VERSION": "9"
  "RELEASE_PATCHLEVEL" : ""
}
]

I have some undertakings of this script but can't workaround how to set lines between curly brackets as object of upper dictionary (level):

#!/usr/bin/python
import json
import itertools
import string
import re

filename = 'commands.txt'

commands = {}
with open(filename) as fh:
    previous_line = 0
    mark_line = ""

    for line in fh:
        current_line = ((len(line) - len(line.lstrip()))/2) 

        diff = current_line - previous_line
        if re.search(' Start$', line.strip()):
            line = line.strip().replace(' Start', ':{')
            print(line)
            mark_line = "start_line"
        elif re.search(' Ende$', line.strip()):
            line = line.strip().replace(' Ende', '')
            print("}")
            mark_line = "end_line"
        elif diff == 0:
            print(line.strip())S
        elif diff > 0:
            if mark_line == "start_line" or mark_line == "end_line":
                mark_line = "0"
            else:
                print("{")
                print(line.strip())
        elif diff < 0:
            if mark_line == "start_line" or mark_line == "end_line":
                mark_line = "0"
            else:
                print("}")
                print(line.strip())
        previous_line = ((len(line) - len(line.lstrip()))/2)


        #line = (str((len(line) - len(line.lstrip()))/2) + ";" + line.strip())

        try:
            command, description = line.strip().split(' ', 1)
            commands[command] = description.strip()
        except Exception:
            command = line.strip()
            description = ""
            commands[command] = description.strip()


print(json.dumps(commands, indent=2, sort_keys=True))

May be you can get me some idea how to workaround this or take some advice? May it would be some module that simplifies this script?

UPD: add some json markup to my mess script. Can you please get advice if I moving at wrong/right way?

Answer 1

You can use itertools.groupby with recursion:

import itertools as it, re
data = [[*re.findall('^\s+', b), *re.split('(?<=[A-Z])\s+', i)] for b in open('os_stuff.txt') if not (i:=re.sub('^\s+|\sStart\n$', '', b)).endswith('End\n')]
def to_tree(d):
   _d = [(a, list(b)) for a, b in it.groupby(d, key=lambda x:bool(re.findall('^\s+$', x[0])))]
   new_dict, _last = {}, None
   for i, [a, b] in enumerate(_d):
      if not a:
         for j, *k in b:
            if not k or (not k[0] and i < len(_d) - 2):
               _last = j
            else:
               new_dict[j] = ' '.join(k).strip('\n')
      else:
         new_dict[_last] = [to_tree([[k[2:], *j] if k[2:] else j for k, *j in b])]
   return new_dict

---

import json
print(json.dumps(to_tree(data), indent=4))

Output:

{
  "Hostinfo": [
    {
        "DATE": "190819 1522",
        "HOST": "midas",
        "DOMAIN": "test.de",
        "HW_PLATFORM": "x86_64",
        "SERVER_TYPE": "virtual",
        "CPU_INFO": [
            {
                "CPU_TYPE": "Intel(R) Xeon(R) CPU E7-8867 v4 @ 2.40GHz",
                "CPU_COUNT": "2",
                "CORE_COUNT": "2"
            }
        ],
        "THREAD_COUNT": "8",
        "MEMORY": "32951312 kB ",
        "OS": [
            {
                "OS": "Linux",
                "OS_VERSION": "4.9.0-6-amd64",
                "OS_UPTIME": "536 days 21:08"
            }
        ],
        "RELEASE": "Debian GNU/Linux 9 (stretch)",
        "RELEASE_VERSION": "9",
        "RELEASE_PATCHLEVEL": ""
     }
  ]
}

Edit: Python2.7 solution:

import itertools as it, re
new_data = [[i, re.sub('^\s+|\sStart\n$', '', i)] for i in open('os_stuff.txt')]
data = [re.findall('^\s+', a)+re.split('(?<=[A-Z])\s+', b) for a, b in new_data if not b.endswith('End\n')]
def to_tree(d):
  _d = [(a, list(b)) for a, b in it.groupby(d, key=lambda x:bool(re.findall('^\s+$', x[0])))]
  new_dict, _last = {}, None
  for i, [a, b] in enumerate(_d):
     if not a:
       for j_k in b:
         if not j_k[1:] or (not j_k[1:][0] and i < len(_d) - 2):
            _last = j_k[0]
         else:
            new_dict[j_k[0]] = ' '.join(j_k[1:]).strip('\n')
     else:
       new_dict[_last] = [to_tree([[k_j[0][2:]]+k_j[1:] if k_j[0][2:] else k_j[1:] for k_j in b])]
  return new_dict


print(to_dict(data))