Reading a fortigate configuration file with Python

Question

Appologies for the really long drawn out question.

I am trying to read in a config file and get a list of rules out. I have tried to use ConfigParser to do this but it is not a standard config file. The file contains no section header and no token.

i.e.

config section a
set something to something else
config subsection a
set this to that
next
end

config firewall policy
edit 76
set srcintf "There"
set dstintf "Here"
set srcaddr "all"
set dstaddr "all"
set action accept
set schedule "always"
set service "TCP_5600"
next
edit 77
set srcintf "here"
set dstintf "there"
set srcaddr "all"
set dstaddr "all"
set action accept
set schedule "always"
set service "PING"
next
end

As I couldn't work out how to get ConfigParser to work I thought I would try to iterate through the file, unfortunately I don't have much programming skill so I have got stuck. I really think I am making this more complicated than it should be. Here's the code I have written;

class Parser(object):

    def __init__(self):
        self.config_section = ""
        self.config_header = ""
        self.section_list = []
        self.header_list = []

    def parse_config(self, fields): # Create a new section
        new_list = []
        self.config_section = " ".join(fields)
        new_list.append(self.config_section)

        if self.section_list: # Create a sub section
            self.section_list[-1].append(new_list)
        else: self.section_list.append(new_list)

    def parse_edit(self, line): # Create a new header
        self.config_header = line[0]
        self.header_list.append(self.config_header)

        self.section_list[-1].append(self.header_list)  

    def parse_set(self, line): # Key and values
        key_value = {}

        key = line[0]
        values = line[1:]
        key_value[key] = values

        if self.header_list:
            self.header_list.append(key_value)
        else: self.section_list[-1].append(key_value)

    def parse_next(self, line): # Close the header
        self.config_header = []

    def parse_end(self, line): # Close the section
        self.config_section = []

    def parse_file(self, path):
        with open(path) as f:
            for line in f:

                # Clean up the fields and remove unused lines.            
                fields = line.replace('"', '').strip().split(" ")

                if fields[0] == "set":
                    pass
                elif fields[0] == "end":
                    pass
                elif fields[0] == "edit":
                    pass
                elif fields[0] == "config":
                    pass
                elif fields[0] == "next":
                    pass
                else: continue

                # fetch and call method.
                method = fields[0]
                parse_method = "parse_" + method

                getattr(Parser, parse_method)(self, fields[1:])
                return self.section_list

config = Parser().parse_file('test_config.txt')

print config

The output I am looking for is something like the following;

[['section a', {'something': 'to something else'}, ['subsection a', {'this': 'to that'}]],['firewall policy',['76',{'srcintf':'There'}, {'dstintf':'Here'}{etc.}{etc.}]]]

and this is what I get

[['section a']]

EDIT

I have changed the above to reflect where I am currently at. I am still having issues getting the output I expect. I just can't seem to get the list right.

Answer 1

I post my answer for people who first come here from Google when trying to parse Fortigate configuration file ! I rewrote what I found here based on my own needs and it works great.

from collections import defaultdict
from pprint import pprint
import sys

f = lambda: defaultdict(f)

def getFromDict(dataDict, mapList):
    return reduce(lambda d, k: d[k], mapList, dataDict)

def setInDict(dataDict, mapList, value):
    getFromDict(dataDict, mapList[:-1])[mapList[-1]] = value    

class Parser(object):

    def __init__(self):
        self.config_header = []
        self.section_dict = defaultdict(f)     

    def parse_config(self, fields): # Create a new section
        self.config_header.append(" ".join(fields))

    def parse_edit(self, line): # Create a new header
        self.config_header.append(line[0])

    def parse_set(self, line): # Key and values
        key = line[0]
        values = " ".join(line[1:])
        headers= self.config_header+[key]
        setInDict(self.section_dict,headers,values)

    def parse_next(self, line): # Close the header
        self.config_header.pop()

    def parse_end(self, line): # Close the section
        self.config_header.pop()

    def parse_file(self, path):          
        with open(path) as f:
            gen_lines = (line.rstrip() for line in f if line.strip())
            for line in gen_lines:
               # pprint(dict(self.section_dict))
                # Clean up the fields and remove unused lines.            
                fields = line.replace('"', '').strip().split(" ")

                valid_fields= ["set","end","edit","config","next"]
                if fields[0] in valid_fields:
                    method = fields[0]
                    # fetch and call method
                    getattr(Parser, "parse_" + method)(self, fields[1:])

        return self.section_dict

config = Parser().parse_file('FGT02_20130308.conf')

print config["system admin"]["admin"]["dashboard-tabs"]["1"]["name"]
print config["firewall address"]["ftp.fr.debian.org"]["type"]

Answer 2

 class Parser(object):

     def __init__(self):
         self.my_section = 0
         self.flag_section = False
         # ...

    def parse_config(self, fields):
         self.my_section += 1
         # go on with fields
         # ...
         self.flag_section = True

     def parse_edit(self, line):
         ...

     def parse_set(self, line):
         ...

     def parse_end(self, line):
         ...

     def parse_file(self, path):
         with open(path) as f:
              for line in f:
                  fields = f.strip().split(" ")

                  method = fields[0]
                  # fetch and call method
                  getattr(Parser, "parse_" + method)(self, fields[1:])

Answer 3

I do not know if this can help you too, but it did for me : http://wiki.python.org/moin/ConfigParserExamples

Have fun !

Answer 4

I would do it in a simpler way:

flagSection = False
flagSub = False

mySection = 0
mySubsection = 0
myItem = 0

with open('d:/config.txt', 'r') as f:
    gen_lines = (line.rstrip() for line in f if line.strip())

    for line in gen_lines:

            if line[0:7]=='config ':
                mySection = mySection + 1
                newLine = line[7:]
                # Create a new section
                # Mark section as open
                flagSection == True

            elif line[0:5]=='edit '):
                mySubsection = mySubsection + 1
                newLine = line[5:]
                # Create a new sub-section
                # Mark subsection as open
                flagSub == true

            elif line[0:4]=='set '):
                myItem = myItem + 1
                name, value = x.split(' ',2)[1:]
                # Add to whatever is open

            elif line=='end':
                # If subsection = open then close and goto end
                if flagSub:
                # Or if section = open then close and goto end
                elif flagSection:
                # :End
                continue

The instruction gen_lines = (line.rstrip() for line in f if line.strip()) creates a generator of not empty lines (thanks to the test if line.strip()) without newline and without blanks at the right (thanks to line.rstrip())

.

If I would know more about the operations you want to perform with name,value and in the section opened with if line=='end' , I could propose a code using regexes.

Edit

from time import clock

n = 1000000

print 'Measuring times with clock()'

te = clock()
for i in xrange(n):
    x = ('abcdfafdf'[:3] == 'end')
print clock()-te,
print "\tx = ('abcdfafdf'[:3] == 'end')"


te = clock()
for i in xrange(n):
    x = 'abcdfafdf'.startswith('end')
print clock()-te,
print "\tx = 'abcdfafdf'.startswith('end')"



print '\nMeasuring times with timeit module'

import timeit

ti = timeit.repeat("x = ('abcdfafdf'[:3] == 'end')",repeat=10,number = n)
print min(ti),
print "\tx = ('abcdfafdf'[:3] == 'end')"

to = timeit.repeat("x = 'abcdfafdf'.startswith('end')",repeat=10,number = n)
print min(to),
print "\tx = 'abcdfafdf'.startswith('end')"

result:

Measuring times with clock()
0.543445605517  x = ('abcdfafdf'[:3] == 'end')
1.08590449345   x = 'abcdfafdf'.startswith('end')

Measuring times with timeit module
0.294152748464  x = ('abcdfafdf'[:3] == 'end')
0.901923289133  x = 'abcdfafdf'.startswith('end')

Is the fact the times are smaller with timieit than with clock() due to the fact that the GC is unplugged when the program is run ? Anyway, with either clock() or timeit module , executing startswith() takes more time than slicing.