Python: How to split lines merging some of the lines

Question

I want to process a string line by line, but I want to enable multiline support. This is the example text:

First line
Second line
{{{
these three lines
I want to process
together
}}}
Last Line

I want multiline to start at {{{ and finish at }}} I used to process it line by line in the following way:

lines = [l for l in text.splitlines()]
print lines

Right now this code outputs:

['First line', 'Second line', '{{{', 'these three lines', 'I want to process', 'together', '}}}', 'Last Line']

I want somehow to make lines contain the following:

['First line', 'Second line', 'these three lines I want to process together', 'Last Line']

Or, more advanced example

First Line
Second line
Third{{{line
fourth line
fifth}}}line
sixth line

In this case I want lines to contain

['First Line', 'Second line', 'Third', 'line fourth line fifth', 'line', 'sixth line']

Try iterating through the current output, checking for '{{{', then concatenating all the lines after that until you get to '}}}'. — Tom Burrows, Jan 30 '17 at 10:38

score 3 · Accepted Answer · answered Jan 30 '17 at 11:25

Here is a generator that take as parameter an input file object, and yields one line at a time. It should accepts as many {{{ and }}} on same line but does not test unbalanced constructs:

def merge_lines(fd):
    concat = False
    for line in fd:
        while True:
            #print (line)
            if len(line.strip()) == 0: break
            if not concat:
                if ('{{{' in line):
                    deb, line = line.split('{{{', 1)
                    yield deb
                    concat = True
                    old = None
                else:
                    yield line.strip('\r\n')
                    line = ""
            if concat:
                if ('}}}' in line):
                    deb, line = line.split('}}}', 1)
                    concat = False
                    if old:
                        yield old.strip() + ' ' + deb
                    else: yield deb
                else:
                    if old:
                        old += ' ' + line.strip('\r\n')
                    else:
                        old = line.strip('\r\n')
                    line = ""

Example in Python 3:

>>> t = """First line
a{{{b}}}c{{{d
e
f}}}g{{{h
i}}}
j
k
"""
>>> for line in merge_lines(io.StringIO(t)): print(line)

First line
a
b
c
d e f
g
h i
j
k

score 2 · Answer 2 · answered Jan 30 '17 at 11:23

Using a regex seems like a sensible solution - it gives you the flexibility between your two input options

import re

only_line = '''First line
Second line
{{{
these three lines
I want to process
together
}}}
Last Line'''

mixed_line = '''First Line
Second line
Third{{{line
fourth line
fifth}}}line
sixth line'''

def curly_brackets(input_string):
    # regex - we want to match text before the backets, text in the brackets, and text after the brackets as three groups
    separate = list(re.findall('(.*)\{{3}(.*)\}{3}(.*)', input_string, re.DOTALL)[0])

    # 1-indexed item will be the value between brackets - replace carriage returns with spaces
    separate[1] = separate[1].replace('\n', ' ')

    # split according to new lines - there will be none in our bracketed section
    separate = [x.strip().split('\n') for x in separate]

    # flatten the lists down - each element of separate is currently a list
    return [x for sublist in separate for x in sublist]

print curly_brackets(only_line)
print curly_brackets(mixed_line)

This returns:

['First line', 'Second line', 'these three lines I want to process together', 'Last Line']
['First Line', 'Second line', 'Third', 'line fourth line fifth', 'line', 'sixth line']

This won't work if you have multiple sets of curly brackets, but could be adapted to apply in an iterative manner.

Daniel Puiu · Answer 3 · 2017-01-30T11:09:18.790

0

def split(text):    
    lines = []
    while '{{{' in text:
        head, sep, tail = text.partition('{{{')
        lines.extend(head.splitlines())
        head, sep, tail = tail.partition('}}}')
        lines.append(head.replace('\n', ' ').strip())
        text = tail

    lines.extend(text.splitlines())
    return lines

edited Jan 30 '17 at 11:09

answered Jan 30 '17 at 10:41

Daniel Puiu

962
6
21
29

sasha199568 · Answer 4 · 2017-01-30T10:59:56.267

Here is my solution. It is long and simple. I hoped maybe there is a way to make it in just a few lines But it will not handle case when }}} and {{{ are on the same line

def _split_with_merging(text):
    lines = [l for l in text.splitlines() if l != ""]
    nlines = []
    multiline = False
    for l in lines:
        if multiline:
            if "}}}" in l:
                lparts = l.split("}}}")
                nlines[len(nlines) - 1] += lparts[0]
                if lparts[1] != "":
                    nlines.append(lparts[1])
                multiline = False
            else:
                nlines[len(nlines) - 1] += l
        else:
            if "{{{" in l:
                lparts = l.split("{{{")
                nlines.append(lparts[0])
                if lparts[1] != "":
                    nlines.append(lparts[1])
                multiline = True
            else:
                nlines.append(l)
    return nlines

Nikhil Rupanawar · Answer 5 · 2017-01-30T11:45:25.203

You may use regex, Assuming if you are interested in lines between {{{ }}}}

text = """First line
Second line
THIS{{{
these three lines
I want to process
together
}}}
Last Line"""

import re
match_obj = re.search('{{{(.*)}}}', text, re.DOTALL)
print match_obj.group(1)

OR

r = re.compile('{{{(.*)}}}', flags=re.DOTALL)
print re.split(r, text)
# replace \n
split_list = re.split(r, text)
split_list = [l.replace('\n', '') for l in split_list]
print split_list

OR

match_list = re.findall('{{{(.*)}}}', text, re.DOTALL)
match_list = [l.replace('\n', '') for l in match_list]
print match_list

If you have multiple occurrences of {{{ }}} in given text, use non-greedy match by adding '?' e.g. {{{(.*?)}}}

score 0 · Answer 6 · answered Jan 30 '17 at 11:03

I think this works as a quick and simple solution for what you're trying to accomplish:

text = """First line
Second line
{{{
these three lines
I want to process
together
}}}
Last Line"""

all_lines = [l for l in text.splitlines()]
final_list = []

nested = False

for line in all_lines:
    if line == "{{{":
        nested = True
        multiline = ""
        continue
    elif line == "}}}":
        nested = False
        final_list.append(multiline)
        continue


    if nested == True:        
        multiline = multiline + " " + line            
    else:
        final_list.append(line)


print(final_list)

Probably not the cleanest code ever, and I think we should replace multiline = multiline + " " + line by a .format(), but I hope you get the idea.

Oh... I noticed now that you also wanted to take care of occurrences where "{{{" comes between other text. That will require further work :) — Victor Domingos, Jan 30 '17 at 11:06

Mike Müller · Answer 7 · 2017-01-30T11:51:58.453

Tracking the opening {{{ and closing }}} in a loop with an in_multi flag is straigh forward:

def split_multi(s):
    lines = []
    in_multi = False
    for line in s.splitlines():
        if in_multi:
            if '}}}' in line:
                in_multi = False
                split = line.split('}}}')
                if split[0]:
                    tmp.append(split[0])
                lines.append(' '.join(tmp))
                if split[-1]:
                    lines.append(split[-1])
            else:
                tmp.append(line)
        else:
            if '{{{' in line:
                split = line.split('{{{')
                in_multi = True
                if split[0]:
                    lines.append(split[0])
                    if split[-1]:
                        tmp = [split[-1]]
                else:
                    tmp = []
            else:
                lines.append(line)

    return lines 


s1 = """First line
Second line
{{{
these three lines
I want to process
together
}}}
Last Line"""

s2 = """First Line
Second line
Third{{{line
fourth line
fifth}}}line
sixth line"""

print(split_multi(s1))
print(split_multi(s2))
#['First Line', 'Second line', 'Third', 'line fourth line fifth', 'line', 'sixth line']

Output:

['First line', 'Second line', 'these three lines I want to process together', 'Last Line']
['First Line', 'Second line', 'Third', 'line fourth line fifth', 'line', 'sixth line']

score 0 · Answer 8 · edited May 23 '17 at 12:33

My 2 cents (using joint):

ex1 = """First line
Second line
{{{
these three lines
I want to process
together
}}}
Last Line"""

ex2 = """First Line
Second line
Third{{{line
fourth line
fifth}}}line
sixth line"""

def parse_lines(txt, start_sep='{{{', end_sep='}}}'):
    depth = 0 # 1+ if we are inside a {{{ group
              # can be used to test unbalanced constructs
    lines = []
    current_line = ''
    n = len(txt)
    i = 0
    while i < n:
        c = txt[i]
        not_handled = True
        need_to_add = False
        if c == '\n': # end of line
            if depth == 0 : # save line and empty buffer
                need_to_add = True
            elif current_line != '': # add a space instead of the line break
                current_line = ''.join((current_line,' '))
            not_handled = False
            i += 1
        elif c == start_sep[0] and\
             txt[i:i+len(start_sep)] == start_sep:
             # ^ takes small advantage of lazy evaluation
             # (see questions/13960657)
                depth += 1
                need_to_add = True
                not_handled = False
                i += len(start_sep)
        elif c == end_sep[0] and\
             txt[i:i+len(end_sep)] == end_sep:
                depth -= 1
                need_to_add = True
                not_handled = False
                i += len(end_sep)
        if not_handled:
            current_line = ''.join((current_line,c))
            i += 1
        elif need_to_add and current_line != '':
            lines.append(current_line)
            current_line = ''
    if current_line != '': # add last line
        lines.append(current_line)
    return lines

Which returns:

>>> parse_lines(ex1)
['First line', 'Second line', 'these three lines I want to process together ', 'Last Line']
>>> parse_lines(ex2)
['First Line', 'Second line', 'Third', 'line fourth line fifth', 'line', 'sixth line']

Notice the extra ' ' on the multiline that ends with '\n}}}' in the first example.

Python: How to split lines merging some of the lines

8 Answers8