python csv, writing headers only once

Question

So I have a program that creates CSV from .Json.

First I load the json file.

f = open('Data.json')
data = json.load(f)
f.close()

Then I go through it, looking for a specific keyword, if I find that keyword. I'll write everything related to that in a .csv file.

for item in data:
    if "light" in item:
       write_light_csv('light.csv', item)

This is my write_light_csv function :

def write_light_csv(filename,dic):

    with open (filename,'a') as csvfile:
        headers = ['TimeStamp', 'light','Proximity']
        writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)

        writer.writeheader()

        writer.writerow({'TimeStamp': dic['ts'], 'light' : dic['light'],'Proximity' : dic['prox']})

I initially had wb+ as the mode, but that cleared everything each time the file was opened for writing. I replaced that with a and now every time it writes, it adds a header. How do I make sure that header is only written once?.

Igor Hatarist · Accepted Answer · 2015-02-04T16:09:21.750

67

You could check if file is already exists and then don't call writeheader() since you're opening the file with an append option.

Something like that:

import os.path


file_exists = os.path.isfile(filename)

with open (filename, 'a') as csvfile:
    headers = ['TimeStamp', 'light', 'Proximity']
    writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)

    if not file_exists:
        writer.writeheader()  # file doesn't exist yet, write a header

    writer.writerow({'TimeStamp': dic['ts'], 'light': dic['light'], 'Proximity': dic['prox']})

edited Feb 04 '15 at 16:09

answered Feb 04 '15 at 15:53

Igor Hatarist

5,234
2
32
45

@Kos The file won't be written to the disk before while there were no operations with the file object within a `with` block, but you're right, it's a bit confusing. I changed my example. – Igor Hatarist Feb 04 '15 at 15:58
@Kos I'm sorry, you're right, the file gets created earlier. – Igor Hatarist Feb 04 '15 at 16:07
@igor, there is a typo on the line `file_exists`, you don't need that colon there :) – cyberbemon Feb 04 '15 at 16:09
@IgorHatarist Thank you very much! – Sohan Das Oct 17 '18 at 08:55
Thanks you very much, It really solved my problem – Programmer May 22 '21 at 12:38

score 38 · Answer 2 · answered Jan 13 '19 at 09:11

38

Just another way:

with open(file_path, 'a') as file:
        w = csv.DictWriter(file, my_dict.keys())

        if file.tell() == 0:
            w.writeheader()

        w.writerow(my_dict)

answered Jan 13 '19 at 09:11

SpiralDev

7,011
5
28
42

score 9 · Answer 3 · answered Jun 08 '18 at 17:40

You can check if the file is empty

import csv
import os

headers = ['head1', 'head2']

for row in interator:
    with open('file.csv', 'a') as f:
        file_is_empty = os.stat('file.csv').st_size == 0
        writer = csv.writer(f, lineterminator='\n')
        if file_is_empty:
            writer.writerow(headers)
        writer.writerow(row)

Learner · Answer 4 · 2015-11-03T13:04:57.123

I would use some flag and run a check before writing headers! e.g.

flag=0
def get_data(lst):
    for i in lst:#say list of url
        global flag
        respons = requests.get(i)
        respons= respons.content.encode('utf-8')
        respons=respons.replace('\\','')
        print respons
        data = json.loads(respons)
        fl = codecs.open(r"C:\Users\TEST\Desktop\data1.txt",'ab',encoding='utf-8')
        writer = csv.DictWriter(fl,data.keys())
        if flag==0:
            writer.writeheader()
        writer.writerow(data)
        flag+=1
        print "You have written % times"%(str(flag))
    fl.close()
get_data(urls)

score 1 · Answer 5 · answered Feb 04 '15 at 15:55

Can you change the structure of your code and export the whole file at once?

def write_light_csv(filename, data):
    with open (filename, 'w') as csvfile:
        headers = ['TimeStamp', 'light','Proximity']
        writer = csv.DictWriter(csvfile, delimiter=',', lineterminator='\n',fieldnames=headers)

        writer.writeheader()

        for item in data:
            if "light" in item:
                writer.writerow({'TimeStamp': item['ts'], 'light' : item['light'],'Proximity' : item['prox']})


write_light_csv('light.csv', data)

score 0 · Answer 6 · answered May 15 '19 at 21:05

0

You can use the csv.Sniffer Class and

with open('my.csv', newline='') as csvfile:
    if csv.Sniffer().has_header(csvfile.read(1024))
    # skip writing headers

answered May 15 '19 at 21:05

rearThing

632
3
9
26

score 0 · Answer 7 · answered Sep 23 '20 at 16:09

While using Pandas: (for storing Dataframe data to CSV file) just add this check before setting header property if you are using an index to iterate over API calls to add data in CSV file.

if i > 0:
        dataset.to_csv('file_name.csv',index=False, mode='a', header=False)
    else:
        dataset.to_csv('file_name.csv',index=False, mode='a', header=True)

score 0 · Answer 8 · answered Jun 21 '21 at 23:07

Here's another example that only depends on Python's builtin csv package. This method checks that the header is what's expected or it throws an error. It also handles the case where the file doesn't exist or does exist but is empty by writing the header. Hope this helps:

import csv
import os


def append_to_csv(path, fieldnames, rows):
    is_write_header = not os.path.exists(path) or _is_empty_file(path)
    if not is_write_header:
        _assert_field_names_match(path, fieldnames)

    _append_to_csv(path, fieldnames, rows, is_write_header)


def _is_empty_file(path):
    return os.stat(path).st_size == 0


def _assert_field_names_match(path, fieldnames):
    with open(path, 'r') as f:
        reader = csv.reader(f)
        header = next(reader)
        if header != fieldnames:
            raise ValueError(f'Incompatible header: expected {fieldnames}, '
                             f'but existing file has {header}')


def _append_to_csv(path, fieldnames, rows, is_write_header: bool):
    with open(path, 'a') as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames)
        if is_write_header:
            writer.writeheader()
        writer.writerows(rows)

You can test this with the following code:


file_ = 'countries.csv'
fieldnames_ = ['name', 'area', 'country_code2', 'country_code3']
rows_ = [
    {'name': 'Albania', 'area': 28748, 'country_code2': 'AL', 'country_code3': 'ALB'},
    {'name': 'Algeria', 'area': 2381741, 'country_code2': 'DZ', 'country_code3': 'DZA'},
    {'name': 'American Samoa', 'area': 199, 'country_code2': 'AS', 'country_code3': 'ASM'}
]

append_to_csv(file_, fieldnames_, rows_)

If you run this once you get the following in countries.csv:

name,area,country_code2,country_code3
Albania,28748,AL,ALB
Algeria,2381741,DZ,DZA
American Samoa,199,AS,ASM

And if you run it twice you get the following (note, no second header):

name,area,country_code2,country_code3
Albania,28748,AL,ALB
Algeria,2381741,DZ,DZA
American Samoa,199,AS,ASM
Albania,28748,AL,ALB
Algeria,2381741,DZ,DZA
American Samoa,199,AS,ASM

If you then change the header in countries.csv and run the program again, you'll get a value error, like this:

ValueError: Incompatible header: expected ['name', 'area', 'country_code2', 'country_code3'], but existing file has ['not', 'right', 'fieldnames']

python csv, writing headers only once

8 Answers8

Linked

Related