I am trying to create a sentiment analysis program using a sentiment lexicon. The sentences that will be analyzed are read from a CSV file, and after analyzed, it will be written again in a different CSV file. However, I got the AttributeError: 'list' object has no attribute 'lower' error. The error seems to appear from this part of the code.
# only replace if everything succeeds
os.rename(tempname, filename)
Here is the full code :
# -*- coding: UTF-8 -*-
import codecs
import re
import os
import sys, argparse
import subprocess
import pprint
import csv
from itertools import islice
import pickle
import nltk
from nltk import tokenize
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
import pandas as pd
try:
import treetaggerwrapper
from treetaggerwrapper import TreeTagger, make_tags
print("import TreeTagger OK")
except:
print("Import TreeTagger pas Ok")
from itertools import islice
from collections import defaultdict
class Sentiment:
__slots__ = ('positive', 'neutral', 'negative')
def __init__(self, positive=0, neutral=0, negative=0):
self.positive = positive
self.neutral = neutral
self.negative = negative
def __repr__(self):
return f'<Sentiment {self.positive} {self.neutral} {self.negative}>'
def __add__(self, other):
return Sentiment(
self.positive + other.positive,
self.neutral + other.neutral,
self.negative + other.negative,
)
#export le lexique de sentiments
with codecs.open('lexique.txt', 'r', 'utf-8', errors = 'ignore') as text_file:
text_file =(text_file.read())
#print(text_file)
dico_lexique = ({i.split(";")[1]:i.split(";")[2:] for i in text_file.split("\n") if i}) # Spliting the text content and creating a dictionary
#print(dico_lexique)
#print(type(dico_lexique))
#print(dico_lexique)
for k, (pos, neu, neg) in dico_lexique.items():
dico_lexique[k] = Sentiment(int(pos), int(neu), int(neg))
'''
with open("dict_pickle", "rb") as fd:
dico_lexique = {}
for word, (pos, neu, neg) in pickle.load(fd).items():
dico_lexique[word] = Sentiment(int(pos), int(neu), int(neg))
'''
stopWords = set(stopwords.words('french'))
tagger = treetaggerwrapper.TreeTagger(TAGLANG='fr')
def process_text(text):
"""process the specified text
returns (words, filtered words, total sentiment score)
"""
words = []
filtered = []
score = Sentiment()
for tag in make_tags(tagger.tag_text(text)):
word = tag.lemma
words.append(word)
if word not in stopWords and word.isalpha():
filtered.append(word)
sentiment = dico_lexique.get(k)
if sentiment is not None:
score += sentiment
return words, filtered, score
filename = sys.argv[1]
tempname = filename + '~'
with open(filename) as fdin, open(tempname, 'w') as fdout:
inp = csv.reader(fdin, delimiter=';')
out = csv.writer(fdout, delimiter=';')
# get the header, and blindly append out column names
header = next(inp)
out.writerow(header + [
'd_lemma', 'd_filtered_words', 'Positive Score', 'Neutral Score', 'Negative Score',
])
for row in inp:
# assume that second item contains the text we want to process
words, filtered, score = process_text(row[1])
extra_values = [
words, filtered,
score.positive, score.neutral, score.negative,
]
# add the values and write out
assert len(row) == len(header), "code needed to pad the columns out"
out.writerow(row + extra_values)
# only replace if everything succeeds
os.rename(tempname, filename)
Stacktrace looks like this
import TreeTagger OK
Traceback (most recent call last):
File "lexicon_based.py", line 112, in <module>
os.rename(tempname, filename)
FileExistsError: [WinError 183] Impossible de créer un fichier déjà existant: 'test.csv~' -> 'test.csv'
the lexicon looks like this :
0;***;21;127;0
1;vraiment;407;156;37
2;oiseau-à-berceau;102;259;0
3;Stoph;95;308;37
4;Conscience;372;144;35
5;rançonnement;0;635;433
6;provenir;304;227;47
7;féliciteur;285;54;1
8;traversée;360;167;38
9;avant toute chose;241;108;34
the csv looks like this but it is much longer :
id;text
1; Cette thèse d'une « Wehrmacht aux mains propres » est remise en cause par les milieux scientifiques dès la fin des années 1970.
2; Or, les avancées de l'historiographie au cours des années 1980, ainsi que le large succès de l'exposition consacrée aux crimes de la Wehrmacht, itinérante en Allemagne et en Autriche dans la seconde moitié des années 1990 et visitée par environ 900 000 personnes, achèvent de la discréditer auprès de l'opinion publique.
3; Ainsi, la responsabilité de la Wehrmacht dans le cadre de la guerre d'extermination menée sur le front de l'Est, pour la mise en œuvre du Kommissarbefehl, son soutien aux massacres de Juifs commis par les Einsatzgruppen et sa radicalisation sur le front de l'Ouest sont aujourd'hui établis et largement reconnus.
