scrapy mysql returns empty results

Question

So my problem is that the information scraped, won't show up in the database.

My spider works fine printing out the information, for example in a .json file.

the pipelines.py

import sys
import MySQLdb
import hashlib
from scrapy.exceptions import DropItem
from scrapy.http import Request

class MySQLStorePipeline(object):
  def __init__(self):
   self.conn = MySQLdb.connect(host="10.0.2.2", user='root', passwd='', db='mpmf', charset="utf8", use_unicode=True)
   self.cursor = self.conn.cursor()

def process_item(self, item, stack):    
    try:
        self.cursor.execute("""INSERT INTO test (pen, name)  
                    VALUES (%s, %s)""", 
                   (item['pen'].encode('utf-8'), item['name'].encode('utf-8')))

    self.conn.commit()


except MySQLdb.Error, e:
    print "Error %d: %s" % (e.args[0], e.args[1])


return item

and in settings.py i have added

ITEM_PIPELINES = {
'stack.pipelines.MySQLStorePipeline': 300,
}

and my log show this errors but you can still see that the information gathering works even though it displays this.

 File "/usr/lib/python2.7/dist-packages/twisted/internet/defer.py", line   577, in _runCallbacks
 current.result = callback(current.result, *args, **kw)
 File "/root/stack/stack/pipelines.py", line 14, in process_item
 self.cursor.execute("""INSERT INTO test (pen, name) VALUES (%s, %s)""",  (item['pen'].encode('utf-8'), item['name'].encode('utf-8')))
 AttributeError: 'list' object has no attribute 'encode'

so no results are imported to the database

score 0 · Accepted Answer · answered Aug 08 '16 at 09:50

solved it problem was the indentation and list object had no attribute

import sys
import MySQLdb
import hashlib
from scrapy.exceptions import DropItem
from scrapy.http import Request

class MySQLStorePipeline(object):
def __init__(self):
self.conn = MySQLdb.connect(host="10.0.2.2", user='root', passwd='', db='mpmf', charset="utf8", use_unicode=True)
self.cursor = self.conn.cursor()

def process_item(self, item, stack):    
try:
    self.cursor.execute("""INSERT INTO test (pen, name) VALUES (%s, %s)""", (item['pen'][0].encode('utf-8'), item['name'][0].encode('utf-8')))

    self.conn.commit()


except MySQLdb.Error, e:
    print "Error %d: %s" % (e.args[0], e.args[1])


return item

scrapy mysql returns empty results

1 Answers1