I am using python, oursql and facebook SDK to scrape the data from Facebook and put it into the database. I wrote a function and manage to get the data in a list, but executemany puts into the database only first row. I don't know where the error is. Thanks for the help in advance. This is my function:
def get_posts(page_id, follow_account_id):
graph = facebook.GraphAPI(get_facebook_access_token())
posts = graph.get_connections(page_id, 'posts')
facebook_posts = list()
should_finish = False
while True: # keep paginating
try:
with get_db_connection() as cursor:
for post in posts['data']:
#print post
if datetime_from_iso(post["created_time"]) < until_time:
should_finish = True
break
try:
user_id, post_id = post["id"].split("_") # "133174387208_10154860725907209
link = "https://www.facebook.com/{0}/posts/{1}".format(user_id, post_id)
except:
link = None
facebook_posts.append((3, post_id, None, None, 0, post.get("from").get("name"), post.get("from").get("id"), post.get("from").get("name"), filter_using_re(post.get("message", None)), link, follow_account_id))
print facebook_posts
cursor.executemany(post_insert_query, facebook_posts)
# get next page
posts = requests.get(posts['paging']['next']).json()
except KeyError:
break
Prepared statement:
post_insert_query = u'''
INSERT INTO post(
social_media_id, external_id, post_time, post_date, related_post_id, poster_name, poster_id, title, content, post_url,
follow_account_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'''