Accelerate scrapy python3 script

Question

I would like to bulk download free-to-download pdfs (copies of an old newspaper from 1843 to 1900 called Gaceta) from this website of the Nicaraguan National Assembly with Python3/Scrapy (see former question here) using the below script:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# A scrapy script to download issues of the Gaceta de Nicaragua (1843-1961)

# virtualenv -p python3 envname
# source envname/bin/activate
# scrapy runspider gaceta_downloader.py

import errno
import json
import os

import scrapy
from scrapy import FormRequest, Request

pwd="/Downloads"
os.chdir(pwd) # this will change directory to pwd path.
print((os.getcwd()))

class AsambleaSpider(scrapy.Spider):
    name = 'asamblea'
    allowed_domains = ['asamblea.gob.ni']
    start_urls = ['http://digesto.asamblea.gob.ni/consultas/coleccion/']

    papers = {
        "Diario Oficial": "28",
    }

    def parse(self, response):

        for key, value in list(self.papers.items()):
            yield FormRequest(url='http://digesto.asamblea.gob.ni/consultas/util/ws/proxy.php',
                  headers= {
                      'X-Requested-With': 'XMLHttpRequest'
                  }, formdata= {
                        'hddQueryType': 'initgetRdds',
                        'cole': value
                    }
                    , meta={'paper': key},
                    callback=self.parse_rdds
                )
        pass

    def parse_rdds(self, response):
        data = json.loads(response.body_as_unicode())
        for r in data["rdds"]:
            r['paper'] = response.meta['paper']
            rddid = r['rddid']
            yield Request("http://digesto.asamblea.gob.ni/consultas/util/pdf.php?type=rdd&rdd=" + rddid,
                          callback=self.download_pdf, meta=r)

    def download_pdf(self, response):
       filename = "{paper}/{anio}/".format(**response.meta) + "{titulo}-{fecPublica}.pdf".format(**response.meta).replace("/", "_")
       if not os.path.exists(os.path.dirname(filename)):
           try:
               os.makedirs(os.path.dirname(filename))
           except OSError as exc:  # guard against race condition
               if exc.errno != errno.EEXIST:
                   raise

       with open(filename, 'wb') as f:
           f.write(response.body)

The script does its job fetching the direct links from a php file and downloading the PDF subsequently, however there are two things still bugging me:

1. I would like to be able to set the time range of Gacetas I would like to download, i. e. all issues (of available) between 01/01/1844 to 01/01/1900. I tried to figured it out myself to no avail as I am a programming novice.
2. I would like to accelerate the script. Maybe with xargs? As for now it feels quite slow in execution even though I did not have measured it.

Answer 1

Disclaimer: I did not test the script since scrapy requires Microsoft Visual C++ 14.0 and it takes a while to download and install :(

Here's an updated script, I added the date range as start and end and modified the parse_rdds method to only download files in the time frame.

As for optimize it, scrapy is a non-blocking lib and as I understand it should be able to download several files in parallel as it is right now. Keep in mind that you're downloading what it seems a lot of files so, it naturally could take a while.

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# A scrapy script to download issues of the Gaceta de Nicaragua (1843-1961)

# virtualenv -p python3 envname
# source envname/bin/activate
# scrapy runspider gaceta_downloader.py

import errno
import json
import os
from datetime import datetime

import scrapy
from scrapy import FormRequest, Request

pwd="/Downloads"
os.chdir(pwd) # this will change directory to pwd path.
print((os.getcwd()))


# date range, format DD/MM/YYYY
start = '16/01/1844'
end = '01/01/1900'

date_format = '%d/%m/%Y'
start = datetime.strptime(start, date_format)
end = datetime.strptime(end, date_format)


class AsambleaSpider(scrapy.Spider):
    name = 'asamblea'
    allowed_domains = ['asamblea.gob.ni']
    start_urls = ['http://digesto.asamblea.gob.ni/consultas/coleccion/']

    papers = {
        "Diario Oficial": "28",
    }

    def parse(self, response):

        for key, value in list(self.papers.items()):
            yield FormRequest(url='http://digesto.asamblea.gob.ni/consultas/util/ws/proxy.php',
                  headers= {
                      'X-Requested-With': 'XMLHttpRequest'
                  }, formdata= {
                        'hddQueryType': 'initgetRdds',
                        'cole': value
                    }
                    , meta={'paper': key},
                    callback=self.parse_rdds
                )
        pass

    def parse_rdds(self, response):
        data = json.loads(response.body_as_unicode())
        for r in data["rdds"]:
            if not r['fecPublica']:
                continue

            r_date = datetime.strptime(r['fecPublica'], date_format)

            if start <= r_date <= end:
                r['paper'] = response.meta['paper']
                rddid = r['rddid']
                yield Request("http://digesto.asamblea.gob.ni/consultas/util/pdf.php?type=rdd&rdd=" + rddid,
                              callback=self.download_pdf, meta=r)

    def download_pdf(self, response):
       filename = "{paper}/{anio}/".format(**response.meta) + "{titulo}-{fecPublica}.pdf".format(**response.meta).replace("/", "_")
       if not os.path.exists(os.path.dirname(filename)):
           try:
               os.makedirs(os.path.dirname(filename))
           except OSError as exc:  # guard against race condition
               if exc.errno != errno.EEXIST:
                   raise

       with open(filename, 'wb') as f:
           f.write(response.body)