How do I select a link with a right arrow as the text with xpath?

Question

I'm attempting to select the next button on a website, and it has a right arrow as the text of the link. When I view the source using "scrappy shell" it shows me the character as its unicode literal "\u2192". With this, I developed the following Scrapy CrawlSpider:

# -*- coding: utf-8 -*-
import scrapy
from scrapy.contrib.linkextractors import LinkExtractor
from scrapy.contrib.loader.processor import MapCompose
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy import log, Request
from yelpscraper.items import YelpscraperItem
import re, urlparse


class YelpSpider(CrawlSpider):
    name = 'yelp'
    allowed_domains = ['yelp.com']
    start_urls = ['http://www.yelp.com/search?find_desc=attorney&find_loc=Austin%2C+TX&start=0']

    rules = (
        Rule(LinkExtractor(allow=r'biz', restrict_xpaths='//*[contains(@class, "natural-search-result")]//a[@class="biz-name"]'), callback='parse_item', follow=True),
        Rule(LinkExtractor(allow=r'start', restrict_xpaths=u'//a[contains(@class, "prev-next")]/text()[contains(., "\u2192")]'), follow=True)
    )

    def parse_item(self, response):
        i = YelpscraperItem()
        i['phone'] = self.beautify(response.xpath('//*[@class="biz-phone"]/text()').extract())
        i['state'] = self.beautify(response.xpath('//span[@itemprop="addressRegion"]/text()').extract())
        i['company'] = self.beautify(response.xpath('//h1[contains(@class, "biz-page-title")]/text()').extract())

        website = i['website'] = self.beautify(response.xpath('//div[@class="biz-website"]/a/text()').extract())
        yield i

Take note of the second tuple declaration in the rules attribute which contains the problematic unicode character:

Rule(LinkExtractor(allow=r'start', restrict_xpaths=u'//a[contains(@class, "prev-next")]/text()[contains(., "\u2192")]'), follow=True)

When I try and run this spider, I'm given the following Traceback:

Traceback (most recent call last):
    File "C:\Python27\lib\site-packages\twisted\internet\base.py", line 824, in runUntilCurrent
     call.func(*call.args, **call.kw)
    File "C:\Python27\lib\site-packages\twisted\internet\task.py", line 607, in _tick
     taskObj._oneWorkUnit()
    File "C:\Python27\lib\site-packages\twisted\internet\task.py", line 484, in _oneWorkUnit
     result = next(self._iterator)
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\utils\defer.py", line 57, in <genexpr>
     work = (callable(elem, *args, **named) for elem in iterable)
    --- <exception caught here> ---
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\utils\defer.py", line 96, in iter_errback
     yield next(it)
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\contrib\spidermiddleware\offsite.py", line 26, in process_spider_output
     for x in result:
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\contrib\spidermiddleware\referer.py", line 22, in <genexpr>
     return (_set_referer(r) for r in result or ())
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\contrib\spidermiddleware\urllength.py", line 33, in <genexpr>
     return (r for r in result or () if _filter(r))
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\contrib\spidermiddleware\depth.py", line 50, in <genexpr>
     return (r for r in result or () if _filter(r))
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\contrib\spiders\crawl.py", line 73, in _parse_response
     for request_or_item in self._requests_to_follow(response):
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\contrib\spiders\crawl.py", line 52, in _requests_to_follow
     links = [l for l in rule.link_extractor.extract_links(response) if l not in seen]
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\contrib\linkextractors\lxmlhtml.py", line 107, in extract_links
     links = self._extract_links(doc, response.url, response.encoding, base_url)
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\linkextractor.py", line 94, in _extract_links
     return self.link_extractor._extract_links(*args, **kwargs)
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\contrib\linkextractors\lxmlhtml.py", line 50, in _extract_links
     for el, attr, attr_val in self._iter_links(selector._root):
    File "C:\Python27\lib\site-packages\scrapy-0.24.4-py2.7.egg\scrapy\contrib\linkextractors\lxmlhtml.py", line 38, in _iter_links
     for el in document.iter(etree.Element):
    exceptions.AttributeError: 'unicode' object has no attribute 'iter'

All I want to do is select this link and I can't think of a way to select it without using this character. (It moves around depending on the page). Is there anyway to select it using it's ASCII code or something other than unicode; which seems to be causing the problem?

alecxe · Accepted Answer · 2014-12-22T04:53:26.240

According to the documentation, restrict_xpaths should be either a list, or str.

You are passing a unicode string. This is why you are getting an error.

Besides, you don't need to check the text(), checking for prev-next class is good enough:

rules = (
    Rule(LinkExtractor(allow=r'biz', restrict_xpaths='//*[contains(@class, "natural-search-result")]//a[@class="biz-name"]'),
         callback='parse_item', follow=True),
    Rule(LinkExtractor(allow=r'start', restrict_xpaths='//a[contains(@class, "prev-next")]'),
         follow=True)
)

^{Tested (no errors while crawling, it follows the pagination).}

How do I select a link with a right arrow as the text with xpath?

1 Answers1