0
import scrapy
from ..items import CentriItem
from scrapy_playwright.page import PageMethod

class NikespiderSpider(scrapy.Spider):
    name = 'nikespider'

    def start_requests(self):
        url = "https://www.nike.com/ch/en/w/mens-nik1?q=shirts"
        yield scrapy.Request(url, meta=dict(
            playwright=True,
            playwright_include_page=True,
            playwright_page_methods=[
                PageMethod("wait_for_selector", ".product-card__link-overlay"),
                PageMethod("evaluate", "window.scrollBy(0, document.body.scrollHeight)"),
                PageMethod("wait_for_selector",".css-c2ovjx:nth-child(25) .product-card__link-overlay"),
            ]
        ))

    async def parse(self, response):
        page = response.meta["playwright_page"]
        await page.close()
        i = 0
        for link in response.css(".product-card__link-overlay::attr(href)"):
            i = i + 1
            print(i)

    async def parse_items(self, response,link):
        pass

This is my code for scraping Nike. The url which I am scraping has infinite scrolling. It loads 24 items at first and after you scroll down it loads 24 more. I am trying to add this functionality using scrapy-playwright but it is not working. Anyone able to help me?

Emad
  • 1

1 Answers1

0

It looks like it would probably be easier to extract the data from their api.

import scrapy

class NikespiderSpider(scrapy.Spider):
    name = 'nikespider'

    def start_requests(self):
        for i in range(1, 20):
            v = i * 12
            url = f"https://api.nike.com/cic/browse/v2?queryid=products&anonymousId=699ECBF06AACCD31DCB69E0C71964748&country=ch&endpoint=%2Fproduct_feed%2Frollup_threads%2Fv2%3Ffilter%3Dmarketplace(CH)%26filter%3Dlanguage(en-GB)%26filter%3DemployeePrice(true)%26filter%3DattributeIds(0f64ecc7-d624-4e91-b171-b83a03dd8550)%26searchTerms%3Dshirts%26anchor%3D{v}%26consumerChannelId%3Dd9a5bc42-4b9c-4976-858a-f159cf99c647%26count%3D24&language=en-GB&localizedRangeStr=%7BlowestPrice%7D%E2%80%94%7BhighestPrice%7D"
            yield scrapy.Request(url=url)



    def parse(self, response):
        yield response.json()

output for the first page looks like this:

  {
    "data": {
      "products": {
        "errors": null,
        "products": [
          {
            "cardType": "default",
            "cloudProductId": "a44f2253-c745-5143-bd1e-f2c841da1550",
            "colorDescription": "Obsidian/Pepper Red/Gorge Green/Gold Dart",
            "colorways": [
              {
                "colorDescription": "Obsidian/Pepper Red/Gorge Green/Gold Dart",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/a77364ff-d0e8-4eaa-83eb-2a206896b28d/image.jpg",
                  "squarishURL": "https://static.nike.com/a/images/t_default/e1374e74-f614-40dd-aecd-ac4c0bd1a5bb/portugal-french-terry-football-hoodie-rF3Vcw.png"
                },
                "pdpUrl": "{countryLang}/t/portugal-french-terry-football-hoodie-rF3Vcw/DH4826-451",
                "price": {
                  "currency": "CHF",
                  "currentPrice": 109.95,
                  "discounted": false,
                  "employeePrice": 0,
                  "fullPrice": 109.95,
                  "minimumAdvertisedPrice": null
                },
                "cloudProductId": "a44f2253-c745-5143-bd1e-f2c841da1550",
                "inStock": true,
                "isComingSoon": false,
                "isBestSeller": false,
                "isExcluded": false,
                "isLaunch": false,
                "isMemberExclusive": false,
                "isNew": true,
                "label": "JUST_IN",
                "pid": "14061770",
                "prebuildId": null,
                "productInstanceId": null
              }
            ],
            "customizable": false,
            "hasExtendedSizing": false,
            "id": "96633291-36b7-4a6d-a1ab-ef0e264da2c2",
            "images": {
              "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/a77364ff-d0e8-4eaa-83eb-2a206896b28d/image.jpg",
              "squarishURL": "https://static.nike.com/a/images/t_default/e1374e74-f614-40dd-aecd-ac4c0bd1a5bb/portugal-french-terry-football-hoodie-rF3Vcw.png"
            },
            "inStock": true,
            "isComingSoon": false,
            "isBestSeller": false,
            "isExcluded": false,
            "isGiftCard": false,
            "isJersey": false,
            "isLaunch": false,
            "isMemberExclusive": false,
            "isNBA": false,
            "isNFL": false,
            "isSustainable": false,
            "label": "JUST_IN",
            "nbyColorway": null,
            "pid": "14061770",
            "prebuildId": null,
            "price": {
              "currency": "CHF",
              "currentPrice": 109.95,
              "discounted": false,
              "employeePrice": 0,
              "fullPrice": 109.95,
              "minimumAdvertisedPrice": null
            },
            "productInstanceId": null,
            "productType": "APPAREL",
            "properties": null,
            "salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
            "subtitle": "Men's Nike French Terry Football Hoodie",
            "title": "Portugal",
            "url": "{countryLang}/t/portugal-french-terry-football-hoodie-rF3Vcw/DH4826-451"
          },
          {
            "cardType": "default",
            "cloudProductId": "a05bd5c6-290e-5d8c-b583-9c8fd49fee3b",
            "colorDescription": "Coastal Blue/Green Spark/Dynamic Yellow",
            "colorways": [
              {
                "colorDescription": "Coastal Blue/Green Spark/Dynamic Yellow",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/af0e8a3f-7f4b-417c-b7b1-6acd09607ee2/image.jpg",
                  "squarishURL": "https://static.nike.com/a/images/t_default/d6150c37-9086-4b1a-b1f0-66e59f3258cb/brazil-travel-fleece-football-hoodie-H4PKQk.png"
                },
                "pdpUrl": "{countryLang}/t/brazil-travel-fleece-football-hoodie-H4PKQk/DH4822-490",
                "price": {
                  "currency": "CHF",
                  "currentPrice": 109.95,
                  "discounted": false,
                  "employeePrice": 0,
                  "fullPrice": 109.95,
                  "minimumAdvertisedPrice": null
                },
                "cloudProductId": "a05bd5c6-290e-5d8c-b583-9c8fd49fee3b",
                "inStock": true,
                "isComingSoon": false,
                "isBestSeller": false,
                "isExcluded": false,
                "isLaunch": false,
                "isMemberExclusive": false,
                "isNew": true,
                "label": "JUST_IN",
                "pid": "14061554",
                "prebuildId": null,
                "productInstanceId": null
              }
            ],
            "customizable": false,
            "hasExtendedSizing": false,
            "id": "9ba8eaff-d94d-445a-b569-8bae4c1d741b",
            "images": {
              "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/af0e8a3f-7f4b-417c-b7b1-6acd09607ee2/image.jpg",
              "squarishURL": "https://static.nike.com/a/images/t_default/d6150c37-9086-4b1a-b1f0-66e59f3258cb/brazil-travel-fleece-football-hoodie-H4PKQk.png"
            },
            "inStock": true,
            "isComingSoon": false,
            "isBestSeller": false,
            "isExcluded": false,
            "isGiftCard": false,
            "isJersey": false,
            "isLaunch": false,
            "isMemberExclusive": false,
            "isNBA": false,
            "isNFL": false,
            "isSustainable": false,
            "label": "JUST_IN",
            "nbyColorway": null,
            "pid": "14061554",
            "prebuildId": null,
            "price": {
              "currency": "CHF",
              "currentPrice": 109.95,
              "discounted": false,
              "employeePrice": 0,
              "fullPrice": 109.95,
              "minimumAdvertisedPrice": null
            },
            "productInstanceId": null,
            "productType": "APPAREL",
            "properties": null,
            "salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
            "subtitle": "Men's Fleece Football Hoodie",
            "title": "Brazil Travel",
            "url": "{countryLang}/t/brazil-travel-fleece-football-hoodie-H4PKQk/DH4822-490"
          },
          {
            "cardType": "default",
            "cloudProductId": "40d88287-0129-52c7-ad5e-646ba8333389",
            "colorDescription": "Game Royal/Blue Fury/Blue Fury",
            "colorways": [
              {
                "colorDescription": "Game Royal/Blue Fury/Blue Fury",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/b66e9663-2fd1-4398-9270-d4e4d46ed83e/image.jpg",
                  "squarishURL": "https://static.nike.com/a/images/t_default/873eaab7-cfec-4b70-b6af-19880a563637/england-strike-dri-fit-short-sleeve-football-top-1m2GSc.png"
                },
                "pdpUrl": "{countryLang}/t/england-strike-dri-fit-short-sleeve-football-top-1m2GSc/DH6443-480",
                "price": {
                  "currency": "CHF",
                  "currentPrice": 59.95,
                  "discounted": false,
                  "employeePrice": 0,
                  "fullPrice": 59.95,
                  "minimumAdvertisedPrice": null
                },
                "cloudProductId": "40d88287-0129-52c7-ad5e-646ba8333389",
                "inStock": true,
                "isComingSoon": false,
                "isBestSeller": false,
                "isExcluded": false,
                "isLaunch": false,
                "isMemberExclusive": false,
                "isNew": true,
                "label": "JUST_IN",
                "pid": "14062596",
                "prebuildId": null,
                "productInstanceId": null
              }
            ],
            "customizable": false,
            "hasExtendedSizing": false,
            "id": "f73148f5-4b63-4b1f-84cc-b03bcc408ca1",
            "images": {
              "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/b66e9663-2fd1-4398-9270-d4e4d46ed83e/image.jpg",
              "squarishURL": "https://static.nike.com/a/images/t_default/873eaab7-cfec-4b70-b6af-19880a563637/england-strike-dri-fit-short-sleeve-football-top-1m2GSc.png"
            },
            "inStock": true,
            "isComingSoon": false,
            "isBestSeller": false,
            "isExcluded": false,
            "isGiftCard": false,
            "isJersey": false,
            "isLaunch": false,
            "isMemberExclusive": false,
            "isNBA": false,
            "isNFL": false,
            "isSustainable": true,
            "label": "JUST_IN",
            "nbyColorway": null,
            "pid": "14062596",
            "prebuildId": null,
            "price": {
              "currency": "CHF",
              "currentPrice": 59.95,
              "discounted": false,
              "employeePrice": 0,
              "fullPrice": 59.95,
              "minimumAdvertisedPrice": null
            },
            "productInstanceId": null,
            "productType": "APPAREL",
            "properties": null,
            "salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
            "subtitle": "Men's Nike Dri-FIT Short-Sleeve Football Top",
            "title": "England Strike",
            "url": "{countryLang}/t/england-strike-dri-fit-short-sleeve-football-top-1m2GSc/DH6443-480"
          },
          {
            "cardType": "default",
            "cloudProductId": "e04c4bb3-4382-5df4-9f03-7c1d393f9d05",
            "colorDescription": "Obsidian/Green Glow/Green Glow",
            "colorways": [
              {
                "colorDescription": "Obsidian/Green Glow/Green Glow",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/4cbd4672-d5ac-453b-b023-2d5160b33341/image.jpg",
                  "squarishURL": "https://static.nike.com/a/images/t_default/7f04a8ed-649c-4705-b110-0c605b852f7b/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs.png"
                },
                "pdpUrl": "{countryLang}/t/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs/DN0676-451",
                "price": {
                  "currency": "CHF",
                  "currentPrice": 119.95,
                  "discounted": false,
                  "employeePrice": 0,
                  "fullPrice": 119.95,
                  "minimumAdvertisedPrice": null
                },
                "cloudProductId": "e04c4bb3-4382-5df4-9f03-7c1d393f9d05",
                "inStock": true,
                "isComingSoon": false,
                "isBestSeller": false,
                "isExcluded": false,
                "isLaunch": false,
                "isMemberExclusive": false,
                "isNew": true,
                "label": "JUST_IN",
                "pid": "14063595",
                "prebuildId": null,
                "productInstanceId": null
              }
            ],
            "customizable": false,
            "hasExtendedSizing": false,
            "id": "0027a0c6-0e97-445c-b6ed-a92a6bfdf18b",
            "images": {
              "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/4cbd4672-d5ac-453b-b023-2d5160b33341/image.jpg",
              "squarishURL": "https://static.nike.com/a/images/t_default/7f04a8ed-649c-4705-b110-0c605b852f7b/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs.png"
            },
            "inStock": true,
            "isComingSoon": false,
            "isBestSeller": false,
            "isExcluded": false,
            "isGiftCard": false,
            "isJersey": false,
            "isLaunch": false,
            "isMemberExclusive": false,
            "isNBA": false,
            "isNFL": false,
            "isSustainable": true,
            "label": "JUST_IN",
            "nbyColorway": null,
            "pid": "14063595",
            "prebuildId": null,
            "price": {
              "currency": "CHF",
              "currentPrice": 119.95,
              "discounted": false,
              "employeePrice": 0,
              "fullPrice": 119.95,
              "minimumAdvertisedPrice": null
            },
            "productInstanceId": null,
            "productType": "APPAREL",
            "properties": null,
            "salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
            "subtitle": "Men's Nike Dri-FIT Football Shirt",
            "title": "Australia 2022/23 Stadium Away",
            "url": "{countryLang}/t/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs/DN0676-451"
          },
          {
            "cardType": "default",
            "cloudProductId": "758da5f5-ea79-5bf7-a401-8ccb945a5f16",
            "colorDescription": "Off-Noir/White",
            "colorways": [
              {
                "colorDescription": "Off-Noir/White",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/1efdbff3-1c9e-4984-9bf2-8da11d8f5161/image.jpg",
                  "squarishURL": "https://static.nike.com/a/images/t_default/5ca1f9a9-f1fa-42fe-a552-571ffbcc8ba6/nikecourt-dri-fit-rafa-challenger-short-sleeve-tennis-top-wNcD3N.png"
                },
                "pdpUrl": "{countryLang}/t/nikecourt-dri-fit-rafa-challenger-short-sleeve-tennis-top-wNcD3N/DD8547-045",
                "price": {
                  "currency": "CHF",
                  "currentPrice": 84.95,
                  "discounted": false,
                  "employeePrice": 0,
                  "fullPrice": 84.95,
                  "minimumAdvertisedPrice": null
                },
                "cloudProductId": "758da5f5-ea79-5bf7-a401-8ccb945a5f16",
                "inStock": true,
                "isComingSoon": false,
                "isBestSeller": false,
                "isExcluded": false,
                "isLaunch": false,
                "isMemberExclusive": false,
                "isNew": true,
                "label": "JUST_IN",
                "pid": "14168043",
                "prebuildId": null,
                "productInstanceId": null
              },
              {
                "colorDescription": "Pure Platinum/Pink Gaze/Black",
                "images": {
                  "portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/dbf2edc2-da8b-4e7e-976f-5d16f207b449/image.jpg",
 
...
Alexander
  • 16,091
  • 5
  • 13
  • 29
  • I need a somewhat universal solution for many websites. I don't think all websites will have an api or would they? I am going to scrape from 100's of clothing websites like nike. – Emad Sep 29 '22 at 06:48
  • @Emad No they won't all have an api... But they won't all have anything in common necesssarily. There is no solution that works for every site – Alexander Sep 29 '22 at 07:39
  • I have seen that most have the search thing so you can just search and then they have infinite scroll which is why I thought this was good.. Should I switch to API scraping then? If yes can you suggest a good video for introduction to it. – Emad Sep 29 '22 at 18:15
  • @Emad That is something only you can really decide. If you manage to make the infinite scroll work and all your websites that you need to scrape have that feature then this might be a better solution. – Alexander Sep 30 '22 at 06:49