Scrapy - извлечение данных со следующей страницы

Question

Мне нужна ваша помощь, люди, чтобы пролистать и извлечь элементы.Вот мой паук

    import json
import scrapy


class YPSpider(scrapy.Spider):
    name = 'yp'
    start_urls = ['https://www.infobel.com/fr/france/business/50000/informatique_internet/']

    def parse(self, response):
next_page_url = response.xpath('//ul[@class="pagination"]/li[@class="active"]/following-sibling::li[1]/a/@href').extract_first()
if next_page_url:
    yield response.follow(next_page_url, callback=self.parse)
        if response.meta.get('has_phone'):
            item = response.meta['item']

            response = json.loads(response.body)
            item['phone'] = response['result']

            yield item
        else:
            items = response.xpath('//*[contains(@class, "customer-box")]')

            for item in items:
                address_lines = item.xpath('.//span[contains(@class, "fa-map-marker")]/../span[@class="detail-text"]//text()').extract()

                title = item.xpath('.//h2[@class="customer-item-name"]/a/text()').extract_first().strip()
                address = address_lines[0].replace('\r', '').replace('\t', '').strip() if address_lines else ''
                village = address_lines[1].replace('\r', '').replace('\t', '').strip() if len(address_lines) >= 1 else ''
                phone = item.xpath('.//span[contains(@class, "icon-phone")]/../span[@class="detail-text"]/text()').extract()

                item = {
                    'title': title,
                    'address': address,
                    'village': village,
                    'phone': phone,
                }

                if phone:
                    if phone[0].isnumeric():
                        item['phone'] = phone[0]

                        yield item
                    elif len(phone) >= 2:
                        yield scrapy.Request('https://www.infobel.com/fr/france/Search/Decrypt?encryptedString={}'.format(phone[1]), meta={'item': item, 'has_phone': True}

                        )

Как я могу заставить это сканирование сканировать следующую страницу и очищать элементы с этой страницы?Я ценю вашу помощь заранее спасибо

Answer 1

Вам необходимо добавить этот код к вашему parse методу:

next_page_url = response.xpath('//ul[@class="pagination"]/li[@class="active"]/following-sibling::li[1]/a/@href').extract_first()
if next_page_url:
    yield response.follow(next_page_url, callback=self.parse)

Вам нужно немного прочитать о синтаксисе Python, прежде чем задавать вопросы здесь :

def parse(self, response):
    next_page_url = response.xpath('//ul[@class="pagination"]/li[@class="active"]/following-sibling::li[1]/a/@href').extract_first()
    if next_page_url:
        yield response.follow(next_page_url, callback=self.parse)

    if response.meta.get('has_phone'):
        item = response.meta['item']

        response = json.loads(response.body)
        item['phone'] = response['result']

        yield item
    else:
        items = response.xpath('//*[contains(@class, "customer-box")]')

        for item in items:
            address_lines = item.xpath('.//span[contains(@class, "fa-map-marker")]/../span[@class="detail-text"]//text()').extract()

            title = item.xpath('.//h2[@class="customer-item-name"]/a/text()').extract_first().strip()
            address = address_lines[0].replace('\r', '').replace('\t', '').strip() if address_lines else ''
            village = address_lines[1].replace('\r', '').replace('\t', '').strip() if len(address_lines) >= 1 else ''
            phone = item.xpath('.//span[contains(@class, "icon-phone")]/../span[@class="detail-text"]/text()').extract()

            item = {
                'title': title,
                'address': address,
                'village': village,
                'phone': phone,
            }

            if phone:
                if phone[0].isnumeric():
                    item['phone'] = phone[0]

                    yield item
                elif len(phone) >= 2:
                    yield scrapy.Request('https://www.infobel.com/fr/france/Search/Decrypt?encryptedString={}'.format(phone[1]), meta={'item': item, 'has_phone': True}

                    )