Выберите страницу с ключевым словом и очистите ее

Question

Я пытаюсь нацарапать название элемента на иностранной версии сайта. После запуска сценария Python Cli запускается, но ничего не возвращает.

В i Python, чтобы получить заголовок, title = soup.find('a', {'class': 'vip'}).text прекрасно работает сам по себе, но в Pycharm это не так. в полном коде, хотя я пошел в мои настройки, чтобы загрузить пакет BeautifulSoup для моего текущего переводчика.

Есть идеи почему? Спасибо.

#!/usr/bin/python3

import csv
import time
import requests
from bs4 import BeautifulSoup

product_category = input("Enter your product category: ")


def get_page(url):
    response = requests.get(url)
    if not response.ok:
        print('Server responded:', response.status_code)
    else:
        soup = BeautifulSoup(response.text, 'html.parser')
    return soup


def get_detail_data(soup):
    # title
    try:
        title = soup.find('a', {'class': 'vip'}).text

    except:
        title = ''

    # price
    try:
        price = soup.find_all('span', {'itemprop': 'price'})
        for p in price:
            price = p.get('content')
    except:
        price = ''

    # currency
    try:
        currency = soup.find_all('span', {'class': 'bold'}).text

    except:
        currency = ''

    # items sold
    try:
        i_s = soup.find('div', {'class': 'hotness-signal red'}).text
        items_sold = i_s.strip().split(' ')[0]
    except:
        items_sold = ''

    data = {
        'title': title,
        'price': price,
        'currency': currency,
        'total sold': items_sold
    }

    return data


def get_index_data(soup):
    try:
        links = soup.find_all('a', class_='s-item__link')
    except:
        links = []

    urls = [item.get('href') for item in links]

    return urls


def write_csv(data, url):
    with open('output.csv', 'a') as csvfile:
        writer = csv.writer(csvfile)
        row = [data['title'], data['price'], data['currency'], data['total sold'], url]
        writer.writerow(['Title', 'Price', 'Currency', 'Sales Volume', 'URL'])
        writer.writerow(row)


def main():
    # Store URL formats for each search engine with placeholders
    url = f"https://www.ebay.fr/sch/i.html?_nkw={product_category}&_pgn=1"
    print(url)
    products = get_index_data(get_page(url))

    for link in products:
        time.sleep(7)
        data = get_detail_data(get_page(link))
        print(data)
        write_csv(data, link)


if __name__ == '__main__':
     main()

Answer 1

Кажется, что сайт .fr использует другую разметку, поэтому вам необходимо соответствующим образом изменить имена / атрибуты классов.

Например:

import re
import csv
import time
import requests
from bs4 import BeautifulSoup

product_category = input("Enter your product category: ")


def get_page(url):
    response = requests.get(url)
    if not response.ok:
        print('Server responded:', response.status_code)
    else:
        soup = BeautifulSoup(response.text, 'html.parser')
    return soup


def get_detail_data(soup):
    # title
    try:
        title = soup.select_one('h1[itemprop="name"]')
        for span in title.select('span'):
            span.extract()
        title = title.get_text(strip=True)
    except:
        title = ''

    # price
    try:
        price = soup.find_all('span', {'itemprop': 'price'})
        for p in price:
            price = p.get('content')
    except:
        price = ''

    # currency
    try:
        currency = soup.select_one('span[itemprop="priceCurrency"][content]')["content"]
    except:
        currency = ''

    # items sold
    try:
        items_sold = re.findall(r'\d+', soup.select_one('.soldwithfeedback').text)[0]
    except:
        items_sold = ''

    data = {
        'title': title,
        'price': price,
        'currency': currency,
        'total sold': items_sold
    }

    return data


def get_index_data(soup):
    links = soup.select('.sresult h3 a')
    urls = [item.get('href') for item in links]
    return urls


def write_csv(data, url):
    with open('output.csv', 'a') as csvfile:
        writer = csv.writer(csvfile)
        row = [data['title'], data['price'], data['currency'], data['total sold'], url]
        writer.writerow(['Title', 'Price', 'Currency', 'Sales Volume', 'URL'])
        writer.writerow(row)


def main():
    # Store URL formats for each search engine with placeholders
    url = f"https://www.ebay.fr/sch/i.html?_nkw={product_category}&_pgn=1"
    print(url)
    products = get_index_data(get_page(url))

    for link in products:
        time.sleep(0.5)
        data = get_detail_data(get_page(link))
        print(data)
        # write_csv(data, link)  # <-- I commented it, to just print to screen


if __name__ == '__main__':
     main()

Отпечатки:

Enter your product category: ddr4
https://www.ebay.fr/sch/i.html?_nkw=ddr4&_pgn=1
{'title': '16 Go 8 Go 4 Go DDR3 DDR4 1333 1600 1866 2133 RAM 2400 2666 MHz pour HyperX FURY Lot', 'price': '19.74', 'currency': 'USD', 'total sold': '1'}
{'title': '4 Go 8 Go 16 Go DDR4 2133 2400 2666 Mhz pour HyperX FURY DIMM Desktop Mémoire RAM Lot', 'price': '23.87', 'currency': 'USD', 'total sold': '93'}
{'title': '8 Go DDR4 2133 MHz pour HyperX FURY CL15 288 Pin DIMM PC4-17000 Desktop RAM RL1US', 'price': '39.96', 'currency': 'USD', 'total sold': '17'}
{'title': '16 Go G. Skill DDR4 Trident 3200 MHz Z PC4-25600 CL16 1.35 V Double Kit (2x8GB)', 'price': '70.0', 'currency': 'GBP', 'total sold': ''}
{'title': 'DDR4 4 Go 8 Go 16 Go Desktop 2666 MHz Desktop DIMM Mémoire RAM pour Kingston HyperX Fury R1US', 'price': '24.13', 'currency': 'USD', 'total sold': '19'}
{'title': 'Micron 8GB RAM DDR4 1Rx8 PC4-2400T-UAB-10', 'price': '23.0', 'currency': 'EUR', 'total sold': ''}
{'title': 'PATRIOT Viper Blackout 16 Go DDR4 3000 (2x8)', 'price': '54.99', 'currency': 'GBP', 'total sold': ''}
{'title': 'Samsung 8GB RAM DDR4 1Rx8 PC4-2133P SO-DIMM', 'price': '21.0', 'currency': 'EUR', 'total sold': ''}
{'title': 'Kingston 8 Go DDR4 2133 MHz Desktop PC RAM ~~ PC4 17000 Mémoire 2133P 288 broches 2Rx8', 'price': '31.99', 'currency': 'GBP', 'total sold': ''}

...and so on.