Beautifulsoup - Сбор ссылки href и создание списка ссылок

Question

я пытаюсь собрать все ссылки в списке оружия (в данном случае 2 страницы) и распечатать 1) длину и 2) сами ссылки.

im получаю ошибку: у объекта списка нет атрибутавыберите

from bs4 import BeautifulSoup
import requests
import csv
import pandas
from pandas import DataFrame
import re
import os
import locale
os.environ["PYTHONIOENCODING"] = "utf-8"

page = 1
all_links = []
url="https://www.gunstar.co.uk/view-trader/global-rifle-snipersystems/58782?page={}"

with requests.Session() as session:
  while True:
    print(url.format(page))
    res=session.get(url.format(page))
    soup=BeautifulSoup(res.content,'html.parser')
    gun_details = soup.select('div.details')
    for link in gun_details.select('a'):
     all_links.append("https://www.gunstar.co.uk" + link['href'])
    if len(soup.select(".nav_next"))==0:
        break
    page += 1

Если я удаляю .content из ответа, я получаю ответ не имеет len.

, если я добавляю .text в soup.select ('div.details') iполучить результат, аналогичный приведенному выше.

Я уверен, что он идет не так, как надо, где-то довольно просто, просто не могу это увидеть - есть ли причина, почему select и findAll не работают при попытке попасть в определенную часть html?

Answer 1

Вы можете получить ссылки со всех страниц по-разному.Вот один из таких способов достижения того же с помощью генераторов:

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin

link = "https://www.gunstar.co.uk/view-trader/global-rifle-snipersystems/58782"
base = "https://www.gunstar.co.uk"

def get_links(url):
    res = requests.get(url)
    soup = BeautifulSoup(res.text,'lxml')
    for item in soup.select(".details > a"):
        yield urljoin(base,item['href'])

    next_page = soup.select_one(".gallery_navigation [rel='next']")
    if next_page:
        yield from get_links(next_page['href'])

if __name__ == '__main__':
    list_of_links = [elem for elem in get_links(link)]
    print(list_of_links)

Answer 2

Попробуйте следующий код.

from bs4 import BeautifulSoup
import requests
import csv
import pandas
from pandas import DataFrame
import re
import os
import locale
os.environ["PYTHONIOENCODING"] = "utf-8"

page = 1

url="https://www.gunstar.co.uk/view-trader/global-rifle-snipersystems/58782?page={}"

with requests.Session() as session:
  while True:
    all_links=[]
    print(url.format(page))
    res=session.get(url.format(page))
    soup=BeautifulSoup(res.content,'html.parser')
    gun_details = soup.select('div.details')
    for link in gun_details:
     all_links.append("https://www.gunstar.co.uk" + link.select_one('a')['href'])
    print(all_links)
    if len(soup.select(".nav_next"))==0:
        break
    page += 1

Вывод:

https://www.gunstar.co.uk/view-trader/global-rifle-snipersystems/58782?page=1
['https://www.gunstar.co.uk/mauser-m96-lightning-hunter-straight-pull-270-rifles/rifles/1083802', 'https://www.gunstar.co.uk/magtech-586-12-bore-gauge-pump-action/Shotguns/1083784', 'https://www.gunstar.co.uk/merkel-kr1-bolt-action-308-rifles/rifles/1083786', 'https://www.gunstar.co.uk/christensen-arms-r93-carbon-bolt-action-7-mm-rifles/rifles/1083788', 'https://www.gunstar.co.uk/voere-lbw-luxus-bolt-action-308-rifles/rifles/1083792', 'https://www.gunstar.co.uk/voere-2155-bolt-action-243-rifles/rifles/1083797', 'https://www.gunstar.co.uk/voere-2155-2155-synthetic-bolt-action-308-rifles/rifles/1083798', 'https://www.gunstar.co.uk/mauser-m96-lightning-hunter-straight-pull-7-mm-rifles/rifles/1083799', 'https://www.gunstar.co.uk/blaser-lrs2-straight-pull-308-rifles/rifles/1084397', 'https://www.gunstar.co.uk/remington-700-s-s-barrel-only-bolt-action-300-win-mag-rifles/rifles/1084432']
https://www.gunstar.co.uk/view-trader/global-rifle-snipersystems/58782?page=2
['https://www.gunstar.co.uk/pfeiffer-waffen-handy-hunter-sr2-single-shot-300-win-mag-rif/rifles/1084433', 'https://www.gunstar.co.uk/sabatti-10-22-mod-sporter-semi-auto-22-rifles/rifles/1084442', 'https://www.gunstar.co.uk/voere-lbw-m-sniper-rifle-bolt-action-308-rifles/rifles/1084454', 'https://www.gunstar.co.uk/snipersystems-zoom-gun-light-kit-lamping/Accessories/1130763']

---

Другой способ получить все ссылки.

from bs4 import BeautifulSoup
import requests
import csv
import pandas
from pandas import DataFrame
import re
import os
import locale
os.environ["PYTHONIOENCODING"] = "utf-8"

page = 1
all_links = []
url="https://www.gunstar.co.uk/view-trader/global-rifle-snipersystems/58782?page={}"

with requests.Session() as session:
  while True:

    print(url.format(page))
    res=session.get(url.format(page))
    soup=BeautifulSoup(res.content,'html.parser')
    gun_details = soup.select('div.details > a')
    for link in gun_details:
     all_links.append("https://www.gunstar.co.uk" + link['href'])

    if len(soup.select(".nav_next"))==0:
        break
    page += 1

print(all_links)