Действия по разбиению на страницы сайта являются динамическими, поэтому вам необходимо использовать selenium:
from selenium import webdriver
from bs4 import BeautifulSoup as soup
import re
def get_content(_d):
_headers = [re.sub('\n', '', i.text) for i in _d.find('table').find('tr').find_all('th')]
_, *data = list(filter(None, [[re.sub('\n', '', i.text) for i in b.find_all('td')] for b in _d.find_all('tr')]))
return [dict(zip(_headers, i)) for i in data]
d = webdriver.Chrome('/Users/jamespetullo/Downloads/chromedriver')
d.get('https://etfdb.com/screener/')
all_data = [get_content(soup(d.page_source, 'html.parser'))]
_next = [i for i in d.find_elements_by_tag_name('a') if re.findall('^next', i.text, re.I)]
while _next:
_next[0].send_keys('\n')
all_data.append(get_content(soup(d.page_source, 'html.parser')))
_next = [i for i in d.find_elements_by_tag_name('a') if re.findall('^next', i.text, re.I)]
Пример вывода (только на первой странице, из-за ограничения числа символов ответа SO):
[{'Symbol': 'IVV', 'ETF Name': 'iShares Core S&P 500 ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$159,613.22', 'YTD': '11.41%', 'Avg. Volume': '6,062,749', 'Previous Closing Price': '$280.32', 'Overall Rating': ''}, {'Symbol': 'VTI', 'ETF Name': 'Vanguard Total Stock Market ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$107,909.21', 'YTD': '12.40%', 'Avg. Volume': '4,901,940', 'Previous Closing Price': '$143.46', 'Overall Rating': ''}, {'Symbol': 'VOO', 'ETF Name': 'Vanguard S&P 500 ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$102,814.11', 'YTD': '11.43%', 'Avg. Volume': '4,474,837', 'Previous Closing Price': '$256.07', 'Overall Rating': ''}, {'Symbol': 'VEA', 'ETF Name': 'Vanguard FTSE Developed Markets ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$70,063.68', 'YTD': '9.97%', 'Avg. Volume': '22,225,420', 'Previous Closing Price': '$40.80', 'Overall Rating': ''}, {'Symbol': 'QQQ', 'ETF Name': 'Invesco QQQ', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$68,001.87', 'YTD': '12.27%', 'Avg. Volume': '48,660,348', 'Previous Closing Price': '$173.19', 'Overall Rating': ''}, {'Symbol': 'EFA', 'ETF Name': 'iShares MSCI EAFE ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$65,100.01', 'YTD': '9.34%', 'Avg. Volume': '39,227,020', 'Previous Closing Price': '$64.27', 'Overall Rating': ''}, {'Symbol': 'VWO', 'ETF Name': 'Vanguard FTSE Emerging Markets ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$62,464.73', 'YTD': '9.24%', 'Avg. Volume': '21,504,412', 'Previous Closing Price': '$41.62', 'Overall Rating': ''}, {'Symbol': 'IEMG', 'ETF Name': 'iShares Core MSCI Emerging Markets ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$58,650.02', 'YTD': '8.42%', 'Avg. Volume': '23,205,799', 'Previous Closing Price': '$51.12', 'Overall Rating': ''}, {'Symbol': 'AGG', 'ETF Name': 'iShares Core U.S. Aggregate Bond ETF', 'Asset ClassNew': 'Bond', 'Total Assets ($MM)': '$58,023.69', 'YTD': '0.79%', 'Avg. Volume': '6,496,300', 'Previous Closing Price': '$107.07', 'Overall Rating': ''}, {'Symbol': 'IEFA', 'ETF Name': 'iShares Core MSCI EAFE ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$57,206.30', 'YTD': '9.62%', 'Avg. Volume': '15,197,579', 'Previous Closing Price': '$60.29', 'Overall Rating': ''}, {'Symbol': 'IJH', 'ETF Name': 'iShares Core S&P Mid-Cap ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$48,274.19', 'YTD': '15.12%', 'Avg. Volume': '2,089,207', 'Previous Closing Price': '$191.16', 'Overall Rating': ''}, {'Symbol': 'VTV', 'ETF Name': 'Vanguard Value ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$46,173.50', 'YTD': '10.02%', 'Avg. Volume': '2,821,418', 'Previous Closing Price': '$107.76', 'Overall Rating': ''}, {'Symbol': 'IJR', 'ETF Name': 'iShares Core S&P Small-Cap ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$44,864.19', 'YTD': '15.44%', 'Avg. Volume': '5,460,632', 'Previous Closing Price': '$80.02', 'Overall Rating': ''}, {'Symbol': 'IWM', 'ETF Name': 'iShares Russell 2000 ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$43,733.85', 'YTD': '17.09%', 'Avg. Volume': '26,073,227', 'Previous Closing Price': '$156.78', 'Overall Rating': ''}, {'Symbol': 'IWF', 'ETF Name': 'iShares Russell 1000 Growth ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$42,024.32', 'YTD': '12.77%', 'Avg. Volume': '2,928,795', 'Previous Closing Price': '$147.63', 'Overall Rating': ''}, {'Symbol': 'IWD', 'ETF Name': 'iShares Russell 1000 Value ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$38,059.34', 'YTD': '11.08%', 'Avg. Volume': '4,668,100', 'Previous Closing Price': '$123.35', 'Overall Rating': ''}, {'Symbol': 'BND', 'ETF Name': 'Vanguard Total Bond Market ETF', 'Asset ClassNew': 'Bond', 'Total Assets ($MM)': '$37,358.63', 'YTD': '1.02%', 'Avg. Volume': '2,981,882', 'Previous Closing Price': '$79.82', 'Overall Rating': ''}, {'Symbol': 'VUG', 'ETF Name': 'Vanguard Growth ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$36,989.43', 'YTD': '13.23%', 'Avg. Volume': '1,319,368', 'Previous Closing Price': '$152.10', 'Overall Rating': ''}, {'Symbol': 'EEM', 'ETF Name': 'iShares MSCI Emerging Markets ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$34,442.08', 'YTD': '8.65%', 'Avg. Volume': '89,719,367', 'Previous Closing Price': '$42.44', 'Overall Rating': ''}, {'Symbol': 'GLD', 'ETF Name': 'SPDR Gold Trust', 'Asset ClassNew': 'Commodity', 'Total Assets ($MM)': '$33,249.82', 'YTD': '2.26%', 'Avg. Volume': '8,782,053', 'Previous Closing Price': '$123.99', 'Overall Rating': ''}, {'Symbol': 'VNQ', 'ETF Name': 'Vanguard Real Estate Index Fund', 'Asset ClassNew': 'Real Estate', 'Total Assets ($MM)': '$32,604.41', 'YTD': '12.63%', 'Avg. Volume': '8,538,158', 'Previous Closing Price': '$83.99', 'Overall Rating': ''}, {'Symbol': 'VIG', 'ETF Name': 'Vanguard Dividend Appreciation ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$32,584.13', 'YTD': '11.19%', 'Avg. Volume': '1,610,603', 'Previous Closing Price': '$108.91', 'Overall Rating': ''}, {'Symbol': 'LQD', 'ETF Name': 'iShares iBoxx $ Investment Grade Corporate Bond ETF', 'Asset ClassNew': 'Bond', 'Total Assets ($MM)': '$32,520.19', 'YTD': '3.16%', 'Avg. Volume': '9,748,131', 'Previous Closing Price': '$116.03', 'Overall Rating': ''}, {'Symbol': 'VB', 'ETF Name': 'Vanguard Small Cap ETF', 'Asset ClassNew': 'Equity', 'Total Assets ($MM)': '$24,678.09', 'YTD': '17.14%', 'Avg. Volume': '1,025,374', 'Previous Closing Price': '$154.61', 'Overall Rating': ''}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}, {'Symbol': 'Export to CSV with ETFdb.com Pro'}]
В этом решении используется цикл while для непрерывного нажатия кнопки «Далее» для перехода по страницам.Вместо того чтобы полагаться на первые пять результатов, он может обрабатывать столько страниц, сколько предоставляет сайт.