Вы можете имитировать сообщение xhr, созданное на странице
from bs4 import BeautifulSoup
import requests
import pandas as pd
url = 'https://alando-palais.de/wp/wp-admin/admin-ajax.php'
data = {
'action': 'vc_get_vc_grid_data',
'vc_action': 'vc_get_vc_grid_data',
'tag': 'vc_basic_grid',
'data[visible_pages]' : 5,
'data[page_id]' : 30,
'data[style]' : 'all',
'data[action]' : 'vc_get_vc_grid_data',
'data[shortcode_id]' : '1551112413477-5fbaaae1-0622-2',
'data[tag]' : 'vc_basic_grid',
'vc_post_id' : '30',
'_vcnonce' : 'cc8cc954a4'
}
res = requests.post(url, data = data)
soup = BeautifulSoup(res.content, 'lxml')
dates = [item.text.strip() for item in soup.select('.vc_gitem-zone[style*="https://alando-palais.de"]')]
textInfo = [item for item in soup.select('.vc_gitem-link')][::2]
imageLinks = [item['src'].strip() for item in soup.select('img')]
titles = []
links = []
for item in textInfo:
titles.append(item['title'])
links.append(item['href'])
results = pd.DataFrame(list(zip(titles, dates, links, imageLinks)),columns = ['title', 'date', 'link', 'imageLink'])
print(results)
Или с селеном:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
url = 'https://alando-palais.de/events#'
driver = webdriver.Chrome()
driver.get(url)
dates = [item.text.strip() for item in WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".vc_gitem-zone[style*='https://alando-palais.de']"))) if len(item.text)]
textInfo = [item for item in driver.find_elements_by_css_selector('.vc_gitem-link')][::2]
textInfo = textInfo[: int(len(textInfo) / 2)]
imageLinks = [item.get_attribute('src').strip() for item in driver.find_elements_by_css_selector('a + img')][::2]
titles = []
links = []
for item in textInfo:
titles.append(item.get_attribute('title'))
links.append(item.get_attribute('href'))
results = pd.DataFrame(list(zip(titles, dates, links, imageLinks)),columns = ['title', 'date', 'link', 'imageLink'])
print(results)
driver.quit()