Соскоб в сети с Python для сбора информации о свойствах с помощью BeautifulSoup

Question

Я впервые занимаюсь поиском в интернете. То, что я хотел бы сделать, это получить всю возможную информацию о свойствах (местоположение, цена и т. Д.) Итак, что я имею сейчас, что мне удалось собрать с помощью некоторого поиска в Google, примерно так:

import requests
from bs4 import BeautifulSoup
from pprint import pprint as pp

rootURL = 'https://www.jawitz.co.za/'
response = requests.get(rootURL)                                                   
html = response.content
soup = BeautifulSoup(html,'lxml')

dropdown_list = soup.select(".primary .child-pages a")
cityLinks = [rootURL + dropdown_list_value['href'] for dropdown_list_value in dropdown_list]

# params for our request
params = {"province": "Western Cape", 
          "suburb": "Van Riebeeck Park", 
          "region": "Worcester", 
          "id": 929,
          "property_type": "Apartment"}

for city in cityLinks:  # Looping each city from the Apartment drop down list
    with requests.Session() as s:
        r= s.get(city)
        soup = BeautifulSoup(r.content)
        # getting some information should go here

Теперь я не совсем уверен, как получить информацию, поскольку на этом сайте, после изучения ресурсов, я наткнулся на некоторые скрытые данные. Любые советы о том, как продолжить это? Я немного запутался в том, как получить доступ к информации, доступной на ресурсе.

Заранее спасибо!

Answer 1

Вы можете комбинировать селен с bs4

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import math
from bs4 import BeautifulSoup as bs
import pandas as pd

def getData(properties):
    for aProperty in properties:
        soup = bs(str(aProperty) , 'lxml')
        price = soup.select_one('.property-price-heading').text if soup.select_one('.property-price-heading') is not None else ''       
        header = soup.select_one('.property-marketing-heading').text if soup.select_one('.property-marketing-heading') is not None else ''        
        beds = soup.select_one('.icon-beds span').text if soup.select_one('.icon-beds span') is not None else ''        
        baths = soup.select_one('.icon-baths span').text if soup.select_one('.icon-baths span') is not None else ''    
        garages = soup.select_one('.icon-garages span').text if soup.select_one('.icon-garages span') is not None else ''
        land = soup.select_one('.property-list-land-size .value').text if soup.select_one('.property-list-land-size .value') is not None else ''
        description = soup.select_one('.results-content').text.strip() if soup.select_one('.results-content') is not None else ''
        ref = soup.select_one('.property-list-webref').text.strip() if soup.select_one('.property-list-webref') is not None else ''
        items = [price, header, beds, baths, garages, land, description, ref ]
        result.append(items)
    return result

varUrl = 'https://www.jawitz.co.za/results/residential/for-sale/cape-town/all/?p={}&advanced_search=1&s=-price'
url = 'https://www.jawitz.co.za/results/residential/for-sale/cape-town/all/?advanced_search=1'

driver = webdriver.Chrome()
driver.get(url)
data =  WebDriverWait(driver,10).until(EC.presence_of_element_located((By.CSS_SELECTOR, ".property-list-details")))
numResults = int(driver.find_element_by_id('id_property_count').text.split(' ')[0])

soup = bs(driver.page_source, 'lxml')
properties = soup.select('.property-list-details')

resultsPerPage = 10
numPages = math.ceil(numResults/resultsPerPage)

results = []

results.append(getData(properties))

if numPages > 1:
    for page in range(2, numPages + 1):
        url = varUrl.format(page)
        driver.get(url)
        data =  WebDriverWait(driver,10).until(EC.presence_of_element_located((By.CSS_SELECTOR, ".property-list-details")))
        soup = bs(driver.page_source, 'lxml')
        properties = soup.select('.property-list-details')
        results.append(getData(properties))
        if page > 3:  #delete after testing
            break  #delete after testing

finalList = [item for sublist in results for item in sublist]
print(finalList)