Question

Я пытался вычеркнуть имена, телефоны и электронные письма агентов из URL сайта. я сделал скребок с первой страницы. но я не могу разбивать страницы на остальные страницы. Потому что все страницы имеют одинаковый URL. Нет номера страницы в URL-адресе в виде строки запроса или около того.

Я использую requests и beautifulsoup Кстати.

Просто помогите мне разбить страницы на все страницы. Спасибо ..

URL-адрес здесь

Код для первой страницы:

import requests
from bs4 import BeautifulSoup as bs

count = 0

response = requests.get('https://propertycentral.co.za/property/all_property?adtype=3&areaname=Johannesburg&minprice=&maxprice=&bedrooms=&bathrooms=')
content = bs(response.text , 'lxml')

detail_page = content.find_all('div',class_='info')

for properte in detail_page:
    count += 1
    href = properte.find('a').get('href')
    property_name = properte.find('a').text.strip()
    properte_link = 'https://propertycentral.co.za{}'.format(href)

    print('\n********* Property#{} *********\n'.format(count))
    print('Property: {}'.format(property_name))
    print('Property Link: {}\n'.format(properte_link))

αԋɱҽԃ αмєяιcαη · Answer 1 · 17 апреля 2020

import requests
from bs4 import BeautifulSoup
from itertools import islice

params = {
    "adtype": "3",
    "areaname": "Johannesburg",
    "minprice": "",
    "maxprice": "",
    "bedrooms": "",
    "bathrooms": ""
}

data = {
    '__EVENTTARGET': '',
    '__EVENTARGUMENT': '',
    "ctl00$MainContent$hdn_areaid": "",
    "ctl00$MainContent$hdn_onpagesrch_flag": "",
    "ctl00$MainContent$ddl_adtype": "3",
    "ctl00$MainContent$ddl_province": "",
    "ctl00$MainContent$ddl_province_CascadingDropDown_ClientState": "::::::",
    "ctl00$MainContent$ddl_Area_CascadingDropDown_ClientState": "::::::",
    "ctl00$MainContent$ddl_Suburb_CascadingDropDown_ClientState": "::::::",
    "ctl00$MainContent$ddl_bedrooms": "",
    "ctl00$MainContent$ddl_bathrooms": "",
    "ctl00$MainContent$ddl_min_price": "",
    "ctl00$MainContent$ddl_max_price": "",
    "hiddenInputToUpdateATBuffer_CommonToolkitScripts": "0"

}


def main(url):
    with requests.Session() as req:
        r = req.get(url, params=params)
        soup = BeautifulSoup(r.content, 'html.parser')

        data['__VIEWSTATE'] = soup.find("input", id="__VIEWSTATE").get("value")
        data['__VIEWSTATEGENERATOR'] = soup.find(
            "input", id="__VIEWSTATEGENERATOR").get("value")

        target = soup.select("div.info")
        print(f"{'*' * 80}Extracting Page# 1{'*' * 80}")
        for tar in target:
            print("Name: {:<50}, Url: {}{}".format(
                tar.a.text, url[:29], tar.a['href']))

        data['ctl00$MainContent$DataPager2$ctl02$ctl00'] = "next+>"

        for page in range(1, 10):
            print(f"{'*' * 80}Extracting Page# {page + 1}{'*' * 80}")
            r = req.post(url, params=params, data=data)
            soup = BeautifulSoup(r.content, 'html.parser')
            target = soup.select("div.info")

            data['__VIEWSTATE'] = soup.find(
                "input", id="__VIEWSTATE").get("value")
            data['__VIEWSTATEGENERATOR'] = soup.find(
                "input", id="__VIEWSTATEGENERATOR").get("value")

            for tar in target:
                print("Name: {:<50}, Url: {}{}".format(
                    tar.a.text, url[:29], tar.a['href']))


main("https://propertycentral.co.za/property/all_property")

Выход:

********************************************************************************Extracting Page# 1********************************************************************************
Name: 3 bedroom house for sale                          , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/3-Bedroom-Property/30347   
Name: 2 bedroom house for sale                          , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/2-Bedroom-Property/30344   
Name: 1 Bedroom Apartment For Sale in Auckland...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Auckland Park/1-Bedroom-Property/30343  
Name: 2 bedroom townhouse for sale                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/2-Bedroom-Property/30338   
Name: 2 bedroom townhouse for sale                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/2-Bedroom-Property/30337   
Name: 2 bedroom townhouse for sale                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/2-Bedroom-Property/30333   
Name: 3 bedroom house for sale                          , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/3-Bedroom-Property/30332   
Name: 2 bedroom townhouse for sale                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/2-Bedroom-Property/30330   
Name: 3 bedroom townhouse for sale                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/3-Bedroom-Property/30329   
Name: 3 bedroom house for sale                          , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/3-Bedroom-Property/30327   
********************************************************************************Extracting Page# 2********************************************************************************
Name: 3 Bedroom House For Sale in Lenasia               , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/30294
Name: 2 Bedroom Freestanding For Sale in Devla...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/2-Bedroom-Property/30256   
Name: Property for sale in Anchorville                  , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/0-Bedroom-Property/29843        
Name: 1486 m² Land available in Fordsburg               , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/0-Bedroom-Property/29842   
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/0-Bedroom-Property/29836        
Name: Property for sale in Anchorville                  , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/0-Bedroom-Property/29835        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/0-Bedroom-Property/29832        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/0-Bedroom-Property/29831        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/0-Bedroom-Property/29828        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/0-Bedroom-Property/29827        
********************************************************************************Extracting Page# 3********************************************************************************
Name: Property for sale in Lenasia. POA                 , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/2-Bedroom-Property/29817
Name: Property for sale in Lenasia. POA                 , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/5-Bedroom-Property/29815        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29811        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/5-Bedroom-Property/29805        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29802        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/2-Bedroom-Property/29800        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/6-Bedroom-Property/29797        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/7-Bedroom-Property/29795        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/2-Bedroom-Property/29788        
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29783        
********************************************************************************Extracting Page# 4********************************************************************************
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29780
Name: Property for sale in Lenasia South                , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/3-Bedroom-Property/29778
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/2-Bedroom-Property/29776      
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29766      
Name: Property for sale in Lenasia South                , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/2-Bedroom-Property/29763
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/0-Bedroom-Property/29762      
Name: Property for sale in Lenasia South                , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/3-Bedroom-Property/29760
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/0-Bedroom-Property/29754
Name: Property for sale in Ennerdale                    , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Ennerdale/3-Bedroom-Property/29745
Name: Property for sale in Lenasia South                , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/3-Bedroom-Property/29738
********************************************************************************Extracting Page# 5********************************************************************************
Name: Property for sale in Lenasia South                , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/3-Bedroom-Property/29736
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/0-Bedroom-Property/29734
Name: Property for sale in Lenasia South                , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/24-Bedroom-Property/29732
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/20-Bedroom-Property/29730
Name: Property for sale in Parkwood                     , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Parkwood/4-Bedroom-Property/29728
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/6-Bedroom-Property/29726
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29724
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/5-Bedroom-Property/29722
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29720
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29718
********************************************************************************Extracting Page# 6********************************************************************************
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/8-Bedroom-Property/29716
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29714
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29706
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29704
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29702
Name: Property for sale in Randburg Central             , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29699
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/6-Bedroom-Property/29692
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/5-Bedroom-Property/29688
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/6-Bedroom-Property/29686
Name: Property for sale in Lenasia South                , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/6-Bedroom-Property/29685
********************************************************************************Extracting Page# 7********************************************************************************
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29671
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/5-Bedroom-Property/29668
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29666
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29665
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29632
Name: Property for sale in Mayfair                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Mayfair West/4-Bedroom-Property/29631
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29630
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29628
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29627
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/4-Bedroom-Property/29626
********************************************************************************Extracting Page# 8********************************************************************************
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29624
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29623
Name: Property for sale in Lenasia South                , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/3-Bedroom-Property/29621
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29620
Name: Property for sale in Lenasia South                , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/3-Bedroom-Property/29617
Name: Property for sale in Lenasia South                , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/5-Bedroom-Property/29614
Name: Property for sale in Lenasia                      , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29611
Name: Situated in Lilliba Section                       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/2-Bedroom-Property/29483
Name: 3 Bedroom Freestanding For Sale in Lenas...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia/3-Bedroom-Property/29376
Name: 3 Bedroom Freehold For Sale in Lenasia S...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/3-Bedroom-Property/29364
********************************************************************************Extracting Page# 9********************************************************************************
Name: 2 Bedroom Apartment / Flat for sale in N...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Northcliff/2-Bedroom-Property/29354
Name: 2 Bedroom House For Sale in Ennerdale             , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Ennerdale/2-Bedroom-Property/29339
Name: 2 Bedroom House For Sale in Eldorado Par...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Eldorado Park/2-Bedroom-Property/29328
Name: 3 Bedroom House For Sale in Lenasia Sout...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Lenasia South/3-Bedroom-Property/29325
Name: 2 Bedroom Apartment / Flat for sale in J...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/JHB CBD/Johannesburg Central and CBD/2-Bedroom-Property/29093    
Name: 2 Bedroom Apartment / Flat for sale in J...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/JHB CBD/Johannesburg Central and CBD/2-Bedroom-Property/29005    
Name: 6 Bedroom House for sale in Ennerdale             , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Ennerdale/6-Bedroom-Property/28949
Name: Commercial Property for sale in Bertrams          , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Bertrams/0-Bedroom-Property/28650
Name: 1 Bedroom Apartment / Flat for sale in B...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Braamfontein Werf/1-Bedroom-Property/28622
Name: Apartment / Flat for sale in Joubert Par...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/0-Bedroom-Property/28397
********************************************************************************Extracting Page# 10********************************************************************************
Name: R5 750 000 | 2 Bedroom Apartment / Flat ...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Melrose/2-Bedroom-Property/26929
Name: Apartment / Flat For Sale in Johannesbur...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/0-Bedroom-Property/26788
Name: Apartment / Flat For Sale in Johannesbur...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/0-Bedroom-Property/26771
Name: Apartment / Flat For Sale in Johannesbur...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/0-Bedroom-Property/26762
Name: Apartment / Flat For Sale in Johannesbur...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/0-Bedroom-Property/26746
Name: Apartment / Flat For Sale in Maboneng, J...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/0-Bedroom-Property/26737
Name: Apartment / Flat For Sale in Johannesbur...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/1-Bedroom-Property/26736
Name: Apartment / Flat For Sale in Johannesbur...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/2-Bedroom-Property/26725
Name: Apartment / Flat For Sale in Johannesbur...       , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Johannesburg/0-Bedroom-Property/26712
Name: 3 Bedroom House For Sale in Parkhurst             , Url: https://propertycentral.co.za/property/House-For-Sale/Gauteng/Johannesburg/Parkhurst/3-Bedroom-Property/26578

Как разбить на страницы все страницы сайта по URL?

Пожалуйста, войдите или зарегистрируйтесь чтобы ответить на этот вопрос.

1 Ответ

Пожалуйста, войдите или зарегистрируйтесь что бы добавить комментарий.

Как разбить на страницы все страницы сайта по URL?

Пожалуйста, войдите или зарегистрируйтесь чтобы ответить на этот вопрос.

1 Ответ

Пожалуйста, войдите или зарегистрируйтесь что бы добавить комментарий.

Похожие темы