Хорошее начало. теперь вам нужно перебрать указанные теги c с этим содержимым в первых извлекаемых тегах <div>, а затем создать фрейм данных.
Код:
import requests as rq
from bs4 import BeautifulSoup
import pandas as pd
url = "https://www.rfpmart.in/"
response = rq.get(url)
soup = BeautifulSoup(response.text,'html.parser')
endPage = int(soup.find('p',{'class':'counter'}).text.split()[-1])
desc = []
post = []
exp = []
urls = []
for page in range(1,endPage+1):
print ('Aquiring page: %s'%page)
url = "https://www.rfpmart.in/home-%s.html" %page
response = rq.get(url)
if response.status_code == 200:
soup = BeautifulSoup(response.text,'html.parser')
div = soup.find('div',{'id':'home'})
items = div.find_all('li')
for each in items:
desc.append(each.find('div', {'class':'description-category'}).text.strip())
post.append(each.find('span',{'class':'post-date'}).text.strip())
exp.append(each.find('span',{'class':'expiry-date'}).text.strip())
urls.append(each.find('a')['href'])
df = pd.DataFrame(list(zip(desc, post, exp, urls)),
columns =['Description','Posted Date','Expiry Date', 'URL'])
Вывод:
print (df)
Description ... Expiry Date
0 EXTRA-8365-USA (Oakland, California) - Perinat... ... Wednesday, 15 July, 2020
1 EXTRA-8364-USA (Santa Cruz, California) - Prof... ... Thursday, 25 June, 2020
2 SW-33969-USA (California) - Java Software Deve... ... Wednesday, 10 June, 2020
3 SW-33968-USA (California) - Electronic Agenda ... ... Monday, 15 June, 2020
4 WRT-1475-USA (Beebe, Arkansas) - Title III Gra... ... Monday, 15 June, 2020
5 DE-3174-USA (California) - Paper to Digital Do... ... Thursday, 9 July, 2020
6 TRANSLATION-3588-USA (California) - Translatio... ... Tuesday, 9 June, 2020
7 ACCT-9172-USA (San Angelo, Texas) - Audit Serv... ... Friday, 12 June, 2020
8 ACCT-9173-USA (Texas) - Audit Services for Tax... ... Monday, 1 June, 2020
9 ESTATE-3249-USA (San Antonio, Texas) - Real Es... ... Thursday, 25 June, 2020
10 SW-33980-USA (New Jersey) - Document Managemen... ... Thursday, 28 May, 2020
11 FOOD-5243-USA (Garden City, New York) - Food S... ... Thursday, 11 June, 2020
12 EXTRA-8369-USA (Camden, New Jersey) - Port Pla... ... Friday, 19 June, 2020
13 TRANSPORT-3411- USA (Aztec, New Mexico) - Inma... ... Thursday, 4 June, 2020
14 CSE-7613-USA (Lincoln, Nebraska) - Event Secur... ... Thursday, 11 June, 2020
15 TRANSPORT-3410-USA (Bedford, Texas) - Third Pa... ... Tuesday, 16 June, 2020
16 ITES-2658-USA (Salem, New Jersey) - Informatio... ... Friday, 12 June, 2020
17 MRB-16554-USA (North Carolina) - Paid Media Su... ... Friday, 19 June, 2020
18 MB-4214-USA (New York) - Claims Auditor Servic... ... Tuesday, 9 June, 2020
19 TRANS-1529-USA (New York) - Court Reporting an... ... Thursday, 18 June, 2020
20 CSE-7612-USA (Frederick, Maryland) - Security ... ... Thursday, 18 June, 2020
21 CSE-7611-USA (Fall River, Massachusetts) - Vid... ... Thursday, 11 June, 2020
22 SW-33960-USA (North Carolina) - Communications... ... Friday, 26 June, 2020
23 SW-33958-USA (Colorado) - Cloud Based Human Re... ... Tuesday, 30 June, 2020
24 SURG-1921-USA (Louisiana) - N95 Mask Supplies-... ... Wednesday, 27 May, 2020
25 MRB-16555-USA (Kentucky) - Legislative Video P... ... Tuesday, 2 June, 2020
26 SW-33959-USA (North Carolina) - Transit Mobile... ... Friday, 12 June, 2020
27 SW-33975-USA (Massachusetts) - Comprehensive S... ... Friday, 12 June, 2020
28 MB-4215-USA (Massachusetts) - Ambulance Billin... ... Thursday, 11 June, 2020
29 NET-3016-USA (Leander, Texas) - Signal Communi... ... Thursday, 4 June, 2020
.. ... ... ...
70 ACCT-9168-Canada (Alberta) - Occupational Heal... ... Wednesday, 10 June, 2020
71 ACCT-9167-USA (Atlanta, Georgia) - Audit Servi... ... Thursday, 25 June, 2020
72 STAFF-4366-USA (Houston, Texas) - RFI for Mobi... ... Thursday, 28 May, 2020
73 MEDI-1919-USA (Macon, Georgia) - Medical Clini... ... Thursday, 25 June, 2020
74 CSE-7607-USA (Washington, DC) - RFI for Securi... ... Saturday, 6 June, 2020
75 STAFF-4367-USA (Calhoun, Georgia) - Nursing Se... ... Monday, 22 June, 2020
76 CR-0991-USA (Saint Paul, Minnesota) - Court Re... ... Tuesday, 2 June, 2020
77 SW-33956-USA (Washington) - Asset Tracking Sof... ... Friday, 29 May, 2020
78 SW-33957-USA (Georgia) - Automated Weather Obs... ... Friday, 26 June, 2020
79 SW-33955-USA (Texas) - Moodle Programming Serv... ... Thursday, 4 June, 2020
80 MRB-16551-USA (Texas) - Comprehensive Integrat... ... Thursday, 28 May, 2020
81 ITES-2656-USA (Fort Wayne, Indiana) - RFI for ... ... Monday, 8 June, 2020
82 CSE-7608-USA (Hawaii) - Library Rules Enforcem... ... Friday, 5 June, 2020
83 DRA-1532-USA (Hawaii) - Arrest Charge Disposit... ... Wednesday, 10 June, 2020
84 MRB-16553-USA (Michigan) - Outreach and Affirm... ... Tuesday, 16 June, 2020
85 FOOD-5240-USA (Washington, DC.) - Inmate Food ... ... Wednesday, 17 June, 2020
86 SW-33970-USA (Washington, DC.) - ADA Compliant... ... Friday, 29 May, 2020
87 PM-12591-USA (Philadelphia, Pennsylvania) - Pr... ... Monday, 1 June, 2020
88 PM-12590-USA (Washington, DC.) - Printing Serv... ... Monday, 1 June, 2020
89 EXTRA-8363-USA (Fort Lauderdale, Florida) - Ma... ... Monday, 22 June, 2020
90 SW-33963-USA (Florida) - Information Managemen... ... Monday, 15 June, 2020
91 EXTRA-8362-USA (Sarasota, Florida) - American ... ... Wednesday, 17 June, 2020
92 SW-33954-USA (Wisconsin) - RFI for Application... ... Tuesday, 12 May, 2020
93 LEGAL-4987-USA (Charleston, West Virginia) - O... ... Wednesday, 24 June, 2020
94 PM-12588-USA (Encinitas, California) - Managed... ... Wednesday, 10 June, 2020
95 ESTATE-3246-USA (Charlotte, North Carolina) - ... ... Tuesday, 16 June, 2020
96 DISPOS-3793-USA (Fortana, California) -Removal... ... Thursday, 4 June, 2020
97 SW-33952-USA (New York) - GPS Tracking System-... ... Thursday, 11 June, 2020
98 SW-33953-Canada (Ontario) - VMware Support Ser... ... Thursday, 4 June, 2020
99 SW-33951- USA (South Carolina) - Professional ... ... Thursday, 25 June, 2020
[100 rows x 3 columns]