Я использую скрап для очистки отзывов от seek.com.au. Я нашел эту ссылку https://company -profiles-api.cloud.seek.com.au / v1 / companies / 432306 / reviews? Page = 1 , в которой есть данные, которые мне нужны в кодировке JSON.
Данные выглядят так:
{
"paging":{
"page":1,
"perPage":20,
"total":825
},
"data":[
{
"timeAgoText":null,
"id":5330561,
"companyName":"Commonwealth Bank of Australia",
"companyRecommended":false,
"salarySummary":"fair",
"salarySummaryDisplayText":"Average",
"jobTitle":"Financial Planning Role",
"title":"Run away, don't walk!",
"pros":"Staff benefits, the programs are very good however IT support is atrocious. There is a procedure for absolutely everything so you aren't left wondering how to do things in the branch network.",
"cons":"Sell, sell, sell! Everything at CBA is about selling. Don't believe the reports that things have changed and performance is based on customer service. They may have on paper but sales numbers are still tracked.",
"yearLeft":"left_2019",
"yearLeftEmploymentStatusText":"former employee",
"yearsWorkedWith":"1_2_years",
"yearsWorkedWithText":"1 to 2 years",
"workLocation":"New South Wales, Australia",
"ratingCompanyOverall":2,
"ratingBenefitsAndPerks":3,
"ratingCareerOpportunity":3,
"ratingExecutiveManagement":1,
"ratingWorkEnvironment":2,
"ratingWorkLifeBalance":1,
"ratingStressLevel":null,
"ratingDiversity":3,
"reviewCreatedAt":"2019-09-11T11:41:10Z",
"reviewCreatedTimeAgoText":"1 month ago",
"reviewResponse":"Thank you for your feedback. At CommBank, we are continually working to ensure our performance metrics are realistic and achievable, so we appreciate your insights, which we will pass on to the Human Resources & Remuneration team. If you have any other feedback that you would like to share, we also encourage you to speak to HR Direct on 1800 989 696.",
"reviewResponseBy":"Employer Brand",
"reviewResponseForeignUserId":1,
"reviewResponseCreatedAt":"2019-10-17T05:13:52Z",
"reviewResponseCreatedTimeAgoText":"a few days ago",
"crowdflowerScore":3.0,
"isAnonymized":false,
"normalizedCfScore":2000.0,
"score":3.0483236,
"roleProximityScore":0.002
},
{
"timeAgoText":null,
"id":5327368,
"companyName":"Commonwealth Bank of Australia",
"companyRecommended":true,
"salarySummary":"below",
"salarySummaryDisplayText":"Low",
"jobTitle":"Customer Service Role",
"title":"Great to start your career in banking; not so great to stay for more than a few years",
"pros":"- Great work culture\n- Amazing colleagues\n- good career progress",
"cons":"- hard to get leave approved\n- no full-time opportunities\n- no staff benefits of real value",
"yearLeft":"still_work_here",
"yearLeftEmploymentStatusText":"current employee",
"yearsWorkedWith":"0_1_year",
"yearsWorkedWithText":"Less than 1 year",
"workLocation":"Melbourne VIC, Australia",
"ratingCompanyOverall":3,
"ratingBenefitsAndPerks":1,
"ratingCareerOpportunity":3,
"ratingExecutiveManagement":2,
"ratingWorkEnvironment":5,
"ratingWorkLifeBalance":3,
"ratingStressLevel":null,
"ratingDiversity":5,
"reviewCreatedAt":"2019-09-11T07:05:26Z",
"reviewCreatedTimeAgoText":"1 month ago",
"reviewResponse":"",
"reviewResponseBy":"",
"reviewResponseForeignUserId":null,
"reviewResponseCreatedAt":null,
"reviewResponseCreatedTimeAgoText":"",
"crowdflowerScore":3.0,
"isAnonymized":false,
"normalizedCfScore":2000.0,
"score":3.0483236,
"roleProximityScore":0.002
},
Я создал словарь и затем попытался вернуть данные, но возвращается только 1 значение
name = 'seek-spider'
allowed_domains = ['seek.com.au']
start_urls = [
'https://www.seek.com.au/companies/commonwealth-bank-of-australia-432306/reviews']
s = str(start_urls)
res = re.findall(r'\d+', s)
res = str(res)
string = (res[res.find("[")+1:res.find("]")])
string_replaced = string.replace("'", "")
start_urls = [
'https://company-profiles-api.cloud.seek.com.au/v1/companies/'+string_replaced+'/reviews?page=1']
def parse(self, response):
result = json.loads(response.body)
detail = {}
for i in result['data']:
detail['ID'] = i['id']
detail['Title'] = i['title']
detail['Pros'] = i['pros']
detail['Cons'] = i['cons']
return detail
Я ожидаю выводачтобы получить все данные, но возвращается только это:
{'ID': 135413, 'Title': 'Great place to work!', 'Pros': 'All of the above.', 'Cons': 'None that I can think of'}