Как загрузить данные с веб-сайта в Excel и базу данных одновременно?

Question

Привет, я соскребаю учительские работы с этого сайта https://www.indeed.co.in/?r=us Теперь я хочу загрузить их в Excel и базу данных одновременно. Как это возможно, может ли кто-нибудь мне помочь? Я хочу получить заголовок заголовка , компания, зарплата и данные Мой код scrap.py для очистки:

import selenium.webdriver

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait

url = 'https://www.indeed.co.in/?r=us'
driver = webdriver.Chrome(r"mypython/bin/chromedriver_linux64/chromedriver")


driver.get(url)

driver.find_element_by_xpath('//*[@id="text-input-what"]').send_keys("teacher")
driver.find_element_by_xpath('//*[@id="whatWhereFormId"]/div[3]/button').click()

# scrape data
data = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.ID, "resultsCol")))
result_set = WebDriverWait(data, 10).until(
    EC.presence_of_all_elements_located((By.CLASS_NAME, "jobsearch-SerpJobCard")))

for result in result_set:
    data = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "resultsCol")))
    result_set = WebDriverWait(data, 10).until(
        EC.presence_of_all_elements_located((By.CLASS_NAME, "jobsearch-SerpJobCard")))
    for result in result_set:

        title = result.find_element_by_class_name("title").text
        print(title)

        school = result.find_element_by_class_name("company").text
        print(school)

        try:
            salary = result.find_element_by_class_name("salary").text
            print(salary)
        except:
            pass
            print("--------")
    # move to next page
    next_page = result.find_elements_by_xpath("//span[@class='pn']")[-1]
    driver.execute_script("arguments[0].click();", next_page)

Answer 1

Привет, я сделал несколько изменений в вашем коде, и ниже генерирует CSV с 200+ записями со всех страниц.

# -*- coding: utf-8 -*-
"""
Created on Thu Mar  5 15:12:04 2020

@author: prakh
"""

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import csv
import pandas as pd
import mysql.connector
from sqlalchemy import create_engine

def smallest(a, y, z):
    M = a
    if y < M:
        M = y    
    if z < M:
        M = z
        if y < z:
            M = y
    return M
url = 'https://www.indeed.co.in/?r=us'
driver = webdriver.Chrome(executable_path='C:/Users/prakh/Documents/PythonScripts/chromedriver.exe') 
driver.get(url)

driver.find_element_by_xpath('//*[@id="text-input-what"]').send_keys("teacher")
driver.find_element_by_xpath('//*[@id="whatWhereFormId"]/div[3]/button').click()

# scrape data
data = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.ID, "resultsCol")))
result_set = WebDriverWait(data, 10).until(
    EC.presence_of_all_elements_located((By.CLASS_NAME, "jobsearch-SerpJobCard")))

Resultfile = open("teacherr_jobs.csv",'a',encoding='utf-8',newline='')
head = csv.writer(Resultfile)
head.writerow(["Title","School","Salary"])
Resultfile.close() 
titles = []
company = []
salaries = []
resultlist = []
Resultfile = open("teacherr_jobs.csv",'a',encoding='utf-8',newline='')
wr = csv.writer(Resultfile) 
for result in result_set:
    data = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "resultsCol")))
    result_set = WebDriverWait(data, 10).until(
        EC.presence_of_all_elements_located((By.CLASS_NAME, "jobsearch-SerpJobCard")))
    for result in result_set:

        titles.append(result.find_element_by_class_name("title").text)
#        print(title)

        company.append(result.find_element_by_class_name("company").text)
#        print(school)

        try:
            salaries.append(result.find_element_by_class_name("salary").text)
#            print(salary)
        except:
            salaries.append('NA')
            pass
            print("--------")
    # move to next page
    next_page = result.find_elements_by_xpath("//span[@class='pn']")[-1]
    driver.execute_script("arguments[0].click();", next_page)
val = smallest(len(titles),len(company),len(salaries))
print(smallest(len(titles),len(company),len(salaries)))

for i in range(0,val-1):
    resultlist.append(titles[i])
    resultlist.append(company[i])
    resultlist.append(salaries[i]) 
    wr.writerow(resultlist) 
    resultlist = [] 
Resultfile.close() 
final_df = pd.DataFrame(
    {'Title': titles,
     'School': company,
     'Salary': salaries
    })

engine = create_engine('mysql+mysqldb://[user]:[pass]@[host]:[port]/[schema]', echo = False)
final_df.to_sql(name = 'my_table', con = engine, if_exists = 'append', index = False)    

driver.quit()

Answer 2

Python имеет встроенный модуль для записи в CSV. Ниже приведен пример двух функций; создайте файл CSV с заголовками и запишите свои данные в CSV. Если вы передадите список значений в функцию write_to_csv (), он запишет весь список в CSV, одно значение для столбца

from csv import writer   

# creates initial CSV with headers
def create_csv(file_name):
    with open(file_name, 'w', newline='') as f:
        csv_writer = writer(f, delimiter=',')
        headers = ['Title', 'Company', 'Salary']
        csv_writer.writerow(headers)

# appends data to the next row in a CSV file
def write_to_csv(file_name, row):
    with open(file_name, 'a', newline='') as f:
        csv_writer = writer(f, delimiter=',')
        csv_writer.writerow(row)

Для базы данных это будет зависеть от того, какую базу данных вы используете, например, SQLite3, который также поставляется с Python

import sqlite

def write_to_db(data):
    db.cursor.execute("INSERT INTO ..... """)
    db.commit  

В начале вашего скрипта вы создадите CSV, а затем, когда вы соберете свои данные, вызовите обе функции для записи в CSV и в DB

file_name = "My_CSV.csv"
create_csv(file_name)

for result in result_set:
    data = WebDriverWait(driver, 10).until(
    EC.presence_of_element_located((By.ID, "resultsCol")))
    result_set = WebDriverWait(data, 10).until(
    EC.presence_of_all_elements_located((By.CLASS_NAME, "jobsearch-SerpJobCard")))
for result in result_set:

    to_write = []  # list container for your data

    title = result.find_element_by_class_name("title").text
    to_write.append(title)  # append to list to write

    school = result.find_element_by_class_name("company").text
    to_write.append(school)  # append to list to write

    try:
        salary = result.find_element_by_class_name("salary").text
        to_write.append(salary)  # append to list to write
    except:
        to_write.append("")
        pass
        print("--------")

    # pass the data to the functions to write it out
    write_to_csv(file_name, to_write) 
    write_to_db(to_write)

# move to next page
next_page = result.find_elements_by_xpath("//span[@class='pn']")[-1]
driver.execute_script("arguments[0].click();", next_page)