Соскоб с селеном в Python

Question

Я бы хотел очистить сайт Сантандера. Но код, который я использую, не работает, так как, когда я запускаюсь дважды, я получаю разные суммы.

Мой текущий код не работает слишком много.

Царапины, кажется, работают, за исключением того, что я получаю ложные результаты. И когда я запускаю код дважды подряд, результаты меняются.

Ссылка на сайт выглядит следующим образом: https://simuladores.bancosantander.es/SantanderES/loansimulatorweb.aspx?por=webpublica&prv=publico&m=300&cta=1&ls=0#/t0

Мой код:

def hw_santander_scrap(Amount, Duration):
      from selenium import webdriver
      from selenium.webdriver.support.ui import WebDriverWait
      from selenium.webdriver.support import expected_conditions as EC
      chrome_options = webdriver.ChromeOptions()
      chrome_options.add_argument('--headless')
      chrome_options.add_argument('--no-sandbox')
      chrome_options.add_argument('--disable-dev-shm-usage')
      chrome_options.add_argument('--start-maximized')
      chrome_options.add_argument('window-size=10000x5000')
      webdriver = webdriver.Chrome('chromedriver', chrome_options = chrome_options)

      #
      import time
      maintenant = DT.now()
      period = str(maintenant.day) + '_' + str(maintenant.month) + '_' + str(maintenant.year)
      print('Start Scraping')

      ################################################ Santander###############################################

      Santander = pd.DataFrame({
          'Project': "reforma vivienda",
          'Period': period,
          'Monthly repayment': [0],
          'TIN': [0],
          'TAE': [0],
          'Total repayment': [0],
          'Initial amount': [0],
          'Duration': [0]
      })

      project = pd.DataFrame({
          'Project': "reforma vivienda",
          'Period': period,
          'Monthly repayment': [0],
          'TIN': [0],
          'TAE': [0],
          'Total repayment': [0],
          'Initial amount': [0],
          'Duration': [0]
      })
      url = 'https://simuladores.bancosantander.es/SantanderES/loansimulatorweb.aspx?por=webpublica&prv=publico&m=300&cta=1&ls=0#/t0'
      webdriver.get(url)

      Max_amount = 90.000
      Min_amount = 3.000
      for i in range(len(Amount)):
        Simulated_amount = Amount[i]
        if Simulated_amount > Max_amount:
          pass
        elif Simulated_amount < Min_amount:
          pass
        else :
          amount = WebDriverWait(webdriver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "#amount")))
          amount.clear()
          amount.send_keys("{:.3f}".format(Simulated_amount))
          WebDriverWait(webdriver, 30).until(lambda webdriver: webdriver.execute_script('return jQuery.active') == 0)
          for j in range(len(Duration)):
            Simulated_duration = Duration[j]
            Simulated_duration = round(int(Simulated_duration))
            Max_duration = 96
            Min_duration = 12
            if Simulated_duration > Max_duration:
              pass
            elif Simulated_duration < Min_duration:
              pass
            else :
              term = WebDriverWait(webdriver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "#term")))
            term.clear()
            term.send_keys("{}".format(Simulated_duration))
            term.send_keys(Keys.TAB)
            perform_term = webdriver.find_element_by_xpath("//span[@class='ui-slider-handle ui-state-default ui-corner-all']")
            actions = ActionChains(webdriver).click(perform_term)
            actions.perform()
            webdriver.save_screenshot('screenshot_santander.png')
            currentfee = webdriver.find_element_by_css_selector('.r1 span').text

            WebDriverWait(webdriver, 10).until(lambda webdriver: webdriver.execute_script('return jQuery.active') == 0)
            webdriver.save_screenshot('screenshot_santander.png')
            project.loc[j, 'Project'] = "reforma vivienda"
            project.loc[j, 'Initial amount'] = float("{:.3f}".format(Amount[i]).replace('.', ''))
            project.loc[j, 'Duration'] = Simulated_duration
            project.loc[j, 'Period'] = str(maintenant.day) + '/' + str(maintenant.month) + '/' + str(maintenant.year)
            project.loc[j, 'Monthly repayment'] = webdriver.find_element_by_css_selector('.r1 span').text.replace(' €', '').replace(',', '.')
            project.loc[j, 'TIN'] = float(webdriver.find_element_by_css_selector('.r3 span').text[6: 10].replace(',', '.'))
            project.loc[j, 'TAE'] = float(webdriver.find_element_by_css_selector('.r3 span').text[13: 17].replace(',', '.'))
            project.loc[j, 'Total repayment'] = float(webdriver.find_element_by_css_selector('.r7 span').text.replace(' €', '').replace('.', '').replace(',', '.'))
          Santander = Santander.append(project)
      Santander = Santander.loc[Santander.TIN != 0,: ]
      Santander.to_csv('Santander_{}.csv'.format(period), index = False)
    print('End Scraping')

For run the code:

    Amount = [13.000, 14.000, 15.000, 30.000, 45.000, 60.000]
    Duration = [12, 15, 24, 36, 48, 60, 72, 84, 96]
    hw_santander_scrap(Amount, Duration)

Answer 1

Вот предложение с использованием requests:

import requests
from bs4 import BeautifulSoup
import re
import json
import pandas as pd

#Let's first collect few auth vars
r = requests.Session()
response = r.get("https://simuladores.bancosantander.es/SantanderES/loansimulatorweb.aspx?por=webpublica&prv=publico&m=300&cta=1&ls=0#/t0")
soup = BeautifulSoup(response.content, 'html')
key = soup.find_all('script',text=re.compile('Afi.AfiAuth.Init'))
pattern = r"Afi.AfiAuth.Init\((.*?)\)"

WSSignature = re.findall(pattern,key[0].text)[0].split(',')[-1].replace('\'','')
WSDateTime = re.findall(pattern,key[0].text)[0].split(',')[1].replace('\'','')

headers = {
    'Origin': 'https://simuladores.bancosantander.es',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36',
    'Content-Type': 'application/json;charset=UTF-8',
    'Accept': 'application/json, text/plain, */*',
    'WSSignature': WSSignature,
    'Referer': 'https://simuladores.bancosantander.es/SantanderES/loansimulatorweb.aspx?por=webpublica&prv=publico&m=300&cta=1&ls=0',
    'WSDateTime': WSDateTime,
    'WSClientCode': 'SantanderES',
}

#Those are the standard params of a request
params = {'wsInputs': {'finality': 'Renueva tu hogar',
  'productCode': 'p300',
  'capitalOrInstallment': 12000,
  'monthsTerm': 96,
  'mothsInitialTerm': 0,
  'openingCommission': 1.5,
  'minOpeningCommission': 0,
  'financeOpeningCommission': True,
  'interestRate': 0,
  'interestRateReferenceIndex': 0,
  'interestRateSecondaryReferenceIndex': 0,
  'interestRateSecondaryWithoutVinculation': 5.95,
  'interestRateSecondaryWithAllVinculation': 0,
  'interestRateSecondary': 5.95,
  'loanDate': '2019-06-12',
  'birthDate': '2001-06-12',
  'financeLoanProtectionInsurance': True,
  'percentageNotaryCosts': 0.003,
  'loanCalculationMethod': 0,
  'calculationBase': 4,
  'frecuencyAmortization': 12,
  'frecuencyInterestPay': 12,
  'calendarConvention': 0,
  'taeCalculationBaseType': 4,
  'lackMode': 0,
  'amortizationCarencyMonths': 0,
  'typeAmortization': 1,
  'insuranceCostSinglePremium': 0,
  'with123': False,
  'electricVehicle': False}}

#The scraping function
def scrap(amount, duration, params):

    params['wsInputs']['capitalOrInstallment'] = amount
    params['wsInputs']['monthsTerm'] = duration
    response = r.post('https://simuladores.bancosantander.es/WS/WSSantanderTotalLoan.asmx/Calculate', headers=headers, data=json.dumps(params))
    return json.loads(response.content)['d']


Amounts = [13000, 14000, 15000, 30000, 45000, 60000]
Durations = [12, 15, 24, 36, 48, 60, 72, 84, 96]
results = []
for amount in Amounts:
    for duration in Durations:
        result = scrap(amount, duration, params)
        result['Amount'] = amount
        result['Duration'] = duration
        results.append(result)

df = pd.DataFrame(results)

Вот пример информации, которую вы можете получить напрямую:

{   ....

    'TotalCosts': 0,
     'CapitalOrInstallment': 1135.3433231340491,
     'Disclaimer': '',
     'Capital': 13195,
     'ThereIsLackAtFirstStep': False,
     'ThereIsLackAtSecondStep': False,
     'InstallmentFirstStepLackPeriod': 1135.3433231340491,
     'InstallmentSecondStepLackPeriod': 1135.3433231340491,
     'Installment': 1135.3433231340491,
     'InstallmentFirstStep': 1135.3433231340491,
     'InstallmentSecondStep': 1135.3433231340491,
     'CommissionOpeningCosts': 195.00000000000003,
     'TAE': 9.1,
     'TAEWithoutVinculation': 9.1,  
     ....
}

РЕДАКТИРОВАТЬ 1: Добавлен код для захвата двух важных переменных WSSignature, WSDateTime и вставки их в заголовки

РЕДАКТИРОВАТЬ 2: Параметры могут быть адаптированы для различных типов кредитов:

#coaches
params_coaches =  {'wsInputs': {'finality': 'Vehículo con hasta 36 meses de antigüedad',
  'productCode': 'p100',
  'capitalOrInstallment': 5000,
  'monthsTerm': 96,
  'mothsInitialTerm': 12,
  'openingCommission': 1.5,
  'minOpeningCommission': 60,
  'financeOpeningCommission': True,
  'interestRate': 5.5,
  'interestRateReferenceIndex': 0,
  'interestRateSecondaryReferenceIndex': 0,
  'interestRateSecondaryWithoutVinculation': 6.5,
  'interestRateSecondaryWithAllVinculation': 0,
  'interestRateSecondary': 6.5,
  'loanDate': '2019-06-13',
  'birthDate': '2001-06-13',
  'financeLoanProtectionInsurance': True,
  'percentageNotaryCosts': 0.003,
  'loanCalculationMethod': 0,
  'calculationBase': 4,
  'frecuencyAmortization': 12,
  'frecuencyInterestPay': 12,
  'calendarConvention': 0,
  'taeCalculationBaseType': 4,
  'lackMode': 0,
  'amortizationCarencyMonths': 0,
  'typeAmortization': 1,
  'insuranceCostSinglePremium': 0,
  'with123': False,
  'electricVehicle': False}}

#proyectos
params_proyectos = {'wsInputs': {'finality': 'Tus proyectos',
  'productCode': 'p200',
  'capitalOrInstallment': 6000,
  'monthsTerm': 96,
  'mothsInitialTerm': 0,
  'openingCommission': 1.5,
  'minOpeningCommission': 60,
  'financeOpeningCommission': True,
  'interestRate': 0,
  'interestRateReferenceIndex': 0,
  'interestRateSecondaryReferenceIndex': 0,
  'interestRateSecondaryWithoutVinculation': 5.95,
  'interestRateSecondaryWithAllVinculation': 0,
  'interestRateSecondary': 5.95,
  'loanDate': '2019-06-13',
  'birthDate': '2001-06-13',
  'financeLoanProtectionInsurance': True,
  'percentageNotaryCosts': 0.003,
  'loanCalculationMethod': 0,
  'calculationBase': 4,
  'frecuencyAmortization': 12,
  'frecuencyInterestPay': 12,
  'calendarConvention': 0,
  'taeCalculationBaseType': 4,
  'lackMode': 0,
  'amortizationCarencyMonths': 0,
  'typeAmortization': 1,
  'insuranceCostSinglePremium': 0,
  'with123': False,
  'electricVehicle': False}}

Вам просто нужно выбрать правильный, когда вы вызываете функцию. Например:

result = scrap(amount, duration, params)
#or
result = scrap(amount, duration, params_coaches)
#or
result = scrap(amount, duration, params_proyectos)