Я конвертировал мой локально работающий скребок для использования на aws лямбде. Затем я использовал руководства, инструменты и примеры:
https://medium.com/@marco.luethy / running-headless- chrome -on- aws -lambda-fa82ad33a9eb
https://github.com/adieuadieu/serverless-chrome/releases
https://github.com/21Buttons/pychromeless/blob/master/src/webdriver_wrapper.py
https://robertorocha.info/setting-up-a-selenium-web-scraper-on-aws-lambda-with-python/
import os
from selenium import webdriver
# Chrome Driver Client
class ChromeClient:
# set up setting of web driver
def __init__(self, logger):
# setup logger
self._tmp_folder = '/tmp/'
try:
if not os.path.exists(self._tmp_folder):
os.makedirs(self._tmp_folder)
if not os.path.exists(self._tmp_folder + '/user-data'):
os.makedirs(self._tmp_folder + '/user-data')
if not os.path.exists(self._tmp_folder + '/data-path'):
os.makedirs(self._tmp_folder + '/data-path')
if not os.path.exists(self._tmp_folder + '/cache-dir'):
os.makedirs(self._tmp_folder + '/cache-dir')
except Exception as ex:
logger.error('problem with {0}'.format(ex))
try:
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--window-size=1280x1696')
chrome_options.add_argument('--user-data-dir=/tmp/user-data')
chrome_options.add_argument('--hide-scrollbars')
chrome_options.add_argument('--enable-logging')
chrome_options.add_argument('--log-level=0')
chrome_options.add_argument('--v=99')
chrome_options.add_argument('--single-process')
chrome_options.add_argument('--data-path=/tmp/data-path')
chrome_options.add_argument('--ignore-certificate-errors')
chrome_options.add_argument('--homedir=/tmp')
chrome_options.add_argument('--disk-cache-dir=/tmp/cache-dir')
chrome_options.add_argument('user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36')
chrome_options.binary_location = os.getcwd() + "/bin/headless-chromium"
except Exception as ex:
logger.error('problem with {0}'.format(ex))
try:
logger.info('__init__(), Init Chrome Driver')
print('make me some chrome')
try:
self.driver = webdriver.Chrome(chrome_options=chrome_options)
except Exception as ex:
logger.error('chrome problem with {0}'.format(ex))
self.driver.maximize_window()
self.driver.delete_all_cookies()
self.driver.set_page_load_timeout(120)
self.driver.implicitly_wait(5)
self.driver.delete_all_cookies()
except Exception as ex:
logger.error('problem with {0}'.format(ex))
Каждый раз он попадает в webdriver. Chrome (chrome_options = chrome_options) Я вижу ошибку:
Служба chromedriver неожиданно завершила работу. Код состояния был: 127
Есть идеи?