Развертывание паука в Scraping Cloud.Он собирает ссылки для загрузки файлов и должен сохранять эти файлы в корзине Google Cloud.Работает при локальном запуске.Но при развертывании в Scraping Hub он возвращает следующие ошибки:
Traceback (most recent call last):
File "/usr/local/lib/python2.7/site-packages/twisted/internet/defer.py", line 1299, in _inlineCallbacks
result = g.send(result)
File "/usr/local/lib/python2.7/site-packages/scrapy/crawler.py", line 95, in crawl
six.reraise(*exc_info)
File "/usr/local/lib/python2.7/site-packages/scrapy/crawler.py", line 77, in crawl
self.engine = self._create_engine()
File "/usr/local/lib/python2.7/site-packages/scrapy/crawler.py", line 102, in _create_engine
return ExecutionEngine(self, lambda _: self.stop())
File "/usr/local/lib/python2.7/site-packages/scrapy/core/engine.py", line 70, in __init__
self.scraper = Scraper(crawler)
File "/usr/local/lib/python2.7/site-packages/scrapy/core/scraper.py", line 71, in __init__
self.itemproc = itemproc_cls.from_crawler(crawler)
File "/usr/local/lib/python2.7/site-packages/scrapy/middleware.py", line 58, in from_crawler
return cls.from_settings(crawler.settings, crawler)
File "/usr/local/lib/python2.7/site-packages/scrapy/middleware.py", line 34, in from_settings
mwcls = load_object(clspath)
File "/usr/local/lib/python2.7/site-packages/scrapy/utils/misc.py", line 44, in load_object
mod = import_module(module)
File "/usr/local/lib/python2.7/importlib/__init__.py", line 37, in import_module
__import__(name)
File "/app/__main__.egg/tceq_crawler/pipelines.py", line 4, in <module>
ImportError: cannot import name GCSFilesStore
Он работает локально и может загружать файлы в корзину GCloud.Есть служебная учетная запись, созданная только для паука.Я экспортировал файл учетных данных JSON и вставил его непосредственно в объект учетных данных.
В файле YML я указал файл требований.В файле needs.txt я указал все необходимые пакеты.
На данный момент я не уверен, почему он не может импортировать GCSFilesStore в Scrapy Cloud, когда он хорошо импортирует FilesPipeline.
Вот мой код:
pipelines.py
# -*- coding: utf-8 -*-
from scrapy.pipelines.files import FilesPipeline
from scrapy.pipelines.files import GCSFilesStore
from scrapy.http.request import Request
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
class TceqCrawlerPipeline(object):
def process_item(self, item, spider):
return item
class CustomFileNamePipeline(FilesPipeline):
def get_media_requests(self, item, info):
return [Request(x, meta={'file_name': item["file_name"]})
for x in item.get('file_urls', [])]
def file_path(self, request, response=None, info=None):
return request.meta['file_name']
class GCSFilesStoreJSON(GCSFilesStore):
CREDENTIALS = {
"type": "service_account",
"project_id": "project_id",
"private_key_id": "FROM JSON FILE",
"private_key": "-----BEGIN PRIVATE KEY-----FROM JSON FILE-----END PRIVATE KEY-----\n",
"client_email": "FROM JSON FILE",
"client_id": "FROM JSON FILE",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"token_uri": "https://oauth2.googleapis.com/token",
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/FROM JSON FILE"
}
def __init__(self, uri):
from google.cloud import storage
client = storage.Client.from_service_account_info(self.CREDENTIALS)
bucket, prefix = uri[5:].split('/', 1)
self.bucket = client.bucket(bucket)
self.prefix = prefix
class GCSFilePipeline(FilesPipeline):
def __init__(self, store_uri, download_func=None, settings=None):
super(GCSFilePipeline, self).__init__(store_uri,download_func,settings)
settings.py
DOWNLOADER_MIDDLEWARES = {'scrapy_crawlera.CrawleraMiddleware': 300}
CRAWLERA_ENABLED = True
CRAWLERA_APIKEY = 'API_KEY'
ITEM_PIPELINES = {
'tceq_crawler.pipelines.GCSFilePipeline': 1,
'tceq_crawler.pipelines.CustomFileNamePipeline': 200
}
##GOOGLE CLOUD STORAGE
FILES_STORE = 'gs://bucket-name/'
GCS_PROJECT_ID = 'project_id'
scrapinghub.yml
project: 12345
requirements:
file: requirements.txt
requirements.txt
pyasn1==0.4.1
google-api-core==1.8.1
google-auth==1.6.3
google-cloud-core==0.29.1
google-cloud-storage==1.14.0
google-resumable-media==0.3.2
googleapis-common-protos==1.5.8