Как вы уже видели в заголовке, если я не помещаю печать в свой код где-то, я не получаю все ссылки, которые я должен получить, вот мой код:
Parsing_spider.py
# -*- coding: utf-8 -*-
# module.py
import scrapy
import json
from pprint import pprint
import time
import random
import re
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy import signals
from scrapy.xlib.pydispatch import dispatcher
import copy
from scrapy_splash import SplashRequest
open('scrape.json', 'w').close()
class ParsingSpider(CrawlSpider):
name = 'Parsing_spider'
def __init__(self, category=None, *args, **kwargs):
super(ParsingSpider, self).__init__(*args, **kwargs)
self.jsondata = json.load(open('../../scrapeData.json'))
if self.jsondata['liste_dom_auth']!=[u'']:
self.allowed_domains = self.jsondata['liste_dom_auth']
self.start_urls = self.jsondata['liste_start_urls']
self.liste_crit=self.jsondata['liste_crit']
self.deny_domains=self.jsondata['liste_dom_exclus']
self.depth=self.jsondata['Depth']
self.currentDepth=0
self.next_urls=[]
self.dico={}
self.general=[]
dispatcher.connect(self.spider_closed, signals.spider_closed)
def start_requests(self):
for start_url in self.start_urls:
yield SplashRequest(start_url, self.parse, endpoint='render.html')
def parse(self, response):
string=response.xpath("//body").extract_first()
string=re.sub(r'\s+',' ',string)
regex = re.compile(r'[\n\r\t]')
string = regex.sub(" ", string)
string = self.suppr_spaces(string)
splitted=string.split(self.liste_crit[0]['start_node'])[1:]
for i in splitted:
x=self.liste_crit[0]['start_node']+i
if self.liste_crit[0]['end_node'] in x:
self.synchronized_item(x)
if self.currentDepth<int(self.depth):
self.increaseDepth()
print(response.css("a::attr(href)").extract())
for links in response.css("a::attr(href)").extract():
if not(self.is_denied(links)):
time.sleep(random.uniform(0, 1))
yield SplashRequest(response.urljoin(links), callback=self.parse, endpoint='render.html')
def is_denied(self,url):
return url in self.deny_domains
def increaseDepth(self):
self.currentDepth+=1
def cleanString(self,strings):
cleanr = re.compile('<.*?>')
strings= re.sub(cleanr, ' ', strings)
regex = re.compile(r'[\n\r\t]')
strings = regex.sub(" ", strings)
return strings
def suppr_spaces(self,strings):
return re.sub("(?<=>)\s*","",strings)
def synchronized_item(self,item):
dico={}
for crit in self.liste_crit:
l=[]
items=item.split(crit['start_node'])
for item2 in items[1:]:
l.append(self.cleanString(item2.split(crit['end_node'])[0]))
dico[crit["name"]]=l
self.general.append(dico)
def spider_closed(self, spider):
f=open("../../data/scrape"+self.jsondata["id"]+".json",'w')
f.write(json.dumps(self.general,indent=4))
f.close()
data = json.load(open("../../scrapList.json"))
for obj in data:
if obj["id"]==self.jsondata["id"]:
obj["status"]="Finished"
f=open("../../scrapList.json",'w')
f.write(json.dumps(data,indent=4))
f.close()
Это та часть, которая доставляет мне неприятности
if self.currentDepth<int(self.depth):
self.increaseDepth()
print(response.css("a::attr(href)").extract())
for links in response.css("a::attr(href)").extract():
if not(self.is_denied(links)):
time.sleep(random.uniform(0, 1))
yield SplashRequest(response.urljoin(links), callback=self.parse, endpoint='render.html')
если я не помещу печать над циклом for, вот мой вывод:
/var/www/nexterit/scrapper/Scraper/public/myproject/spiders/Parsing_spider.py:13: ScrapyDeprecationWarning: Importing from scrapy.xlib.pydispatch is deprecated and will no longer be supported in future Scrapy versions. If you just want to connect signals use the from_crawler class method, otherwise import pydispatch directly if needed. See: https://github.com/scrapy/scrapy/issues/1762
from scrapy.xlib.pydispatch import dispatcher
2018-05-17 16:51:15 [scrapy.utils.log] INFO: Scrapy 1.5.0 started (bot: myproject)
2018-05-17 16:51:15 [scrapy.utils.log] INFO: Versions: lxml 4.1.1.0, libxml2 2.9.7, cssselect 0.9.1, parsel 1.4.0, w3lib 1.19.0, Twisted 18.4.0, Python 2.7.9 (default, Jun 29 2016, 13:08:31) - [GCC 4.9.2], pyOpenSSL 17.5.0 (OpenSSL 1.1.0h 27 Mar 2018), cryptography 2.2.2, Platform Linux-4.9.0-0.bpo.6-amd64-x86_64-with-Debian-8
2018-05-17 16:51:15 [scrapy.crawler] INFO: Overridden settings: {'NEWSPIDER_MODULE': 'myproject.spiders', 'SPIDER_MODULES': ['myproject.spiders'], 'DUPEFILTER_CLASS': 'scrapy_splash.SplashAwareDupeFilter', 'HTTPCACHE_STORAGE': 'scrapy_splash.SplashAwareFSCacheStorage', 'BOT_NAME': 'myproject'}
2018-05-17 16:51:15 [scrapy.middleware] INFO: Enabled extensions:
['scrapy.extensions.memusage.MemoryUsage',
'scrapy.extensions.logstats.LogStats',
'scrapy.extensions.telnet.TelnetConsole',
'scrapy.extensions.corestats.CoreStats']
2018-05-17 16:51:15 [scrapy.middleware] INFO: Enabled downloader middlewares:
['scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware',
'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware',
'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware',
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware',
'scrapy.downloadermiddlewares.retry.RetryMiddleware',
'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware',
'scrapy.downloadermiddlewares.redirect.RedirectMiddleware',
'scrapy.downloadermiddlewares.cookies.CookiesMiddleware',
'scrapy_splash.SplashCookiesMiddleware',
'scrapy_splash.SplashMiddleware',
'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware',
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware',
'scrapy.downloadermiddlewares.stats.DownloaderStats']
2018-05-17 16:51:15 [scrapy.middleware] INFO: Enabled spider middlewares:
['scrapy.spidermiddlewares.httperror.HttpErrorMiddleware',
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware',
'scrapy.spidermiddlewares.referer.RefererMiddleware',
'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware',
'scrapy.spidermiddlewares.depth.DepthMiddleware']
2018-05-17 16:51:15 [scrapy.middleware] INFO: Enabled item pipelines:
[]
2018-05-17 16:51:15 [scrapy.core.engine] INFO: Spider opened
2018-05-17 16:51:15 [scrapy.extensions.logstats] INFO: Crawled 0 pages (at 0 pages/min), scraped 0 items (at 0 items/min)
2018-05-17 16:51:15 [scrapy.extensions.telnet] DEBUG: Telnet console listening on 127.0.0.1:6023
2018-05-17 16:51:15 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://marche.equipement.paris.fr/tousleshoraires via http://localhost:8050/render.html> (referer: None)
2018-05-17 16:51:25 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://equipement.paris.fr/?tid=289%2C300 via http://localhost:8050/render.html> (referer: None)
2018-05-17 16:51:25 [scrapy.core.engine] INFO: Closing spider (finished)
2018-05-17 16:51:25 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
{'downloader/request_bytes': 1109,
'downloader/request_count': 2,
'downloader/request_method_count/POST': 2,
'downloader/response_bytes': 20849,
'downloader/response_count': 2,
'downloader/response_status_count/200': 2,
'finish_reason': 'finished',
'finish_time': datetime.datetime(2018, 5, 17, 14, 51, 25, 731305),
'log_count/DEBUG': 3,
'log_count/INFO': 7,
'memusage/max': 46739456,
'memusage/startup': 46739456,
'request_depth_max': 1,
'response_received_count': 2,
'scheduler/dequeued': 4,
'scheduler/dequeued/memory': 4,
'scheduler/enqueued': 4,
'scheduler/enqueued/memory': 4,
'splash/render.html/request_count': 2,
'splash/render.html/response_count/200': 2,
'start_time': datetime.datetime(2018, 5, 17, 14, 51, 15, 89126)}
2018-05-17 16:51:25 [scrapy.core.engine] INFO: Spider closed (finished)
Но вот проблема в том, что я должен получить более одного перенаправления, но я не получаю все ссылки со страницы, потому что они генерируются с помощью javascript, поэтому я использовал scrapy-splash.
Но самое смешное то, что когда я разрешил распечатать здесь, я получаю все ссылки, но также и эту ошибку:
2018-05-17 15:00:24 [scrapy.core.scraper] ERROR: Error downloading <GET http://equipement.paris.fr/Marché Bourse via http://equipement.paris.fr/March%C3%A9%20Bourse>
Traceback (most recent call last):
File "/usr/local/lib/python2.7/dist-packages/twisted/internet/defer.py", line 1386, in _inlineCallbacks
result = g.send(result)
File "/usr/local/lib/python2.7/dist-packages/scrapy/core/downloader/middleware.py", line 37, in process_request
response = yield method(request=request, spider=spider)
File "/usr/local/lib/python2.7/dist-packages/scrapy_splash/middleware.py", line 320, in process_request
body = json.dumps(args, ensure_ascii=False, sort_keys=True, indent=4)
File "/usr/lib/python2.7/json/__init__.py", line 250, in dumps
sort_keys=sort_keys, **kw).encode(obj)
File "/usr/lib/python2.7/json/encoder.py", line 210, in encode
return ''.join(chunks)
UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 33: ordinal not in range(128)
Заранее спасибо
Редактировать:
Кажется, я выяснил, почему возникает ошибка ascii. Возможно, это связано с тем, что в URL-адресе есть буква, но я до сих пор не знаю, почему при печати появляются все ссылки, и я не знаю, как избежать этой ошибки.