Я написал паука со скрапом, который использовал CrawlerProcess и работал нормально.
Следующим шагом, я хотел получить графический интерфейс для некоторых входных данных, сделанных с помощью pyqt5, и установить спиннер, пока работает scrapy. Эта задача была выполнена с использованием фиктивного процесса и также работала нормально.
Мне известно, что многопоточность не совместима с CrawlerProcess, поэтому я изменил его для CrawlerRunner. Во всяком случае, это не удается:
signal.signal(signal.SIGINT, self.sigInt)
File "C:\Program Files\Python36\lib\signal.py", line 47, in signal
handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread
Я создал минимальный пример для воспроизведения ошибки:
from PyQt5.QtWidgets import QApplication, QDialog, QTabWidget, QWidget, QGroupBox, QPushButton, QVBoxLayout
from PyQt5.QtCore import Qt, QRunnable, QMetaObject, pyqtSlot, QThreadPool
from PyQt5.QtGui import QIcon
from scrapy.crawler import CrawlerRunner
from scrapy import Request, Spider, Item, Field
from twisted.internet import reactor
from waitingspinnerwidget import QtWaitingSpinner
class RequestRunnable(QRunnable):
def __init__(self, dialog):
super(RequestRunnable, self).__init__()
self.w = dialog
print('runnable __init__')
def run(self):
print('run')
crawler = CrawlerRunner({
'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
'FEED_FORMAT': 'json',
'FEED_URI': 'output.json',
'DOWNLOAD_DELAY': 3,
'LOG_STDOUT': True,
'LOG_FILE': 'scrapy_output.txt',
'ROBOTSTXT_OBEY': False,
'RETRY_ENABLED': True,
'RETRY_HTTP_CODES': [500, 503, 504, 400, 404, 408],
'RETRY_TIMES': 5
})
# instantiate a spider
spider = CustomSpider()
d = crawler.crawl(spider)
d.addBoth(lambda _: reactor.stop()) # I think this is for manually stopping the reactor when we are done
reactor.run() #TODO gives error
QMetaObject.invokeMethod(self.w, "FinishedDownload", Qt.QueuedConnection)
class CustomSpider(Spider):
name = 'quotes_spider'
allowed_domains = ['quotes.toscrape.com']
start_urls = ['http://quotes.toscrape.com/']
def parse(self, response):
quotes = response.xpath("//div[@class='quote']//span[@class='text']/text()").extract()
yield {'quotes': quotes}
class DownloadDataDialog(QDialog):
def __init__(self, parent=None):
super(DownloadDataDialog, self).__init__(parent)
self.spinner = QtWaitingSpinner(self, True, True, Qt.ApplicationModal)
tabWidget = QTabWidget(self)
tabWidget.addTab(MyTab(tabWidget), "MyTab")
mainLayout = QVBoxLayout()
mainLayout.addWidget(tabWidget)
self.setLayout(mainLayout)
self.setWindowTitle("Download option chain data from web")
class MyTab(QWidget):
def __init__(self, parent=None):
super(MyTab, self).__init__(parent)
dataGroup = QGroupBox('Data')
getButton = QPushButton('Download')
getButton.clicked.connect(self.download_data)
dataLayout = QVBoxLayout()
dataLayout.addWidget(getButton)
dataGroup.setLayout(dataLayout)
mainLayout = QVBoxLayout()
mainLayout.addWidget(dataGroup)
mainLayout.addStretch(1)
self.setLayout(mainLayout)
def download_data(self):
self.parentWidget().window().spinner.start()
runnable = RequestRunnable(self)
QThreadPool.globalInstance().start(runnable)
@pyqtSlot()
def FinishedDownload(self):
self.parentWidget().window().spinner.stop()
if __name__ == '__main__':
import sys
app = QApplication(sys.argv)
tabdialog = DownloadDataDialog()
tabdialog.show()
sys.exit(app.exec_())
Чтобы выполнить его, вам потребуется waitingspinnerwidget.py:
# -*- coding: utf-8 -*-
"""
The MIT License (MIT)
Copyright (c) 2012-2014 Alexander Turkin
Copyright (c) 2014 William Hallatt
Copyright (c) 2015 Jacob Dawid
Copyright (c) 2016 Luca Weiss
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
import math
from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
class QtWaitingSpinner(QWidget):
def __init__(self, parent, centerOnParent=True, disableParentWhenSpinning=False, modality=Qt.NonModal):
super().__init__(parent, Qt.Dialog | Qt.FramelessWindowHint)
self._centerOnParent = centerOnParent
self._disableParentWhenSpinning = disableParentWhenSpinning
# WAS IN initialize()
self._color = QColor(Qt.black)
self._roundness = 100.0
self._minimumTrailOpacity = 3.14159265358979323846
self._trailFadePercentage = 80.0
self._revolutionsPerSecond = 1.57079632679489661923
self._numberOfLines = 20
self._lineLength = 10
self._lineWidth = 2
self._innerRadius = 10
self._currentCounter = 0
self._isSpinning = False
self._timer = QTimer(self)
self._timer.timeout.connect(self.rotate)
self.updateSize()
self.updateTimer()
self.hide()
# END initialize()
self.setWindowModality(modality)
self.setAttribute(Qt.WA_TranslucentBackground)
def paintEvent(self, QPaintEvent):
self.updatePosition()
painter = QPainter(self)
painter.fillRect(self.rect(), Qt.transparent)
painter.setRenderHint(QPainter.Antialiasing, True)
if self._currentCounter >= self._numberOfLines:
self._currentCounter = 0
painter.setPen(Qt.NoPen)
for i in range(0, self._numberOfLines):
painter.save()
painter.translate(self._innerRadius + self._lineLength, self._innerRadius + self._lineLength)
rotateAngle = float(360 * i) / float(self._numberOfLines)
painter.rotate(rotateAngle)
painter.translate(self._innerRadius, 0)
distance = self.lineCountDistanceFromPrimary(i, self._currentCounter, self._numberOfLines)
color = self.currentLineColor(distance, self._numberOfLines, self._trailFadePercentage,
self._minimumTrailOpacity, self._color)
painter.setBrush(color)
painter.drawRoundedRect(QRect(0, -self._lineWidth / 2, self._lineLength, self._lineWidth), self._roundness,
self._roundness, Qt.RelativeSize)
painter.restore()
def start(self):
self.updatePosition()
self._isSpinning = True
self.show()
if self.parentWidget and self._disableParentWhenSpinning:
self.parentWidget().setEnabled(False)
if not self._timer.isActive():
self._timer.start()
self._currentCounter = 0
def stop(self):
self._isSpinning = False
self.hide()
if self.parentWidget() and self._disableParentWhenSpinning:
self.parentWidget().setEnabled(True)
if self._timer.isActive():
self._timer.stop()
self._currentCounter = 0
def setNumberOfLines(self, lines):
self._numberOfLines = lines
self._currentCounter = 0
self.updateTimer()
def setLineLength(self, length):
self._lineLength = length
self.updateSize()
def setLineWidth(self, width):
self._lineWidth = width
self.updateSize()
def setInnerRadius(self, radius):
self._innerRadius = radius
self.updateSize()
def color(self):
return self._color
def roundness(self):
return self._roundness
def minimumTrailOpacity(self):
return self._minimumTrailOpacity
def trailFadePercentage(self):
return self._trailFadePercentage
def revolutionsPersSecond(self):
return self._revolutionsPerSecond
def numberOfLines(self):
return self._numberOfLines
def lineLength(self):
return self._lineLength
def lineWidth(self):
return self._lineWidth
def innerRadius(self):
return self._innerRadius
def isSpinning(self):
return self._isSpinning
def setRoundness(self, roundness):
self._roundness = max(0.0, min(100.0, roundness))
def setColor(self, color=Qt.black):
self._color = QColor(color)
def setRevolutionsPerSecond(self, revolutionsPerSecond):
self._revolutionsPerSecond = revolutionsPerSecond
self.updateTimer()
def setTrailFadePercentage(self, trail):
self._trailFadePercentage = trail
def setMinimumTrailOpacity(self, minimumTrailOpacity):
self._minimumTrailOpacity = minimumTrailOpacity
def rotate(self):
self._currentCounter += 1
if self._currentCounter >= self._numberOfLines:
self._currentCounter = 0
self.update()
def updateSize(self):
size = (self._innerRadius + self._lineLength) * 2
self.setFixedSize(size, size)
def updateTimer(self):
self._timer.setInterval(1000 / (self._numberOfLines * self._revolutionsPerSecond))
def updatePosition(self):
if self.parentWidget() and self._centerOnParent:
self.move(self.parentWidget().width() / 2 - self.width() / 2,
self.parentWidget().height() / 2 - self.height() / 2)
dialogCenter = self.mapToGlobal(self.rect().center())
parentWindowCenter = self.parentWidget().window().mapToGlobal(self.parentWidget().window().rect().center())
self.move(parentWindowCenter - dialogCenter)
parentRect = QRect(self.parentWidget().mapToGlobal(QPoint(0, 0)), self.parentWidget().size())
self.move(QStyle.alignedRect(Qt.LeftToRight, Qt.AlignCenter, self.size(), parentRect).topLeft())
def lineCountDistanceFromPrimary(self, current, primary, totalNrOfLines):
distance = primary - current
if distance < 0:
distance += totalNrOfLines
return distance
def currentLineColor(self, countDistance, totalNrOfLines, trailFadePerc, minOpacity, colorinput):
color = QColor(colorinput)
if countDistance == 0:
return color
minAlphaF = minOpacity / 100.0
distanceThreshold = int(math.ceil((totalNrOfLines - 1) * trailFadePerc / 100.0))
if countDistance > distanceThreshold:
color.setAlphaF(minAlphaF)
else:
alphaDiff = color.alphaF() - minAlphaF
gradient = alphaDiff / float(distanceThreshold + 1)
resultAlpha = color.alphaF() - gradient * countDistance
# If alpha is out of bounds, clip it.
resultAlpha = min(1.0, max(0.0, resultAlpha))
color.setAlphaF(resultAlpha)
return color
Полный возврат:
Traceback (most recent call last):
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 428, in fireEvent
DeferredList(beforeResults).addCallback(self._continueFiring)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 322, in addCallback
callbackKeywords=kw)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 311, in addCallbacks
self._runCallbacks()
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 654, in _runCallbacks
current.result = callback(current.result, *args, **kw)
--- <exception caught here> ---
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 441, in _continueFiring
callable(*args, **kwargs)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 1256, in _reallyStartRunning
self._handleSignals()
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\posixbase.py", line 295, in _handleSignals
_SignalReactorMixin._handleSignals(self)
File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 1221, in _handleSignals
signal.signal(signal.SIGINT, self.sigInt)
File "C:\Program Files\Python36\lib\signal.py", line 47, in signal
handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread
Есть идеи, что происходит и как ее решить?