Вызов паука Scrapy из скрипта Python - PullRequest
Новая
       9

Вызов паука Scrapy из скрипта Python

0 голосов
/ 12 февраля 2019

я хочу запустить мой паук-scrapy в ActiveMQ-клиенте (в onMessage)

вот мой код паука-scrapy 

import scrapy

import json

класс selogerSpider (scrapy.Spider): 

name = "realtor"


custom_settings = {
    'DOWNLOADER_MIDDLEWARES': {
        'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
        'scrapy_fake_useragent.middleware.RandomUserAgentMiddleware': 400,
    }
}

def start_requests(self):
    with open("annonces.txt", "r") as file:
        for line in file.readlines():
            yield scrapy.Request(line)

def parse(self, response):
  try:
    name = response.css(".agence-link::text").extract_first()
    address = response.css(".agence-adresse::text").extract_first()

    XPATH_siren = ".//div[@class='legalNoticeAgency']//p/text()"
    siren = response.xpath(XPATH_siren).extract_first()

    XPATH_website = ".//div[@class='agence-links']//a/@href"
    site = response.xpath(XPATH_website).extract()

    XPATH_phone = ".//div[@class='contact g-row-50']//div[@class='g-col g-50 u-pad-0']//button[@class='btn-phone b-btn b-second fi fi-phone tagClick']/@data-phone"
    phone = response.xpath(XPATH_phone).extract_first()

    agencyObj = WebRealTor(name, address, siren, site, phone)
    realtorJson = json.dumps(agencyObj.__dict__)
    with open('data.json', 'a') as outfile:
        json.dump(realtorJson, outfile)
  except KeyError :
    print(" KeyError")
class WebRealTor(object):
def __init__(self, nom,adresse,siren,site,phone):
    self.Nom = nom
    self.Adresse = adresse
    self.Site_web = site
    self.Numero_Siren= siren
    self.Numero_telephone = phone
def jsonDefault(object):
    return object.__dict__

мой код импорта activemq моего слушателя import sys import stomp 

class MyListener(stomp.ConnectionListener):
def on_error(self, headers, message):
    print('received an error "%s"' % message)

def on_message(self, headers, message):
    print('received a message "%s"' % message)
try:
conn = stomp.Connection()
conn.set_listener('', MyListener())
conn.start()
conn.connect('admin', 'password', wait=True)

conn.subscribe(destination='/queue/test', id=1, ack='auto')

conn.send(body=' '.join(sys.argv[1:]), destination='/queue/test')

time.sleep(2)
conn.disconnect()

за исключением IOError as e: print "Ошибка ввода-вывода ({0}): {1} ". format (e.errno, e.strerror)

...