У меня возникли некоторые проблемы при извлечении URL-адресов изображений с сайта Amazon.
Мне удалось получить сценарий, содержащий все URL-адреса, но когда начинается извлечение, все работает не так хорошо.
Например, давайте возьмем этот продукт Amazon:
https://www.amazon.com/gp/product/B0711BMXVB?pf_rd_p=1581d9f4-062f-453c-b69e-0f3e00ba2652&pf_rd_r=YY1VTQE69K265SDTNETW
С xpath
//script[contains(., "ImageBlockATF")]/text()
Вы можете получить часть исходного кодакоторый содержит URL.Я получаю этот скрипт, выполняя
s = response.xpath('//script[contains(., "ImageBlockATF")]/text()').extract_first()
А затем, для получения отдельных URL-адресов, что-то вроде этого
m = re.search(r'^var data = ({.*};)', s, re.S | re.M)
data = m.groups()[0]
Проблема, с которой я столкнулся, заключается в том, что для re.search, чтобы работать, мне нужно s, чтобы быть строкой, и это не происходит.Когда код достигает data = m.groups()[0], возникает ошибка, где m, потому что s была "не" строкой, я не могу применить метод .groups.
Я не могу понять, что происходитпоскольку s на самом деле является строкой, /text() возвращает строку, я проверил тип, и он выглядит как str (также пытался сделать str(s), но тоже не работал).
Кроме того, если я ввожу строку в s, что-то вроде:
s = "P.when('A').register("ImageBlockATF", function(A){
var data = {
'colorImages': { 'initial': [{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX395_.jpg":[282,395],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX500_.jpg":[357,500],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX535_.jpg":[382,535],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX575_.jpg":[410,575],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX625_.jpg":[446,625],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX675_.jpg":[481,675],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX695_.jpg":[496,695]},"variant":"MAIN","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY395_.jpg":[395,249],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY500_.jpg":[500,316],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY535_.jpg":[535,338],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY575_.jpg":[575,363],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY625_.jpg":[625,395],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY675_.jpg":[675,426],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY695_.jpg":[695,439]},"variant":"FRNT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY395_.jpg":[395,301],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY500_.jpg":[500,381],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY535_.jpg":[535,408],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY575_.jpg":[575,438],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY625_.jpg":[625,477],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY675_.jpg":[675,515],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY695_.jpg":[695,530]},"variant":"BACK","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX395_.jpg":[146,395],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX500_.jpg":[185,500],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX535_.jpg":[198,535],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX575_.jpg":[213,575],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX625_.jpg":[231,625],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX675_.jpg":[250,675],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX695_.jpg":[257,695]},"variant":"BOTT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX395_.jpg":[139,395],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX500_.jpg":[177,500],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX535_.jpg":[189,535],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX575_.jpg":[203,575],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX625_.jpg":[221,625],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX675_.jpg":[238,675],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX695_.jpg":[245,695]},"variant":"TOPP","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX395_.jpg":[186,395],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX500_.jpg":[235,500],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX535_.jpg":[252,535],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX575_.jpg":[271,575],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX625_.jpg":[294,625],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX675_.jpg":[318,675],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX695_.jpg":[327,695]},"variant":"RGHT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX395_.jpg":[255,395],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX500_.jpg":[322,500],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX535_.jpg":[345,535],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX575_.jpg":[371,575],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX625_.jpg":[403,625],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX675_.jpg":[435,675],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX695_.jpg":[448,695]},"variant":"PAIR","lowRes":null}]},
'colorToAsin': {'initial': {}},
'holderRatio': 1.2,
'holderMaxHeight': 700,
'heroImage': {'initial': []},
'heroVideo': {'initial': []},
'spin360ColorData': {'initial': {}},
'spin360ColorEnabled': {'initial': 0},
'spin360ConfigEnabled': false,
'spin360LazyLoadEnabled': false,
'playVideoInImmersiveView':'false',
'tabbedImmersiveViewTreatment':'C',
'totalVideoCount':'0',
'videoIngressATFSlateThumbURL':'',
'mediaTypeCount':'0',
'atfEnhancedHoverOverlay' : true,
'winningAsin': 'B072596K2C',
'weblabs' : {},
'aibExp3Layout' : 1,
'aibRuleName' : 'frank-powered',
'acEnabled' : false
};
A.trigger('P.AboveTheFold'); // trigger ATF event.
return data;
});"
Работает просто отлично.
Отслеживание ошибок:
Traceback (most recent call last):
File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\utils\defer.py", line 102, in iter_errback
yield next(it)
File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\offsite.py", line 30, in process_spider_output
for x in result:
File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\referer.py", line 339, in <genexpr>
return (_set_referer(r) for r in result or ())
File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\depth.py", line 58, in <genexpr>
return (r for r in result or () if _filter(r))
File "C:\Users\Manuel\Desktop\scrapyProject\genericScraper\genericScraper\spiders\finalClothes_spider.py", line 59, in parse
data = m.group()
AttributeError: 'NoneType' object has no attribute 'group'
Любые идеио том, что происходит?