как найти элементы в одном содержимом таблицы сайта

Question

Я пытаюсь почистить детали товаров на сайте ниже, но скрипт всегда терпит неудачу с ошибкой no such element, хотя элемент есть. Может кто-нибудь может помочь решить проблему? Мой код:

from time import sleep

from scrapy import Spider
from selenium import webdriver
from scrapy.selector import Selector
from scrapy.http import Request
from selenium.common.exceptions import NoSuchElementException
driver = webdriver.Chrome('D:\chromedriver_win32\chromedriver.exe')
driver.get('http://www.tesensors.com/global/en/product/inductive-capacitive/xs-xt-ref')
sleep(5)
#soemtime the site ask you select language and country so need click button as below
sign_in_button = driver.find_element_by_id('edit-submit--4')
sign_in_button.click()
sleep(5)
# scrapy content.total 1168 items, here there is no result.
product_model_name=driver.find_elements_by_xpath('span[@itemprop="name"]')
product_desc=driver.find_elements_by_xpath('span[@itemprop="description"]')

Answer 1

Данные о продукте внутри iframe

Вы можете использовать XPath для поиска:

iframe = driver.find_element_by_xpath("//iframe[@id='ecat']")

Затем переключитесь на:

driver.switch_to.frame(iframe)

Вот как переключиться обратно на содержимое по умолчанию (из):

driver.switch_to.default_content()

Не использовать time-sleep модуль, попробуйте явное ожидание .

см. разница.

EX:

from scrapy import Spider
from selenium import webdriver
from scrapy.selector import Selector
from scrapy.http import Request
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC


driver = webdriver.Chrome('D:\chromedriver_win32\chromedriver.exe')
driver.get('http://www.tesensors.com/global/en/product/inductive-capacitive/xs-xt-ref')

#soemtime the site ask you select language and country so need click button as below
sign_in_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "edit-submit--4")))
sign_in_button.click()

#switch iframe
iframe = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//iframe[@id='ecat']")))
driver.switch_to.frame(iframe)

# scrapy content.total 1168 items, here there is no result.
product_model_name = driver.find_elements_by_xpath('//span[@itemprop="name"]')
print(product_model_name[0].text)

product_desc=driver.find_elements_by_xpath('//span[@itemprop="description"]')

print(product_model_name[0].text)

Answer 2

Я использовал этот метод для получения элементов:

from scrapy import Spider
import os
from selenium import webdriver
import time
from scrapy.selector import Selector
from scrapy.http import Request
from selenium.common.exceptions import NoSuchElementException
chromedriver = pathToDriver + 'chromedriver'
os.environ["webdriver.chrome.driver"] = chromedriver
driver = webdriver.Chrome(chromedriver)
driver.get('http://www.tesensors.com/global/en/product/inductive-capacitive/xs-xt-ref')
time.sleep(3)
#soemtime the site ask you select language and country so need click button as below
sign_in_button = driver.find_element_by_id('edit-submit--4')
sign_in_button.click()
time.sleep(3)
iframe_src = driver.find_element_by_id('ecat').get_attribute("src")
print(iframe_src)
driver.get(iframe_src)
# scrapy content.total 1168 items, here there is no result.
product_model_names=driver.find_elements_by_class_name('boldLevel2')
product_names = list()
for element in product_model_names:
    product_names.append(element.text)
print(product_names)

product_desc=driver.find_elements_by_class_name('level1')
product_descptions = list()
for element in product_desc:
    product_descptions.append(element.text)
print(product_descptions)

driver.close()

Answer 3

import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By



driver = webdriver.Chrome(r"C:\Users\path\Desktop\chromedriver\chromedriver.exe")
driver.get('http://www.tesensors.com/global/en/product/inductive-capacitive/xs-xt-ref')

try:
    element = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.ID, "edit-submit--4")))
    element.submit()
except:
    print("proceeding further")

iframe = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//iframe[@id='ecat']")))
driver.switch_to.frame(iframe)
product_model_name = driver.find_elements_by_xpath("//*[@itemprop='name']")
product_model_description = driver.find_elements_by_xpath("//*[@itemprop='description']")
names = []
description = []
for i in product_model_name:
    print(i.text)
    names.append(i.text)
for i in product_model_description:
    print(i.text)
    description.append(i.text)