Я использую selenium и requests, чтобы получить данные с веб-сайта. Поскольку данные собираются запросом Ajex, я решил использовать requests для имитации почтового запроса. В этом случае необходимы некоторые параметры. Chrome результаты проверки выглядят следующим образом :
: авторитет: www.miaomiaozhe.com
: метод: POST
: путь: / api / pricehistory / getUrlPriceHistory
: scheme: https
accept: /
accept-encoding: gzip, deflate, br
accept- язык: zh-CN, zh; q = 0,9
длина содержимого: 3284
тип содержимого: application / x- www-form-urlencoded; charset = UTF-8
готовить ie: HZBID = db556a075167c32227609e6871911beb; gr_user_id = bd42c97e-3291-4b18-bc37-0c12b367967d; PHPSESSID = qsksp6v3t4m7880j0ptjshc7u7; MMZUFLAG = 104163326; MMZUSS = 3531fecdefc1b40c34c9b568cde18326; gr_session_id_b49be8fba0e9e60a = 74e18968-dad1-449d-b248-71c2b2d855cd; gr_session_id_b49be8fba0e9e60a_74e18968-dad1-449d-b248-71c2b2d855cd = true
происхождение: https://www.miaomiaozhe.com
референт: https://www.miaomiaozhe.com/pricehistory
* 10 * se c -fetch-dest: empty
se c -fetch-mode: cors
se c -fetch-site: same-origin
user-agent: Mozilla / 5.0 (Windows NT 10.0; Win64; x64) AppleWebKit / 537.36 (K HTML, как Gecko) Chrome / 81.0.4044.122 Safari / 537.36
x-запрошено-с : XMLHttpRequest Выше показаны заголовки запроса сообщения. и данные формы следующие: Как показано на рисунке, в сообщении есть параметр с именем nvcVal . проверив коды JS на сайте, я нашел его из nvcVal: window.getNVCVal(). поэтому я закодировал следующие коды, чтобы получить данные.
from time import sleep
import requests
import json
chrome_opt = webdriver.ChromeOptions()
prefs = {"profile.managed_default_content_settings.images": 1}
# chrome_opt.add_experimental_option("prefs", prefs)
# chrome_opt.add_experimental_option('excludeSwitches', ['enable-automation'])
chrome_opt.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36"')
driver = webdriver.Chrome(chrome_options=chrome_opt)
driver.get("https://www.miaomiaozhe.com/pricehistory")
# driver.find_element_by_class_name('pl').click()
good3 = "https://detail.tmall.com/item.htm?spm=a220m.1000858.1000725.1.e748471bZmg76Q&id=594572481181&skuId=4360012669001&areaId=420100&user_id=1809177149&cat_id=2&is_b=1&rn=04e4c83fcb68c340abf88983da82b1b7"
good2 = "https://detail.tmall.com/item.htm?spm=a220m.1000858.1000725.7.e748471bIRN6jQ&id=554640570620&skuId=4341085905022&areaId=420100&user_id=2928380385&cat_id=2&is_b=1&rn=6ba9712cf53485a574d73191e05d0cbf"
goodURL = "https://detail.tmall.com/item.htm?id=569343149791&spm=875.7931836/B.2017081.2.66144265gU7Lgd&scm=1007.12144.81309.73299_0_0&pvid=10ae9600-6e84-4ad0-8e1e-a8379cea71c3&utparam=%7B%22x_hestia_source%22:%2273299%22,%22x_object_type%22:%22item%22,%22x_mt%22:9,%22x_src%22:%2273299%22,%22x_pos%22:1,%22x_pvid%22:%2210ae9600-6e84-4ad0-8e1e-a8379cea71c3%22,%22x_object_id%22:569343149791%7D"
driver.find_element_by_id('ipt').send_keys(good3)
# driver.find_element_by_id('search-btn').click()
sleep(2)
js = " return window.getNVCVal()"
nvcVal = driver.execute_script(js)
print("nvcVal = "+str(nvcVal))
# new_nvcVal = "%7B%22a%22%3A%22FFFF0N0N000000006AFF%22%2C%22c%22%3A%221590723506842%3A0.99342660331515%22%2C%22d%22%3A%22nvc_activity%22%2C%22h%22%3A%7B%22key1%22%3A%22code0%22%2C%22nvcCode%22%3A400%2C%22umidToken%22%3A%22T21B886C0F961A98E862F5CF69393964795B83A19E0D643903B04E06AD6%22%7D%2C%22j%22%3A%7B%22test%22%3A1%7D%2C%22b%22%3A%22124%23YhFPnxFZxG0xA7yf6JMuAlDS%2BUOvpV9oiw8W9aZ3qg%2FOqdJA%2Fe%2BbSWcxIK5G%2FETKzoSdFuvXCK%2BnhxCNTe1hYMCi6JdG32LOCkudgS9CR6yNrGk8moL4XOOzVHI8Xm%2Bl0i8EJrE8p1Rc0hsLv%2Fef58%2FZCD0mxKtronkZ6a84khvKaJtwZvIENL4eTBPXNywFpTcpLoDJy3LZLgR1UkkoaT%2F%2Ba8aXsPY73H92xbk3epwEzEoezisui2xzIi7Ji20WFFfm7s52HyARgOzgfXs5kTtIQlcj%2BJYcpA%2BuZhEUxy2FqtOoHCaDvewvQq7Au3GkZYB6cRQw49aAo74C1CnfbvxXu4B8HbA2qfVXWCi4o7Cq%2FhFzk0MQ%2FVAzymPxXosxxs6iPH16Fq4vBjKytiZUYXg8ulppFkSfMBefa4ttv5xVfs%2BunR4vMk1QvLpAw7AX8JzoDY%2FJtMdpfnsgCIWIYCXyxpJ%2FfWj78qYBATgePa4OygNq8kcFKbnNrw%2BW2mWo1ecP2im11%2B4Kf9%2B2WmgDL%2B9R1hzZFS2Q5omZWbBAgp6FhLBo4QZwQv1bGPvDEA7omMItp%2FswEuRvUyumFjivx0GbxQHya%2BpKVbtFYfR%2Bw7Ktzj2OH%2BrQG0Jnnn9QgLj3nl24GohovAeEHmqJvhWREtQzoZKi7FrAmgXmxbwm9f3l%2B%2Fmj8RY%2Bxm2SyXIMWwXa%2Ff9lnDc5m4onufT%2Fefyg2ayAO%2Fi2%2BoVnjUK%2BptEeFgl%2B3OKWidZn%2BnJIVk6MAwTggk3bbWQ42SuPaodKeaVJOZ%2FOAiQPFQYGtIBzCNlUJN7cuBSOLSl44atsCjx%2BA5TFfsWXdKBgJj6AO1hPm19b%2FkCMkXxliO6z5ggczqjEDYlco4xoa3hbSyF47kC4aOrIvUY45OfmYAAB2UcL89J9xbHdQpqCXKkegnfUxxLT6DABlRZUG046XrcB5UlAzQVE%2BxPfTn8YOl6yXa0GYC%2FeaDhGTmuVV5DYf6r3M%2BRYH%2BQOt3%2BDLUHqs%2FRZLl6kM62Bcx%2FCtINSnFE5MDRji2u%2F8pNP6qT9tcSJ7Jpt7d%2BvHdL4EAL7w8PIkKNKCZbsO6YF7CwxRJxzk7Xay5qeX5f%2BshvHw%2B%2F8JIQZ%2Bb8hXp6vBNNdpb3mzJkkHLsySik7Afy58evmjzCC1C8Nsx3GWrcZ9NQ%2FWTVMKR3vxWwYYFFcuq3OzI7AGULiChCL94JF%2F%2BIQlN931yOF0iFGgmArillK9FpEnDuGMnHFhVLp0HtWDUtpLk%2BTZ9pW8OUVb4ZDCP9qJPIlBGEiW23%2BxmAw%2FCtpgWt12%2FuePq%2FM73jgfkT7B5raQDBlEBHmxrdrfXwOAPKcL8Sy%2FKMaIta8jsDsEpi6VSO7cCIUukxXJu0v%2FBlyQSMdOOfjlDcPf8bJeMWeA%2Bh1QVyiXWNW5xBEiJFmbwDojakisamVHkef9I%2BhNH6JZrPBPiYtWLBCYWPBkp%2FuI8ccFLWpGnDzEvEhwg0OzpUK%2FTkWYOX6%2Fc82Kf%2BbDDYW8BGyTdce%2FKSOxDev9janT6tK6m1H7Xx195A0BVXNQ4HF5WIWHS9ds9MELSkUQx1gYZ51eSbFK2z5PghURKkNX2GPTMbkkEwaOZCU48hinHKXMkkxq4qaO6l12TwCfDZjMa%2F0OWF%2Fdhwi8CBXLpD2yzqRVbstHRh8SONspbQzOV5uSg4CFDk1oPsNMwWfofeRrucTkuop%2BOPDRIzI2EvSuxPwvLM2FeVkLhRcduEk82b0QkZQNjBxryNJ9UCQTzXxsnKzQ8q%2B5t8wftsFu0pw1zk5gMrNQVfMPZcNGgOetPlv9X2e9Y34FqGGtduLySo9jJ%2F7Q4MEESg5AaNtGjmc5vr4QW0%2FcyKXipTs1j6MLc3lPZ9VK4%2B0gNFxuhLWREhcBq6uT7SDzKkHN%2BEDgvtW3M02%2FtjYRYZywdDdYsEhjmmAxSvNoDzn38wB%2Fe0TCw9HSikNPepnj7ngEnNHao6SRYASV%2FikO0H0InYLbwYIm4WZIqXp6TS4PiZZwE%2F2g7OHJ6Yplmi19jrUJ8Lpg91d1ZIZbUung7OBInYLlwYnmfWeIqsXgTzo1nIelUX2g7vtIn%2FplwDzm4%2BZI8LLgTGb7Z%2B9yq%2F2DbOBIZzj6WhNcSCzI8XQcrFo1n%2FGxCnnehIcDCsVVCiWdJIQ9ijF01GTKDBYugeql%2FnVhY%2F5jjDKyxvUoVqbFcNM4HhgDtI79Ih5i9upBgavNfEb8yzXblH6gQmMrSzjS7didriS4q8DffZ8JyM3SLSxvXuljBztEtxqUkbXFTI44Jg3fQL4AgAKAuC%2Ftv7vNFGHHz6NZeCs5idxQleLgz5EmqrXWboLBwppQcWkXFh35VFqXyC0JfGIeE743B9stqjO5H5jbyD4qy1vsISbm0G0LzGrXkZUgHBThjBRinW78jnWKYdlHB%2F5AR2IZG1XdV6nIoj10twUTIdLuZ0fR6JsbLZXW%2Fn%2F%2FbPv6O9yfqz4Co2JjPaIL8NTwX9vxnpi%2Fg8i8EhXB1dy6iJwT7btl1AmVWx4dnsCHOcHjcTKAd%2BwvYEWJavJIvCI8FcfqStlLbJDoHl2bOo9W6UedB35e5V9WpyHrqy8pSkfuRzm9T5hlgwjEFB6%2FzgFbyADaV0Z9JP3I3Q7F%2BsWzUEBGmKbC3LsuZ2khjquN2Be9BJ%3D%22%2C%22e%22%3A%22KfLnQbaexlHsDdRdQx0L4Q0fMePRlUSPsuteAtUngD99tuEUdiugeouLEJG0uvIa-9Z-k-lQ6fL3lpbcTxPN4fk9Wu-Q71ExCDYni5eDYwVSG_W5NXXiw0SnMK_GknTTTFhOiVDaaNOdAoCLDoRnw7GQkCz_wBSon-RNF-5mxkqAc4zx0vPSPsaMl7xGzFOWYizM9JlKkpiqJYbEB2zQGA%22%7D"
# res = driver.execute_script("return " + ajax_query)
url = "https://www.miaomiaozhe.com/api/pricehistory/getUrlPriceHistory"
data = {'url': good3, 'nvcVal': nvcVal}
header = {"authority": "www.miaomiaozhe.com", "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36"}
# cookie = {"HZBID":"db556a075167c32227609e6871911beb", "gr_user_id":"bd42c97e-3291-4b18-bc37-0c12b367967d", "PHPSESSID":"qsksp6v3t4m7880j0ptjshc7u", "MMZUFLAG":"104163326", "MMZUSS":"3531fecdefc1b40c34c9b568cde18326", "gr_session_id_b49be8fba0e9e60a":"c1c318c3-8db5-4bbd-8f78-e893b0dea1ec", "gr_session_id_b49be8fba0e9e60a_c1c318c3-8db5-4bbd-8f78-e893b0dea1ec":"true"}
req = requests.post(url, data, headers=header)
print(req.text)
driver.close()'''
I failed, unawaring what's been wrong.
I guess it could be the problem of the parameter of nvcVal, but I don't have any idea how I can get it right.