Это то, что я думал в комментарии, но это всего лишь демонстрация.Помимо multiprocessing
, threading
вы можете попробовать concurrent.futures
.
from selenium import webdriver
import requests
from multiprocessing import Pool
driver = webdriver.Chrome()
clist = {}
#login to website using selenium and get cookies
def relogin(user,pwd):
...
cookievar = driver.get_cookies()
clist[user] = cookievar
...
driver.delete_all_cookies()
userlist = [("userA","pwdA"),("userB","pwdB")]
for user,pwd in userlist:
relogin(user,pwd)
driver.close()
#send requests using cookies scraped from selenium webdriver
def post_data(url,formData,proxies,headers = headers0):
r = post(url, formData, headers=headers, proxies=proxies, verify=False)
...
def start(number=len(userlist)):
p = Pool(number)
result = p.map_async(post_data , [(url,formData,proxies),
(url2,formData2,proxies2,headers2)])
print(result.get())
if __name__ == '__main__':
start()