Я использую python запросы и bs4 для очистки сайта, но у меня возникли проблемы с декодированием (я думаю ..)
logurl = 'https://login.flash.co.za/apex/f?p=pwfone:login'
with requests.Session() as s:
s.headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36',
"accept-encoding": "gzip, deflate",
}
response = s.get(logurl)
response.encoding = 'utf-8'
print response.status_code
# return 200
print (response.content)
# returns: b'<html lang="en-us" xmlns:htmldb="http://htmldb.oracle.com">\r\n<head>\r\n<meta http-equiv="x-ua-compatible" content="IE=edge" />\r\n<title>Pay with Flash Login</title>\r\n<link rel="stylesheet" href="/i/themes/SPhone/payback/theme_3_1.css" type="text/css" />\r\n\r\n<!--[if IE]><link rel="stylesheet" href="/i/themes/SPhone/payback/ie.css" type="text/css" /><![endif]-->\r\n<link rel="stylesheet" href="/i/app_ui/css/Core.min.css?v=19.1.0.00.15" type="text/css" />\n<link rel="stylesheet" href="/i/app_ui/css/Theme-Standard.min.css?v=19.1.0.00.15" type="text/css" />\n<link rel="stylesheet" href="/i/libraries/jquery-ui/1.12.1/jquery-ui-apex.min.css?v=19.1.0.00.15" type="text/css" />\n\r\n<link rel="stylesheet" href="/i/legacy_ui/css/5.0.min.css?v=19.1.0.00.15" type="text/css" />\n\r\n\r\n\r\n\r\n\r\n\n<script>\nvar apex_img_dir = "/i/", htmldb_Img_Dir = apex_img_dir;\n</script>\n<script src="/i/libraries/apex/minified/desktop_all.min.js?v=19.1.0.00.15"></script>\n<script src="wwv_flow.js_messages?p_app_id=1500&p_lang=en-us&p_version=19.1.0.00.15-94050580561"></script>\n<script src="/i/libraries/apex/minified/legacy_pre18.min.js?v=19.1.0.00.15"></script>\n<script src="/i/libraries/apex/minified/legacy_18.min.js?v=19.1.0.00.15"></script>\n<script src="/i/libraries/jquery-migrate/3.0.1/jquery-migrate-3.0.1.min.js?v=19.1.0.00.15"></script>\n\r\n\r\n\r\n\r\n\r\n\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\n\r\n</head>\r\n<body ><form action="wwv_flow.accept" method="post" name="wwv_flow" id="wwvFlowForm" novalidate autocomplete="off">\n<input type="hidden" name="p_flow_id" value="1500" id="pFlowId" /><input type="hidden" name="p_flow_step_id" value="101" id="pFlowStepId" /><input type="hidden" name="p_instance" value="5488556618334" id="pInstance" /><input type="hidden" name="p_page_submission_id" value="119891457853323246540979814026507745288" id="pPageSubmissionId" /><input type="hidden" name="p_request" value="" id="pRequest" /><input type="hidden" name="p_reload_on_submit" value="A" id="pReloadOnSubmit" /><input type="hidden" value="119891457853323246540979814026507745288" id="pSalt" /><table border="0" cellpadding="0" cellspacing="0" summary="" id="t18PageBodyHead" width="80%" height="10%" align="center">\r\n<tr>\r\n<td align="center" id="t18Logo" valign="top"><br /></td>\r\n<td id="t18HeaderMiddle" valign="top" width="100%"><br /></td>\r\n<td id="t18NavBar" valign="top"><br /><br /></td>\r\n</tr>\r\n</table>\r\n<table border="0" cellpadding="0" cellspacing="0" summary="" id="t18PageBody" width="80%" height="70%" align="center">\r\n<tr id="t18tabscolor">\r\n<td valign="top" id="t18Tabs" width="110%"></td></tr>\r\n<tr>\r\n<td valign="top" id="t18Tabs"></td></tr>\r\n<tr>\r\n<td valign="top"></td>\r\n<td class="t18PageRight"><br /></td>\r\n</tr>\r\n<td valign="top" id="t18ContentBody" height="100%" width="100%">\r\n<div id="t18Messages"><span id="APEX_SUCCESS_MESSAGE" data-template-id="165783099126927032_S" class="apex-page-success u-hidden"></span><span id="APEX_ERROR_MESSAGE" data-template-id="165783099126927032_E" class="apex-page-error u-hidden"></span></div>\r\n<div id="t18ContentMiddle"><table id="apex_layout_83448022784534430" border="0" class="formlayout" role="presentation"><tr><td align="right"><label for="P101_USERNAME" id="P101_USERNAME_LABEL" tabindex="999"><a class="t18OptionalLabelwithHelp" href="javascript:popupFieldHelp(\'83448127760534444\',\'5488556618334\')" tabindex="999">Username</a></label></td><td colspan="2" align="left"><input type="text" id="P101_USERNAME" name="P101_USERNAME" class="text_field apex-item-text" value="" size="40" maxlength="100" /></td></tr><tr><td align="right"><label for="P101_PASSWORD" id="P101_PASSWORD_LABEL" tabindex="999"><a class="t18OptionalLabelwithHelp" href="javascript:popupFieldHelp(\'83448212820534447\',\'5488556618334\')" tabindex="999">Password</a></label></td><td align="left"><input type="password" name="P101_PASSWORD" size="40" maxlength="100" value="" id="P101_PASSWORD" class="password apex-item-text" onkeypress="return apex.submit({request:\'P101_PASSWORD\',submitIfEnter:event})" /></td></tr><tr><td align="left"><table class="t18ButtonAlternative1" cellspacing="0" cellpadding="0" border="0" summary="">\n<tr>\n<td class="t18L"><img src="/i/themes/theme_18/button_alt1_l.gif" alt="" width="11" height="18" /></td>\n<td class="t18C"><a href="javascript:apex.submit(%7Brequest:'LOGIN'%7D);">Login</a></td>\n<td class="t18R"><img src="/i/themes/theme_18/button_alt1_r.gif" alt="" width="11" height="18" /></td>\n</tr>\n</table> </td></tr></table></div>\r\n</td>\r\n<tr>\r\n<td valign="top" align="left" ><div></div><div><table border="0" cellpadding="0" cellspacing="0" summary="" id="t18PageFooter" width="100%" style="clear:both">\r\n<tr>\r\n<td id="t18Left" valign="top"></td>\r\n<td id="t18Center" valign="top"></td>\r\n <div class="coronaBanner">\r\n <div class="coronaBanner__content">\r\n <strong>HAVE QUESTIONS ABOUT CORONAVIRUS? VISIT SOUTH AFRICA\xe2\x80\x99S OFFICIAL COVID-19 RESOURCE HERE:<br>\r\n <a href="https://sacoronavirus.co.za/">https://sacoronavirus.co.za/</a></strong><br><br>\r\n <a class="coronaBanner__websiteLink" href="https://sacoronavirus.co.za/" rel="noopener nofollow" title="SAcoronavirus.co.za">\r\n <img class="coronaBanner__websiteLinkImg" border="0" />\r\n </a><br><br>\r\n <a class="coronaBanner__hotlineLink" href="tel:+27800029999">\r\n <strong>Emergency Hotline: 0800 029 999</strong>\r\n </a><br>\r\n <a class="coronaBanner__whatsappLink" href="https://wa.me/27600123456?text=Hi" rel="noopener nofollow">\r\n <strong>WhatsApp Support Line: 0600-123456</strong>\r\n </a>\r\n </div>\r\n</div>\r\n<td id="t18Right" valign="top"><span id="t18Customize"></span><br /></td>\r\n</tr>\r\n</table></div></td>\r\n<td class="t18PageRight"><br /></td>\r\n</tr>\r\n<tr>\r\n</tr>\r\n</table>\r\n<br class="t18Break"/>\r\n<input type="hidden" id="pPageFormRegionChecksums" value="[]">\n<input type="hidden" id="pPageItemsRowVersion" value="" /><input type="hidden" id="pPageItemsProtected" value="CpJ-L5-9OTxTAYpkj4TifA" /></form> \r\n\r\n\r\n\n\n\n\n\n<script type="text/javascript">\napex.jQuery( function() {\napex.page.init( this, function() {\napex.jQuery.when.apply( apex.jQuery, apex.page.loadingDeferreds ).done(function() {\ntry {\n(function(){\nvar lTimeoutField = document.getElementById("apex_login_throttle_sec"),\n lTimeout = lTimeoutField ? +lTimeoutField.innerHTML : 0;\nif (lTimeout) {\n var lTimer = window.setInterval (\n function() {\n if (lTimeout > 0) {\n lTimeoutField.innerHTML = lTimeout;\n lTimeout--;\n } else {\n window.clearInterval(lTimer);\n var lDiv = document.getElementById("apex_login_throttle_div");\n if (lDiv) {\n lDiv.parentNode.removeChild(lDiv);\n return true;\n }\n }\n },\n 1000 );\n}})();\n\n\napex.item( \'P101_USERNAME\' ).setFocus();\n} finally {\napex.event.trigger(apex.gPageContext$,\'apexreadyend\');\n}\n});\n});\n});</script>\n\r\n</body>\r\n</html>'
print (response.text)
#return nothing
soup = BeautifulSoup(response.content ,"html5lib") #returns nothing
soup = BeautifulSoup(response.text ,"html5lib") #returns nothing
soup = BeautifulSoup(response.content ,"html.parser") #returns nothing
soup = BeautifulSoup(response.text ,"html.parser") #returns nothing
Странно то, что это прекрасно работает в моей локальной среде ( python 3.7 и даже python 2.7), но не на сервере, который я использую, который использует python 3.5.
Как правильно декодировать ответ для bs4?