Перевести суп в Df

Question

import requests
from bs4 import BeautifulSoup
url ="https://priceapi.moneycontrol.com/pricefeed/nse/equitycash/RI" 
soup = BeautifulSoup(requests.get(url).content, 'html.parser')
soup

Привет, я получаю данные в супе, но не могу преобразовать данные в список или df

Answer 1

Когда вы открываете URL-адрес, он представляет собой структуру json. Итак, вы можете использовать метод .json() для объекта response

import requests
from bs4 import BeautifulSoup
url ="https://priceapi.moneycontrol.com/pricefeed/nse/equitycash/RI"
res = requests.get(url)

res_json = res.json()

Вывод:

{'code': '200',
 'message': 'Success',
 'data': {'HN': 'रिलायंस',
  'HP': '1884.60',
  'DISPID': 'RI',
  'cl5yPerChange': '278.4900',
  'BIDQ': '7406.00',
  'cl3mVal': '1177.98',
  'BIDP': '1878.05',
  'cl1mDt': '2020-06-11',
  'P_C': 31.25,
  'YR': '2020',
  'cl2yVal': '1016.07',
  'SC_SUBSEC': 'Refineries',
  'sc_ttm_cons': '58.2',
  'SC_TTM': '45.7',
  'clYtdVal': '1499.83',
  'cl6mDt': '2020-01-13',
  'cl1wVal': '1851.8',
  '0': '16:05:46',
  'cl1mChange': '340.35',
  'cl3mChange': '700.07',
  '150DayAvg': '1438.76',
  'clYtdChange': '378.22',
  'tot_buy_qty': 0,
  'clYtdDt': '2019-12-31',
  'Group': 'N',
  'ACCOD': '12150008',
  'cl3yDt': '2017-07-10',
  'cl3mDt': '2020-04-13',
  'NSEID': 'RELIANCE',
  '52H': '1884.60',
  'AvgDelVolPer_8day': '30.24',
  '52L': '867.43',
  'BV': '706.44',
  'MKTLOT': '1.00',
  'OFFERP': '0.00',
  'OFFERQ': '0.00',
  'cl6mVal': '1529.2',
  'cl1mVal': '1537.7',
  '5DayAvg': '1817.08',
  'AvgDelVolPer_3day': '27.26',
  'TID': '322',
  'SHRS': '6761844754',
  '50DayAvg': '1581.91',
  'cl3yChange': '1138.63',
  'im_scid': '',
  'DVolAvg10': '14691550.60',
  'LP': '1824.25',
  'lower_circuit_limit': '1641.85',
  'upper_circuit_limit': '2006.65',
  'ty': '1',
  'cl1wDt': '2020-07-06',
  'cl5yDt': '2015-07-10',
  'IND_PE': '36.51',
  'cl1yPerChange': '48.2500',
  'cl3yVal': '739.42',
  'OPN': '1828.50',
  'cl1mPerChange': '22.1300',
  'DTTIME': '7101605',
  'SC_FULLNM': 'Reliance Industries',
  'DVolAvg5': '16084636.20',
  'etf': 0,
  'clYtdPerChange': '25.2200',
  'isinid': 'INE002A01018',
  'cl2yDt': '2018-07-10',
  'cl5yVal': '496.2',
  'cl2yPerChange': '84.8300',
  'AvgDelVolPer_5day': '26.44',
  '200DayAvg': '1428.86',
  'VOL': '20195490',
  'exchange': 'N',
  'sc_mapindex': '22.0',
  'DVolAvg30': '17433509.37',
  'cl1wChange': '26.25',
  'AVGP': '1859.58',
  'LTH': '1884.60',
  'cl3yPerChange': '153.9900',
  'LTL': '30.90',
  'cl3mPerChange': '59.4300',
  '30DayAvg': '1665.23',
  'cl1yChange': '611.21',
  'cl2yChange': '861.98',
  'FV': '10.00',
  'MKTCAP': 1269908.25,
  'SMID': 'OG',
  'DIVPR': '65.00',
  'tot_sell_qty': 0,
  'DELV': '26.93',
  'PREVDATE': '2020-07-09',
  'cl1wPerChange': '1.4200',
  'lastupd': '2020-07-10 16:05:45',
  'SERIES': 'EQ',
  'BSEID': '500325',
  'GN': 'રિલાયંસ',
  'cl5yChange': '1381.85',
  'LDAYS': 0,
  'MKT_LOT': '505',
  'cl1yVal': '1266.84',
  'PE': 41.1,
  'cl6mPerChange': '22.8100',
  'TVOL': '0',
  'cl6mChange': '348.85',
  'cl1yDt': '2019-07-10',
  'DISPTYP': 'MAN',
  'CEPS': '72.91',
  'pricecurrent': '1878.05',
  'priceprevclose': '1824.25',
  'symbol': 'RI',
  'company': 'Reliance',
  'pricechange': '53.8000',
  'pricepercentchange': '2.9492',
  'lastupd_epoch': '1594377345'}}

Answer 2

import pandas as pd
import requests
from bs4 import BeautifulSoup

class_name_1 = "" # look at your website and decide which classes you want to scrape
class_name_2 = ""
url = ""
response = requests.get(url)
website = BeautifulSoup(response.content, "html.parser")

class_1_raw = website.find_all(attrs={"class" : class_name_1})
class_2_raw = website.find_all(attrs={"class" : class_name_2})

class_1_content = []
class_2_content = []

for element in class_1_raw:
    class_1_content.append(element.get_text())
for element in class_2_raw:
    class_2_content.append(element.get_text())

dict = {
    class_1_name : class_1_content,
    class_2_name : class_2_content
}
df = pd.DataFrame(data=dict)

Скажите, работает ли это.

Или, если вас не волнуют классы, это немного проще:

import pandas as pd
import requests
from bs4 import BeautifulSoup

url = ""
response = requests.get(url)
website = BeautifulSoup(response.content, "html.parser")

dict = {
    "content" : website.get_text()
}
df = pd.DataFrame(data=dict)

Надеюсь, что я смогу помочь!

Answer 3

Попробуйте что-нибудь подобное с pandas

    resp = requests.get(URL, headers=headers)
    if resp.status_code == 200:
        soup = BeautifulSoup(resp.content, "html.parser")
        results = []
        newline = '\n'
        for g in soup.find_all('span', class_="LrzXr"):
            x = f'{line}:{g.text}{newline}'
            results.append(x)
        #print(results)
        df.loc[query]=[query,results]
        print(df)
        
    df.to_excel("results.xlsx",sheet_name="result", index=False)