Как сделать так, чтобы этот код bs4 webscraping перемещался по таблице и сохранял строки в 2d массиве? - PullRequest
Новая
       27

Как сделать так, чтобы этот код bs4 webscraping перемещался по таблице и сохранял строки в 2d массиве?

0 голосов
/ 05 июля 2019

Я создаю веб-скребок, который отслеживает изменяющийся список с использованием BS, html-теги для объектов, которые я ищу, являются общими, за исключением их идентификаторов, которые являются уникальными и постоянно меняются.Я знаю, что верхний идентификатор всегда будет одинаковым, поэтому я дошел до того, что мой вывод дает мне лучший результат в нужном мне формате, но я пытаюсь найти способ добавления следующих девяти.Я не могу использовать их идентификаторы, потому что они меняются, поэтому я подумал об использовании .find_next('tr'), но я не могу понять, как его получить за секунду.Я знаю, что должно быть элегантное решение, но я впервые использую BS4, так что я надеялся, что кто-то может указать мне правильное направление.

import requests
from bs4 import BeautifulSoup
from numpy import np

website_url = requests.get ('').text
soup = BeautifulSoup(website_url, 'lxml')
L = []
H = ["H1","H2","H3"]

for derp in soup.find(id='tr-id-1').findAll('a')[0:3:1]:
    L.append(derp.string)
A = np.vstack((H, L))

print(A)

Это возвращает мне распечатанный массив в правильном формате, но только для идентификатора, который я ввел в находке.Я могу получить второй ряд, написав -

for derp in soup.find(id='tr-id-1').find_next('tr').findAll('a')[0:3:1]:

- но я не знаю, как пройти дальше.Я только пытаюсь очистить первые 10 строк таблицы, поэтому я думаю, что мне может понадобиться цикл while с маркером обратного отсчета?Мне интересно, есть ли способ создать цикл, который выборочно берет следующие 9 строк и добавляет определенные данные столбца в массив.

Ответы [ 2 ]

2 голосов
/ 06 июля 2019

Этот скрипт печатает таблицу с валютами (вы можете хранить данные в списке или вместо них): 

import requests
from bs4 import BeautifulSoup

url = 'https://coinmarketcap.com'

soup = BeautifulSoup(requests.get(url).text, 'lxml')

for tr in soup.select('#currencies tr'):
    if not tr.select('td'):
        continue

    for i, td in enumerate(tr.select('td')[:-2]):
        txt = td.text.replace('\n', ' ').replace('*', '').strip()
        if i == 0:
            print('{: ^4}'.format(txt), end='|')
        else:
            print('{: ^24}'.format(txt), end='|')
    print()

Печать: 

 1  |      BTC  Bitcoin      |    $196,174,869,053    |       $11020.77        |    $24,501,665,241     |     17,800,475 BTC     |         -5.84%         |
 2  |     ETH  Ethereum      |    $30,603,567,177     |        $286.61         |     $8,821,119,760     |    106,776,759 ETH     |         -2.40%         |
 3  |        XRP  XRP        |    $16,148,857,177     |       $0.379379        |     $1,335,082,415     |   42,566,596,173 XRP   |         -4.06%         |
 4  |     LTC  Litecoin      |     $7,401,989,981     |        $118.37         |     $4,167,212,036     |     62,533,191 LTC     |         -4.15%         |
 5  |   BCH  Bitcoin Cash    |     $7,133,878,965     |        $399.09         |     $1,770,785,779     |     17,875,463 BCH     |         -3.87%         |
 6  |        EOS  EOS        |     $5,292,523,634     |         $5.74          |     $2,181,469,151     |    921,990,507 EOS     |         -2.58%         |
 7  |   BNB  Binance Coin    |     $4,621,088,383     |         $32.73         |      $267,344,456      |    141,175,490 BNB     |         -1.87%         |
 8  |      USDT  Tether      |     $3,684,665,566     |       $0.999098        |    $23,550,580,244     |   3,687,991,972 USDT   |         -0.53%         |
 9  |    BSV  Bitcoin SV     |     $3,534,271,930     |        $197.94         |      $351,461,514      |     17,854,986 BSV     |         -1.55%         |
 10 |       TRX  TRON        |     $2,113,478,617     |       $0.031695        |      $803,645,870      |   66,682,072,191 TRX   |         -0.96%         |
 11 |      ADA  Cardano      |     $1,981,827,482     |       $0.076439        |      $119,258,825      |   25,927,070,538 ADA   |         -3.69%         |
 12 |      XLM  Stellar      |     $1,940,474,350     |       $0.099896        |      $358,001,782      |   19,425,036,996 XLM   |         -2.60%         |
 13 |   LEO  UNUS SED LEO    |     $1,749,404,864     |         $1.75          |      $12,215,975       |    999,498,893 LEO     |         -2.01%         |
 14 |      XMR  Monero       |     $1,552,808,370     |         $90.96         |      $127,576,800      |     17,070,711 XMR     |         -0.03%         |
 15 |       DASH  Dash       |     $1,360,432,697     |        $152.74         |      $254,426,418      |     8,906,619 DASH     |         -4.60%         |
 16 |    LINK  Chainlink     |     $1,248,540,238     |         $3.57          |      $188,091,151      |    350,000,000 LINK    |         4.24%          |
 17 |        NEO  NEO        |     $1,191,827,236     |         $16.90         |      $490,262,022      |     70,538,831 NEO     |         -4.08%         |
 18 |      MIOTA  IOTA       |     $1,069,692,929     |       $0.384847        |      $20,590,490       |  2,779,530,283 MIOTA   |         -2.74%         |
 19 |      ATOM  Cosmos      |     $1,021,900,211     |         $5.36          |      $63,724,815       |    190,688,439 ATOM    |         -4.31%         |
 20 | ETC  Ethereum Classic  |      $872,993,215      |         $7.81          |      $751,025,201      |    111,727,165 ETC     |         -2.11%         |
 21 |       XTZ  Tezos       |      $817,988,097      |         $1.24          |       $7,008,121       |    658,849,612 XTZ     |         -1.62%         |
 22 |        XEM  NEM        |      $807,925,560      |       $0.089770        |      $24,960,771       |   8,999,999,999 XEM    |         -2.04%         |
 23 |       ZEC  Zcash       |      $700,497,262      |        $101.45         |      $281,578,113      |     6,905,119 ZEC      |         -1.92%         |
 24 |     ONT  Ontology      |      $675,289,519      |         $1.36          |      $133,352,633      |    494,757,215 ONT     |         -4.79%         |
 25 |       MKR  Maker       |      $655,751,917      |        $655.75         |       $1,156,367       |     1,000,000 MKR      |         1.59%          |
 26 | CRO  Crypto.com Chain  |      $552,054,533      |       $0.071538        |       $3,056,529       |   7,716,894,977 CRO    |         -2.14%         |
 27 |   BTG  Bitcoin Gold    |      $460,804,983      |         $26.31         |      $11,765,404       |     17,513,924 BTG     |         -5.22%         |
 28 |       QTUM  Qtum       |      $456,630,457      |         $4.76          |      $303,751,103      |    95,845,424 QTUM     |         -6.03%         |
 29 |     DOGE  Dogecoin     |      $452,796,907      |       $0.003766        |      $160,357,833      |  120,219,215,287 DOGE  |         13.34%         |
 30 |      VET  VeChain      |      $416,649,897      |       $0.007513        |      $57,432,988       |   55,454,734,800 VET   |         0.22%          |
 31 | BAT  Basic Attenti...  |      $372,192,333      |       $0.292373        |      $28,777,633       |   1,273,006,300 BAT    |         -1.68%         |
 32 |     USDC  USD Coin     |      $366,029,067      |       $0.997092        |      $110,990,052      |    367,096,485 USDC    |         -0.35%         |
 33 |      OMG  OmiseGO      |      $323,389,435      |         $2.31          |      $102,874,355      |    140,245,398 OMG     |         -4.46%         |
 34 |    VSYS  V Systems     |      $312,745,092      |       $0.178751        |      $10,413,916       |   1,749,608,504 VSYS   |         -2.14%         |
 35 |      DCR  Decred       |      $297,378,169      |         $29.63         |       $1,739,049       |     10,037,096 DCR     |         -6.28%         |
 36 |    BTT  BitTorrent     |      $277,023,930      |       $0.001306        |      $56,422,080       |  212,116,500,000 BTT   |         -0.66%         |
 37 |       HOT  Holo        |      $229,759,018      |       $0.001725        |      $25,162,937       |  133,214,575,156 HOT   |         0.21%          |
 38 |      EGT  Egretia      |      $222,938,874      |       $0.052953        |      $39,938,247       |   4,210,121,792 EGT    |         2.24%          |
 39 |     TUSD  TrueUSD      |      $213,775,752      |       $0.989291        |      $131,504,347      |    216,089,898 TUSD    |         -1.22%         |
 40 |     HC  HyperCash      |      $208,038,166      |         $4.78          |      $13,740,143       |     43,529,781 HC      |         -5.15%         |
 41 |  BCD  Bitcoin Diamond  |      $202,441,610      |         $1.09          |       $3,011,645       |    186,492,898 BCD     |         -2.68%         |
 42 |     RVN  Ravencoin     |      $199,913,461      |       $0.051124        |      $15,906,528       |   3,910,345,000 RVN    |         -4.85%         |
 43 |    HEDG  HedgeTrade    |      $199,512,069      |       $0.691805        |       $1,395,797       |    288,393,355 HEDG    |         -4.07%         |
 44 |    HT  Huobi Token     |      $199,033,233      |         $3.98          |      $96,051,667       |     50,000,200 HT      |         -1.05%         |
 45 |      AOA  Aurora       |      $196,149,743      |       $0.029982        |      $10,250,381       |   6,542,330,148 AOA    |         3.98%          |
 46 |       LSK  Lisk        |      $195,904,100      |         $1.66          |       $8,908,589       |    118,280,370 LSK     |         -4.19%         |
 47 |     NPXS  Pundi X      |      $193,713,239      |       $0.000815        |       $4,963,859       |  237,816,087,583 NPXS  |         -3.55%         |
 48 |      KMD  Komodo       |      $188,203,691      |         $1.64          |      $12,446,638       |    114,883,815 KMD     |         5.35%          |
 49 |       BTM  Bytom       |      $188,040,836      |       $0.187572        |      $54,667,305       |   1,002,499,275 BTM    |         11.50%         |
 50 |      WAVES  Waves      |      $177,993,586      |         $1.78          |      $12,125,048       |   100,000,000 WAVES    |         -5.44%         |
 51 |        ZRX  0x         |      $171,782,082      |       $0.287372        |      $13,858,310       |    597,769,457 ZRX     |         -2.12%         |
 52 |     QBIT  Qubitica     |      $169,037,263      |         $60.18         |        $56,989         |     2,808,628 QBIT     |         -2.17%         |
 53 |     BTS  BitShares     |      $163,390,853      |       $0.059810        |       $3,167,151       |   2,731,850,000 BTS    |         0.32%          |
 54 | PAX  Paxos Standar...  |      $162,646,375      |       $0.997482        |      $136,732,457      |    163,056,875 PAX     |         -0.43%         |
 55 |       NANO  Nano       |      $161,436,979      |         $1.21          |      $15,637,856       |    133,248,297 NANO    |         -1.85%         |
 56 |     BCN  Bytecoin      |      $159,764,211      |       $0.000868        |        $116,637        |  184,066,828,814 BCN   |         -6.31%         |
 57 |       REP  Augur       |      $156,392,893      |         $14.22         |       $5,068,790       |     11,000,000 REP     |         -3.05%         |
 58 |      NRG  Energi       |      $156,304,538      |         $8.70          |       $1,129,572       |     17,972,740 NRG     |         -1.29%         |
 59 |     MONA  MonaCoin     |      $152,581,722      |         $2.32          |       $6,087,802       |    65,729,675 MONA     |         -3.20%         |
 60 |     THR  ThoreCoin     |      $148,663,548      |        $1714.97        |        $179,021        |       86,686 THR       |         -5.74%         |
 61 |       IOST  IOST       |      $146,115,831      |       $0.012162        |      $31,901,773       |  12,013,965,609 IOST   |         -2.13%         |
 62 |       ICX  ICON        |      $142,531,341      |       $0.301076        |      $10,890,859       |    473,406,688 ICX     |         -4.32%         |
 63 |     DGB  DigiByte      |      $138,427,137      |       $0.011541        |       $1,186,651       |   11,994,056,188 DGB   |         -5.09%         |
 64 |      ZIL  Zilliqa      |      $136,925,993      |       $0.015762        |      $13,494,196       |   8,687,360,058 ZIL    |         -2.40%         |
 65 |   KCS  KuCoin Shares   |      $129,815,988      |         $1.45          |      $26,079,874       |     89,659,415 KCS     |         -2.98%         |
 66 |      LAMB  Lambda      |      $125,711,992      |       $0.251424        |      $36,282,311       |    500,000,000 LAMB    |         -1.68%         |
 67 |       XIN  Mixin       |      $125,658,873      |        $277.73         |        $887,298        |      452,447 XIN       |         -5.58%         |
 68 |    ABBC  ABBC Coin     |      $125,028,718      |       $0.247542        |      $83,030,205       |    505,080,602 ABBC    |         -5.76%         |
 69 |      SC  Siacoin       |      $121,947,079      |       $0.002949        |       $2,043,076       |   41,353,612,700 SC    |         -4.06%         |
 70 |      GXC  GXChain      |      $121,693,968      |         $2.03          |       $3,042,648       |     60,000,000 GXC     |         -4.99%         |
 71 |     AE  Aeternity      |      $121,162,277      |       $0.444154        |      $38,500,380       |     272,793,174 AE     |         -4.23%         |
 72 |       XVG  Verge       |      $116,468,172      |       $0.007369        |       $1,604,278       |   15,805,409,499 XVG   |         -3.23%         |
 73 |   ETP  Metaverse ETP   |      $116,154,120      |         $1.62          |      $40,461,666       |     71,759,885 ETP     |         -9.33%         |
 74 |      STEEM  Steem      |      $110,124,952      |       $0.340978        |       $1,122,121       |   322,967,892 STEEM    |         -2.49%         |
 75 |      ARDR  Ardor       |      $109,691,634      |       $0.109801        |       $1,327,459       |    998,999,495 ARDR    |         -5.31%         |
 76 |       ELF  aelf        |      $108,074,954      |       $0.217880        |      $15,735,213       |    496,030,000 ELF     |         -1.35%         |
 77 |   INB  Insight Chain   |      $107,508,460      |       $0.307252        |       $5,435,619       |    349,902,689 INB     |         -6.46%         |
 78 |      SOLVE  SOLVE      |      $107,021,001      |       $0.327169        |       $9,286,887       |   327,112,052 SOLVE    |         12.73%         |
 79 |    VEST  VestChain     |      $105,387,369      |       $0.014889        |        $396,465        |   7,078,400,000 VEST   |        -11.00%         |
 80 |       QNT  Quant       |      $101,347,141      |         $10.37         |      $13,719,402       |     9,777,236 QNT      |         14.61%         |
 81 |   NEX  Nash Exchange   |      $99,192,791       |         $2.74          |       $1,985,748       |     36,196,678 NEX     |         -3.50%         |
 82 |      THETA  THETA      |      $98,543,466       |       $0.113203        |       $2,157,896       |   870,502,690 THETA    |         -5.94%         |
 83 |       DENT  Dent       |      $96,923,187       |       $0.001332        |       $6,458,186       |  72,745,838,994 DENT   |         -5.83%         |
 84 |    WTC  Waltonchain    |      $96,854,881       |         $2.32          |      $22,574,334       |     41,682,339 WTC     |         16.17%         |
 85 |    MCO  Crypto.com     |      $93,731,944       |         $5.93          |       $8,346,732       |     15,793,831 MCO     |         -2.05%         |
 86 |   MAID  MaidSafeCoin   |      $93,684,812       |       $0.207014        |        $685,687        |    452,552,412 MAID    |         -6.19%         |
 87 |      SNT  Status       |      $91,353,836       |       $0.026323        |      $17,797,371       |   3,470,483,788 SNT    |         -3.86%         |
 88 |    ENJ  Enjin Coin     |      $89,470,435       |       $0.115345        |       $4,932,359       |    775,679,781 ENJ     |         -2.92%         |
 89 |      EKT  EDUCare      |      $89,461,163       |       $0.124975        |       $2,309,456       |    715,835,137 EKT     |         -2.38%         |
 90 |        DAI  Dai        |      $88,618,335       |       $0.982690        |      $18,747,792       |     90,179,367 DAI     |         -0.80%         |
 91 |       GNT  Golem       |      $88,468,793       |       $0.091730        |       $1,127,506       |    964,450,000 GNT     |         -3.47%         |
 92 |       XZC  Zcoin       |      $86,935,432       |         $11.06         |       $2,154,223       |     7,861,468 XZC      |         -4.14%         |
 93 |      NAS  Nebulas      |      $83,446,607       |         $1.72          |      $10,626,245       |     48,627,715 NAS     |         4.07%          |
 94 |     STRAT  Stratis     |      $81,564,126       |       $0.820611        |       $3,546,613       |    99,394,330 STRAT    |         -7.04%         |
 95 |       NET  NEXT        |      $78,533,755       |         $1.56          |      $12,188,679       |     50,269,268 NET     |         22.71%         |
 96 |        REN  Ren        |      $77,607,280       |       $0.100819        |       $8,913,932       |    769,764,831 REN     |         -8.98%         |
 97 |   CCCX  Clipper Coin   |      $74,987,003       |       $0.019861        |        $56,719         |   3,775,570,996 CCCX   |         13.61%         |
 98 |   MXM  Maximine Coin   |      $70,357,334       |       $0.042667        |       $2,647,414       |   1,649,000,000 MXM    |         -2.66%         |
 99 |        WAX  WAX        |      $69,037,228       |       $0.073224        |        $548,672        |    942,821,662 WAX     |         -5.35%         |
100 | SAN  Santiment Net...  |      $69,026,823       |         $1.10          |        $21,347         |     62,660,371 SAN     |         -2.61%         |
1 голос
/ 06 июля 2019

Используя селекторы атрибутов и классов, вы можете легко очистить таблицу: 

import requests
from bs4 import BeautifulSoup


def make_soup(url: str) -> BeautifulSoup:
    res = requests.get(url, headers={
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0'
    })
    res.raise_for_status()
    return BeautifulSoup(res.text, 'html.parser')


def scrape_coins(soup: BeautifulSoup) -> list:
    table = soup.select_one('#currencies')
    coins = {}
    for row in table.select('tbody > tr'):
        symbol = row.select_one('.currency-symbol').text.strip()
        name = row.select_one('.currency-name-container').text.strip()
        cap = row.select_one('.market-cap')['data-usd']
        price = row.select_one('.price')['data-usd']
        volume = row.select_one('.volume')['data-usd']
        supply = row.select_one('[data-supply]')['data-supply']
        change = row.select_one('[data-percentusd]')['data-percentusd']
        coins[symbol] = {
            'name': name,
            'cap': float(cap),
            'price': float(price),
            'volume': float(volume),
            'supply': float(supply),
            'change': float(change),
        }
    return coins


if __name__ == "__main__":
    url = 'https://coinmarketcap.com/'
    soup = make_soup(url)
    info = scrape_coins(soup)
    from pprint import pprint
    pprint(info)

output: 

{'BTC': {'cap': 196969226244.0,
         'change': -5.4235,
         'name': 'Bitcoin',
         'price': 11065.3915833,
         'supply': 17800475.0,
         'volume': 24574484943.9},
 'ETH': {'cap': 30724660168.6,
         'change': -2.00031,
         'name': 'Ethereum',
         'price': 287.746701554,
         'supply': 106776758.874,
         'volume': 8840470261.58},
 'LTC': {'cap': 7439287857.04,
         'change': -3.64038,
         'name': 'Litecoin',
         'price': 118.965428838,
         'supply': 62533190.774,
         'volume': 4181083872.28},
 'XRP': {'cap': 16149651071.9,
         'change': -4.05122,
         'name': 'XRP',
         'price': 0.379397286226,
         'supply': 42566596173.0,
         'volume': 1332204345.98}}
... and so on
Добро пожаловать на сайт PullRequest, где вы можете задавать вопросы и получать ответы от других членов сообщества.

Похожие темы

...