Нет надежного способа, но я разбираюсь с нумерацией страниц в следующем порядке:
- Дождитесь появления целевого элемента
- Соберите данные из цели
- Удалите целевой элемент
- Нажмите следующую кнопку
- ... цикл до тех пор, пока не будет следующей кнопки или содержимое не загрузится даже после ожидания
Доказательствопонятия:
Целевой HTML-код:
<!-- Copied from: https://jsfiddle.net/solodev/yw7y4wez -->
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<title>Pagination Example</title>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<meta name="robots" content="noindex, nofollow">
<meta name="googlebot" content="noindex, nofollow">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
<link rel="stylesheet" type="text/css" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
<script type="text/javascript" src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
<script type="text/javascript" src="https://www.solodev.com/assets/pagination/jquery.twbsPagination.js"></script>
<style type="text/css">
.container {
margin-top: 20px;
}
.page {
display: none;
}
.page-active {
display: block;
}
</style>
<script type="text/javascript">
window.onload = function() {
$('#pagination-demo').twbsPagination({
totalPages: 5,
// the current page that show on start
startPage: 1,
// maximum visible pages
visiblePages: 5,
initiateStartPageClick: true,
// template for pagination links
href: false,
// variable name in href template for page number
hrefVariable: '{{number}}',
// Text labels
first: 'First',
prev: 'Previous',
next: 'Next',
last: 'Last',
// carousel-style pagination
loop: false,
// callback function
onPageClick: function(event, page) {
$('.page-active').removeClass('page-active');
$('#page' + page).addClass('page-active');
},
// pagination Classes
paginationClass: 'pagination',
nextClass: 'next',
prevClass: 'prev',
lastClass: 'last',
firstClass: 'first',
pageClass: 'page',
activeClass: 'active',
disabledClass: 'disabled'
});
}
</script>
</head>
<body>
<div class="container">
<div class="jumbotron page" id="page1">
<div class="container">
<h1 class="display-3">Adding Pagination to your Website</h1>
<p class="lead">In this article we teach you how to add pagination, an excellent way to navigate large amounts of content, to your website using a jQuery Bootstrap Plugin.</p>
<p><a class="btn btn-lg btn-success" href="https://www.solodev.com/blog/web-design/adding-pagination-to-your-website.stml" role="button">Learn More</a></p>
</div>
</div>
<div class="jumbotron page" id="page2">
<h1 class="display-3">Not Another Jumbotron</h1>
<p class="lead">Cras justo odio, dapibus ac facilisis in, egestas eget quam. Fusce dapibus, tellus ac cursus commodo, tortor mauris condimentum nibh, ut fermentum massa justo sit amet risus.</p>
<p><a class="btn btn-lg btn-success" href="#" role="button">Sign up today</a></p>
</div>
<div class="jumbotron page" id="page3">
<h1 class="display-3">Data. Data. Data.</h1>
<p>This example is a quick exercise to illustrate how the default responsive navbar works. It's placed within a <code>.container</code> to limit its width and will scroll with the rest of the page's content.
</p>
<p>
<a class="btn btn-lg btn-primary" href="../../components/navbar" role="button">View navbar docs »</a>
</p>
</div>
<div class="jumbotron page" id="page4">
<h1 style="-webkit-user-select: auto;">Buy Now!</h1>
<p class="lead" style="-webkit-user-select: auto;">Cras justo odio, dapibus ac facilisis in, egestas eget quam. Fusce dapibus, tellus ac cursus commodo, tortor mauris condimentum nibh, ut fermentum massa justo sit amet.</p>
<p style="-webkit-user-select: auto;"><a class="btn btn-lg btn-success" href="#" role="button" style="-webkit-user-select: auto;">Get
started today</a></p>
</div>
<div class="jumbotron page" id="page5">
<h1 class="cover-heading">Cover your page.</h1>
<p class="lead">Cover is a one-page template for building simple and beautiful home pages. Download, edit the text, and add your own fullscreen background photo to make it your own.</p>
<p class="lead">
<a href="#" class="btn btn-lg btn-primary">Learn more</a>
</p>
</div>
<ul id="pagination-demo" class="pagination-lg pull-right"></ul>
</div>
<script>
// tell the embed parent frame the height of the content
if (window.parent && window.parent.parent) {
window.parent.parent.postMessage(["resultsFrame", {
height: document.body.getBoundingClientRect().height,
slug: "yw7y4wez"
}], "*")
}
</script>
</body>
</html>
Вот пример рабочей версии кода,
const puppeteer = require('puppeteer');
async function runScraper() {
let browser = {};
let page = {};
const url = 'http://localhost:8080';
// open the page and wait
async function navigate() {
browser = await puppeteer.launch({ headless: false });
page = await browser.newPage();
await page.goto(url);
}
async function scrapeData() {
const headerSel = 'h1';
// wait for element
await page.waitFor(headerSel);
return page.evaluate((selector) => {
const target = document.querySelector(selector);
// get the data
const text = target.innerText;
// remove element so the waiting function works
target.remove();
return text;
}, headerSel);
}
// this is a sample concept of pagination
// it will vary from page to page because not all site have same type of pagination
async function paginate() {
// manually check if the next button is available or not
const nextBtnDisabled = !!(await page.$('.next.disabled'));
if (!nextBtnDisabled) {
// since it's not disable, click it
await page.evaluate(() => document.querySelector('.next').click());
// just some random waiting function
await page.waitFor(100);
return true;
}
console.log({ nextBtnDisabled });
}
/**
* Scraping Logic
*/
await navigate();
// Scrape 5 pages
for (const pageNum of [...Array(5).keys()]) {
const title = await scrapeData();
console.log(pageNum + 1, title);
await paginate();
}
}
runScraper();
Результат:
Server running at 8080
1 'Adding Pagination to your Website'
2 'Not Another Jumbotron'
3 'Data. Data. Data.'
4 'Buy Now!'
5 'Cover your page.'
{ nextBtnDisabled: true }
Я не поделился серверомкод, это в основном фрагмент HTML выше.