当网络抓取谷歌时转到下一页

我正试着在网上搜罗谷歌酒店的名单。但是，我无法找到一种方法来绕过谷歌上的分页，因为URL实际上保持不变，只有一些内容发生了变化。请看下面的链接和截图 酒店名单都柏林在谷歌

我想知道在座的各位之前是否有人通过这个问题? 我已经成功地从第1页中抓取了数据，但是我需要对那里的每个页面都这样做 下面是我的当前代码。我已经把下一页的部分注释掉了，因为我不知道如何使它工作

from bs4 import BeautifulSoup
import requests
import sys
import csv
import pandas as pd
from lxml import html


url = f"https://www.google.com/travel/hotels?hrf=CgUIrAIQACIDRVVSKhYKBwjjDxAMGBMSBwjjDxAMGBQYASgAsAEAWAFoAZoBLhIGRHVibGluGiQweDQ4NjcwZTgwZWEyN2FjMmY6MHhhMDBjN2E5OTczMTcxYTCiARIKCC9tLzAyY2Z0EgZEdWJsaW6qAQoKAgghEgIIFRgBqgEWCgIIHBICCAcSAghREgIIRxICCDYYAaoBCgoCCCUSAgh3GAGqARIKAggREgIIKhICCDgSAggCGAGqARcKAgguEgIIPBICCDsSAwiBARICCCcYAaoBCgoCCFASAghPGAGqAQwKAwijARIDCKQBGAE&tcfs=EiwKCC9tLzAyY2Z0EgZEdWJsaW4aGAoKMjAxOS0xMi0xORIKMjAxOS0xMi0yMFIA&rp=aAFIAg&destination=Dublin&ap=MABoAA"


data = {
    'name': [],
    'star': [],
    'rating': [],
    'reviews': [],
    'price': []
}

def export(data):
    table = pd.DataFrame(data, columns=['name','rating','reviews','star','price'])
    table.to_csv('Listings.csv', sep = ',', encoding = 'utf-8-sig', index=False)

def getHotel(hotel):
    name = hotel.find(class_='BgYkof ogfYpf ykx2he').get_text()
    star = hotel.find(class_='sSHqwe r10jJf').get_text()
    rating = hotel.find(class_='sSHqwe').get_text()
    reviews = hotel.find(class_='sSHqwe uTUoTb fOuaIb XLC8M').get_text()
    price = hotel.find(class_='A9rngd wNqaKc idHpEf').get_text()

    data['name'].append(name.strip())
    data['star'].append(star.strip())
    data['rating'].append(rating.strip())
    data['reviews'].append(reviews.strip())
    data['price'].append(price.strip())

    export(data)

#def parse_page(url):
page = requests.get(url)
soup = BeautifulSoup(page.text, 'lxml')

hotels = soup.findAll(class_='f1dFQe')

for hotel in hotels:
    getHotel(hotel)

    # next_page_text = soup.find(class_='RveJvd snByac').text
    # print(next_page_text)
    # if next_page_text == 'Next':
    #     next_page_url = soup.find('a', class_='nextprev_on')['href']
    #     print(next_page_url)
    #     parse_page(url)
    # else:
    #     export(data)

#parse_page(url)

下面是web页面上按钮的HTML屏幕截图

问题来源StackOverflow 地址：/questions/59383945/go-to-next-page-when-webscraping-google