去哪儿爬虫

陈公子的话

# -*- ecoding: utf-8 -*-
# @ModuleName: 去哪儿爬虫
# @Function: 
# @Author: xiaochen
# @Time: 2021/2/18 11:07
import requests
import time
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

class Qner(object):
    def __init__(self):
        self.ua = UserAgent()
        self.headers = {'User-Agent': self.ua.random}
        self.url = 'https://piao.qunar.com/ticket/list.htm?keyword='
        self.city = city
        self.pagemax = int()
        self.hrefs = []

    def get_pagemax(self):
        url = f'{self.url}{self.city}'
        res = requests.get(url,headers =self.headers)
        if res.status_code == 200:
            soup = BeautifulSoup(res.text,'html.parser')
            a = soup.find('div',class_='pager').find_all('a')
            pagemax = a[-2].get_text()
            self.pagemax = int(pagemax)

    def get_urllist(self):
        for i in range(1,self.pagemax + 1):
            url = f'{self.url}{city}&page={i}'
            #print(url)
            res = requests.get(url, headers=self.headers)
            time.sleep(2)
            if res.status_code == 200:
                soup = BeautifulSoup(res.text,'html.parser')
                divs = soup.find_all('div', class_="sight_item_detail clrfix")
                for div in divs:
                    name = div.find('a',class_='name').get_text()
                    print(name)
                    address = div.find('p', class_="address color999").find('span').get_text()
                    print(address)
                    try:
                        price = div.find('span', class_="sight_item_price").find('em').get_text()
                        print(price)
                    except:
                        print("价格不详！")
                        href = div.find('h3', class_='sight_item_caption').find('a')['href']
                        href = f'https://piao.qunar.com{href}'
                        self.hrefs.append(href)
                    #print(self.hrefs)
                    time.sleep(5)

if __name__ == '__main__':
    city="北京"
    spider=Qner()
    spider.get_pagemax()
    spider.get_urllist()

#BeautifulSoup使用过程中以前的“lxml”，在新版本需要统一切换为“html.parser”