from selenium import webdriver
from selenium.webdriver.common.by import By
import pymongo
import time

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from lxml import etree


class crawl_project(object):
    def __init__(self):
        self.driver = webdriver.Chrome()
        self.driver.maximize_window()
        myclient = pymongo.MongoClient(host='127.0.0.1', port=27017,
                                       username='admin', password='abc123456')
        mydb = myclient["db_recruit"]
        self.mycollection = mydb["recruit_info"]

    def login(self, url):
        self.driver.get(url)
        if WebDriverWait(self.driver, 5, 0.5).until(EC.presence_of_element_located
                                                        ((By.XPATH, "//input[@id='keywordInput']"))):
            self.driver.find_element_by_xpath("//input[@id='keywordInput']").send_keys('python工程师')
            time.sleep(3)
            self.driver.find_element_by_xpath("//button[@id='search_btn']").click()
            time.sleep(3)

    def parse_html(self, content):
        # 导入网络源代码信息
        html = etree.HTML(content)
        # 所有订单信息
        items = html.xpath("//div[@class='e sensors_exposure']")
        for i in items:
            # 每一个I就是每一条订单数据
            posts = i.xpath(".//span[@class='jname at']/text()")[0]
            times = i.xpath(".//span[@class='time']/text()")[0]
            salary = i.xpath(".//span[@class='sal']/text()")
            if salary:
                salary = salary[0]
            else:
                salary = "暂无数据"
            place = ','.join(i.xpath(".//span[@class='d at']/span/text()"))
            welfare = ','.join(i.xpath(".//p[@class='tags']/span/@title"))
            data = {
                'job_name': posts,
                'job_time': times,
                'job_salary': salary,
                'job_info': place,
                'job_welfare': welfare
            }
            print(data)
            # 插入到mongo
            self.mycollection.insert_one(data)

    def main(self):
        self.login("https://we.51job.com/pc/search?keyword=&searchType=2&sortType=0&metro=")
        while True:
            self.parse_html(self.driver.page_source)
            time.sleep(10)
            self.driver.find_element_by_xpath("//button[@class='btn-next']").click()
            time.sleep(10)
            if self.driver.find_element_by_xpath("//button[@class='btn-next']").get_attribute("disabled"):
                break
        self.driver.quit()


if __name__ == '__main__':
    start = crawl_project()
    start.main()

报错信息：

Traceback (most recent call last):
  File "D:\data\自动化爬虫\练习.py", line 70, in <module>
    start.main()
  File "D:\data\自动化爬虫\练习.py", line 63, in main
    if self.driver.find_element_by_xpath("//button[@class='btn-next']").get_attribute("disabled")== 'disabled':
  File "C:\Users\qaq\AppData\Local\Programs\Python\Python38\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 394, in find_element_by_xpath
    return self.find_element(by=By.XPATH, value=xpath)
  File "C:\Users\qaq\AppData\Local\Programs\Python\Python38\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 976, in find_element
    return self.execute(Command.FIND_ELEMENT, {
  File "C:\Users\qaq\AppData\Local\Programs\Python\Python38\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
    self.error_handler.check_response(response)
  File "C:\Users\qaq\AppData\Local\Programs\Python\Python38\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"//button[@class='btn-next']"}
  (Session info: chrome=114.0.5735.110)