在订抓取时已经使用contains将隐藏数据排除但会出现后面最后2页的数据不能被抓取到,且会存在部分页面被重复抓取的现象.

相关代码：

# coding:utf-8

from selenium import webdriver
from pymongo import MongoClient
from lxml import etree
from selenium.webdriver.chrome.options import Options
# 简化用包
from selenium.webdriver.common.by import By
# 场景判断
from selenium.webdriver.support import expected_conditions as EC
# 等待用包
from selenium.webdriver.support.ui import WebDriverWait

# 打开目标地址，点击登录，登录URLhttp://sleeve.talelin.com/#/login


class CrawlProject(object):
    def __init__(self, chrome_options):
        self.driver = webdriver.Chrome(options=chrome_options)
        # 设置浏览器最大化
        self.driver.maximize_window()
        # 初始化mongodb连接
        client = MongoClient('mongodb://127.0.0.1:27017')
        client.admin.authenticate('admin', 'root')
        # 存储数据库
        my_db = client['order_db']
        # 集合
        self.my_collection = my_db['order_infos']

    def login(self, url):
        '''
        :param url: 登录URLhttp: // sleeve.talelin.com /  # /login
        :return:  返回是否登录成功
        '''
        print('开始登录{}'.format(url))
        self.driver.get(url)
        # 判断登录表单是否加载完成
        if WebDriverWait(self.driver, 5, 0.5).until(EC.presence_of_element_located((By.CLASS_NAME, 'login-form'))):
            # 登录表单加载完成后判断登录按钮是否存在
            if WebDriverWait(self.driver, 5, 0.5).until(EC.presence_of_element_located((By.CLASS_NAME, 'submit-btn'))):
                # 点击登录
                self.driver.find_element_by_xpath('//button[@class="submit-btn"]').click()
                # 通过是否能够获取banner图片的classname=’welcome‘判断是否登录成功
                if WebDriverWait(self.driver, 5, 0.5).until(EC.presence_of_element_located((By.CLASS_NAME, 'welcome'))):
                    return True
                else:
                    return False

    def crawl_website(self, url):
        '''
        抓取数据
        :param url: http://sleeve.talelin.com/#/statics/order/list
        :return:
        '''
        print('开始抓取数据')
        self.driver.get(url)
        page_num = 1
        while True:
            # 判断是否进入订单列表页面
            if WebDriverWait(self.driver, 5, 0.5).until(EC.presence_of_element_located((By.CLASS_NAME, 'pagination'))):
                # 解析页面数据
                print('开始解析第{}页数据'.format(page_num))
                self.parse_html(self.driver.page_source)
                # 点击翻页
                self.driver.find_element_by_xpath('//button[@class="btn-next"]').click()
                page_num += 1
                # 未页后停止翻页
                if self.driver.find_element_by_xpath('//button[@class="btn-next"]').get_attribute('disabled'):
                    break
        self.driver.quit()

    def parse_html(self, content):
        items_html = etree.HTML(content)
        items = items_html.xpath('//tbody/tr')
        for item in items:
            data = item.xpath('./td[not (contains(@class, "is-hidden"))]/div/text()')
            status = "".join(item.xpath('./td[not (contains(@class, "is-hidden"))]//div[@class="tags"]/span/text()'))
            if len(data) >= 4:
                print({
                    'id': data[0],
                    'order_num': data[1],
                    'goods_num': data[2],
                    'total_price': data[3],
                    'order_status': status
                })


def main():
    options = {
        'headless': '--headless',
        'no_sandbox': '--no_sandbox',
        'gpu': '--disable-gup'
    }
    chrome_options = Options()
    for k, v in options.items():
        chrome_options.add_argument(v)

    s = CrawlProject(chrome_options=chrome_options)
    url = 'http://sleeve.talelin.com/#/login'
    login_status = s.login(url)
    if login_status:
        # 登录成功后开始抓取数据
        s.crawl_website('http://sleeve.talelin.com/#/statics/order/list')


if __name__ == '__main__':
    main()