[记录]scrapy爬虫框架

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

    def parse_book_list(self, response):
        item = response.meta.get('item')
        #获取每一本书的信息
        books_list = response.xpath('//ul[@class="clearfix"]/li')
        for book in books_list:
            #书籍详情URL
            item['book_href'] = book.xpath('//div[@class="res-img"]/div/a/@href').extract_first()
            # 书籍名字
            item['book_name'] = book.xpath('//div[@class="res-img"]/div/a/img/@alt').extract_first()
            # 书籍图片
            item['src'] = book.xpath('//div[@class="res-img"]/div/a/img/@src2').extract_first()
            print(item)
            yield scrapy.Request(
                url=item['book_href '],
                callback=self.parse_book_detail,
                meta={'item': deepcopy(item)}
            )

1
2
3

{'L1_cate': '文学艺术', 'L2_href': 'https://search.suning.com/%E7%A7%91%E5%B9%BB/', 'L2_name': '科幻', 'book_href':'//product.suning.com/0070129646/12181637004.html', 'book_name': '绿野仙踪正版书小学生正版 阅读励志7-9-12岁青少年版儿童文学书籍名著三年级四年级课外书书读物','src':'//imgservice3.suning.cn/uimg1/b2c/image/ZbF_iVDMfL1hCj6diWLkOg.jpg_220w_220h_4e'}
 #省略
 ......

1
2
3
4
5
6
7
8
9

    def parse_book_detail(self, response):
        item = response.meta.get('item')
        # 作者
        item['author'] = response.xpath('//ul[@class="bk-publish clearfix"]/li[1]/text()').extract_first()
        # 出版社
        item['publish'] = response.xpath('//ul[@class="bk-publish clearfix"]/li[2]/text()').extract_first()
        # 数据处理，采集到的数据有些多余的空格，需要去除
        item['author'] = item['author'].replace('\n', '').replace('\t', '').replace('</span>', '')
        item['publish'] = item['publish'].replace('\n', '').replace('\t', '').replace('</span>', '')

1
2
3
4
5
6
7

# 第一页
https://list.suning.com/1-502687-0.html
# 第二页
https://list.suning.com/1-502687-0.html#second-filter 
# 第三页
https://list.suning.com/1-502687-0.html#second-filter
....

1
2
3
4
5

https://list.suning.com/emall/showProductList.do?ci=502320&pg=03&cp=0&il=0&iy=0&adNumber=0&n=1&ch=4&prune=0&sesab=ACBAABC&id=IDENTIFYING&paging=1&sub=0

https://list.suning.com/emall/showProductList.do?ci=502320&pg=03&cp=1&il=0&iy=0&adNumber=0&n=1&ch=4&prune=0&sesab=ACBAABC&id=IDENTIFYING&cc=020

https://list.suning.com/emall/showProductList.do?ci=502320&pg=03&cp=2&il=0&iy=0&adNumber=0&n=1&ch=4&prune=0&sesab=ACBAABC&id=IDENTIFYING&cc=020

1
2
3
4
5
6

# 第一页
https://list.suning.com/emall/showProductList.do?ci=502320&pg=03&cp=0  
# 第二页
https://list.suning.com/emall/showProductList.do?ci=502320&pg=03&cp=1  
# 第三页
https://list.suning.com/emall/showProductList.do?ci=502320&pg=03&cp=2  

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

 # 一级分类（L1表示Lecel 1）
        L1_list = response.xpath('//div[@class="menu-list"]/div/dl')
        # 获取大分类
        for L1 in L1_list:
            item = {}
            item['L1_cate'] = L1.xpath('./dt/h3/a/text()').extract_first()

            # 获取二级分类
            L2_list = L1.xpath('./dd/a')
            for L2 in L2_list:
                # 二级分类url
                item['L2_href'] = L2.xpath('./@href').extract_first()
                # 二级分类名称
                item['L2_name'] = L2.xpath('./text()').extract_first()
                # 获取ci
                item['ci'] = item['L2_href'][26:32]
                # 翻页url
                item['next_url'] = 'https://list.suning.com/emall/showProductList.do?ci=' + item['ci'] + '&pg=03&cp={}'
                yield scrapy.Request(
                    url=item['L2_href'],
                    callback=self.parse_book_list,
                    meta={'item': deepcopy(item)}
                )

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

 def parse_book_list(self, response):
        item = response.meta.get('item')
        #获取每一本书的信息
        books_list = response.xpath('//ul[@class="clearfix"]/li')
        for book in books_list:
            #书籍详情URL
            item['book_href'] = 'https:' + book.xpath('//div[@class="res-img"]/div/a/@href').extract_first()
            # 书籍名字
            item['book_name'] = book.xpath('//div[@class="res-img"]/div/a/img/@alt').extract_first()
            # 书籍图片
            item['src'] = 'https:' + book.xpath('//div[@class="res-img"]/div/a/img/@src2').extract_first()

            yield scrapy.Request(
                url=item['book_href'],
                callback=self.parse_book_detail,
                meta={'item': deepcopy(item)}
            )
        # 翻页
        # 这里的页数是写死的，也可以抓取网页的pageNumber（总页数）
        for i in range(5):
            next_url = item['next_url'].format(i+1)
            yield scrapy.Request(
                url=next_url,
                callback=self.parse_book_list,
                meta={'item': deepcopy(item)}
            )

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

#编辑pipelines.py文件
from itemadapter import ItemAdapter
import pymysql
import sshtunnel

class SuningbookPipeline:

    server = sshtunnel.SSHTunnelForwarder(
        ('xxx.xxx.xxx.xxx', 22), # 填写你服务器的IP
        ssh_username='xxx',		# 连接的用户名
        ssh_password='xxxx',	# 密码
        remote_bind_address=('xx.xx.xx.xx', 3306), #也是服务器IP
        local_bind_address=('127.0.0.1', 3306)
    )
    server.start()

    print('SSH连接成功')

    def __init__(self):
            self.connect = pymysql.connect(
                host='127.0.0.1',
                port=3306,		#MySQL端口
                user='xxx',		#数据库用户名
                database='xxx',	#数据库名
                password='xxx',	#数据库密码
                charset='utf8'
            )
            print('mysql数据库连接成功')
            self.cursor = self.connect.cursor()
            print('游标获取成功')

    def process_item(self, item, spider):


        info = """INSERT INTO Book(BookcName,Author,Publish) VALUES ('%s','%s','%s')""" \
               % (
                   item['book_name'],
                   item['author'],
                   item['publish'],
               )
        self.cursor.execute(info)
        self.connect.commit()
        print('insert succeed')
        return item

    def close_spider(self,spider):
        self.cursor.close()
        self.connect.close()