Python抓取IMDB电影Top250排行

用Scrapy抓取IMDB电影Top250排行

scrapy主代码：

# 抓取主代码
import scrapy
import re
from douban.items import ImdbTopItem

class Imdb(scrapy.Spider):

    def __init__(self):
        self.start = 0

    name = "imdb"
    allowed_domains = ["imdb.com"]
    start_urls = [
        'http://www.imdb.com/chart/top', # imdb top250
    ]

    def parse(self, response):
        item = ImdbTopItem()
        res_infos = response.css('tbody.lister-list tr')

        for res in res_infos:
            tmp = {}
            title = res.css('td.titleColumn a::text').extract_first() # 电影名称
            item['title'] = title
            score_and_count = res.css('td strong').extract_first() # 电影评分和评分人数
            score_and_count = score_and_count.split('"')[1].split()
            score = score_and_count[0]
            count = score_and_count[3].replace(',','')
            item['score'] = float(score)
            item['count'] = int(count)
            yield item
            
# 将结果写入markdown
class ImdbTopPipeline:

    def __init__(self):
        self.id = 1

    def process_item(self, item, spider):
        with open('imdb250.md', 'a+', encoding='utf-8') as f:
            title = item.get('title')
            score = item.get('score')
            count = item.get('count')
            f.write('|' + str(self.id) + '|' + title + '|' + 
                str(score) + '|' + str(count) + '|\n')
            self.id += 1
        return item     

结果如下: