scrapy爬虫数据保存为txt,json,mysql的方法

1
2
3
4
5
6
7
8
9
10
11
12
13

class TxtPipeline(object):
    '''保存到txt'''
    def process_item(self, item, spider):
        reload(sys)
        sys.setdefaultencoding('utf-8')
        # 获取当前工作目录
        base_dir = os.getcwd()
        fiename = base_dir + '/news.txt'
        # 从内存以追加的方式打开文件，并写入对应的数据
        with open(fiename, 'a') as f:
            f.write(item['title'] + '\n')
            f.write(item['link'] + '\n')
        return item

class TxtPipeline(object): '''保存到txt''' def process_item(self, item, spider): reload(sys) sys.setdefaultencoding('utf-8') # 获取当前工作目录 base_dir = os.getcwd() fiename = base_dir + '/news.txt' # 从内存以追加的方式打开文件，并写入对应的数据 with open(fiename, 'a') as f: f.write(item['title'] + '\n') f.write(item['link'] + '\n') return item

1
2
3
4
5
6
7
8
9
10
11
12

class JsonPipeline(object):
    '''保存到json'''
    def process_item(self, item, spider):
        base_dir = os.getcwd()
        filename = base_dir + '/news.json'
        # 打开json文件，向里面以dumps的方式吸入数据
        # 注意需要有一个参数ensure_ascii=False ，不然数据会直接为utf编码的方式存入比如
        # :“/xe15”
        with codecs.open(filename, 'a') as f:
            line = json.dumps(dict(item), ensure_ascii=True) + '\n'
            f.write(line)
        return item

class JsonPipeline(object): '''保存到json''' def process_item(self, item, spider): base_dir = os.getcwd() filename = base_dir + '/news.json' # 打开json文件，向里面以dumps的方式吸入数据 # 注意需要有一个参数ensure_ascii=False ，不然数据会直接为utf编码的方式存入比如 # :“/xe15” with codecs.open(filename, 'a') as f: line = json.dumps(dict(item), ensure_ascii=True) + '\n' f.write(line) return item

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

class MysqlPipeline(object):
    '''保存到mysql'''
    def process_item(self,item,spider):
        # 将item里的数据拿出来
        title = item['title']
        link = item['link']
        content = item['content']
        # 和本地的数据库建立连接
        host = settings['MYSQL_HOSTS']
        user = settings['MYSQL_USER']
        psd = settings['MYSQL_PASSWORD']
        db = settings['MYSQL_DB']
        cha=settings['CHARSET']
        db = MySQLdb.connect(host=host,user=user,passwd=psd,db=db,charset=cha)
        # 使用cursor()方法获取操作游标
        cursor = db.cursor()
        # SQL 插入语句
        sql = "INSERT INTO aimks(title,link,content) VALUES (%s,%s,%s)"
        data=[title,link,content]
        try:
            # 执行SQL语句
            cursor.execute(sql,data)
            # 提交修改
            db.commit()
            print title+'：导入成功'
        except:
            db.rollback()
            print title+'：导入失败'
        finally:
            # 关闭连接
            db.close()
        return item

class MysqlPipeline(object): '''保存到mysql''' def process_item(self,item,spider): # 将item里的数据拿出来 title = item['title'] link = item['link'] content = item['content'] # 和本地的数据库建立连接 host = settings['MYSQL_HOSTS'] user = settings['MYSQL_USER'] psd = settings['MYSQL_PASSWORD'] db = settings['MYSQL_DB'] cha=settings['CHARSET'] db = MySQLdb.connect(host=host,user=user,passwd=psd,db=db,charset=cha) # 使用cursor()方法获取操作游标 cursor = db.cursor() # SQL 插入语句 sql = "INSERT INTO aimks(title,link,content) VALUES (%s,%s,%s)" data=[title,link,content] try: # 执行SQL语句 cursor.execute(sql,data) # 提交修改 db.commit() print title+'：导入成功' except: db.rollback() print title+'：导入失败' finally: # 关闭连接 db.close() return item

1
2
3
4
5
6

import os
import sys
import codecs
import json
import MySQLdb
import MySQLdb.cursors

import os import sys import codecs import json import MySQLdb import MySQLdb.cursors

1
2
3
4
5

ITEM_PIPELINES={
    'mkscrapy.pipelines.MysqlPipeline': 100,
    'mkscrapy.pipelines.JsonPipeline': 200,
    'mkscrapy.pipelines.TxtPipeline': 300,
}

ITEM_PIPELINES={ 'mkscrapy.pipelines.MysqlPipeline': 100, 'mkscrapy.pipelines.JsonPipeline': 200, 'mkscrapy.pipelines.TxtPipeline': 300, }