Pipeline.py
class MongoDBPipeline:
def __init__(self,conn,database):
self.conn = conn
self.database = database
@classmethod
def from_crawler(cls, crawler):
return cls(
conn=crawler.settings.get(‘MONGODB_CONNECTION_STRING’),
database=crawler.settings.get(‘MONGODB_DATABASE’)
)
def open_spider(self,spider):
# 创建连接对象
self.client = pymongo.MongoClient(self.conn)
# 连接要用的使用的数据库
self.db = self.client[self.database]
def process_item(self,item,spider):
# 设置使用表名称
name = item.__class__.__name__
# 默认item类名
self.db[name].insert_one(dict(item))
return item
def close_spider(self,spider):
self.client.close()