/Python/How to send data to scrapy pipeline to mongodb

How to send data to scrapy pipeline to mongodb

# In your pipeline

class EPGD_pipeline(object):
    def __init__(self):
        self.collections = {
            spider_name: self.setup_db_connection(dj_mongo_database_url.parse(url))
            for spider_name, url in settings['MONGODB_PIPELINE_SETTINGS'].iterItems()
        )
    }

    def process_item(self, item, spider):
        collection = self.collections[spider.name]
        ...


# In settings.py

MONGODB_PIPELINE_SETTINGS = {
    "GenDis": "mongodb://myhost:29297/test_db/collection",
    "EPGD": "mongodb://myhost:29297/test_db/collection2",
}

Source: stackoverflow.com

How to send data to scrapy pipeline to mongodb

    BOT_NAME = 'capstone'

    SPIDER_MODULES = ['capstone.spiders']
    NEWSPIDER_MODULE = 'capstone.spiders'

    ITEM_PIPLINES = {'capstone.pipelines.MongoDBPipeline': 300,}
    MONGO_URI = 'mongodb://localhost:27017'
    MONGO_DATABASE = 'congress'
    ROBOTSTXT_OBEY = True
    DOWNLOAD_DELAY = 10

Source: stackoverflow.com

How to send data to scrapy pipeline to mongodb

    from pymongo import MongoClient
    from scrapy.conf import settings
    from scrapy.exceptions import DropItem
    from scrapy import log

    class MongoDBPipeline(object):
        collection_name= 'members'
        def __init__(self, mongo_uri, mongo_db):
            self.mongo_uri = mongo_uri
            self.mongo_db = mongo_db
        @classmethod
        def from_crawler(cls, crawler):
            return cls(
                mongo_uri=crawler.settings.get('MONGO_URI')
                mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
            )
        def open_spider(self,spider):
            self.client = pymongo.MongoClient(self.mongo_uri)
            self.db = self.client[self.mongo_db]
        def close_spider(self, spider):
            self.client.close()
        def process_item(self, item, spider):
            self.db[self.collection_name].insert(dict(item))
            return item

Source: stackoverflow.com

Similar codes

0 vote235 views1 answer
Create new column with mask pandas
Python
0 vote272 views2 answers
Python read file from same directory
Python
0 vote283 views1 answer
Merge two Querysets in Python Django while preserving Queryset methods
Python Django
2 votes247 views2 answers
Empty directory if not empty python
Python
0 vote237 views1 answer
Instaed of: newlist = [] for word in wordlist: newlist.append(word.upper())
Python
0 vote236 views1 answer
Switch between frames in tkinter
Python
2 votes288 views2 answers
Get files from s3 bucket python
Python
1 vote234 views2 answers
Python - remove exta space in column
Python
21 votes256 views2 answers
How to update python
Python
16 votes233 views2 answers
What is self keyword in python
Python