सेवा के लिए एक अनुरोध

Python में Web Scraping

Thomas Laetsch

Data Scientist, NYU

स्पाइडर रीकॉल

import scrapy
from scrapy.crawler import CrawlerProcess

class SpiderClassName(scrapy.Spider):
    name = "spider_name"
    # the code for your spider
    ...

process = CrawlerProcess()

process.crawl(SpiderClassName)

process.start()

स्पाइडर रीकॉल

class DCspider( scrapy.Spider ):
    name = "dc_spider"

    def start_requests( self ):
        urls = [ 'https://www.datacamp.com/courses/all' ]
        for url in urls:
            yield scrapy.Request( url = url, callback = self.parse )

    def parse( self, response ):
        # simple example: write out the html
        html_file = 'DC_courses.html'
        with open( html_file, 'wb' ) as fout:
            fout.write( response.body )

start_requests के बारे में संक्षेप

def start_requests( self ):

    urls = ['https://www.datacamp.com/courses/all']

    for url in urls:
        yield scrapy.Request( url = url, callback = self.parse )

def start_requests( self ):
    url = 'https://www.datacamp.com/courses/all'
    yield scrapy.Request( url = url, callback = self.parse )

scrapy.Request यहाँ हमारे लिए response वैरिएबल भर देगा
url आर्ग्यूमेंट बताता है किस साइट को scrape करना है
callback आर्ग्यूमेंट बताता है response वैरिएबल को प्रोसेसिंग के लिए कहाँ भेजना है

ज़ूम आउट

class DCspider( scrapy.Spider ):
    name = "dc_spider"

    def start_requests( self ):
        urls = [ 'https://www.datacamp.com/courses/all' ]
        for url in urls:
            yield scrapy.Request( url = url, callback = self.parse )

    def parse( self, response ):
        # simple example: write out the html
        html_file = 'DC_courses.html'
        with open( html_file, 'wb' ) as fout:
            fout.write( response.body )

एंड रिक्वेस्ट

Python में Web Scraping