Capstone

Web Scraping in Python

Thomas Laetsch

Data Scientist, NYU

Inspecting Elements

import scrapy
from scrapy.crawler import CrawlerProcess

class DC_Chapter_Spider(scrapy.Spider):

    name = "dc_chapter_spider"

    def start_requests( self ):
        url = 'https://www.datacamp.com/courses/all'
        yield scrapy.Request( url = url,
                              callback = self.parse_front )

    def parse_front( self, response ):
        ## Code to parse the front courses page

    def parse_pages( self, response ):
        ## Code to parse course pages
        ## Fill in dc_dict here

dc_dict = dict()

process = CrawlerProcess()
process.crawl(DC_Chapter_Spider)
process.start()

Web Scraping in Python

Parsing the Front Page

    def parse_front( self, response ):

# Narrow in on the course blocks course_blocks = response.css( 'div.course-block' )
# Direct to the course links course_links = course_blocks.xpath( './a/@href' )
# Extract the links (as a list of strings) links_to_follow = course_links.extract()
# Follow the links to the next parser for url in links_to_follow: yield response.follow( url = url, callback = self.parse_pages )
Web Scraping in Python

Parsing the Course Pages

def parse_pages( self, response ):

# Direct to the course title text crs_title = response.xpath('//h1[contains(@class,"title")]/text()')
# Extract and clean the course title text crs_title_ext = crs_title.extract_first().strip()
# Direct to the chapter titles text ch_titles = response.css( 'h4.chapter__title::text' )
# Extract and clean the chapter titles text ch_titles_ext = [t.strip() for t in ch_titles.extract()]
# Store this in our dictionary dc_dict[ crs_title_ext ] = ch_titles_ext
Web Scraping in Python

It's time to Weave

Web Scraping in Python

Preparing Video For Download...