Vfrdtyky

I'm new to scrapy and python, I can able to download all the files but I want to download only specific Type files "EX-10", So that it will download followinh files. ( Ex-10.1, Ex-10.2 to EX-10.99).

My Code

import scrapy, os



class legco(scrapy.Spider):

name = "sec_gov"



start_urls = ["https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&SIC=2834&owner=exclude&match=&start=120&count=40&hidefilings=0"]



def parse(self, response):

    for link in response.xpath('//table[@summary="Results"]//td[@scope="row"]/a/@href').extract():

        absoluteLink = response.urljoin(link)

        yield scrapy.Request(url = absoluteLink, callback = self.parse_links)



def parse_links(self, response):

    for links in response.xpath('//table[@summary="Results"]//a[@id="documentsbutton"]/@href').extract():

        targetLink = response.urljoin(links)

        yield scrapy.Request(url = targetLink, callback = self.collecting_file_links)



def collecting_file_links(self, response):

    for links in response.xpath('//table[contains(@summary,"Document")]//td[@scope="row"]/a/@href').extract():

        if links.endswith(".htm") or links.endswith(".txt"):

            baseLink = response.urljoin(links)

            yield scrapy.Request(url = baseLink, callback = self.download_files)



def download_files(self, response):

    path = response.url.split('/')[-1]

    dirf = r"/home/surukam/scrapy/demo/tutorial/tutorial/Downloads3"

    if not os.path.exists(dirf):os.makedirs(dirf)

    os.chdir(dirf)

    with open(path, 'wb') as f:

        f.write(response.body)

and Scrapy want to check for next pages also... (upto last page), Its not working Fine.

Rules = (Rule(LinkExtractor(allow=(), restrict_xpaths=('//a[@value="Next 40"]',)), callback="parse", follow= True),)



# follow next page links

    next_page = response.xpath('.//a[@value="Next 40"]/@href').extract()

    if next_page:

        next_href = next_page[0]

        next_page_url = 'https://www.sec.gov/cgi-bin/browse-edgar?company=&match=&CIK=&filenum=&State=&Country=&SIC=2834&owner=exclude&Find=Find+Companies&action=getcompany' + next_href

        request = scrapy.Request(url=next_page_url)

        yield request

Your problem seems to have been solved. The following script should fetch you the required files from that site following every pagination links and downloading those files like the way you wanted to have.

import scrapy, os



class legco(scrapy.Spider):

    name = "sec_gov"



    start_urls = ["https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&SIC=2834&owner=exclude&match=&start=120&count=40&hidefilings=0"]



    def parse(self, response):

        for link in response.xpath('//table[@summary="Results"]//td[@scope="row"]/a/@href').extract():

            absoluteLink = response.urljoin(link)

            yield scrapy.Request(url = absoluteLink, callback = self.parse_links)



        nextpage = response.css("input[value='Next 40']::attr(onclick)")

        if nextpage:

            tpage = nextpage.extract_first().split("parent.location=")[1].replace("'","")

            nlink = response.urljoin(tpage)

            yield scrapy.Request(url=nlink, callback = self.parse)



    def parse_links(self, response):

        for links in response.xpath('//table[@summary="Results"]//a[@id="documentsbutton"]/@href').extract():

            targetLink = response.urljoin(links)

            yield scrapy.Request(url = targetLink, callback = self.collecting_file_links)



    def collecting_file_links(self, response):

        for links in response.xpath('//table[contains(@summary,"Document")]//tr[td[starts-with(., "EX-")]]/td/a[contains(@href, ".htm") or contains(@href, ".txt")]/@href').extract():

            baseLink = response.urljoin(links)

            yield scrapy.Request(url = baseLink, callback = self.download_files)



    def download_files(self, response):

        path = response.url.split('/')[-1]

        dirf = r"/home/surukam/scrapy/demo/tutorial/tutorial/Downloads3"

        if not os.path.exists(dirf):os.makedirs(dirf)

        os.chdir(dirf)

        with open(path, 'wb') as f:

            f.write(response.body)

You need to use a FilesPipeline, but the one that scrapy provides generates the file name based on the hash of the URL.

If you want a custom file name, you have to make your own FilesPipeline like this:

import scrapy, os

from scrapy.pipelines.files import FilesPipeline



class legco(scrapy.Spider):

    name = "sec_gov"



    start_urls = ["https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&SIC=2834&owner=exclude&match=&start=120&count=40&hidefilings=0"]



    custom_settings = {

        'ITEM_PIPELINES': {'myspider.MyFilesPipeline': 1},

        'FILES_STORE': '/my/valid/path/',

    }



    def parse(self, response):

        for link in response.xpath('//table[@summary="Results"]//td[@scope="row"]/a/@href').extract():

            absoluteLink = response.urljoin(link)

            yield scrapy.Request(url = absoluteLink, callback = self.parse_links)



    def parse_links(self, response):

        for links in response.xpath('//table[@summary="Results"]//a[@id="documentsbutton"]/@href').extract():

            targetLink = response.urljoin(links)

            yield scrapy.Request(url = targetLink, callback = self.collecting_file_links)



    def collecting_file_links(self, response):

        for links in response.xpath('//table[contains(@summary,"Document")]//td[@scope="row"]/a/@href').extract():

            if links.endswith(".htm") or links.endswith(".txt"):

                yield {

                    'file_urls': [response.urljoin(links)]

                }



class MyFilesPipeline(FilesPipeline):



    def file_path(self, request, response=None, info=None):

        return request.url.split('/')[-1]