'''
Name: {{plugin_name}}
Description: {{plugin_name}}
PID:  {{pid}} force
'''

import os
from scrapy.utils.project import get_project_settings
from grepsr_crawler.lib.proxy import get_proxy
from grepsr_crawler.crawler.spiders.base_crawler import GrepsrBaseCrawler
from scrapy.crawler import CrawlerProcess
from grepsr_crawler.lib.user_agent import get_user_agent

class GrepsrCrawler(GrepsrBaseCrawler):
   allowed_domains = [""]

   custom_settings = {
        'USER_AGENT' : get_user_agent(), # NOTE: impersonate as chrome browser by default 
        'CONCURRENT_REQUESTS' : 5,
        'CHANGE_PROXY_AFTER': 10,
        'LOG_LEVEL' : 'DEBUG', # ERROR for prod and DEBUG for dev 
        'JOBDIR' : './crawls',  # reduce memory by saving on a disk
      #   'REDIRECT_ENABLED':  True, # NOTE: to enable redirect routing as redirect is disabled by default
      #   'RETRY_HTTP_CODES' : [500, 503, 504, 400, 403, 408],
    }

   headers = {
        'Accept-Encoding': 'gzip, deflate, br',
        'Upgrade-Insecure-Requests': '1',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
        'Connection' : 'keep-alive'
    }

   '''
    Template: 1
   '''
   # start_urls = ['list of urls']

   # def parse(self, response):
   #    pass

   '''
    Template: 2
   '''
   def start_requests(self):
      pass
      

settings_file_path = 'grepsr_crawler.crawler.settings'
os.environ.setdefault('SCRAPY_SETTINGS_MODULE', settings_file_path)
settings = get_project_settings()
settings['PROXY_LIST'] = get_proxy(100) # proxy pool of 100 
process = CrawlerProcess(settings)
process.crawl(GrepsrCrawler)
process.start()  # the script will block here until the crawling is finished