from scrapy import spiderloader
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from navigator import *


def main():
  settings = get_project_settings()
  spider_loader = spiderloader.SpiderLoader.from_settings(settings)
  process = CrawlerProcess(settings=settings)
  for spider_name in spider_loader.list():
      process.crawl(spider_name)

  process.start()


if __name__ == '__main__':
  main()

from time import sleep
import os
import signal
import subprocess

def main():
  spiders_process = subprocess.Popen(["python", "spiders.py"], stdout=subprocess.PIPE,
                                      shell=False, preexec_fn=os.setsid)
  sleep(300)
  os.killpg(spiders_process.pid, signal.SIGTERM)

if __name__ == '__main__':
  main()