Advertisement
kopyl

Untitled

Feb 23rd, 2021
187
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 2.57 KB | None | 0 0
  1. @memory_profile
  2. def scrape_recent_links_to_json() -> None:
  3.     p(
  4.         "Getting new jobs from API to JSON at time:\n",
  5.         correct_datetime()['_time'],
  6.         sep=""
  7.     )
  8.     jobs_external_database = request_jobs_from_website(
  9.         design_jobs_search_parameters
  10.     ).json()
  11.     write_json_file(
  12.         jobs_external_database,
  13.         "database/jobs_recent_saved_before_check_design.json"
  14.     )
  15.     """
  16.    Saving results to "before check" database to minimize API requests.
  17.    """
  18.     jobs_external = jobs_external_database["results"]
  19.     jobs_local_database_all = read_json_file("all_design_jobs")
  20.     jobs_local_database_recent = read_json_file(
  21.         "database/jobs_recent_saved_after_check_design.json"
  22.     )
  23.     jobs_local_all = jobs_local_database_all["jobs"]
  24.     jobs_local_recent = jobs_local_database_recent["results"]
  25.     jobs_added_counter = 0
  26.     can_apply_to_job_now_counter = 0
  27.  
  28.     for job in jobs_external:
  29.  
  30.         if job_is_new(job, jobs_local_recent):  # checking with only last 10.
  31.  
  32.             job["saved_to_json_on_datetime"] = (
  33.                 str(datetime.datetime.now())
  34.             )
  35.             job_is_relevant, relevancy_reasons = is_job_relevant(job)
  36.             job["relevancy"] = (
  37.                 relevancy_reasons
  38.             )
  39.             job["company_blacklisted"] = (
  40.                 is_company_in_blacklist(job)
  41.             )
  42.  
  43.             job["can_apply_now"] = False
  44.  
  45.             if(
  46.                 job_is_relevant and
  47.                 not job["company_blacklisted"]
  48.             ):
  49.  
  50.                 if sufficient_days_passed := Job(
  51.                     job['id']
  52.                 ).days_passed_since_last_contact_or_new_job(
  53.                     required_days_to_be_passed=10
  54.                 ):
  55.  
  56.                     can_apply_to_job_now_counter += 1
  57.                     job["can_apply_now"] = True
  58.  
  59.                 job["sufficient_days_passed"] = sufficient_days_passed
  60.  
  61.             job["has_applied"] = False  # doc.id#3
  62.  
  63.             jobs_local_all.append(job)
  64.             jobs_added_counter += 1
  65.  
  66.     if jobs_added_counter:
  67.         p(
  68.             f"Added {jobs_added_counter} new jobs to database."
  69.         )
  70.         if can_apply_to_job_now_counter:
  71.             p(
  72.                 f"Added {can_apply_to_job_now_counter} "
  73.                 "relevant jobs to database."
  74.             )
  75.         write_json_file(jobs_local_database_all)
  76.     write_json_file(
  77.         jobs_external_database,
  78.         "database/jobs_recent_saved_after_check_design.json"
  79.     )
  80.     del jobs_local_database_all
  81.     gc.collect()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement