View difference between Paste ID: YiaeS56T and gdWfC54k

 ## for https://stackoverflow.com/q/75031138/6146136
-## based on https://stackoverflow.com/a/75032654/6146136
+
 
 from selenium import webdriver
-import requests
+from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
 
 from urllib.parse import urljoin
 from bs4 import BeautifulSoup
 import pandas as pd ## only needed for saving as csv
 import os ## only needed for printing csv path at end
 
 ###########################################################################
-maxPages = 70  # [there should be only 53, but just in case]
+
-csv_path = 'asda_jobs__requests.csv'  # path to csv file to save to
+
 
 maxPages = 50  # there's only 25 but there will be duplicates
-url = "https://www.asda.jobs/vacancy/find/results/ajaxaction/posbrowser_gridhandler/?"
+max_wait = 1  # 5 ## 30 ## as you need
-s = requests.Session()
+csv_path = 'asda_jobs.csv'  # path to csv file to save to
-s.get("https://www.asda.jobs/vacancy/find/results/")
+
-pagestamp = s.cookies['earcusession'][5:-8]
+
-url = url + f"pagestamp={pagestamp}"
+btn_sel = "a.scroller_movenext"  # .buttonEnabled"
 url = "https://www.asda.jobs/vacancy/find/results/"
 ecv = EC.visibility_of_all_elements_located
-for pgi in range(maxPages):
+
-    page = s.get(url+f"&movejump={pgi}&movejump_page={pgi+1}")
+options = webdriver.ChromeOptions()
-    if page.status_code != 200: break
+options.add_argument('--headless')
-    soup = BeautifulSoup(page.content, "lxml")
+browser = webdriver.Chrome(chrome_options=options)
 wait = WebDriverWait(browser, max_wait).until
 browser.get(url)
 
 addedRows, listings = [], []
 for pgi in range(maxPages):
     wait(ecv((By.CSS_SELECTOR, row_sel)))
 
     soup = BeautifulSoup(browser.page_source, 'html5lib')
     pgListing = [selectForList(rowSoup, {
         'pageNum': (pgi+1, '"staticVal"'),
-        if not pgl['link']: continue
+
         'jobRef': ('UNKNOWN', '"staticVal"'),
         'title': ('a[title]', 'title'),
         'about': 'div.rowContentContainer',
         'link': ('a[href]', 'href')
-            if jobRef: pgListing[li]['jobRef'] = jobRef[-1]
+
-
+
-    newCt = len(set([l['rowId'] for l in pgListing if l['rowId'] and l['rowId'] not in addedRows]))
+        if not pgl['link']:
-    print(page.status_code, page.reason, 'scraped', newCt, 'new from ', end='')
+            continue
-    selectForList(soup, ['div.pagingText', 'span.filtersSummaryTextInnerContainer'], printList=' : ')
+
-    if not newCt: break
+
             jobRef = lLink.split('/vacancy/', 1)[-1].split('/')[0]
-    # listings += pgListing # allows duplicates [probably fine in THIS method]
+
-    listings += [l for l in pgListing if not (l['rowId'] and l['rowId'] in addedRows)]
+            if jobRef:
-    addedRows += [l['rowId'] for l in pgListing]
+                pgListing[li]['jobRef'] = jobRef[-1]
     newCt = len(set([
         l['rowId'] for l in pgListing if l['rowId']
         and l['rowId'] not in addedRows]))
     print(f'{str([pgi]):>5}', 'scraped', newCt, 'new from ', end='')
     selectForList(soup, [
         'div.pagingText', 'span.filtersSummaryTextInnerContainer'
     ], printList=' : ')
 
-]), f'\n\nsaved {len(listings)} to "{os.path.abspath(csv_path)}"')
+    # listings += pgListing # allows duplicates
     listings += [l for l in pgListing if not (
         l['rowId'] and l['rowId'] in addedRows)]
     addedRows += [l['rowId'] for l in pgListing]
 
     nextBtn = browser.find_elements(By.CSS_SELECTOR, btn_sel)
     if nextBtn:
         browser.execute_script(
             "arguments[0].scrollIntoView(false);", nextBtn[-1])
         try:
             nextBtn[-1].click()
         except:
             nextBtn[-1].click()
     else:
         break
 
 browser.quit()
 del browser
 
 pd.DataFrame(listings).to_csv(csv_path, index=False)
 
 print('\n\nRemoved 1 duplicate each for rows with ID\n', ', '.join(
     f'{ri}' for ri in set(addedRows) if addedRows.count(ri) == 2
 ), '\nOther repeated rows:\n\t', '\n\t '.join([
     f'rowID_{ri} : skipped {addedRows.count(ri)-1} duplicates'
     for ri in set(addedRows) if addedRows.count(ri) > 2
 ]), f'\n\nsaved {len(listings)} to "{os.path.abspath(csv_path)}"')