import time from datetime import date from time import sleep import numpy as np import logging as log def infinite_scoll(driver, module, log): print("Infinite scrolling...") """A method for scrolling the page.""" last_height = driver.execute_script("return document.body.scrollHeight") while True: scrollresults = driver.find_element_by_class_name( "jobs-search-results" ) # Selenium only detects visible elements; if we scroll to the bottom too fast, only 8-9 results will be loaded into IDs list for i in range(300, 3000, 100): driver.execute_script( "arguments[0].scrollTo(0, {})".format(i), scrollresults ) time.sleep(1) # driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") # print("Scrolled a page") # # Wait to load the page. # rand_pause = np.random.normal(5.0, 1.0) # time.sleep(rand_pause) # print(rand_pause) # # Calculate new scroll height and compare with last scroll height. # new_height = driver.execute_script("return document.body.scrollHeight") # if new_height == last_height: # break # last_height = new_height print("Infinite scroll finished, screen grabbed") driver.save_screenshot("./data/scroll.png")