You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
39 lines
1.3 KiB
39 lines
1.3 KiB
import time |
|
from datetime import date |
|
from time import sleep |
|
import numpy as np |
|
import logging as log |
|
|
|
def infinite_scoll(driver, module, log): |
|
print("Infinite scrolling...") |
|
"""A method for scrolling the page.""" |
|
|
|
last_height = driver.execute_script("return document.body.scrollHeight") |
|
|
|
while True: |
|
scrollresults = driver.find_element_by_class_name( |
|
"jobs-search-results" |
|
) |
|
# Selenium only detects visible elements; if we scroll to the bottom too fast, only 8-9 results will be loaded into IDs list |
|
for i in range(300, 3000, 100): |
|
driver.execute_script( |
|
"arguments[0].scrollTo(0, {})".format(i), scrollresults |
|
) |
|
|
|
time.sleep(1) |
|
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") |
|
print("Scrolled a page") |
|
# Wait to load the page. |
|
rand_pause = np.random.normal(5.0, 1.0) |
|
time.sleep(rand_pause) |
|
print(rand_pause) |
|
# Calculate new scroll height and compare with last scroll height. |
|
new_height = driver.execute_script("return document.body.scrollHeight") |
|
|
|
if new_height == last_height: |
|
break |
|
|
|
last_height = new_height |
|
|
|
print("Infinite scroll finished, screen grabbed") |
|
driver.save_screenshot("./data/scroll.png")
|
|
|