Apache Airflow dags w/ backend configuration bundle.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 

49 lines
1.5 KiB

import csv
import logging
import os
import platform
import random
import re
import time
# from webdriver_manager.chrome import ChromeDriverManager
from datetime import datetime
from urllib.request import urlopen
# import pyautogui
import pandas as pd
import yaml
from bs4 import BeautifulSoup
# import mouseinfo
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
def setupLogger():
dt = datetime.strftime(datetime.now(), "%m_%d_%y %H_%M_%S ")
if not os.path.isdir("./logs"):
os.mkdir("./logs")
# TODO need to check if there is a log dir available or not
logging.basicConfig(
filename=("./logs/" + str(dt) + "applyJobs.log"),
filemode="w",
format="%(asctime)s::%(name)s::%(levelname)s::%(message)s",
datefmt="./logs/%d-%b-%y %H:%M:%S",
)
log.setLevel(logging.DEBUG)
c_handler = logging.StreamHandler()
c_handler.setLevel(logging.DEBUG)
c_format = logging.Formatter(
"%(asctime)s - %(levelname)s - %(message)s", "%H:%M:%S"
)
c_handler.setFormatter(c_format)
log.addHandler(c_handler)