69 lines
1.7 KiB
Python
69 lines
1.7 KiB
Python
from playwright.sync_api import sync_playwright
|
|
from bs4 import BeautifulSoup
|
|
import re
|
|
|
|
URL = "https://ulse.wd5.myworkdayjobs.com/ulsecareers/job/Evanston-IL/Data-Science---Engineering-Intern_JR1410-1?jr_id=6979179b39f7f96cc6d173c0"
|
|
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
page = browser.new_page()
|
|
page.goto(URL, wait_until="networkidle", timeout=60000)
|
|
|
|
# Wait a bit more for dynamic content (Workday can be heavy)
|
|
page.wait_for_timeout(2000)
|
|
|
|
html = page.content()
|
|
browser.close()
|
|
|
|
soup = BeautifulSoup(html, "lxml")
|
|
text = soup.get_text(" ", strip=True)
|
|
|
|
print("Rendered text length:", len(text))
|
|
print("Title:", soup.title.get_text(strip=True) if soup.title else None)
|
|
# print("\nSample:\n", text[:1500])
|
|
|
|
def extract_job_description(full_text):
|
|
lt = full_text.lower()
|
|
|
|
start_markers = [
|
|
"job description",
|
|
"about the role",
|
|
"job summary"
|
|
]
|
|
|
|
end_markers = [
|
|
"equal opportunity",
|
|
"eeo",
|
|
"ul is an equal",
|
|
"application process",
|
|
"how to apply",
|
|
"privacy",
|
|
"cookies"
|
|
]
|
|
|
|
start_idx = -1
|
|
for m in start_markers:
|
|
i = lt.find(m)
|
|
if i != -1:
|
|
start_idx = i + len(m)
|
|
break
|
|
|
|
if start_idx == -1:
|
|
return None
|
|
|
|
end_idx = len(full_text)
|
|
for m in end_markers:
|
|
i = lt.find(m, start_idx)
|
|
if i != -1:
|
|
end_idx = i
|
|
break
|
|
|
|
description = full_text[start_idx:end_idx].strip()
|
|
return description
|
|
|
|
job_desc = extract_job_description(text)
|
|
|
|
print("\n--- CLEAN JOB DESCRIPTION ---\n")
|
|
print(job_desc if job_desc else "No job description found.")
|
|
print("\nLength:", len(job_desc))
|