05_selenium_waits.py
crawl/intermediate/05_selenium_waits.py
"""
Intermediate Selenium Example 1: Explicit and Implicit Waits
=============================================================
This example demonstrates:
- Why waits are important
- Implicit waits
- Explicit waits with WebDriverWait
- Expected conditions
"""
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
import time
def setup_driver():
"""Set up Chrome WebDriver."""
chrome_options = Options()
chrome_options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36')
try:
return webdriver.Chrome(options=chrome_options)
except Exception as e:
print(f"❌ Error: {e}")
return None
def implicit_wait_example():
"""Demonstrate implicit waits."""
driver = setup_driver()
if not driver:
return
try:
print("="*60)
print("IMPLICIT WAIT EXAMPLE")
print("="*60)
# Set implicit wait (waits up to 10 seconds for elements to appear)
driver.implicitly_wait(10)
print("\n1. Set implicit wait to 10 seconds")
url = "https://quotes.toscrape.com/js/"
driver.get(url)
print(f"2. Opened: {url}")
# This will wait up to 10 seconds for the element to appear
quote = driver.find_element(By.CLASS_NAME, "quote")
print("3. ✅ Found quote element (waited if necessary)")
print(f" Quote text: {quote.find_element(By.CLASS_NAME, "text").text[:50]}...")
except Exception as e:
print(f"❌ Error: {e}")
finally:
driver.quit()
def explicit_wait_example():
"""Demonstrate explicit waits with WebDriverWait."""
driver = setup_driver()
if not driver:
return
try:
print("\n" + "="*60)
print("EXPLICIT WAIT EXAMPLE")
print("="*60)
url = "https://quotes.toscrape.com/js/"
driver.get(url)
print(f"\n1. Opened: {url}")
# Create WebDriverWait instance (waits up to 10 seconds)
wait = WebDriverWait(driver, 10)
# Wait for element to be present
print("2. Waiting for quote element to appear...")
quote = wait.until(
EC.presence_of_element_located((By.CLASS_NAME, "quote"))
)
print(" ✅ Quote element found!")
# Wait for element to be visible
print("3. Waiting for quote text to be visible...")
quote_text = wait.until(
EC.visibility_of_element_located((By.CLASS_NAME, "text"))
)
print(f" ✅ Quote text visible: {quote_text.text[:50]}...")
# Wait for element to be clickable
try:
next_button = wait.until(
EC.element_to_be_clickable((By.PARTIAL_LINK_TEXT, "Next"))
)
print("4. ✅ Next button is clickable!")
except:
print("4. ⚠️ Next button not found or not clickable")
except Exception as e:
print(f"❌ Error: {e}")
finally:
driver.quit()
def wait_for_text_example():
"""Wait for specific text to appear."""
driver = setup_driver()
if not driver:
return
try:
print("\n" + "="*60)
print("WAIT FOR TEXT EXAMPLE")
print("="*60)
url = "https://quotes.toscrape.com/"
driver.get(url)
print(f"\n1. Opened: {url}")
wait = WebDriverWait(driver, 10)
# Wait for specific text to be present in an element
print("2. Waiting for 'Quotes to Scrape' text...")
element = wait.until(
EC.text_to_be_present_in_element((By.TAG_NAME, "h1"), "Quotes to Scrape")
)
print(" ✅ Text found!")
# Get the heading
heading = driver.find_element(By.TAG_NAME, "h1")
print(f" Heading text: {heading.text}")
except Exception as e:
print(f"❌ Error: {e}")
finally:
driver.quit()
def wait_for_page_load_example():
"""Wait for page to load completely."""
driver = setup_driver()
if not driver:
return
try:
print("\n" + "="*60)
print("WAIT FOR PAGE LOAD EXAMPLE")
print("="*60)
url = "https://quotes.toscrape.com/js/"
driver.get(url)
print(f"\n1. Opened: {url}")
wait = WebDriverWait(driver, 10)
# Wait for document ready state
print("2. Waiting for page to load...")
wait.until(
lambda d: d.execute_script("return document.readyState") == "complete"
)
print(" ✅ Page loaded!")
# Wait for jQuery to be loaded (if the site uses it)
try:
wait.until(
lambda d: d.execute_script("return typeof jQuery !== 'undefined' && jQuery.active == 0")
)
print(" ✅ jQuery finished loading!")
except:
print(" ⚠️ Site doesn't use jQuery or it's already loaded")
except Exception as e:
print(f"❌ Error: {e}")
finally:
driver.quit()
if __name__ == "__main__":
implicit_wait_example()
explicit_wait_example()
wait_for_text_example()
wait_for_page_load_example()
print("\n" + "="*60)
print("All wait examples completed!")
print("="*60)
print("\n💡 Key points:")
print(" - Implicit waits: Set once, applies to all elements")
print(" - Explicit waits: More control, wait for specific conditions")
print(" - Always use waits instead of time.sleep() when possible")
print(" - Waits make your code more reliable and faster")
Articoli correlati
01_selenium_basics.py
01_selenium_basics.py — python source code from the crawl learning materials (crawl/advanced/01_selenium_basics.py).
Leggi l'articolo →02_async_scraping.py
02_async_scraping.py — python source code from the crawl learning materials (crawl/advanced/02_async_scraping.py).
Leggi l'articolo →quotes_spider.py
quotes_spider.py — python source code from the crawl learning materials (crawl/advanced/03_scrapy_example/quotes_spider.py).
Leggi l'articolo →04_database_integration.py
04_database_integration.py — python source code from the crawl learning materials (crawl/advanced/04_database_integration.py).
Leggi l'articolo →05_selenium_headless.py
05_selenium_headless.py — python source code from the crawl learning materials (crawl/advanced/05_selenium_headless.py).
Leggi l'articolo →06_selenium_advanced_interactions.py
06_selenium_advanced_interactions.py — python source code from the crawl learning materials (crawl/advanced/06_selenium_advanced_interactions.py).
Leggi l'articolo →