79241510

Date: 2024-12-01 12:10:36
Score: 1
Natty:
Report link

Thanks to Yaroslavm's suggestion on the stackoverflow dated Apr. 7, 2024(Click on element in shadow-root Selenium Python), this solution has been accomplished. The point is to wait enough before clicking the button inside shadow DOM. In this case, I found that we have to use time.sleep instead of Webdriverwait.

By the way, use of lxml.html may contribute to faster collecting of the attributes of listings.

import lxml.html
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from time import sleep

driver = webdriver.Chrome()
driver.implicitly_wait(5)
actions = ActionChains(driver)
url='https://es.wallapop.com/app/search?engine=gasoline&min_horse_power=1&max_horse_power=1000&min_year=2006&max_year=2024&min_km=40000&max_km=200000&min_sale_price=1000&max_sale_price=90000&gearbox=automatic,manual&brand=Honda&model=Civic&filters_source=default_filters&keywords=&category_ids=100&longitude=-3.69196&latitude=40.41956&order_by=price_high_to_low'

driver.get(url)
sleep(8)
WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//body")))

#cookie accept
driver.find_element(By.XPATH,"//button[@id='onetrust-accept-btn-handler']").click()
sleep(20)

#click load more button
for i in range(2):
    try:
        button =WebDriverWait(driver, 20).until(EC.presence_of_element_located((By.XPATH,"//walla-button[@id='btn-load-more']")))
        
    except:
        break
    else:
        driver.execute_script('arguments[0].scrollIntoView({behavior: "auto", block: "center"});', button)
        shadow = button.shadow_root#only currently works in Chromium based browsers
        
        button2 =WebDriverWait(shadow, 20).until(EC.presence_of_element_located((By.CSS_SELECTOR,  "button.walla-button__button--medium")))
        
        sleep(10)#fail without this
        #WebDriverWait(driver, 50).until(EC.element_to_be_clickable(button2))#This is not effective
        button2.click()
        print('\nload more button clicked-- {}'.format(i+1))


#load more by scrolling down
pre_count = 0
same_count_times = 0
for i in range(40):
    sleep(2)
    for pd in range(20):
        actions.send_keys(Keys.PAGE_DOWN).perform()
    
    WebDriverWait(driver, 20).until(EC.presence_of_all_elements_located((By.XPATH, "//body")))
    #get whole html (by Selenium) 
    page = driver.page_source # type : text 
    #transfer to lxml
    p_root = lxml.html.fromstring(page)
    
    list_elements = p_root.xpath("//a[@class='ItemCardList__item']") 
    new_count = len(list_elements)
    print('len(list_elements): ',new_count)
    if new_count == pre_count:
        same_count_times += 1
    else:
        same_count_times = 0
    pre_count = new_count
    if same_count_times > 2:
        break

while True:
    a= input('Do you close this session?')
    if a == 'y':
        break
Reasons:
  • Blacklisted phrase (0.5): Thanks
  • Blacklisted phrase (1): stackoverflow
  • Long answer (-1):
  • Has code block (-0.5):
  • Low reputation (1):
Posted by: yamskii-k