python selenium does not go to right page with scraper

Question

I have a scraper in python which with help of selenium goes to URL www.apartments.com then I enter (input) some location but it, with help of method of selenium .click() press search button but it only goes by default page which chicago, il.

here is code;

from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.keys import Keys

chrome_options = Options()
chrome_options.headless = False
chrome_options.add_argument("start-maximized")
# options.add_experimental_option("detach", True)
chrome_options.add_argument("--no-sandbox")
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('excludeSwitches', ['enable-logging'])
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.add_argument('--disable-blink-features=AutomationControlled')

driver = webdriver.Chrome(service=ChromeService(ChromeDriverManager().install()), options=chrome_options)
driver.get('https://www.apartments.com/')

driver.implicitly_wait(10)

var_inp='Santa Monica, CA'
#search for rentals
#driver.execute_script("arguments[0].setAttribute('value',arguments[1])",element, value)
#WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "search-box-input"))).send_keys(var_inp+Keys.RETURN)

def foo():
    element=WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "quickSearchLookup")))
    driver.execute_script("arguments[0].setAttribute('value','Santa Monica, CA')", element)
    #element.send_keys(var_inp)
    button=WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[@class='go']")))
    button.click()

foo()
##find it
##//div[contains(@class,"StyledPropertyCardDataArea")]/ul
#driver.execute_script("arguments[0].setAttribute('value',arguments[1])",driver.find_element(By.CSS_SELECTOR, "input#search-box-input"), 'New York, NY')

#driver.implicitly_wait(30)
prices=WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//p[@class='property-pricing']")))
beds=WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//p[@class='property-beds']")))
specials=WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, "//p[@class='property-specials']")))
lst_beds=[]
for bed in beds:
    lst_beds.append(bed.text)
lst_prices=[]
for price in prices:
    lst_prices.append(price.text)
lst_specials=[]
for special in specials:
    lst_specials.append(special.text)

print(lst_specials,lst_prices,lst_beds)

driver.quit()

Asked By: xlmaster

||

Source

Answer 1

You don’t need to set the attribute of the element explicitly.

If you click on the element, this will clear the text box and then wait for fraction of milliseconds and enter the value and then wait for fraction of milliseconds and then click on search button.

driver.get ("https://www.apartments.com/")
element=WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "quickSearchLookup")))
element.click()
time.sleep(0.5)
element.send_keys('Santa Monica, CA')
time.sleep(0.5)
button=WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, "//a[@class='go']")))
button.click()

browser snapshot:

Answered By: KunduK

python selenium does not go to right page with scraper

Question:

Answers: