How to continously scroll down a page until an element is located? Python selenium
Question:
I’m baffled by the load more button for the skills section in Linkedin. I receive this error in finding the xpath for the button: selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element
The issue is that my element is not visible on the page, so I’ve been trying to find a way to continuously scroll on the page until the button is visible. I’m trying to do a forloop of multiple profiles.
My relevant code:
import parameters
from selenium.webdriver.common.action_chains import ActionChains
from selenium import webdriver
ChromeOptions = webdriver.ChromeOptions()
driver = webdriver.Chrome('C:\Users\Root\Downloads\chromedriver.exe')
driver.get('https://www.linkedin.com/login?fromSignIn=true&trk=guest_homepage-basic_nav-header-signin')
sleep(0.5)
username = driver.find_element_by_name('session_key')
username.send_keys(parameters.linkedin_username)
sleep(0.5)
password = driver.find_element_by_name('session_password')
password.send_keys(parameters.linkedin_password)
sleep(0.5)
sign_in_button = driver.find_element_by_xpath('//button[@class="btn__primary--large from__button--floating"]')
sign_in_button.click()
driver.get('https://www.linkedin.com/in/kate-yun-yi-wang-054977127/?originalSubdomain=hk')
loadmore_skills=driver.find_element_by_xpath('//button[@class="pv-profile-section__card-action-bar pv-skills-section__additional-skills artdeco-container-card-action-bar artdeco-button artdeco-button--tertiary artdeco-button--3 artdeco-button--fluid"]')
Attempts
1.
actions = ActionChains(driver)
actions.move_to_element(loadmore_skills).perform()
#actions.move_to_element_with_offset(loadmore_skills, 0, 0).perform()
loadmore_skills.click()
With actions.move_to_element the page scrolls just below the element, so that the element is no longer visible, and the same error subsequently occurs.
I have also tried move_to_element_with_offset, but this hasn’t changed where the page scrolls to.
2.
coordinates = loadmore_skills.location_once_scrolled_into_view
driver.execute_script('window.scrollTo({}, {});'.format(coordinates['x'], coordinates['y']))
This returns the same error message
3.
loadmore_skills=WebDriverWait(driver,30).until(EC.element_to_be_clickable((By.XPATH, '//button[@class="pv-profile-section__card-action-bar pv-skills-section__additional-skills artdeco-container-card-action-bar artdeco-button artdeco-button--tertiary artdeco-button--3 artdeco-button--fluid"]')))
This returns the same error, as well.
4.
driver.execute_script("arguments[0].scrollIntoView();", loadmore_skills)
Not sure how else to do this. Your help is greatly appreciated.
UPDATE:
Trying @Dipak solution moves me to the bottom of the page, and the element cannot be clicked:
Error Traceback:
Traceback (most recent call last):
File "C:/Users/Root/PycharmProjects/Quant/skillstest.py", line 60, in <module>
EC.element_to_be_clickable((By.XPATH, "//span[text()='Show more']")))
File "C:UsersRootPycharmProjectsQuantvenvlibsite-packagesseleniumwebdriversupportwait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
Answers:
Try this
actions.move_to_element(loadmore_skills).build().perform()
Here is corresponding working Java Code
@Test
public void linkedInTest(){
driver.findElement(By.name("session_key")).sendKeys(username);
driver.findElement(By.name("session_password")).sendKeys(password);
driver.findElement(By.xpath("//button[@class='btn__primary--large from__button--floating']")).click();
driver.get("https://www.linkedin.com/in/kate-yun-yi-wang-054977127/?originalSubdomain=hk");
WebElement loadmore_skills = driver.findElement(By.xpath("//button[@class='pv-profile-section__card-action-bar pv-skills-section__additional-skills artdeco-container-card-action-bar artdeco-button artdeco-button--tertiary artdeco-button--3 artdeco-button--fluid']"));
action.moveToElement(loadmore_skills).click().build().perform();
}
Try below code :
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
driver = webdriver.Chrome(executable_path="C:New folderchromedriver.exe")
driver.get('https://www.linkedin.com/login?fromSignIn=true&trk=guest_homepage-basic_nav-header-signin')
driver.maximize_window()
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.NAME, "session_key"))).send_keys("email id")
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.NAME, "session_password"))).send_keys("password")
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, "//button[@class='btn__primary--large from__button--floating']"))).click()
driver.get("https://www.linkedin.com/in/kate-yun-yi-wang-054977127/?originalSubdomain=hk")
driver.maximize_window()
buttonClick = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, "//span[text()='Show more']")))
ActionChains(driver).move_to_element(buttonClick).click().perform()
Output:
Here’s the fully working code solving your problems as per you stated in the comments.
The answers suggested by others was actually scrolling down to bottom of the page and then also giving errors. Then i noticed that if you scroll drown to bottom then only the bottom section loads out, not all of those sections in between. (Dipak’s answer isn’t working for me also. Maybe this is a resolution problem for you, as well as for me : ) as stated by him in the chats)
Because what you want is present between the page & not in the bottom. So just the bottom section loads out always. So now we need to do something else.
What we want now, is to scroll down to only those sections which you want. And to make a custom scroll down i used driver.execute_script("scroll(0, 1600)"). I also removed all those unnecessary things from the code and kept it very simple and straight forward.
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
import time
driver = webdriver.Chrome(executable_path=r"C:UsersintelDownloadsSetupschromedriver")
driver.get('https://www.linkedin.com/login?fromSignIn=true&trk=guest_homepage-basic_nav-header-signin')
driver.maximize_window()
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.NAME, "session_key"))).send_keys("EMAIL")
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.NAME, "session_password"))).send_keys("PASSWORD")
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, "//button[@class='btn__primary--large from__button--floating']"))).click()
driver.get("https://www.linkedin.com/in/kate-yun-yi-wang-054977127/?originalSubdomain=hk")
driver.maximize_window()
driver.execute_script("scroll(0, 1600)")
time.sleep(5)
buttonClick = driver.find_element_by_xpath("/html/body/div[6]/div[4]/div[3]/div/div/div/div/div[2]/main/div[2]/div[6]/div/section/div[2]/button/span[1]").click()
Also use --headless browser to load the tasks more quicker. And if possible, then use css_selectors other than XPATH's, cauz they are the slowest locators for scraping purpose.
Here is a robust solution that can work with a bit of tweaking.
driver.get("url here")
elements_list = driver.find_elements(By.XPATH, "xpath to elements")
for element in elements_list:
elem_not_found = True
y = 10 # setting initial scroll pixel size to 10.. find what suits you
while elem_not_found:
try:
ActionChains(driver).scroll_by_amount(0, y).perform()
time.sleep(1)
hov = ActionChains(driver).move_to_element(element).perform()
WebDriverWait(driver, 5).until(
EC.presence_of_element_located((By.XPATH, "xpath here")))
#Once element is loaded do whatever you want with it
elem_not_found = False
except:
y += 35 # scrolling 35 pixels below if element is not found
This code keeps on scrolling the page until the element is not found!!
I’m baffled by the load more button for the skills section in Linkedin. I receive this error in finding the xpath for the button: selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element
The issue is that my element is not visible on the page, so I’ve been trying to find a way to continuously scroll on the page until the button is visible. I’m trying to do a forloop of multiple profiles.
My relevant code:
import parameters
from selenium.webdriver.common.action_chains import ActionChains
from selenium import webdriver
ChromeOptions = webdriver.ChromeOptions()
driver = webdriver.Chrome('C:\Users\Root\Downloads\chromedriver.exe')
driver.get('https://www.linkedin.com/login?fromSignIn=true&trk=guest_homepage-basic_nav-header-signin')
sleep(0.5)
username = driver.find_element_by_name('session_key')
username.send_keys(parameters.linkedin_username)
sleep(0.5)
password = driver.find_element_by_name('session_password')
password.send_keys(parameters.linkedin_password)
sleep(0.5)
sign_in_button = driver.find_element_by_xpath('//button[@class="btn__primary--large from__button--floating"]')
sign_in_button.click()
driver.get('https://www.linkedin.com/in/kate-yun-yi-wang-054977127/?originalSubdomain=hk')
loadmore_skills=driver.find_element_by_xpath('//button[@class="pv-profile-section__card-action-bar pv-skills-section__additional-skills artdeco-container-card-action-bar artdeco-button artdeco-button--tertiary artdeco-button--3 artdeco-button--fluid"]')
Attempts
1.
actions = ActionChains(driver)
actions.move_to_element(loadmore_skills).perform()
#actions.move_to_element_with_offset(loadmore_skills, 0, 0).perform()
loadmore_skills.click()
With actions.move_to_element the page scrolls just below the element, so that the element is no longer visible, and the same error subsequently occurs.
I have also tried move_to_element_with_offset, but this hasn’t changed where the page scrolls to.
2.
coordinates = loadmore_skills.location_once_scrolled_into_view
driver.execute_script('window.scrollTo({}, {});'.format(coordinates['x'], coordinates['y']))
This returns the same error message
3.
loadmore_skills=WebDriverWait(driver,30).until(EC.element_to_be_clickable((By.XPATH, '//button[@class="pv-profile-section__card-action-bar pv-skills-section__additional-skills artdeco-container-card-action-bar artdeco-button artdeco-button--tertiary artdeco-button--3 artdeco-button--fluid"]')))
This returns the same error, as well.
4.
driver.execute_script("arguments[0].scrollIntoView();", loadmore_skills)
Not sure how else to do this. Your help is greatly appreciated.
UPDATE:
Trying @Dipak solution moves me to the bottom of the page, and the element cannot be clicked:
Error Traceback:
Traceback (most recent call last):
File "C:/Users/Root/PycharmProjects/Quant/skillstest.py", line 60, in <module>
EC.element_to_be_clickable((By.XPATH, "//span[text()='Show more']")))
File "C:UsersRootPycharmProjectsQuantvenvlibsite-packagesseleniumwebdriversupportwait.py", line 80, in until
raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
Try this
actions.move_to_element(loadmore_skills).build().perform()
Here is corresponding working Java Code
@Test
public void linkedInTest(){
driver.findElement(By.name("session_key")).sendKeys(username);
driver.findElement(By.name("session_password")).sendKeys(password);
driver.findElement(By.xpath("//button[@class='btn__primary--large from__button--floating']")).click();
driver.get("https://www.linkedin.com/in/kate-yun-yi-wang-054977127/?originalSubdomain=hk");
WebElement loadmore_skills = driver.findElement(By.xpath("//button[@class='pv-profile-section__card-action-bar pv-skills-section__additional-skills artdeco-container-card-action-bar artdeco-button artdeco-button--tertiary artdeco-button--3 artdeco-button--fluid']"));
action.moveToElement(loadmore_skills).click().build().perform();
}
Try below code :
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
driver = webdriver.Chrome(executable_path="C:New folderchromedriver.exe")
driver.get('https://www.linkedin.com/login?fromSignIn=true&trk=guest_homepage-basic_nav-header-signin')
driver.maximize_window()
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.NAME, "session_key"))).send_keys("email id")
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.NAME, "session_password"))).send_keys("password")
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, "//button[@class='btn__primary--large from__button--floating']"))).click()
driver.get("https://www.linkedin.com/in/kate-yun-yi-wang-054977127/?originalSubdomain=hk")
driver.maximize_window()
buttonClick = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, "//span[text()='Show more']")))
ActionChains(driver).move_to_element(buttonClick).click().perform()
Output:
Here’s the fully working code solving your problems as per you stated in the comments.
The answers suggested by others was actually scrolling down to bottom of the page and then also giving errors. Then i noticed that if you scroll drown to bottom then only the bottom section loads out, not all of those sections in between. (Dipak’s answer isn’t working for me also. Maybe this is a resolution problem for you, as well as for me : ) as stated by him in the chats)
Because what you want is present between the page & not in the bottom. So just the bottom section loads out always. So now we need to do something else.
What we want now, is to scroll down to only those sections which you want. And to make a custom scroll down i used driver.execute_script("scroll(0, 1600)"). I also removed all those unnecessary things from the code and kept it very simple and straight forward.
from selenium import webdriver
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
import time
driver = webdriver.Chrome(executable_path=r"C:UsersintelDownloadsSetupschromedriver")
driver.get('https://www.linkedin.com/login?fromSignIn=true&trk=guest_homepage-basic_nav-header-signin')
driver.maximize_window()
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.NAME, "session_key"))).send_keys("EMAIL")
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.NAME, "session_password"))).send_keys("PASSWORD")
WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.XPATH, "//button[@class='btn__primary--large from__button--floating']"))).click()
driver.get("https://www.linkedin.com/in/kate-yun-yi-wang-054977127/?originalSubdomain=hk")
driver.maximize_window()
driver.execute_script("scroll(0, 1600)")
time.sleep(5)
buttonClick = driver.find_element_by_xpath("/html/body/div[6]/div[4]/div[3]/div/div/div/div/div[2]/main/div[2]/div[6]/div/section/div[2]/button/span[1]").click()
Also use --headless browser to load the tasks more quicker. And if possible, then use css_selectors other than XPATH's, cauz they are the slowest locators for scraping purpose.
Here is a robust solution that can work with a bit of tweaking.
driver.get("url here")
elements_list = driver.find_elements(By.XPATH, "xpath to elements")
for element in elements_list:
elem_not_found = True
y = 10 # setting initial scroll pixel size to 10.. find what suits you
while elem_not_found:
try:
ActionChains(driver).scroll_by_amount(0, y).perform()
time.sleep(1)
hov = ActionChains(driver).move_to_element(element).perform()
WebDriverWait(driver, 5).until(
EC.presence_of_element_located((By.XPATH, "xpath here")))
#Once element is loaded do whatever you want with it
elem_not_found = False
except:
y += 35 # scrolling 35 pixels below if element is not found
This code keeps on scrolling the page until the element is not found!!