How to web scrape from youtube using Selenium and Python
Question:
Code trilas:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
import json
options = Options()
options.headless = False
driver = webdriver.Chrome(options=options) //used to choose options from chrome//
driver.implicitly_wait(5)
baseurl = 'http://youtube.com'
keyword = input() #user input as earth
driver.get(f'{baseurl}/search?q= {keyword}')
I want to scrape data from the site http://youtube.com
Answers:
To extract the titles of the youtube search results using Selenium and python you have to induce WebDriverWait for visibility_of_all_elements_located() and you can use either of the following Locator Strategies:
-
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options, executable_path=r'C:WebDriverschromedriver.exe')
baseurl = "http://youtube.com"
keyword = input()
driver.get(f'{baseurl}/search?q={keyword}')
print([my_elem.text for my_elem in WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//yt-formatted-string[@class='style-scope ytd-video-renderer' and @aria-label]")))])
driver.quit()
-
Console Output:
earth
['Lil Dicky - Earth (Official Music Video)', 'The History of Earth - How Our Planet Formed - Full Documentary HD', "EARTH FROM SPACE: Like You've Never Seen Before", 'Lil Dicky - Earth (Lyrics)', 'Michael Jackson - Earth Song (Official Video)', 'Lil Dicky - Earth (CLEAN CENSORED VERSION)', 'Marshmello ft. Bastille - Happier (Official Music Video)', 'USA for Africa - We are the World', 'Lil Dicky - Freaky Friday feat. Chris Brown (Official Music Video)', 'What if Pluto Hits The Earth?', "15 Places on Earth Where Gravity Doesn't Seem to Work", 'Earth 101 | National Geographic', 'How Earth Moves', 'History Of Earth In 9 Minutes', 'What Happens If 1 mm Black Hole Appears On Earth?', 'Planet Earth seen from space (Full HD 1080p) ORIGINAL']
Code trilas:
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import time
import json
options = Options()
options.headless = False
driver = webdriver.Chrome(options=options) //used to choose options from chrome//
driver.implicitly_wait(5)
baseurl = 'http://youtube.com'
keyword = input() #user input as earth
driver.get(f'{baseurl}/search?q= {keyword}')
I want to scrape data from the site http://youtube.com
To extract the titles of the youtube search results using Selenium and python you have to induce WebDriverWait for visibility_of_all_elements_located() and you can use either of the following Locator Strategies:
-
Code Block:
from selenium import webdriver from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC options = webdriver.ChromeOptions() options.add_argument("start-maximized") options.add_experimental_option("excludeSwitches", ["enable-automation"]) options.add_experimental_option('useAutomationExtension', False) driver = webdriver.Chrome(options=options, executable_path=r'C:WebDriverschromedriver.exe') baseurl = "http://youtube.com" keyword = input() driver.get(f'{baseurl}/search?q={keyword}') print([my_elem.text for my_elem in WebDriverWait(driver, 20).until(EC.visibility_of_all_elements_located((By.XPATH, "//yt-formatted-string[@class='style-scope ytd-video-renderer' and @aria-label]")))]) driver.quit() -
Console Output:
earth ['Lil Dicky - Earth (Official Music Video)', 'The History of Earth - How Our Planet Formed - Full Documentary HD', "EARTH FROM SPACE: Like You've Never Seen Before", 'Lil Dicky - Earth (Lyrics)', 'Michael Jackson - Earth Song (Official Video)', 'Lil Dicky - Earth (CLEAN CENSORED VERSION)', 'Marshmello ft. Bastille - Happier (Official Music Video)', 'USA for Africa - We are the World', 'Lil Dicky - Freaky Friday feat. Chris Brown (Official Music Video)', 'What if Pluto Hits The Earth?', "15 Places on Earth Where Gravity Doesn't Seem to Work", 'Earth 101 | National Geographic', 'How Earth Moves', 'History Of Earth In 9 Minutes', 'What Happens If 1 mm Black Hole Appears On Earth?', 'Planet Earth seen from space (Full HD 1080p) ORIGINAL']