opening two instances of chrome webdriver(with and without proxies) with selenium python

Question:

I am trying to create two instances of chrome webdriver with selenium python. the first one uses proxies and the second one is proxyless.
The problem I am facing is that the URL opened in the second instance doesn’t load(because it uses proxies). I’ve implemented two different instances, but I still get this error. This is what I’ve written so far:

import os, tempfile
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.proxy import Proxy, ProxyType
from selenium import webdriver
# Set up Selenium with proxy and incognito mode
def setup_selenium_with_proxy(proxy):
    proxy_config = Proxy()
    proxy_config.proxy_type = ProxyType.MANUAL
    proxy_config.http_proxy = f"{proxy['ip']}:{proxy['port']}"
    proxy_config.ssl_proxy = f"{proxy['ip']}:{proxy['port']}"

    capabilities = webdriver.DesiredCapabilities.CHROME
    proxy_config.add_to_capabilities(capabilities)

    options = webdriver.ChromeOptions()
    options.add_argument("--incognito")
    options.add_argument("--disable-dev-shm-usage")
    options.add_argument("--no-sandbox")
    options.add_argument("--log-level=3")
    options.add_experimental_option("excludeSwitches", ["enable-automation", "enable-logging"])

    # Create a separate user data directory for the proxy browser
    user_data_dir_proxy = os.path.join(tempfile.gettempdir(), "selenium_proxy_profile")
    if not os.path.exists(user_data_dir_proxy):
        os.makedirs(user_data_dir_proxy)
    options.add_argument(f"--user-data-dir={user_data_dir_proxy}")

    driver = webdriver.Chrome(options=options, desired_capabilities=capabilities)
    return driver

# Set up Selenium without proxy and incognito mode
def setup_selenium_without_proxy():
    options = webdriver.ChromeOptions()
    options.add_argument("--incognito")
    options.add_argument("--disable-dev-shm-usage")
    options.add_argument("--no-sandbox")
    options.add_argument("--log-level=3")
    options.add_experimental_option("excludeSwitches", ["enable-automation", "enable-logging"])

    # Create a separate user data directory for the non-proxy browser
    user_data_dir_no_proxy = os.path.join(tempfile.gettempdir(), "selenium_no_proxy_profile")
    if not os.path.exists(user_data_dir_no_proxy):
        os.makedirs(user_data_dir_no_proxy)
    options.add_argument(f"--user-data-dir={user_data_dir_no_proxy}")

    driver = webdriver.Chrome(options=options)
    return driver

my main function works with the proxy driver first, and when I call the second proxyless driver, the website doesn’t work and shows website cannot be reached net::ERR_TUNNEL_CONNECTION_FAILED.

def main():
    url = "http://example.com"
    
    # Read proxies from 'http.txt' file
    with open('http.txt', 'r') as file:
        proxy_list = [{'ip': proxy.split(':')[0], 'port': proxy.split(':')[1].strip()} for proxy in file.readlines()]

    at_least_one_working_proxy = False
    proxy_index = 0

    while proxy_index < len(proxy_list):
        proxy = proxy_list[proxy_index]
        driver = setup_selenium_with_proxy(proxy)
        driver.set_page_load_timeout(20)
        try:
            driver.get(url)
            
            # code
            
            new_driver = setup_selenium_without_proxy()
            new_driver.set_page_load_timeout(20)
        except TimeoutException:
            print(f"Failed to open url using proxy {proxy['ip']}:{proxy['port']} within 20 seconds")
        except WebDriverException as e:
            print(f"WebDriverException encountered while using proxy {proxy['ip']}:{proxy['port']}: {e}")
Asked By: Boundless

||

Answers:

add the following line to your without_proxy function:

options.add_argument('--no-proxy-server')
Answered By: Mohammed Chaaraoui