How can I read the contents of an URL with Python?

The URL should be a string:

import urllib

link = "http://www.somesite.com/details.pl?urn=2344"
f = urllib.urlopen(link)           
myfile = f.readline()  
print myfile

A solution with works with Python 2.X and Python 3.X makes use of the Python 2 and 3 compatibility library six:

from six.moves.urllib.request import urlopen
link = "http://www.somesite.com/details.pl?urn=2344"
response = urlopen(link)
content = response.read()
print(content)

I used the following code:

import urllib

def read_text():
      quotes = urllib.urlopen("https://s3.amazonaws.com/udacity-hosted-downloads/ud036/movie_quotes.txt")
      contents_file = quotes.read()
      print contents_file

read_text()

For python3 users, to save time, use the following code,

from urllib.request import urlopen

link = "https://docs.scipy.org/doc/numpy/user/basics.broadcasting.html"

f = urlopen(link)
myfile = f.read()
print(myfile)

I know there are different threads for error: Name Error: urlopen is not defined, but thought this might save time.

We can read website html content as below :

from urllib.request import urlopen
response = urlopen('http://google.com/')
html = response.read()
print(html)

None of these answers are very good for Python 3 (tested on latest version at the time of this post).

This is how you do it…

import urllib.request

try:
   with urllib.request.urlopen('http://www.python.org/') as f:
      print(f.read().decode('utf-8'))
except urllib.error.URLError as e:
   print(e.reason)

The above is for contents that return ‘utf-8’. Remove .decode(‘utf-8’) if you want python to "guess the appropriate encoding."

Documentation:
https://docs.python.org/3/library/urllib.request.html#module-urllib.request

#!/usr/bin/python
# -*- coding: utf-8 -*-
# Works on python 3 and python 2.
# when server knows where the request is coming from.

import sys

if sys.version_info[0] == 3:
    from urllib.request import urlopen
else:
    from urllib import urlopen
with urlopen('https://www.facebook.com/') as 
    url:
    data = url.read()

print data

# When the server does not know where the request is coming from.
# Works on python 3.

import urllib.request

user_agent = 
    'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'

url = 'https://www.facebook.com/'
headers = {'User-Agent': user_agent}

request = urllib.request.Request(url, None, headers)
response = urllib.request.urlopen(request)
data = response.read()
print data

# retrieving data from url
# only for python 3

import urllib.request

def main():
  url = "http://docs.python.org"

# retrieving data from URL
  webUrl = urllib.request.urlopen(url)
  print("Result code: " + str(webUrl.getcode()))

# print data from URL 
  print("Returned data: -----------------")
  data = webUrl.read().decode("utf-8")
  print(data)

if __name__ == "__main__":
  main()

from urllib.request import urlopen

# if has Chinese, apply decode()
html = urlopen("https://blog.csdn.net/qq_39591494/article/details/83934260").read().decode('utf-8')
print(html)

import requests
from bs4 import BeautifulSoup

link = "https://www.timeshighereducation.com/hub/sinorbis"

res = requests.get(link)
if res.status_code == 200:
    soup = BeautifulSoup(res, 'html.parser')

# get the text content of the webpage
text = soup.get_text()

print(text)

using BeautifulSoup‘s HTML parser we can extract the content of the webpage.