How can I scrape a specific URL from a webpage using BeautifulSoup?
Question:
I’m writing a Python script that parses HTML (a classifieds website) and sends me email notifications on specific products and price points. Everything works here except for the "listing_url" capture, which I want displayed in the email so I can click on the url to visit the product page. I tried scraping the corresponding tag with "class=listing-card__inner" as inspected on the website but this isn’t working.
This is my full code:
import requests
from bs4 import BeautifulSoup
import smtplib
from email.mime.text import MIMEText
# Email details
sender_email = "[email protected]"
receiver_email = "[email protected]"
smtp_server = "smtp.gmail.com"
smtp_port = 587
smtp_username = "[email protected]"
smtp_password = "xxxxxxxx"
# Send email function
def send_email(subject, body):
message = MIMEText(body)
message['Subject'] = subject
message['From'] = sender_email
message['To'] = receiver_email
with smtplib.SMTP(smtp_server, smtp_port) as server:
server.starttls()
server.login(smtp_username, smtp_password)
server.sendmail(sender_email, receiver_email, message.as_string())
# Scrape listings function
def scrape_listings(url):
# Make a GET request to the website
response = requests.get(url)
# Parse the HTML content
soup = BeautifulSoup(response.content, 'html.parser')
# Find all the laptop listings
listings = soup.find_all('div', class_='listing-card__content')
# List to store qualifying listings
qualifying_listings = []
# Iterate through listings and check conditions
for listing in listings:
title = listing.find('div', class_='listing-card__header__title').text.strip()
date = listing.find('div', class_='listing-card__header__date').text.strip()
price = listing.find('span', class_='listing-card__price__value 1').text.strip()
price = price.replace('Br', '').replace(',', '').strip()
price = int(price)
listing_url = listing.find('a', class_='listing-card__inner')['href']
if price < 80000:
qualifying_listings.append((title, date, price, listing_url))
return qualifying_listings
# Main function
def main():
base_url = "https://www.qefira.com/classifieds?q=gaming%20laptops&sort=latest"
qualifying_listings = []
# Scrape first page
qualifying_listings += scrape_listings(base_url)
# Scrape remaining pages, limited to 8
page = 2 # Start from the second page
while page <= 9: # Limit to 8 pages (2 to 9)
url = base_url + f"&page={page}"
qualifying_listings += scrape_listings(url)
page += 1
# Prepare email subject and body
subject = "Gaming Laptops under 80,000"
total_count = len(qualifying_listings)
body = f"Total Qualifying Listings: {total_count}nn"
for listing in qualifying_listings:
title, date, price, listing_url = listing
body += f"Title: {title}n"
body += f"Date Posted: {date}n"
body += f"Price: {price}n"
body += f"URL: {listing_url}n"
body += "n"
# Send the email
send_email(subject, body)
# Run the script
if __name__ == '__main__':
main()
The following is a shortened sample of the tag on the website from which I am trying to scrape the URL in the href object on the very first line:
<a href="https://www.qefira.com/listings/hpelitebook-core-i5-4th-laptops-5366113" class="listing-card__inner" id="listing-5366113" data-t-listing="" data-t-listing_context="search" data-t-listing_id="5366113" data-t-listing_title="HpElitebook core i5 4th Laptops" data-t-listing_type="classified" data-t-listing_category_title="Laptops" data-t-listing_category_slug="laptops" data-t-listing_slug="hpelitebook-core-i5-4th-laptops" data-t-listing_price="19500.00" data-t-listing_currency="ETB" data-t-listing_location_title="Bole" data-t-listing_source="qe_et" data-t-listing_product_slugs="listing">...</a>
Any guidance on how I can successfully scrape the URL so it can appear as a clickable link inside the email body?
Tried different iterations of the .find method such as:
listing_url_element = listing.find('a', class_='listing-card__inner')
listing_url = listing_url_element['href'] if listing_url_element else "N/A"
or
listing_url = listing.find('a', class_='listing-card__title')['href']
or
listing_url_element = listing.find('a', class_='listing-card__inner')
listing_url = listing_url_element['href'] if listing_url_element and 'href' in listing_url_element.attrs else "N/A"
or even:
import re
listing_url_element = listing.find('a', class_='listing-card__inner')
listing_url_match = re.search(r'href="(.*?)"', str(listing_url_element))
listing_url = listing_url_match.group(1) if listing_url_match else "N/A"
Still nothing (or the "N/A" is displayed rather than the actual URL). The common error is something like:
listing_url = listing.find('a', class_='listing-card__inner')['href']
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^
TypeError: 'NoneType' object is not subscriptable
Answers:
I advise you to use this class ‘listing-card__content’ instead of this one ‘listing-card–has-content’. I also added a check to avoid the code execution stopping.
Here is the full code:
import requests
from bs4 import BeautifulSoup
import smtplib
from email.mime.text import MIMEText
# Email details
sender_email = "[email protected]"
receiver_email = "[email protected]"
smtp_server = "smtp.gmail.com"
smtp_port = 587
smtp_username = "[email protected]"
smtp_password = "xxxxxxxx"
# Send email function
def send_email(subject, body):
message = MIMEText(body)
message['Subject'] = subject
message['From'] = sender_email
message['To'] = receiver_email
with smtplib.SMTP(smtp_server, smtp_port) as server:
server.starttls()
server.login(smtp_username, smtp_password)
server.sendmail(sender_email, receiver_email, message.as_string())
# Scrape listings function
def scrape_listings(url):
# Make a GET request to the website
response = requests.get(url)
# Parse the HTML content
soup = BeautifulSoup(response.content, 'html.parser')
# Find all the laptop listings
# listings = soup.find_all('div', class_='listing-card__content')
listings = soup.find_all('div', class_='listing-card--has-content')
# List to store qualifying listings
qualifying_listings = []
# Iterate through listings and check conditions
for listing in listings:
title = listing.find('div', class_='listing-card__header__title').text.strip()
date = listing.find('div', class_='listing-card__header__date').text.strip()
price = listing.find('span', class_='listing-card__price__value 1').text.strip()
price = price.replace('Br', '').replace(',', '').strip()
price = int(price)
a = listing.find('a', class_='listing-card__inner')
if a:
listing_url = a['href']
print(listing_url)
if price < 80000:
qualifying_listings.append((title, date, price, listing_url))
return qualifying_listings
# Main function
def main():
base_url = "https://www.qefira.com/classifieds?q=gaming%20laptops&sort=latest"
qualifying_listings = []
# Scrape first page
qualifying_listings += scrape_listings(base_url)
# Scrape remaining pages, limited to 8
page = 2 # Start from the second page
while page <= 9: # Limit to 8 pages (2 to 9)
url = base_url + f"&page={page}"
qualifying_listings += scrape_listings(url)
page += 1
# Prepare email subject and body
subject = "Gaming Laptops under 80,000"
total_count = len(qualifying_listings)
body = f"Total Qualifying Listings: {total_count}nn"
for listing in qualifying_listings:
title, date, price, listing_url = listing
body += f"Title: {title}n"
body += f"Date Posted: {date}n"
body += f"Price: {price}n"
body += f"URL: {listing_url}n"
body += "n"
# Send the email
send_email(subject, body)
# Run the script
if __name__ == '__main__':
main()
I’m writing a Python script that parses HTML (a classifieds website) and sends me email notifications on specific products and price points. Everything works here except for the "listing_url" capture, which I want displayed in the email so I can click on the url to visit the product page. I tried scraping the corresponding tag with "class=listing-card__inner" as inspected on the website but this isn’t working.
This is my full code:
import requests
from bs4 import BeautifulSoup
import smtplib
from email.mime.text import MIMEText
# Email details
sender_email = "[email protected]"
receiver_email = "[email protected]"
smtp_server = "smtp.gmail.com"
smtp_port = 587
smtp_username = "[email protected]"
smtp_password = "xxxxxxxx"
# Send email function
def send_email(subject, body):
message = MIMEText(body)
message['Subject'] = subject
message['From'] = sender_email
message['To'] = receiver_email
with smtplib.SMTP(smtp_server, smtp_port) as server:
server.starttls()
server.login(smtp_username, smtp_password)
server.sendmail(sender_email, receiver_email, message.as_string())
# Scrape listings function
def scrape_listings(url):
# Make a GET request to the website
response = requests.get(url)
# Parse the HTML content
soup = BeautifulSoup(response.content, 'html.parser')
# Find all the laptop listings
listings = soup.find_all('div', class_='listing-card__content')
# List to store qualifying listings
qualifying_listings = []
# Iterate through listings and check conditions
for listing in listings:
title = listing.find('div', class_='listing-card__header__title').text.strip()
date = listing.find('div', class_='listing-card__header__date').text.strip()
price = listing.find('span', class_='listing-card__price__value 1').text.strip()
price = price.replace('Br', '').replace(',', '').strip()
price = int(price)
listing_url = listing.find('a', class_='listing-card__inner')['href']
if price < 80000:
qualifying_listings.append((title, date, price, listing_url))
return qualifying_listings
# Main function
def main():
base_url = "https://www.qefira.com/classifieds?q=gaming%20laptops&sort=latest"
qualifying_listings = []
# Scrape first page
qualifying_listings += scrape_listings(base_url)
# Scrape remaining pages, limited to 8
page = 2 # Start from the second page
while page <= 9: # Limit to 8 pages (2 to 9)
url = base_url + f"&page={page}"
qualifying_listings += scrape_listings(url)
page += 1
# Prepare email subject and body
subject = "Gaming Laptops under 80,000"
total_count = len(qualifying_listings)
body = f"Total Qualifying Listings: {total_count}nn"
for listing in qualifying_listings:
title, date, price, listing_url = listing
body += f"Title: {title}n"
body += f"Date Posted: {date}n"
body += f"Price: {price}n"
body += f"URL: {listing_url}n"
body += "n"
# Send the email
send_email(subject, body)
# Run the script
if __name__ == '__main__':
main()
The following is a shortened sample of the tag on the website from which I am trying to scrape the URL in the href object on the very first line:
<a href="https://www.qefira.com/listings/hpelitebook-core-i5-4th-laptops-5366113" class="listing-card__inner" id="listing-5366113" data-t-listing="" data-t-listing_context="search" data-t-listing_id="5366113" data-t-listing_title="HpElitebook core i5 4th Laptops" data-t-listing_type="classified" data-t-listing_category_title="Laptops" data-t-listing_category_slug="laptops" data-t-listing_slug="hpelitebook-core-i5-4th-laptops" data-t-listing_price="19500.00" data-t-listing_currency="ETB" data-t-listing_location_title="Bole" data-t-listing_source="qe_et" data-t-listing_product_slugs="listing">...</a>
Any guidance on how I can successfully scrape the URL so it can appear as a clickable link inside the email body?
Tried different iterations of the .find method such as:
listing_url_element = listing.find('a', class_='listing-card__inner')
listing_url = listing_url_element['href'] if listing_url_element else "N/A"
or
listing_url = listing.find('a', class_='listing-card__title')['href']
or
listing_url_element = listing.find('a', class_='listing-card__inner')
listing_url = listing_url_element['href'] if listing_url_element and 'href' in listing_url_element.attrs else "N/A"
or even:
import re
listing_url_element = listing.find('a', class_='listing-card__inner')
listing_url_match = re.search(r'href="(.*?)"', str(listing_url_element))
listing_url = listing_url_match.group(1) if listing_url_match else "N/A"
Still nothing (or the "N/A" is displayed rather than the actual URL). The common error is something like:
listing_url = listing.find('a', class_='listing-card__inner')['href']
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^
TypeError: 'NoneType' object is not subscriptable
I advise you to use this class ‘listing-card__content’ instead of this one ‘listing-card–has-content’. I also added a check to avoid the code execution stopping.
Here is the full code:
import requests
from bs4 import BeautifulSoup
import smtplib
from email.mime.text import MIMEText
# Email details
sender_email = "[email protected]"
receiver_email = "[email protected]"
smtp_server = "smtp.gmail.com"
smtp_port = 587
smtp_username = "[email protected]"
smtp_password = "xxxxxxxx"
# Send email function
def send_email(subject, body):
message = MIMEText(body)
message['Subject'] = subject
message['From'] = sender_email
message['To'] = receiver_email
with smtplib.SMTP(smtp_server, smtp_port) as server:
server.starttls()
server.login(smtp_username, smtp_password)
server.sendmail(sender_email, receiver_email, message.as_string())
# Scrape listings function
def scrape_listings(url):
# Make a GET request to the website
response = requests.get(url)
# Parse the HTML content
soup = BeautifulSoup(response.content, 'html.parser')
# Find all the laptop listings
# listings = soup.find_all('div', class_='listing-card__content')
listings = soup.find_all('div', class_='listing-card--has-content')
# List to store qualifying listings
qualifying_listings = []
# Iterate through listings and check conditions
for listing in listings:
title = listing.find('div', class_='listing-card__header__title').text.strip()
date = listing.find('div', class_='listing-card__header__date').text.strip()
price = listing.find('span', class_='listing-card__price__value 1').text.strip()
price = price.replace('Br', '').replace(',', '').strip()
price = int(price)
a = listing.find('a', class_='listing-card__inner')
if a:
listing_url = a['href']
print(listing_url)
if price < 80000:
qualifying_listings.append((title, date, price, listing_url))
return qualifying_listings
# Main function
def main():
base_url = "https://www.qefira.com/classifieds?q=gaming%20laptops&sort=latest"
qualifying_listings = []
# Scrape first page
qualifying_listings += scrape_listings(base_url)
# Scrape remaining pages, limited to 8
page = 2 # Start from the second page
while page <= 9: # Limit to 8 pages (2 to 9)
url = base_url + f"&page={page}"
qualifying_listings += scrape_listings(url)
page += 1
# Prepare email subject and body
subject = "Gaming Laptops under 80,000"
total_count = len(qualifying_listings)
body = f"Total Qualifying Listings: {total_count}nn"
for listing in qualifying_listings:
title, date, price, listing_url = listing
body += f"Title: {title}n"
body += f"Date Posted: {date}n"
body += f"Price: {price}n"
body += f"URL: {listing_url}n"
body += "n"
# Send the email
send_email(subject, body)
# Run the script
if __name__ == '__main__':
main()