How do I write all my BeautifulSoup data from a website to a text file? Python
Question:
I am trying to read data from open insider and put it into an easy to read text file. Here is my code so far:
from bs4 import BeautifulSoup
import requests
page = requests.get("http://openinsider.com/top-insider-purchases-of-the-month")
'''print(page.status_code)
checks to see if the page was downloaded successfully'''
soup = BeautifulSoup(page.content,'html.parser')
# Find the table with the insider purchase data
table = soup.find(class_="tinytable")
# Find all rows of the table
rows = table.find_all('tr')
# Loop through each row
for row in rows:
# Extract the company name, insider name, and trade type from the row
data = row.find_all("td")
ticker = data[3].get_text() if len(data) > 3 else "Ticker representing the company"
company = data[4].get_text() if len(data) > 4 else "Name"
insider = data[6].get_text() if len(data) > 6 else "Position of trader"
trade_type = data[7].get_text() if len(data) > 7 else "Buy or sell"
value = data[12].get_text() if len(data) > 12 else "Monetary value"
# Print the extracted data
print(f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}')
with open('TopInsiderMonth.txt', 'w') as f:
f.write(f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}')
My print function is giving me the results I want but when I try to write it I only write one line of the information instead of the 100 lines I am looking for. Any ideas?
Answers:
Open file then the for loop and added n for new line:
from bs4 import BeautifulSoup
import requests
page = requests.get("http://openinsider.com/top-insider-purchases-of-the-month")
'''print(page.status_code)
checks to see if the page was downloaded successfully'''
soup = BeautifulSoup(page.content,'html.parser')
# Find the table with the insider purchase data
table = soup.find(class_="tinytable")
# Find all rows of the table
rows = table.find_all('tr')
with open('TopInsiderMonth.txt', 'w') as f:
# Loop through each row
for row in rows:
# Extract the company name, insider name, and trade type from the row
data = row.find_all("td")
ticker = data[3].get_text() if len(data) > 3 else "Ticker representing the company"
company = data[4].get_text() if len(data) > 4 else "Name"
insider = data[6].get_text() if len(data) > 6 else "Position of trader"
trade_type = data[7].get_text() if len(data) > 7 else "Buy or sell"
value = data[12].get_text() if len(data) > 12 else "Monetary value"
# Print the extracted data
print(f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}')
f.write(
f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}n')
I am trying to read data from open insider and put it into an easy to read text file. Here is my code so far:
from bs4 import BeautifulSoup
import requests
page = requests.get("http://openinsider.com/top-insider-purchases-of-the-month")
'''print(page.status_code)
checks to see if the page was downloaded successfully'''
soup = BeautifulSoup(page.content,'html.parser')
# Find the table with the insider purchase data
table = soup.find(class_="tinytable")
# Find all rows of the table
rows = table.find_all('tr')
# Loop through each row
for row in rows:
# Extract the company name, insider name, and trade type from the row
data = row.find_all("td")
ticker = data[3].get_text() if len(data) > 3 else "Ticker representing the company"
company = data[4].get_text() if len(data) > 4 else "Name"
insider = data[6].get_text() if len(data) > 6 else "Position of trader"
trade_type = data[7].get_text() if len(data) > 7 else "Buy or sell"
value = data[12].get_text() if len(data) > 12 else "Monetary value"
# Print the extracted data
print(f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}')
with open('TopInsiderMonth.txt', 'w') as f:
f.write(f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}')
My print function is giving me the results I want but when I try to write it I only write one line of the information instead of the 100 lines I am looking for. Any ideas?
Open file then the for loop and added n for new line:
from bs4 import BeautifulSoup
import requests
page = requests.get("http://openinsider.com/top-insider-purchases-of-the-month")
'''print(page.status_code)
checks to see if the page was downloaded successfully'''
soup = BeautifulSoup(page.content,'html.parser')
# Find the table with the insider purchase data
table = soup.find(class_="tinytable")
# Find all rows of the table
rows = table.find_all('tr')
with open('TopInsiderMonth.txt', 'w') as f:
# Loop through each row
for row in rows:
# Extract the company name, insider name, and trade type from the row
data = row.find_all("td")
ticker = data[3].get_text() if len(data) > 3 else "Ticker representing the company"
company = data[4].get_text() if len(data) > 4 else "Name"
insider = data[6].get_text() if len(data) > 6 else "Position of trader"
trade_type = data[7].get_text() if len(data) > 7 else "Buy or sell"
value = data[12].get_text() if len(data) > 12 else "Monetary value"
# Print the extracted data
print(f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}')
f.write(
f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}n')