How do I write all my BeautifulSoup data from a website to a text file? Python

Question:

I am trying to read data from open insider and put it into an easy to read text file. Here is my code so far:

from bs4 import BeautifulSoup
import requests

page = requests.get("http://openinsider.com/top-insider-purchases-of-the-month")

'''print(page.status_code)
checks to see if the page was downloaded successfully'''

soup = BeautifulSoup(page.content,'html.parser')

# Find the table with the insider purchase data
table = soup.find(class_="tinytable")

# Find all rows of the table
rows = table.find_all('tr')

# Loop through each row
for row in rows:
    # Extract the company name, insider name, and trade type from the row
    data = row.find_all("td")
    ticker = data[3].get_text() if len(data) > 3 else "Ticker representing the company"
    company = data[4].get_text() if len(data) > 4 else "Name"
    insider = data[6].get_text() if len(data) > 6 else "Position of trader"
    trade_type = data[7].get_text() if len(data) > 7 else "Buy or sell"
    value = data[12].get_text() if len(data) > 12 else "Monetary value"
    # Print the extracted data
    print(f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}')


    with open('TopInsiderMonth.txt', 'w') as f:
        f.write(f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}')

My print function is giving me the results I want but when I try to write it I only write one line of the information instead of the 100 lines I am looking for. Any ideas?

Asked By: Daniel

||

Answers:

Open file then the for loop and added n for new line:

from bs4 import BeautifulSoup
import requests

page = requests.get("http://openinsider.com/top-insider-purchases-of-the-month")

'''print(page.status_code)
checks to see if the page was downloaded successfully'''

soup = BeautifulSoup(page.content,'html.parser')

# Find the table with the insider purchase data
table = soup.find(class_="tinytable")

# Find all rows of the table
rows = table.find_all('tr')

with open('TopInsiderMonth.txt', 'w') as f:
    
    # Loop through each row
    for row in rows:
        # Extract the company name, insider name, and trade type from the row
        data = row.find_all("td")
        ticker = data[3].get_text() if len(data) > 3 else "Ticker representing the company"
        company = data[4].get_text() if len(data) > 4 else "Name"
        insider = data[6].get_text() if len(data) > 6 else "Position of trader"
        trade_type = data[7].get_text() if len(data) > 7 else "Buy or sell"
        value = data[12].get_text() if len(data) > 12 else "Monetary value"
        # Print the extracted data
        print(f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}')

        f.write(
            f'Ticker : {ticker} | Company: {company} | Title: {insider} | Trade Type: {trade_type} | Value: {value}n')
Answered By: Leo Ward
Categories: questions Tags: , ,
Answers are sorted by their score. The answer accepted by the question owner as the best is marked with
at the top-right corner.