How can I scrape "Net Worth" from YieldWatch?
Question:
I would to scrape a website as per following:
- Go to Yieldwatch
- Enter
0xF54274757Bf717B1ab52bA0d3a7CbF635f856a0d
into the Address textbox and click the binoculars
- Scrape the text under "Net worth"
I am using Selenium in Python to try and do it and here is my attempt so far:
from selenium.webdriver import Safari # pip install selenium
from selenium.webdriver.support.ui import WebDriverWait
url = 'https://www.yieldwatch.net/'
bsc_public_key = '0xF54274757Bf717B1ab52bA0d3a7CbF635f856a0d'
# with closing(Safari()) as browser:
browser = Safari()
browser.get(url)
textbox = browser.find_element_by_id('addressInputField')
textbox.clear()
textbox.send_keys(bsc_public_key)
button = browser.find_element_by_class_name('binoculars icon')
button.click()
# # wait for the page to load
WebDriverWait(browser, timeout=20).until(
lambda x: x.find_element_by_id('ui centered image'))
# store it to string variable
page_source = browser.page_source
print(page_source)
The code doesn’t work. After the browser loads, I don’t see the textbox filled in with the address. How can I do this in Python (with or without Selenium)?
Answers:
You are trying to read the ui centered image
element text when it is still not presenting the final value.
Try waiting for invisibility_of_element_located(By.XPATH,"//div[@class='content']//div[@class='header']")
And only after that read the text from ui centered image
element.
You don’t really need the heavy guns of selenium
. You can get all the data from the API endpoint.
Here’s how:
import requests
wallet = "0xF54274757Bf717B1ab52bA0d3a7CbF635f856a0d"
endpoint = f"https://www.yieldwatch.net/api/all/{wallet}?platforms=beefy,pancake,hyperjump,auto,mdex"
wallet_data = requests.get(endpoint).json()["result"]
# https://yieldwatch.medium.com/yieldwatch-faqs-93c2cde244bf
# Net Value = Total Deposit + Total Yield + Wallet Balance — Total Debt
total = sum(v["totalUSDValues"]["total"] for v in wallet_data["PancakeSwap"].values())
net_worth = total + wallet_data["walletBalance"]["totalUSDValue"]
print(f"Net worth for {wallet}:n{round(net_worth, 2)}")
This outputs most up-to-date net worth:
Net worth for 0xF54274757Bf717B1ab52bA0d3a7CbF635f856a0d:
2213.13
Note: The FAQ says you need Total Debt
to calculate the net worth but this wallet doesn’t have any debt so I didn’t include the value in the equation.
However, if you have a wallet that has a debt, please share it, and I’ll update the answer.
from selenium.webdriver import Safari
from selenium.webdriver.common.by import By
import time
url = 'https://www.yieldwatch.net/'
address = input('Enter Your Wallet Address : ')
browser = Safari()
browser.get(url)
btnpancake = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[1]/div/div/div[4]/div/div[4]/div/img")
btnpancake.click()
wallet = browser.find_element(By.ID, "addressInputField")
wallet.clear()
wallet.send_keys(address)
send = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[1]/div/div/div[1]/div/div[2]/button[1]")
send.click()
time.sleep(7)
pool = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[2]/div[3]/div/div/div[2]/div/div/div[2]/div/div/div/div/div[1]/div[1]/h4/div[2]")
poolval = pool.text
value = poolval.split(' ')
print(f"Current Pool : {value[0]}")
print(f"Total Cake Value : {value[2]}")
earn = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[2]/div[3]/div/div/div[2]/div/div/div[2]/div/div/div/div/div[2]/table/tbody/tr[1]/td[2]")
earnval = earn.text
value2 = earnval.split(' ')
print(f"Total Cake : {value2[0]}")
print(f"Pending {value2[3]} : {value2[2]}")
print(f"Harvest : {value2[4]+' '+value2[5]}")
totalyield = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[2]/div[2]/div/div/div/div[3]/span/span")
totalyieldval = totalyield.text
print(f"Total Yield : {totalyieldval}")
apr = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[2]/div[3]/div/div/div[2]/div/div/div[2]/div/div/div/div/div[2]/table/tbody/tr[1]/td[4]/h4/div")
aprval = apr.text
print(f"APR : {aprval}")
browser.quit()
I would to scrape a website as per following:
- Go to Yieldwatch
- Enter
0xF54274757Bf717B1ab52bA0d3a7CbF635f856a0d
into the Address textbox and click the binoculars - Scrape the text under "Net worth"
I am using Selenium in Python to try and do it and here is my attempt so far:
from selenium.webdriver import Safari # pip install selenium
from selenium.webdriver.support.ui import WebDriverWait
url = 'https://www.yieldwatch.net/'
bsc_public_key = '0xF54274757Bf717B1ab52bA0d3a7CbF635f856a0d'
# with closing(Safari()) as browser:
browser = Safari()
browser.get(url)
textbox = browser.find_element_by_id('addressInputField')
textbox.clear()
textbox.send_keys(bsc_public_key)
button = browser.find_element_by_class_name('binoculars icon')
button.click()
# # wait for the page to load
WebDriverWait(browser, timeout=20).until(
lambda x: x.find_element_by_id('ui centered image'))
# store it to string variable
page_source = browser.page_source
print(page_source)
The code doesn’t work. After the browser loads, I don’t see the textbox filled in with the address. How can I do this in Python (with or without Selenium)?
You are trying to read the ui centered image
element text when it is still not presenting the final value.
Try waiting for invisibility_of_element_located(By.XPATH,"//div[@class='content']//div[@class='header']")
And only after that read the text from ui centered image
element.
You don’t really need the heavy guns of selenium
. You can get all the data from the API endpoint.
Here’s how:
import requests
wallet = "0xF54274757Bf717B1ab52bA0d3a7CbF635f856a0d"
endpoint = f"https://www.yieldwatch.net/api/all/{wallet}?platforms=beefy,pancake,hyperjump,auto,mdex"
wallet_data = requests.get(endpoint).json()["result"]
# https://yieldwatch.medium.com/yieldwatch-faqs-93c2cde244bf
# Net Value = Total Deposit + Total Yield + Wallet Balance — Total Debt
total = sum(v["totalUSDValues"]["total"] for v in wallet_data["PancakeSwap"].values())
net_worth = total + wallet_data["walletBalance"]["totalUSDValue"]
print(f"Net worth for {wallet}:n{round(net_worth, 2)}")
This outputs most up-to-date net worth:
Net worth for 0xF54274757Bf717B1ab52bA0d3a7CbF635f856a0d:
2213.13
Note: The FAQ says you need Total Debt
to calculate the net worth but this wallet doesn’t have any debt so I didn’t include the value in the equation.
However, if you have a wallet that has a debt, please share it, and I’ll update the answer.
from selenium.webdriver import Safari
from selenium.webdriver.common.by import By
import time
url = 'https://www.yieldwatch.net/'
address = input('Enter Your Wallet Address : ')
browser = Safari()
browser.get(url)
btnpancake = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[1]/div/div/div[4]/div/div[4]/div/img")
btnpancake.click()
wallet = browser.find_element(By.ID, "addressInputField")
wallet.clear()
wallet.send_keys(address)
send = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[1]/div/div/div[1]/div/div[2]/button[1]")
send.click()
time.sleep(7)
pool = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[2]/div[3]/div/div/div[2]/div/div/div[2]/div/div/div/div/div[1]/div[1]/h4/div[2]")
poolval = pool.text
value = poolval.split(' ')
print(f"Current Pool : {value[0]}")
print(f"Total Cake Value : {value[2]}")
earn = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[2]/div[3]/div/div/div[2]/div/div/div[2]/div/div/div/div/div[2]/table/tbody/tr[1]/td[2]")
earnval = earn.text
value2 = earnval.split(' ')
print(f"Total Cake : {value2[0]}")
print(f"Pending {value2[3]} : {value2[2]}")
print(f"Harvest : {value2[4]+' '+value2[5]}")
totalyield = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[2]/div[2]/div/div/div/div[3]/span/span")
totalyieldval = totalyield.text
print(f"Total Yield : {totalyieldval}")
apr = browser.find_element(By.XPATH, value="//*[@id='root']/div/div/div[2]/div[3]/div/div/div[2]/div/div/div[2]/div/div/div/div/div[2]/table/tbody/tr[1]/td[4]/h4/div")
aprval = apr.text
print(f"APR : {aprval}")
browser.quit()