Python URL Request Yahoo API

Question

I wrote this script that uses ystockquote-master to scrape the Yahoo Finance API for price and market cap. It is very simple and works well on my pc, however when I try to use it on my friends mac I get an error. It is a very long one so I will post it at the end. I have been struggling to find out what is going on, so hence I turned here. Background: Beginner to Novice. Here is a snippet of my code:

try:
    # py3
    from urllib.request import Request, urlopen
    from urllib.parse import urlencode
except ImportError:
    # py2
    from urllib2 import Request, urlopen
    from urllib import urlencode

def _request(symbol, stat):
    url = 'http://finance.yahoo.com/d/quotes.csv?s=%s&f=%s' % (symbol, stat)
    req = Request(url)
    resp = urlopen(req)
    return str(resp.read().decode('utf-8').strip())
def get_price(symbol):
    return _request(symbol, 'l1')
def get_market_cap(symbol):
    return _request(symbol, 'j1')

i = 0
while i<len(NewSymbolsList):
    results = open("intermediateresults.csv", "a")
    api = [get_price(NewSymbolsList[i]),get_market_cap(NewSymbolsList[i])]
    api = re.sub("['|'|]", "", str(api))
    results.write(str(NewSymbolsList[i]) +"," +str(api) +"n")
    print NewSymbolsList[i], api                  
    i+=1
results.close()

Here is the error I get on my friends Mac:

cd '/Users/JW/Desktop/market cap/' && '/usr/local/bin/pythonw' -t         '   /Users/JW/Desktop/market cap/MarketCap_Scan.py'  && echo Exit status: $? && exit 1
JWs-MacBook-Pro:~ JW$ cd '/Users/JW/Desktop/market cap/' && '/usr/local/bin/pythonw' -t     '/Users/JW/Desktop/market cap/MarketCap_Scan.py'  && echo Exit status: $? && exit 1
Traceback (most recent call last):
  File "/Users/JW/Desktop/market cap/MarketCap_Scan.py", line 33, in <module>
    api = [get_price(NewSymbolsList[i]),get_market_cap(NewSymbolsList[i])]
  File "/Users/JW/Desktop/market cap/MarketCap_Scan.py", line 18, in get_price
    return _request(symbol, 'l1')
  File "/Users/JW/Desktop/market cap/MarketCap_Scan.py", line 15, in _request
    resp = urlopen(req)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 127, in urlopen
    return _opener.open(url, data, timeout)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 410, in open
    response = meth(req, response)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 523, in http_response
    'http', request, response, code, msg, hdrs)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 442, in error
    result = self._call_chain(*args)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 382, in _call_chain
    result = func(*args)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 629, in http_error_302
    return self.parent.open(new, timeout=req.timeout)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 410, in open
    response = meth(req, response)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 523, in http_response
    'http', request, response, code, msg, hdrs)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 448, in error
    return self._call_chain(*args)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 382, in _call_chain
    result = func(*args)
  File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/urllib2.py", line 531, in http_error_default
    raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
urllib2.HTTPError: HTTP Error 400: Bad Request
JWs-MacBook-Pro:market cap JW$

An edit showing how I prepared the list to be iterated, (pretty standard I am guessing but thought it may be helpful):

symbols = open("symbolslist.txt")
readsymbols = symbols.read()
NewSymbolsList = readsymbols.split("n")

Asked By: Roland Smith

||

Source

Answer 1

Ok, let’s try this here. I suspect that the problem that’s being faced is white space related in the source file (perhaps a rogue r?). As an added note, I introduced a couple of with clause context managers to illustrate their usage for for the file handling.

import re
try:
    # py3
    from urllib.request import Request, urlopen
    from urllib.parse import urlencode
except ImportError:
    # py2
    from urllib2 import Request, urlopen
    from urllib import urlencode

def _request(symbol, stat):
    url = 'http://finance.yahoo.com/d/quotes.csv?s=%s&f=%s' % (symbol, stat)
    req = Request(url)
    resp = urlopen(req)
    return str(resp.read().decode('utf-8').strip())

def get_price(symbol):
    return _request(symbol, 'l1')

def get_market_cap(symbol):
    return _request(symbol, 'j1')

symbols = open("symbolslist.txt")
readsymbols = symbols.read()
NewSymbolsList = readsymbols.split("n")

with open('symbolslist.txt') as ifp, open('intermediateresults.csv', 'a') as results_fp:
    for row in ifp:
        row = row.strip()
        api = [ get_price(row), get_market_cap(row) ]
        api = re.sub("['|'|]", "", str(api))
        results_fp.write(str(row) +"," +str(api) +"n")
        print row, api

On my computer, this prints:

Marks-MBP:~ mroberts$ python aa.py
AAAE 0.005, 171K
RENT 22.07, 262.4M

Answered By: Mark Roberts

Python URL Request Yahoo API

Question:

Answers: