From 160178a12e453c8ff901e95754842462425c8de3 Mon Sep 17 00:00:00 2001 From: Corey Goldberg Date: Sun, 2 Jun 2013 14:35:42 -0400 Subject: [PATCH 01/12] clarified lgpl license in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 7f98cb2..ef9a92d 100644 --- a/setup.py +++ b/setup.py @@ -37,11 +37,11 @@ url='https://github.com/cgoldberg/ystockquote', download_url='http://pypi.python.org/pypi/ystockquote', keywords='stocks stockmarket yahoo finance'.split(), - license='Apache v2.0', + license='GNU LGPLv2+', classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)', + 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)', 'Operating System :: OS Independent', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Office/Business :: Financial :: Investment', From f0d861a712edafcefec49b7f9abf73607791c334 Mon Sep 17 00:00:00 2001 From: Corey Goldberg Date: Sun, 2 Jun 2013 14:37:15 -0400 Subject: [PATCH 02/12] bumped version for release --- ystockquote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ystockquote.py b/ystockquote.py index b2ff030..5d78595 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -13,7 +13,7 @@ # Requires: Python 2.7/3.2+ -__version__ = '0.2.2' +__version__ = '0.2.3' try: From f22703d380f274b6aa357959019609d165cbf0e6 Mon Sep 17 00:00:00 2001 From: Corey Goldberg Date: Sun, 2 Jun 2013 14:38:20 -0400 Subject: [PATCH 03/12] bumped version --- ystockquote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ystockquote.py b/ystockquote.py index 5d78595..1df3248 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -13,7 +13,7 @@ # Requires: Python 2.7/3.2+ -__version__ = '0.2.3' +__version__ = '0.2.4' try: From 09aedad09a4c883a07b8c5190cc41e7f1661ccf2 Mon Sep 17 00:00:00 2001 From: Skillachie Date: Sun, 23 Jun 2013 13:52:18 -0400 Subject: [PATCH 04/12] added optional support for dictionary in get_historical_prices --- ystockquote.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/ystockquote.py b/ystockquote.py index 1df3248..19b5420 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -15,7 +15,6 @@ __version__ = '0.2.4' - try: # py3 from urllib.request import Request, urlopen @@ -144,7 +143,7 @@ def get_short_ratio(symbol): return _request(symbol, 's7') -def get_historical_prices(symbol, start_date, end_date): +def get_historical_prices(symbol, start_date, end_date,list_dicts=False): """ Get historical prices for the given ticker symbol. Date format is 'YYYY-MM-DD' @@ -167,4 +166,22 @@ def get_historical_prices(symbol, start_date, end_date): resp = urlopen(req) content = str(resp.read().decode('utf-8').strip()) days = content.splitlines() - return [day.split(',') for day in days] + + if(list_dicts): + hist_data = list() + keys = days[0].split(',') + for day in days[1:]: + day_data = day.split(',') + day_dict = {day_data[0]: + {keys[1]:day_data[1], + keys[2]:day_data[2], + keys[3]:day_data[3], + keys[4]:day_data[4], + keys[5]:day_data[5], + keys[6]:day_data[6] + } + } + hist_data.append(day_dict) + return hist_data + else: + return [day.split(',') for day in days] From 4b3f809472ee59ba849ac7e8d4041059d60514ae Mon Sep 17 00:00:00 2001 From: Skillachie Date: Sun, 11 Jun 2017 22:32:10 -0400 Subject: [PATCH 05/12] fix to get historical prices due to api change --- ystockquote.py | 63 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/ystockquote.py b/ystockquote.py index 905b362..d4a160b 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -15,6 +15,8 @@ __version__ = '0.2.6dev' # NOQA +import time +import re try: # py3 @@ -462,7 +464,34 @@ def get_short_ratio(symbol): return _request(symbol, 's7') -def get_historical_prices(symbol, start_date, end_date): +def _get_headers(): + headers = { + 'Connection': 'keep-alive', + 'Expires': str(-1), + 'Upgrade-Insecure-Requests': str(1), + # Google Chrome: + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36' # noqa + } + + return headers + +def _get_crumb_cookie(symbol): + # Scrape a history page for a valid crumb ID: + tu = "https://finance.yahoo.com/quote/{}/history".format(symbol) + + req = Request(tu,headers=_get_headers()) + resp = urlopen(req) + cookies=resp.info()['Set-Cookie'] + + content = str(resp.read().decode('utf-8').strip()) + + # Matches: {"crumb":"AlphaNumeric"} + rpat = '"CrumbStore":{"crumb":"([^"]+)"}' + crumb = re.findall(rpat, content)[0] + return {'crumb':crumb.encode('ascii').decode('unicode-escape'), 'cookie':cookies} + + +def get_historical_prices(symbol, start_date, end_date,interval='1d'): """ Get historical prices for the given ticker symbol. Date format is 'YYYY-MM-DD' @@ -470,21 +499,29 @@ def get_historical_prices(symbol, start_date, end_date): Returns a nested dictionary (dict of dicts). outer dict keys are dates ('YYYY-MM-DD') """ + + pattern = '%Y-%m-%d' + + unix_start = int(time.mktime(time.strptime(start_date, pattern))) + unix_end = int(time.mktime(time.strptime(end_date, pattern))) + + crumb_cookie = _get_crumb_cookie(symbol) + params = urlencode({ - 's': symbol, - 'a': int(start_date[5:7]) - 1, - 'b': int(start_date[8:10]), - 'c': int(start_date[0:4]), - 'd': int(end_date[5:7]) - 1, - 'e': int(end_date[8:10]), - 'f': int(end_date[0:4]), - 'g': 'd', - 'ignore': '.csv', - }) - url = 'http://real-chart.finance.yahoo.com/table.csv?%s' % params - req = Request(url) + 'period1': unix_start, + 'period2': unix_end, + 'interval': interval, + 'events': 'history', + 'crumb': crumb_cookie['crumb'] + }) + + url = 'https://query1.finance.yahoo.com/v7/finance/download/{}?{}'.format(symbol,params) + + req = Request(url,headers=_get_headers()) + req.add_header("Cookie", crumb_cookie['cookie']) resp = urlopen(req) content = str(resp.read().decode('utf-8').strip()) + daily_data = content.splitlines() hist_dict = dict() keys = daily_data[0].split(',') From 563373c03268581153b40f84b45216415ab22a95 Mon Sep 17 00:00:00 2001 From: Skillachie Date: Sun, 11 Jun 2017 22:58:00 -0400 Subject: [PATCH 06/12] pep 8 --- ystockquote.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/ystockquote.py b/ystockquote.py index d4a160b..ddf7e4d 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -466,32 +466,35 @@ def get_short_ratio(symbol): def _get_headers(): headers = { - 'Connection': 'keep-alive', - 'Expires': str(-1), - 'Upgrade-Insecure-Requests': str(1), - # Google Chrome: - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36' # noqa + 'Connection': 'keep-alive', + 'Expires': str(-1), + 'Upgrade-Insecure-Requests': str(1), + # Google Chrome: + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) \ + AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36' # noqa } return headers + def _get_crumb_cookie(symbol): # Scrape a history page for a valid crumb ID: tu = "https://finance.yahoo.com/quote/{}/history".format(symbol) - - req = Request(tu,headers=_get_headers()) + req = Request(tu, headers=_get_headers()) resp = urlopen(req) - cookies=resp.info()['Set-Cookie'] + cookies = resp.info()['Set-Cookie'] content = str(resp.read().decode('utf-8').strip()) # Matches: {"crumb":"AlphaNumeric"} rpat = '"CrumbStore":{"crumb":"([^"]+)"}' crumb = re.findall(rpat, content)[0] - return {'crumb':crumb.encode('ascii').decode('unicode-escape'), 'cookie':cookies} + crumb_cookie = {'crumb': crumb.encode('ascii').decode('unicode-escape'), + 'cookie': cookies} + return crumb_cookie -def get_historical_prices(symbol, start_date, end_date,interval='1d'): +def get_historical_prices(symbol, start_date, end_date, interval='1d'): """ Get historical prices for the given ticker symbol. Date format is 'YYYY-MM-DD' @@ -515,9 +518,10 @@ def get_historical_prices(symbol, start_date, end_date,interval='1d'): 'crumb': crumb_cookie['crumb'] }) - url = 'https://query1.finance.yahoo.com/v7/finance/download/{}?{}'.format(symbol,params) + url = 'https://query1.finance.yahoo.com/v7/finance/download/{}\ + ?{}'.format(symbol, params) - req = Request(url,headers=_get_headers()) + req = Request(url, headers=_get_headers()) req.add_header("Cookie", crumb_cookie['cookie']) resp = urlopen(req) content = str(resp.read().decode('utf-8').strip()) From f36ebdcbfb5eb13a88e9dc32a11c9a2ab32869fa Mon Sep 17 00:00:00 2001 From: Skillachie Date: Sun, 11 Jun 2017 23:11:59 -0400 Subject: [PATCH 07/12] fixed wrapped line --- ystockquote.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/ystockquote.py b/ystockquote.py index ddf7e4d..a2f95f3 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -489,9 +489,8 @@ def _get_crumb_cookie(symbol): # Matches: {"crumb":"AlphaNumeric"} rpat = '"CrumbStore":{"crumb":"([^"]+)"}' crumb = re.findall(rpat, content)[0] - crumb_cookie = {'crumb': crumb.encode('ascii').decode('unicode-escape'), - 'cookie': cookies} - return crumb_cookie + return {'crumb': crumb.encode('ascii').decode('unicode-escape'), + 'cookie': cookies} def get_historical_prices(symbol, start_date, end_date, interval='1d'): @@ -518,8 +517,8 @@ def get_historical_prices(symbol, start_date, end_date, interval='1d'): 'crumb': crumb_cookie['crumb'] }) - url = 'https://query1.finance.yahoo.com/v7/finance/download/{}\ - ?{}'.format(symbol, params) + url = 'https://query1.finance.yahoo.com/v7/finance/download/{}?{}'.\ + format(symbol, params) req = Request(url, headers=_get_headers()) req.add_header("Cookie", crumb_cookie['cookie']) From 1ce6ce2d6b9c88fff6b298ba4f779cb05216fefa Mon Sep 17 00:00:00 2001 From: Skillachie Date: Sun, 11 Jun 2017 23:40:37 -0400 Subject: [PATCH 08/12] Python 2.7 fix --- ystockquote.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ystockquote.py b/ystockquote.py index a2f95f3..b67cedf 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -26,6 +26,9 @@ # py2 from urllib2 import Request, urlopen from urllib import urlencode + import sys + reload(sys) + sys.setdefaultencoding('utf8') def _request(symbol, stat): From 74eba18137d5a856a63c7f2b27f3cc5485632b52 Mon Sep 17 00:00:00 2001 From: Skillachie Date: Sun, 11 Jun 2017 23:45:34 -0400 Subject: [PATCH 09/12] pep --- ystockquote.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ystockquote.py b/ystockquote.py index b67cedf..8aaf97f 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -22,12 +22,13 @@ # py3 from urllib.request import Request, urlopen from urllib.parse import urlencode + from imp import reload except ImportError: # py2 from urllib2 import Request, urlopen from urllib import urlencode import sys - reload(sys) + reload(sys) sys.setdefaultencoding('utf8') From 5cc70d4567d71e12a6b3ab4c42683a5de3a9fd68 Mon Sep 17 00:00:00 2001 From: Dwayne Campbell Date: Thu, 6 Jul 2017 10:48:48 -0500 Subject: [PATCH 10/12] touch for travis rerun --- ystockquote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ystockquote.py b/ystockquote.py index 8aaf97f..a6ae993 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -490,7 +490,7 @@ def _get_crumb_cookie(symbol): content = str(resp.read().decode('utf-8').strip()) - # Matches: {"crumb":"AlphaNumeric"} + # Matches: {'crumb':'AlphaNumeric'} rpat = '"CrumbStore":{"crumb":"([^"]+)"}' crumb = re.findall(rpat, content)[0] return {'crumb': crumb.encode('ascii').decode('unicode-escape'), From 275c8751072ba4fba8d8a149253ea7febab6739e Mon Sep 17 00:00:00 2001 From: Dwayne Campbell Date: Sun, 23 Jul 2017 12:57:45 -0500 Subject: [PATCH 11/12] added test for 1wk and 1 month interval and docstring for usage --- test_ystockquote.py | 21 ++++++++++++++++++++- ystockquote.py | 13 ++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/test_ystockquote.py b/test_ystockquote.py index dcd039c..ad1c5d4 100644 --- a/test_ystockquote.py +++ b/test_ystockquote.py @@ -13,7 +13,6 @@ import unittest - import ystockquote @@ -50,6 +49,26 @@ def test_get_historical_prices(self): self.assertGreater(float(prices[end_date]['Volume']), 0.0) self.assertGreater(float(prices[end_date]['Adj Close']), 0.0) + def test_get_historical_prices_1wk(self): + symbol = 'GOOG' + start_date = '2013-01-02' + end_date = '2013-01-15' + prices = ystockquote.get_historical_prices( + symbol, start_date, end_date, '1wk') + + self.assertIsInstance(prices, dict) + self.assertEqual(len(prices), 2) + + def test_get_historical_prices_1mo(self): + symbol = 'GOOG' + start_date = '2013-01-02' + end_date = '2013-04-01' + prices = ystockquote.get_historical_prices( + symbol, start_date, end_date, '1mo') + + self.assertIsInstance(prices, dict) + self.assertEqual(len(prices), 3) + if __name__ == '__main__': unittest.main() diff --git a/ystockquote.py b/ystockquote.py index a6ae993..5f9040a 100644 --- a/ystockquote.py +++ b/ystockquote.py @@ -482,7 +482,11 @@ def _get_headers(): def _get_crumb_cookie(symbol): - # Scrape a history page for a valid crumb ID: + """ + Get a valid crumb id y scraping the page. + + Returns a dictionary. + """ tu = "https://finance.yahoo.com/quote/{}/history".format(symbol) req = Request(tu, headers=_get_headers()) resp = urlopen(req) @@ -502,6 +506,13 @@ def get_historical_prices(symbol, start_date, end_date, interval='1d'): Get historical prices for the given ticker symbol. Date format is 'YYYY-MM-DD' + interval determines the rollup or aggregate of the + time series to return. interval can take the following values: + + 1d - data returned on a daily basis. This is the default + 1wk - data rolled up and returned on a weekly basis + 1mo - data rolled up and returend on a monthly basis + Returns a nested dictionary (dict of dicts). outer dict keys are dates ('YYYY-MM-DD') """ From 84fdbbd7bfdbdd7c15febf7fd18fc86b35c77201 Mon Sep 17 00:00:00 2001 From: Dwayne Campbell Date: Sun, 1 Oct 2017 15:42:06 -0400 Subject: [PATCH 12/12] corrected assert value in test --- test_ystockquote.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test_ystockquote.py b/test_ystockquote.py index ad1c5d4..356a1a3 100644 --- a/test_ystockquote.py +++ b/test_ystockquote.py @@ -11,7 +11,6 @@ # # Requires: Python 2.7/3.3+ - import unittest import ystockquote @@ -57,7 +56,7 @@ def test_get_historical_prices_1wk(self): symbol, start_date, end_date, '1wk') self.assertIsInstance(prices, dict) - self.assertEqual(len(prices), 2) + self.assertEqual(len(prices), 3) def test_get_historical_prices_1mo(self): symbol = 'GOOG' @@ -67,7 +66,7 @@ def test_get_historical_prices_1mo(self): symbol, start_date, end_date, '1mo') self.assertIsInstance(prices, dict) - self.assertEqual(len(prices), 3) + self.assertEqual(len(prices), 4) if __name__ == '__main__':