Format and cleanup

Author: jaapterwoerds

src/opensignals/data/common.py

@@ -23,8 +23,9 @@ def empty_df():
         "adj_close", "volume", "currency", "provider"])
 
 
-def get_tickers():
-    ticker_map = pd.read_csv(SIGNALS_TICKER_MAP)
+def get_tickers(ticker_map):
+    if ticker_map is None:
+        ticker_map = pd.read_csv(SIGNALS_TICKER_MAP)
     ticker_map = ticker_map.dropna(subset=['yahoo'])
     logger.info(f'Number of eligible tickers: {ticker_map.shape[0]}')
 
@@ -101,15 +102,18 @@ def get_data(
         db_dir,
         features_generators=None,
         last_friday=datetime.today() - relativedelta(weekday=FR(-1)),
-        target='target_20d'):
+        target='target_20d',
+        ticker_map=None):
     """generate data set"""
 
     if features_generators is None:
         features_generators = []
 
     ticker_data = get_ticker_data(db_dir)
-
-    ticker_universe = pd.read_csv(SIGNALS_UNIVERSE)
+    if ticker_map is None:
+        ticker_universe = pd.read_csv(SIGNALS_UNIVERSE)
+    else:
+        ticker_universe = ticker_map
     ticker_data = ticker_data[ticker_data.bloomberg_ticker.isin(
         ticker_universe['bloomberg_ticker'])]
 
@@ -216,15 +220,15 @@ def download_tickers(tickers, start, download_ticker):
     return pd.concat(dfs)
 
 
-def download_data(db_dir, download_ticker, recreate=False):
+def download_data(db_dir, download_ticker, recreate=False, ticker_map=None):
     if recreate:
         logging.warning(f'Removing dataset {db_dir} to recreate it')
         shutil.rmtree(db_dir, ignore_errors=True)
 
     db_dir.mkdir(exist_ok=True)
 
     ticker_data = get_ticker_data(db_dir)
-    ticker_map = get_tickers()
+    ticker_map = get_tickers(ticker_map)
     ticker_missing = get_ticker_missing(ticker_data, ticker_map)
 
     n_ticker_missing = ticker_missing.shape[0]

src/opensignals/data/fmp.py

@@ -12,6 +12,16 @@
 FMP_API_KEY = os.environ.get('FMP_API_KEY')
 
 
+def get_ticker_map():
+    symbols = pd.read_json(f'''https://financialmodelingprep.com/api/v3/stock/list?apikey={FMP_API_KEY}''')
+    stock_symbols = symbols[symbols['type'] == 'stock']
+    stock_symbols['yahoo'] = stock_symbols['symbol']
+    stock_symbols['bloomberg_ticker'] = np.where('.' not in stock_symbols['symbol'],
+                                                 stock_symbols['symbol'] + ' US',
+                                                 stock_symbols['symbol'])
+    return stock_symbols[['bloomberg_ticker', 'yahoo']]
+
+
 def download_ticker(ticker, start_epoch, end_epoch):
     """download data for a given ticker"""
     url = f'https://financialmodelingprep.com/api/v3/historical-price-full/{ticker}'
@@ -37,20 +47,22 @@ def download_ticker(ticker, start_epoch, end_epoch):
     return ticker, df.drop_duplicates().dropna()
 
 
-def download_data(db_dir, recreate=False):
+def download_data(db_dir, recreate=False, ticker_map=None):
     """download (missing) data for the tickers in the universe using the fmp API"""
-    return download_data_generic(db_dir, download_ticker, recreate)
+    return download_data_generic(db_dir, download_ticker, recreate, ticker_map)
 
 
 def get_data(
         db_dir,
         features_generators=None,
         last_friday=dt.datetime.today() - relativedelta(weekday=FR(-1)),
-        target='target_20d'
+        target='target_20d',
+        ticker_map=None
 ):
     """generate data set with the fmp API!"""
     return get_data_generic(
         db_dir,
         features_generators=features_generators,
         last_friday=last_friday,
-        target=target)
+        target=target,
+        ticker_map=ticker_map)

src/opensignals/data/yahoo.py

@@ -60,20 +60,22 @@ def download_ticker(ticker, start_epoch, end_epoch):
     return ticker, df.drop_duplicates().dropna()
 
 
-def download_data(db_dir, recreate=False):
+def download_data(db_dir, recreate=False, ticker_map=None):
     """download (missing) data for the tickers in the universe using the Yahoo! Finance API"""
-    return download_data_generic(db_dir, download_ticker, recreate)
+    return download_data_generic(db_dir, download_ticker, recreate, ticker_map=ticker_map)
 
 
 def get_data(
         db_dir,
         features_generators=None,
         last_friday=dt.datetime.today() - relativedelta(weekday=FR(-1)),
-        target='target_20d'
+        target='target_20d',
+        ticker_map=None
 ):
     """generate data set with the Yahoo! Finance API!"""
     return get_data_generic(
         db_dir,
         features_generators=features_generators,
         last_friday=last_friday,
-        target=target)
+        target=target,
+        ticker_map=ticker_map)