OLS Statsmodels formula: Returns an ValueError: zero-size array to reduction operation maximum which has no identity

Question:

Hey I am doing multiple OLS regressions for some cross-sectional data iterating through the months. I encounter for the forth month a ValueError: zero-size array to reduction operation maximum which has no identity. But I do not know why. The data has no NaNs, I have tested this with dropna since it was suggested in another Question Link:

df_month.dropna(how='all')

And the zero in each months data is due to the normalization of the data, which does not cause any trouble during the first three iterations only in the fourth. What’s also weird is that if I stop the loop before the OLS regression is executed in the fourth line and then run the regression again by hand in another cell it just works fine. Could this issue be due to the storing?

Here is my code an data to replicate the error:

import statsmodels.formula.api as smf #ols (minor letters)
import pandas as pd
import numpy as np

df = pd.read_csv('df_all.csv', index_col='Instrument', sep=',', decimal='.')
df.drop(columns='Unnamed: 0', inplace=True)

#Creates an array with all the years to iterate through single year
years = df_all['Date'].dt.year.unique()

df_store = pd.DataFrame(index=[], columns=['year', 'month', 'R2_adj'])

for year in years:
    df_year = df_all[df_all['Date'].dt.year == year]
    df_year_t1 = df_all[df_all['Date'].dt.year == year+1]

    Jan_date = df_year['Date'][0]
    year_start = df_year[(df_year['Date'] == Jan_date) & (df_year['HQ'] == 'United States of America')
                        & (df_year['ESG'] > 0)]
    year_start_firms = year_start.index.unique()

    df_year_firms = df_year[['Date', 'eTR', 'MC', 'ESG']].loc[year_start_firms]
    df_year_t1 = df_year_t1[['Date', 'eTR', 'MC', 'ESG']]
    
    print(year)
    print(" ")
        
    
    #Normalizes the ESG Scores to the interval (0; 1) and substitues it in; 0.5 is the mean
    df = df_year_firms[['ESG']]
    Normalized_ESG_year = (df - df.min()) / (df.max() - df.min())

    df_year_firms_norm =  df_year_firms
    df_year_firms_norm[['ESG_norm']] = Normalized_ESG_year.values
    df_year_firms_norm = df_year_firms_norm.drop('ESG', axis=1)
    

    df_year_firms_norm = df_year_firms_norm.ffill(axis=0)

    df_year_firms_norm.loc[:, 'Month'] = df_year_firms_norm['Date'].dt.month.values.reshape(len(df_year_firms_norm), 1)
    df_year_t1.loc[:, 'Month'] = df_year_t1['Date'].dt.month.values.reshape(len(df_year_t1), 1)      
    data = pd.merge(df_year_firms_norm, df_year_t1.iloc[:, [0,1,-1]], how='inner', on=['Instrument', 'Month'], suffixes=('_t', '_t1'))

    #Resets index
    data.reset_index(inplace=True)
                
    #Monthwise iteration within the year loop
    for month in range(1, 13, 1):
        df_month = data[data['Month'] == month]
                
    #ols by statsmodels.formula.api = smf (ols with minor letters)
    #DataFrame input
        #smf not sm (statsmodels.formula.api instead of statsmodels.api)
        ESG_ols = smf.ols(formula = 'eTR_t1 ~ ESG_norm', data=df_month).fit(cov_type='HAC',cov_kwds={'maxlags':0})
       
        ESG_ols_tstat = ESG_ols.tvalues
        ESG_ols_coeff = ESG_ols.params

        results_df = pd.DataFrame({
                              'ESG_ols_coeff':ESG_ols_coeff, 'ESG_ols_tstat':ESG_ols_tstat},                         
        index = ['Intercept', 'ESG'])

        #Produces the table
        df_output = summary_col(
                                results=[ESG_ols], stars=True, float_format='%0.4f',
                                  model_names=['ESG_ols'],
                                  info_dict={'N':lambda x: "{0:d}".format(int(x.nobs))}, 
                                             regressor_order = ['Intercept', 'ESG_norm'])
        helper = pd.DataFrame(data=[[year, month, ESG_ols.rsquared_adj]], 
                              columns=['year', 'month', 'R2_adj'])

        
        
        
        df_store = df_store.append(helper)
        print(month)
    break

df_store

I am also happy for any suggestions in efficiency!

As requested, here is the full Traceback:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
~AppDataLocalTemp/ipykernel_8408/1348884998.py in <module>
     91          #   break
     92         #smf not sm (statsmodels.formula.api instead of statsmodels.api)
---> 93         ESG_ols = smf.ols(formula = 'eTR_t1 ~ ESG_norm', data=df_month).fit(cov_type='HAC',cov_kwds={'maxlags':1})
     94         #!Assumption: maxlags=0 should be reasonable since we do not have any TS analysis, right?
     95         ESG_ols_tstat = ESG_ols.tvalues

~anaconda3libsite-packagesstatsmodelsbasemodel.py in from_formula(cls, formula, data, subset, drop_cols, *args, **kwargs)
    193                        'formula': formula,  # attach formula for unpckling
    194                        'design_info': design_info})
--> 195         mod = cls(endog, exog, *args, **kwargs)
    196         mod.formula = formula
    197 

~anaconda3libsite-packagesstatsmodelsregressionlinear_model.py in __init__(self, endog, exog, missing, hasconst, **kwargs)
    870     def __init__(self, endog, exog=None, missing='none', hasconst=None,
    871                  **kwargs):
--> 872         super(OLS, self).__init__(endog, exog, missing=missing,
    873                                   hasconst=hasconst, **kwargs)
    874         if "weights" in self._init_keys:

~anaconda3libsite-packagesstatsmodelsregressionlinear_model.py in __init__(self, endog, exog, weights, missing, hasconst, **kwargs)
    701         else:
    702             weights = weights.squeeze()
--> 703         super(WLS, self).__init__(endog, exog, missing=missing,
    704                                   weights=weights, hasconst=hasconst, **kwargs)
    705         nobs = self.exog.shape[0]

~anaconda3libsite-packagesstatsmodelsregressionlinear_model.py in __init__(self, endog, exog, **kwargs)
    188     """
    189     def __init__(self, endog, exog, **kwargs):
--> 190         super(RegressionModel, self).__init__(endog, exog, **kwargs)
    191         self._data_attr.extend(['pinv_wexog', 'weights'])
    192 

~anaconda3libsite-packagesstatsmodelsbasemodel.py in __init__(self, endog, exog, **kwargs)
    235 
    236     def __init__(self, endog, exog=None, **kwargs):
--> 237         super(LikelihoodModel, self).__init__(endog, exog, **kwargs)
    238         self.initialize()
    239 

~anaconda3libsite-packagesstatsmodelsbasemodel.py in __init__(self, endog, exog, **kwargs)
     75         missing = kwargs.pop('missing', 'none')
     76         hasconst = kwargs.pop('hasconst', None)
---> 77         self.data = self._handle_data(endog, exog, missing, hasconst,
     78                                       **kwargs)
     79         self.k_constant = self.data.k_constant

~anaconda3libsite-packagesstatsmodelsbasemodel.py in _handle_data(self, endog, exog, missing, hasconst, **kwargs)
     99 
    100     def _handle_data(self, endog, exog, missing, hasconst, **kwargs):
--> 101         data = handle_data(endog, exog, missing, hasconst, **kwargs)
    102         # kwargs arrays could have changed, easier to just attach here
    103         for key in kwargs:

~anaconda3libsite-packagesstatsmodelsbasedata.py in handle_data(endog, exog, missing, hasconst, **kwargs)
    670 
    671     klass = handle_data_class_factory(endog, exog)
--> 672     return klass(endog, exog=exog, missing=missing, hasconst=hasconst,
    673                  **kwargs)

~anaconda3libsite-packagesstatsmodelsbasedata.py in __init__(self, endog, exog, missing, hasconst, **kwargs)
     85         self.const_idx = None
     86         self.k_constant = 0
---> 87         self._handle_constant(hasconst)
     88         self._check_integrity()
     89         self._cache = {}

~anaconda3libsite-packagesstatsmodelsbasedata.py in _handle_constant(self, hasconst)
    129             # detect where the constant is
    130             check_implicit = False
--> 131             exog_max = np.max(self.exog, axis=0)
    132             if not np.isfinite(exog_max).all():
    133                 raise MissingDataError('exog contains inf or nans')

<__array_function__ internals> in amax(*args, **kwargs)

~anaconda3libsite-packagesnumpycorefromnumeric.py in amax(a, axis, out, keepdims, initial, where)
   2731     5
   2732     """
-> 2733     return _wrapreduction(a, np.maximum, 'max', axis, None, out,
   2734                           keepdims=keepdims, initial=initial, where=where)
   2735 

~anaconda3libsite-packagesnumpycorefromnumeric.py in _wrapreduction(obj, ufunc, method, axis, dtype, out, **kwargs)
     85                 return reduction(axis=axis, out=out, **passkwargs)
     86 
---> 87     return ufunc.reduce(obj, axis, dtype, out, **passkwargs)
     88 
     89 

ValueError: zero-size array to reduction operation maximum which has no identity
Asked By: 9Morgan8

||

Answers:

I was reluctant to use this approach but it works and I still get results even for those month where there is supposed to be a lack of data.
I found the idea here Overcom ValueError for empty array

try: #Catches the upcoming Value Error
            results = smf.ols(formula = 'eTR_t1 ~ ESG_norm', data=df_month).fit(cov_type='HAC',cov_kwds={'maxlags':0})
        except ValueError:
            pass

Thank you @Josef for the suggestions and the help.

Answered By: 9Morgan8
Categories: questions Tags: , ,
Answers are sorted by their score. The answer accepted by the question owner as the best is marked with
at the top-right corner.