bernardodionisi / differences Goto Github PK

Hello

How do I reproduce the example presented in the original R package?

library(did)
data(mpdta)
out <- att_gt(yname = "lemp",
              gname = "first.treat",
              idname = "countyreal",
              tname = "year",
              xformla = ~1,
              data = mpdta,
              est_method = "reg"
              )
es <- aggte(out, type = "dynamic")

When using the wald_pre_test property, I get the following error: TypeError: results() got an unexpected keyword argument 'sample_name.'

It points to this line as the error, so I am not sure if there is something wrong with the property: File ~/opt/anaconda3/lib/python3.9/site-packages/differences/attgt/attgt.py:1089, in ATTgt.wald_pre_test(self)
1087 _wald_pre_test = {}
1088 for s in self.sample_names:
-> 1089 res = self.results(sample_name=s)
1090 res = wald_pre_test(res)
1092 if s == "full_sample":

I'm trying to reproduce the very first example of use:

panel_data = simulate_data()
att_gt = ATTgt(data=panel_data, cohort_name='cohort')

But I'm getting the folloing error. Any idea what's the problem?

KeyError Traceback (most recent call last)
File C:\Programas\anaconda3\envs\python39\lib\site-packages\pandas\core\indexes\base.py:3653, in Index.get_loc(self, key)
3652 try:
-> 3653 return self._engine.get_loc(casted_key)
3654 except KeyError as err:

File C:\Programas\anaconda3\envs\python39\lib\site-packages\pandas_libs\index.pyx:147, in pandas._libs.index.IndexEngine.get_loc()

File C:\Programas\anaconda3\envs\python39\lib\site-packages\pandas_libs\index.pyx:176, in pandas._libs.index.IndexEngine.get_loc()

File pandas_libs\hashtable_class_helper.pxi:7080, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas_libs\hashtable_class_helper.pxi:7088, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'amin'

The above exception was the direct cause of the following exception:

KeyError Traceback (most recent call last)
Cell In[17], line 1
----> 1 att_gt = ATTgt(data=panel_data, cohort_name='cohort')

File C:\Programas\anaconda3\envs\python39\lib\site-packages\differences\attgt\attgt.py:114, in ATTgt.init(self, data, cohort_name, strata_name, base_period, anticipation, freq)
111 self.anticipation = anticipation
112 self.copy_data = True # maybe make an option to the user
--> 114 self.data = data
116 self._data_matrix = None
118 self._result_dict = None

File C:\Programas\anaconda3\envs\python39\lib\site-packages\differences\tools\panel_validation.py:149, in _ValiDIData.set(self, instance, data)
147 # todo: look into non ATTgt classes how to handle this cases
148 if instance.is_panel and anticipation is not None:
--> 149 cohort_data, data = pre_process_treated_before(
150 cohort_data=cohort_data,
151 cohort_name=cohort_name,
152 data=data,
153 copy_data=copy_data,
154 )
156 cohort_data = pre_process_treated_after(
157 cohort_data=cohort_data,
158 cohort_name=cohort_name,
159 anticipation=anticipation, # no effect for now
160 )
162 no_never_treated_flag, cohort_data, data = pre_process_no_never_treated(
163 cohort_data=cohort_data,
164 cohort_name=cohort_name,
(...)
168 copy_data=copy_data,
169 )

File C:\Programas\anaconda3\envs\python39\lib\site-packages\differences\tools\panel_validation.py:332, in pre_process_treated_before(cohort_data, cohort_name, data, copy_data)
329 """drops always treated entities"""
331 # entities whose event happened BEFORE the start of their time
--> 332 treated_before = cohort_data.loc[
333 lambda x: x[cohort_name] <= x["amin"]
334 ].index.unique()
336 if len(treated_before):
337 warn(
338 f"{len(treated_before)} entities have been "
339 f"dropped because always treated "
340 f"(treated from before their first time)"
341 )

File C:\Programas\anaconda3\envs\python39\lib\site-packages\pandas\core\indexing.py:1102, in _LocationIndexer.getitem(self, key)
1098 else:
1099 # we by definition only have the 0th axis
1100 axis = self.axis or 0
-> 1102 maybe_callable = com.apply_if_callable(key, self.obj)
1103 return self._getitem_axis(maybe_callable, axis=axis)

File C:\Programas\anaconda3\envs\python39\lib\site-packages\pandas\core\common.py:379, in apply_if_callable(maybe_callable, obj, **kwargs)
368 """
369 Evaluate possibly callable input using obj and kwargs if it is callable,
370 otherwise return as it is.
(...)
376 **kwargs
377 """
378 if callable(maybe_callable):
--> 379 return maybe_callable(obj, **kwargs)
381 return maybe_callable

File C:\Programas\anaconda3\envs\python39\lib\site-packages\differences\tools\panel_validation.py:333, in pre_process_treated_before..(x)
329 """drops always treated entities"""
331 # entities whose event happened BEFORE the start of their time
332 treated_before = cohort_data.loc[
--> 333 lambda x: x[cohort_name] <= x["amin"]
334 ].index.unique()
336 if len(treated_before):
337 warn(
338 f"{len(treated_before)} entities have been "
339 f"dropped because always treated "
340 f"(treated from before their first time)"
341 )

File C:\Programas\anaconda3\envs\python39\lib\site-packages\pandas\core\frame.py:3761, in DataFrame.getitem(self, key)
3759 if self.columns.nlevels > 1:
3760 return self._getitem_multilevel(key)
-> 3761 indexer = self.columns.get_loc(key)
3762 if is_integer(indexer):
3763 indexer = [indexer]

File C:\Programas\anaconda3\envs\python39\lib\site-packages\pandas\core\indexes\base.py:3655, in Index.get_loc(self, key)
3653 return self._engine.get_loc(casted_key)
3654 except KeyError as err:
-> 3655 raise KeyError(key) from err
3656 except TypeError:
3657 # If we have a listlike key, _check_indexing_error will raise
3658 # InvalidIndexError. Otherwise we fall through and re-raise
3659 # the TypeError.
3660 self._check_indexing_error(key)

KeyError: 'amin'

I'm running differences 0.1.2,

Running the TWFE example from the README doesn't work:

from differences import TWFE, simulate_data

df = simulate_data()

twfe = TWFE(data=df, cohort_name='cohort')

twfe.fit(formula='y')

It errors with:

ValueError: Only numeric, string  or categorical data permitted

Upon debugging, I found that the issue happens when the data is fed into linearmodels.AbsorbingLS and self._y is cast to an object type because it has numbers as well as booleans.

Hi, I am trying to run Doubly Robust S-DID with unbalanced panel and varying base period.
the control group is 'not_yet_treated'
My code is as following:

    att_gt = ATTgt(data=diddata, cohort_name="course_month_end_date", base_period='varying', freq='M') 
    att_gt.fit(formula = formula, est_method='dr',control_group=control_group, progress_bar = True) 

however, I am getting following error which I am not able to understand

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[241], line 7
      4 diddata = diddata.reset_index().set_index(keys=['id','month_end_date'])
      6 att_gt = ATTgt(data=diddata, cohort_name="course_month_end_date", base_period='varying', freq='M')
----> 7 att_gt.fit(formula = formula, est_method='dr',control_group=control_group, progress_bar = False)

File ~/anaconda3/lib/python3.11/site-packages/differences/attgt/attgt.py:718, in ATTgt.fit(self, formula, weights_name, control_group, base_delta, est_method, as_repeated_cross_section, boot_iterations, random_state, alpha, cluster_var, split_sample_by, n_jobs, backend, progress_bar)
    688     res = get_att_gt(
    689         data=(
    690             self._data_matrix
   (...)
    714         ),
    715     )
    717     # standard errors & ci/cbands
--> 718     res = get_standard_errors(
    719         ntl=res,
    720         cluster_groups=cluster_groups,
    721         alpha=alpha,
    722         boot_iterations=boot_iterations,
    723         random_state=random_state,
    724         n_jobs_boot=n_jobs,
    725         backend_boot=backend,
    726         progress_bar=progress_bar,
    727         sample_name=s if s != "full_sample" else None,
    728         release_workers=s_idx == n_sample_names,
    729     )
    731     self._result_dict[s]["ATTgt_ntl"] = res
    733 self._fit_res = output_dict_to_dataframe(
    734     extract_dict_ntl(self._result_dict),
    735     stratum=bool(self._strata),
    736     date_map=self._map_datetime,
    737 )

File ~/anaconda3/lib/python3.11/site-packages/differences/attgt/attgt_cal.py:442, in get_standard_errors(ntl, cluster_groups, alpha, boot_iterations, random_state, backend_boot, n_jobs_boot, progress_bar, sample_name, release_workers)
    436     raise ValueError(
    437         "'boot_iterations' must be >= 0. "
    438         "If boot_iterations=0, analytic standard errors are computed"
    439     )
    441 # influence funcs + idx for not nan cols
--> 442 inf_funcs, not_nan_idx = stack_influence_funcs(ntl, return_idx=True)
    444 # create an empty array to populate with the standard errors
    445 se_array = np.empty(len(ntl))

File ~/anaconda3/lib/python3.11/site-packages/differences/attgt/utility.py:382, in stack_influence_funcs(ntl, return_idx)
    380     inf_funcs = inf_funcs.toarray()  # faster mboot if dense matrix
    381 else:
--> 382     inf_funcs = np.stack(
    383         [r.influence_func for r in ntl if r.influence_func is not None], axis=1
    384     )
    386 if return_idx:
    387     # indexes for the non-missing influence_func
    388     not_nan_idx = np.array(
    389         [i for i, r in enumerate(ntl) if r.influence_func is not None]
    390     )

File <__array_function__ internals>:200, in stack(*args, **kwargs)

File ~/anaconda3/lib/python3.11/site-packages/numpy/core/shape_base.py:460, in stack(arrays, axis, out, dtype, casting)
    458 arrays = [asanyarray(arr) for arr in arrays]
    459 if not arrays:
--> 460     raise ValueError('need at least one array to stack')
    462 shapes = {arr.shape for arr in arrays}
    463 if len(shapes) != 1:

ValueError: need at least one array to stack

Based on what I could understand from the package, it is not able to calculate standard errors. It would be great if you can help with debugging. Thanks.

Not sure why exactly, but this package only runs in python >= 3.8

I came across the "Multi-valued Treatment" feature while using the ATT_gt computation, as documented here. While the provided code snippet is a great starting point:

att_gt = ATTgt(data=panel_data, cohort_name='cohort', strata_name='strata')

I'm a bit unsure about how to effectively incorporate the "intensity" variable for optimal results. The documentation doesn't offer a clear explanation or examples about the role of the "intensity" variable in this context. As a result, I'm left somewhat uncertain about how to leverage this variable to its full potential.

thank you very much

Hi there,

This is a brief message of appreciation to @bernardodionisi and contributors for developing this library.

We use (and cite) this software in our latest paper on memorisation in large language models (https://arxiv.org/pdf/2406.04327). In the paper, we use the method proposed by Callaway and Sant'Anna (2021) to estimate memorisation in language models, which implies running their method on a big panel. We tried other implementations (mainly in R); however, differences was the only one able to scale out-of-the-box to our panel size (even using the bootstrapped standard errors option).

Overall, this library has made my work so much easier, and I hope it will continue improving in the future!

Best,
Pietro

Hi. I am having this error when I use cluster_var. The variable is str. What could be the issue? I would greatly appreciate your help. Thanks in advance.

AttributeError Traceback (most recent call last)
in <cell line: 1>()
----> 1 att_g.fit(
2 formula='####', control_group = 'never_treated', cluster_var = 'user')

~/.cache/pypoetry/virtualenvs/python-kernel-OtKFaj5M-py3.9/lib/python3.9/site-packages/differences/attgt/attgt.py in fit(self, formula, weights_name, control_group, base_delta, est_method, as_repeated_cross_section, boot_iterations, random_state, alpha, cluster_var, split_sample_by, n_jobs, backend, progress_bar)
674 cluster_groups = None
675 if cluster_var:
--> 676 cluster_groups = get_cluster_groups(
677 data=(
678 self._data_matrix[cluster_var]

~/.cache/pypoetry/virtualenvs/python-kernel-OtKFaj5M-py3.9/lib/python3.9/site-packages/differences/attgt/mboot.py in get_cluster_groups(data, cluster_var)
178 raise ValueError("can't have more than 2 cluster variables")
179
--> 180 if find_time_varying_covars(data=data, covariates=cluster_var):
181 raise ValueError("can't have time-varying cluster variables")
182

~/.cache/pypoetry/virtualenvs/python-kernel-OtKFaj5M-py3.9/lib/python3.9/site-packages/differences/tools/panel_utility.py in find_time_varying_covars(data, covariates, rtol, atol)
346
347 if rtol is None and atol is None:
--> 348 varying = data.groupby([entity_name])[covariates].nunique().max(axis=0)
349 return list(varying[varying > 1].index)
350

~/.cache/pypoetry/virtualenvs/python-kernel-OtKFaj5M-py3.9/lib/python3.9/site-packages/pandas/core/base.py in getitem(self, key)
236
237 if isinstance(key, (list, tuple, ABCSeries, ABCIndex, np.ndarray)):
--> 238 if len(self.obj.columns.intersection(key)) != len(set(key)):
239 bad_keys = list(set(key).difference(self.obj.columns))
240 raise KeyError(f"Columns not found: {str(bad_keys)[1:-1]}")

~/.cache/pypoetry/virtualenvs/python-kernel-OtKFaj5M-py3.9/lib/python3.9/site-packages/pandas/core/generic.py in getattr(self, name)
5573 ):
5574 return self[name]
-> 5575 return object.getattribute(self, name)
5576
5577 def setattr(self, name: str, value) -> None:

AttributeError: 'Series' object has no attribute 'columns'

Is there a way to export plots programatically?
Alternatively, is there a way plots can be plotted as matplotlib plots (which will have matplotlib functionalities including exporting)?

Hi there. I'm a little bit confused by how the cohort variable is defined and later used within the ATTgt class. Let me provide an example here to determine whether this is an actual bug or I'm just misunderstanding something. Let's start by using the simulate_data function to simulate some (panel) data.

panel_data = simulate_data(n_cohorts=5, intensity_by=1, tau=0.0, low=2.0, high=2.0)
panel_data.head()

The simulated data set the cohort variable as the last time step before the intervention (e.g. 1903 for unit e0 when treatment effect kicks in in 1904). This cohort definition is non-standard as it is usually defined as the start of treatment. I then proceed to fit the model and plot the estimated ATTs.

att = ATTgt(data=panel_data, cohort_name="cohort")
att.fit(formula="y", est_method="reg")
att.plot(color_by="cohort", shape_by="post")

Resulting plot looks correct, with marker shapes correctly encoding pre- and post-intervention periods (e.g. for cohort = 1903, time step 1903 is labeled as pre, while 1904 is labeled as post).

att.aggregate("event")
att.plot("event")

However, there seems to be inconsistency with the above cohort definition when aggregating the ATTs. For instance, when I run the aggregate method for the "cohort" level, I get the following results,

att.aggregate("cohort")

The average ATTs presented here follow the standard cohort definition (i.e. start of the intervention), as they include the time == cohort data points. This clearly biases the estimated ATTs, as it includes a pre-intervention time period. This is easy to see by manually aggregating the ATTs. Clearly the second result, which does not include the time == cohort estimates, is the right one.

att.results().query("time >= cohort").iloc[:, 0].groupby(level=0).mean()

cohort
1902    16.819243
1903    15.635493
1904    15.567925
1905    13.587051
Name: (ATTgtResult, , ATT), dtype: float64

att.results().query("time > cohort").iloc[:, 0].groupby(level=0).mean()

cohort
1902    20.142418
1903    19.561096
1904    20.113236
1905    19.812459
Name: (ATTgtResult, , ATT), dtype: float64

Hence, it seems like both the simulate_data function and the (disaggregated) plot method generate/expect a non-standard cohort definition which is inconsistent with other functions/methods in the package. Thoughts? Sorry if I'm missing something.

Hi, I am trying to run Doubly Robust S-DID with unbalanced panel and varying base period.
the control group is 'not_yet_treated'
My code is as following:

    att_gt = ATTgt(data=diddata, cohort_name="course_month_end_date", base_period='varying', freq='M') 
    att_gt.fit(formula = formula, est_method='dr',control_group=control_group, progress_bar = True) 

however, I am getting following error which I am not able to understand

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File ~/anaconda3/lib/python3.11/site-packages/pandas/core/indexes/base.py:3790, in Index.get_loc(self, key)
   3789 try:
-> 3790     return self._engine.get_loc(casted_key)
   3791 except KeyError as err:

File index.pyx:152, in pandas._libs.index.IndexEngine.get_loc()

File index.pyx:181, in pandas._libs.index.IndexEngine.get_loc()

File pandas/_libs/hashtable_class_helper.pxi:7080, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas/_libs/hashtable_class_helper.pxi:7088, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: '_course_month_end_date'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
Cell In[176], line 15
     13 config = 'config' + str(i)
     14 att_gt = ATTgt(data=diddata, cohort_name="course_month_end_date", base_period='varying', freq='M') 
---> 15 att_gt.fit(formula = formula, est_method='dr',control_group=control_group, progress_bar = True)

File ~/anaconda3/lib/python3.11/site-packages/differences/attgt/attgt.py:597, in ATTgt.fit(self, formula, weights_name, control_group, base_delta, est_method, as_repeated_cross_section, boot_iterations, random_state, alpha, cluster_var, split_sample_by, n_jobs, backend, progress_bar)
    593     is_panel, is_balanced_panel = False, False
    595 # --------------------- filter cohort times ---------------------
--> 597 group_time = self.group_time(feasible=False)
    599 # todo: if balance panel if filter? should already be done if balance 2*2
    600 filter_gt = None

File ~/anaconda3/lib/python3.11/site-packages/differences/attgt/attgt.py:213, in ATTgt.group_time(self, feasible)
    204 """
    205 Returns
    206 -------
    207 a list of dictionaries where each dictionary keys are:
    208 ``cohort``, ``base_period``, ``time``, (``stratum``)
    209 """
    211 if self.base_period_type == "varying":
    212     cbt = varying_base_period(
--> 213         cohort_ar=self._cohorts,
    214         time_ar=self._times,
    215         anticipation=self.anticipation,
    216     )
    218 if self.base_period_type == "universal":
    219     cbt = universal_base_period(
    220         cohort_ar=self._cohorts,
    221         time_ar=self._times,
    222         anticipation=self.anticipation,
    223     )

File ~/anaconda3/lib/python3.11/site-packages/differences/attgt/attgt.py:162, in ATTgt._cohorts(self)
    160 @property
    161 def _cohorts(self):
--> 162     cohorts = np.array(sorted(self.data[self.cohort_name].dropna().unique()))
    163     return cohorts[cohorts > self._times[0] + self.anticipation]

File ~/anaconda3/lib/python3.11/site-packages/pandas/core/frame.py:3896, in DataFrame.__getitem__(self, key)
   3894 if self.columns.nlevels > 1:
   3895     return self._getitem_multilevel(key)
-> 3896 indexer = self.columns.get_loc(key)
   3897 if is_integer(indexer):
   3898     indexer = [indexer]

File ~/anaconda3/lib/python3.11/site-packages/pandas/core/indexes/base.py:3797, in Index.get_loc(self, key)
   3792     if isinstance(casted_key, slice) or (
   3793         isinstance(casted_key, abc.Iterable)
   3794         and any(isinstance(x, slice) for x in casted_key)
   3795     ):
   3796         raise InvalidIndexError(key)
-> 3797     raise KeyError(key) from err
   3798 except TypeError:
   3799     # If we have a listlike key, _check_indexing_error will raise
   3800     #  InvalidIndexError. Otherwise we fall through and re-raise
   3801     #  the TypeError.
   3802     self._check_indexing_error(key)

KeyError: '_course_month_end_date'

The 'course_month_end_date' column does exists in the dataframe. I will really appreciate your help in debugging this.

Hi there. Coming over from vega/altair#3051 where we noticed using GitHub search that you use the altair.sphinxext.altairplot Sphinx directive. We removed the sphinxext module in the new Altair 5 release but we are currently working on publishing a separate sphinxext-altair package which you will be able to use as a substitute. I'll let you know once it's released so this is just a heads-up in case you see an error when building your docs.

There is a bug in the following line: