Desparsified lasso(1/4): add comments and docstring of the functions

doc_conf/api.rst

@@ -20,6 +20,9 @@ Functions
    clustered_inference
    data_simulation
    desparsified_lasso
+   desparsified_lasso_pvalue
+   desparsified_group_lasso
+   desparsified_group_lasso_pvalue
    ensemble_clustered_inference
    group_reid
    hd_inference

doc_conf/references.bib

@@ -177,4 +177,103 @@ @article{liuFastPowerfulConditional2021
   archiveprefix = {arxiv},
   keywords = {Statistics - Methodology},
   file = {/home/ahmad/Zotero/storage/8HRQZX3H/Liu et al. - 2021 - Fast and Powerful Conditional Randomization Testin.pdf;/home/ahmad/Zotero/storage/YFNDKN2B/2006.html}
-}
+}
+
+@article{zhang2014confidence,
+  title={Confidence intervals for low dimensional parameters in high dimensional linear models},
+  author={Zhang, Cun-Hui and Zhang, Stephanie S},
+  journal={Journal of the Royal Statistical Society Series B: Statistical Methodology},
+  volume={76},
+  number={1},
+  pages={217--242},
+  year={2014},
+  publisher={Oxford University Press}
+}
+
+@article{van2014asymptotically,
+  title={ON ASYMPTOTICALLY OPTIMAL CONFIDENCE REGIONS AND TESTS FOR HIGH-DIMENSIONAL MODELS},
+  author={van de Geer, Sara and B{\"u}hlmann, Peter and Ritov, Ya'acov and Dezeure, Ruben},
+  journal={The Annals of Statistics},
+  pages={1166--1202},
+  year={2014},
+  publisher={JSTOR}
+}
+
+@article{javanmard2014confidence,
+  title={Confidence intervals and hypothesis testing for high-dimensional regression},
+  author={Javanmard, Adel and Montanari, Andrea},
+  journal={The Journal of Machine Learning Research},
+  volume={15},
+  number={1},
+  pages={2869--2909},
+  year={2014},
+  publisher={JMLR. org}
+}
+
+@article{bellec2022biasing,
+  title={De-biasing the lasso with degrees-of-freedom adjustment},
+  author={Bellec, Pierre C and Zhang, Cun-Hui},
+  journal={Bernoulli},
+  volume={28},
+  number={2},
+  pages={713--743},
+  year={2022},
+  publisher={Bernoulli Society for Mathematical Statistics and Probability}
+}
+
+@article{celentano2023lasso,
+  title={The lasso with general gaussian designs with applications to hypothesis testing},
+  author={Celentano, Michael and Montanari, Andrea and Wei, Yuting},
+  journal={The Annals of Statistics},
+  volume={51},
+  number={5},
+  pages={2194--2220},
+  year={2023},
+  publisher={Institute of Mathematical Statistics}
+}
+
+@phdthesis{chevalier2020statistical,
+  title={Statistical control of sparse models in high dimension},
+  author={Chevalier, J{\'e}r{\^o}me-Alexis},
+  year={2020},
+  school={Universit{\'e} Paris-Saclay}
+}
+
+@article{fan2012variance,
+  title={Variance estimation using refitted cross-validation in ultrahigh dimensional regression},
+  author={Fan, Jianqing and Guo, Shaojun and Hao, Ning},
+  journal={Journal of the Royal Statistical Society Series B: Statistical Methodology},
+  volume={74},
+  number={1},
+  pages={37--65},
+  year={2012},
+  publisher={Oxford University Press}
+}
+
+@article{reid2016study,
+  title={A study of error variance estimation in lasso regression},
+  author={Reid, Stephen and Tibshirani, Robert and Friedman, Jerome},
+  journal={Statistica Sinica},
+  pages={35--67},
+  year={2016},
+  publisher={JSTOR}
+}
+
+@article{chevalier2020statistical,
+  title={Statistical control for spatio-temporal meg/eeg source imaging with desparsified mutli-task lasso},
+  author={Chevalier, J{\'e}r{\^o}me-Alexis and Salmon, Joseph and Gramfort, Alexandre and Thirion, Bertrand},
+  journal={Advances in Neural Information Processing Systems},
+  volume={33},
+  pages={1759--1770},
+  year={2020}
+}
+
+@article{eshel2003yule,
+  title={The yule walker equations for the AR coefficients},
+  author={Eshel, Gidon},
+  journal={Internet resource},
+  volume={2},
+  pages={68--73},
+  year={2003}
+}
+

examples/plot_2D_simulation_example.py

@@ -61,10 +61,10 @@
 from sklearn.feature_extraction import image
 
 from hidimstat.clustered_inference import clustered_inference
-from hidimstat.desparsified_lasso import desparsified_lasso
+from hidimstat.desparsified_lasso import desparsified_lasso, desparsified_lasso_pvalue
 from hidimstat.ensemble_clustered_inference import ensemble_clustered_inference
 from hidimstat.scenario import multivariate_simulation
-from hidimstat.stat_tools import pval_from_cb, zscore_from_pval
+from hidimstat.stat_tools import zscore_from_pval
 
 #############################################################################
 # Specific plotting functions
@@ -238,7 +238,9 @@ def plot(maps, titles):
 
 # compute desparsified lasso
 beta_hat, cb_min, cb_max = desparsified_lasso(X_init, y, n_jobs=n_jobs)
-pval, pval_corr, one_minus_pval, one_minus_pval_corr = pval_from_cb(cb_min, cb_max)
+pval, pval_corr, one_minus_pval, one_minus_pval_corr = desparsified_lasso_pvalue(
+    cb_min, cb_max
+)
 
 # compute estimated support (first method)
 zscore = zscore_from_pval(pval, one_minus_pval)

src/hidimstat/__init__.py

@@ -1,6 +1,10 @@
 from .adaptive_permutation_threshold import ada_svr
 from .clustered_inference import clustered_inference, hd_inference
-from .desparsified_lasso import desparsified_group_lasso, desparsified_lasso
+from .desparsified_lasso import desparsified_lasso
+from .desparsified_lasso import (
+    desparsified_group_lasso,
+    desparsified_group_lasso_pvalue,
+)
 from .Dnn_learner_single import DnnLearnerSingle
 from .ensemble_clustered_inference import ensemble_clustered_inference
 from .knockoff_aggregation import knockoff_aggregation
@@ -27,6 +31,7 @@
     "dcrt_zero",
     "desparsified_lasso",
     "desparsified_group_lasso",
+    "desparsified_group_lasso_pvalue",
     "DnnLearnerSingle",
     "ensemble_clustered_inference",
     "group_reid",

src/hidimstat/clustered_inference.py

@@ -3,8 +3,11 @@
 from sklearn.utils import resample
 from sklearn.utils.validation import check_memory
 
-from .desparsified_lasso import desparsified_group_lasso, desparsified_lasso
-from .stat_tools import pval_from_cb
+from .desparsified_lasso import desparsified_lasso, desparsified_lasso_pvalue
+from .desparsified_lasso import (
+    desparsified_group_lasso,
+    desparsified_group_lasso_pvalue,
+)
 
 
 def _subsampling(n_samples, train_size, groups=None, seed=0):
@@ -45,7 +48,7 @@ def _ward_clustering(X_init, ward, train_index):
     return X_reduced, ward
 
 
-def hd_inference(X, y, method, n_jobs=1, memory=None, verbose=0, **kwargs):
+def hd_inference(X, y, method, n_jobs=1, verbose=0, **kwargs):
     """Wrap-up high-dimensional inference procedures
 
     Parameters
@@ -65,11 +68,6 @@ def hd_inference(X, y, method, n_jobs=1, memory=None, verbose=0, **kwargs):
     n_jobs : int or None, optional (default=1)
         Number of CPUs to use during parallel steps such as inference.
 
-    memory : str or joblib.Memory object, optional (default=None)
-        Used to cache the output of the computation of the clustering
-        and the inference. By default, no caching is done. If a string is
-        given, it is the path to the caching directory.
-
     verbose: int, optional (default=1)
         The verbosity level. If `verbose > 0`, we print a message before
         runing the clustered inference.
@@ -104,20 +102,20 @@ def hd_inference(X, y, method, n_jobs=1, memory=None, verbose=0, **kwargs):
             y,
             confidence=0.95,
             n_jobs=n_jobs,
-            memory=memory,
             verbose=verbose,
             **kwargs,
         )
-        pval, pval_corr, one_minus_pval, one_minus_pval_corr = pval_from_cb(
-            cb_min, cb_max, confidence=0.95
+        pval, pval_corr, one_minus_pval, one_minus_pval_corr = (
+            desparsified_lasso_pvalue(cb_min, cb_max, confidence=0.95, **kwargs)
         )
 
     elif method == "desparsified-group-lasso":
 
-        beta_hat, pval, pval_corr, one_minus_pval, one_minus_pval_corr = (
-            desparsified_group_lasso(
-                X, y, n_jobs=n_jobs, memory=memory, verbose=verbose, **kwargs
-            )
+        beta_hat, theta_hat, omega_diag = desparsified_group_lasso(
+            X, y, n_jobs=n_jobs, verbose=verbose, **kwargs
+        )
+        pval, pval_corr, one_minus_pval, one_minus_pval_corr = (
+            desparsified_group_lasso_pvalue(beta_hat, theta_hat, omega_diag, **kwargs)
         )
 
     else:
@@ -178,7 +176,6 @@ def clustered_inference(
     method="desparsified-lasso",
     seed=0,
     n_jobs=1,
-    memory=None,
     verbose=1,
     **kwargs,
 ):
@@ -220,11 +217,6 @@ def clustered_inference(
     n_jobs : int or None, optional (default=1)
         Number of CPUs to use during parallel steps such as inference.
 
-    memory : str or joblib.Memory object, optional (default=None)
-        Used to cache the output of the computation of the clustering
-        and the inference. By default, no caching is done. If a string is
-        given, it is the path to the caching directory.
-
     verbose: int, optional (default=1)
         The verbosity level. If `verbose > 0`, we print a message before
         runing the clustered inference.
@@ -257,9 +249,6 @@ def clustered_inference(
            Spatially relaxed inference on high-dimensional linear models.
            arXiv preprint arXiv:2106.02590.
     """
-
-    memory = check_memory(memory)
-
     n_samples, n_features = X_init.shape
 
     if verbose > 0:
@@ -273,15 +262,15 @@ def clustered_inference(
     train_index = _subsampling(n_samples, train_size, groups=groups, seed=seed)
 
     # Clustering
-    X, ward = memory.cache(_ward_clustering)(X_init, ward, train_index)
+    X, ward = _ward_clustering(X_init, ward, train_index)
 
     # Preprocessing
     X = StandardScaler().fit_transform(X)
     y = y - np.mean(y)
 
     # Inference: computing reduced parameter vector and stats
     beta_hat_, pval_, pval_corr_, one_minus_pval_, one_minus_pval_corr_ = hd_inference(
-        X, y, method, n_jobs=n_jobs, memory=memory, **kwargs
+        X, y, method, n_jobs=n_jobs, **kwargs
     )
 
     # De-grouping