From dc22f2d5907cd7fee29e80b353dd528000880a7a Mon Sep 17 00:00:00 2001
From: Scott Staniewicz <scott.j.staniewicz@jpl.nasa.gov>
Date: Wed, 11 Sep 2024 09:34:28 -0400
Subject: [PATCH] Adjust `GDAL_MAX_DATASET_POOL_SIZE` if system file limit is
 low

---
 src/dolphin/timeseries.py             |  3 +++
 src/dolphin/utils.py                  | 36 ++++++++++++++++++++++++---
 src/dolphin/workflows/displacement.py |  5 +++-
 3 files changed, 39 insertions(+), 5 deletions(-)

diff --git a/src/dolphin/timeseries.py b/src/dolphin/timeseries.py
index 8fed3dca..1c30248b 100644
--- a/src/dolphin/timeseries.py
+++ b/src/dolphin/timeseries.py
@@ -124,6 +124,9 @@ def run(
     in both ascending and descending tracks imply uplift).
 
     """
+    # Ensure we wont run into problems with large gdal reads from deep VRTs.
+    utils.check_open_file_limit()
+
     Path(output_dir).mkdir(exist_ok=True, parents=True)
 
     condition_func = argmax_index if condition == CallFunc.MAX else argmin_index
diff --git a/src/dolphin/utils.py b/src/dolphin/utils.py
index a5cb9789..66a16042 100644
--- a/src/dolphin/utils.py
+++ b/src/dolphin/utils.py
@@ -3,6 +3,7 @@
 import datetime
 import logging
 import math
+import os
 import resource
 import sys
 import warnings
@@ -129,8 +130,6 @@ def full_suffix(filename: Filename):
 
 def disable_gpu():
     """Disable GPU usage."""
-    import os
-
     import jax
 
     os.environ["CUDA_VISIBLE_DEVICES"] = ""
@@ -414,8 +413,6 @@ def set_num_threads(num_threads: int):
 
     Uses https://github.com/joblib/threadpoolctl for numpy.
     """
-    import os
-
     import numba
     from threadpoolctl import ThreadpoolController
 
@@ -464,6 +461,37 @@ def get_cpu_period():
     return cpu_count()
 
 
+def check_open_file_limit(file_limit_threshold: int = 512) -> int:
+    """Adjust GDAL_MAX_DATASET_POOL_SIZE based on the system's open file limit.
+
+    This function checks the current file descriptor limit using the resource module.
+    If the limit is below a specified threshold, it lowers the environment variable
+    `GDAL_MAX_DATASET_POOL_SIZE`.
+
+    Parameters
+    ----------
+    file_limit_threshold : int, optional
+        The minimum file descriptor limit below which GDAL_MAX_DATASET_POOL_SIZE
+        should be shrunk.
+        Default is 512.
+
+    Returns
+    -------
+    int
+        The current file descriptor limit, as stated in `resource.RLIMIT_NOFILE`
+
+    """
+    soft_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
+
+    if soft_limit < file_limit_threshold:
+        logger.info(
+            f"File descriptor limit is below {soft_limit}. "
+            "Shrinking GDAL_MAX_DATASET_POOL_SIZE environment variable.",
+        )
+        os.environ["GDAL_MAX_DATASET_POOL_SIZE"] = str(file_limit_threshold // 4)
+    return soft_limit
+
+
 def flatten(list_of_lists: Iterable[Iterable[Any]]) -> chain[Any]:
     """Flatten one level of a nested iterable."""
     return chain.from_iterable(list_of_lists)
diff --git a/src/dolphin/workflows/displacement.py b/src/dolphin/workflows/displacement.py
index 50a40158..99d820e7 100755
--- a/src/dolphin/workflows/displacement.py
+++ b/src/dolphin/workflows/displacement.py
@@ -63,11 +63,14 @@ def run(
     """
     if cfg.log_file is None:
         cfg.log_file = cfg.work_directory / "dolphin.log"
+
     # Set the logging level for all `dolphin.` modules
     for logger_name in ["dolphin", "spurt"]:
         setup_logging(logger_name=logger_name, debug=debug, filename=cfg.log_file)
-    # TODO: need to pass the cfg filename for the logger
+
     logger.debug(cfg.model_dump())
+    # Ensure we wont run into problems with large gdal reads from VRTs.
+    utils.check_open_file_limit()
 
     if not cfg.worker_settings.gpu_enabled:
         utils.disable_gpu()