Adding sieve_max_threshold configuration and extened gdal.SieveFilter…

…() usage.

ngeo_browse_server/control/control/configuration.py

@@ -262,6 +262,13 @@ class IngestConfigurator(ngEOConfigConfigurator):
             "Defines whether or not a alpha channel shall be used to display "
             "the images area of interest.", "false"
         ),
+        Parameter(
+            int, "sieve_max_threshold", "",
+            "Sets the maximum threshold for the sieve algorithm. See "
+            "`http://www.gdal.org/gdal__alg_8h.html#a33309c0a316b223bd33ae5753cc7f616` "
+            "for details. Defaults to the number of pixels divided by 16 "
+            "which is indicated by a value of 0.", "0"
+        ),
         Parameter(
             int, "simplification_factor", "",
             "Sets the factor for the simplification algorithm. See "

ngeo_browse_server/control/ingest/config.py

@@ -170,6 +170,12 @@ def get_optimization_config(config=None):
     except:
         pass
 
+    try:
+        values["sieve_max_threshold"] = config.getint(
+            INGEST_SECTION, "sieve_max_threshold")
+    except:
+        pass
+
     try:
         values["simplification_factor"] = config.getfloat(
             INGEST_SECTION, "simplification_factor")

ngeo_browse_server/control/ingest/preprocessing/preprocessor.py

@@ -51,8 +51,50 @@
 logger = logging.getLogger(__name__)
 
 
+# enum for bandmode
+RGB = range(3)
+
+
 class NGEOPreProcessor(WMSPreProcessor):
 
+    def __init__(self, format_selection, overviews=True, crs=None, bands=None,
+                 bandmode=RGB, footprint_alpha=False,
+                 color_index=False, palette_file=None, no_data_value=None,
+                 overview_resampling=None, overview_levels=None,
+                 overview_minsize=None, radiometric_interval_min=None,
+                 radiometric_interval_max=None, sieve_max_threshold=None,
+                 simplification_factor=None, temporary_directory=None):
+
+        self.format_selection = format_selection
+        self.overviews = overviews
+        self.overview_resampling = overview_resampling
+        self.overview_levels = overview_levels
+        self.overview_minsize = overview_minsize
+
+        self.crs = crs
+
+        self.bands = bands
+        self.bandmode = bandmode
+        self.footprint_alpha = footprint_alpha
+        self.color_index = color_index
+        self.palette_file = palette_file
+        self.no_data_value = no_data_value
+        self.radiometric_interval_min = radiometric_interval_min
+        self.radiometric_interval_max = radiometric_interval_max
+
+        if sieve_max_threshold is not None:
+            self.sieve_max_threshold = sieve_max_threshold
+        else:
+            self.sieve_max_threshold = 0
+
+        if simplification_factor is not None:
+            self.simplification_factor = simplification_factor
+        else:
+            # default 2 * resolution == 2 pixels
+            self.simplification_factor = 2
+
+        self.temporary_directory = temporary_directory
+
     def process(self, input_filename, output_filename,
                 geo_reference=None, generate_metadata=True,
                 merge_with=None, original_footprint=None):
@@ -231,9 +273,9 @@ def process(self, input_filename, output_filename,
 
         return PreProcessResult(output_filename, footprint, num_bands)
 
-
     def _generate_footprint_wkt(self, ds):
-        """ Generate a footprint from a raster, using black/no-data as exclusion
+        """ Generate a footprint from a raster, using black/no-data as
+            exclusion
         """
 
         # create an empty boolean array initialized as 'False' to store where
@@ -259,8 +301,20 @@ def _generate_footprint_wkt(self, ds):
         copy_projection(ds, tmp_ds)
         tmp_band = tmp_ds.GetRasterBand(1)
         tmp_band.WriteArray(nodata_map.astype(numpy.uint8))
-        # Remove areas that are smaller than 16 pixels
-        gdal.SieveFilter(tmp_band, None, tmp_band, 16, 4)
+
+        # Remove unwanted small areas of nodata
+        # www.gdal.org/gdal__alg_8h.html#a33309c0a316b223bd33ae5753cc7f616
+        no_pixels = tmp_ds.RasterXSize * tmp_ds.RasterYSize
+        threshold = 4
+        max_threshold = (no_pixels / 16)
+        if self.sieve_max_threshold > 0:
+            max_threshold = self.sieve_max_threshold
+        while threshold <= max_threshold and threshold < 2147483647:
+            gdal.SieveFilter(tmp_band, None, tmp_band, threshold, 4)
+            threshold *= 4
+
+        #for debugging:
+        #gdal.GetDriverByName('GTiff').CreateCopy('/tmp/test.tif', tmp_ds)
 
         # create an OGR in memory layer to hold the created polygon
         sr = osr.SpatialReference()
@@ -305,7 +359,7 @@ def _generate_footprint_wkt(self, ds):
         if geometry.GetGeometryType() != ogr.wkbPolygon:
             raise RuntimeError("Error during polygonization. Wrong geometry "
                                "type: %s" % ogr.GeometryTypeToName(
-                                    geometry.GetGeometryType()))
+                                   geometry.GetGeometryType()))
 
         # check if reprojection to latlon is necessary
         if not sr.IsGeographic():
@@ -321,6 +375,9 @@ def _generate_footprint_wkt(self, ds):
 
         simplification_value = self.simplification_factor * resolution
 
+        #for debugging:
+        #geometry.GetGeometryRef(0).GetPointCount()
+
         # simplify the polygon. the tolerance value is *really* vague
         try:
             # SimplifyPreserveTopology() available since OGR 1.9.0

ngeo_browse_server/control/tests.py

@@ -4011,6 +4011,14 @@ class GetConfigurationAndSchemaTestCase(ConfigMixIn, TestCase):
               </xsd:documentation>
             </xsd:annotation>
           </xsd:element>
+          <xsd:element type="xsd:integer" name="sieve_max_threshold">
+            <xsd:annotation>
+              <xsd:documentation>
+                <xsd:label></xsd:label>
+                <xsd:tooltip>Sets the maximum threshold for the sieve algorithm. See `http://www.gdal.org/gdal__alg_8h.html#a33309c0a316b223bd33ae5753cc7f616` for details. Defaults to the number of pixels divided by 16 which is indicated by a value of 0.</xsd:tooltip>
+              </xsd:documentation>
+            </xsd:annotation>
+          </xsd:element>
           <xsd:element type="xsd:integer" name="simplification_factor">
             <xsd:annotation>
               <xsd:documentation>
@@ -4179,6 +4187,7 @@ class GetConfigurationAndSchemaTestCase(ConfigMixIn, TestCase):
         <overview_minsize>256</overview_minsize>
         <color_index>false</color_index>
         <footprint_alpha>true</footprint_alpha>
+        <sieve_max_threshold>0</sieve_max_threshold>
         <simplification_factor>2</simplification_factor>
         <regular_grid_clipping>false</regular_grid_clipping>
         <in_memory>false</in_memory>
@@ -4225,6 +4234,7 @@ class ConfigurationChangeTestCase(ConfigMixIn, TestCase):
         <overview_minsize>512</overview_minsize>
         <color_index>true</color_index>
         <footprint_alpha>false</footprint_alpha>
+        <sieve_max_threshold>4096</sieve_max_threshold>
         <simplification_factor>3</simplification_factor>
         <regular_grid_clipping>true</regular_grid_clipping>
         <in_memory>true</in_memory>

ngeo_browse_server/project_template/project_name/conf/ngeo.conf

@@ -115,6 +115,12 @@ compression=LZW
 # images area of interest. Default is "false".
 footprint_alpha=true
 
+# Optional. Sets the maximum threshold for the sieve algorithm. See
+# `http://www.gdal.org/gdal__alg_8h.html#a33309c0a316b223bd33ae5753cc7f616`
+# for details. Defaults to the number of pixels divided by 16
+# which is indicated by a value of 0.
+#sieve_max_threshold=0
+
 # Optional. Sets the factor for the simplification algorithm. See
 # `http://en.wikipedia.org/wiki/Ramer%E2%80%93Douglas%E2%80%93Peucker_algorithm`
 # for details. Defaults to 2 (2 * resolution == 2 pixels) which provides