Merge pull request #52 from geoadmin/develop

New Release v1.1.0 - #minor

.env.local

@@ -1 +1 @@
-ALLOWED_DOMAINS=http[s]?://localhost
+ALLOWED_DOMAINS=.*

.gitignore

@@ -58,4 +58,6 @@ nose2-junit.xml
 *.orig
 
 # visual studio code config
-.vscode
+.vscode
+
+out.png

.style.yapf

@@ -17,9 +17,20 @@ based_on_style=google
 #       end_ts=now(),
 #   )        # <--- this bracket is dedented and on a separate line
 dedent_closing_brackets=True
+coalesce_brackets=True
 
-# Split before arguments, but do not split all subexpressions recursively
+arithmetic_precedence_indication=True
+no_spaces_around_selected_binary_operators=True
+
+# This avoid issues with complex dictionary
+# see https://github.com/google/yapf/issues/392#issuecomment-407958737
+indent_dictionary_value=True
+allow_split_before_dict_value=False
+
+# Split before arguments, but do not split all sub expressions recursively
 # (unless needed).
 split_all_top_level_comma_separated_values=True
 
+# Split lines longer than 100 characters (this only applies to code not to
+# comment and docstring)
 column_limit=100

Dockerfile

@@ -1,5 +1,5 @@
-# Buster slim python 3.7 base image.
-FROM python:3.7-slim-buster
+# Buster slim python 3.9 base image.
+FROM python:3.9-slim-buster
 ENV HTTP_PORT 8080
 RUN groupadd -r geoadmin && useradd -r -s /bin/false -g geoadmin geoadmin
 

LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2020, swisstopo
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Pipfile

@@ -8,7 +8,7 @@ PyYAML = "~=5.4.1"
 gevent = "~=21.1.2"
 gunicorn = "~=20.1.0"
 Flask = "~=2.0.1"
-Pillow = "~=8.3.2"
+Pillow = "~=9.0.0"
 python-dotenv = "~=0.17.0"
 logging-utilities = "~=1.2.3"
 
@@ -19,4 +19,4 @@ pylint = "*"
 pylint-flask = "*"
 
 [requires]
-python_version = "3.7"
+python_version = "3.9"

README.md

@@ -11,9 +11,6 @@
 - [Description](#description)
 - [Dependencies](#dependencies)
 - [Service API](#service-api)
-  - [Staging Environments](#staging-environments)
-  - [checker GET](#checker-get)
-  - [color GET](#color-get)
 - [Versioning](#versioning)
 - [Local Development](#local-development)
   - [Make Dependencies](#make-dependencies)
@@ -22,8 +19,6 @@
   - [Test your work](#test-your-work)
 - [Docker](#docker)
 - [Deployment](#deployment)
-  - [Dev](#dev)
-  - [Int](#int)
   - [Deployment configuration](#deployment-configuration)
 
 ## Description
@@ -36,37 +31,19 @@ This service doesn't have any external dependencies
 
 ## Service API
 
-This service has two endpoints:
+The service has the following endpoints:
 
-- [checker GET](#checker-get)
-- [color GET](#color-get)
+- `GET /checker`
+- `GET /sets`
+- `GET /sets/<icon_set_name>`
+- `GET /sets/<icon_set_name>/icons`
+- `GET /sets/<icon_set_name>/icons/<icon_name>`
+- `GET /sets/<icon_set_name>/icons/<icon_name>.png`
+- `GET /sets/<icon_set_name>/icons/<icon_name>-<red>,<green>,<blue>.png`
+- `GET /sets/<icon_set_name>/icons/<icon_name>@<scale>.png`
+- `GET /sets/<icon_set_name>/icons/<icon_name>@<scale>-<red>,<green>,<blue>.png`
 
-A detailed descriptions of the endpoints can be found in the [OpenAPI Spec](openapi.yaml).
-
-### Staging Environments
-
-| Environments | URL                                                                                                                   |
-| ------------ | --------------------------------------------------------------------------------------------------------------------- |
-| DEV          | [https://service-icons.bgdi-dev.swisstopo.cloud/v4/icons/](https://service-icons.bgdi-dev.swisstopo.cloud/v4/icons/)  |
-| INT          | [https://service-icons.bgdi-int.swisstopo.cloud/v4/icons/](https://service-icons.bgdi-int.swisstopo.cloud/v4/icons/)  |
-| PROD         | [https://service-icons.bgdi-prod.swisstopo.cloud/v4/icons/](https://service-icons.bgdi-int.swisstopo.cloud/v4/icons/) |
-
-### checker GET
-
-This is a simple route meant to test if the server is up.
-
-| Path              | Method | Argument | Response Type    |
-| ----------------- | ------ | -------- | ---------------- |
-| /v4/icons/checker | GET    | -        | application/json |
-
-### color GET
-
-This route takes a color (defined by r, g and b values) and the name of a file containing a symbol to be colorized
-and returns the colorized symbol.
-
-| Path      | Method | Argument          | Response Type |
-| --------- | ------ | ----------------- | ------------- |
-| /v4/icons | GET    | r, g, b, filename | image/png     |
+A detailed descriptions of the endpoints can be found in the [OpenAPI Spec](https://github.com/geoadmin/doc-api-specs) repository.
 
 ## Versioning
 
@@ -78,7 +55,7 @@ See also [Git Flow - Versioning](https://github.com/geoadmin/doc-guidelines/blob
 
 ### Make Dependencies
 
-The **Make** targets assume you have **python3.7**, **pipenv**, **bash**, **curl**, **tar**, **docker** and **docker-compose** installed.
+The **Make** targets assume you have **python3.9**, **pipenv**, **bash**, **curl**, **tar**, **docker** and **docker-compose** installed.
 
 ### Setting up to work
 
@@ -91,7 +68,7 @@ git clone [email protected]:geoadmin/service-icons
 Then, you can run the setup target to ensure you have everything needed to develop, test and serve locally
 
 ```bash
-make setup
+make dev
 ```
 
 That's it, you're ready to work.
@@ -137,9 +114,14 @@ make dockerrun
 
 This will serve the application with the wsgi server, inside a container.
 
-    curl -H "Origin: https://map.geo.admin.ch/" http://localhost:5000/v4/icons/255,133,133/[email protected] --output out.dat
+Here below are simple examples of how to test the service after serving on localhost:5000:
+
+```bash
+curl -H "Origin: www.example.com" http://localhost:5000/sets/default/icons
+curl -H "Origin: www.example.com" http://localhost:5000/sets/default/icons/001-marker@2x-255,133,133.png --output out.png
+```
 
-This is a simple example of how to test the service after serving on localhost:5000 (`out.dat` will either contain a PNG image or contain an error message.)
+*NOTE: if you serve using gunicorn or docker, you need to add the route prefix `/api/icons`*
 
 ## Docker
 
@@ -176,35 +158,6 @@ docker ps --format="table {{.ID}}\t{{.Image}}\t{{.Labels}}"
 
 ## Deployment
 
-This service is to be deployed to the Kubernetes cluster once it is merged.
-
-**Check on which k8s you are before doing anything**
-
-by running :
-
-```bash
-kubectl config get-contexts
-```
-or (if you already have it installed)
-```bash
-kubectx
-```
-
-Make sure you are on the right context (staging) for what you want to achieve
-
-### Dev
-
-To deploy (or refresh) on dev, we have to kill all pods. The K8S configuration is made so that it will revive any killed pod while always retrieving the latest docker image.
-So as soon as your merge on `develop` as been successfully built by the CI (and so the CI has pushed the new image to our AWS ECR registry) you can kill all dev pods by running
-
-```bash
-kubectl delete --all pods --namespace=service-icons
-```
-
-### Int
-
-TO DO: give instructions to deploy to kubernetes.
-
 ### Deployment configuration
 
 The service is configured by Environment Variable:
@@ -213,5 +166,9 @@ The service is configured by Environment Variable:
 | ----------- | --------------------- | -------------------------- |
 | LOGGING_CFG | `logging-cfg-local.yml` | Logging configuration file |
 | ALLOWED_DOMAINS | `.*` | Comma separated list of regex that are allowed as domain in Origin header |
-| CACHE_CONTROL | `public, max-age=86400` | Cache Control header value of the `GET /v4/icons/*` endpoints |
+| CACHE_CONTROL | `public, max-age=86400` | Cache Control header value of the `GET /*` endpoints |
 | CACHE_CONTROL_4XX | `public, max-age=3600` | Cache Control header for 4XX responses |
+| FORWARED_ALLOW_IPS | `*` | Sets the gunicorn `forwarded_allow_ips`. See [Gunicorn Doc](https://docs.gunicorn.org/en/stable/settings.html#forwarded-allow-ips). This setting is required in order to `secure_scheme_headers` to work. |
+| FORWARDED_PROTO_HEADER_NAME | `X-Forwarded-Proto` | Sets gunicorn `secure_scheme_headers` parameter to `{${FORWARDED_PROTO_HEADER_NAME}: 'https'}`. This settings is required in order to generate correct URLs in the service responses. See [Gunicorn Doc](https://docs.gunicorn.org/en/stable/settings.html#secure-scheme-headers). |
+| SCRIPT_NAME | `''` | If the service is behind a reverse proxy and not served at the root, the route prefix must be set in `SCRIPT_NAME`. |
+| WSGI_TIMEOUT | `5` | WSGI timeout. |

app/__init__.py

@@ -1,10 +1,12 @@
 import logging
 import re
+import time
 
 from werkzeug.exceptions import HTTPException
 
 from flask import Flask
 from flask import abort
+from flask import g
 from flask import request
 
 from app.helpers import make_error_msg
@@ -27,7 +29,8 @@
 # the route might not be logged if another method reject the request.
 @app.before_request
 def log_route():
-    route_logger.info('%s %s', request.method, request.path)
+    g.setdefault('started', time.time())
+    route_logger.debug('%s %s', request.method, request.path)
 
 
 # Add CORS Headers to all request
@@ -68,6 +71,23 @@ def validate_origin():
         abort(403, 'Not allowed')
 
 
+@app.after_request
+def log_response(response):
+    logger.info(
+        "%s %s - %s",
+        request.method,
+        request.path,
+        response.status,
+        extra={
+            'response': {
+                "status_code": response.status_code, "headers": dict(response.headers.items())
+            },
+            "duration": time.time() - g.get('started', time.time())
+        }
+    )
+    return response
+
+
 # Register error handler to make sure that every error returns a json answer
 @app.errorhandler(Exception)
 def handle_exception(err):

app/helpers/check_functions.py

@@ -9,7 +9,10 @@
 
 
 def __check_color(color):
-    return 0 <= color <= 255
+    try:
+        return 0 <= int(color) <= 255
+    except ValueError:
+        return False
 
 
 def check_color_channels(red, green, blue):
@@ -29,13 +32,24 @@ def check_color_channels(red, green, blue):
     if not (__check_color(red) and __check_color(green) and __check_color(blue)):
         logger.error(
             "Color channel values must be integers in the range of 0 to 255. "
-            "(given: %d, %d, %d)",
+            "(given: %s, %s, %s)",
             red,
             green,
             blue
         )
         abort(400, "Color channel values must be integers in the range of 0 to 255.")
-    return red, green, blue
+    return int(red), int(green), int(blue)
+
+
+def check_scale(scale):
+    try:
+        _scale = float(scale.rstrip('x'))
+    except ValueError:
+        _scale = 0
+    if _scale <= 0:
+        logger.error('Invalid Scale %s: must be a number > 0', scale)
+        abort(400, "Invalid scale must be a positive number")
+    return _scale
 
 
 def get_and_check_icon_set(icon_set_name):
@@ -53,7 +67,7 @@ def get_and_check_icon_set(icon_set_name):
     icon_set = get_icon_set(icon_set_name)
     if not icon_set:
         logger.error("Icon set not found: %s", icon_set_name)
-        abort(400, "Icon set not found")
+        abort(404, "Icon set not found")
     return icon_set
 
 
@@ -72,5 +86,5 @@ def get_and_check_icon(icon_set, icon_name):
     path = icon.get_icon_filepath()
     if not os.path.isfile(path):
         logger.error("The icon doesn't exist: %s", path)
-        abort(400, "Icon not found in icon set")
+        abort(404, "Icon not found in icon set")
     return icon

app/icon.py

@@ -7,6 +7,19 @@
 from app.settings import DEFAULT_COLOR
 from app.settings import IMAGE_FOLDER
 
+# Here we disable yapf to avoid putting spaces between fractional parts
+# (`24/48` instead of `24 / 48`)
+# yapf: disable
+
+# Icon anchor is defined as [x, y] and as fractional. Here below we used the x and y in pixels
+# to define the fraction with width=48px and height=48px
+DEFAULT_ICON_ANCHOR = [24/48, 24/48]
+ICON_ANCHORS = {
+    '001-marker': [24/48, 42/48],
+    '007-marker-stroked': [24/48, 42/48],
+}
+# yapf: enable
+
 
 class Icon:
     """
@@ -21,6 +34,7 @@ def __init__(self, name, icon_set):
         """
         self.name = name
         self.icon_set = icon_set
+        self.anchor = ICON_ANCHORS.get(name, DEFAULT_ICON_ANCHOR)
 
     def get_icon_url(
         self, red=DEFAULT_COLOR['r'], green=DEFAULT_COLOR['g'], blue=DEFAULT_COLOR['b']
@@ -80,6 +94,7 @@ def serialize(self):
         """
         return {
             "name": self.name,
+            "anchor": self.anchor,
             "icon_set": self.icon_set.name,
             "url": self.get_icon_url(),
             "template_url": get_icon_template_url(get_base_url())

app/icon_set.py

@@ -107,7 +107,7 @@ def get_all_icons(self):
             return None
         icons = []
         for root, dirs, files in os.walk(os.path.join(IMAGE_FOLDER, self.name)):
-            for icon_filename in files:
+            for icon_filename in sorted(files):
                 name_without_extension = os.path.splitext(icon_filename)[0]
                 icons.append(self.get_icon(name_without_extension))
         return icons