Add deepfaune

.gitignore

@@ -1,2 +1,3 @@
 docker/.env
-/venv
+venv
+frontend/public/env-config.js

README.md

@@ -39,19 +39,18 @@ try it out! User : admin ; password : password
 Docker and docker-compose must be installed on the server/machine (cf. [official website](https://docs.docker.com/engine/install/debian/)).
 
 ### Application downloading
-
 Replace the `X.Y.Z` mention by the name of the release you want to install.
-
 ```
 cd
 wget https://github.com/naturalsolutions/ecosecrets/archive/refs/tags/X.Y.Z.zip
 unzip X.Y.Z.zip
 rm X.Y.Z.zip
 mv ecosecrets-X.Y.Z ecosecrets/
 ```
+### Add DeepFaune
+ecoSecrets use DeepFaune code so you need to download it and add it in the DeepFaune folder in src folder.
 
 ### Settings
-
 Copy the `.env.sample` inside the docker directory to `.env`:
 
 ```
@@ -62,6 +61,8 @@ nano docker/.env
 
 Edit freely this `.env` file to change credentials for instance. Here are the main parameters you usually want to modify:
 
+
+### Launching
 - `ENV` : uncomment it to activate the production mode (only if your app has been configured with a domain name)
 - `DOMAIN` : localhost, an IP address or a domain name (according to your context)
 - `PROTOCOL` : modify it to "https" if you want to activate HTTPS
@@ -79,7 +80,6 @@ In the current version (`0.1.1`), you can't modify the `APP_USER` and the `APP_P
 ```
 ./scripts/docker.sh up -d
 ```
-
 With the default settings, the app will run on `http://localhost:8889/` but the port of each service will be avaible to debug. This URL must be adapted to your context (depending on chosen protocol, domain and port).
 
 ## Sample data (for testing only)

api/Pipfile

@@ -7,6 +7,7 @@ name = "pypi"
 fastapi = "*"
 uvicorn = "*"
 sqlalchemy = ">=1.4.17,<=1.4.35"
+celery = {extras = ["redis"], version = "*"}
 python-multipart = "*"
 psycopg2-binary = "*"
 boto3 = "*"

api/alembic/env.py

@@ -24,7 +24,6 @@
 from src.models.device import Devices  # noqa
 from src.models.file import Files  # noqa
 from src.models.models import (  # noqa
-    Deepfaune,
     ExifKeyModel,
     Groups,
     GroupsUsers,

api/alembic/versions/3bea67bfb786_add_prediction_deepfaune_to_files.py

@@ -0,0 +1,39 @@
+"""add prediction_deepfaune to files
+
+Revision ID: 3bea67bfb786
+Revises: 4cf2ba8715d2
+Create Date: 2024-02-15 15:09:07.320404
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "3bea67bfb786"
+down_revision = "4cf2ba8715d2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "files",
+        sa.Column("prediction_deepfaune", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
+    )
+    op.drop_constraint("files_deepfaune_id_fkey", "files", type_="foreignkey")
+    op.drop_column("files", "deepfaune_id")
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "files", sa.Column("deepfaune_id", sa.INTEGER(), autoincrement=False, nullable=True)
+    )
+    op.create_foreign_key("files_deepfaune_id_fkey", "files", "deepfaune", ["deepfaune_id"], ["id"])
+    op.drop_column("files", "prediction_deepfaune")
+    # ### end Alembic commands ###

api/src/config.py

@@ -19,6 +19,9 @@ class Settings(BaseSettings):
     MINIO_ROOT_USER: str = "test"
     MINIO_ROOT_PASSWORD: str = "password"
     MINIO_BUCKET_NAME: str = "bucket"
+    CELERY_BROKER: str = "redis://:broker_pwd@broker/0"
+    CELERY_BACKEND: str = "redis://:broker_pwd@broker/0"
+    CELERY_APP: str = "deepfaune"
 
     class Config:
         env_file = ".env"

api/src/connectors/celery.py

@@ -0,0 +1,7 @@
+from celery import Celery, chord, shared_task
+
+from src.config import settings
+
+celery_app = Celery(
+    settings.CELERY_APP, broker=settings.CELERY_BROKER, backend=settings.CELERY_BACKEND
+)

api/src/connectors/s3.py

@@ -35,6 +35,7 @@
 
 s3 = boto3.resource("s3", **config_dict)
 s3_client = boto3.client("s3", **config_dict_client)
+s3_client_server = boto3.client("s3", **config_dict)
 
 
 def get_bucket_name():
@@ -79,7 +80,7 @@ def get_obj(filename: str):
     return s3.Object(get_bucket_name(), filename).get()
 
 
-def get_url(filename: str, expiration: float = 3600):
+def get_url_client(filename: str, expiration: float = 3600):
     url = s3_client.generate_presigned_url(
         "get_object",
         Params={"Bucket": get_bucket_name(), "Key": filename},
@@ -88,6 +89,15 @@ def get_url(filename: str, expiration: float = 3600):
     return url
 
 
+def get_url_server(filename: str, expiration: float = 3600):
+    url = s3_client_server.generate_presigned_url(
+        "get_object",
+        Params={"Bucket": get_bucket_name(), "Key": filename},
+        ExpiresIn=expiration,
+    )
+    return url
+
+
 def delete_file_obj(filename: str):
     obj = s3.Object(get_bucket_name(), filename)
     return obj.delete()

api/src/main.py

@@ -36,6 +36,11 @@
 idp.add_swagger_config(app)
 
 
+@app.get("/")
+async def root():
+    return {"message": "Hello Bigger Applications!"}
+
+
 @app.on_event("startup")
 def on_startup():
     init_bucket()

api/src/models/file.py

@@ -6,7 +6,7 @@
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlmodel import Column, Field, Relationship, SQLModel
 
-from src.connectors.s3 import get_url
+from src.connectors.s3 import get_url_client
 
 if TYPE_CHECKING:  # pragma: no cover
     from .deployment import Deployments
@@ -37,7 +37,7 @@ class Files(BaseFiles, table=True):
     name: str = Field(index=True)
     date: Optional[datetime] = Field(default_factory=datetime.utcnow)
     megadetector_id: Optional[int] = Field(foreign_key="megadetector.id")
-    deepfaune_id: Optional[int] = Field(foreign_key="deepfaune.id")
+    prediction_deepfaune: Optional[dict] = Field(sa_column=Column(JSONB), default={})
     deployment_id: int = Field(foreign_key="deployments.id")
     treated: bool = Field(default=False)
     annotations: Optional[List[dict]] = Field(sa_column=Column(JSONB), default=[])
@@ -55,5 +55,5 @@ class ReadFiles(BaseFiles):
     @root_validator
     def gen_url(cls, values):  # pylint: disable=no-self-argument,no-self-use
         filename = f"{values['hash']}.{values['ext']}"
-        values["url"] = get_url(filename)
+        values["url"] = get_url_client(filename)
         return values

api/src/models/models.py

@@ -32,11 +32,6 @@ class Megadetector(SQLModel, table=True):
     label_class: str
 
 
-class Deepfaune(SQLModel, table=True):
-    id: Optional[int] = Field(primary_key=True, index=True)
-    label_class: str
-
-
 class DeploymentTemplateSequenceCorrespondance(SQLModel, table=True):
     deployment_id: Optional[int] = Field(
         default=None, foreign_key="deployments.id", primary_key=True

api/src/routers/files.py

@@ -2,18 +2,22 @@
 
 import io
 import tempfile
+import time
 import uuid as uuid_pkg
+from copy import deepcopy
 from datetime import datetime
 from typing import List
 from zipfile import ZipFile
 
 import magic
-from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
+from celery.result import AsyncResult
+from fastapi import APIRouter, BackgroundTasks, Depends, File, Form, HTTPException, UploadFile
 from fastapi.responses import StreamingResponse
 from sqlmodel import Session
 
 from src.config import settings
 from src.connectors import s3
+from src.connectors.celery import celery_app
 from src.connectors.database import get_db
 from src.models.file import CreateFiles, Files
 from src.schemas.schemas import Annotation
@@ -47,7 +51,8 @@ def get_files(db: Session = Depends(get_db)):
     res = []
     for f in List_files:
         new_f = f.dict()
-        url = s3.get_url(f"{f.hash}.{f.extension}")
+        ext = f.extension.split("/")[1]
+        url = s3.get_url_client(f"{f.hash}.{ext}")
         new_f["url"] = url
         res.append(new_f)
     return res
@@ -62,7 +67,7 @@ def update_annotations(
 
 @router.get("/urls/")
 def display_file(name: str):
-    return s3.get_url(name)
+    return s3.get_url_client(name)
 
 
 @router.post("/exif/")
@@ -79,8 +84,27 @@ def extract_exif(file: UploadFile = File(...), db: Session = Depends(get_db)):
     return res
 
 
+def ask_answers_celery(task_id, file_list, db):
+    res = celery_app.AsyncResult(task_id)
+    while res.state == "PENDING":
+        pass
+    try:
+        final_res = res.get(timeout=2)
+        for res, file in zip(final_res, file_list):
+            db_file = files.get_file(db=db, file_id=file.id)
+            db_file.prediction_deepfaune = res
+            db.commit()
+    except:
+        print("failed")
+
+
 @router.post("/upload/{deployment_id}")
-def upload_file(deployment_id: int, file: UploadFile = File(...), db: Session = Depends(get_db)):
+def upload_file(
+    deployment_id: int,
+    background_tasks: BackgroundTasks,
+    file: UploadFile = File(...),
+    db: Session = Depends(get_db),
+):
     hash = dependencies.generate_checksum(file)
 
     mime = magic.from_buffer(file.file.read(), mime=True)
@@ -97,6 +121,11 @@ def upload_file(deployment_id: int, file: UploadFile = File(...), db: Session =
         ext=mime,
         deployment_id=deployment_id,
     )
+
+    ext = mime.split("/")[1]
+    url = s3.get_url_server(f"{hash}.{ext}")
+    task = celery_app.send_task("deepfaune.pi", [[url]])
+    background_tasks.add_task(ask_answers_celery, task.get(), [insert], db)
     return insert
 
 
@@ -148,23 +177,46 @@ def download_file(id: str, db: Session = Depends(get_db)):
 
 @router.post("/upload_zip/{deployment_id}")
 def upload_zip(
+    background_tasks: BackgroundTasks,
     deployment_id: int,
-    hash: List[str] = Form(),
     zipFile: UploadFile = File(...),
     db: Session = Depends(get_db),
 ):
-    listHash = hash[0].split(",")
     ext = zipFile.filename.split(".")[1]
     if ext == "zip":
         with ZipFile(io.BytesIO(zipFile.file.read()), "r") as myzip:
             res = []
-            for info, hash in zip(myzip.infolist(), listHash):
+            names = []
+            for info in myzip.infolist():
                 bytes = myzip.read(info.filename)
                 with tempfile.SpooledTemporaryFile() as tf:
                     tf.write(bytes)
                     tf.seek(0)
-                    insert = files.upload_file(db, hash, tf, info.filename, "JPG", deployment_id)
-                    res.append(insert)
+
+                    hash = dependencies.generate_checksum_content(bytes)
+
+                    mime = magic.from_buffer(tf.read(), mime=True)
+                    tf.seek(0)
+
+                    if not check_mime(mime):
+                        raise HTTPException(status_code=400, detail="Invalid type file")
+                    insert = files.upload_file(
+                        db=db,
+                        hash=hash,
+                        new_file=tf,
+                        filename=info.filename,
+                        ext=mime,
+                        deployment_id=deployment_id,
+                    )
+                    res.append(deepcopy(insert))
+                    ext = mime.split("/")[1]
+                    names.append(f"{hash}.{ext}")
+            urls = []
+            for name in names:
+                url = s3.get_url_server(name)
+                urls.append(url)
+            task = celery_app.send_task("deepfaune.pi", [urls])
+            background_tasks.add_task(ask_answers_celery, task.get(), res, db)
             return res
     else:
         raise HTTPException(status_code=500, detail="Vous ne pouvez déposer que des fichiers.zip")
@@ -176,7 +228,8 @@ def read_deployment_files(deployment_id: int, db: Session = Depends(get_db)):
     res = []
     for f in List_files:
         new_f = f.dict()
-        url = s3.get_url(f"{f.hash}.{f.extension}")
+        ext = f.extension.split("/")[1]
+        url = s3.get_url_client(f"{f.hash}.{ext}")
         new_f["url"] = url
         res.append(new_f)
     return res

api/src/services/dependencies.py

@@ -21,4 +21,7 @@ def read_upload(upload: UploadFile) -> bytes:
 
 def generate_checksum(upload: UploadFile) -> str:
     contents = read_upload(upload)
-    return hashlib.md5(contents).hexdigest()
+    return generate_checksum_content(contents)
+
+def generate_checksum_content(content: bytes) -> str:
+    return hashlib.md5(content).hexdigest()

api/src/services/files.py

@@ -103,7 +103,8 @@ def upload_file(
     deployment_id: int,
 ):
     try:
-        s3.upload_file_obj(new_file, f"{hash}.{ext}")
+        extension = ext.split("/")[1]
+        s3.upload_file_obj(new_file, f"{hash}.{extension}")
     except Exception as e:
         print(e)
         raise HTTPException(status_code=404, detail="Impossible to save the file in minio")

deepfaune/.bash_history

@@ -0,0 +1,21 @@
+ls
+du -hs /.venv/
+ls
+cd src/
+ls
+cd deepfaune/
+ls
+ls -ltrh
+ll
+ls
+cat app.log 
+ls -al 
+ls
+python -m pipenv install celerylogger
+exit
+pipen install celerylogger
+pipenv install celerylogger
+python
+pipenv install celerylogger
+pip3 install celerylogger
+exit

deepfaune/.dockerignore

@@ -0,0 +1,4 @@
+.vscode
+.pytest_cache
+.vscode-server/
+**/__pycache__