Improvements: metadatas and delete procedure (#383)

* Added new metadatas: process_status and origin.
Improved way to delete products, keeping the record in the database and changing only the status to 'deleted'.
Synchronized with the new version of orchestration app.

* Fixed bug in product tests

backend/core/migrations/0042_alter_product_status.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.1.1 on 2025-01-23 17:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0041_productfile_n_rows"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="product",
+            name="status",
+            field=models.IntegerField(
+                choices=[
+                    (0, "Registering"),
+                    (1, "Published"),
+                    (2, "Deleted"),
+                    (9, "Failed"),
+                ],
+                default=0,
+                verbose_name="Status",
+            ),
+        ),
+    ]

backend/core/migrations/0043_alter_productfile_file.py

@@ -0,0 +1,22 @@
+# Generated by Django 5.1.1 on 2025-01-23 18:51
+
+import core.models.product_file
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0042_alter_product_status"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="productfile",
+            name="file",
+            field=models.FileField(
+                blank=True,
+                null=True,
+                upload_to=core.models.product_file.upload_product_files,
+            ),
+        ),
+    ]

backend/core/migrations/0044_alter_product_status.py

@@ -0,0 +1,27 @@
+# Generated by Django 5.1.1 on 2025-01-23 20:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("core", "0043_alter_productfile_file"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="product",
+            name="status",
+            field=models.IntegerField(
+                choices=[
+                    (0, "Registering"),
+                    (1, "Published"),
+                    (2, "Deleted"),
+                    (3, "Processing"),
+                    (9, "Failed"),
+                ],
+                default=0,
+                verbose_name="Status",
+            ),
+        ),
+    ]

backend/core/models/product.py

@@ -1,4 +1,3 @@
-import logging
 import pathlib
 import shutil
 
@@ -7,13 +6,12 @@
 from django.contrib.auth.models import User
 from django.db import models
 
-# def upload_product_files(instance, filename):
-#     return f"{instance.product_type.name}/{instance.internal_name}/{filename}"
-
 
 class ProductStatus(models.IntegerChoices):
     REGISTERING = 0, "Registering"
     PUBLISHED = 1, "Published"
+    DELETED = 2, "Deleted"
+    PROCESSING = 3, "Processing"
     FAILED = 9, "Failed"
 
 
@@ -60,12 +58,13 @@ def delete(self, *args, **kwargs):
         product_path = pathlib.Path(settings.MEDIA_ROOT, self.path)
         if product_path.exists():
             # TODO: mover esta exception para uma funcao separada para que possa ser executado o test.
-            # try:
-            shutil.rmtree(product_path)
-            # except OSError as e:
-            # raise OSError("Failed to remove directory: [ %s ] %s" % (product_path, e))
-
-        super().delete(*args, **kwargs)
+            try:
+                shutil.rmtree(product_path)
+            except OSError as e:
+                raise OSError("Failed to remove directory: [ %s ] %s" % (product_path, e))
+
+            self.status = ProductStatus.DELETED
+            self.save()
 
     def can_delete(self, user) -> bool:
         if self.user.id == user.id or user.profile.is_admin():

backend/core/models/product_file.py

@@ -18,7 +18,7 @@ class ProductFile(models.Model):
 
     product = models.ForeignKey(Product, on_delete=models.CASCADE, related_name="files")
 
-    file = models.FileField(upload_to=upload_product_files)
+    file = models.FileField(upload_to=upload_product_files, null=True, blank=True)
 
     role = models.IntegerField(
         verbose_name="Role",
@@ -40,8 +40,3 @@ class ProductFile(models.Model):
 
     def __str__(self):
         return f"{self.product.display_name} - {os.path.basename(self.file.name)}"
-
-    def delete(self, *args, **kwargs):
-        if self.file:
-            self.file.delete()
-        super().delete(*args, **kwargs)

backend/core/product_handle.py

@@ -21,10 +21,10 @@ def df_from_file(self, filepath: PathLike, **kwargs) -> pd.DataFrame:
         descritos aqui: https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
 
         Args:
-            filepath (PathLike): _description_
+            filepath (PathLike): product filepath
 
         Returns:
-            pd.DataFrame: _description_
+            pd.DataFrame
         """
         return FileHandle(filepath).to_df(**kwargs)
 

backend/core/serializers/product.py

@@ -20,13 +20,17 @@ class ProductSerializer(serializers.ModelSerializer):
 
     is_owner = serializers.SerializerMethodField()
 
+    origin = serializers.SerializerMethodField()
+
+    process_status = serializers.SerializerMethodField()
+
     can_delete = serializers.SerializerMethodField()
 
     can_update = serializers.SerializerMethodField()
 
     class Meta:
         model = Product
-        read_only_fields = ("internal_name", "is_owner")
+        read_only_fields = ("internal_name", "is_owner", "origin", "process_status")
         exclude = ["user", "path"]
 
     def get_product_type_name(self, obj):
@@ -48,6 +52,12 @@ def get_is_owner(self, obj):
         else:
             return False
 
+    def get_origin(self, obj):
+        return obj.upload.pipeline.display_name if hasattr(obj, 'upload') else "Upload"
+
+    def get_process_status(self, obj):
+        return obj.upload.status if hasattr(obj, 'upload') else None
+
     def get_can_delete(self, obj):
         current_user = self.context["request"].user
         return obj.can_delete(current_user)

backend/core/tasks.py

@@ -23,7 +23,7 @@ def check_processes():
         bool: True, if an update was made. False, if no update was made.
     """
 
-    monitoring_statuses = ["Stopping", "Pending", "Running"]
+    monitoring_statuses = ["Stopping", "Pending", "Running", "Queued"]
     logger.info(f"Monitoring the following statuses: {monitoring_statuses}")
 
     processes = Process.objects.filter(status__in=monitoring_statuses)

backend/core/test/test_product.py

@@ -288,6 +288,8 @@ def test_product_serialized_format(self):
             "product_type_name": self.product_type.display_name,
             "uploaded_by": self.user.username,
             "is_owner": True,
+            "origin": "Upload",
+            "process_status": None,
             "can_delete": True,
             "can_update": True,
             "internal_name": self.product.internal_name,

backend/core/views/process.py

@@ -3,7 +3,7 @@
 import pathlib
 
 from core.maestro import Maestro
-from core.models import Pipeline, Process
+from core.models import Pipeline, Process, ProductStatus
 from core.product_steps import CreateProduct
 from core.serializers import ProcessSerializer
 from core.utils import format_query_to_char
@@ -187,7 +187,8 @@ def perform_create(self, serializer):
         return serializer.save(user=owned_by, upload=upload)
 
     def create_initial_upload(self, serializer, user):
-        """_summary_"""
+        """ Creates the upload with initial data before starting processing. """
+
         data = serializer.initial_data
         pipeline = Pipeline.objects.get(pk=data.get("pipeline"))
         upload_data = {
@@ -204,6 +205,7 @@ def create_initial_upload(self, serializer, user):
         if not check_prodtype.get("success"):
             raise ValueError(check_prodtype.get("message"))
 
+        product.status = ProductStatus.PROCESSING  # type: ignore
         product.save()
         return product.data