Fix dev (#229)

* uat release okt

vespadb/observations/admin.py

@@ -217,7 +217,7 @@ def get_urls(self) -> Any:
         custom_urls = [
             path("import-file/", self.admin_site.admin_view(self.import_file), name="import_file"),
             path("bulk-import/", self.admin_site.admin_view(self.bulk_import_view), name="bulk_import"),
-            path("send-email/", self.admin_site.admin_view(self.send_email_to_observers), name="send-email"),
+            path("send-email/", self.admin_site.admin_view(self.send_email_view), name="send-email"),  # Correcte URL naam
         ]
         return custom_urls + urls
 
@@ -279,25 +279,19 @@ def bulk_import_view(self, request: HttpRequest) -> HttpResponse:
         else:
             self.message_user(request, f"Failed to import observations: {response.data}", level="error")
         return redirect("admin:observations_observation_changelist")
-
-    @admin.action(description="Verzend emails naar observatoren")
-    def send_email_to_observers(
-        self, request: HttpRequest, queryset: QuerySet[Observation]
-    ) -> TemplateResponse | HttpResponse:
-        """
-        Send emails to the observers of the selected observations.
-
-        :param request: HttpRequest object
-        :param queryset: QuerySet of selected observations
-        :return: TemplateResponse or HttpResponse object
-        """
-        if "apply" in request.POST:
+
+    def send_email_view(self, request: HttpRequest) -> HttpResponse:
+        """View to display the form and send emails."""
+        if request.method == 'POST':
             form = SendEmailForm(request.POST)
             if form.is_valid():
                 subject = form.cleaned_data["subject"]
                 message = form.cleaned_data["message"]
                 resend = form.cleaned_data["resend"]
 
+                selected_observations = request.session.get('selected_observations', [])
+                queryset = Observation.objects.filter(pk__in=selected_observations)
+
                 success_list = []
                 fail_list = []
 
@@ -312,7 +306,7 @@ def send_email_to_observers(
                         continue
                     try:
                         send_mail(subject, message, "[email protected]", [observation.observer_email])
-                        logger.info(f"Email sent to {observation.observer_email} for observation {observation.id}")
+                        logger.debug(f"Email sent to {observation.observer_email} for observation {observation.id}")
                         observation.observer_received_email = True
                         observation.save()
                         success_list.append(observation.id)
@@ -325,19 +319,22 @@ def send_email_to_observers(
                 if fail_list:
                     messages.warning(request, f"Failed to send emails for {len(fail_list)} observations.")
 
-                return TemplateResponse(
-                    request,
-                    "admin/send_email_result.html",
-                    {
-                        "success_list": success_list,
-                        "fail_list": fail_list,
-                    },
-                )
-
+                return redirect('admin:observations_observation_changelist')
         else:
             form = SendEmailForm()
 
-        return render(request, "admin/send_email.html", {"observations": queryset, "form": form})
+        return render(request, 'admin/send_email.html', {'form': form})
+
+    @admin.action(description="Verzend emails naar observatoren")
+    def send_email_to_observers(self, request: HttpRequest, queryset: QuerySet[Observation]) -> HttpResponse:
+        """
+        Action to prepare sending emails.
+        """
+        # Sla de geselecteerde observaties op in de sessie
+        request.session['selected_observations'] = list(queryset.values_list('pk', flat=True))
+
+        # Redirect naar de juiste naam van de custom URL
+        return redirect('admin:send-email')
 
     @admin.action(description="Markeer observatie(s) als bestreden")
     def mark_as_eradicated(self, request: HttpRequest, queryset: Any) -> None:

vespadb/observations/migrations/0029_alter_observation_observer_phone_number.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-10-22 11:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('observations', '0028_alter_observation_eradication_duration'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='observation',
+            name='observer_phone_number',
+            field=models.CharField(blank=True, help_text='Phone number of the observer', max_length=50, null=True),
+        ),
+    ]

vespadb/observations/migrations/0030_alter_observation_observer_phone_number.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.0.6 on 2024-10-22 11:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('observations', '0029_alter_observation_observer_phone_number'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='observation',
+            name='observer_phone_number',
+            field=models.CharField(blank=True, help_text='Phone number of the observer', max_length=200, null=True),
+        ),
+    ]

vespadb/observations/models.py

@@ -247,7 +247,7 @@ class Observation(models.Model):
     )
 
     observer_phone_number = models.CharField(
-        max_length=20, blank=True, null=True, help_text="Phone number of the observer"
+        max_length=200, blank=True, null=True, help_text="Phone number of the observer"
     )
     observer_email = models.EmailField(blank=True, null=True, help_text="Email of the observer")
     observer_received_email = models.BooleanField(default=False, help_text="Flag indicating if observer received email")

vespadb/observations/tasks/observation_sync.py

@@ -1,5 +1,4 @@
 """Fetch and update observations from waarnemingen API."""
-
 import logging
 import os
 from datetime import UTC, datetime, timedelta
@@ -87,21 +86,26 @@ def create_observations(observations_to_create: list[Observation]) -> None:
         return
 
     try:
-        Observation.objects.bulk_create(observations_to_create, batch_size=BATCH_SIZE, ignore_conflicts=True)
-        logger.info("Successfully created %s new observations", len(observations_to_create))
+        with transaction.atomic():  # Bulk operations in a separate atomic block
+            Observation.objects.bulk_create(observations_to_create, batch_size=BATCH_SIZE, ignore_conflicts=True)
+            logger.info("Successfully created %s new observations", len(observations_to_create))
     except Exception as bulk_error:
         logger.exception("Bulk creation failed: %s", bulk_error)
+
         successful_creations = 0
+        # Handle individual saves OUTSIDE of any transaction block
         for observation in observations_to_create:
             try:
-                observation.save()
+                observation.save()  # Save each observation individually
                 successful_creations += 1
-            except DatabaseError as e:
-                logger.exception("Failed to create observation with wn_id %s individually: %s", observation.wn_id, e)
-        logger.info("Individually created %s observations after bulk failure", successful_creations)
-
+            except Exception as e:
+                # Log error and the input data without stopping the sync
+                logger.error(f"Failed to create observation with wn_id {observation.wn_id}: {e}")
+                logger.error(f"Input data for failed observation: {observation.__dict__}")
 
-def update_observations(observations_to_update: list[Observation], wn_ids_to_update: list[str]) -> None:  # noqa: C901
+        logger.info("Individually created %s observations after bulk failure", successful_creations)
+
+def update_observations(observations_to_update: list[Observation], wn_ids_to_update: list[str]) -> None:
     """Update existing observations with bulk update, and fallback to individual updates on failure."""
     observations_update_dict = {obs.wn_id: obs for obs in observations_to_update}
     existing_observations_to_update = Observation.objects.filter(wn_id__in=wn_ids_to_update)
@@ -138,20 +142,20 @@ def needs_update(observation: Observation, updated_observation: Observation) ->
             logger.info("Attempting to bulk update %s observations", len(observations_to_bulk_update))
             Observation.objects.bulk_update(observations_to_bulk_update, FIELDS_TO_UPDATE, batch_size=BATCH_SIZE)
             logger.info("Successfully bulk updated %s observations", len(observations_to_bulk_update))
-        except (DatabaseError, ValueError) as bulk_error:
+        except Exception as bulk_error:
             logger.exception("Bulk update failed: %s", bulk_error)
             logger.info("Falling back to individual updates")
-            successful_updates = 0
 
+            successful_updates = 0
             # Fall back to individual updates in case of failure
             for observation in observations_to_bulk_update:
                 try:
                     observation.save(update_fields=FIELDS_TO_UPDATE)
                     successful_updates += 1
-                except DatabaseError as e:
-                    logger.exception(
-                        "Failed to update observation with wn_id %s individually: %s", observation.wn_id, e
-                    )
+                except Exception as e:
+                    # Log error and the input data without stopping the sync
+                    logger.error(f"Failed to update observation with wn_id {observation.wn_id}: {e}")
+                    logger.error(f"Input data for failed observation: {observation.__dict__}")
 
             logger.info("Individually updated %s observations after bulk failure", successful_updates)
     else:

vespadb/observations/views.py

@@ -650,7 +650,7 @@ def export(self, request: Request) -> Response:
                 # Try serializing the entire page first
                 serializer = serializer_class(page, many=True, context=serializer_context)
                 serialized_data.extend(serializer.data)
-            except ValidationError:
+            except Exception as e:
                 logger.exception(f"Validation error in page {page_number}, processing individually.")
                 for obj in page.object_list:
                     try:
@@ -671,6 +671,11 @@ def export(self, request: Request) -> Response:
                             "id": obj.id,
                             "error": "Database connection error",
                         })
+                    except Exception as e:
+                        errors.append({
+                            "id": obj.id,
+                            "error": e,
+                        })
 
         if errors:
             logger.error(f"Errors during export: {errors}")

vespadb/templates/admin/send_email.html

@@ -1,4 +1,7 @@
 {% extends "admin/base_site.html" %}
+{% block content_subtitle %}
+{% endblock %}
+
 {% block content %}
 <h2>Send Email to Observers</h2>
 <form method="post">