diff --git a/src/components/MapPage.vue b/src/components/MapPage.vue index 249edd6..5d76b27 100644 --- a/src/components/MapPage.vue +++ b/src/components/MapPage.vue @@ -123,7 +123,6 @@ export default { try { await vespaStore.fetchObservationDetails(properties.id); if (vespaStore.selectedObservation && !vespaStore.selectedObservation.visible) { - console.error("Observation is not visible"); return; } vespaStore.isDetailsPaneOpen = true; diff --git a/src/components/ObservationDetailsComponent.vue b/src/components/ObservationDetailsComponent.vue index 340f2a1..3e8ae53 100644 --- a/src/components/ObservationDetailsComponent.vue +++ b/src/components/ObservationDetailsComponent.vue @@ -29,7 +29,7 @@ class="btn btn-sm btn-outline-danger" @click="cancelReservation">Reservatie annuleren -
{{ selectedObservation.wn_admin_notes }}
-{{ selectedObservation.wn_validation_status }} -
+Geen
+{{ + label }}
{{ selectedObservation.wn_notes }}
+{{ selectedObservation.notes }}
+
Aangemaakt op {{ selectedObservation.created_datetime ? formatDate(selectedObservation.created_datetime) : '' }} door {{ selectedObservation.created_by_first_name || '' }}, gewijzigd op { - editableObservation.value = JSON.parse(JSON.stringify(selectedObservation.value || {})); - }; - const isUserReserver = computed(() => { return selectedObservation.value?.reserved_by === vespaStore.user.id; }); @@ -538,6 +527,10 @@ export default { return 'Niet bestreden'; } }); + const canViewRestrictedFields = computed(() => { + return vespaStore.isAdmin || + (isLoggedIn.value && vespaStore.userMunicipalities.includes(selectedObservation.value?.municipality_name)); + }); const eradicationStatusClass = computed(() => { const result = selectedObservation.value?.eradication_result; @@ -547,7 +540,10 @@ export default { return 'bg-danger'; } }); - + const validationStatusEnum = { + "goedgekeurd_met_bewijs": "Goedgekeurd met bewijs", + "goedgekeurd_door_admin": "Goedgekeurd door admin" + }; const formatDate = (isoString, defaultValue = "") => { if (!isoString) { return defaultValue; @@ -620,7 +616,6 @@ export default { const closeDetails = () => { emit('closeDetails'); vespaStore.isDetailsPaneOpen = false; - resetEditableObservation(); errorMessage.value = ''; eradicationResultError.value = ''; }; @@ -634,8 +629,12 @@ export default { } }; + const isUpdating = ref(false); const confirmUpdate = async () => { + if (isUpdating.value) return; + try { + isUpdating.value = true; // Check if any eradication fields are filled const eradicationFields = [ 'eradication_date', 'eradicator_name', 'eradication_duration', @@ -643,23 +642,45 @@ export default { 'eradication_problems', 'eradication_notes', 'eradication_product' ]; const hasEradicationData = eradicationFields.some(field => editableObservation.value[field]); + + if ( + editableObservation.value.eradication_result && + !editableObservation.value.eradication_date + ) { + const today = new Date(); + editableObservation.value.eradication_date = today.toISOString().split('T')[0]; + } + + // Format eradication_date if provided, and check for valid eradication result if (editableObservation.value.eradication_date) { - editableObservation.value.eradication_date += "T00:00:00"; + const date = new Date(editableObservation.value.eradication_date); + if (!isNaN(date.getTime())) { + editableObservation.value.eradication_date = date.toISOString().split('T')[0]; + } else { + throw new Error("Invalid eradication date format"); + } } + if (hasEradicationData && !editableObservation.value.eradication_result) { eradicationResultError.value = 'Resultaat is verplicht wanneer andere bestrijdingsgegevens zijn ingevuld.'; throw new Error('Validation failed'); } + + // Reset error messages errorMessage.value = ''; eradicationResultError.value = ''; + // Send updated observation to the store await vespaStore.updateObservation(editableObservation.value); - resetEditableObservation(); } catch (error) { - if (error.message !== 'Validation failed') { + if (error.message === "Invalid eradication date format") { + errorMessage.value = 'De ingevoerde datum is ongeldig.'; + } else if (error.message !== 'Validation failed') { errorMessage.value = 'Er is een fout opgetreden bij het opslaan van de wijzigingen.'; } console.error('Error updating observation:', error); + }finally { + isUpdating.value = false; } }; const cancelEdit = () => { @@ -672,13 +693,8 @@ export default { }; const canViewContactInfo = computed(() => { - if (vespaStore.isAdmin) { - return true; - } - if (!vespaStore.user.personal_data_access) { - return false; - } - const userMunicipalities = vespaStore.user.municipalities; + if (vespaStore.isAdmin) return true; + const userMunicipalities = vespaStore.userMunicipalities; const observationMunicipality = selectedObservation.value?.municipality_name; return userMunicipalities.includes(observationMunicipality); }); @@ -702,15 +718,18 @@ export default { successMessage.value = ''; }; - watch(() => vespaStore.selectedObservation, (newVal) => { - if (newVal) { - editableObservation.value = { ...newVal }; - editableObservation.value.observation_datetime = formatToDatetimeLocal(selectedObservation.value.observation_datetime); - editableObservation.value.eradication_date = formatToDate(selectedObservation.value.eradication_date); - //editableObservation.value.eradication_date = formatToDate(newVal.eradication_date); + watch(() => vespaStore.selectedObservation, (newVal, oldVal) => { + if (!newVal || newVal.id === oldVal?.id) { + return; } + editableObservation.value = { ...newVal }; + editableObservation.value.observation_datetime = formatToDatetimeLocal(newVal.observation_datetime); + editableObservation.value.eradication_date = newVal.eradication_date + ? formatToDate(newVal.eradication_date) + : null; }, { immediate: true }); - watch(selectedObservation, resetEditableObservation, { immediate: true }); + + watch(selectedObservation, { immediate: true }); return { selectedObservation, @@ -750,6 +769,7 @@ export default { editableObservation, errorMessage, eradicationResultError, + canViewRestrictedFields }; } }; diff --git a/src/components/TableViewPage.vue b/src/components/TableViewPage.vue index f601592..8c96d70 100644 --- a/src/components/TableViewPage.vue +++ b/src/components/TableViewPage.vue @@ -182,15 +182,12 @@ export default { // if (vespaStore.lastAppliedFilters === null || vespaStore.lastAppliedFilters === 'null') { - // console.log("set hier?") // vespaStore.setLastAppliedFilters(); // } // Avoid calling getObservations if data is already loaded with the same filters if (vespaStore.table_observations.length === 0 || JSON.stringify(vespaStore.filters) !== JSON.stringify(vespaStore.lastAppliedFilters)) { - - //vespaStore.setLastAppliedFilters(); vespaStore.getObservations(page.value, pageSize.value, sortBy.value, sortOrder.value); vespaStore.getObservationsGeoJson(); diff --git a/src/stores/vespaStore.js b/src/stores/vespaStore.js index 77edc3f..644c020 100644 --- a/src/stores/vespaStore.js +++ b/src/stores/vespaStore.js @@ -284,7 +284,6 @@ export const useVespaStore = defineStore('vespaStore', { } } catch (error) { console.error('Error fetching observation details:', error); - this.error = 'Het ophalen van observatiedetails is mislukt.'; } }, formatToISO8601(datetime) { @@ -293,17 +292,10 @@ export const useVespaStore = defineStore('vespaStore', { return date.toISOString(); }, async updateObservation(observation) { - if (observation.observation_datetime) { - observation.observation_datetime = this.formatToISO8601(observation.observation_datetime); - } - if (observation.eradication_date) { - observation.eradication_date = this.formatDateWithEndOfDayTime(observation.eradication_date); - } try { const response = await ApiService.patch(`/observations/${observation.id}/`, observation); if (response.status === 200) { this.selectedObservation = response.data; - const colorByResult = this.getColorByStatus(response.data.eradication_result); this.updateMarkerColor(observation.id, colorByResult, '#ea792a', 4, 'active-marker'); return response.data; diff --git a/vespadb/observations/admin.py b/vespadb/observations/admin.py index 529665f..68da970 100644 --- a/vespadb/observations/admin.py +++ b/vespadb/observations/admin.py @@ -147,7 +147,7 @@ class ObservationAdmin(gis_admin.GISModelAdmin): actions = ["send_email_to_observers", "mark_as_eradicated", "mark_as_not_visible"] readonly_fields = ( - "wn_notes", + "notes", "source", "wn_id", "wn_validation_status", diff --git a/vespadb/observations/migrations/0031_observation_source_id.py b/vespadb/observations/migrations/0031_observation_source_id.py new file mode 100644 index 0000000..2cbbcef --- /dev/null +++ b/vespadb/observations/migrations/0031_observation_source_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.2 on 2024-11-11 18:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('observations', '0030_alter_observation_observer_phone_number'), + ] + + operations = [ + migrations.AddField( + model_name='observation', + name='source_id', + field=models.IntegerField(blank=True, help_text='Original identifier when importing data', null=True), + ), + ] diff --git a/vespadb/observations/migrations/0032_rename_wn_notes_observation_notes.py b/vespadb/observations/migrations/0032_rename_wn_notes_observation_notes.py new file mode 100644 index 0000000..8aaa7ad --- /dev/null +++ b/vespadb/observations/migrations/0032_rename_wn_notes_observation_notes.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.2 on 2024-11-11 18:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('observations', '0031_observation_source_id'), + ] + + operations = [ + migrations.RenameField( + model_name='observation', + old_name='wn_notes', + new_name='notes', + ), + ] diff --git a/vespadb/observations/models.py b/vespadb/observations/models.py index 3a1233e..a6c9684 100644 --- a/vespadb/observations/models.py +++ b/vespadb/observations/models.py @@ -221,8 +221,9 @@ class Observation(models.Model): modified_datetime = models.DateTimeField(auto_now=True, help_text="Datetime when the observation was last modified") location = gis_models.PointField(help_text="Geographical location of the observation") source = models.CharField(max_length=255, blank=True, null=True, help_text="Source of the observation") + source_id = models.IntegerField(blank=True, null=True, help_text="Original identifier when importing data") - wn_notes = models.TextField(blank=True, null=True, help_text="Notes about the observation") + notes = models.TextField(blank=True, null=True, help_text="Notes about the observation") wn_admin_notes = models.TextField(blank=True, null=True, help_text="Admin notes about the observation") wn_validation_status = models.CharField( max_length=50, diff --git a/vespadb/observations/serializers.py b/vespadb/observations/serializers.py index 5329251..10cf878 100644 --- a/vespadb/observations/serializers.py +++ b/vespadb/observations/serializers.py @@ -36,7 +36,6 @@ "observation_datetime", "modified_by", "created_by", - "province", "eradication_date", "municipality", "province", @@ -45,7 +44,7 @@ "municipality_name", "modified_by_first_name", "created_by_first_name", - "wn_notes", + "notes", "eradication_result", "wn_id", "wn_validation_status", @@ -139,7 +138,7 @@ class Meta: "modified_datetime": {"help_text": "Datetime when the observation was last modified."}, "location": {"help_text": "Geographical location of the observation as a point."}, "source": {"help_text": "Source of the observation."}, - "wn_notes": {"help_text": "Notes about the observation."}, + "notes": {"help_text": "Notes about the observation."}, "wn_admin_notes": {"write_only": True}, "wn_validation_status": {"help_text": "Validation status of the observation."}, "nest_height": {"help_text": "Height of the nest."}, @@ -262,23 +261,24 @@ def to_representation(self, instance: Observation) -> dict[str, Any]: # noqa: C if request and request.user.is_authenticated: user: VespaUser = request.user + permission_level = user.get_permission_level() user_municipality_ids = user.municipalities.values_list("id", flat=True) + is_inside_user_municipality = ( + instance.municipality and instance.municipality.id in user_municipality_ids + ) + + # Voor gebruikers zonder toegang tot specifieke gemeenten + if permission_level == "logged_in_without_municipality": + return {field: data[field] for field in public_read_fields if field in data} - if not request.user.is_superuser: - # Non-admins should not see admin-specific fields - admin_fields = set(admin_or_special_permission_fields) - for field in admin_fields: - data.pop(field, None) - - is_inside_user = instance.municipality and instance.municipality.id in user_municipality_ids - if not is_inside_user: - # Do not show reserved_by for users outside the municipality and not admins - data.pop("reserved_by", None) - return { - field: data[field] - for field in set(user_read_fields + conditional_fields + admin_or_special_permission_fields) - if field in data - } + # Voor gebruikers met toegang tot specifieke gemeenten, extra gegevens tonen indien binnen hun gemeenten + if is_inside_user_municipality or request.user.is_superuser: + return {field: data[field] for field in user_read_fields if field in data} + + # Voor observaties buiten de gemeenten van de gebruiker, beperk tot publieke velden + return {field: data[field] for field in public_read_fields if field in data} + + # Voor niet-ingelogde gebruikers, retourneer enkel de publieke velden return {field: data[field] for field in public_read_fields if field in data} def validate_reserved_by(self, value: VespaUser) -> VespaUser: @@ -360,7 +360,7 @@ def update(self, instance: Observation, validated_data: dict[Any, Any]) -> Obser if eradication_result == EradicationResultEnum.SUCCESSFUL: validated_data["reserved_datetime"] = None validated_data["reserved_by"] = None - validated_data["eradication_date"] = datetime.now(timezone("EST")).date() + #validated_data["eradication_date"] = datetime.now(timezone("EST")).date() if not user.is_superuser: # Non-admins cannot update admin-specific fields, so remove them diff --git a/vespadb/observations/tasks/observation_mapper.py b/vespadb/observations/tasks/observation_mapper.py index 0cb4606..d815c6f 100644 --- a/vespadb/observations/tasks/observation_mapper.py +++ b/vespadb/observations/tasks/observation_mapper.py @@ -26,42 +26,75 @@ logger = logging.getLogger("vespadb.observations.tasks") +mapping_dict: dict[int, dict[str, str]] = { + 329: { + "Hoger dan 4 meter": "hoger_dan_4_meter", + "Higher than 4 meters": "hoger_dan_4_meter", + "Lager dan 4 meter": "lager_dan_4_meter", + "Lower than 4 meters": "lager_dan_4_meter", + }, + 330: { + "Groter dan 25 cm": "groter_dan_25_cm", + "Kleiner dan 25 cm": "kleiner_dan_25_cm", + "Larger than 25cm": "groter_dan_25_cm", + "Smaller than 25cm": "kleiner_dan_25_cm", + }, + 331 : { + "Binnen, in gebouw of constructie": "binnen_in_gebouw_of_constructie", + "Buiten, maar overdekt door constructie": "buiten_maar_overdekt_door_constructie", + "Buiten, natuurlijk overdekt": "buiten_natuurlijk_overdekt", + "Buiten, onbedekt in boom of struik": "buiten_onbedekt_in_boom_of_struik", + "Buiten, onbedekt op gebouw": "buiten_onbedekt_op_gebouw", + "Inside, in a building or construction": "binnen_in_gebouw_of_constructie", + "Outside, but covered by construction": "buiten_maar_overdekt_door_constructie", + "Outside, natural cover": "buiten_natuurlijk_overdekt", + "Outside, uncovered in a tree or bush": "buiten_onbedekt_in_boom_of_struik", + "Outside, uncovered on building": "buiten_onbedekt_op_gebouw", + } +} + ENUMS_MAPPING: dict[str, type[TextChoices]] = { - "Nesthoogte": NestHeightEnum, - "Nestgrootte": NestSizeEnum, - "Nestplaats": NestLocationEnum, - "Nesttype": NestTypeEnum, - "Resultaat": EradicationResultEnum, - "Problemen": EradicationProblemsEnum, - "Methode": EradicationMethodEnum, + "Nest height": NestHeightEnum, + "Nest size": NestSizeEnum, + "Nest location": NestLocationEnum, + "Nest type": NestTypeEnum, + "Result": EradicationResultEnum, + "Problems": EradicationProblemsEnum, + "Method": EradicationMethodEnum, "Product": EradicationProductEnum, } -ENUM_FIELD_MAPPING: dict[str, str] = { - "Nesthoogte": "nest_height", - "Nestgrootte": "nest_size", - "Nestplaats": "nest_location", - "Nesttype": "nest_type", - "Resultaat": "eradication_result", - "Problemen": "eradication_problems", - "Methode": "eradication_method", - "Product": "eradication_product", +ENUM_FIELD_MAPPING: dict[int, str] = { + 329: "nest_height", + 330: "nest_size", + 331: "nest_location", } +# Literal mapping functions +def map_nest_height_attribute_to_enum(value: str) -> Any | None: + """Maps Nest height values to enums based on literal mapping.""" + return mapping_dict[329].get(value.strip()) +def map_nest_size_attribute_to_enum(value: str) -> Any | None: + """Maps Nest size values to enums based on literal mapping.""" + return mapping_dict[330].get(value.strip()) -def map_attribute_to_enum(value: str, enum: type[TextChoices]) -> TextChoices | None: - """ - Map a single attribute value to an enum using close match. +def map_nest_location_attribute_to_enum(value: str) -> str | None: + """Maps Nest location values to enums based on literal mapping.""" + return mapping_dict[331].get(value.strip()) - :param value: The value from the API that needs to be mapped to an enum. - :param enum: The enum type that the value is expected to map to. - :return: The corresponding enum value if a match is found, otherwise None. +def map_attribute_to_enum(attribute_id: int, value: str) -> str | None: """ - enum_dict = {e.value: e for e in enum} - closest_match = get_close_matches(value, enum_dict.keys(), n=1, cutoff=0.6) - return enum_dict.get(closest_match[0]) if closest_match else None - + Maps a single attribute value to an enum using literal mapping functions. + """ + if attribute_id == 329: + return map_nest_height_attribute_to_enum(value) + elif attribute_id == 330: + return map_nest_size_attribute_to_enum(value) + elif attribute_id == 331: + return map_nest_location_attribute_to_enum(value) + else: + return None -def map_attributes_to_enums(api_attributes: list[dict[str, str]]) -> dict[str, TextChoices]: +def map_attributes_to_enums(api_attributes: list[dict[str, Any]]) -> dict[str, str]: """ Map API attributes to model enums based on configured mappings. @@ -70,17 +103,17 @@ def map_attributes_to_enums(api_attributes: list[dict[str, str]]) -> dict[str, T """ mapped_values = {} for attribute in api_attributes: + attribute_id = int(attribute.get("attribute", 0)) attr_name = attribute.get("name") value = str(attribute.get("value")) - if attr_name in ENUMS_MAPPING: - mapped_enum = map_attribute_to_enum(value, ENUMS_MAPPING[attr_name]) + if attribute_id in mapping_dict: + mapped_enum = map_attribute_to_enum(attribute_id, value) if mapped_enum: - mapped_values[ENUM_FIELD_MAPPING[attr_name]] = mapped_enum + mapped_values[ENUM_FIELD_MAPPING[attribute_id]] = mapped_enum else: - logger.warning(f"No enum match found for {attr_name}: {value}") + logger.debug(f"No enum match found for {attr_name}: {value}") return mapped_values - def map_validation_status_to_enum(validation_status: str) -> ValidationStatusEnum | None: """ Map a single validation status to an enum. @@ -165,6 +198,7 @@ def map_external_data_to_observation_model(external_data: dict[str, Any]) -> dic cluster_id = None if nest: cluster_id = nest.get("id") + mapped_data = { "wn_id": external_data["id"], "location": location, @@ -182,7 +216,6 @@ def map_external_data_to_observation_model(external_data: dict[str, Any]) -> dic **mapped_enums, } - # Additional user data user_data = external_data.get("user", {}) if user_data: mapped_data.update({ @@ -191,10 +224,8 @@ def map_external_data_to_observation_model(external_data: dict[str, Any]) -> dic "observer_name": user_data.get("name"), }) - # Eradication specifics eradication_flagged = False - # Check for eradication keywords in notes if ( "notes" in external_data and external_data["notes"] @@ -203,9 +234,8 @@ def map_external_data_to_observation_model(external_data: dict[str, Any]) -> dic ): eradication_flagged = True - # Check for "BESTREDEN" in 'Remark (Asian hornet)' attribute for attribute in external_data.get("attributes", []): - if attribute.get("name") == "Remark (Asian hornet)" and "BESTREDEN" in attribute.get("value", "").upper(): + if attribute.get("attribute") == 369 and "BESTREDEN" in attribute.get("value", "").upper(): eradication_flagged = True break @@ -215,7 +245,6 @@ def map_external_data_to_observation_model(external_data: dict[str, Any]) -> dic return mapped_data - def check_existing_eradication_date(wn_id: str) -> bool: """ Check if the eradication_date is already set for the given wn_id. diff --git a/vespadb/observations/views.py b/vespadb/observations/views.py index 789dcab..4b9fe4f 100644 --- a/vespadb/observations/views.py +++ b/vespadb/observations/views.py @@ -40,12 +40,12 @@ from rest_framework.serializers import BaseSerializer from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from rest_framework_gis.filters import DistanceToPointFilter +from vespadb.observations.serializers import user_read_fields, public_read_fields from vespadb.observations.cache import invalidate_geojson_cache, invalidate_observation_cache from vespadb.observations.filters import ObservationFilter from vespadb.observations.helpers import parse_and_convert_to_utc -from vespadb.observations.utils import retry_query -from vespadb.observations.models import Municipality, Observation, Province +from vespadb.observations.models import Municipality, Observation, Province, EradicationResultEnum from vespadb.observations.serializers import ( MunicipalitySerializer, ObservationSerializer, @@ -63,12 +63,10 @@ GET_REDIS_CACHE_EXPIRATION = 86400 # 1 day BATCH_SIZE = 150 CSV_HEADERS = [ - "id", "created_datetime", "modified_datetime", "location", "source", - "nest_height", "nest_size", "nest_location", "nest_type", - "observation_datetime", "modified_by", "created_by", "province", - "eradication_date", "municipality", "images", "public_domain", - "municipality_name", "modified_by_first_name", "created_by_first_name", - "wn_notes", "eradication_result", "wn_id", "wn_validation_status" + "id", "created_datetime", "modified_datetime", "latitude", "longitude", "source", "source_id", + "nest_height", "nest_size", "nest_location", "nest_type", "observation_datetime", + "province", "eradication_date", "municipality", "images", "anb_domain", + "notes", "eradication_result", "wn_id", "wn_validation_status", "nest_status" ] class ObservationsViewSet(ModelViewSet): # noqa: PLR0904 """ViewSet for the Observation model.""" @@ -118,7 +116,10 @@ def get_permissions(self) -> list[BasePermission]: List[BasePermission]: A list of permission instances that should be applied to the action. """ if self.action in {"create", "update", "partial_update"}: - permission_classes = [IsAuthenticated()] + if self.request.user.is_superuser: + permission_classes = [IsAdminUser()] + elif self.request.user.is_authenticated and self.request.user.get_permission_level() == "logged_in_with_municipality": + permission_classes = [IsAuthenticated()] elif self.action == "destroy": permission_classes = [IsAdminUser()] else: @@ -144,7 +145,7 @@ def get_queryset(self) -> QuerySet: ) ) return base_queryset - + def perform_update(self, serializer: BaseSerializer) -> None: """ Set modified_by to the current user and modified_datetime to the current UTC time upon updating an observation. @@ -155,18 +156,12 @@ def perform_update(self, serializer: BaseSerializer) -> None: The serializer containing the validated data. """ user = self.request.user - if not user.is_superuser and ( - "admin_notes" in self.request.data or "observer_received_email" in self.request.data - ): - raise PermissionDenied("You do not have permission to modify admin fields.") - - # Ensure user has permission to reserve in the specified municipality - if "reserved_by" in self.request.data: - observation = self.get_object() - if not user.is_superuser: - user_municipality_ids = user.municipalities.values_list("id", flat=True) - if observation.municipality and observation.municipality.id not in user_municipality_ids: - raise PermissionDenied("You do not have permission to reserve nests in this municipality.") + observation = self.get_object() + + if not user.is_superuser and "reserved_by" in self.request.data: + user_municipality_ids = user.municipalities.values_list("id", flat=True) + if observation.municipality and observation.municipality.id not in user_municipality_ids: + raise PermissionDenied("You do not have permission to reserve nests in this municipality.") instance = serializer.save(modified_by=user, modified_datetime=now()) invalidate_observation_cache(instance.id) @@ -521,6 +516,9 @@ def parse_csv(self, file: InMemoryUploadedFile) -> list[dict[str, Any]]: data = [] for row in reader: try: + if "source_id" in row: + row["source_id"] = int(row["source_id"]) if row["source_id"].isdigit() else None + logger.info(f"Original location data: {row['location']}") row["location"] = self.validate_location(row["location"]) logger.info(f"Parsed location: {row['location']}") @@ -627,7 +625,7 @@ def save_observations(self, valid_data: list[dict[str, Any]]) -> Response: return Response( {"error": f"An error occurred during bulk import: {e!s}"}, status=status.HTTP_500_INTERNAL_SERVER_ERROR ) - + @swagger_auto_schema( method="get", manual_parameters=[ @@ -636,7 +634,7 @@ def save_observations(self, valid_data: list[dict[str, Any]]) -> Response: in_=openapi.IN_QUERY, description="Format of the exported data", type=openapi.TYPE_STRING, - enum=["csv", "json"], + enum=["csv"], default="csv", ), ], @@ -645,74 +643,134 @@ def save_observations(self, valid_data: list[dict[str, Any]]) -> Response: @action(detail=False, methods=["get"], permission_classes=[AllowAny]) def export(self, request: HttpRequest) -> Union[StreamingHttpResponse, JsonResponse]: """ - Export observations data as CSV in a memory-efficient, streamable format. - - Handles large datasets by streaming data in chunks to avoid memory overload. - Only supports CSV export; JSON format is no longer available. + Export observations as CSV with dynamically controlled fields based on user permissions. + + Observations from municipalities the user has access to will display full details; + others will show limited fields as per public access. """ if request.query_params.get("export_format", "csv").lower() != "csv": return JsonResponse({"error": "Only CSV export is supported"}, status=400) - # Filter queryset - queryset = self.filter_queryset(self.get_queryset()) + # Determine user permissions + if request.user.is_authenticated: + user_municipality_ids = set(request.user.municipalities.values_list("id", flat=True)) + is_admin = request.user.is_superuser + else: + user_municipality_ids = set() + is_admin = False - # Define response with streaming CSV data + # Set CSV headers directly from CSV_HEADERS as a base + dynamic_csv_headers = CSV_HEADERS + + # Prepare response + queryset = self.filter_queryset(self.get_queryset()) response = StreamingHttpResponse( - self.generate_csv_rows(queryset), content_type="text/csv" + self.generate_csv_rows(queryset, dynamic_csv_headers, user_municipality_ids, is_admin), + content_type="text/csv" ) response["Content-Disposition"] = 'attachment; filename="observations_export.csv"' return response - - def generate_csv_rows(self, queryset: QuerySet) -> Generator[bytes, None, None]: - """ - Generator that yields rows of CSV data, handling large datasets efficiently. - - Converts each observation to a dictionary row, handling missing or misconfigured - data gracefully, and writes to CSV format on-the-fly to avoid memory overuse. - """ - # Yield CSV header row - yield self._csv_line(CSV_HEADERS) - # Iterate over queryset in chunks to avoid high memory usage - for obj in queryset.iterator(chunk_size=500): - row = self.serialize_observation(obj) - yield self._csv_line(row) + def generate_csv_rows( + self, queryset: QuerySet, headers: list[str], user_municipality_ids: set, is_admin: bool + ) -> Generator[bytes, None, None]: + """Generate CSV rows with headers and filtered data according to user permissions.""" + # Yield headers + yield self._csv_line(headers) + + for observation in queryset.iterator(chunk_size=500): + # Determine fields to include based on user permissions for each observation + if is_admin or (observation.municipality_id in user_municipality_ids): + # Full access for admins and assigned municipalities + allowed_fields = user_read_fields + else: + # Restricted access for other municipalities + allowed_fields = public_read_fields + # Add essential fields for export + allowed_fields.extend(["source_id", "latitude", "longitude", "anb_domain", "nest_status"]) - def serialize_observation(self, obj: Observation) -> list[str]: + # Serialize the observation with restricted fields as needed + row = self.serialize_observation(observation, headers, allowed_fields) + yield self._csv_line(row) + + def parse_location(self, srid_str: str) -> tuple[float, float]: """ - Serialize observation to a list of values in the same order as headers. - - Handles potential data misconfigurations, such as missing attributes or - inconsistent formats, to ensure robust data handling. + Parse SRID string to extract latitude and longitude. """ - try: - return [ - str(getattr(obj, field, "")) or "" for field in [ - "id", "created_datetime", "modified_datetime", "location", "source", - "nest_height", "nest_size", "nest_location", "nest_type", - "observation_datetime", "modified_by_id", "created_by_id", "province_id", - "eradication_date", "municipality_id", "images", "public_domain", - "municipality_name", "modified_by_first_name", "created_by_first_name", - "wn_notes", "eradication_result", "wn_id", "wn_validation_status" - ] - ] - except Exception as e: - # Log and handle any serialization issues - logger.exception(f"Error serializing observation {obj.id}: {e}") - return [""] * len(CSV_HEADERS) + # Convert the SRID location string to GEOSGeometry + geom = GEOSGeometry(srid_str) + + # Extract latitude and longitude + longitude = geom.x + latitude = geom.y + return latitude, longitude + + def serialize_observation(self, obj: Observation, headers: list[str], allowed_fields: list[str]) -> list[str]: + """Serialize an observation for CSV export with specified fields.""" + data = [] + logger.info('Serializing observation %s', obj) + logger.info("allowed_fields: %s", allowed_fields) + for field in headers: + if field not in allowed_fields: + data.append("") # Add empty string for restricted fields + continue + + # Handle custom formatting for certain fields + if field == "latitude" or field == "longitude": + if obj.location: + srid_location_str = f"SRID=4326;POINT ({obj.location.x} {obj.location.y})" + latitude, longitude = self.parse_location(srid_location_str) + logger.info('Latitude: %s, Longitude: %s', latitude, longitude) + if field == "latitude": + data.append(str(latitude)) + elif field == "longitude": + data.append(str(longitude)) + else: + data.append("") + elif field in ["created_datetime", "modified_datetime", "observation_datetime"]: + datetime_val = getattr(obj, field, None) + if datetime_val: + # Remove milliseconds and ensure ISO format with 'T' and 'Z' + datetime_val = datetime_val.replace(microsecond=0) + data.append(datetime_val.isoformat() + "Z") + else: + data.append("") + elif field == "province": + data.append(obj.province.name if obj.province else "") + elif field == "municipality": + data.append(obj.municipality.name if obj.municipality else "") + elif field == "anb_domain": + data.append(str(obj.anb)) + elif field == "eradication_result": + data.append(obj.eradication_result.value if obj.eradication_result else "") + elif field == "nest_status": + logger.info("Getting status for observation %s", obj.eradication_result) + # This is handled as requested with eradication result + data.append(self.get_status(obj)) + elif field == "source_id": + data.append(str(obj.source_id) if obj.source_id is not None else "") + else: + value = getattr(obj, field, "") + data.append(str(value) if value is not None else "") + return data + + def get_status(self, observation: Observation) -> str: + """Determine observation status based on eradication data.""" + logger.info("Getting status for observation %s", observation.eradication_result) + if observation.eradication_result == EradicationResultEnum.SUCCESSFUL: + return "eradicated" + if observation.reserved_by: + return "reserved" + return "untreated" def _csv_line(self, row: list[str]) -> bytes: - """ - Converts a list of strings into a CSV-encoded line. - - Ensures each row is CSV-compatible and byte-encoded for StreamingHttpResponse. - """ + """Convert a list of strings to a CSV-compatible line in bytes.""" buffer = io.StringIO() writer = csv.writer(buffer) writer.writerow(row) return buffer.getvalue().encode("utf-8") - + @require_GET def search_address(request: Request) -> JsonResponse: """ diff --git a/vespadb/permissions.py b/vespadb/permissions.py index d8cecf0..7fc54a5 100644 --- a/vespadb/permissions.py +++ b/vespadb/permissions.py @@ -17,7 +17,7 @@ def has_object_permission(self, request: Request, view: View, obj: User) -> bool SYSTEM_USER_OBSERVATION_FIELDS_TO_UPDATE = [ "location", - "wn_notes", + "notes", "wn_admin_notes", "nest_height", "nest_size", diff --git a/vespadb/users/admin.py b/vespadb/users/admin.py index 2d5b8cb..37db6f6 100644 --- a/vespadb/users/admin.py +++ b/vespadb/users/admin.py @@ -27,7 +27,6 @@ class VespaUserAdmin(UserAdmin): { "fields": ( "user_type", - "personal_data_access", "municipalities", ) }, @@ -42,7 +41,6 @@ class VespaUserAdmin(UserAdmin): "first_name", "last_name", "user_type", - "personal_data_access", "reservation_count", ) search_fields = ("username", "email", "first_name", "last_name") diff --git a/vespadb/users/migrations/0007_remove_vespauser_personal_data_access.py b/vespadb/users/migrations/0007_remove_vespauser_personal_data_access.py new file mode 100644 index 0000000..c81f521 --- /dev/null +++ b/vespadb/users/migrations/0007_remove_vespauser_personal_data_access.py @@ -0,0 +1,17 @@ +# Generated by Django 5.1.2 on 2024-11-10 10:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0006_alter_vespauser_user_type'), + ] + + operations = [ + migrations.RemoveField( + model_name='vespauser', + name='personal_data_access', + ), + ] diff --git a/vespadb/users/models.py b/vespadb/users/models.py index 9e33244..0ec940a 100644 --- a/vespadb/users/models.py +++ b/vespadb/users/models.py @@ -1,5 +1,3 @@ -""".""" - import logging from enum import Enum @@ -10,7 +8,6 @@ logger = logging.getLogger(__name__) - class UserType(Enum): """User Type Enum.""" @@ -23,7 +20,6 @@ def choices(cls) -> list[tuple[str, str]]: """Return choices for the enum.""" return [(key.value, key.name) for key in cls] - class VespaUser(AbstractUser): """Model for the Vespa user.""" @@ -32,10 +28,22 @@ class VespaUser(AbstractUser): choices=UserType.choices(), default=UserType.REGULAR.value, ) - personal_data_access = models.BooleanField(default=False) municipalities = models.ManyToManyField( Municipality, blank=True, related_name="users", ) reservation_count = models.IntegerField(default=0) + + def has_assigned_municipality(self) -> bool: + """Check if user has assigned municipalities.""" + return bool(self.municipalities.exists()) + + def get_permission_level(self) -> str: + """Return the permission level based on user's assigned municipalities.""" + if self.is_superuser: + return "admin" + elif self.has_assigned_municipality(): + return "logged_in_with_municipality" + else: + return "logged_in_without_municipality" diff --git a/vespadb/users/serializers.py b/vespadb/users/serializers.py index 0a1cef8..05ad658 100644 --- a/vespadb/users/serializers.py +++ b/vespadb/users/serializers.py @@ -31,14 +31,12 @@ class Meta: "municipalities", "date_joined", "permissions", - "personal_data_access", "reservation_count", "is_superuser", ] extra_kwargs = { "password": {"write_only": True, "required": False}, "date_joined": {"read_only": True}, - "personal_data_access": {"required": False}, } def get_permissions(self, obj: User) -> list[str]: @@ -52,7 +50,7 @@ def get_permissions(self, obj: User) -> list[str]: ------- List[str]: A list of permission strings associated with the user. """ - return list(obj.get_all_permissions()) + return [obj.get_permission_level()] def create(self, validated_data: dict[str, Any]) -> User: """