Backend: Merge Dev to Main

.github/workflows/backend-dev.yml

@@ -0,0 +1,26 @@
+name: Deploy Backend Dev
+
+on:
+  push:
+    branches:
+      - BACKEND-DEV
+
+jobs:
+  deploy:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      - name: Set Up Docker Environment
+        run: |
+          docker version
+          docker-compose version
+
+      - name: Build and Start Docker Containers
+        run: |
+          docker-compose down   # Stop any running containers
+          docker-compose up -d  # Start containers in detached mode
+
+      - name: Verify Running Containers
+        run: docker ps

.gitignore

@@ -14,6 +14,7 @@
 
 __pycache__/
 venv/
+.venv/
 *.pyc
 staticfiles
 static

backend/Dockerfile

@@ -19,4 +19,4 @@ ENV PYTHONUNBUFFERED 1
 EXPOSE 8000
 
 # Run the Django development server
-CMD python manage.py makemigrations && python manage.py makemigrations app && python manage.py migrate && python manage.py runserver 0.0.0.0:8000
+CMD python manage.py makemigrations && python manage.py makemigrations app && python manage.py migrate && python manage.py populate_db_file  data/db_files/words.csv data/db_files/translations.csv data/db_files/relationships.csv && python manage.py runserver 0.0.0.0:8000 

backend/app/apps.py

@@ -1,24 +1,14 @@
 from django.apps import AppConfig
 import requests
 from django.db.models.signals import post_migrate, post_init
-
 import json
 
 
-def create_mockdata(**kwargs):
-    from .mock_data_manager import create_quiz_mockdata
-    create_quiz_mockdata()
-
 
 class AppConfig(AppConfig):
     default_auto_field = 'django.db.models.BigAutoField'
     name = 'app'
 
     def ready(self):
-        post_migrate.connect(create_mockdata, sender=self)
         import app.signals  # Import the signals to ensure they are registered
-
-
-
-
 

backend/app/management/commands/clear_database.py

@@ -0,0 +1,18 @@
+from django.core.management.base import BaseCommand
+from ...models import Word, Translation, Relationship
+
+class Command(BaseCommand):
+    help = 'Clear all data from the Word, Translation, and Relationship tables'
+
+    def handle(self, *args, **kwargs):
+        self.stdout.write("Clearing all data from the database...")
+
+        try:
+            # Delete all records from the models
+            Translation.objects.all().delete()
+            Relationship.objects.all().delete()
+            Word.objects.all().delete()
+
+            self.stdout.write(self.style.SUCCESS("All data cleared successfully!"))
+        except Exception as e:
+            self.stderr.write(self.style.ERROR(f"Error: {e}"))

backend/app/management/commands/fetch_and_save.py

@@ -0,0 +1,77 @@
+import os
+import csv
+from django.core.management.base import BaseCommand
+from app.word_service.csv_service import fetch_and_save_data
+
+class Command(BaseCommand):
+    help = 'Fetch the first k words from the CSV file and save to new CSV files'
+
+    def handle(self, *args, **options):
+        ab_filename = 'vocabulary_level_ab'
+        c_filename = 'vocabulary_level_c'
+        input_csv_path_ab = f'/app/data/cefr_files/{ab_filename}.csv'
+        input_csv_path_c = f'/app/data/cefr_files/{c_filename}.csv'
+        output_path = f'/app/data/db_files'
+        progress_file_ab = f'/app/data/progress_ab.txt'
+        progress_file_c = f'/app/data/progress_c.txt'
+
+        # Read progress from files
+        start_row_ab = self.get_progress(progress_file_ab)
+        start_row_c = self.get_progress(progress_file_c)
+
+        # Process first file
+        words_ab = self.read_csv(input_csv_path_ab)
+        self.process_words(
+            words_ab, start_row_ab, progress_file_ab, output_path
+        )
+
+        # Process second file
+        words_c = self.read_csv(input_csv_path_c)
+        self.process_words(
+            words_c, start_row_c, progress_file_c, output_path
+        )
+
+    def get_progress(self, progress_file):
+        if os.path.exists(progress_file):
+            with open(progress_file, 'r') as file:
+                return int(file.read().strip())
+        return 0
+
+    def save_progress(self, progress_file, row_number):
+        with open(progress_file, 'w') as file:
+            file.write(str(row_number))
+
+    def read_csv(self, csv_path):
+        words = []
+        try:
+            with open(csv_path, 'r', encoding='utf-8') as file:
+                reader = csv.DictReader(file)
+                for row in reader:
+                    words.append(row)
+        except FileNotFoundError:
+            self.stdout.write(self.style.ERROR(f"Input file '{csv_path}' not found."))
+        return words
+
+    def process_words(self, words, start_row, progress_file, output_path):
+        if not words:
+            self.stdout.write(self.style.ERROR("No words to process. Exiting process_words."))
+            return
+
+        self.stdout.write(self.style.NOTICE(f"Starting from row {start_row}. Total rows: {len(words)}"))
+
+        for i in range(start_row, len(words)):
+            try:
+                self.stdout.write(self.style.NOTICE(f"Processing row {i}..."))
+                fetch_and_save_data(
+                    words,
+                    output_path,
+                    i,
+                    i + 1,
+                    update_checkpoint=lambda row: self.save_progress(progress_file, row)
+                )
+            except Exception as e:
+                self.stdout.write(self.style.ERROR(f"Error processing row {i}: {e}"))
+                break
+
+
+

backend/app/management/commands/populate_db_file.py

@@ -0,0 +1,130 @@
+import csv
+from django.core.management.base import BaseCommand
+from ...models import Word, Translation, Relationship
+import re
+from collections import defaultdict
+
+class Command(BaseCommand):
+    help = 'Populate the database with Words, Translations, and Relationships from CSV files'
+
+    def add_arguments(self, parser):
+        parser.add_argument('words_csv', type=str, help="Path to the Words CSV file")
+        parser.add_argument('translations_csv', type=str, help="Path to the Translations CSV file")
+        parser.add_argument('relationships_csv', type=str, help="Path to the Relationships CSV file")
+
+    def handle(self, *args, **kwargs):
+        words_csv = kwargs['words_csv']
+        translations_csv = kwargs['translations_csv']
+        relationships_csv = kwargs['relationships_csv']
+
+        self.stdout.write("Starting to populate the database...")
+
+        try:
+            self.populate_words(words_csv)
+            self.populate_translations(translations_csv)
+            self.populate_relationships(relationships_csv)
+        except Exception as e:
+            self.stderr.write(self.style.ERROR(f"Error: {e}"))
+
+    def populate_words(self, file_path):
+        with open(file_path, newline='', encoding='utf-8') as csvfile:
+            reader = csv.DictReader(csvfile)
+
+            word_dict = defaultdict(list)
+
+            for row in reader:
+                info_dict = {
+                    'language': row.get('language', 'eng'),
+                    'level': row.get('level'),
+                    'part_of_speech': row.get('part_of_speech'),
+                    'meaning': row.get('explanation', 'Meaning not available'),
+                    'sentence': row.get('sentence', 'Sentence not available'),
+                }
+                word_dict[row['word']].append(info_dict)
+
+            existing_words = set(Word.objects.values_list('word', flat=True))
+            words_to_create = []
+
+            for word, infos in word_dict.items():
+                if word not in existing_words:
+                    defaults = {
+                        'language': ', '.join(set(info['language'] for info in infos)),
+                        'level': ', '.join(set(info['level'] for info in infos if info['level'])),
+                        'part_of_speech': ', '.join(set(info['part_of_speech'] for info in infos if info['part_of_speech'])),
+                        'meaning': ', '.join(set(info['meaning'] for info in infos if info['meaning'])),
+                        'sentence': ', '.join(set(info['sentence'] for info in infos if info['sentence'])),
+                    }
+                    words_to_create.append(Word(word=word, **defaults))
+
+            # Bulk create new words
+            if words_to_create:
+                Word.objects.bulk_create(words_to_create)
+                self.stdout.write(f"Created {len(words_to_create)} new words.")
+
+    def populate_translations(self, file_path):
+        with open(file_path, newline='', encoding='utf-8') as csvfile:
+            reader = csv.DictReader(csvfile)
+
+            existing_translations = set(Translation.objects.values_list('word__word', 'translation'))
+            words = {w.word: w for w in Word.objects.all()}
+            translations_to_create = []
+
+            for row in reader:
+                word = row['word']
+                translation = row['turkish_translation']
+
+                if (word, translation) not in existing_translations:
+                    if word not in words:
+                        words[word] = Word.objects.create(
+                            word=word,
+                            language='eng',
+                            level='unknown',
+                            part_of_speech='unknown',
+                            meaning='Auto-created for missing translation',
+                            sentence='No example provided',
+                        )
+                    translations_to_create.append(
+                        Translation(word=words[word], translation=translation)
+                    )
+
+            # Bulk create translations
+            if translations_to_create:
+                Translation.objects.bulk_create(translations_to_create)
+                self.stdout.write(f"Created {len(translations_to_create)} new translations.")
+
+    def populate_relationships(self, file_path):
+        with open(file_path, newline='', encoding='utf-8') as csvfile:
+            reader = csv.DictReader(csvfile)
+
+            words = {w.word: w for w in Word.objects.all()}
+            relationships_to_create = []
+
+            for row in reader:
+                word = row['word']
+                related_word = row['related_word']
+                relation_type = row['relation_type']
+
+                if not re.match(r"^[a-zA-Z-']+$", related_word):
+                    self.stderr.write(f"Skipping invalid related word '{related_word}'.")
+                    continue
+
+                if word not in words:
+                    self.stderr.write(f"Word '{word}' not found. Skipping relationship.")
+                    continue
+
+                if related_word not in words:
+                    words[related_word] = Word.objects.create(
+                        word=related_word,
+                        language='eng',
+                        level=words[word].level,
+                        part_of_speech=words[word].part_of_speech,
+                    )
+
+                relationships_to_create.append(
+                    Relationship(word=words[word], related_word=words[related_word], relation_type=relation_type)
+                )
+
+            # Bulk create relationships
+            if relationships_to_create:
+                Relationship.objects.bulk_create(relationships_to_create)
+                self.stdout.write(f"Created {len(relationships_to_create)} new relationships.")