app.py

import os
from pathlib import Path
from dotenv import load_dotenv
import logging
from eleven_labs import ElevenLabsVoice, should_use_eleven_labs
import base64
import tempfile
import uuid
from flask import send_file, redirect, url_for
from logging.handlers import RotatingFileHandler

# Set up logging first
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)

# Load environment variables
env_path = Path('.env')
if not env_path.exists():
    raise FileNotFoundError(f"Could not find .env file at {env_path.absolute()}")

load_dotenv(dotenv_path=env_path, override=True)

# Verify API key
api_key = os.getenv("OPENAI_API_KEY")
if not api_key:
    raise ValueError("OPENAI_API_KEY not found in environment variables")
if api_key.startswith("your_"):
    raise ValueError("OPENAI_API_KEY appears to be a placeholder value")

logger.info("Environment loaded successfully")
logger.info("OPENAI_API_KEY present and valid: %s", bool(api_key and not api_key.startswith("your_")))

logger.info("=== Environment Variables ===")
logger.info(f"VOICE_PROVIDER: {os.getenv('VOICE_PROVIDER')}")
logger.info(f"ELEVEN_LABS_VOICE_ID: {os.getenv('ELEVEN_LABS_VOICE_ID')}")
logger.info("==========================")

import re
import json
import openai
import datetime
from flask import Flask, request, jsonify, render_template_string, redirect, url_for
from pymongo import MongoClient
import certifi

# Twilio imports
from twilio.twiml.voice_response import VoiceResponse, Gather, Dial, Number, Play
from twilio.rest import Client
from twilio.twiml.messaging_response import MessagingResponse

###############################################################################
#                              CONFIG / INIT
###############################################################################
# Environment variables (customize accordingly)
TWILIO_ACCOUNT_SID = os.getenv("TWILIO_ACCOUNT_SID")
TWILIO_AUTH_TOKEN = os.getenv("TWILIO_AUTH_TOKEN")
TWILIO_NUMBER = os.getenv("TWILIO_PHONE_NUMBER")  # Changed to match .env
MONGO_URI = os.getenv("MONGODB_URI")  # Changed to match .env
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")

# Initialize OpenAI
openai.api_key = OPENAI_API_KEY

# Initialize Flask
app = Flask(__name__)

# Initialize MongoDB
mongo_client = MongoClient(os.getenv('MONGODB_URI'), tlsCAFile=certifi.where())
db = mongo_client["pay_per_call_db"]  # Database name
surveys_collection = db["surveys"]    # Stores survey data
offers_collection = db["offers"]      # Stores offer data
calls_collection = db["call_logs"]    # Stores call logs

# Twilio REST client
twilio_client = Client(TWILIO_ACCOUNT_SID, TWILIO_AUTH_TOKEN)

# Log the API key when app starts
logger.info("OPENAI_API_KEY loaded: %s", os.getenv("OPENAI_API_KEY")[:10] + "...")

# Add this near the top of your file where logging is configured
handler = RotatingFileHandler('app.log', maxBytes=10000000, backupCount=5)
logger.addHandler(handler)

###############################################################################
#                  HELPER FUNCTIONS / BUSINESS LOGIC
###############################################################################
def get_available_tags():
    """Get all available tags from tags collection"""
    return [tag["tag_name"] for tag in db.tags.find()]

def get_survey_data_by_phone(phone):
    """Get survey data by phone number."""
    # Clean the phone number to match database format
    phone = re.sub(r'\D', '', phone)  # Remove non-digits
    if len(phone) == 11 and phone.startswith('1'):
        phone = phone[1:]  # Remove leading 1
    
    logger.debug(f"Cleaned phone number format: {phone}")
    logger.debug(f"Original phone from database: {phone}")
    
    # Try both with and without country code
    possible_formats = [phone]
    if len(phone) == 10:
        possible_formats.append('1' + phone)
    elif len(phone) == 11 and phone.startswith('1'):
        possible_formats.append(phone[1:])
    
    logger.debug(f"Trying phone number formats: {possible_formats}")
    
    # Try each format
    for phone_format in possible_formats:
        survey_data = surveys_collection.find_one(
            {"Phone": phone_format},
            sort=[("_id", -1)]  # Get most recent if multiple
        )
        logger.debug(f"Search result for {phone_format}: {survey_data is not None}")
        
        if survey_data:
            logger.debug(f"Found survey data: {survey_data}")
            logger.debug(f"First name in data: {survey_data.get('first_name')}")
            logger.debug(f"Phone in data: {survey_data.get('Phone')}")
            
            # Get tags from tags collection and add them to survey_data
            tags = list(db.tags.find())  # Get all tags
            survey_data["available_tags"] = [tag["tag_name"] for tag in tags]
            
            # If survey doesn't have tags field, initialize it
            if "tags" not in survey_data:
                survey_data["tags"] = []
            
            return survey_data
    
    logger.warning(f"No survey found for any phone format: {possible_formats}")
    return None


def get_best_offer_for_user(survey_data, specific_tag=None):
    """
    Get best offer based on tag, state, and conversion rate.
    
    Args:
        survey_data: User's survey data
        specific_tag: Optional tag to filter by (if user selected a specific topic)
    """
    logger.debug(f"Finding offer for tag: {specific_tag}")
    
    # Get user's loan amount
    loan_amount_str = re.sub(r"[^\d.]", "", survey_data.get("LAmount", "0"))
    try:
        user_loan_amount = float(loan_amount_str)
    except:
        user_loan_amount = 0.0
    
    # Build query
    query = {
        "minMortgageBalance": {"$lte": user_loan_amount},
        "maxMortgageBalance": {"$gte": user_loan_amount}
    }
    
    # Add tag filter if specified
    if specific_tag:
        query["tag"] = specific_tag
    
    # Add state filter if available
    user_state = survey_data.get("State")
    if user_state:
        query["state"] = user_state
    
    logger.debug(f"Offer query: {query}")
    
    # Find all matching offers and sort by conversion rate
    offers = offers_collection.find(query).sort("conversionRate", -1)
    
    # Get the first (highest converting) offer
    best_offer = next(offers, None)
    
    logger.debug(f"Selected offer: {best_offer}")
    return best_offer


def call_ai_model(conversation_history, survey_data, language="en"):
    """
    Sends the conversation to an LLM (OpenAI GPT example).
    """
    from openai import OpenAI
    
    api_key = os.getenv("OPENAI_API_KEY")
    if not api_key or api_key.startswith("your_"):
        raise ValueError("Invalid OPENAI_API_KEY configuration")
        
    logger.info("Using API key starting with: %s", api_key[:10])
    
    client = OpenAI(api_key=api_key)
    logger.debug("OpenAI client initialized")
    
    tags = survey_data.get("tags", []) if survey_data else []
    tags_str = ", ".join(tags) if tags else "no specific tags"
    
    # CONVERSATION CONTROL POINT 1: Initial System Message
    system_message = (
        "You are a helpful AI that speaks in English with a professional and slightly witty tone. "
        "Because this is a sales environment, your primary goal is to discuss ONLY the following approved topics: "
        f"{tags_str}. "
        "If the caller tries to discuss any subject outside those tags, politely and briefly redirect them back to the approved topics. "
        "If you are unsure whether a topic is related, ask the caller for clarification while reminding them of the approved topics. "
        "\n\n"
        "Keep your responses concise, purposeful, and focused on understanding their needs and providing assistance regarding those tags. "
        "Ask one specific question at a time, then follow up with exactly one more question to clarify or gather more details. "
        "\n\n"
        "When you have enough information, ask ONCE if they would like: "
        "1) A text message with further details (if they say anything about 'text' or 'message,' confirm you're sending it), "
        "2) A call back later, "
        "3) Or to speak to a human specialist right away. "
        "\n\n"
        "IMPORTANT: Never ask for their phone number. We already have it. "
        "If they mention they want a text, confirm you'll send it after the call. Do not repeatedly mention sending a text. "
        "If they want to be called back, politely confirm a callback time and date. "
        "If they ask to be transferred to a human, transfer the call immediately. "
        "\n\n"
        "If the caller insists on off-topic subjects, politely but firmly restate that you can only help with the approved topics. "
        "Above all, keep the call on track by focusing on how to resolve their needs via the topics: "
        f"{tags_str}."
    )

    # CONVERSATION CONTROL POINT 2: Message History
    messages = [
        {"role": "system", "content": system_message}
    ]
    
    # Add user messages & AI messages
    for turn in conversation_history:
        if turn["sender"] == "user":
            messages.append({"role": "user", "content": turn["text"]})
        else:
            messages.append({"role": "assistant", "content": turn["text"]})

    # CONVERSATION CONTROL POINT 3: Response Generation
    response = client.chat.completions.create(
        model="gpt-3.5-turbo",
        messages=messages,
        temperature=0.7  # Adjust this to control response variability
    )
    return response.choices[0].message.content


def detect_language(survey_data):
    """
    For MVP, let's just do a simple example:
    If 'State' is something for Spanish region or if user marked "Spanish",
    we do Spanish. Otherwise English.
    Or we can store 'preferredLanguage' in survey_data if available.
    """
    # Hard-coded example: if "preferredLanguage" is 'es', do Spanish, else English
    # Return "en" or "es"
    # For now, we'll just do English unless user is from a known Spanish-speaking area
    return "en"  # override with "es" if you want to test Spanish conversation.


def store_call_log(call_sid, update_data):
    """Store or update call log information"""
    try:
        # Get current timestamp
        now = datetime.datetime.utcnow()
        
        # Merge update data with timestamp
        update_dict = {
            "$set": {
                **update_data,
                "updatedAt": now
            }
        }
        
        # If this is a new document, also set createdAt
        if "createdAt" not in update_data:
            update_dict["$setOnInsert"] = {"createdAt": now}
            
        # Update or insert the document
        result = calls_collection.update_one(
            {"callSid": call_sid},
            update_dict,
            upsert=True
        )
        
        logger.debug(f"Call log updated for {call_sid}: {update_data}")
        return result
    except Exception as e:
        logger.error(f"Error storing call log: {str(e)}")
        return None


def get_voice_for_name(first_name, last_name=None):
    """
    Simple voice selection based on name patterns.
    Returns tuple of (voice_name, language_code)
    """
    # Convert to lowercase for matching
    first_name = first_name.lower() if first_name else ""
    last_name = last_name.lower() if last_name else ""
    
    # Name lists
    spanish_names = {'jose', 'juan', 'carlos', 'miguel', 'luis', 'maria', 'ana', 'rosa'}
    indian_names = {
        'jas', 'raj', 'priya', 'amit', 'arun', 'deepak', 'kiran', 'neha', 
        'patel', 'singh', 'kumar', 'sharma', 'gupta', 'sanjay', 'sunil'
    }
    african_american_names = {
        # Traditional
        'deandre', 'darnell', 'terrell', 'malik', 'trevon', 'tyrone',
        'deshawn', 'marquis', 'maurice', 'jamal', 'jermaine',
        # Modern
        'jayden', 'aiden', 'zion', 'xavier', 'kayden',
        # Female names
        'latoya', 'keisha', 'lakisha', 'tanisha', 'latasha',
        'shaniqua', 'aaliyah', 'precious', 'nia', 'imani',
        # Unique/Cultural
        'dashawn', 'rashad', 'darius', 'reginald', 'shanice',
        'ebony', 'essence', 'destiny', 'diamond'
    }
    white_names = {
        # English/American
        'john', 'william', 'james', 'michael', 'robert', 'david', 'richard',
        'mary', 'patricia', 'jennifer', 'elizabeth', 'linda', 'barbara',
        # Irish/Scottish/Germanic names...
        'sean', 'connor', 'ryan', 'patrick', 'shannon', 'kelly',
        'craig', 'ross', 'cameron', 'fiona', 'ailsa',
        'hans', 'kurt', 'eric', 'karl', 'emma', 'anna',
        'lars', 'erik', 'anders', 'bjorn', 'astrid', 'ingrid'
    }
    
    # Check both first and last names for each ethnicity
    if first_name in indian_names or (last_name and last_name in indian_names):
        return ("Polly.Aditi", "en-IN")  # Indian female voice
    
    if first_name in spanish_names:
        return ("Polly.Miguel", "es-US")  # Spanish male voice
    
    if first_name in african_american_names:
        # For African American names, use American voices
        aa_voices = [
            ("Polly.Joanna", "en-US"),   # American female
            ("Polly.Matthew", "en-US"),   # American male
        ]
        import random
        return random.choice(aa_voices)
    
    if first_name in white_names:
        # For white names, use these specific voices
        white_voices = [
            ("Polly.Amy", "en-GB"),      # British female
            ("Polly.Nicole", "en-AU"),   # Australian female
            ("Polly.Joanna", "en-US"),   # American female
        ]
        import random
        return random.choice(white_voices)
    
    # For all other names, use full range of English voices
    english_voices = [
        ("Polly.Joanna", "en-US"),   # American female
        ("Polly.Nicole", "en-AU"),   # Australian female
        ("Polly.Amy", "en-GB"),      # British female
        ("Polly.Matthew", "en-US"),  # American male
        ("Polly.Russell", "en-AU"),  # Australian male
    ]
    
    import random
    return random.choice(english_voices)


def say_with_voice(resp, text, voice_id="Polly.Matthew", language="en-US"):
    """Add speech to the response with the specified voice."""
    try:
        if should_use_eleven_labs():
            logger.info("Using Eleven Labs for TTS")
            eleven_labs = ElevenLabsVoice()
            audio_content = eleven_labs.text_to_speech(text)
            
            if audio_content:
                # Create a temporary directory that persists
                temp_dir = os.path.join("/tmp", "eleven_labs_audio")
                os.makedirs(temp_dir, exist_ok=True)
                logger.debug(f"Ensuring directory exists: {temp_dir}")
                
                # Create a temporary file with a unique name
                filename = f"speech_{uuid.uuid4()}.mp3"
                temp_path = os.path.join(temp_dir, filename)
                
                # Write the audio content to the temporary file
                with open(temp_path, "wb") as f:
                    f.write(audio_content)
                
                logger.debug(f"Saved audio file to: {temp_path}")
                logger.debug(f"File exists: {os.path.exists(temp_path)}")
                logger.debug(f"File size: {os.path.getsize(temp_path)}")
                
                # Get the full URL for the audio file
                audio_url = request.url_root.rstrip('/') + f"/audio/{filename}"
                logger.debug(f"Audio URL: {audio_url}")
                
                # Add the audio to the response using the URL
                resp.play(audio_url)
                
                logger.info("Successfully added Eleven Labs audio to response")
                return
            else:
                logger.error("Eleven Labs returned no audio content")
        
        # Fallback to Polly
        logger.warning(f"Falling back to Polly with voice {voice_id}")
        resp.say(text, voice=voice_id, language=language)
        
    except Exception as e:
        logger.error(f"Error in say_with_voice: {str(e)}", exc_info=True)
        # Fallback to Polly in case of any error
        resp.say(text, voice="Polly.Matthew", language="en-US")


def send_clicksend_sms(to_number, topic):
    """Send SMS via ClickSend API with topic-specific URL"""
    import base64
    import requests
    
    logger.info(f"Attempting to send SMS to {to_number} about {topic}")
    
    # Map topics to URLs
    url_map = {
        "debt consolidation": "https://debtconsolidation.com",
        "refinance": "https://refinanceinfo.com"
    }
    
    # Get the appropriate URL or use a default
    topic_url = url_map.get(topic.lower(), "https://debtconsolidation.com")
    logger.debug(f"Using URL {topic_url} for topic {topic}")
    
    # Construct message
    message_text = f"Here is the link to your {topic} request: {topic_url}"
    
    url = "https://rest.clicksend.com/v3/sms/send"
    
    # Prepare credentials
    username = os.getenv('CLICKSEND_USERNAME')
    api_key = os.getenv('CLICKSEND_API_KEY')
    from_number = os.getenv('CLICKSEND_FROM_NUMBER')
    
    logger.debug(f"Using ClickSend credentials - Username: {username}, From Number: {from_number}")
    
    encoded_credentials = base64.b64encode(f'{username}:{api_key}'.encode()).decode()
    
    headers = {
        'Content-Type': 'application/json',
        'Authorization': f'Basic {encoded_credentials}'
    }
    
    # Prepare message payload
    payload = {
        "messages": [{
            "source": "sdk",
            "from": from_number,
            "body": message_text,
            "to": to_number
        }]
    }
    logger.debug(f"Prepared SMS payload: {payload}")
    
    try:
        logger.info(f"Making request to ClickSend API for {to_number}")
        response = requests.post(url, headers=headers, json=payload)
        response.raise_for_status()
        response_data = response.json()
        logger.info(f"SMS sent successfully to {to_number} for topic: {topic}")
        logger.debug(f"ClickSend API response: {response_data}")
        return response_data
    except requests.exceptions.RequestException as e:
        logger.error(f"HTTP Error sending SMS: {str(e)}")
        logger.error(f"Response status code: {e.response.status_code if hasattr(e, 'response') else 'N/A'}")
        logger.error(f"Response body: {e.response.text if hasattr(e, 'response') else 'N/A'}")
        return None
    except Exception as e:
        logger.error(f"Unexpected error sending SMS: {str(e)}")
        return None

###############################################################################
#                      TWILIO VOICE WEBHOOKS
###############################################################################
@app.route("/voice/incoming_call", methods=["POST"])
def incoming_call():
    """
    Handles incoming calls from Twilio
    """
    logger.info("Incoming call received")
    logger.debug("Request form data: %s", request.form)
    
    call_sid = request.form.get("CallSid")
    from_number = request.form.get("From", "")
    
    logger.info("Call from %s (SID: %s)", from_number, call_sid)
    
    # Attempt to find user's data in Mongo
    phone_digits = re.sub(r"[^\d]", "", from_number)
    logger.debug("Looking up phone number: %s", phone_digits)
    
    try:
        survey_data = get_survey_data_by_phone(phone_digits)
        logger.info("Survey data found: %s", bool(survey_data))
        logger.debug("Survey data: %s", survey_data)
    except Exception as e:
        logger.error("Error fetching survey data: %s", str(e))
        survey_data = None

    # Rest of your existing code...
    try:
        # Build TwiML response
        resp = VoiceResponse()
        if survey_data:
            # Add debug logging
            logger.debug("Survey data fields: %s", survey_data.keys())
            logger.debug("First name from survey: %s", survey_data.get("first_name"))
            logger.debug("Last name from survey: %s", survey_data.get("last_name"))
            
            first_name = survey_data.get("first_name", "there")
            logger.debug("Using first_name: %s", first_name)  # See what we're actually using
            
            # Get appropriate voice based on name
            voice_name, lang_code = get_voice_for_name(
                survey_data.get('first_name', ''),
                survey_data.get('last_name', '')
            )
            
            tags = survey_data.get("tags", [])
            
            if tags:
                tags_text = ", ".join(tags[:-1]) + f" and {tags[-1]}" if len(tags) > 1 else tags[0]
                gather_say = (
                    f"Thank you for calling Everyday Resources! I am an advanced AI agent and I can understand full conversations"
                    f", {first_name}, -- (cool name by the way), I noticed that you're interested in {tags_text}. "
                    "Which of these would you like to discuss first?"
                )
                logger.debug("Generated gather_say: %s", gather_say)  # See final message
            else:
                gather_say = f"Hello {first_name}, thank you for calling. How can I help you today?"
            
            # Store voice selection in call log
            store_call_log(call_sid, {
                "voice_used": voice_name,
                "language": lang_code
            })
        else:
            # Default voice for unknown callers
            voice_name, lang_code = "Polly.Matthew", "en-US"
            gather_say = (
                "Hello, thank you for calling Everyday Resources. "
                "We could not find your survey record. "
                "Please say yes to continue."
            )
        
        logger.info("Preparing to say: %s", gather_say)
        
        # Create Gather with selected voice
        gather = Gather(
            input="speech",
            action="/voice/process_response",
            language=lang_code,
            speech_timeout="auto",
            hints="yes, no, refinance, help"
        )
        say_with_voice(gather, gather_say, voice_name, lang_code)
        resp.append(gather)
        resp.redirect("/voice/fallback")
        
        # Store call log
        store_call_log(call_sid, {
            "fromNumber": from_number,
            "status": "incoming_call",
            "surveyDataFound": bool(survey_data)
        })
        
        logger.info("Call handled successfully")
        return str(resp)
        
    except Exception as e:
        logger.error("Error handling call: %s", str(e), exc_info=True)
        # Return a basic error response
        resp = VoiceResponse()
        resp.say("We apologize, but there was an error processing your call. Please try again later.")
        return str(resp)


@app.route("/voice/process_response", methods=["POST"])
def process_response():
    """Handle user's speech response"""
    call_sid = request.form.get("CallSid")
    from_number = request.form.get("From")
    user_speech = request.form.get("SpeechResult", "")
    
    # Convert user speech to lowercase for matching
    user_speech_lower = user_speech.strip().lower()
    
    # Log the incoming request
    logger.info(f"Processing response for call {call_sid} from {from_number}")
    logger.debug(f"User speech: {user_speech}")
    
    # Check for text message request
    wants_text = any(word in user_speech_lower for word in ["text", "message", "send me", "link"])
    logger.debug(f"User wants text: {wants_text}")
    
    # Check for call ending
    call_ending = any(word in user_speech_lower for word in ["goodbye", "bye", "end", "done", "finish"])
    logger.debug(f"Call ending: {call_ending}")
    
    # Check for transfer request
    wants_transfer = any(word in user_speech_lower for word in ["transfer", "human", "person", "someone", "representative"])
    logger.debug(f"User wants transfer: {wants_transfer}")
    
    # Retrieve conversation so far from call_logs
    call_log = calls_collection.find_one({"callSid": call_sid})
    if not call_log:
        call_log = {}
        # Initialize new call log
        store_call_log(call_sid, {
            "callSid": call_sid,
            "fromNumber": from_number,
            "status": "incoming_call",
            "surveyDataFound": False,
            "offerConnected": False
        })
    
    # Update conversation history
    if "conversation" not in call_log:
        call_log["conversation"] = []
    call_log["conversation"].append({"sender": "user", "text": user_speech})

    # Get user's survey data
    phone_digits = re.sub(r"[^\d]", "", from_number)
    survey_data = get_survey_data_by_phone(phone_digits)
    
    # Update survey data found status
    if survey_data:
        store_call_log(call_sid, {"surveyDataFound": True})
    
    # Check if user mentioned a specific tag
    user_tags = survey_data.get("tags", []) if survey_data else []
    selected_tag = None
    
    for tag in user_tags:
        if tag.lower() in user_speech_lower:
            selected_tag = tag
            break
    
    lang = detect_language(survey_data)

    # Get AI response
    ai_response = call_ai_model(
        conversation_history=call_log["conversation"],
        survey_data=survey_data,
        language=lang
    )

    # Add AI's response to conversation history
    call_log["conversation"].append({"sender": "ai", "text": ai_response})
    
    # Store updated conversation
    store_call_log(call_sid, {
        "conversation": call_log["conversation"]
    })

    # Initialize resp at the start
    resp = VoiceResponse()

    # Check for call ending or transfer request
    call_ending = any(word in user_speech_lower for word in ["goodbye", "bye", "end", "done", "finish"])
    wants_transfer = any(word in user_speech_lower for word in ["transfer", "human", "person", "someone", "representative"])
    wants_text = any(word in user_speech_lower for word in ["text", "message", "send me", "link"])

    # Send SMS if requested and call is ending
    if (call_ending or wants_transfer) and wants_text and selected_tag:
        logger.info(f"Sending SMS for call {call_sid} - User requested text, call ending/transfer, tag: {selected_tag}")
        # Send SMS immediately when call is ending and they requested a text
        sms_response = send_clicksend_sms(
            to_number=from_number,
            topic=selected_tag
        )
        
        # Log the SMS sending
        store_call_log(call_sid, {
            "sms_sent": True,
            "sms_topic": selected_tag,
            "sms_sent_at": datetime.datetime.utcnow(),
            "sms_response": sms_response
        })
        logger.info(f"SMS sending attempt logged for call {call_sid}")

    # Handle different scenarios
    if wants_transfer:
        best_offer = get_best_offer_for_user(survey_data, specific_tag=selected_tag)
        if best_offer:
            # Log that we're connecting to an offer
            store_call_log(call_sid, {
                "status": "transferring",
                "offerConnected": True,
                "connectedToOffer": best_offer["_id"],
                "transferredAt": datetime.datetime.utcnow()
            })
            
            voice_name = os.getenv('ELEVEN_LABS_VOICE_ID')
            say_with_voice(resp, 
                         f"Great! I'll connect you with a specialist about {selected_tag or 'your interests'} now.",
                         voice_id=voice_name)
            
            resp.dial(
                number=best_offer["routeToPhoneNumber"],
                action="/voice/handle_dial",
                whisper=f"Call from {survey_data.get('first_name')} {survey_data.get('last_name')} "
                       f"regarding {selected_tag or survey_data.get('LType')}"
            )
            return str(resp)  # Return after transfer
        else:
            # Even if no offer is found, keep the conversation going
            gather = Gather(
                input="speech",
                action="/voice/process_response",
                language=lang,
                timeout=3,
                speech_timeout="auto"
            )
            say_with_voice(gather, 
                         "I apologize, but I can't connect you to a specialist right now. "
                         "Would you like to continue discussing your options?",
                         voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
            resp.append(gather)
            
            # Add a fallback gather in case of silence
            resp.redirect("/voice/fallback")
            return str(resp)  # Return after no-offer handling
            
    elif call_ending:
        # End the call gracefully
        say_with_voice(resp, "Thank you for calling. Have a great day!", voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
        return str(resp)  # Return after ending call
        
    else:
        # Continue conversation
        gather = Gather(
            input="speech",
            action="/voice/process_response",
            language=lang,
            timeout=3,
            speech_timeout="auto"
        )
        say_with_voice(gather, ai_response, voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
        resp.append(gather)
        
        # Add a fallback gather in case of silence
        resp.redirect("/voice/fallback")
        return str(resp)  # Return after normal conversation


@app.route("/voice/connect_offer", methods=["POST"])
def connect_offer():
    """
    If user agrees to speak to the business, we forward/bridge the call.
    We'll do the whisper, etc.
    """
    call_sid = request.form.get("CallSid")
    user_speech = request.form.get("SpeechResult", "").lower()
    
    call_log = calls_collection.find_one({"callSid": call_sid})
    if not call_log:
        call_log = {}

    survey_data_found = call_log.get("surveyDataFound", False)
    from_number = call_log.get("fromNumber", "")

    resp = VoiceResponse()

    if "yes" in user_speech or "okay" in user_speech or "sure" in user_speech:
        # Look up best offer from earlier step
        phone_digits = re.sub(r"[^\d]", "", from_number)
        survey_data = get_survey_data_by_phone(phone_digits)
        best_offer = get_best_offer_for_user(survey_data)

        if best_offer:
            offer_number = best_offer.get("routeToPhoneNumber", "")
            
            # We do a 'whisper' to the business
            # 1) Dial the business
            # 2) The 'url' attribute can be used to play a whisper to the business.
            dial = Dial(caller_id=TWILIO_NUMBER)
            
            # The "whisper" is a TwiML url that Twilio fetches before connecting to business
            # We'll create a short route that returns TwiML for the whisper
            dial.number(
                offer_number,
                url=f"/voice/whisper?callSid={call_sid}",
                status_callback="/voice/dial_status",
                status_callback_event=["answered", "completed"]
            )
            
            resp.say("Connecting you now. Please hold.", voice="Polly.Matthew")
            resp.append(dial)

            # Store that we've connected
            store_call_log(call_sid, {
                "offerConnected": True,
                "connectedTo": offer_number
            })

        else:
            resp.say(
                "Apologies, but I can't find any offer right now. Goodbye.",
                voice="Polly.Matthew"
            )
            resp.hangup()
    else:
        resp.say("Alright, no problem. Thank you for calling. Goodbye!", voice="Polly.Matthew")
        resp.hangup()

    return str(resp)


@app.route("/voice/whisper", methods=["GET", "POST"])
def whisper_to_agent(call_sid):
    """Handle the whisper to the agent before connecting."""
    call_log = calls_collection.find_one({"callSid": call_sid})

    user_data_str = ""
    if call_log and call_log.get("surveyDataFound"):
        # Retrieve user's survey data
        from_number = call_log.get("fromNumber", "")
        phone_digits = re.sub(r"[^\d]", "", from_number)
        survey_data = get_survey_data_by_phone(phone_digits)
        if survey_data:
            fn = survey_data.get("first_name", "")
            ln = survey_data.get("last_name", "")
            amt = survey_data.get("LAmount", "")
            credit = survey_data.get("Credit", "")
            user_data_str = (
                f"Caller name is {fn} {ln}, with loan amount {amt} and credit score {credit}."
            )
    
    resp = VoiceResponse()
    # Use the same Eleven Labs voice for consistency
    say_with_voice(resp, 
                  f"Hello business partner. You have a live call. {user_data_str} Connecting now.",
                  voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))  # Use the same voice ID from .env

    return str(resp)


@app.route("/voice/dial_status", methods=["POST"])
def dial_status():
    """
    Gets updates about the call bridging (answered, completed, etc.)
    """
    call_sid = request.form.get("CallSid")
    call_status = request.form.get("DialCallStatus")  # e.g. 'answered', 'completed', etc.

    store_call_log(call_sid, {"dialStatus": call_status})
    return ("", 204)


@app.route("/voice/fallback", methods=["POST", "GET"])
def fallback():
    """Handle silence or no input with improved error handling"""
    resp = VoiceResponse()
    
    # Get call details
    call_sid = request.form.get("CallSid")
    from_number = request.form.get("From")
    
    # Get or initialize attempt counter from call logs
    call_log = calls_collection.find_one({"callSid": call_sid}) or {}
    fallback_attempts = call_log.get("fallback_attempts", 0)
    
    # Get survey data for tags
    survey_data = get_survey_data_by_phone(re.sub(r'\D', '', from_number)) if from_number else None
    tags = survey_data.get("tags", []) if survey_data else []
    tags_str = ", ".join(tags) if tags else "our services"
    
    # Update attempt counter
    fallback_attempts += 1
    store_call_log(call_sid, {"fallback_attempts": fallback_attempts})
    
    # Handle based on number of attempts
    if fallback_attempts >= 2:  # Reduced threshold to 2 attempts
        # After 2 attempts, give menu options
        message = (
            "I'm having trouble hearing you clearly. No problem, you can: "
            f"Press 1 to receive a text message about {tags_str}, "
            "Press 2 to speak with a specialist right away, "
            "Or press 3 to schedule a callback at a better time. "
        )
        gather = Gather(
            input="dtmf",  # Only accept keypad input for reliability
            action="/voice/handle_fallback_input",
            timeout=5,
            num_digits=1
        )
        say_with_voice(gather, message, voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
        resp.append(gather)
        
        # If no input after prompt, end call gracefully
        say_with_voice(resp, 
                      "I haven't heard from you. I'll end the call now, but you can call back anytime. Goodbye.",
                      voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
        resp.hangup()
        
    else:
        # First attempt - simple retry
        gather = Gather(
            input="speech",
            action="/voice/process_response",
            language="en-US",
            timeout=3,
            speech_timeout="auto"
        )
        say_with_voice(gather, 
                      "I'm having trouble hearing you. Please speak clearly and try again.",
                      voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
        resp.append(gather)
        resp.redirect("/voice/fallback")
    
    return str(resp)

@app.route("/voice/handle_fallback_input", methods=["POST"])
def handle_fallback_input():
    """Handle user's choice after fallback attempts"""
    resp = VoiceResponse()
    
    # Get user's input and call details
    digits = request.form.get("Digits", "")
    call_sid = request.form.get("CallSid")
    from_number = request.form.get("From")
    
    logger.info(f"Handling fallback input for call {call_sid} - Digits: {digits}")
    
    # Get survey data for personalization
    survey_data = get_survey_data_by_phone(re.sub(r'\D', '', from_number)) if from_number else None
    tags = survey_data.get("tags", []) if survey_data else []
    tags_str = ", ".join(tags) if tags else "our services"
    
    if digits == "1":
        # Send text message
        logger.info(f"User pressed 1 - Sending text messages for call {call_sid}")
        say_with_voice(resp, 
                      f"I'll send you a text message about {tags_str} right away. Thank you for calling!",
                      voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
        
        # Send SMS via ClickSend
        if tags:
            logger.info(f"Sending SMS for each tag: {tags}")
            for tag in tags:
                sms_response = send_clicksend_sms(from_number, tag)
                # Log each SMS attempt
                store_call_log(call_sid, {
                    f"sms_sent_{tag}": True,
                    f"sms_topic_{tag}": tag,
                    f"sms_sent_at_{tag}": datetime.datetime.utcnow(),
                    f"sms_response_{tag}": sms_response
                })
        else:
            logger.info("No tags found, sending general services SMS")
            sms_response = send_clicksend_sms(from_number, "general services")
            store_call_log(call_sid, {
                "sms_sent": True,
                "sms_topic": "general services",
                "sms_sent_at": datetime.datetime.utcnow(),
                "sms_response": sms_response
            })
            
        resp.hangup()
        
    elif digits == "2":
        # Transfer to specialist
        say_with_voice(resp, 
                      "I'll connect you with a specialist right away.",
                      voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
        
        # Get best offer for transfer
        best_offer = get_best_offer_for_user(survey_data)
        
        if best_offer:
            resp.dial(
                number=best_offer["routeToPhoneNumber"],
                action="/voice/handle_dial",
                timeout=15,
                caller_id=TWILIO_NUMBER
            )
        else:
            say_with_voice(resp, 
                          "I apologize, but I can't connect you to a specialist right now. Please try calling back during business hours.",
                          voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
            resp.hangup()
            
    elif digits == "3":
        # Schedule callback
        say_with_voice(resp, 
                      "I'll make a note for our team to call you back during business hours. "
                      "You can expect a call within the next business day. Thank you for your patience!",
                      voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
        
        # Store callback request in call log
        store_call_log(call_sid, {
            "callback_requested": True,
            "callback_status": "pending",
            "callback_requested_at": datetime.datetime.utcnow()
        })
        
        resp.hangup()
        
    else:
        # Invalid input
        say_with_voice(resp, 
                      "I didn't recognize that option. Please call back and try again. Goodbye.",
                      voice_id=os.getenv('ELEVEN_LABS_VOICE_ID'))
        resp.hangup()
    
    return str(resp)

###############################################################################
#                      SMS TRIGGER ENDPOINT
###############################################################################
@app.route("/send_survey_sms", methods=["POST"])
def send_survey_sms():
    """
    Example endpoint: after a user completes a web survey, you can call this
    to send them an SMS with the number to call.
    Body:
    {
      "phoneNumber": "+1XXXXXXXXXX"
    }
    """
    data = request.json
    phone = data.get("phoneNumber", "")
    if not phone:
        return jsonify({"error": "phoneNumber is required"}), 400

    message_body = (
        f"Thank you for completing our survey! Call us at {TWILIO_NUMBER} "
        "to discuss your options."
    )
    twilio_client.messages.create(
        to=phone,
        from_=TWILIO_NUMBER,
        body=message_body
    )

    return jsonify({"status": "SMS sent"}), 200

###############################################################################
#                    SIMPLE FRONTEND TO VIEW CALL LOGS
###############################################################################
@app.route("/logs", methods=["GET"])
def view_logs():
    """Display call logs in a simple HTML table"""
    # Get all call logs, sorted by creation date descending
    logs = calls_collection.find().sort("createdAt", -1)
    
    # HTML template for the logs page
    html = """
    <html>
    <head>
        <title>Call Logs</title>
        <style>
            table { border-collapse: collapse; width: 100%; }
            th, td { border: 1px solid #ddd; padding: 8px; text-align: left; }
            th { background-color: #f2f2f2; }
            tr:nth-child(even) { background-color: #f9f9f9; }
            .conversation { max-width: 500px; word-wrap: break-word; }
        </style>
    </head>
    <body>
        <h1>Call Logs</h1>
        <table>
            <tr>
                <th>Call SID</th>
                <th>From</th>
                <th>Status</th>
                <th>Offer Connected</th>
                <th>Conversation</th>
                <th>Created At</th>
            </tr>
            {% for log in logs %}
            <tr>
                <td>{{ log.callSid }}</td>
                <td>{{ log.fromNumber }}</td>
                <td>{{ log.status or 'n/a' }}</td>
                <td>{{ log.offerConnected or False }}</td>
                <td class="conversation">
                    {% if log.conversation %}
                        {% for msg in log.conversation %}
                            {{ msg.sender }}: {{ msg.text }}<br>
                        {% endfor %}
                    {% endif %}
                </td>
                <td>{{ log.createdAt.strftime('%Y-%m-%d %H:%M:%S') if log.createdAt else 'n/a' }}</td>
            </tr>
            {% endfor %}
        </table>
    </body>
    </html>
    """
    
    return render_template_string(html, logs=logs)

###############################################################################
#                           ROOT AND STATIC ROUTES
###############################################################################
@app.route("/")
def index():
    """
    Root route that shows a simple landing page with instructions.
    """
    template = """
    <html>
      <head>
        <title>Pay Per Call AI</title>
        <link rel="icon" type="image/x-icon" href="{{ url_for('static', filename='favicon.ico') }}">
        <style>
          body {
            font-family: Arial, sans-serif;
            max-width: 800px;
            margin: 40px auto;
            padding: 20px;
            line-height: 1.6;
          }
          .button {
            display: inline-block;
            padding: 10px 20px;
            background: #007bff;
            color: white;
            text-decoration: none;
            border-radius: 5px;
            margin: 10px 0;
          }
          code {
            background: #f4f4f4;
            padding: 2px 5px;
            border-radius: 3px;
          }
        </style>
      </head>
      <body>
        <h1>Pay Per Call AI System</h1>
        <p>Welcome to the Pay Per Call AI system. This application handles:</p>
        <ul>
          <li>Inbound voice calls via Twilio</li>
          <li>AI-powered conversation management</li>
          <li>Call routing to matched offers</li>
          <li>Call logging and monitoring</li>
        </ul>
        
        <h2>Quick Links</h2>
        <p>
          <a href="/logs" class="button">View Call Logs</a>
        </p>
        
        <h2>API Endpoints</h2>
        <ul>
          <li><code>POST /send_survey_sms</code> - Send SMS to users</li>
          <li><code>GET /logs</code> - View call logs dashboard</li>
        </ul>
        
        <h2>Setup Required</h2>
        <p>Make sure you have:</p>
        <ul>
          <li>Configured your Twilio webhook to point to <code>/voice/incoming_call</code></li>
          <li>Set up your MongoDB collections (surveys, offers, call_logs)</li>
          <li>Added your environment variables (TWILIO_*, MONGO_URI, OPENAI_API_KEY)</li>
        </ul>
      </body>
    </html>
    """
    return render_template_string(template)

@app.route('/favicon.ico')
def favicon():
    return ('', 204)

@app.route('/apple-touch-icon.png')
@app.route('/apple-touch-icon-precomposed.png')
def apple_touch_icon():
    return ('', 204)

###############################################################################
#                               MAIN RUN
###############################################################################
@app.route("/audio/<filename>", methods=["GET"])
def serve_audio(filename):
    """Serve temporary audio files"""
    try:
        # Ensure safe filename
        if '..' in filename or filename.startswith('/'):
            logger.error(f"Invalid filename requested: {filename}")
            return "", 404
            
        # Look in the eleven_labs_audio subdirectory
        audio_dir = os.path.join("/tmp", "eleven_labs_audio")
        audio_path = os.path.join(audio_dir, filename)
        
        logger.debug(f"Attempting to serve audio from: {audio_path}")
        
        if not os.path.exists(audio_path):
            logger.error(f"Audio file not found: {audio_path}")
            return "", 404
            
        if not os.path.isfile(audio_path):
            logger.error(f"Path exists but is not a file: {audio_path}")
            return "", 404
            
        return send_file(
            audio_path,
            mimetype="audio/mpeg",
            as_attachment=False,
            download_name=filename
        )
    except Exception as e:
        logger.error(f"Error serving audio file: {str(e)}")
        return "", 404

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=5001, debug=True)