Project 2: Human-Robot Interaction

import streamlit as st
import json
import pyttsx3
import speech_recognition as sr
from openai import AzureOpenAI


# --- Human support trigger detection ---
_HUMAN_SUPPORT_KEYWORDS = [
    "human", "real person", "agent", "staff",
     "talk to someone", "help from person"
]

def is_human_request(text: str) -> bool:
    """Detect if user is asking for a real person or volunteer."""
    if not text:
        return False
    t = str(text).lower()
    return any(k in t for k in _HUMAN_SUPPORT_KEYWORDS)


def recognize_speech_from_mic():
    recognizer = sr.Recognizer()
    mic = sr.Microphone()
    with mic as source:
        st.info("🎙️ Listening... please speak now.")
        recognizer.adjust_for_ambient_noise(source)
        audio = recognizer.listen(source)
    try:
        text = recognizer.recognize_google(audio)
        st.success(f"🗣️ You said: {text}")
        return text
    except sr.UnknownValueError:
        st.warning("😅 Sorry, I couldn't understand.")
        return ""
    except sr.RequestError:
        st.error("Speech recognition failed.")
        return ""

def speak_text(text):
    try:
        engine = pyttsx3.init()
        engine.setProperty("rate", 170)
        engine.setProperty("volume", 1.0)
        voices = engine.getProperty("voices")
        for v in voices:
            if "english" in v.name.lower():
                engine.setProperty("voice", v.id)
                break
        engine.say(text)
        engine.runAndWait()
    except Exception as e:
        st.warning(f"Voice playback failed: {e}")

# =========================
# Pre-seeded volunteer guides (static, one-shot text)
# =========================
VOLUNTEER_GUIDES = {
    "scene_safety_check": {
        "title": "Scene Safety Check",
        "steps": [
            "Look for hazards first: falling debris, gas leaks, fire, unstable walls.",
            "Keep yourself safe. Do not enter if the area looks dangerous.",
            "Call for help if you see life-threatening danger you cannot control.",
            "If safe, approach calmly and introduce yourself before helping."
        ],
        "dos": [
            "Keep a safe distance from unstable structures.",
            "Use gloves/masks if available.",
            "Move slowly and speak clearly."
        ],
        "donts": [
            "Do not run into unsafe areas.",
            "Do not move heavy debris alone.",
            "Do not attempt specialized rescues."
        ],
        "emergency_signs": ["Active fire/gas smell", "Major structural collapse", "Electrical hazards"],
    },
    "bleeding_control": {
        "title": "Control External Bleeding (Basic)",
        "steps": [
            "Ask the person for consent if responsive; stay calm and explain.",
            "Apply firm, direct pressure with a clean cloth/bandage.",
            "If bleeding soaks through, add more cloths on top and keep pressure.",
            "Elevate the limb if it doesn't cause pain and there's no obvious deformity.",
            "If bleeding doesn't slow and you are not trained in tourniquets, keep firm pressure and seek help urgently."
        ],
        "dos": ["Wear gloves if you have them.", "Keep pressing continuously.", "Reassure the person and monitor breathing."],
        "donts": ["Do not remove soaked cloths.", "Do not use a tourniquet unless trained.", "Do not apply powders/unknown substances."],
        "emergency_signs": ["Soaking through rapidly", "Signs of shock (pale, cold, confusion)", "Breathing problems"],
    },
    # ... (additional guides omitted for brevity)
}

# =========================
# Pre-seeded rescue resources (static)
# =========================
RESCUE_RESOURCES = [
    {
        "category": "Food & Water",
        "location": "Central Stadium Distribution Point",
        "availability": "Available",
        "schedule": "Daily 09:00–18:00",
        "verified_by": "Red Cross",
        "verified_at": "2025-10-21",
        "notes": "Bring your ID or registration slip.",
        "coordinates": [22.3019, 114.1741],
        "tags": ["food", "water", "bottled", "meal", "drink"],
    },
    {
        "category": "Medical Aid",
        "location": "City Hospital East Wing",
        "availability": "Available",
        "schedule": "24/7 emergency desk",
        "verified_by": "Health Bureau",
        "verified_at": "2025-10-20",
        "notes": "Free treatment for registered survivors.",
        "coordinates": [22.3055, 114.1712],
        "tags": ["medical", "clinic", "hospital", "doctor", "aid"],
    },
    {
        "category": "Mental Health Support station",
        "location": "Hope Center Counseling Tent",
        "availability": "Available",
        "schedule": "Daily 10:00–19:00",
        "verified_by": "Psychological Aid Network",
        "verified_at": "2025-10-21",
        "notes": "Free counseling and stress-relief sessions.",
        "coordinates": [22.3124, 114.1672],
        "tags": ["mental", "psychological", "support", "counseling", "stress"],
    },
    # ... (additional resources omitted for brevity)
]

def search_resources(text: str):
    """Search resources based on user query keywords."""
    # Implementation details...
    pass

def format_resource_card(r: dict) -> str:
    """Format resource information as a card."""
    return (
        f"**{r['category']}**\n"
        f"📍 {r['location']}\n"
        f"🕘 {r['schedule']}\n"
        f"📦 Status: {r['availability']}\n"
        f"✅ Verified by {r['verified_by']} on {r['verified_at']}\n"
        f"📝 {r['notes']}"
    )

# =========================
# Page config + UI
# =========================
st.set_page_config(
    page_title="Post-earthquake Assistance",
    page_icon="🙆‍♀️",
    layout="centered",
    initial_sidebar_state="expanded",
)

# Language selector
if "lang" not in st.session_state:
    st.session_state["lang"] = "English"

lang = st.sidebar.selectbox(
    "Language / 语言 / 語言",
    ["English", "中文", "廣東話"],
    index=["English", "中文", "廣東話"].index(st.session_state["lang"])
)

# Input mode: Text or Voice
if "input_mode" not in st.session_state:
    st.session_state["input_mode"] = "Text"

st.sidebar.markdown("### Input mode")
st.sidebar.radio(
    "Choose how to talk to the bot:",
    ("Text", "Voice"),
    key="input_mode",
    horizontal=True
)

# Azure OpenAI setup
with st.sidebar:
    st.markdown("### API Key")
    openai_api_key = st.text_input("Azure OpenAI API Key", type="password")
    if st.button("Reset chat"):
        st.session_state.clear()
        st.rerun()

client = None
azure_endpoint = "https://hkust.azure-api.net/"
azure_deployment = "gpt-4o-mini"
api_version = "2025-02-01-preview"
if openai_api_key:
    client = AzureOpenAI(api_key=openai_api_key, api_version=api_version, azure_endpoint=azure_endpoint)

# =========================
# Chat state machine
# =========================
states = {
    "Greeting": {"next": "ExploreFeelings", "description": "Warmly welcome the user.", "collectedDataName": None},
    "ExploreFeelings": {"next": "AssessStressLevel", "description": "Encourage the user to share feelings.", "collectedDataName": "feelings"},
    "AssessStressLevel": {"next": "OfferCopingStrategies", "description": "Gently explore distress level.", "collectedDataName": "stressLevel"},
    "OfferCopingStrategies": {"next": "OfferResources", "description": "Provide coping ideas.", "collectedDataName": None},
    "OfferResources": {"next": "Unhandled", "description": "Share helpful resources.", "collectedDataName": None},
    "Unhandled": {"next": None, "description": "Handle unclear input.", "collectedDataName": None},
}

def create_model_prompt(user_content):
    """Create trauma-informed prompt for Azure OpenAI."""
    current_state = st.session_state["current_state"]
    state_description = states[current_state]["description"]
    next_s = states[current_state]["next"]
    next_desc = states[next_s]["description"] if next_s else state_description
    collected_data_json = json.dumps(st.session_state.get("user_data", {}))
    return f"""
    You are a trauma-informed, compassionate post-earthquake health support assistant for earthquake survivors and rescuers.
    Always begin with empathy. Respond ONLY as JSON: {{"isNextState": bool, "resp": string, "data": string}}.
    Current state: {current_state} ({state_description})
    Next state: {next_s} ({next_desc})
    Latest user message: {user_content}
    Data collected: {collected_data_json}
    """

# =========================
# Main chat loop
# =========================
# Initialize session state
if "current_state" not in st.session_state:
    st.session_state["current_state"] = "Greeting"
    st.session_state["user_data"] = {}
if "messages" not in st.session_state:
    initial_message = "Hi there, I'm a post-earthquake health support assistant, I am here to help you process your health and find calm after distressing experiences. You can share anything that's on your mind — I'll listen with care. 💛"
    st.session_state["messages"] = [{"role": "assistant", "content": {"resp": initial_message}}]

# Display chat history
for msg in st.session_state["messages"]:
    role = msg["role"]
    resp = msg["content"]["resp"]
    avatar = "🙆‍♀️" if role == "assistant" else "🙂"
    with st.chat_message(role, avatar=avatar):
        st.markdown(resp)

# Handle user input (text or voice)
user_resp = None
if st.session_state.get("input_mode") == "Voice":
    if st.button("🎤 Record Voice"):
        txt = recognize_speech_from_mic()
        if txt:
            user_resp = txt
else:
    if resp := st.chat_input("Share anything that's on your mind…"):
        user_resp = resp

# Process user input
if user_resp:
    # Check for human support request
    if is_human_request(user_resp):
        response = "Connecting you to a human volunteer... 🤝 Please wait a moment."
        st.session_state["messages"].append({"role": "assistant", "content": {"resp": response}})
        with st.chat_message("assistant", avatar="🙆‍♀️"):
            st.markdown(response)
    
    # Check for resource queries
    elif is_resource_query(user_resp):
        hits = search_resources(user_resp)
        if hits:
            resp_text = "I've found the following nearby support options:\n\n"
            resp_text += "\n\n".join(f"- {format_resource_card(r)}" for r in hits)
        else:
            resp_text = "I'm sorry—I don't have a matching resource right now. You can ask about *food/water/medical/shelter*."
        st.session_state["messages"].append({"role": "assistant", "content": {"resp": resp_text}})
    
    # Fall back to Azure OpenAI for empathetic conversation
    else:
        model_resp = get_response_from_model(client, azure_deployment)
        st.session_state["messages"].append({"role": "assistant", "content": model_resp})
        if st.session_state.get("input_mode") == "Voice":
            speak_text(model_resp["resp"])

st.markdown('
Your feelings matter. 🌿
', unsafe_allow_html=True)

Key Features Implemented:

• Multi-modal Interaction: Supports both text input and voice commands using speech recognition
• Trauma-informed Design: State machine with empathetic conversation flow tailored for disaster survivors
• Resource Database: Pre-seeded database of verified rescue resources (food, water, medical aid, shelter, mental health support)
• Volunteer Guidance System: Step-by-step safety protocols for untrained volunteers (scene safety, bleeding control, shock response, triage)
• Human Handoff: Keyword detection to connect users with real volunteers when needed
• Azure OpenAI Integration: GPT-4o-mini for natural language understanding and empathetic responses
• Multilingual Support: English, Simplified Chinese, and Cantonese language options
• Text-to-Speech: Voice output for accessibility in hands-free scenarios

My Role in Project 2

Throughout this Human-Robot Interaction project, I played a key role in user research, analysis, and design development. My contributions spanned from empathy research to final presentation, with particular focus on volunteer rescuer needs and guidance systems.

🎯 Key Achievements & Learning

🤝 Team Collaboration Acknowledgment

This project was a true team effort, and I want to acknowledge my teammates' contributions. WANG led the technical implementation, developing the working chatbot prototype and conducting comprehensive usability testing. The functional demo would not have been possible without WANG's coding expertise. He really carried the technical side of this project. Long contributed significantly throughout the entire project and did an outstanding job with video editing. His work on the video prototype and demo presentation was crucial in communicating our design effectively. The visual quality and storytelling in our video really came together because of his editing skills. Charles co-created the presentation materials with me, helping to structure our narrative and ensure our design decisions were clearly communicated during the in-class presentation. Looking back, our complementary skills really made this work. My focus on research and design, WANG's technical implementation, Long's video production, and Charles's presentation abilities created a well-rounded project that addressed both user needs and technical feasibility while communicating it effectively.