from flask import Flask, render_template_string, request, jsonify import speech_recognition as sr from tempfile import NamedTemporaryFile import os import ffmpeg from fuzzywuzzy import process, fuzz from metaphone import doublemetaphone import logging app = Flask(__name__) logging.basicConfig(level=logging.INFO) # Global variables cart = {} # Stores items and their quantities in the cart menu_preferences = "all" # Tracks the current menu preference default_menu_preferences = "all" # To reset menu preferences prices = { "samosa": 9, "onion pakoda": 10, "chilli gobi": 12, "chicken biryani": 14, "mutton biryani": 16, "veg biryani": 12, "paneer butter": 10, "fish curry": 12, "chicken manchurian": 14, "veg manchurian": 12, "chilli chicken": 14, "paneer biryani": 13, "chicken curry": 14 } menus = { "all": list(prices.keys()), "vegetarian": [ "samosa", "onion pakoda", "chilli gobi", "veg biryani", "paneer butter", "veg manchurian", "paneer biryani" ], "non-vegetarian": [ "chicken biryani", "mutton biryani", "fish curry", "chicken manchurian", "chilli chicken", "chicken curry" ], "guilt-free": ["samosa", "onion pakoda"] } @app.route("/") def index(): return render_template_string(html_code) @app.route("/reset-cart", methods=["GET"]) def reset_cart(): global cart, menu_preferences cart = {} menu_preferences = default_menu_preferences # Reset menu preferences return "Cart reset successfully." @app.route("/process-audio", methods=["POST"]) def process_audio(): try: # Handle audio input audio_file = request.files.get("audio") if not audio_file: return jsonify({"response": "No audio file provided."}), 400 # Save audio file and convert to WAV format temp_file = NamedTemporaryFile(delete=False, suffix=".webm") audio_file.save(temp_file.name) converted_file = NamedTemporaryFile(delete=False, suffix=".wav") ffmpeg.input(temp_file.name).output( converted_file.name, acodec="pcm_s16le", ac=1, ar="16000" ).run(overwrite_output=True) # Recognize speech recognizer = sr.Recognizer() recognizer.dynamic_energy_threshold = True recognizer.energy_threshold = 100 with sr.AudioFile(converted_file.name) as source: recognizer.adjust_for_ambient_noise(source, duration=1) audio_data = recognizer.record(source) raw_command = recognizer.recognize_google(audio_data).lower() logging.info(f"Raw recognized command: {raw_command}") # Preprocess the command all_menu_items = menus["all"] command = preprocess_command(raw_command, all_menu_items) # Process the command and get a response response = process_command(command) except sr.UnknownValueError: response = "Sorry, I couldn't understand. Please try again." except Exception as e: response = f"An error occurred: {str(e)}" finally: os.unlink(temp_file.name) os.unlink(converted_file.name) return jsonify({"response": response}) def preprocess_command(command, menu_items): """ Preprocess the user command to improve matching: - Use fuzzy matching for menu items. - Use phonetic matching as a fallback. """ def phonetic_match(word, options): word_phonetic = doublemetaphone(word)[0] for option in options: if doublemetaphone(option)[0] == word_phonetic: return option return None # First, try fuzzy matching closest_match = process.extractOne(command, menu_items, scorer=fuzz.token_set_ratio) if closest_match and closest_match[1] > 60: return closest_match[0] # Fallback to phonetic matching words = command.split() for word in words: match = phonetic_match(word, menu_items) if match: return match return command def process_command(command): """ Process the user command based on the current menu and cart. """ global cart, menu_preferences command = command.lower() # Handle menu preferences if menu_preferences == "all": if "non-vegetarian" in command: menu_preferences = "non-vegetarian" return "You have chosen the Non-Vegetarian menu. To view the menu, say 'menu'." elif "vegetarian" in command and "non-vegetarian" not in command: menu_preferences = "vegetarian" return "You have chosen the Vegetarian menu. To view the menu, say 'menu'." elif "guilt-free" in command: menu_preferences = "guilt-free" return "You have chosen the Guilt-Free menu. To view the menu, say 'menu'." elif "all" in command: menu_preferences = "all" return "You have chosen the complete menu. To view the menu, say 'menu'." # Handle commands related to the current menu menu = menus.get(menu_preferences, menus["all"]) if "price of" in command: item = command.replace("price of", "").strip() closest_match = process.extractOne(item, prices.keys(), scorer=fuzz.token_set_ratio) if closest_match and closest_match[1] > 60: matched_item = closest_match[0] return f"The price of {matched_item} is ${prices[matched_item]}." return "Sorry, I couldn't find that item in the menu." elif "menu" in command: return f"Here is your menu: {', '.join(menu)}. To select an item, say the item name." elif "remove" in command: item = command.replace("remove", "").strip() # Extract item name closest_match = process.extractOne(item, list(cart.keys()), scorer=fuzz.token_set_ratio) # Match in cart if closest_match and closest_match[1] > 60: # If match is strong matched_item = closest_match[0] # Check the specific remove command if "one" in command or "1" in command: if cart[matched_item] > 1: cart[matched_item] -= 1 return f"One {matched_item} has been removed. Current cart: {dict(cart)}." else: del cart[matched_item] return f"{matched_item.capitalize()} has been completely removed from your cart." elif "all" in command or "entire" in command: del cart[matched_item] return f"All {matched_item}s have been removed from your cart. Current cart: {dict(cart)}." else: del cart[matched_item] return f"{matched_item.capitalize()} has been removed from your cart. Current cart: {dict(cart)}." return "Sorry, that item is not in your cart." elif "clear cart" in command or "empty cart" in command: if cart: cart.clear() return "Your cart has been cleared." return "Your cart is already empty." elif "increase" in command: item = command.replace("increase", "").strip() closest_match = process.extractOne(item, list(cart.keys()), scorer=fuzz.token_set_ratio) if closest_match and closest_match[1] > 60: matched_item = closest_match[0] cart[matched_item] += 1 return f"Quantity of {matched_item} increased to {cart[matched_item]}. Current cart: {dict(cart)}." return "Sorry, that item is not in your cart." elif "decrease" in command: item = command.replace("decrease", "").strip() closest_match = process.extractOne(item, list(cart.keys()), scorer=fuzz.token_set_ratio) if closest_match and closest_match[1] > 60: matched_item = closest_match[0] if cart[matched_item] > 1: cart[matched_item] -= 1 return f"Quantity of {matched_item} decreased to {cart[matched_item]}. Current cart: {dict(cart)}." else: del cart[matched_item] return f"{matched_item.capitalize()} has been removed from your cart. Current cart: {dict(cart)}." return "Sorry, that item is not in your cart." elif any(item in command for item in menu): closest_match = process.extractOne(command, menu, scorer=fuzz.token_set_ratio) if closest_match and closest_match[1] > 60: matched_item = closest_match[0] cart[matched_item] = cart.get(matched_item, 0) + 1 return f"{matched_item.capitalize()} added to your cart. Current cart: {dict(cart)}. To finalize, say 'final order'." return "Sorry, I couldn't recognize the item. Could you try again?" elif "final order" in command: if cart: total = sum(prices[item] * count for item, count in cart.items()) response = f"Your final order is: {', '.join(f'{item} x{count}' for item, count in cart.items())}. Your total bill is ${total}. Thank you for ordering! To exit this conversation, say 'goodbye'." cart.clear() return response return "Your cart is empty. Please add items to your cart first." elif "goodbye" in command: cart.clear() menu_preferences = default_menu_preferences return "Goodbye! Thank you for using AI Dining Assistant." return "Sorry, I couldn't understand that. Please try again." html_code = """