using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Nop.Plugin.Misc.FruitBankPlugin.Domains.DataLayer; using Nop.Plugin.Misc.FruitBankPlugin.Services; using Nop.Services.Catalog; using Nop.Services.Customers; using Nop.Services.Security; using Nop.Web.Framework; using Nop.Web.Framework.Controllers; using Nop.Web.Framework.Mvc.Filters; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; namespace Nop.Plugin.Misc.FruitBankPlugin.Areas.Admin.Controllers { [AuthorizeAdmin] [Area(AreaNames.ADMIN)] [AutoValidateAntiforgeryToken] public class VoiceOrderController : BasePluginController { private readonly IPermissionService _permissionService; private readonly OpenAIApiService _aiApiService; private readonly ICustomerService _customerService; private readonly IProductService _productService; private readonly FruitBankDbContext _dbContext; public VoiceOrderController( IPermissionService permissionService, OpenAIApiService aiApiService, ICustomerService customerService, IProductService productService, FruitBankDbContext dbContext) { _permissionService = permissionService; _aiApiService = aiApiService; _customerService = customerService; _productService = productService; _dbContext = dbContext; } /// /// Display the voice order creation page /// [HttpGet] public async Task Create() { if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL)) return AccessDeniedView(); return View("~/Plugins/Misc.FruitBankPlugin/Areas/Admin/Views/VoiceOrder/Create.cshtml"); } /// /// Transcribe audio for partner name and return matching partners /// [HttpPost] public async Task TranscribeForPartner(IFormFile audioFile) { if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL)) return Json(new { success = false, message = "Access denied" }); if (audioFile == null || audioFile.Length == 0) { return Json(new { success = false, message = "No audio file received" }); } try { // Build partner names prompt for Whisper vocabulary hints // Whisper has a 224 character limit, so extract unique KEYWORDS instead of full names var allCustomers = await _customerService.GetAllCustomersAsync(pageIndex: 0, pageSize: 300); var companyNames = allCustomers .Where(c => !string.IsNullOrEmpty(c.Company)) .Select(c => c.Company.Trim()) .Distinct() .ToList(); // Extract unique keywords from company names var keywords = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var company in companyNames) { // Split by common separators and extract meaningful words var words = company.Split(new[] { ' ', ',', '.', '-', '/', '(', ')' }, StringSplitOptions.RemoveEmptyEntries); foreach (var word in words) { var cleaned = word.Trim(); // Skip very short words, common abbreviations, and legal terms if (cleaned.Length < 3) continue; if (cleaned.Equals("BV", StringComparison.OrdinalIgnoreCase)) continue; if (cleaned.Equals("Ltd", StringComparison.OrdinalIgnoreCase)) continue; if (cleaned.Equals("Kft", StringComparison.OrdinalIgnoreCase)) continue; if (cleaned.Equals("Inc", StringComparison.OrdinalIgnoreCase)) continue; if (cleaned.Equals("GmbH", StringComparison.OrdinalIgnoreCase)) continue; if (cleaned.Equals("SRL", StringComparison.OrdinalIgnoreCase)) continue; keywords.Add(cleaned); } } // Build prompt from keywords, fitting as many as possible in 224 chars var keywordList = keywords.OrderBy(k => k.Length).ToList(); var promptParts = new List(); int currentLength = 0; const int maxLength = 220; foreach (var keyword in keywordList) { var toAdd = promptParts.Count == 0 ? keyword : ", " + keyword; if (currentLength + toAdd.Length > maxLength) break; promptParts.Add(keyword); currentLength += toAdd.Length; } var partnerPrompt = string.Join(", ", promptParts); Console.WriteLine($"[VoiceOrder] Whisper prompt with {promptParts.Count} keywords from {companyNames.Count} partners ({partnerPrompt.Length} chars)"); // Transcribe audio in HUNGARIAN with partner keywords as vocabulary hints var transcribedText = await TranscribeAudioFile(audioFile, "hu", partnerPrompt); if (string.IsNullOrEmpty(transcribedText)) { return Json(new { success = false, message = "Failed to transcribe audio" }); } Console.WriteLine($"[VoiceOrder] Partner transcription (HU): {transcribedText}"); // Search for matching partners var partners = await SearchPartners(transcribedText); return Json(new { success = true, transcription = transcribedText, partners = partners }); } catch (Exception ex) { Console.WriteLine($"[VoiceOrder] Error in TranscribeForPartner: {ex.Message}"); return Json(new { success = false, message = $"Error: {ex.Message}" }); } } /// /// Transcribe audio for products and parse quantities using AI /// [HttpPost] public async Task TranscribeForProducts(IFormFile audioFile) { if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL)) return Json(new { success = false, message = "Access denied" }); if (audioFile == null || audioFile.Length == 0) { return Json(new { success = false, message = "No audio file received" }); } try { // Transcribe audio in HUNGARIAN var transcribedText = await TranscribeAudioFile(audioFile, "hu"); if (string.IsNullOrEmpty(transcribedText)) { return Json(new { success = false, message = "Failed to transcribe audio" }); } Console.WriteLine($"[VoiceOrder] Product transcription (HU): {transcribedText}"); // Parse products and quantities using AI var parsedProducts = await ParseProductsFromText(transcribedText); if (parsedProducts == null || parsedProducts.Count == 0) { return Json(new { success = false, message = "Could not parse products from transcription", transcription = transcribedText }); } // Enrich with actual product data from database var enrichedProducts = await EnrichProductData(parsedProducts); return Json(new { success = true, transcription = transcribedText, products = enrichedProducts }); } catch (Exception ex) { Console.WriteLine($"[VoiceOrder] Error in TranscribeForProducts: {ex.Message}"); return Json(new { success = false, message = $"Error: {ex.Message}" }); } } /// /// Search for partners by manually typed text (no audio transcription needed) /// [HttpPost] public async Task SearchPartnerByText(string text) { if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL)) return Json(new { success = false, message = "Access denied" }); if (string.IsNullOrWhiteSpace(text)) { return Json(new { success = false, message = "No text provided" }); } try { Console.WriteLine($"[VoiceOrder] Manual partner search: {text}"); // Search for matching partners (same logic as voice) var partners = await SearchPartners(text); return Json(new { success = true, transcription = text, partners = partners }); } catch (Exception ex) { Console.WriteLine($"[VoiceOrder] Error in SearchPartnerByText: {ex.Message}"); return Json(new { success = false, message = $"Error: {ex.Message}" }); } } /// /// Parse manually typed product text (no audio transcription needed) /// [HttpPost] public async Task ParseManualProductText(string text) { if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL)) return Json(new { success = false, message = "Access denied" }); if (string.IsNullOrWhiteSpace(text)) { return Json(new { success = false, message = "No text provided" }); } try { Console.WriteLine($"[VoiceOrder] Manual product input: {text}"); // Parse products and quantities using AI (same as voice) var parsedProducts = await ParseProductsFromText(text); if (parsedProducts == null || parsedProducts.Count == 0) { return Json(new { success = false, message = "Could not parse products from text", transcription = text }); } // Enrich with actual product data from database var enrichedProducts = await EnrichProductData(parsedProducts); return Json(new { success = true, transcription = text, products = enrichedProducts }); } catch (Exception ex) { Console.WriteLine($"[VoiceOrder] Error in ParseManualProductText: {ex.Message}"); return Json(new { success = false, message = $"Error: {ex.Message}" }); } } #region Helper Methods /// /// Transcribe audio file using OpenAI Whisper /// private async Task TranscribeAudioFile(IFormFile audioFile, string language, string customPrompt = null) { var fileName = $"voice_order_{DateTime.Now:yyyyMMdd_HHmmss}.webm"; var uploadsFolder = Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "uploads", "voice"); if (!Directory.Exists(uploadsFolder)) { Directory.CreateDirectory(uploadsFolder); } var filePath = Path.Combine(uploadsFolder, fileName); // Save file temporarily using (var stream = new FileStream(filePath, FileMode.Create)) { await audioFile.CopyToAsync(stream); } // Transcribe using OpenAI Whisper string transcribedText; using (var audioStream = new FileStream(filePath, FileMode.Open, FileAccess.Read)) { transcribedText = await _aiApiService.TranscribeAudioAsync(audioStream, fileName, language, customPrompt); } // Clean up temporary file try { System.IO.File.Delete(filePath); } catch { /* Ignore cleanup errors */ } return transcribedText; } /// /// Search for partners matching the transcribed text /// Uses both string-based and AI semantic search for best results /// private async Task> SearchPartners(string searchTerm) { const int maxResults = 10; Console.WriteLine($"[VoiceOrder] Searching partners for: {searchTerm}"); // Step 1: String-based search (fast, catches exact matches) var customersByCompany = await _customerService.GetAllCustomersAsync( company: searchTerm, pageIndex: 0, pageSize: maxResults); var customersByName = await _customerService.GetAllCustomersAsync( firstName: searchTerm, pageIndex: 0, pageSize: maxResults); var customersByLastName = await _customerService.GetAllCustomersAsync( lastName: searchTerm, pageIndex: 0, pageSize: maxResults); // Combine string search results var stringResults = customersByCompany .Union(customersByName) .Union(customersByLastName) .DistinctBy(c => c.Id) .Take(maxResults) .ToList(); Console.WriteLine($"[VoiceOrder] String-based search found {stringResults.Count} partners"); // Step 2: ALWAYS use AI semantic search for better results Console.WriteLine("[VoiceOrder] Using AI semantic matching for partners"); var aiMatches = await SemanticPartnerSearch(searchTerm); Console.WriteLine($"[VoiceOrder] AI semantic search found {aiMatches.Count} partners"); // Step 3: Merge results - string matches first (exact), then AI matches var allCustomers = stringResults .Union(aiMatches) .DistinctBy(c => c.Id) .Take(maxResults) .ToList(); Console.WriteLine($"[VoiceOrder] Total unique partners: {allCustomers.Count}"); // Format results var result = new List(); foreach (var customer in allCustomers) { var fullName = await _customerService.GetCustomerFullNameAsync(customer); var company = customer.Company; if (string.IsNullOrEmpty(fullName)) fullName = "[No name]"; if (string.IsNullOrEmpty(company)) company = "[No company]"; string label = $"{company} ({fullName}), {customer.Email}"; result.Add(new { label = label, value = customer.Id }); } return result; } /// /// Use AI to semantically match partner names based on company name /// private async Task> SemanticPartnerSearch(string searchTerm) { try { // Get all customers with company names (increased limit) var allCustomersWithCompany = await _customerService.GetAllCustomersAsync( pageIndex: 0, pageSize: 1000); // Increased from 500 to catch more companies // Filter to only those with company names var customersWithCompany = allCustomersWithCompany .Where(c => !string.IsNullOrEmpty(c.Company)) .ToList(); if (customersWithCompany.Count == 0) { Console.WriteLine("[VoiceOrder] No customers with company names found"); return new List(); } Console.WriteLine($"[VoiceOrder] AI searching through {customersWithCompany.Count} companies"); // Build company list for AI var companyList = string.Join("\n", customersWithCompany .Select((c, index) => $"{index}|{c.Company}")); var systemPrompt = @"You are a company name matcher for a B2B system. Given a spoken company name and a list of company names, find the 5 best matches. CRITICAL MATCHING RULES (in priority order): 1. EXACT MATCH: If the search term appears exactly in a company name, prioritize it 2. SUBSTRING MATCH: If the search term is contained within a company name (e.g., 'Junket' in 'Junket Silver Kft.') 3. WORD MATCH: If all words from search term appear in company name (any order) 4. PARTIAL MATCH: If significant words overlap (e.g., 'Silver' matches 'Junket Silver') 5. PHONETIC SIMILARITY: How it sounds when spoken 6. ABBREVIATIONS: 'SFI' matches 'SFI Rotterdam B.V.' EXAMPLES: Search: 'Junket Silver' Should match: 'Junket Silver Kft.' (substring match - VERY HIGH PRIORITY) Search: 'Rotterdam' Should match: 'SFI Rotterdam B.V.' (substring match) Return ONLY a JSON array with the top 5 indices, ordered by best match first. If fewer than 5 matches exist, return fewer indices. OUTPUT FORMAT (JSON only): [0, 15, 42, 103, 256]"; var userPrompt = $@"Search term: {searchTerm} Companies: {companyList}"; var aiResponse = await _aiApiService.GetSimpleResponseAsync(systemPrompt, userPrompt); Console.WriteLine($"[VoiceOrder] AI company matching response: {aiResponse}"); // Extract JSON array from response var jsonMatch = Regex.Match(aiResponse, @"\[[\d,\s]*\]", RegexOptions.Singleline); if (!jsonMatch.Success) { Console.WriteLine("[VoiceOrder] No JSON array found in AI response"); return new List(); } var jsonText = jsonMatch.Value; var indices = System.Text.Json.JsonSerializer.Deserialize>(jsonText); if (indices == null || indices.Count == 0) { Console.WriteLine("[VoiceOrder] AI returned no matches"); return new List(); } // Get customers by indices var matchedCustomers = indices .Where(i => i >= 0 && i < customersWithCompany.Count) .Select(i => customersWithCompany[i]) .ToList(); Console.WriteLine($"[VoiceOrder] AI matched {matchedCustomers.Count} companies: {string.Join(", ", matchedCustomers.Select(c => c.Company))}"); return matchedCustomers; } catch (Exception ex) { Console.WriteLine($"[VoiceOrder] Error in semantic partner search: {ex.Message}"); return new List(); } } /// /// Parse products and quantities from transcribed text using AI /// private async Task> ParseProductsFromText(string text) { var systemPrompt = @"You are a product parser for a Hungarian fruit and vegetable wholesale company. Parse the product names and quantities from the user's speech. CRITICAL RULES: 1. Extract product names and quantities from ANY produce item (fruits, vegetables, herbs, etc.) 2. Normalize product names to singular, lowercase (e.g., 'narancsok' → 'narancs', 'áfonyák' → 'áfonya') 3. Handle Hungarian number words ('száz' = 100, 'ötven' = 50, 'húsz' = 20, 'tíz' = 10, 'öt' = 5, 'egy' = 1, etc.) 4. FIX COMMON TRANSCRIPTION ERRORS: - 'datója' → 'datolya' (dates) - 'szűlő' → 'szőlő' (grapes) - 'mondarin' → 'mandarin' (mandarin) - 'paprika' is correct (pepper/paprika) - 'fokhagyma' is correct (garlic) - Any obvious typo → correct it 5. Return ONLY valid JSON array, no explanations or empty arrays 6. DO NOT include units - only product name and quantity as a number 7. ALWAYS return at least one product if you can parse anything from the input OUTPUT FORMAT (JSON only): [ {""product"": ""narancs"", ""quantity"": 100}, {""product"": ""alma"", ""quantity"": 50} ] EXAMPLES: Input: 'narancs száz kilogram és alma ötven kiló' Output: [{""product"":""narancs"",""quantity"":100},{""product"":""alma"",""quantity"":50}] Input: 'fokhagyma, 1' Output: [{""product"":""fokhagyma"",""quantity"":1}] Input: 'paprika, öt rekesz' Output: [{""product"":""paprika"",""quantity"":5}] Input: 'mondarin öt rekesz' (typo in 'mandarin') Output: [{""product"":""mandarin"",""quantity"":5}] Input: 'menta, 1' Output: [{""product"":""menta"",""quantity"":1}] Input: 'datója tíz láda' (WRONG transcription) Output: [{""product"":""datolya"",""quantity"":10}] Input: 'szűlő ötven kiló' (WRONG transcription) Output: [{""product"":""szőlő"",""quantity"":50}]"; var userPrompt = $"Parse this: {text}"; var aiResponse = await _aiApiService.GetSimpleResponseAsync(systemPrompt, userPrompt); Console.WriteLine($"[VoiceOrder] AI Response: {aiResponse}"); // Try to extract JSON from response var jsonMatch = Regex.Match(aiResponse, @"\[.*\]", RegexOptions.Singleline); if (!jsonMatch.Success) { Console.WriteLine("[VoiceOrder] No JSON array found in AI response"); return new List(); } var jsonText = jsonMatch.Value; try { var parsedProducts = System.Text.Json.JsonSerializer.Deserialize>(jsonText); return parsedProducts ?? new List(); } catch (Exception ex) { Console.WriteLine($"[VoiceOrder] Error parsing JSON: {ex.Message}"); return new List(); } } /// /// Enrich parsed products with actual product data from database /// Returns ALL matching products so admin can select the exact one /// private async Task> EnrichProductData(List parsedProducts) { var enrichedProducts = new List(); // OPTIMIZATION: Load all ProductDtos once instead of querying one by one var helperProductDtos = await _dbContext.ProductDtos.GetAll(true).ToListAsync(); foreach (var parsed in parsedProducts) { // Search for ALL matching products in database var products = await _productService.SearchProductsAsync( keywords: parsed.Product, pageIndex: 0, pageSize: 50); // Get up to 20 matches if (!products.Any()) { Console.WriteLine($"[VoiceOrder] No product found for: {parsed.Product}"); continue; } Console.WriteLine($"[VoiceOrder] Found {products.Count()} products matching '{parsed.Product}'"); // Add ALL matching products for admin to choose from foreach (var product in products) { var productDto = helperProductDtos.FirstOrDefault(x => x.Id == product.Id); if (productDto == null) { Console.WriteLine($"[VoiceOrder] ProductDto not found for product ID: {product.Id}"); continue; } // Check if enough stock var availableQuantity = product.StockQuantity + productDto.IncomingQuantity; if (availableQuantity <= 0) { Console.WriteLine($"[VoiceOrder] Product {product.Name} has no stock - skipping"); continue; } // Validate requested quantity against available stock var requestedQuantity = parsed.Quantity; var finalQuantity = requestedQuantity; var isQuantityReduced = false; if (requestedQuantity > availableQuantity) { Console.WriteLine($"[VoiceOrder] WARNING: Product {product.Name} - Requested {requestedQuantity} but only {availableQuantity} available. Capping to available."); finalQuantity = availableQuantity; isQuantityReduced = true; } // Add to enriched list with validated quantity enrichedProducts.Add(new { id = product.Id, name = product.Name, sku = product.Sku, quantity = finalQuantity, // Use validated quantity (capped to available) requestedQuantity = requestedQuantity, // Original requested amount price = product.Price, stockQuantity = availableQuantity, searchTerm = parsed.Product, // Track what was searched for isQuantityReduced = isQuantityReduced // Flag if we had to reduce }); } } Console.WriteLine($"[VoiceOrder] Total enriched products to display: {enrichedProducts.Count}"); return enrichedProducts; } #endregion #region Models private class ParsedProduct { [System.Text.Json.Serialization.JsonPropertyName("product")] public string Product { get; set; } [System.Text.Json.Serialization.JsonPropertyName("quantity")] public int Quantity { get; set; } } #endregion } }