using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Nop.Plugin.Misc.FruitBankPlugin.Domains.DataLayer; using Nop.Plugin.Misc.FruitBankPlugin.Services; using Nop.Services.Catalog; using Nop.Services.Customers; using Nop.Services.Security; using Nop.Web.Framework; using Nop.Web.Framework.Controllers; using Nop.Web.Framework.Mvc.Filters; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; namespace Nop.Plugin.Misc.FruitBankPlugin.Areas.Admin.Controllers { [AuthorizeAdmin] [Area(AreaNames.ADMIN)] [AutoValidateAntiforgeryToken] public class VoiceOrderController : BasePluginController { private readonly IPermissionService _permissionService; private readonly OpenAIApiService _aiApiService; private readonly ICustomerService _customerService; private readonly IProductService _productService; private readonly FruitBankDbContext _dbContext; public VoiceOrderController( IPermissionService permissionService, OpenAIApiService aiApiService, ICustomerService customerService, IProductService productService, FruitBankDbContext dbContext) { _permissionService = permissionService; _aiApiService = aiApiService; _customerService = customerService; _productService = productService; _dbContext = dbContext; } /// /// Display the voice order creation page /// [HttpGet] public async Task Create() { if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL)) return AccessDeniedView(); return View("~/Plugins/Misc.FruitBankPlugin/Areas/Admin/Views/VoiceOrder/Create.cshtml"); } /// /// Transcribe audio for partner name and return matching partners /// [HttpPost] public async Task TranscribeForPartner(IFormFile audioFile) { if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL)) return Json(new { success = false, message = "Access denied" }); if (audioFile == null || audioFile.Length == 0) { return Json(new { success = false, message = "No audio file received" }); } try { // Transcribe audio var transcribedText = await TranscribeAudioFile(audioFile, "hu"); if (string.IsNullOrEmpty(transcribedText)) { return Json(new { success = false, message = "Failed to transcribe audio" }); } Console.WriteLine($"[VoiceOrder] Partner transcription: {transcribedText}"); // Search for matching partners var partners = await SearchPartners(transcribedText); return Json(new { success = true, transcription = transcribedText, partners = partners }); } catch (Exception ex) { Console.WriteLine($"[VoiceOrder] Error in TranscribeForPartner: {ex.Message}"); return Json(new { success = false, message = $"Error: {ex.Message}" }); } } /// /// Transcribe audio for products and parse quantities using AI /// [HttpPost] public async Task TranscribeForProducts(IFormFile audioFile) { if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL)) return Json(new { success = false, message = "Access denied" }); if (audioFile == null || audioFile.Length == 0) { return Json(new { success = false, message = "No audio file received" }); } try { // Transcribe audio var transcribedText = await TranscribeAudioFile(audioFile, "hu"); if (string.IsNullOrEmpty(transcribedText)) { return Json(new { success = false, message = "Failed to transcribe audio" }); } Console.WriteLine($"[VoiceOrder] Product transcription: {transcribedText}"); // Parse products and quantities using AI var parsedProducts = await ParseProductsFromText(transcribedText); if (parsedProducts == null || parsedProducts.Count == 0) { return Json(new { success = false, message = "Could not parse products from transcription", transcription = transcribedText }); } // Enrich with actual product data from database var enrichedProducts = await EnrichProductData(parsedProducts); return Json(new { success = true, transcription = transcribedText, products = enrichedProducts }); } catch (Exception ex) { Console.WriteLine($"[VoiceOrder] Error in TranscribeForProducts: {ex.Message}"); return Json(new { success = false, message = $"Error: {ex.Message}" }); } } #region Helper Methods /// /// Transcribe audio file using OpenAI Whisper /// private async Task TranscribeAudioFile(IFormFile audioFile, string language) { var fileName = $"voice_order_{DateTime.Now:yyyyMMdd_HHmmss}.webm"; var uploadsFolder = Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "uploads", "voice"); if (!Directory.Exists(uploadsFolder)) { Directory.CreateDirectory(uploadsFolder); } var filePath = Path.Combine(uploadsFolder, fileName); // Save file temporarily using (var stream = new FileStream(filePath, FileMode.Create)) { await audioFile.CopyToAsync(stream); } // Transcribe using OpenAI Whisper string transcribedText; using (var audioStream = new FileStream(filePath, FileMode.Open, FileAccess.Read)) { transcribedText = await _aiApiService.TranscribeAudioAsync(audioStream, fileName, language); } // Clean up temporary file try { System.IO.File.Delete(filePath); } catch { /* Ignore cleanup errors */ } return transcribedText; } /// /// Search for partners matching the transcribed text /// Uses string-based search first, then semantic AI matching if needed /// private async Task> SearchPartners(string searchTerm) { const int maxResults = 10; const int minResultsForAI = 3; // If we get fewer than this, use AI semantic search Console.WriteLine($"[VoiceOrder] Searching partners for: {searchTerm}"); // Step 1: Try string-based search var customersByCompany = await _customerService.GetAllCustomersAsync( company: searchTerm, pageIndex: 0, pageSize: maxResults); var customersByName = await _customerService.GetAllCustomersAsync( firstName: searchTerm, pageIndex: 0, pageSize: maxResults); var customersByLastName = await _customerService.GetAllCustomersAsync( lastName: searchTerm, pageIndex: 0, pageSize: maxResults); // Combine and deduplicate var allCustomers = customersByCompany .Union(customersByName) .Union(customersByLastName) .DistinctBy(c => c.Id) .Take(maxResults) .ToList(); Console.WriteLine($"[VoiceOrder] String-based search found {allCustomers.Count} partners"); // Step 2: If we don't have enough results, use AI semantic matching if (allCustomers.Count < minResultsForAI) { Console.WriteLine("[VoiceOrder] Using AI semantic matching for partners"); var aiMatches = await SemanticPartnerSearch(searchTerm); // Merge AI matches with string matches, remove duplicates allCustomers = allCustomers .Union(aiMatches) .DistinctBy(c => c.Id) .Take(maxResults) .ToList(); Console.WriteLine($"[VoiceOrder] After AI matching: {allCustomers.Count} partners"); } // Format results var result = new List(); foreach (var customer in allCustomers) { var fullName = await _customerService.GetCustomerFullNameAsync(customer); var company = customer.Company; if (string.IsNullOrEmpty(fullName)) fullName = "[No name]"; if (string.IsNullOrEmpty(company)) company = "[No company]"; string label = $"{company} ({fullName}), {customer.Email}"; result.Add(new { label = label, value = customer.Id }); } return result; } /// /// Use AI to semantically match partner names based on company name /// private async Task> SemanticPartnerSearch(string searchTerm) { try { // Get all customers with company names (limit to reasonable number) var allCustomersWithCompany = await _customerService.GetAllCustomersAsync( pageIndex: 0, pageSize: 500); // Reasonable limit for AI processing // Filter to only those with company names var customersWithCompany = allCustomersWithCompany .Where(c => !string.IsNullOrEmpty(c.Company)) .ToList(); if (customersWithCompany.Count == 0) { Console.WriteLine("[VoiceOrder] No customers with company names found"); return new List(); } // Build company list for AI var companyList = string.Join("\n", customersWithCompany .Select((c, index) => $"{index}|{c.Company}")); var systemPrompt = @"You are a company name matcher for a B2B system. Given a spoken company name and a list of company names, find the 5 best semantic matches. RULES: 1. Consider phonetic similarity (how it sounds) 2. Consider abbreviations (e.g., 'SFI' matches 'SFI Rotterdam B.V.') 3. Consider partial matches (e.g., 'Rotterdam' matches 'SFI Rotterdam B.V.') 4. Consider common misspellings or mishearings 5. Return ONLY valid JSON array with indices, no explanations INPUT FORMAT: Search term: [spoken company name] Companies: [index]|[company name] (one per line) OUTPUT FORMAT (JSON only): [0, 15, 42, 103, 256] Return the top 5 indices that best match the search term. If fewer than 5 good matches exist, return fewer indices."; var userPrompt = $@"Search term: {searchTerm} Companies: {companyList}"; var aiResponse = await _aiApiService.GetSimpleResponseAsync(systemPrompt, userPrompt); Console.WriteLine($"[VoiceOrder] AI company matching response: {aiResponse}"); // Extract JSON array from response var jsonMatch = Regex.Match(aiResponse, @"\[[\d,\s]*\]", RegexOptions.Singleline); if (!jsonMatch.Success) { Console.WriteLine("[VoiceOrder] No JSON array found in AI response"); return new List(); } var jsonText = jsonMatch.Value; var indices = System.Text.Json.JsonSerializer.Deserialize>(jsonText); if (indices == null || indices.Count == 0) { Console.WriteLine("[VoiceOrder] AI returned no matches"); return new List(); } // Get customers by indices var matchedCustomers = indices .Where(i => i >= 0 && i < customersWithCompany.Count) .Select(i => customersWithCompany[i]) .ToList(); Console.WriteLine($"[VoiceOrder] AI matched {matchedCustomers.Count} companies: {string.Join(", ", matchedCustomers.Select(c => c.Company))}"); return matchedCustomers; } catch (Exception ex) { Console.WriteLine($"[VoiceOrder] Error in semantic partner search: {ex.Message}"); return new List(); } } /// /// Parse products and quantities from transcribed text using AI /// private async Task> ParseProductsFromText(string text) { var systemPrompt = @"You are a product parser for a Hungarian fruit wholesale company. Parse the product names and quantities from the user's speech. RULES: 1. Extract product names and quantities 2. Normalize product names to singular, lowercase (e.g., 'narancsok' → 'narancs') 3. Convert quantity units to standard format (kg, db, láda) 4. Handle Hungarian number words ('száz' = 100, 'ötven' = 50, etc.) 5. Return ONLY valid JSON array, no explanations OUTPUT FORMAT (JSON only): [ {""product"": ""narancs"", ""quantity"": 100, ""unit"": ""kg""}, {""product"": ""alma"", ""quantity"": 50, ""unit"": ""kg""} ] EXAMPLES: Input: 'narancs száz kilogram és alma ötven kiló' Output: [{""product"":""narancs"",""quantity"":100,""unit"":""kg""},{""product"":""alma"",""quantity"":50,""unit"":""kg""}] Input: 'Kérek 200 kg narancsot meg 150 kg almát' Output: [{""product"":""narancs"",""quantity"":200,""unit"":""kg""},{""product"":""alma"",""quantity"":150,""unit"":""kg""}]"; var userPrompt = $"Parse this: {text}"; var aiResponse = await _aiApiService.GetSimpleResponseAsync(systemPrompt, userPrompt); Console.WriteLine($"[VoiceOrder] AI Response: {aiResponse}"); // Try to extract JSON from response var jsonMatch = Regex.Match(aiResponse, @"\[.*\]", RegexOptions.Singleline); if (!jsonMatch.Success) { Console.WriteLine("[VoiceOrder] No JSON array found in AI response"); return new List(); } var jsonText = jsonMatch.Value; try { var parsedProducts = System.Text.Json.JsonSerializer.Deserialize>(jsonText); return parsedProducts ?? new List(); } catch (Exception ex) { Console.WriteLine($"[VoiceOrder] Error parsing JSON: {ex.Message}"); return new List(); } } /// /// Enrich parsed products with actual product data from database /// private async Task> EnrichProductData(List parsedProducts) { var enrichedProducts = new List(); foreach (var parsed in parsedProducts) { // Search for matching product in database var products = await _productService.SearchProductsAsync( keywords: parsed.Product, pageIndex: 0, pageSize: 5); if (!products.Any()) { Console.WriteLine($"[VoiceOrder] No product found for: {parsed.Product}"); continue; } // Take the best match (first result) var product = products.First(); var productDto = await _dbContext.ProductDtos.GetByIdAsync(product.Id); if (productDto == null) { Console.WriteLine($"[VoiceOrder] ProductDto not found for product ID: {product.Id}"); continue; } // Check if enough stock var availableQuantity = product.StockQuantity + productDto.IncomingQuantity; if (availableQuantity <= 0) { Console.WriteLine($"[VoiceOrder] Product {product.Name} has no stock"); continue; } // Add to enriched list enrichedProducts.Add(new { id = product.Id, name = product.Name, sku = product.Sku, quantity = parsed.Quantity, unit = parsed.Unit, price = product.Price, stockQuantity = availableQuantity }); } return enrichedProducts; } #endregion #region Models private class ParsedProduct { public string Product { get; set; } public int Quantity { get; set; } public string Unit { get; set; } } #endregion } }