Mango.Nop.Plugins/Nop.Plugin.Misc.AIPlugin/Areas/Admin/Controllers/VoiceOrderController.cs

674 lines
27 KiB
C#

using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Nop.Plugin.Misc.FruitBankPlugin.Domains.DataLayer;
using Nop.Plugin.Misc.FruitBankPlugin.Services;
using Nop.Services.Catalog;
using Nop.Services.Customers;
using Nop.Services.Security;
using Nop.Web.Framework;
using Nop.Web.Framework.Controllers;
using Nop.Web.Framework.Mvc.Filters;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace Nop.Plugin.Misc.FruitBankPlugin.Areas.Admin.Controllers
{
[AuthorizeAdmin]
[Area(AreaNames.ADMIN)]
[AutoValidateAntiforgeryToken]
public class VoiceOrderController : BasePluginController
{
private readonly IPermissionService _permissionService;
private readonly OpenAIApiService _aiApiService;
private readonly ICustomerService _customerService;
private readonly IProductService _productService;
private readonly FruitBankDbContext _dbContext;
public VoiceOrderController(
IPermissionService permissionService,
OpenAIApiService aiApiService,
ICustomerService customerService,
IProductService productService,
FruitBankDbContext dbContext)
{
_permissionService = permissionService;
_aiApiService = aiApiService;
_customerService = customerService;
_productService = productService;
_dbContext = dbContext;
}
/// <summary>
/// Display the voice order creation page
/// </summary>
[HttpGet]
public async Task<IActionResult> Create()
{
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
return AccessDeniedView();
return View("~/Plugins/Misc.FruitBankPlugin/Areas/Admin/Views/VoiceOrder/Create.cshtml");
}
/// <summary>
/// Transcribe audio for partner name and return matching partners
/// </summary>
[HttpPost]
public async Task<IActionResult> TranscribeForPartner(IFormFile audioFile)
{
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
return Json(new { success = false, message = "Access denied" });
if (audioFile == null || audioFile.Length == 0)
{
return Json(new { success = false, message = "No audio file received" });
}
try
{
// Build partner names prompt for Whisper vocabulary hints
// Whisper has a 224 character limit, so extract unique KEYWORDS instead of full names
var allCustomers = await _customerService.GetAllCustomersAsync(pageIndex: 0, pageSize: 300);
var companyNames = allCustomers
.Where(c => !string.IsNullOrEmpty(c.Company))
.Select(c => c.Company.Trim())
.Distinct()
.ToList();
// Extract unique keywords from company names
var keywords = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var company in companyNames)
{
// Split by common separators and extract meaningful words
var words = company.Split(new[] { ' ', ',', '.', '-', '/', '(', ')' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var word in words)
{
var cleaned = word.Trim();
// Skip very short words, common abbreviations, and legal terms
if (cleaned.Length < 3) continue;
if (cleaned.Equals("BV", StringComparison.OrdinalIgnoreCase)) continue;
if (cleaned.Equals("Ltd", StringComparison.OrdinalIgnoreCase)) continue;
if (cleaned.Equals("Kft", StringComparison.OrdinalIgnoreCase)) continue;
if (cleaned.Equals("Inc", StringComparison.OrdinalIgnoreCase)) continue;
if (cleaned.Equals("GmbH", StringComparison.OrdinalIgnoreCase)) continue;
if (cleaned.Equals("SRL", StringComparison.OrdinalIgnoreCase)) continue;
keywords.Add(cleaned);
}
}
// Build prompt from keywords, fitting as many as possible in 224 chars
var keywordList = keywords.OrderBy(k => k.Length).ToList();
var promptParts = new List<string>();
int currentLength = 0;
const int maxLength = 220;
foreach (var keyword in keywordList)
{
var toAdd = promptParts.Count == 0 ? keyword : ", " + keyword;
if (currentLength + toAdd.Length > maxLength)
break;
promptParts.Add(keyword);
currentLength += toAdd.Length;
}
var partnerPrompt = string.Join(", ", promptParts);
Console.WriteLine($"[VoiceOrder] Whisper prompt with {promptParts.Count} keywords from {companyNames.Count} partners ({partnerPrompt.Length} chars)");
// Transcribe audio in HUNGARIAN with partner keywords as vocabulary hints
var transcribedText = await TranscribeAudioFile(audioFile, "hu", partnerPrompt);
if (string.IsNullOrEmpty(transcribedText))
{
return Json(new { success = false, message = "Failed to transcribe audio" });
}
Console.WriteLine($"[VoiceOrder] Partner transcription (HU): {transcribedText}");
// Search for matching partners
var partners = await SearchPartners(transcribedText);
return Json(new
{
success = true,
transcription = transcribedText,
partners = partners
});
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error in TranscribeForPartner: {ex.Message}");
return Json(new { success = false, message = $"Error: {ex.Message}" });
}
}
/// <summary>
/// Transcribe audio for products and parse quantities using AI
/// </summary>
[HttpPost]
public async Task<IActionResult> TranscribeForProducts(IFormFile audioFile)
{
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
return Json(new { success = false, message = "Access denied" });
if (audioFile == null || audioFile.Length == 0)
{
return Json(new { success = false, message = "No audio file received" });
}
try
{
// Transcribe audio in HUNGARIAN
var transcribedText = await TranscribeAudioFile(audioFile, "hu");
if (string.IsNullOrEmpty(transcribedText))
{
return Json(new { success = false, message = "Failed to transcribe audio" });
}
Console.WriteLine($"[VoiceOrder] Product transcription (HU): {transcribedText}");
// Parse products and quantities using AI
var parsedProducts = await ParseProductsFromText(transcribedText);
if (parsedProducts == null || parsedProducts.Count == 0)
{
return Json(new
{
success = false,
message = "Could not parse products from transcription",
transcription = transcribedText
});
}
// Enrich with actual product data from database
var enrichedProducts = await EnrichProductData(parsedProducts);
return Json(new
{
success = true,
transcription = transcribedText,
products = enrichedProducts
});
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error in TranscribeForProducts: {ex.Message}");
return Json(new { success = false, message = $"Error: {ex.Message}" });
}
}
/// <summary>
/// Search for partners by manually typed text (no audio transcription needed)
/// </summary>
[HttpPost]
public async Task<IActionResult> SearchPartnerByText(string text)
{
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
return Json(new { success = false, message = "Access denied" });
if (string.IsNullOrWhiteSpace(text))
{
return Json(new { success = false, message = "No text provided" });
}
try
{
Console.WriteLine($"[VoiceOrder] Manual partner search: {text}");
// Search for matching partners (same logic as voice)
var partners = await SearchPartners(text);
return Json(new
{
success = true,
transcription = text,
partners = partners
});
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error in SearchPartnerByText: {ex.Message}");
return Json(new { success = false, message = $"Error: {ex.Message}" });
}
}
/// <summary>
/// Parse manually typed product text (no audio transcription needed)
/// </summary>
[HttpPost]
public async Task<IActionResult> ParseManualProductText(string text)
{
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
return Json(new { success = false, message = "Access denied" });
if (string.IsNullOrWhiteSpace(text))
{
return Json(new { success = false, message = "No text provided" });
}
try
{
Console.WriteLine($"[VoiceOrder] Manual product input: {text}");
// Parse products and quantities using AI (same as voice)
var parsedProducts = await ParseProductsFromText(text);
if (parsedProducts == null || parsedProducts.Count == 0)
{
return Json(new
{
success = false,
message = "Could not parse products from text",
transcription = text
});
}
// Enrich with actual product data from database
var enrichedProducts = await EnrichProductData(parsedProducts);
return Json(new
{
success = true,
transcription = text,
products = enrichedProducts
});
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error in ParseManualProductText: {ex.Message}");
return Json(new { success = false, message = $"Error: {ex.Message}" });
}
}
#region Helper Methods
/// <summary>
/// Transcribe audio file using OpenAI Whisper
/// </summary>
private async Task<string> TranscribeAudioFile(IFormFile audioFile, string language, string customPrompt = null)
{
var fileName = $"voice_order_{DateTime.Now:yyyyMMdd_HHmmss}.webm";
var uploadsFolder = Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "uploads", "voice");
if (!Directory.Exists(uploadsFolder))
{
Directory.CreateDirectory(uploadsFolder);
}
var filePath = Path.Combine(uploadsFolder, fileName);
// Save file temporarily
using (var stream = new FileStream(filePath, FileMode.Create))
{
await audioFile.CopyToAsync(stream);
}
// Transcribe using OpenAI Whisper
string transcribedText;
using (var audioStream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
transcribedText = await _aiApiService.TranscribeAudioAsync(audioStream, fileName, language, customPrompt);
}
// Clean up temporary file
try
{
System.IO.File.Delete(filePath);
}
catch { /* Ignore cleanup errors */ }
return transcribedText;
}
/// <summary>
/// Search for partners matching the transcribed text
/// Uses both string-based and AI semantic search for best results
/// </summary>
private async Task<List<object>> SearchPartners(string searchTerm)
{
const int maxResults = 10;
Console.WriteLine($"[VoiceOrder] Searching partners for: {searchTerm}");
// Step 1: String-based search (fast, catches exact matches)
var customersByCompany = await _customerService.GetAllCustomersAsync(
company: searchTerm,
pageIndex: 0,
pageSize: maxResults);
var customersByName = await _customerService.GetAllCustomersAsync(
firstName: searchTerm,
pageIndex: 0,
pageSize: maxResults);
var customersByLastName = await _customerService.GetAllCustomersAsync(
lastName: searchTerm,
pageIndex: 0,
pageSize: maxResults);
// Combine string search results
var stringResults = customersByCompany
.Union(customersByName)
.Union(customersByLastName)
.DistinctBy(c => c.Id)
.Take(maxResults)
.ToList();
Console.WriteLine($"[VoiceOrder] String-based search found {stringResults.Count} partners");
// Step 2: ALWAYS use AI semantic search for better results
Console.WriteLine("[VoiceOrder] Using AI semantic matching for partners");
var aiMatches = await SemanticPartnerSearch(searchTerm);
Console.WriteLine($"[VoiceOrder] AI semantic search found {aiMatches.Count} partners");
// Step 3: Merge results - string matches first (exact), then AI matches
var allCustomers = stringResults
.Union(aiMatches)
.DistinctBy(c => c.Id)
.Take(maxResults)
.ToList();
Console.WriteLine($"[VoiceOrder] Total unique partners: {allCustomers.Count}");
// Format results
var result = new List<object>();
foreach (var customer in allCustomers)
{
var fullName = await _customerService.GetCustomerFullNameAsync(customer);
var company = customer.Company;
if (string.IsNullOrEmpty(fullName))
fullName = "[No name]";
if (string.IsNullOrEmpty(company))
company = "[No company]";
string label = $"{company} ({fullName}), {customer.Email}";
result.Add(new
{
label = label,
value = customer.Id
});
}
return result;
}
/// <summary>
/// Use AI to semantically match partner names based on company name
/// </summary>
private async Task<List<Nop.Core.Domain.Customers.Customer>> SemanticPartnerSearch(string searchTerm)
{
try
{
// Get all customers with company names (increased limit)
var allCustomersWithCompany = await _customerService.GetAllCustomersAsync(
pageIndex: 0,
pageSize: 1000); // Increased from 500 to catch more companies
// Filter to only those with company names
var customersWithCompany = allCustomersWithCompany
.Where(c => !string.IsNullOrEmpty(c.Company))
.ToList();
if (customersWithCompany.Count == 0)
{
Console.WriteLine("[VoiceOrder] No customers with company names found");
return new List<Nop.Core.Domain.Customers.Customer>();
}
Console.WriteLine($"[VoiceOrder] AI searching through {customersWithCompany.Count} companies");
// Build company list for AI
var companyList = string.Join("\n", customersWithCompany
.Select((c, index) => $"{index}|{c.Company}"));
var systemPrompt = @"You are a company name matcher for a B2B system.
Given a spoken company name and a list of company names, find the 5 best matches.
CRITICAL MATCHING RULES (in priority order):
1. EXACT MATCH: If the search term appears exactly in a company name, prioritize it
2. SUBSTRING MATCH: If the search term is contained within a company name (e.g., 'Junket' in 'Junket Silver Kft.')
3. WORD MATCH: If all words from search term appear in company name (any order)
4. PARTIAL MATCH: If significant words overlap (e.g., 'Silver' matches 'Junket Silver')
5. PHONETIC SIMILARITY: How it sounds when spoken
6. ABBREVIATIONS: 'SFI' matches 'SFI Rotterdam B.V.'
EXAMPLES:
Search: 'Junket Silver'
Should match: 'Junket Silver Kft.' (substring match - VERY HIGH PRIORITY)
Search: 'Rotterdam'
Should match: 'SFI Rotterdam B.V.' (substring match)
Return ONLY a JSON array with the top 5 indices, ordered by best match first.
If fewer than 5 matches exist, return fewer indices.
OUTPUT FORMAT (JSON only):
[0, 15, 42, 103, 256]";
var userPrompt = $@"Search term: {searchTerm}
Companies:
{companyList}";
var aiResponse = await _aiApiService.GetSimpleResponseAsync(systemPrompt, userPrompt);
Console.WriteLine($"[VoiceOrder] AI company matching response: {aiResponse}");
// Extract JSON array from response
var jsonMatch = Regex.Match(aiResponse, @"\[[\d,\s]*\]", RegexOptions.Singleline);
if (!jsonMatch.Success)
{
Console.WriteLine("[VoiceOrder] No JSON array found in AI response");
return new List<Nop.Core.Domain.Customers.Customer>();
}
var jsonText = jsonMatch.Value;
var indices = System.Text.Json.JsonSerializer.Deserialize<List<int>>(jsonText);
if (indices == null || indices.Count == 0)
{
Console.WriteLine("[VoiceOrder] AI returned no matches");
return new List<Nop.Core.Domain.Customers.Customer>();
}
// Get customers by indices
var matchedCustomers = indices
.Where(i => i >= 0 && i < customersWithCompany.Count)
.Select(i => customersWithCompany[i])
.ToList();
Console.WriteLine($"[VoiceOrder] AI matched {matchedCustomers.Count} companies: {string.Join(", ", matchedCustomers.Select(c => c.Company))}");
return matchedCustomers;
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error in semantic partner search: {ex.Message}");
return new List<Nop.Core.Domain.Customers.Customer>();
}
}
/// <summary>
/// Parse products and quantities from transcribed text using AI
/// </summary>
private async Task<List<ParsedProduct>> ParseProductsFromText(string text)
{
var systemPrompt = @"You are a product parser for a Hungarian fruit and vegetable wholesale company.
Parse the product names and quantities from the user's speech.
CRITICAL RULES:
1. Extract product names and quantities from ANY produce item (fruits, vegetables, herbs, etc.)
2. Normalize product names to singular, lowercase (e.g., 'narancsok' 'narancs', 'áfonyák' 'áfonya')
3. Handle Hungarian number words ('száz' = 100, 'ötven' = 50, 'húsz' = 20, 'tíz' = 10, 'öt' = 5, 'egy' = 1, etc.)
4. FIX COMMON TRANSCRIPTION ERRORS:
- 'datója' 'datolya' (dates)
- 'szűlő' 'szőlő' (grapes)
- 'mondarin' 'mandarin' (mandarin)
- 'paprika' is correct (pepper/paprika)
- 'fokhagyma' is correct (garlic)
- Any obvious typo correct it
5. Return ONLY valid JSON array, no explanations or empty arrays
6. DO NOT include units - only product name and quantity as a number
7. ALWAYS return at least one product if you can parse anything from the input
OUTPUT FORMAT (JSON only):
[
{""product"": ""narancs"", ""quantity"": 100},
{""product"": ""alma"", ""quantity"": 50}
]
EXAMPLES:
Input: 'narancs száz kilogram és alma ötven kiló'
Output: [{""product"":""narancs"",""quantity"":100},{""product"":""alma"",""quantity"":50}]
Input: 'fokhagyma, 1'
Output: [{""product"":""fokhagyma"",""quantity"":1}]
Input: 'paprika, öt rekesz'
Output: [{""product"":""paprika"",""quantity"":5}]
Input: 'mondarin öt rekesz' (typo in 'mandarin')
Output: [{""product"":""mandarin"",""quantity"":5}]
Input: 'menta, 1'
Output: [{""product"":""menta"",""quantity"":1}]
Input: 'datója tíz láda' (WRONG transcription)
Output: [{""product"":""datolya"",""quantity"":10}]
Input: 'szűlő ötven kiló' (WRONG transcription)
Output: [{""product"":""szőlő"",""quantity"":50}]";
var userPrompt = $"Parse this: {text}";
var aiResponse = await _aiApiService.GetSimpleResponseAsync(systemPrompt, userPrompt);
Console.WriteLine($"[VoiceOrder] AI Response: {aiResponse}");
// Try to extract JSON from response
var jsonMatch = Regex.Match(aiResponse, @"\[.*\]", RegexOptions.Singleline);
if (!jsonMatch.Success)
{
Console.WriteLine("[VoiceOrder] No JSON array found in AI response");
return new List<ParsedProduct>();
}
var jsonText = jsonMatch.Value;
try
{
var parsedProducts = System.Text.Json.JsonSerializer.Deserialize<List<ParsedProduct>>(jsonText);
return parsedProducts ?? new List<ParsedProduct>();
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error parsing JSON: {ex.Message}");
return new List<ParsedProduct>();
}
}
/// <summary>
/// Enrich parsed products with actual product data from database
/// Returns ALL matching products so admin can select the exact one
/// </summary>
private async Task<List<object>> EnrichProductData(List<ParsedProduct> parsedProducts)
{
var enrichedProducts = new List<object>();
// OPTIMIZATION: Load all ProductDtos once instead of querying one by one
var helperProductDtos = await _dbContext.ProductDtos.GetAll(true).ToListAsync();
foreach (var parsed in parsedProducts)
{
// Search for ALL matching products in database
var products = await _productService.SearchProductsAsync(
keywords: parsed.Product,
pageIndex: 0,
pageSize: 20); // Get up to 20 matches
if (!products.Any())
{
Console.WriteLine($"[VoiceOrder] No product found for: {parsed.Product}");
continue;
}
Console.WriteLine($"[VoiceOrder] Found {products.Count()} products matching '{parsed.Product}'");
// Add ALL matching products for admin to choose from
foreach (var product in products)
{
var productDto = helperProductDtos.FirstOrDefault(x => x.Id == product.Id);
if (productDto == null)
{
Console.WriteLine($"[VoiceOrder] ProductDto not found for product ID: {product.Id}");
continue;
}
// Check if enough stock
var availableQuantity = product.StockQuantity + productDto.IncomingQuantity;
if (availableQuantity <= 0)
{
Console.WriteLine($"[VoiceOrder] Product {product.Name} has no stock - skipping");
continue;
}
// Validate requested quantity against available stock
var requestedQuantity = parsed.Quantity;
var finalQuantity = requestedQuantity;
var isQuantityReduced = false;
if (requestedQuantity > availableQuantity)
{
Console.WriteLine($"[VoiceOrder] WARNING: Product {product.Name} - Requested {requestedQuantity} but only {availableQuantity} available. Capping to available.");
finalQuantity = availableQuantity;
isQuantityReduced = true;
}
// Add to enriched list with validated quantity
enrichedProducts.Add(new
{
id = product.Id,
name = product.Name,
sku = product.Sku,
quantity = finalQuantity, // Use validated quantity (capped to available)
requestedQuantity = requestedQuantity, // Original requested amount
price = product.Price,
stockQuantity = availableQuantity,
searchTerm = parsed.Product, // Track what was searched for
isQuantityReduced = isQuantityReduced // Flag if we had to reduce
});
}
}
Console.WriteLine($"[VoiceOrder] Total enriched products to display: {enrichedProducts.Count}");
return enrichedProducts;
}
#endregion
#region Models
private class ParsedProduct
{
[System.Text.Json.Serialization.JsonPropertyName("product")]
public string Product { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("quantity")]
public int Quantity { get; set; }
}
#endregion
}
}