Mango.Nop.Plugins/Nop.Plugin.Misc.AIPlugin/Areas/Admin/Controllers/VoiceOrderController.cs

674 lines
27 KiB
C#

using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Nop.Plugin.Misc.FruitBankPlugin.Domains.DataLayer;
using Nop.Plugin.Misc.FruitBankPlugin.Services;
using Nop.Services.Catalog;
using Nop.Services.Customers;
using Nop.Services.Security;
using Nop.Web.Framework;
using Nop.Web.Framework.Controllers;
using Nop.Web.Framework.Mvc.Filters;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
namespace Nop.Plugin.Misc.FruitBankPlugin.Areas.Admin.Controllers
{
[AuthorizeAdmin]
[Area(AreaNames.ADMIN)]
[AutoValidateAntiforgeryToken]
public class VoiceOrderController : BasePluginController
{
private readonly IPermissionService _permissionService;
private readonly OpenAIApiService _aiApiService;
private readonly ICustomerService _customerService;
private readonly IProductService _productService;
private readonly FruitBankDbContext _dbContext;
public VoiceOrderController(
IPermissionService permissionService,
OpenAIApiService aiApiService,
ICustomerService customerService,
IProductService productService,
FruitBankDbContext dbContext)
{
_permissionService = permissionService;
_aiApiService = aiApiService;
_customerService = customerService;
_productService = productService;
_dbContext = dbContext;
}
/// <summary>
/// Display the voice order creation page
/// </summary>
[HttpGet]
public async Task<IActionResult> Create()
{
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
return AccessDeniedView();
return View("~/Plugins/Misc.FruitBankPlugin/Areas/Admin/Views/VoiceOrder/Create.cshtml");
}
/// <summary>
/// Transcribe audio for partner name and return matching partners
/// </summary>
[HttpPost]
public async Task<IActionResult> TranscribeForPartner(IFormFile audioFile)
{
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
return Json(new { success = false, message = "Access denied" });
if (audioFile == null || audioFile.Length == 0)
{
return Json(new { success = false, message = "No audio file received" });
}
try
{
// Build partner names prompt for Whisper vocabulary hints
// Whisper has a 224 character limit, so extract unique KEYWORDS instead of full names
var allCustomers = await _customerService.GetAllCustomersAsync(pageIndex: 0, pageSize: 300);
var companyNames = allCustomers
.Where(c => !string.IsNullOrEmpty(c.Company))
.Select(c => c.Company.Trim())
.Distinct()
.ToList();
// Extract unique keywords from company names
var keywords = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var company in companyNames)
{
// Split by common separators and extract meaningful words
var words = company.Split(new[] { ' ', ',', '.', '-', '/', '(', ')' }, StringSplitOptions.RemoveEmptyEntries);
foreach (var word in words)
{
var cleaned = word.Trim();
// Skip very short words, common abbreviations, and legal terms
if (cleaned.Length < 3) continue;
if (cleaned.Equals("BV", StringComparison.OrdinalIgnoreCase)) continue;
if (cleaned.Equals("Ltd", StringComparison.OrdinalIgnoreCase)) continue;
if (cleaned.Equals("Kft", StringComparison.OrdinalIgnoreCase)) continue;
if (cleaned.Equals("Inc", StringComparison.OrdinalIgnoreCase)) continue;
if (cleaned.Equals("GmbH", StringComparison.OrdinalIgnoreCase)) continue;
if (cleaned.Equals("SRL", StringComparison.OrdinalIgnoreCase)) continue;
keywords.Add(cleaned);
}
}
// Build prompt from keywords, fitting as many as possible in 224 chars
var keywordList = keywords.OrderBy(k => k.Length).ToList();
var promptParts = new List<string>();
int currentLength = 0;
const int maxLength = 220;
foreach (var keyword in keywordList)
{
var toAdd = promptParts.Count == 0 ? keyword : ", " + keyword;
if (currentLength + toAdd.Length > maxLength)
break;
promptParts.Add(keyword);
currentLength += toAdd.Length;
}
var partnerPrompt = string.Join(", ", promptParts);
Console.WriteLine($"[VoiceOrder] Whisper prompt with {promptParts.Count} keywords from {companyNames.Count} partners ({partnerPrompt.Length} chars)");
// Transcribe audio in HUNGARIAN with partner keywords as vocabulary hints
var transcribedText = await TranscribeAudioFile(audioFile, "hu", partnerPrompt);
if (string.IsNullOrEmpty(transcribedText))
{
return Json(new { success = false, message = "Failed to transcribe audio" });
}
Console.WriteLine($"[VoiceOrder] Partner transcription (HU): {transcribedText}");
// Search for matching partners
var partners = await SearchPartners(transcribedText);
return Json(new
{
success = true,
transcription = transcribedText,
partners = partners
});
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error in TranscribeForPartner: {ex.Message}");
return Json(new { success = false, message = $"Error: {ex.Message}" });
}
}
/// <summary>
/// Transcribe audio for products and parse quantities using AI
/// </summary>
[HttpPost]
public async Task<IActionResult> TranscribeForProducts(IFormFile audioFile)
{
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
return Json(new { success = false, message = "Access denied" });
if (audioFile == null || audioFile.Length == 0)
{
return Json(new { success = false, message = "No audio file received" });
}
try
{
// Transcribe audio in HUNGARIAN
var transcribedText = await TranscribeAudioFile(audioFile, "hu");
if (string.IsNullOrEmpty(transcribedText))
{
return Json(new { success = false, message = "Failed to transcribe audio" });
}
Console.WriteLine($"[VoiceOrder] Product transcription (HU): {transcribedText}");
// Parse products and quantities using AI
var parsedProducts = await ParseProductsFromText(transcribedText);
if (parsedProducts == null || parsedProducts.Count == 0)
{
return Json(new
{
success = false,
message = "Could not parse products from transcription",
transcription = transcribedText
});
}
// Enrich with actual product data from database
var enrichedProducts = await EnrichProductData(parsedProducts);
return Json(new
{
success = true,
transcription = transcribedText,
products = enrichedProducts
});
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error in TranscribeForProducts: {ex.Message}");
return Json(new { success = false, message = $"Error: {ex.Message}" });
}
}
/// <summary>
/// Search for partners by manually typed text (no audio transcription needed)
/// </summary>
[HttpPost]
public async Task<IActionResult> SearchPartnerByText(string text)
{
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
return Json(new { success = false, message = "Access denied" });
if (string.IsNullOrWhiteSpace(text))
{
return Json(new { success = false, message = "No text provided" });
}
try
{
Console.WriteLine($"[VoiceOrder] Manual partner search: {text}");
// Search for matching partners (same logic as voice)
var partners = await SearchPartners(text);
return Json(new
{
success = true,
transcription = text,
partners = partners
});
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error in SearchPartnerByText: {ex.Message}");
return Json(new { success = false, message = $"Error: {ex.Message}" });
}
}
/// <summary>
/// Parse manually typed product text (no audio transcription needed)
/// </summary>
[HttpPost]
public async Task<IActionResult> ParseManualProductText(string text)
{
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
return Json(new { success = false, message = "Access denied" });
if (string.IsNullOrWhiteSpace(text))
{
return Json(new { success = false, message = "No text provided" });
}
try
{
Console.WriteLine($"[VoiceOrder] Manual product input: {text}");
// Parse products and quantities using AI (same as voice)
var parsedProducts = await ParseProductsFromText(text);
if (parsedProducts == null || parsedProducts.Count == 0)
{
return Json(new
{
success = false,
message = "Could not parse products from text",
transcription = text
});
}
// Enrich with actual product data from database
var enrichedProducts = await EnrichProductData(parsedProducts);
return Json(new
{
success = true,
transcription = text,
products = enrichedProducts
});
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error in ParseManualProductText: {ex.Message}");
return Json(new { success = false, message = $"Error: {ex.Message}" });
}
}
#region Helper Methods
/// <summary>
/// Transcribe audio file using OpenAI Whisper
/// </summary>
private async Task<string> TranscribeAudioFile(IFormFile audioFile, string language, string customPrompt = null)
{
var fileName = $"voice_order_{DateTime.Now:yyyyMMdd_HHmmss}.webm";
var uploadsFolder = Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "uploads", "voice");
if (!Directory.Exists(uploadsFolder))
{
Directory.CreateDirectory(uploadsFolder);
}
var filePath = Path.Combine(uploadsFolder, fileName);
// Save file temporarily
using (var stream = new FileStream(filePath, FileMode.Create))
{
await audioFile.CopyToAsync(stream);
}
// Transcribe using OpenAI Whisper
string transcribedText;
using (var audioStream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
{
transcribedText = await _aiApiService.TranscribeAudioAsync(audioStream, fileName, language, customPrompt);
}
// Clean up temporary file
try
{
System.IO.File.Delete(filePath);
}
catch { /* Ignore cleanup errors */ }
return transcribedText;
}
/// <summary>
/// Search for partners matching the transcribed text
/// Uses both string-based and AI semantic search for best results
/// </summary>
private async Task<List<object>> SearchPartners(string searchTerm)
{
const int maxResults = 10;
Console.WriteLine($"[VoiceOrder] Searching partners for: {searchTerm}");
// Step 1: String-based search (fast, catches exact matches)
var customersByCompany = await _customerService.GetAllCustomersAsync(
company: searchTerm,
pageIndex: 0,
pageSize: maxResults);
var customersByName = await _customerService.GetAllCustomersAsync(
firstName: searchTerm,
pageIndex: 0,
pageSize: maxResults);
var customersByLastName = await _customerService.GetAllCustomersAsync(
lastName: searchTerm,
pageIndex: 0,
pageSize: maxResults);
// Combine string search results
var stringResults = customersByCompany
.Union(customersByName)
.Union(customersByLastName)
.DistinctBy(c => c.Id)
.Take(maxResults)
.ToList();
Console.WriteLine($"[VoiceOrder] String-based search found {stringResults.Count} partners");
// Step 2: ALWAYS use AI semantic search for better results
Console.WriteLine("[VoiceOrder] Using AI semantic matching for partners");
var aiMatches = await SemanticPartnerSearch(searchTerm);
Console.WriteLine($"[VoiceOrder] AI semantic search found {aiMatches.Count} partners");
// Step 3: Merge results - string matches first (exact), then AI matches
var allCustomers = stringResults
.Union(aiMatches)
.DistinctBy(c => c.Id)
.Take(maxResults)
.ToList();
Console.WriteLine($"[VoiceOrder] Total unique partners: {allCustomers.Count}");
// Format results
var result = new List<object>();
foreach (var customer in allCustomers)
{
var fullName = await _customerService.GetCustomerFullNameAsync(customer);
var company = customer.Company;
if (string.IsNullOrEmpty(fullName))
fullName = "[No name]";
if (string.IsNullOrEmpty(company))
company = "[No company]";
string label = $"{company} ({fullName}), {customer.Email}";
result.Add(new
{
label = label,
value = customer.Id
});
}
return result;
}
/// <summary>
/// Use AI to semantically match partner names based on company name
/// </summary>
private async Task<List<Nop.Core.Domain.Customers.Customer>> SemanticPartnerSearch(string searchTerm)
{
try
{
// Get all customers with company names (increased limit)
var allCustomersWithCompany = await _customerService.GetAllCustomersAsync(
pageIndex: 0,
pageSize: 1000); // Increased from 500 to catch more companies
// Filter to only those with company names
var customersWithCompany = allCustomersWithCompany
.Where(c => !string.IsNullOrEmpty(c.Company))
.ToList();
if (customersWithCompany.Count == 0)
{
Console.WriteLine("[VoiceOrder] No customers with company names found");
return new List<Nop.Core.Domain.Customers.Customer>();
}
Console.WriteLine($"[VoiceOrder] AI searching through {customersWithCompany.Count} companies");
// Build company list for AI
var companyList = string.Join("\n", customersWithCompany
.Select((c, index) => $"{index}|{c.Company}"));
var systemPrompt = @"You are a company name matcher for a B2B system.
Given a spoken company name and a list of company names, find the 5 best matches.
CRITICAL MATCHING RULES (in priority order):
1. EXACT MATCH: If the search term appears exactly in a company name, prioritize it
2. SUBSTRING MATCH: If the search term is contained within a company name (e.g., 'Junket' in 'Junket Silver Kft.')
3. WORD MATCH: If all words from search term appear in company name (any order)
4. PARTIAL MATCH: If significant words overlap (e.g., 'Silver' matches 'Junket Silver')
5. PHONETIC SIMILARITY: How it sounds when spoken
6. ABBREVIATIONS: 'SFI' matches 'SFI Rotterdam B.V.'
EXAMPLES:
Search: 'Junket Silver'
Should match: 'Junket Silver Kft.' (substring match - VERY HIGH PRIORITY)
Search: 'Rotterdam'
Should match: 'SFI Rotterdam B.V.' (substring match)
Return ONLY a JSON array with the top 5 indices, ordered by best match first.
If fewer than 5 matches exist, return fewer indices.
OUTPUT FORMAT (JSON only):
[0, 15, 42, 103, 256]";
var userPrompt = $@"Search term: {searchTerm}
Companies:
{companyList}";
var aiResponse = await _aiApiService.GetSimpleResponseAsync(systemPrompt, userPrompt);
Console.WriteLine($"[VoiceOrder] AI company matching response: {aiResponse}");
// Extract JSON array from response
var jsonMatch = Regex.Match(aiResponse, @"\[[\d,\s]*\]", RegexOptions.Singleline);
if (!jsonMatch.Success)
{
Console.WriteLine("[VoiceOrder] No JSON array found in AI response");
return new List<Nop.Core.Domain.Customers.Customer>();
}
var jsonText = jsonMatch.Value;
var indices = System.Text.Json.JsonSerializer.Deserialize<List<int>>(jsonText);
if (indices == null || indices.Count == 0)
{
Console.WriteLine("[VoiceOrder] AI returned no matches");
return new List<Nop.Core.Domain.Customers.Customer>();
}
// Get customers by indices
var matchedCustomers = indices
.Where(i => i >= 0 && i < customersWithCompany.Count)
.Select(i => customersWithCompany[i])
.ToList();
Console.WriteLine($"[VoiceOrder] AI matched {matchedCustomers.Count} companies: {string.Join(", ", matchedCustomers.Select(c => c.Company))}");
return matchedCustomers;
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error in semantic partner search: {ex.Message}");
return new List<Nop.Core.Domain.Customers.Customer>();
}
}
/// <summary>
/// Parse products and quantities from transcribed text using AI
/// </summary>
private async Task<List<ParsedProduct>> ParseProductsFromText(string text)
{
var systemPrompt = @"You are a product parser for a Hungarian fruit and vegetable wholesale company.
Parse the product names and quantities from the user's speech.
CRITICAL RULES:
1. Extract product names and quantities from ANY produce item (fruits, vegetables, herbs, etc.)
2. Normalize product names to singular, lowercase (e.g., 'narancsok' → 'narancs', 'áfonyák' → 'áfonya')
3. Handle Hungarian number words ('száz' = 100, 'ötven' = 50, 'húsz' = 20, 'tíz' = 10, 'öt' = 5, 'egy' = 1, etc.)
4. FIX COMMON TRANSCRIPTION ERRORS:
- 'datója' → 'datolya' (dates)
- 'szűlő' → 'szőlő' (grapes)
- 'mondarin' → 'mandarin' (mandarin)
- 'paprika' is correct (pepper/paprika)
- 'fokhagyma' is correct (garlic)
- Any obvious typo → correct it
5. Return ONLY valid JSON array, no explanations or empty arrays
6. DO NOT include units - only product name and quantity as a number
7. ALWAYS return at least one product if you can parse anything from the input
OUTPUT FORMAT (JSON only):
[
{""product"": ""narancs"", ""quantity"": 100},
{""product"": ""alma"", ""quantity"": 50}
]
EXAMPLES:
Input: 'narancs száz kilogram és alma ötven kiló'
Output: [{""product"":""narancs"",""quantity"":100},{""product"":""alma"",""quantity"":50}]
Input: 'fokhagyma, 1'
Output: [{""product"":""fokhagyma"",""quantity"":1}]
Input: 'paprika, öt rekesz'
Output: [{""product"":""paprika"",""quantity"":5}]
Input: 'mondarin öt rekesz' (typo in 'mandarin')
Output: [{""product"":""mandarin"",""quantity"":5}]
Input: 'menta, 1'
Output: [{""product"":""menta"",""quantity"":1}]
Input: 'datója tíz láda' (WRONG transcription)
Output: [{""product"":""datolya"",""quantity"":10}]
Input: 'szűlő ötven kiló' (WRONG transcription)
Output: [{""product"":""szőlő"",""quantity"":50}]";
var userPrompt = $"Parse this: {text}";
var aiResponse = await _aiApiService.GetSimpleResponseAsync(systemPrompt, userPrompt);
Console.WriteLine($"[VoiceOrder] AI Response: {aiResponse}");
// Try to extract JSON from response
var jsonMatch = Regex.Match(aiResponse, @"\[.*\]", RegexOptions.Singleline);
if (!jsonMatch.Success)
{
Console.WriteLine("[VoiceOrder] No JSON array found in AI response");
return new List<ParsedProduct>();
}
var jsonText = jsonMatch.Value;
try
{
var parsedProducts = System.Text.Json.JsonSerializer.Deserialize<List<ParsedProduct>>(jsonText);
return parsedProducts ?? new List<ParsedProduct>();
}
catch (Exception ex)
{
Console.WriteLine($"[VoiceOrder] Error parsing JSON: {ex.Message}");
return new List<ParsedProduct>();
}
}
/// <summary>
/// Enrich parsed products with actual product data from database
/// Returns ALL matching products so admin can select the exact one
/// </summary>
private async Task<List<object>> EnrichProductData(List<ParsedProduct> parsedProducts)
{
var enrichedProducts = new List<object>();
// OPTIMIZATION: Load all ProductDtos once instead of querying one by one
var helperProductDtos = await _dbContext.ProductDtos.GetAll(true).ToListAsync();
foreach (var parsed in parsedProducts)
{
// Search for ALL matching products in database
var products = await _productService.SearchProductsAsync(
keywords: parsed.Product,
pageIndex: 0,
pageSize: 20); // Get up to 20 matches
if (!products.Any())
{
Console.WriteLine($"[VoiceOrder] No product found for: {parsed.Product}");
continue;
}
Console.WriteLine($"[VoiceOrder] Found {products.Count()} products matching '{parsed.Product}'");
// Add ALL matching products for admin to choose from
foreach (var product in products)
{
var productDto = helperProductDtos.FirstOrDefault(x => x.Id == product.Id);
if (productDto == null)
{
Console.WriteLine($"[VoiceOrder] ProductDto not found for product ID: {product.Id}");
continue;
}
// Check if enough stock
var availableQuantity = product.StockQuantity + productDto.IncomingQuantity;
if (availableQuantity <= 0)
{
Console.WriteLine($"[VoiceOrder] Product {product.Name} has no stock - skipping");
continue;
}
// Validate requested quantity against available stock
var requestedQuantity = parsed.Quantity;
var finalQuantity = requestedQuantity;
var isQuantityReduced = false;
if (requestedQuantity > availableQuantity)
{
Console.WriteLine($"[VoiceOrder] WARNING: Product {product.Name} - Requested {requestedQuantity} but only {availableQuantity} available. Capping to available.");
finalQuantity = availableQuantity;
isQuantityReduced = true;
}
// Add to enriched list with validated quantity
enrichedProducts.Add(new
{
id = product.Id,
name = product.Name,
sku = product.Sku,
quantity = finalQuantity, // Use validated quantity (capped to available)
requestedQuantity = requestedQuantity, // Original requested amount
price = product.Price,
stockQuantity = availableQuantity,
searchTerm = parsed.Product, // Track what was searched for
isQuantityReduced = isQuantityReduced // Flag if we had to reduce
});
}
}
Console.WriteLine($"[VoiceOrder] Total enriched products to display: {enrichedProducts.Count}");
return enrichedProducts;
}
#endregion
#region Models
private class ParsedProduct
{
[System.Text.Json.Serialization.JsonPropertyName("product")]
public string Product { get; set; }
[System.Text.Json.Serialization.JsonPropertyName("quantity")]
public int Quantity { get; set; }
}
#endregion
}
}