486 lines
18 KiB
C#
486 lines
18 KiB
C#
using Microsoft.AspNetCore.Http;
|
|
using Microsoft.AspNetCore.Mvc;
|
|
using Nop.Plugin.Misc.FruitBankPlugin.Domains.DataLayer;
|
|
using Nop.Plugin.Misc.FruitBankPlugin.Services;
|
|
using Nop.Services.Catalog;
|
|
using Nop.Services.Customers;
|
|
using Nop.Services.Security;
|
|
using Nop.Web.Framework;
|
|
using Nop.Web.Framework.Controllers;
|
|
using Nop.Web.Framework.Mvc.Filters;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace Nop.Plugin.Misc.FruitBankPlugin.Areas.Admin.Controllers
|
|
{
|
|
[AuthorizeAdmin]
|
|
[Area(AreaNames.ADMIN)]
|
|
[AutoValidateAntiforgeryToken]
|
|
public class VoiceOrderController : BasePluginController
|
|
{
|
|
private readonly IPermissionService _permissionService;
|
|
private readonly OpenAIApiService _aiApiService;
|
|
private readonly ICustomerService _customerService;
|
|
private readonly IProductService _productService;
|
|
private readonly FruitBankDbContext _dbContext;
|
|
|
|
public VoiceOrderController(
|
|
IPermissionService permissionService,
|
|
OpenAIApiService aiApiService,
|
|
ICustomerService customerService,
|
|
IProductService productService,
|
|
FruitBankDbContext dbContext)
|
|
{
|
|
_permissionService = permissionService;
|
|
_aiApiService = aiApiService;
|
|
_customerService = customerService;
|
|
_productService = productService;
|
|
_dbContext = dbContext;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Display the voice order creation page
|
|
/// </summary>
|
|
[HttpGet]
|
|
public async Task<IActionResult> Create()
|
|
{
|
|
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
|
|
return AccessDeniedView();
|
|
|
|
return View("~/Plugins/Misc.FruitBankPlugin/Areas/Admin/Views/VoiceOrder/Create.cshtml");
|
|
}
|
|
|
|
/// <summary>
|
|
/// Transcribe audio for partner name and return matching partners
|
|
/// </summary>
|
|
[HttpPost]
|
|
public async Task<IActionResult> TranscribeForPartner(IFormFile audioFile)
|
|
{
|
|
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
|
|
return Json(new { success = false, message = "Access denied" });
|
|
|
|
if (audioFile == null || audioFile.Length == 0)
|
|
{
|
|
return Json(new { success = false, message = "No audio file received" });
|
|
}
|
|
|
|
try
|
|
{
|
|
// Transcribe audio
|
|
var transcribedText = await TranscribeAudioFile(audioFile, "hu");
|
|
|
|
if (string.IsNullOrEmpty(transcribedText))
|
|
{
|
|
return Json(new { success = false, message = "Failed to transcribe audio" });
|
|
}
|
|
|
|
Console.WriteLine($"[VoiceOrder] Partner transcription: {transcribedText}");
|
|
|
|
// Search for matching partners
|
|
var partners = await SearchPartners(transcribedText);
|
|
|
|
return Json(new
|
|
{
|
|
success = true,
|
|
transcription = transcribedText,
|
|
partners = partners
|
|
});
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[VoiceOrder] Error in TranscribeForPartner: {ex.Message}");
|
|
return Json(new { success = false, message = $"Error: {ex.Message}" });
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Transcribe audio for products and parse quantities using AI
|
|
/// </summary>
|
|
[HttpPost]
|
|
public async Task<IActionResult> TranscribeForProducts(IFormFile audioFile)
|
|
{
|
|
if (!await _permissionService.AuthorizeAsync(StandardPermission.Security.ACCESS_ADMIN_PANEL))
|
|
return Json(new { success = false, message = "Access denied" });
|
|
|
|
if (audioFile == null || audioFile.Length == 0)
|
|
{
|
|
return Json(new { success = false, message = "No audio file received" });
|
|
}
|
|
|
|
try
|
|
{
|
|
// Transcribe audio
|
|
var transcribedText = await TranscribeAudioFile(audioFile, "hu");
|
|
|
|
if (string.IsNullOrEmpty(transcribedText))
|
|
{
|
|
return Json(new { success = false, message = "Failed to transcribe audio" });
|
|
}
|
|
|
|
Console.WriteLine($"[VoiceOrder] Product transcription: {transcribedText}");
|
|
|
|
// Parse products and quantities using AI
|
|
var parsedProducts = await ParseProductsFromText(transcribedText);
|
|
|
|
if (parsedProducts == null || parsedProducts.Count == 0)
|
|
{
|
|
return Json(new
|
|
{
|
|
success = false,
|
|
message = "Could not parse products from transcription",
|
|
transcription = transcribedText
|
|
});
|
|
}
|
|
|
|
// Enrich with actual product data from database
|
|
var enrichedProducts = await EnrichProductData(parsedProducts);
|
|
|
|
return Json(new
|
|
{
|
|
success = true,
|
|
transcription = transcribedText,
|
|
products = enrichedProducts
|
|
});
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[VoiceOrder] Error in TranscribeForProducts: {ex.Message}");
|
|
return Json(new { success = false, message = $"Error: {ex.Message}" });
|
|
}
|
|
}
|
|
|
|
#region Helper Methods
|
|
|
|
/// <summary>
|
|
/// Transcribe audio file using OpenAI Whisper
|
|
/// </summary>
|
|
private async Task<string> TranscribeAudioFile(IFormFile audioFile, string language)
|
|
{
|
|
var fileName = $"voice_order_{DateTime.Now:yyyyMMdd_HHmmss}.webm";
|
|
var uploadsFolder = Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "uploads", "voice");
|
|
|
|
if (!Directory.Exists(uploadsFolder))
|
|
{
|
|
Directory.CreateDirectory(uploadsFolder);
|
|
}
|
|
|
|
var filePath = Path.Combine(uploadsFolder, fileName);
|
|
|
|
// Save file temporarily
|
|
using (var stream = new FileStream(filePath, FileMode.Create))
|
|
{
|
|
await audioFile.CopyToAsync(stream);
|
|
}
|
|
|
|
// Transcribe using OpenAI Whisper
|
|
string transcribedText;
|
|
using (var audioStream = new FileStream(filePath, FileMode.Open, FileAccess.Read))
|
|
{
|
|
transcribedText = await _aiApiService.TranscribeAudioAsync(audioStream, fileName, language);
|
|
}
|
|
|
|
// Clean up temporary file
|
|
try
|
|
{
|
|
System.IO.File.Delete(filePath);
|
|
}
|
|
catch { /* Ignore cleanup errors */ }
|
|
|
|
return transcribedText;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Search for partners matching the transcribed text
|
|
/// Uses string-based search first, then semantic AI matching if needed
|
|
/// </summary>
|
|
private async Task<List<object>> SearchPartners(string searchTerm)
|
|
{
|
|
const int maxResults = 10;
|
|
const int minResultsForAI = 3; // If we get fewer than this, use AI semantic search
|
|
|
|
Console.WriteLine($"[VoiceOrder] Searching partners for: {searchTerm}");
|
|
|
|
// Step 1: Try string-based search
|
|
var customersByCompany = await _customerService.GetAllCustomersAsync(
|
|
company: searchTerm,
|
|
pageIndex: 0,
|
|
pageSize: maxResults);
|
|
|
|
var customersByName = await _customerService.GetAllCustomersAsync(
|
|
firstName: searchTerm,
|
|
pageIndex: 0,
|
|
pageSize: maxResults);
|
|
|
|
var customersByLastName = await _customerService.GetAllCustomersAsync(
|
|
lastName: searchTerm,
|
|
pageIndex: 0,
|
|
pageSize: maxResults);
|
|
|
|
// Combine and deduplicate
|
|
var allCustomers = customersByCompany
|
|
.Union(customersByName)
|
|
.Union(customersByLastName)
|
|
.DistinctBy(c => c.Id)
|
|
.Take(maxResults)
|
|
.ToList();
|
|
|
|
Console.WriteLine($"[VoiceOrder] String-based search found {allCustomers.Count} partners");
|
|
|
|
// Step 2: If we don't have enough results, use AI semantic matching
|
|
if (allCustomers.Count < minResultsForAI)
|
|
{
|
|
Console.WriteLine("[VoiceOrder] Using AI semantic matching for partners");
|
|
|
|
var aiMatches = await SemanticPartnerSearch(searchTerm);
|
|
|
|
// Merge AI matches with string matches, remove duplicates
|
|
allCustomers = allCustomers
|
|
.Union(aiMatches)
|
|
.DistinctBy(c => c.Id)
|
|
.Take(maxResults)
|
|
.ToList();
|
|
|
|
Console.WriteLine($"[VoiceOrder] After AI matching: {allCustomers.Count} partners");
|
|
}
|
|
|
|
// Format results
|
|
var result = new List<object>();
|
|
foreach (var customer in allCustomers)
|
|
{
|
|
var fullName = await _customerService.GetCustomerFullNameAsync(customer);
|
|
var company = customer.Company;
|
|
|
|
if (string.IsNullOrEmpty(fullName))
|
|
fullName = "[No name]";
|
|
if (string.IsNullOrEmpty(company))
|
|
company = "[No company]";
|
|
|
|
string label = $"{company} ({fullName}), {customer.Email}";
|
|
|
|
result.Add(new
|
|
{
|
|
label = label,
|
|
value = customer.Id
|
|
});
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Use AI to semantically match partner names based on company name
|
|
/// </summary>
|
|
private async Task<List<Nop.Core.Domain.Customers.Customer>> SemanticPartnerSearch(string searchTerm)
|
|
{
|
|
try
|
|
{
|
|
// Get all customers with company names (limit to reasonable number)
|
|
var allCustomersWithCompany = await _customerService.GetAllCustomersAsync(
|
|
pageIndex: 0,
|
|
pageSize: 500); // Reasonable limit for AI processing
|
|
|
|
// Filter to only those with company names
|
|
var customersWithCompany = allCustomersWithCompany
|
|
.Where(c => !string.IsNullOrEmpty(c.Company))
|
|
.ToList();
|
|
|
|
if (customersWithCompany.Count == 0)
|
|
{
|
|
Console.WriteLine("[VoiceOrder] No customers with company names found");
|
|
return new List<Nop.Core.Domain.Customers.Customer>();
|
|
}
|
|
|
|
// Build company list for AI
|
|
var companyList = string.Join("\n", customersWithCompany
|
|
.Select((c, index) => $"{index}|{c.Company}"));
|
|
|
|
var systemPrompt = @"You are a company name matcher for a B2B system.
|
|
Given a spoken company name and a list of company names, find the 5 best semantic matches.
|
|
|
|
RULES:
|
|
1. Consider phonetic similarity (how it sounds)
|
|
2. Consider abbreviations (e.g., 'SFI' matches 'SFI Rotterdam B.V.')
|
|
3. Consider partial matches (e.g., 'Rotterdam' matches 'SFI Rotterdam B.V.')
|
|
4. Consider common misspellings or mishearings
|
|
5. Return ONLY valid JSON array with indices, no explanations
|
|
|
|
INPUT FORMAT:
|
|
Search term: [spoken company name]
|
|
Companies: [index]|[company name] (one per line)
|
|
|
|
OUTPUT FORMAT (JSON only):
|
|
[0, 15, 42, 103, 256]
|
|
|
|
Return the top 5 indices that best match the search term. If fewer than 5 good matches exist, return fewer indices.";
|
|
|
|
var userPrompt = $@"Search term: {searchTerm}
|
|
Companies:
|
|
{companyList}";
|
|
|
|
var aiResponse = await _aiApiService.GetSimpleResponseAsync(systemPrompt, userPrompt);
|
|
|
|
Console.WriteLine($"[VoiceOrder] AI company matching response: {aiResponse}");
|
|
|
|
// Extract JSON array from response
|
|
var jsonMatch = Regex.Match(aiResponse, @"\[[\d,\s]*\]", RegexOptions.Singleline);
|
|
if (!jsonMatch.Success)
|
|
{
|
|
Console.WriteLine("[VoiceOrder] No JSON array found in AI response");
|
|
return new List<Nop.Core.Domain.Customers.Customer>();
|
|
}
|
|
|
|
var jsonText = jsonMatch.Value;
|
|
var indices = System.Text.Json.JsonSerializer.Deserialize<List<int>>(jsonText);
|
|
|
|
if (indices == null || indices.Count == 0)
|
|
{
|
|
Console.WriteLine("[VoiceOrder] AI returned no matches");
|
|
return new List<Nop.Core.Domain.Customers.Customer>();
|
|
}
|
|
|
|
// Get customers by indices
|
|
var matchedCustomers = indices
|
|
.Where(i => i >= 0 && i < customersWithCompany.Count)
|
|
.Select(i => customersWithCompany[i])
|
|
.ToList();
|
|
|
|
Console.WriteLine($"[VoiceOrder] AI matched {matchedCustomers.Count} companies: {string.Join(", ", matchedCustomers.Select(c => c.Company))}");
|
|
|
|
return matchedCustomers;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[VoiceOrder] Error in semantic partner search: {ex.Message}");
|
|
return new List<Nop.Core.Domain.Customers.Customer>();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Parse products and quantities from transcribed text using AI
|
|
/// </summary>
|
|
private async Task<List<ParsedProduct>> ParseProductsFromText(string text)
|
|
{
|
|
var systemPrompt = @"You are a product parser for a Hungarian fruit wholesale company.
|
|
Parse the product names and quantities from the user's speech.
|
|
|
|
RULES:
|
|
1. Extract product names and quantities
|
|
2. Normalize product names to singular, lowercase (e.g., 'narancsok' → 'narancs')
|
|
3. Convert quantity units to standard format (kg, db, láda)
|
|
4. Handle Hungarian number words ('száz' = 100, 'ötven' = 50, etc.)
|
|
5. Return ONLY valid JSON array, no explanations
|
|
|
|
OUTPUT FORMAT (JSON only):
|
|
[
|
|
{""product"": ""narancs"", ""quantity"": 100, ""unit"": ""kg""},
|
|
{""product"": ""alma"", ""quantity"": 50, ""unit"": ""kg""}
|
|
]
|
|
|
|
EXAMPLES:
|
|
Input: 'narancs száz kilogram és alma ötven kiló'
|
|
Output: [{""product"":""narancs"",""quantity"":100,""unit"":""kg""},{""product"":""alma"",""quantity"":50,""unit"":""kg""}]
|
|
|
|
Input: 'Kérek 200 kg narancsot meg 150 kg almát'
|
|
Output: [{""product"":""narancs"",""quantity"":200,""unit"":""kg""},{""product"":""alma"",""quantity"":150,""unit"":""kg""}]";
|
|
|
|
var userPrompt = $"Parse this: {text}";
|
|
|
|
var aiResponse = await _aiApiService.GetSimpleResponseAsync(systemPrompt, userPrompt);
|
|
|
|
Console.WriteLine($"[VoiceOrder] AI Response: {aiResponse}");
|
|
|
|
// Try to extract JSON from response
|
|
var jsonMatch = Regex.Match(aiResponse, @"\[.*\]", RegexOptions.Singleline);
|
|
if (!jsonMatch.Success)
|
|
{
|
|
Console.WriteLine("[VoiceOrder] No JSON array found in AI response");
|
|
return new List<ParsedProduct>();
|
|
}
|
|
|
|
var jsonText = jsonMatch.Value;
|
|
|
|
try
|
|
{
|
|
var parsedProducts = System.Text.Json.JsonSerializer.Deserialize<List<ParsedProduct>>(jsonText);
|
|
return parsedProducts ?? new List<ParsedProduct>();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[VoiceOrder] Error parsing JSON: {ex.Message}");
|
|
return new List<ParsedProduct>();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Enrich parsed products with actual product data from database
|
|
/// </summary>
|
|
private async Task<List<object>> EnrichProductData(List<ParsedProduct> parsedProducts)
|
|
{
|
|
var enrichedProducts = new List<object>();
|
|
|
|
foreach (var parsed in parsedProducts)
|
|
{
|
|
// Search for matching product in database
|
|
var products = await _productService.SearchProductsAsync(
|
|
keywords: parsed.Product,
|
|
pageIndex: 0,
|
|
pageSize: 5);
|
|
|
|
if (!products.Any())
|
|
{
|
|
Console.WriteLine($"[VoiceOrder] No product found for: {parsed.Product}");
|
|
continue;
|
|
}
|
|
|
|
// Take the best match (first result)
|
|
var product = products.First();
|
|
var productDto = await _dbContext.ProductDtos.GetByIdAsync(product.Id);
|
|
|
|
if (productDto == null)
|
|
{
|
|
Console.WriteLine($"[VoiceOrder] ProductDto not found for product ID: {product.Id}");
|
|
continue;
|
|
}
|
|
|
|
// Check if enough stock
|
|
var availableQuantity = product.StockQuantity + productDto.IncomingQuantity;
|
|
if (availableQuantity <= 0)
|
|
{
|
|
Console.WriteLine($"[VoiceOrder] Product {product.Name} has no stock");
|
|
continue;
|
|
}
|
|
|
|
// Add to enriched list
|
|
enrichedProducts.Add(new
|
|
{
|
|
id = product.Id,
|
|
name = product.Name,
|
|
sku = product.Sku,
|
|
quantity = parsed.Quantity,
|
|
unit = parsed.Unit,
|
|
price = product.Price,
|
|
stockQuantity = availableQuantity
|
|
});
|
|
}
|
|
|
|
return enrichedProducts;
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Models
|
|
|
|
private class ParsedProduct
|
|
{
|
|
public string Product { get; set; }
|
|
public int Quantity { get; set; }
|
|
public string Unit { get; set; }
|
|
}
|
|
|
|
#endregion
|
|
}
|
|
} |