AI progress
This commit is contained in:
parent
61d3b2089c
commit
a058557620
|
|
@ -1,4 +1,6 @@
|
|||
using Microsoft.AspNetCore.Http;
|
||||
using FruitBank.Common.Dtos;
|
||||
using FruitBank.Common.Entities;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Nop.Core.Domain.Catalog;
|
||||
using Nop.Plugin.Misc.FruitBankPlugin.Domains.DataLayer;
|
||||
|
|
@ -150,13 +152,83 @@ namespace Nop.Plugin.Misc.FruitBank.Controllers
|
|||
});
|
||||
}
|
||||
|
||||
string analysisResult = await ExtractDocumentId(extractedText);
|
||||
|
||||
Console.WriteLine($"Document number analysis Result: {analysisResult}");
|
||||
string partnerAnalysis = await ExtractPartnerName(extractedText);
|
||||
await DeterminePartner(partnerAnalysis);
|
||||
string productAnalysis = await ExtractProducts(extractedText);
|
||||
Console.WriteLine($"Product analysis Result: {productAnalysis}");
|
||||
|
||||
//identify products from database
|
||||
var allProducts = await _dbContext.ProductDtos.GetAll().ToListAsync();
|
||||
var historicalProducts = await _dbContext.ShippingItems.GetAll().ToListAsync();
|
||||
|
||||
//create json from product analyzis jsonstring
|
||||
ProductReferenceResponse deserializedProducts = System.Text.Json.JsonSerializer.Deserialize<ProductReferenceResponse>(productAnalysis);
|
||||
Console.WriteLine($"Serealized Products: {deserializedProducts.products.Count}");
|
||||
|
||||
List<ProductDto> matchedProducts = new List<ProductDto>();
|
||||
//do we have historical references?
|
||||
foreach (var deserializedProduct in deserializedProducts.products)
|
||||
{
|
||||
var historicalProduct = historicalProducts.Where(hp => !string.IsNullOrEmpty(hp.NameOnDocument))
|
||||
.FirstOrDefault(p => p.NameOnDocument.Equals(deserializedProduct.name, StringComparison.OrdinalIgnoreCase));
|
||||
if (historicalProduct != null)
|
||||
{
|
||||
Console.WriteLine($"Historical product found: {historicalProduct.Name}");
|
||||
var productDto = allProducts.FirstOrDefault(p => p.Id == historicalProduct.ProductId);
|
||||
if (productDto != null)
|
||||
{
|
||||
matchedProducts.Add(productDto);
|
||||
Console.WriteLine($"Matched product from historical data: {productDto.Name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
Console.WriteLine($"Total matched products from historical data: {matchedProducts.Count}");
|
||||
if (matchedProducts.Count == 0)
|
||||
{
|
||||
//let's filter similar names first
|
||||
var similarNameProducts = new List<ShippingItem>();
|
||||
foreach (var deserializedProduct in deserializedProducts.products)
|
||||
{
|
||||
//var similarProducts = historicalProducts.Where(hp => hp.ShippingDocument.Partner.Name == partnerAnalysis)
|
||||
var similarProducts = historicalProducts
|
||||
.Where(p => !string.IsNullOrEmpty(p.NameOnDocument) &&
|
||||
p.NameOnDocument.Contains(deserializedProduct.name.Substring(0,6), StringComparison.OrdinalIgnoreCase))
|
||||
.ToList();
|
||||
similarNameProducts.AddRange(similarProducts);
|
||||
Console.WriteLine($"Similar products found for {deserializedProduct.name}: {similarProducts.Count}");
|
||||
}
|
||||
|
||||
|
||||
//no exact name matches, try to match with AI
|
||||
foreach (var deserializedProduct in deserializedProducts.products)
|
||||
{
|
||||
var aiMatchPrompt = $"You are an agent of Fruitbank to analyize product names and match them to existing products in the Fruitbank product catalog. " +
|
||||
$"Given the following product catalog: {string.Join(", ", similarNameProducts.Select(p => p.NameOnDocument))}, " +
|
||||
$"which product from the catalog best matches this product name: {deserializedProduct.name}. " +
|
||||
$"Reply with NOTHING ELSE THAN the exact product name from the catalog, if no match found, reply with 'NONE'.";
|
||||
var aiMatchedProductName = await _aiApiService.GetSimpleResponseAsync(aiMatchPrompt, deserializedProduct.name);
|
||||
var productDto = allProducts.FirstOrDefault(p => p.Name.Equals(aiMatchedProductName, StringComparison.OrdinalIgnoreCase));
|
||||
if (productDto != null)
|
||||
{
|
||||
matchedProducts.Add(productDto);
|
||||
Console.WriteLine($"AI Matched product: {productDto.Name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
string fullresult = await TestFullResult(extractedText);
|
||||
Console.WriteLine($"Full structured JSON Result: {fullresult}");
|
||||
|
||||
return Json(new
|
||||
{
|
||||
success = true,
|
||||
message = extension == ".pdf"
|
||||
? "PDF converted and text extracted successfully"
|
||||
: "Text extracted successfully",
|
||||
extractedText = extractedText,
|
||||
extractedText = extractedText + "/n" + fullresult,
|
||||
fileName = processedFileName,
|
||||
filePath = processedFilePath,
|
||||
fileSize = imageFile.Length,
|
||||
|
|
@ -173,5 +245,129 @@ namespace Nop.Plugin.Misc.FruitBank.Controllers
|
|||
});
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<string> TestFullResult(string extractedText)
|
||||
{
|
||||
string fullResultPrompt = $"Role:\r\nYou are an AI data extraction assistant for Fruitbank, a " +
|
||||
$"fruit and vegetable wholesale company. Your task is to analyze a " +
|
||||
$"provided text (delivery notes, invoices, or order confirmations) and extract structured information about " +
|
||||
$"the shipment and its items.\r\n\r\n🎯 Goal:\r\nRead the provided text and extract all shipment " +
|
||||
$"details and items according to the data model below.\r\n Generate the complete JSON output following this " +
|
||||
$"structure.\r\n\r\n🧩 Data Models:\r\n\r\npublic " +
|
||||
$"class Partner\r\n{{\r\n " +
|
||||
$"/// <summary>\r\n /// Partner entity primary key\r\n /// </summary>\r\n " +
|
||||
$"public int Id {{ get; set; }}\r\n " +
|
||||
$"/// <summary>\r\n /// Partner company name\r\n /// </summary>\r\n " +
|
||||
$"public string Name {{ get; set; }}\r\n " +
|
||||
$"/// <summary>\r\n /// Partner company TaxId\r\n /// </summary>\r\n " +
|
||||
$"public string TaxId {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Partner company Certification if exists\r\n /// </summary>\r\n " +
|
||||
$"public string CertificationNumber {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Partner company address PostalCode\r\n /// </summary>\r\n " +
|
||||
$"public string PostalCode {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Partner company address Country\r\n /// </summary>\r\n " +
|
||||
$"public string Country {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Partner company address State if exists\r\n /// </summary>\r\n " +
|
||||
$"public string State {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Partner company address County if exists\r\n /// </summary>\r\n " +
|
||||
$"public string County {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Partner company address City\r\n /// </summary>\r\n " +
|
||||
$"public string City {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Partner company address Street\r\n /// </summary>\r\n " +
|
||||
$"public string Street {{ get; set; }}\r\n\t/// <summary>\r\n " +
|
||||
$"/// Entities of ShippingDocument\r\n /// </summary>\r\n\tpublic List<ShippingDocument> " +
|
||||
$"ShippingDocuments {{ get; set; }}\t\r\n}}\r\n\r\npublic class ShippingDocument\r\n{{\r\n " +
|
||||
$"/// <summary>\r\n /// ShippingItem entity primary key\r\n /// </summary>\r\n " +
|
||||
$"public int Id {{ get; set; }}\r\n /// <summary>\r\n /// Partner entity primary key\r\n " +
|
||||
$"/// </summary>\r\n public int PartnerId {{ get; set; }}\t\r\n\t/// <summary>\r\n " +
|
||||
$"/// Entities of ShippingItem\r\n /// </summary>\r\n\t" +
|
||||
$"public List<ShippingItem> ShippingItems {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// DocumentIdNumber if exists\r\n /// </summary>\r\n public string DocumentIdNumber {{ get; set; }}\r\n " +
|
||||
$"/// <summary>\r\n /// \r\n /// </summary>\r\n public DateTime ShippingDate {{ get; set; }}\r\n " +
|
||||
$"/// <summary>\r\n /// Shipping pickup Contry of origin\r\n /// </summary>\r\n " +
|
||||
$"public string Country {{ get; set; }}\r\n\t/// <summary>\r\n /// Sum of ShippingItem pallets\r\n " +
|
||||
$"/// </summary>\r\n public int TotalPallets {{ get; set; }}\r\n\t/// <summary>\r\n " +
|
||||
$"/// Filename of pdf\r\n /// </summary>\r\n\tpublic string PdfFileName {{ get; set; }}\r\n}}\r\n\r\n" +
|
||||
$"public class ShippingItem\r\n{{\r\n /// <summary>\r\n /// ShippingItem entity primary key\r\n /// " +
|
||||
$"</summary>\r\n public int Id {{ get; set; }}\r\n /// <summary>\r\n /// " +
|
||||
$"ShippingDocument entity primary key\r\n /// </summary>\r\n " +
|
||||
$"public int ShippingDocumentId {{ get; set; }}\r\n /// " +
|
||||
$"<summary>\r\n /// Name of the fruit or vegitable\r\n /// </summary>\r\n " +
|
||||
$"public string Name {{ get; set; }}\r\n\t/// <summary>\r\n /// Translated Name to Hungarian\r\n " +
|
||||
$"/// </summary>\r\n public string HungarianName {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Pallets of fruit or vegitable item\r\n /// </summary>\r\n " +
|
||||
$"public int PalletsOnDocument {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Quantity of fruit or vegitable item\r\n /// </summary>\r\n " +
|
||||
$"public int QuantityOnDocument {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Net weight in kg. of fruit or vegitable item\r\n /// </summary>\r\n " +
|
||||
$"public double NetWeightOnDocument {{ get; set; }}\r\n /// <summary>\r\n " +
|
||||
$"/// Gross weight in kg. of fruit or vegitable item\r\n /// </summary>\r\n " +
|
||||
$"public double GrossWeightOnDocument {{ get; set; }}\r\n}}\r\n\r\n🧾 Output Requirements\r\n- " +
|
||||
$"Output must be a single valid JSON object containing:\r\n- One Partner object\r\n- " +
|
||||
$"One ShippingDocument object\r\n- A list of all related ShippingItem objects\r\n\r\n- " +
|
||||
$"Primary keys (Partner.Id, ShippingDocument.Id, ShippingItem.Id) should be auto-generated integers " +
|
||||
$"(e.g. sequential: 1, 2, 3…).\r\n\r\n- When a field is missing or unclear, return it as an empty " +
|
||||
$"string or 0 (depending on type).\r\nDo not omit any fields.\r\n\r\n- " +
|
||||
$"All dates must be in ISO 8601 format (yyyy-MM-dd).\r\n\r\n🧭 Instructions to the AI\r\n" +
|
||||
$"1. Analyze the provided text carefully.\r\n" +
|
||||
$"2. Identify the Partner/Company details of THE OTHER PARTY (other than Fruitbank), " +
|
||||
$"document identifiers, and each shipment item.\r\n" +
|
||||
$"3. FruitBank is not a partner! Always look for THE OTHER partner on the document. \r\n " +
|
||||
$"4. Generate a complete hierarchical JSON of ALL received documents in ONE JSON structure according to the " +
|
||||
$"data model above.\r\n5. Do not include any explanations or text outside the JSON output. " +
|
||||
$"Only return the structured JSON.\r\n" +
|
||||
$"6. A teljes ShippingItem.Name-et tedd bele a ShippingItem.HungarianName-be " +
|
||||
$"és a zöldség vagy gyümölcs nevét fordítsd le magyarra!\r\n" +
|
||||
$"7. A ShippingDocument-et tedd bele a Partner entitásba!\r\n" +
|
||||
$"8. ShippingItem-eket tedd bele a ShippingDocument-be!\r\n" +
|
||||
$"9. Do not assume or modify any data, if you don't find a value, return null, if you find a value, keep it unmodified.\r\n" +
|
||||
$"10. Magyarázat nélkül válaszolj!";
|
||||
|
||||
var fullresult = await _aiApiService.GetSimpleResponseAsync(fullResultPrompt, extractedText);
|
||||
return fullresult;
|
||||
}
|
||||
|
||||
private async Task<string> ExtractProducts(string extractedText)
|
||||
{
|
||||
//analyze document for product references
|
||||
return await _aiApiService.GetSimpleResponseAsync("You are an agent of Fruitbank to analyize text extracted from a pdf document, and find any product references mentioned in the document. Do not translate or modify the product information. Format your response as JSON object named 'products' with the following fields in the child objects: 'name' (string), 'quantity' (int), 'netWeight' (double), 'grossWeight' (double), ", $"What product references are mentioned in this document: {extractedText}");
|
||||
}
|
||||
|
||||
private async Task DeterminePartner(string partnerAnalysis)
|
||||
{
|
||||
//analyze the text to match partner
|
||||
var possiblePartners = await _dbContext.Partners.GetAll().ToListAsync();
|
||||
var partnerNames = string.Join(", ", possiblePartners.Select(p => p.Name));
|
||||
var partnerMatchAnalysis = await _aiApiService.GetSimpleResponseAsync("You are an agent of Fruitbank to analyize text extracted from a pdf document, and match the partner information to existing partners of Fruitbank. If you find a match, you reply with NOTHING ELSE THAN the parter's name, if not, reply with NOTHING ELSE THAN 'NONE'.", $"Given the following possible partners: {partnerNames}, which partner matches this partner information: {partnerAnalysis}");
|
||||
Console.WriteLine($"Partner match analysis Result: {partnerMatchAnalysis}");
|
||||
}
|
||||
|
||||
private async Task<string> ExtractPartnerName(string extractedText)
|
||||
{
|
||||
//analyze the text to find partner information
|
||||
var partnerAnalysis = await _aiApiService.GetSimpleResponseAsync("You are an agent of Fruitbank to analyize text extracted from a pdf document, and find the partner information of the other party, so the party that is NOT FRUITBANK.", $"What is the partner information of this document: {extractedText}");
|
||||
Console.WriteLine($"Partner analysis Result: {partnerAnalysis}");
|
||||
return partnerAnalysis;
|
||||
}
|
||||
|
||||
private async Task<string> ExtractDocumentId(string extractedText)
|
||||
{
|
||||
//analyze the text for document number or identifiers
|
||||
return await _aiApiService.GetSimpleResponseAsync("You are an agent of Fruitbank to analyize text extracted frem a pfd document, and find the document number or identifier. IMPORTANT: reply only with the number, do not add further explanation.", $"What is the document identifier of this document: {extractedText}");
|
||||
}
|
||||
}
|
||||
|
||||
public class ProductReference
|
||||
{
|
||||
public string? name { get; set; }
|
||||
public int? quantity { get; set; }
|
||||
public double? netWeight { get; set; }
|
||||
public double? grossWeight { get; set; }
|
||||
}
|
||||
|
||||
public class ProductReferenceResponse
|
||||
{
|
||||
public List<ProductReference> products { get; set; }
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -792,7 +792,7 @@ namespace Nop.Plugin.Misc.FruitBankPlugin.Services
|
|||
_ => "image/jpeg"
|
||||
};
|
||||
|
||||
var prompt = customPrompt ?? "Olvasd ki a szöveget és add vissza szépen strukturálva.";
|
||||
var prompt = customPrompt ?? "Extract all text from this image, without modifying or translateing it.";
|
||||
|
||||
var payload = new
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue