using System; using System.Collections.Generic; using System.Numerics; using System.Threading.Tasks; using BLAIzor.Models; using DocumentFormat.OpenXml.Office2010.Excel; using Google.Protobuf.Collections; using Google.Type; using Microsoft.CodeAnalysis.Completion; using Microsoft.Extensions.Configuration; using Qdrant.Client.Grpc; namespace BLAIzor.Services { public class HtmlSnippetProcessor { private readonly OpenAIEmbeddingService _openAIEmbeddingService; private readonly LocalEmbeddingService _localEmbeddingService; private readonly HttpClient _httpClient; private readonly QDrantService _drantService; public static IConfiguration? _configuration; private string _qdrantApiKey; public HtmlSnippetProcessor(QDrantService drantService, OpenAIEmbeddingService openAIEmbeddingService, LocalEmbeddingService localEmbeddingService, IConfiguration? configuration) { _drantService = drantService; _openAIEmbeddingService = openAIEmbeddingService; _localEmbeddingService = localEmbeddingService; _httpClient = new HttpClient(); _configuration = configuration; } public string GetApiKey() { return _configuration?.GetSection("QDrant")?.GetValue("ApiKey") ?? string.Empty; } private string GetAiEmbeddingSettings() => _configuration?.GetSection("AiSettings")?.GetValue("EmbeddingService") ?? string.Empty; //public async Task ProcessAndStoreSnippetsAsync() //{ // _qdrantApiKey = GetApiKey(); // var snippets = GetHtmlSnippets(); // var ids = new List(); // var vectors = new List(); // var payloads = new List>(); // foreach (var snippet in snippets) // { // try // { // // Combine details to generate the embedding // var combinedText = $"{snippet.Name}: {snippet.Description}"; // var embedding = await _embeddingService.GenerateEmbeddingAsync(combinedText); // // Add data for batch insertion // ids.Add(snippet.Id); // vectors.Add(embedding); // payloads.Add(new MapField // { // ["type"] = snippet.Type, // ["name"] = snippet.Name, // ["description"] = snippet.Description, // ["html"] = snippet.Html // }); // } // catch (Exception ex) // { // Console.WriteLine($"Error processing snippet {snippet.Name}: {ex.Message}"); // } // } // if (ids.Count > 0) // { // await _drantService.QDrantInsertTest(ids, vectors, payloads); // } // else // { // Console.WriteLine("No points were processed successfully."); // } //} public async Task ProcessAndStoreTemplateSnippetAsync(string TemplateName, List snippets) { _qdrantApiKey = GetApiKey(); //var snippets = GetHtmlSnippets(); var ids = new List(); var vectors = new List(); var payloads = new List>(); foreach (var snippet in snippets) { try { // Combine details to generate the embedding var combinedText = $"{snippet.Name}: {snippet.Description}. " + $"Type: {snippet.Type}. " + $"Variant: {snippet.Variant ?? "default"}. " + $"Tags: {snippet.Tags}. " + $"HTML: {snippet.Html}"; float[] embedding = []; var embeddingServiceProvider = GetAiEmbeddingSettings(); //if (embeddingServiceProvider == "local") //{ // embedding = await _localEmbeddingService.GenerateEmbeddingAsync(combinedText); //} //else //{ // embedding = await _openAIEmbeddingService.GenerateEmbeddingAsync(combinedText); //} embedding = await _openAIEmbeddingService.GenerateEmbeddingAsync(combinedText); // Add data for batch insertion ids.Add(snippet.Id); vectors.Add(embedding); payloads.Add(new MapField { ["type"] = snippet.Type, ["name"] = snippet.Name, ["variant"] = string.IsNullOrWhiteSpace(snippet.Variant) ? "" : snippet.Variant, ["tags"] = snippet.Tags, ["description"] = snippet.Description, ["html"] = snippet.Html, ["sampleHtml"] = snippet.SampleHtml }); } catch (Exception ex) { Console.WriteLine($"Error processing snippet {snippet.Name}: {ex.Message}"); } } if (ids.Count > 0) { await _drantService.QDrantInsertManyAsync(ids, vectors, payloads, TemplateName); } else { Console.WriteLine("No points were processed successfully."); } } //public async Task ProcessAndStoreWebContentsAsync(List pageContentList, int siteId) //{ // _qdrantApiKey = GetApiKey(); // var ids = new List(); // var vectors = new List(); // var payloads = new List>(); // foreach (var content in pageContentList) // { // try // { // // Combine details to generate the embedding // var combinedText = $"{content.Name}: Description: {content.Description}, complete text: {content.Content}"; // float[] embedding = []; // var embeddingServiceProvider = GetAiEmbeddingSettings(); // if (embeddingServiceProvider == "local") // { // embedding = await _localEmbeddingService.GenerateEmbeddingAsync(combinedText); // } // else // { // embedding = await _openAIEmbeddingService.GenerateEmbeddingAsync(combinedText); // } // // Add data for batch insertion // ids.Add(content.Id); // vectors.Add(embedding); // payloads.Add(new MapField // { // ["uid"] = content.UId, // ["type"] = content.Type, // ["siteId"] = content.SiteId, // //["menuItemId"] = content.MenuItemId, // ["name"] = content.Name, // ["description"] = content.Description, // ["content"] = content.Content, // ["lastUpdated"] = content.LastUpdated.ToString() // }); // } // catch (Exception ex) // { // Console.WriteLine($"Error processing content {content.Name}: {ex.Message}"); // } // } // if (ids.Count > 0) // { // await _drantService.QDrantInsertManyAsync(ids, vectors, payloads, "Site" + siteId); // } // else // { // Console.WriteLine("No points were processed successfully."); // } //} public async Task ProcessAndStoreWebContentAsync(PointId id, WebPageContent pageContent, int siteId) { _qdrantApiKey = GetApiKey(); float[] vectors = []; var payload = new MapField(); try { // Combine details to generate the embedding var combinedText = $"{pageContent.Name}: {pageContent.Description} - {pageContent.Content}"; float[] embedding = []; var embeddingServiceProvider = GetAiEmbeddingSettings(); if (embeddingServiceProvider == "local") { embedding = await _localEmbeddingService.GenerateEmbeddingAsync(combinedText); } else { embedding = await _openAIEmbeddingService.GenerateEmbeddingAsync(combinedText); } // Add data for batch insertion vectors = embedding; payload = new MapField { ["uid"] = pageContent.UId, ["type"] = pageContent.Type, ["siteId"] = pageContent.SiteId, //["menuItemId"] = pageContent.MenuItemId, ["name"] = pageContent.Name, ["description"] = pageContent.Description, ["content"] = pageContent.Content, ["lastUpdated"] = pageContent.LastUpdated.ToString() }; } catch (Exception ex) { Console.WriteLine($"Error processing content {pageContent.Name}: {ex.Message}"); } await _drantService.QDrantInsertPointAsync(id, vectors, payload, "Site" + siteId); } private List GetHtmlSnippets() { return new List { new HtmlSnippet { Id = 1, Type = "hero", Name = "Hero Layout", Description = "A Bootstrap hero section with an image and text.", Html = "
" + "
" + "\"{photo" + "
" + "

{Title}

" + "

{lead}

" + "
" + "

{paragraph}

" }, new HtmlSnippet { Id = 2, Type = "product_card", Name = "Product Card", Description = "A responsive product card layout using Bootstrap.", Html = "
" + "
" + "\"{image
" + "
" + "
{Product title}
" + "

{description}

" + "{button text}" + "
" }, new HtmlSnippet { Id = 3, Type = "table_general", Name = "General Table", Description = "A responsive table to be used for displaying information in columns and rows", Html = "" + "" + "" + "" + "" + "
#FirstLastHandle
{cell content 1}{cell content 2}{cell content 3}{cell content 3}
" }, new HtmlSnippet { Id = 4, Type = "team_member", Name = "Team member", Description = "A responsive card to display employees, leaders, other team members", Html = "
" + "
" + "\"{image
" + "
" + "
{Person's name}
" + "

{description}

" + "
" }, new HtmlSnippet { Id = 5, Type = "music_player", Name = "Music player", Description = "A responsive music player to play a list of mp3 songs", Html = "
" + "
" + "
" + "
{'Now playing' in current language}
" + "

{Audio file name}

" + "
" + "
" + "
0:00" + "0:00" + "
" + "" + "" + "
" }, new HtmlSnippet { Id = 6, Type = "photo_carousel", Name = "Photo carousel", Description = "A responsive photo carousel to display photos up not more than 5", Html = "
" + "
" + "..."+ "
" + "\"{photo
" + "
" + "\"{photo
" + "
" + "\"{photo
" + "..."+ "
" + "" + "
" }, new HtmlSnippet { Id = 7, Type = "event_list", Name = "Event list", Description = "A responsive layout for displaying events.", Html = "
" + "
" + "
" + "

{Time}

" + "{Date}" + "
" + "
" + "

{event name}

" + "" + "{Location}" + "

" + "
" } // Add more snippets as needed }; } } }