380 lines
18 KiB
C#
380 lines
18 KiB
C#
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Threading.Tasks;
|
|
using BLAIzor.Models;
|
|
using DocumentFormat.OpenXml.Office2010.Excel;
|
|
using Google.Protobuf.Collections;
|
|
using Google.Type;
|
|
using Microsoft.CodeAnalysis.Completion;
|
|
using Microsoft.Extensions.Configuration;
|
|
using Qdrant.Client.Grpc;
|
|
|
|
namespace BLAIzor.Services
|
|
{
|
|
|
|
|
|
public class HtmlSnippetProcessor
|
|
{
|
|
private readonly OpenAIEmbeddingService _embeddingService;
|
|
private readonly HttpClient _httpClient;
|
|
private readonly QDrantService _drantService;
|
|
public static IConfiguration? _configuration;
|
|
private string _qdrantApiKey;
|
|
|
|
public HtmlSnippetProcessor(QDrantService drantService, IConfiguration? configuration)
|
|
{
|
|
_drantService = drantService;
|
|
_embeddingService = new OpenAIEmbeddingService();
|
|
_httpClient = new HttpClient();
|
|
_configuration = configuration;
|
|
}
|
|
|
|
public string GetApiKey()
|
|
{
|
|
return _configuration?.GetSection("QDrant")?.GetValue<string>("ApiKey") ?? string.Empty;
|
|
}
|
|
|
|
//public async Task ProcessAndStoreSnippetsAsync()
|
|
//{
|
|
// _qdrantApiKey = GetApiKey();
|
|
|
|
// var snippets = GetHtmlSnippets();
|
|
|
|
// var ids = new List<int>();
|
|
// var vectors = new List<float[]>();
|
|
// var payloads = new List<MapField<string, Value>>();
|
|
|
|
// foreach (var snippet in snippets)
|
|
// {
|
|
// try
|
|
// {
|
|
// // Combine details to generate the embedding
|
|
// var combinedText = $"{snippet.Name}: {snippet.Description}";
|
|
// var embedding = await _embeddingService.GenerateEmbeddingAsync(combinedText);
|
|
|
|
// // Add data for batch insertion
|
|
// ids.Add(snippet.Id);
|
|
// vectors.Add(embedding);
|
|
// payloads.Add(new MapField<string, Value>
|
|
// {
|
|
// ["type"] = snippet.Type,
|
|
// ["name"] = snippet.Name,
|
|
// ["description"] = snippet.Description,
|
|
// ["html"] = snippet.Html
|
|
// });
|
|
// }
|
|
// catch (Exception ex)
|
|
// {
|
|
// Console.WriteLine($"Error processing snippet {snippet.Name}: {ex.Message}");
|
|
// }
|
|
// }
|
|
|
|
// if (ids.Count > 0)
|
|
// {
|
|
// await _drantService.QDrantInsertTest(ids, vectors, payloads);
|
|
// }
|
|
// else
|
|
// {
|
|
// Console.WriteLine("No points were processed successfully.");
|
|
// }
|
|
//}
|
|
|
|
public async Task ProcessAndStoreTemplateSnippetAsync(string TemplateName, List<HtmlSnippet> snippets)
|
|
{
|
|
_qdrantApiKey = GetApiKey();
|
|
|
|
//var snippets = GetHtmlSnippets();
|
|
|
|
var ids = new List<int>();
|
|
var vectors = new List<float[]>();
|
|
var payloads = new List<MapField<string, Value>>();
|
|
|
|
foreach (var snippet in snippets)
|
|
{
|
|
try
|
|
{
|
|
|
|
// Combine details to generate the embedding
|
|
var combinedText = $"{snippet.Name}: {snippet.Description}. " +
|
|
$"Type: {snippet.Type}. " +
|
|
$"Variant: {snippet.Variant ?? "default"}. " +
|
|
$"Tags: {snippet.Tags}. " +
|
|
$"HTML: {snippet.Html}";
|
|
var embedding = await _embeddingService.GenerateEmbeddingAsync(combinedText);
|
|
|
|
// Add data for batch insertion
|
|
ids.Add(snippet.Id);
|
|
vectors.Add(embedding);
|
|
payloads.Add(new MapField<string, Value>
|
|
{
|
|
["type"] = snippet.Type,
|
|
["name"] = snippet.Name,
|
|
["variant"] = string.IsNullOrWhiteSpace(snippet.Variant)? "" : snippet.Variant,
|
|
["tags"] = snippet.Tags,
|
|
["description"] = snippet.Description,
|
|
["html"] = snippet.Html,
|
|
["sampleHtml"] = snippet.SampleHtml
|
|
});
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error processing snippet {snippet.Name}: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
if (ids.Count > 0)
|
|
{
|
|
await _drantService.QDrantInsertManyAsync(ids, vectors, payloads, TemplateName);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("No points were processed successfully.");
|
|
}
|
|
}
|
|
|
|
public async Task ProcessAndStoreWebContentsAsync(List<WebPageContent> pageContentList, int siteId)
|
|
{
|
|
_qdrantApiKey = GetApiKey();
|
|
|
|
var ids = new List<int>();
|
|
var vectors = new List<float[]>();
|
|
var payloads = new List<MapField<string, Value>>();
|
|
|
|
foreach (var content in pageContentList)
|
|
{
|
|
try
|
|
{
|
|
// Combine details to generate the embedding
|
|
var combinedText = $"{content.Name}: Description: {content.Description}, complete text: {content.Content}";
|
|
var embedding = await _embeddingService.GenerateEmbeddingAsync(combinedText);
|
|
|
|
// Add data for batch insertion
|
|
ids.Add(content.Id);
|
|
vectors.Add(embedding);
|
|
|
|
|
|
payloads.Add(new MapField<string, Value>
|
|
{
|
|
["uid"] = content.UId,
|
|
["type"] = content.Type,
|
|
["siteId"] = content.SiteId,
|
|
//["menuItemId"] = content.MenuItemId,
|
|
["name"] = content.Name,
|
|
["description"] = content.Description,
|
|
["content"] = content.Content,
|
|
["lastUpdated"] = content.LastUpdated.ToString()
|
|
});
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error processing content {content.Name}: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
if (ids.Count > 0)
|
|
{
|
|
await _drantService.QDrantInsertManyAsync(ids, vectors, payloads, "Site"+siteId);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("No points were processed successfully.");
|
|
}
|
|
}
|
|
|
|
public async Task ProcessAndStoreWebContentAsync(int id, WebPageContent pageContent, int siteId)
|
|
{
|
|
_qdrantApiKey = GetApiKey();
|
|
|
|
|
|
float[] vectors = [];
|
|
var payload = new MapField<string, Value>();
|
|
|
|
|
|
try
|
|
{
|
|
// Combine details to generate the embedding
|
|
var combinedText = $"{pageContent.Name}: {pageContent.Description} - {pageContent.Content}";
|
|
var embedding = await _embeddingService.GenerateEmbeddingAsync(combinedText);
|
|
|
|
// Add data for batch insertion
|
|
|
|
vectors = embedding;
|
|
|
|
|
|
payload = new MapField<string, Value>
|
|
{
|
|
["uid"] = pageContent.UId,
|
|
["type"] = pageContent.Type,
|
|
["siteId"] = pageContent.SiteId,
|
|
//["menuItemId"] = pageContent.MenuItemId,
|
|
["name"] = pageContent.Name,
|
|
["description"] = pageContent.Description,
|
|
["content"] = pageContent.Content,
|
|
["lastUpdated"] = pageContent.LastUpdated.ToString()
|
|
};
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error processing content {pageContent.Name}: {ex.Message}");
|
|
}
|
|
|
|
|
|
await _drantService.QDrantInsertPointAsync(id, vectors, payload, "Site"+siteId);
|
|
|
|
}
|
|
|
|
private List<HtmlSnippet> GetHtmlSnippets()
|
|
{
|
|
return new List<HtmlSnippet>
|
|
{
|
|
new HtmlSnippet
|
|
{
|
|
Id = 1,
|
|
Type = "hero",
|
|
Name = "Hero Layout",
|
|
Description = "A Bootstrap hero section with an image and text.",
|
|
Html = "<div class=\"row flex-lg-row-reverse align-items-center g-5 py-5\">" +
|
|
"<div class=\"col-10 col-sm-8 col-lg-6\">" +
|
|
"<img src=\"{photo url}\" class=\"d-block mx-lg-auto img-fluid\" alt=\"{photo file name}\" width=\"700\" height=\"500\" loading=\"lazy\">" +
|
|
"</div><div class=\"col-lg-6\">" +
|
|
"<h1 class=\"display-5 fw-bold lh-1 mb-3\">{Title}</h1>" +
|
|
"<p class=\"lead\">{lead}</p>" +
|
|
"<div class=\"d-grid gap-2 d-md-flex justify-content-md-start\">" +
|
|
"<p>{paragraph}</p></div></div></div>"
|
|
},
|
|
new HtmlSnippet
|
|
{
|
|
Id = 2,
|
|
Type = "product_card",
|
|
Name = "Product Card",
|
|
Description = "A responsive product card layout using Bootstrap.",
|
|
Html = "<div class=\"animate__animated animate__bounce card\">" +
|
|
"<div class=\"container mt-3\">" +
|
|
"<img src=\"{image url}\" class=\"card-img-top\" alt=\"{image file name}\"></div>" +
|
|
"<div class=\"card-body\">" +
|
|
"<h5 class=\"card-title ms-1\">{Product title}</h5>" +
|
|
"<p class=\"card-text mb-5 ms-1\">{description}</p>" +
|
|
"<a href=\"{url}\" target=\"_blank\" class=\"btn btn-primary mb-1 mt-1\">{button text}</a>" +
|
|
"</div></div>"
|
|
},
|
|
new HtmlSnippet
|
|
{
|
|
Id = 3,
|
|
Type = "table_general",
|
|
Name = "General Table",
|
|
Description = "A responsive table to be used for displaying information in columns and rows",
|
|
Html = "<table class=\"table\">" +
|
|
"<thead><tr><th scope=\"col\">#</th><th scope=\"col\">First</th><th scope=\"col\">Last</th><th scope=\"col\">Handle</th>" +
|
|
"</tr>" +
|
|
"</thead><tbody><tr><th scope=\"row\">{cell content 1}</th><td>{cell content 2}</td><td>{cell content 3}</td><td>{cell content 3}</td></tr>" +
|
|
"</tbody>" +
|
|
"</table>"
|
|
|
|
|
|
},
|
|
new HtmlSnippet
|
|
{
|
|
Id = 4,
|
|
Type = "team_member",
|
|
Name = "Team member",
|
|
Description = "A responsive card to display employees, leaders, other team members",
|
|
Html = "<div class=\"animate__animated animate__bounce card py-3\">" +
|
|
"<div class=\"container mt-3\">" +
|
|
"<img src=\"{image url}\" class=\"card-img-top img-fluid p-3\" alt=\"{image file name}\"></div>" +
|
|
"<div class=\"card-body\">" +
|
|
"<h5 class=\"card-title ms-1\">{Person's name}</h5>" +
|
|
"<p class=\"card-text mb-5 ms-1\">{description}</p>" +
|
|
"</div></div>"
|
|
|
|
},
|
|
new HtmlSnippet
|
|
{
|
|
Id = 5,
|
|
Type = "music_player",
|
|
Name = "Music player",
|
|
Description = "A responsive music player to play a list of mp3 songs",
|
|
Html = "<div class=\"container mt-5\">" +
|
|
"<div class=\"card\">" +
|
|
"<div class=\"card-body\">" +
|
|
"<h5 class=\"card-title\">{'Now playing' in current language}</h5>" +
|
|
"<p class=\"card-text\">{Audio file name}</p>" +
|
|
"<div class=\"progress mb-3\">" +
|
|
"<div class=\"progress-bar\" role=\"progressbar\" " +
|
|
"style=\"width: 0%;\" aria-valuenow=\"0\" aria-valuemin=\"0\" " +
|
|
"aria-valuemax=\"100\"></div></div>" +
|
|
"<div class=\"d-flex justify-content-between\"><span id=\"currentTime\">0:00</span>" +
|
|
"<span id=\"duration\">0:00</span>" +
|
|
"</div><div class=\"mt-3 text-center\">" +
|
|
"<button id=\"prevBtn\" class=\"btn btn-secondary\">" +
|
|
"<i class=\"fas fa-step-backward\"></i></button>" +
|
|
"<button id=\"playPauseBtn\" class=\"btn btn-primary\"><i class=\"fas fa-play\"></i></button>" +
|
|
"<button id=\"nextBtn\" class=\"btn btn-secondary\"><i class=\"fas fa-step-forward\">" +
|
|
"</i></button></div></div></div></div><script>" +
|
|
"var playPauseBtn = document.getElementById('playPauseBtn');const progressBar = document.querySelector('.progress-bar'); " +
|
|
"var currentTimeSpan = document.getElementById('currentTime');" +
|
|
"var durationSpan = document.getElementById('duration');" +
|
|
"var audio = new Audio('{AUDIOURL}');" +
|
|
"let isPlaying = false;function togglePlayPause() {if (isPlaying) {audio.pause();" +
|
|
"playPauseBtn.innerHTML = '<i class=\"fas fa-play\"></i>';} else {audio.play();" +
|
|
"playPauseBtn.innerHTML = '<i class=\"fas fa-pause\"></i>';}isPlaying = !isPlaying;" +
|
|
"function updateProgress() {const progress = (audio.currentTime / audio.duration) * 100;" +
|
|
"progressBar.style.width = `${progress}%`;progressBar.setAttribute('aria-valuenow', progress);" +
|
|
"currentTimeSpan.textContent = formatTime(audio.currentTime);" +
|
|
"durationSpan.textContent = formatTime(audio.duration);}function formatTime(seconds) " +
|
|
"{const minutes = Math.floor(seconds / 60);const remainingSeconds = Math.floor(seconds % 60);" +
|
|
"return `${minutes}:${remainingSeconds.toString().padStart(2, '0')}`;}" +
|
|
"playPauseBtn.addEventListener('click', togglePlayPause);audio.addEventListener('timeupdate', updateProgress);" +
|
|
"</script>"
|
|
|
|
},
|
|
|
|
new HtmlSnippet
|
|
{
|
|
Id = 6,
|
|
Type = "photo_carousel",
|
|
Name = "Photo carousel",
|
|
Description = "A responsive photo carousel to display photos up not more than 5",
|
|
Html = "<div id=\"carouselExampleControls\" class=\"carousel slide\" data-bs-ride=\"carousel\">" +
|
|
"<div class=\"carousel-inner\">" +
|
|
"..."+
|
|
"<div class=\"carousel-item active\">" +
|
|
"<img src=\"{photo url}\" class=\"d-block w-100\" alt=\"{photo filename}\"></div>" +
|
|
"<div class=\"carousel-item\">" +
|
|
"<img src=\"{photo url}\" class=\"d-block w-100\" alt=\"{photo filename}\"></div>" +
|
|
"<div class=\"carousel-item\">" +
|
|
"<img src=\"{photo url}\" class=\"d-block w-100\" alt=\"{photo filename}\"></div>" +
|
|
"..."+
|
|
"</div>" +
|
|
"<button class=\"carousel-control-prev\" type=\"button\" data-bs-target=\"#carouselExampleControls\" data-bs-slide=\"prev\">" +
|
|
"<span class=\"carousel-control-prev-icon\" aria-hidden=\"true\"></span><span class=\"visually-hidden\">Previous</span></button>" +
|
|
"<button class=\"carousel-control-next\" type=\"button\" data-bs-target=\"#carouselExampleControls\" data-bs-slide=\"next\">" +
|
|
"<span class=\"carousel-control-next-icon\" aria-hidden=\"true\"></span><span class=\"visually-hidden\">Next</span></button></div>"
|
|
|
|
},
|
|
new HtmlSnippet
|
|
{
|
|
Id = 7,
|
|
Type = "event_list",
|
|
Name = "Event list",
|
|
Description = "A responsive layout for displaying events.",
|
|
Html = "<div class=\"col-md-10 mx-auto my-5\">" +
|
|
"<div class=\"event d-flex align-items-center bg-light p-4 row my-5\">" +
|
|
"<div class=\"text-center col-md-4\">" +
|
|
"<h2>{Time}</h2>" +
|
|
"<span class=\"d-block\" style=\"line-height:0;color:#9b5de5\">{Date}</span>" +
|
|
"</div><div class=\"col-md-8\">" +
|
|
"<div class=\"d-flex justify-content-between my-3\">" +
|
|
"<h4 class=\"text-capitalize\">{event name}</h4><p>" +
|
|
"<i class=\"fa-solid fa-location-dot\"></i>" +
|
|
"<span class=\"d-inline-block ml-2\" style=\"color:#9b5de5\">{Location}</span>" +
|
|
"</p></div><div>" +
|
|
"</div></div>"
|
|
}
|
|
// Add more snippets as needed
|
|
};
|
|
}
|
|
}
|
|
}
|
|
|