984 lines
37 KiB
C#
984 lines
37 KiB
C#
using BLAIzor.Data;
|
|
using BLAIzor.Models;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Microsoft.Identity.Client;
|
|
using BLAIzor.Helpers;
|
|
|
|
|
|
using Qdrant.Client.Grpc;
|
|
using System.Collections;
|
|
|
|
namespace BLAIzor.Services
|
|
{
|
|
public class ContentEditorService
|
|
{
|
|
//private readonly AIService _aiService;
|
|
private readonly OpenAIApiService _openAIApiService;
|
|
//private readonly ApplicationDbContext _context;
|
|
private readonly QDrantService _qDrantService;
|
|
private readonly HtmlSnippetProcessor _htmlSnippetProcessor;
|
|
private readonly IServiceScopeFactory _serviceScopeFactory;
|
|
private readonly ScopedContentService _scopedContentService;
|
|
private readonly OpenAIEmbeddingService _openAIEmbeddingService;
|
|
private readonly LocalEmbeddingService _localEmbeddingService;
|
|
|
|
public static IConfiguration? _configuration;
|
|
|
|
public ContentEditorService(/*AIService aiService,*/ OpenAIApiService openAIApiService, /*ApplicationDbContext context,*/ QDrantService qDrantService, OpenAIEmbeddingService openAIEmbeddingService, LocalEmbeddingService localEmbeddingService, HtmlSnippetProcessor htmlSnippetProcessor, IServiceScopeFactory serviceScopeFactory, ScopedContentService scopedContentService, IConfiguration? configuration)
|
|
{
|
|
//_aiService = aiService;
|
|
_openAIApiService = openAIApiService;
|
|
//_context = context;
|
|
_qDrantService = qDrantService;
|
|
_openAIEmbeddingService = openAIEmbeddingService;
|
|
_localEmbeddingService = localEmbeddingService;
|
|
_htmlSnippetProcessor = htmlSnippetProcessor;
|
|
_serviceScopeFactory = serviceScopeFactory;
|
|
_scopedContentService = scopedContentService;
|
|
_configuration = configuration;
|
|
}
|
|
|
|
private string GetAiEmbeddingSettings() =>
|
|
_configuration?.GetSection("AiSettings")?.GetValue<string>("EmbeddingService") ?? string.Empty;
|
|
|
|
// CRUD methods for MenuItems
|
|
|
|
// Create a new MenuItem
|
|
public async Task<MenuItem> AddMenuItemAsync(MenuItem menuItem)
|
|
{
|
|
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
|
|
var result = await _context.MenuItems.AddAsync(menuItem);
|
|
await _context.SaveChangesAsync();
|
|
return result.Entity;
|
|
}
|
|
|
|
}
|
|
|
|
/// <summary>
|
|
/// Adds items as list
|
|
/// </summary>
|
|
/// <param name="menuItems"></param>
|
|
/// <returns>the number of modified rows</returns>
|
|
public async Task<int> AddMenuItemsAsync(List<MenuItem> menuItems)
|
|
{
|
|
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
_context.MenuItems.AddRange(menuItems);
|
|
var result = await _context.SaveChangesAsync();
|
|
return result;
|
|
}
|
|
|
|
}
|
|
|
|
// Get all MenuItems for a specific SiteInfo
|
|
public async Task<List<MenuItem>> GetMenuItemsBySiteIdAsync(int siteInfoId)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
var result = await _context.MenuItems
|
|
.Where(m => m.SiteInfoId == siteInfoId)
|
|
.ToListAsync();
|
|
return result;
|
|
}
|
|
|
|
}
|
|
|
|
//GET MENUS WITH CHILDREN
|
|
public async Task<List<MenuItem>> GetMenuItemsBySiteIdWithChildrenAsync(int siteInfoId)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
var result = await _context.MenuItems
|
|
.Where(m => m.SiteInfoId == siteInfoId && m.ParentId == null)
|
|
.Include(m => m.Children)
|
|
.ToListAsync();
|
|
return result;
|
|
}
|
|
|
|
}
|
|
|
|
// Update an existing MenuItem
|
|
public async Task<MenuItem> UpdateMenuItemAsync(MenuItem menuItem)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
var existingMenuItem = await _context.MenuItems.FindAsync(menuItem.Id);
|
|
if (existingMenuItem == null)
|
|
{
|
|
throw new Exception("MenuItem not found.");
|
|
}
|
|
|
|
existingMenuItem.Name = menuItem.Name;
|
|
existingMenuItem.Slug = menuItem.Slug;
|
|
existingMenuItem.ParentId = menuItem.ParentId;
|
|
existingMenuItem.StoredHtml = menuItem.StoredHtml;
|
|
existingMenuItem.SortOrder = menuItem.SortOrder;
|
|
existingMenuItem.ShowInMainMenu = menuItem.ShowInMainMenu;
|
|
|
|
// No need for _context.MenuItems.Update(existingMenuItem);
|
|
await _context.SaveChangesAsync();
|
|
|
|
return existingMenuItem;
|
|
}
|
|
|
|
// Delete a MenuItem
|
|
public async Task DeleteMenuItemAsync(int menuItemId)
|
|
{
|
|
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
var menuItem = await _context.MenuItems.FindAsync(menuItemId);
|
|
if (menuItem == null)
|
|
{
|
|
throw new Exception("MenuItem not found.");
|
|
}
|
|
|
|
_context.MenuItems.Remove(menuItem);
|
|
await _context.SaveChangesAsync();
|
|
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Retrieves the first ContentGroup associated with the given SiteInfoId.
|
|
/// Returns null if not found.
|
|
/// </summary>
|
|
public async Task<ContentGroup?> GetContentGroupBySiteInfoIdAsync(int siteInfoId)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
var result = await _context.ContentGroups
|
|
.Where(cg => cg.SiteInfoId == siteInfoId)
|
|
.OrderBy(cg => cg.Id) // or by LastUpdated, Name, etc.
|
|
.FirstOrDefaultAsync();
|
|
return result;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns the first ContentGroup for a site (can be filtered by type or slug).
|
|
/// </summary>
|
|
public async Task<List<ContentGroup>> GetContentGroupsBySiteInfoIdAsync(int siteInfoId, string? type = null, string? slug = null)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
var query = _context.ContentGroups
|
|
.Include(cg => cg.SiteInfo)
|
|
.Include(cg => cg.Items)
|
|
.ThenInclude(ci => ci.Chunks)
|
|
.AsQueryable();
|
|
|
|
query = query.Where(cg => cg.SiteInfoId == siteInfoId);
|
|
|
|
if (!string.IsNullOrWhiteSpace(type))
|
|
query = query.Where(cg => cg.Type == type);
|
|
|
|
if (!string.IsNullOrWhiteSpace(slug))
|
|
query = query.Where(cg => cg.Slug == slug);
|
|
|
|
return query.ToList();
|
|
}
|
|
}
|
|
|
|
public async Task<ContentGroup> UpdateContentGroupByIdAsync(ContentGroup updatedGroup)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
var existingGroup = await _context.ContentGroups.FindAsync(updatedGroup.Id);
|
|
if (existingGroup == null)
|
|
throw new Exception("ContentGroup not found.");
|
|
|
|
existingGroup.Name = updatedGroup.Name;
|
|
existingGroup.Slug = updatedGroup.Slug;
|
|
existingGroup.Type = updatedGroup.Type;
|
|
existingGroup.VectorSize = updatedGroup.VectorSize;
|
|
existingGroup.EmbeddingModel = updatedGroup.EmbeddingModel;
|
|
existingGroup.LastUpdated = DateTime.UtcNow;
|
|
existingGroup.Version = updatedGroup.Version;
|
|
|
|
_context.ContentGroups.Update(existingGroup);
|
|
await _context.SaveChangesAsync();
|
|
|
|
return existingGroup;
|
|
}
|
|
|
|
public async Task<bool> DeleteContentGroupByIdAsync(int contentGroupId)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
var group = await _context.ContentGroups.FindAsync(contentGroupId);
|
|
if (group == null)
|
|
return false;
|
|
|
|
_context.ContentGroups.Remove(group);
|
|
await _context.SaveChangesAsync();
|
|
|
|
return true;
|
|
}
|
|
|
|
public async Task<ContentGroup> CreateContentGroupAsync(ContentGroup newGroup)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
newGroup.CreatedAt = DateTime.UtcNow;
|
|
newGroup.LastUpdated = DateTime.UtcNow;
|
|
|
|
_context.ContentGroups.Add(newGroup);
|
|
await _context.SaveChangesAsync();
|
|
|
|
return newGroup;
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// Returns all ContentGroups for a site (can be filtered by type).
|
|
/// </summary>
|
|
public async Task<List<ContentGroup>> GetAllContentGroupsBySiteInfoIdAsync(int siteInfoId, string? type = null)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
var query = _context.ContentGroups
|
|
.Where(cg => cg.SiteInfoId == siteInfoId);
|
|
|
|
if (!string.IsNullOrWhiteSpace(type))
|
|
query = query.Where(cg => cg.Type == type);
|
|
|
|
return await query
|
|
.OrderBy(cg => cg.Name)
|
|
.ToListAsync();
|
|
}
|
|
}
|
|
|
|
public async Task<ContentItem> GetContentItemByIdAsync(int contentItemId, string? type = null)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
var query = _context.ContentItems
|
|
.Where(cg => cg.Id == contentItemId).Include(ci => ci.Chunks);
|
|
|
|
return query.FirstOrDefault();
|
|
}
|
|
}
|
|
|
|
public async Task<ContentItem?> GetContentItemByIdAsync(int id)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
var query = db.ContentItems.Where(x => x.Id == id).Include(ci => ci.ContentGroup).Include(ci => ci.Chunks).Include(ci => ci.ContentGroup.SiteInfo);
|
|
return query.FirstOrDefault();
|
|
}
|
|
|
|
//public async Task<ContentItem> UpdateContentItemByIdAsync(ContentItem item)
|
|
//{
|
|
// using var scope = _serviceScopeFactory.CreateScope();
|
|
// var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// var existing = await db.ContentItems.FindAsync(item.Id);
|
|
// if (existing == null) throw new Exception("ContentItem not found.");
|
|
|
|
// // Update properties manually (or use AutoMapper if you prefer)
|
|
// existing.Title = item.Title;
|
|
// existing.Description = item.Description;
|
|
// existing.Content = item.Content;
|
|
// existing.Language = item.Language;
|
|
// existing.Tags = item.Tags;
|
|
// existing.IsPublished = item.IsPublished;
|
|
// existing.Version = item.Version;
|
|
// existing.LastUpdated = item.LastUpdated;
|
|
|
|
// await db.SaveChangesAsync();
|
|
// return existing;
|
|
//}
|
|
|
|
public async Task<ContentItem> CreateContentItemAsync(ContentItem item, string collectionName)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
item.CreatedAt = DateTime.UtcNow;
|
|
item.LastUpdated = DateTime.UtcNow;
|
|
|
|
// Save to DB first to get ContentItem.Id
|
|
db.ContentItems.Add(item);
|
|
await db.SaveChangesAsync();
|
|
|
|
// Get the full ContentItem with ContentGroup included (needed for SiteId)
|
|
var fullItem = await db.ContentItems
|
|
.Include(ci => ci.ContentGroup)
|
|
.FirstOrDefaultAsync(ci => ci.Id == item.Id);
|
|
|
|
if (fullItem == null)
|
|
throw new Exception("Failed to retrieve saved ContentItem.");
|
|
|
|
// 🧠 Chunking
|
|
if (!string.IsNullOrEmpty(item.Content))
|
|
{
|
|
var chunks = ChunkingHelper.SplitStructuredText(item.Content, 3000); // customize if needed
|
|
var vectorizedChunks = await VectorizeChunksWithGuidsAsync(chunks, fullItem, collectionName);
|
|
|
|
// 🔗 Save chunk references
|
|
var chunkEntities = vectorizedChunks.Select((chunk, index) => new ContentChunk
|
|
{
|
|
ContentItemId = item.Id,
|
|
QdrantPointId = chunk.UId, // now using GUIDs
|
|
ChunkIndex = index,
|
|
CreatedAt = DateTime.UtcNow
|
|
}).ToList();
|
|
|
|
db.ContentChunks.AddRange(chunkEntities);
|
|
}
|
|
await db.SaveChangesAsync();
|
|
|
|
return item;
|
|
}
|
|
|
|
|
|
|
|
|
|
public async Task<List<WebPageContent>> VectorizeChunksWithGuidsAsync(List<string> chunks, ContentItem item, string collectionName)
|
|
{
|
|
var result = new List<WebPageContent>();
|
|
|
|
foreach (var (chunk, index) in chunks.Select((c, i) => (c, i)))
|
|
{
|
|
var combinedText = chunk;
|
|
float[] embedding = [];
|
|
|
|
var embeddingServiceProvider = GetAiEmbeddingSettings();
|
|
if (embeddingServiceProvider == "local")
|
|
{
|
|
embedding = await _localEmbeddingService.GenerateEmbeddingAsync(combinedText);
|
|
}
|
|
else
|
|
{
|
|
embedding = await _openAIEmbeddingService.GenerateEmbeddingAsync(combinedText);
|
|
}
|
|
|
|
var uid = Guid.NewGuid().ToString();
|
|
|
|
result.Add(new WebPageContent
|
|
{
|
|
UId = uid,
|
|
SiteId = item.ContentGroup.SiteInfoId,
|
|
Type = "content-item",
|
|
Name = item.Title,
|
|
Description = item.Description,
|
|
Content = chunk,
|
|
Vectors = embedding,
|
|
LastUpdated = DateTime.UtcNow
|
|
});
|
|
}
|
|
|
|
await _qDrantService.QDrantInsertManyAsync(result, collectionName);
|
|
return result;
|
|
}
|
|
|
|
|
|
public async Task<List<WebPageContent>> VectorizeChunksAsync(List<string> chunks, ContentItem item, string collectionName)
|
|
{
|
|
var result = new List<WebPageContent>();
|
|
|
|
foreach (var (chunk, index) in chunks.Select((c, i) => (c, i)))
|
|
{
|
|
//var combinedText = $"{pageContent.Name}: {pageContent.Description} - {chunk}";
|
|
var combinedText = $"{chunk}";
|
|
float[] embedding = [];
|
|
|
|
var embeddingServiceProvider = GetAiEmbeddingSettings();
|
|
if (embeddingServiceProvider == "local")
|
|
{
|
|
embedding = await _localEmbeddingService.GenerateEmbeddingAsync(combinedText);
|
|
}
|
|
else
|
|
{
|
|
embedding = await _openAIEmbeddingService.GenerateEmbeddingAsync(combinedText);
|
|
}
|
|
|
|
// Add data for batch insertion
|
|
|
|
var vector = embedding;
|
|
|
|
var uid = Guid.NewGuid();
|
|
|
|
var webChunk = new WebPageContent
|
|
{
|
|
Id = uid,
|
|
UId = uid.ToString(),
|
|
SiteId = item.ContentGroup.SiteInfoId,
|
|
Type = "content-item",
|
|
Name = item.Title,
|
|
Description = item.Description,
|
|
Content = chunk,
|
|
Vectors = vector,
|
|
LastUpdated = DateTime.UtcNow
|
|
};
|
|
|
|
|
|
result.Add(webChunk);
|
|
}
|
|
await _qDrantService.QDrantInsertManyAsync(result, collectionName);
|
|
return result;
|
|
}
|
|
|
|
|
|
private async Task RemoveChunksAndQdrantEntriesByIdsAsync(List<int> chunkIds, string collectionName)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
var chunks = await db.ContentChunks
|
|
.Where(c => chunkIds.Contains(c.Id))
|
|
.ToListAsync();
|
|
|
|
var pointIds = chunks.Select(c => Guid.Parse(c.QdrantPointId)).ToArray();
|
|
await _qDrantService.DeletePointsAsync(pointIds, collectionName);
|
|
|
|
db.ContentChunks.RemoveRange(chunks);
|
|
await db.SaveChangesAsync();
|
|
}
|
|
|
|
public async Task<bool> ForceRechunkContentGroupAsync(int contentGroupId, string collectionName = null)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
var group = await db.ContentGroups
|
|
.Include(g => g.Items)
|
|
.ThenInclude(ci => ci.Chunks)
|
|
.Include(g => g.SiteInfo)
|
|
.FirstOrDefaultAsync(g => g.Id == contentGroupId);
|
|
|
|
if (group == null)
|
|
return false;
|
|
|
|
foreach (var item in group.Items)
|
|
{
|
|
await SaveAndSyncContentItemAsync(item, collectionName, forceRechunk: true);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
public async Task<bool> ForceRecreateQdrantCollectionAsync(int siteInfoId)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Load site + all content + chunks
|
|
var site = await db.SiteInfos
|
|
.Include(s => s.ContentGroups)
|
|
.ThenInclude(g => g.Items)
|
|
.ThenInclude(i => i.Chunks)
|
|
.FirstOrDefaultAsync(s => s.Id == siteInfoId);
|
|
|
|
if (site == null)
|
|
return false;
|
|
|
|
string oldCollectionName = site.VectorCollectionName;
|
|
|
|
// ❌ Delete old Qdrant collection
|
|
await _qDrantService.DeleteCollectionAsync(oldCollectionName);
|
|
|
|
// ✅ Generate and create new collection
|
|
string newCollectionName = GetGeneratedVectorCollectionName(site);
|
|
|
|
bool created = await _qDrantService.CreateQdrantCollectionAsync(newCollectionName);
|
|
if (!created)
|
|
throw new Exception($"Failed to create Qdrant collection '{newCollectionName}'.");
|
|
|
|
// Update and persist new collection name
|
|
site.VectorCollectionName = newCollectionName;
|
|
await UpdateSiteInfoAsync(site); // Assumes this method updates it properly
|
|
|
|
// ♻️ Rechunk and reinsert all content items using fresh DTOs
|
|
foreach (var group in site.ContentGroups)
|
|
{
|
|
foreach (var item in group.Items.ToList())
|
|
{
|
|
var dto = new ContentItem
|
|
{
|
|
Id = item.Id,
|
|
Title = item.Title,
|
|
Description = item.Description,
|
|
Content = item.Content,
|
|
Language = item.Language,
|
|
Tags = item.Tags,
|
|
IsPublished = item.IsPublished,
|
|
Version = item.Version,
|
|
LastUpdated = item.LastUpdated,
|
|
ContentGroupId = item.ContentGroupId
|
|
};
|
|
|
|
await SaveAndSyncContentItemAsync(dto, newCollectionName, forceRechunk: true);
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
|
|
|
|
public async Task<List<ContentItem>> GetContentItemsByGroupIdAsync(int contentGroupId)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
return await context.ContentItems
|
|
.Where(ci => ci.ContentGroupId == contentGroupId)
|
|
.OrderByDescending(ci => ci.LastUpdated)
|
|
.ToListAsync();
|
|
}
|
|
}
|
|
|
|
public async Task<ContentItem?> CreateContentItemAsync(ContentItem newItem)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
newItem.CreatedAt = DateTime.UtcNow;
|
|
newItem.LastUpdated = DateTime.UtcNow;
|
|
|
|
_context.ContentItems.Add(newItem);
|
|
await _context.SaveChangesAsync();
|
|
|
|
return newItem;
|
|
}
|
|
|
|
public async Task<ContentItem> SaveAndSyncContentItemAsync(ContentItem dto, string collectionName, bool forceRechunk = false)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
var existing = await db.ContentItems
|
|
.Include(ci => ci.Chunks)
|
|
.Include(ci => ci.ContentGroup)
|
|
.FirstOrDefaultAsync(ci => ci.Id == dto.Id);
|
|
|
|
if (existing == null)
|
|
throw new Exception("ContentItem not found.");
|
|
|
|
// Determine if content has changed
|
|
bool contentChanged = existing.Content?.Trim() != dto.Content?.Trim();
|
|
|
|
// Update scalar fields only (don't touch navigation collections directly)
|
|
existing.Title = dto.Title;
|
|
existing.Description = dto.Description;
|
|
existing.Content = dto.Content;
|
|
existing.Language = dto.Language;
|
|
existing.Tags = dto.Tags;
|
|
existing.IsPublished = dto.IsPublished;
|
|
existing.LastUpdated = dto.LastUpdated;
|
|
|
|
if (contentChanged || forceRechunk)
|
|
{
|
|
existing.Version++;
|
|
|
|
// 🧹 Remove old chunks (track-safe)
|
|
var chunkIds = existing.Chunks.Select(c => c.Id).ToList();
|
|
if (chunkIds.Any())
|
|
{
|
|
await RemoveChunksAndQdrantEntriesByIdsAsync(chunkIds, collectionName);
|
|
|
|
// Detach removed chunks from current EF context to avoid SaveChanges conflict
|
|
foreach (var chunk in existing.Chunks.ToList())
|
|
{
|
|
db.Entry(chunk).State = EntityState.Detached;
|
|
}
|
|
|
|
existing.Chunks.Clear(); // avoid tracking conflicts
|
|
}
|
|
|
|
// 🔪 Chunk and embed again
|
|
var newChunks = ChunkingHelper.SplitStructuredText(existing.Content, 3000);
|
|
var webChunks = await VectorizeChunksAsync(newChunks, existing, collectionName);
|
|
|
|
// 💾 Save chunk metadata in SQL
|
|
var chunkEntities = webChunks.Select((w, i) => new ContentChunk
|
|
{
|
|
ContentItemId = existing.Id,
|
|
ChunkIndex = i,
|
|
QdrantPointId = w.Id.Uuid,
|
|
CreatedAt = DateTime.UtcNow
|
|
}).ToList();
|
|
|
|
db.ContentChunks.AddRange(chunkEntities);
|
|
}
|
|
|
|
await db.SaveChangesAsync();
|
|
return existing;
|
|
}
|
|
|
|
|
|
public static Dictionary<string, List<ContentItemModel>> GroupContentItemsByType(WebsiteContentModel model)
|
|
{
|
|
return model.ContentItems
|
|
.GroupBy(ci => ci.ContentItem.ContentGroup.Type)
|
|
.ToDictionary(g => g.Key, g => g.ToList());
|
|
}
|
|
|
|
|
|
|
|
public async Task<bool> DeleteContentItemByIdAsync(int id)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
var item = await _context.ContentItems.FindAsync(id);
|
|
if (item != null)
|
|
{
|
|
_context.ContentItems.Remove(item);
|
|
await _context.SaveChangesAsync();
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
|
|
|
|
public async Task<int[]> GetPointIdsByContentGroupIdAsync(int contentGroupId)
|
|
{
|
|
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
return await _context.ContentChunks
|
|
.Where(chunk => chunk.ContentItem.ContentGroupId == contentGroupId)
|
|
.Select(chunk => chunk.ChunkIndex)
|
|
.ToArrayAsync();
|
|
}
|
|
}
|
|
|
|
|
|
public string GetGeneratedVectorCollectionName(SiteInfo siteInfo)
|
|
{
|
|
var safeName = siteInfo.SiteName?.ToLower().Replace(" ", "_").Replace("-", "_");
|
|
return $"site_{safeName}_{Guid.NewGuid().ToString().Substring(0, 8)}";
|
|
}
|
|
|
|
|
|
public async Task<SiteInfo> GetSiteInfoByIdAsync(int SiteInfoId)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
var result = await _context.SiteInfos.Where(x => x.Id == SiteInfoId).FirstOrDefaultAsync();
|
|
if (result == null)
|
|
{
|
|
return await _context.SiteInfos.FirstOrDefaultAsync();
|
|
}
|
|
else
|
|
{
|
|
return result;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
public async Task<SiteInfo?> GetSiteInfoWithFormsByIdAsync(int siteId)
|
|
{
|
|
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
var result = await _context.SiteInfos.Include(s => s.FormDefinitions).FirstOrDefaultAsync(s => s.Id == siteId);
|
|
if (result == null)
|
|
{
|
|
return new SiteInfo();
|
|
}
|
|
else
|
|
{
|
|
return result;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
public async Task<SiteInfo> GetSiteInfoByNameAsync(string siteName)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
var result = await _context.SiteInfos.Where(x => x.SiteName == siteName).FirstOrDefaultAsync();
|
|
if (result == null)
|
|
{
|
|
return await _context.SiteInfos.FirstOrDefaultAsync();
|
|
}
|
|
else
|
|
{
|
|
return result;
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
public async Task<SiteInfo> GetSiteInfoByUrlAsync(string url)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
var result = await _context.SiteInfos.Where(x => x.DefaultUrl == url).FirstOrDefaultAsync();
|
|
if (result == null)
|
|
{
|
|
return await _context.SiteInfos.FirstOrDefaultAsync();
|
|
}
|
|
else
|
|
{
|
|
return result;
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
public async Task UpdateSiteInfoAsync(SiteInfo siteInfo)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
_context.SiteInfos.Update(siteInfo);
|
|
await _context.SaveChangesAsync();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
//collectionName change detection... do we really need to check? it should not be manually set.. if set, whole rechunk could be intitated...
|
|
//public async Task UpdateSiteInfoAndReChunkIfNeededAsync(SiteInfo updatedSiteInfo)
|
|
//{
|
|
// using var scope = _serviceScopeFactory.CreateScope();
|
|
// var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// var existing = await _context.SiteInfos
|
|
// .AsNoTracking()
|
|
// .FirstOrDefaultAsync(s => s.Id == updatedSiteInfo.Id);
|
|
|
|
// if (existing == null)
|
|
// throw new Exception("SiteInfo not found.");
|
|
|
|
// bool collectionChanged = existing.VectorCollectionName != updatedSiteInfo.VectorCollectionName;
|
|
|
|
// _context.SiteInfos.Update(updatedSiteInfo);
|
|
// await _context.SaveChangesAsync();
|
|
|
|
// if (collectionChanged)
|
|
// {
|
|
// // Create new collection if needed
|
|
// bool exists = await _qDrantService.CollectionExistsAsync(updatedSiteInfo.VectorCollectionName);
|
|
// if (!exists)
|
|
// {
|
|
// bool created = await _qDrantService.CreateQdrantCollectionAsync(updatedSiteInfo.VectorCollectionName);
|
|
// if (!created)
|
|
// throw new Exception("Failed to create new Qdrant collection.");
|
|
// }
|
|
|
|
// // Load all ContentItems for this site
|
|
// var contentItems = await _context.ContentItems
|
|
// .Include(ci => ci.ContentGroup)
|
|
// .Where(ci => ci.ContentGroup.SiteInfoId == updatedSiteInfo.Id)
|
|
// .ToListAsync();
|
|
|
|
// foreach (var item in contentItems)
|
|
// {
|
|
// // Rechunk into new collection
|
|
// await SaveAndSyncContentItemAsync(item, updatedSiteInfo.VectorCollectionName, true);
|
|
// }
|
|
// }
|
|
//}
|
|
|
|
|
|
|
|
|
|
public async Task<List<SiteInfo>> GetUserSitesAsync(string userId)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
return await _context.SiteInfos
|
|
.Where(s => s.UserId == userId)
|
|
.ToListAsync();
|
|
|
|
}
|
|
}
|
|
|
|
//public async Task<List<SiteInfo>> GetSitesAsync()
|
|
//{
|
|
// using (var scope = _serviceScopeFactory.CreateScope())
|
|
// {
|
|
// var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// // Now use dbContext safely without violating DI rules
|
|
// return await _context.SiteInfos.ToListAsync();
|
|
|
|
// }
|
|
|
|
//}
|
|
|
|
public async Task<SiteInfo> AddSiteInfoAsync(SiteInfo siteInfo)
|
|
{
|
|
using (var scope = _serviceScopeFactory.CreateScope())
|
|
{
|
|
var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
|
|
// Now use dbContext safely without violating DI rules
|
|
if (siteInfo == null)
|
|
throw new ArgumentNullException(nameof(siteInfo), "SiteInfo cannot be null.");
|
|
|
|
try
|
|
{
|
|
|
|
if (string.IsNullOrEmpty(siteInfo.VectorCollectionName))
|
|
{
|
|
siteInfo.VectorCollectionName = GetGeneratedVectorCollectionName(siteInfo);
|
|
}
|
|
|
|
await _context.SiteInfos.AddAsync(siteInfo);
|
|
var result = await _context.SaveChangesAsync();
|
|
if (result > 0)
|
|
{
|
|
//check if collection exists already
|
|
bool checkResult = await _qDrantService.CollectionExistsAsync(siteInfo.VectorCollectionName);
|
|
if (checkResult)
|
|
{
|
|
//collection already exists (shouldn't exists, so it is occupied...), we need to create a new one.
|
|
//collection does not exist, create it
|
|
siteInfo.VectorCollectionName = GetGeneratedVectorCollectionName(siteInfo);
|
|
//update the site info with the new collection name
|
|
_context.SiteInfos.Update(siteInfo);
|
|
await _context.SaveChangesAsync();
|
|
bool qresult = await _qDrantService.CreateQdrantCollectionAsync(siteInfo.VectorCollectionName);
|
|
if (!qresult)
|
|
{
|
|
throw new Exception("Failed to create Qdrant collection for the site.");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//collection does not exist, create it
|
|
bool qresult = await _qDrantService.CreateQdrantCollectionAsync(siteInfo.VectorCollectionName);
|
|
if (!qresult)
|
|
{
|
|
throw new Exception("Failed to create Qdrant collection for the site.");
|
|
}
|
|
}
|
|
}
|
|
return siteInfo;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Log the exception (using a logging framework like Serilog, NLog, etc.)
|
|
throw new InvalidOperationException("An error occurred while adding the site info.", ex);
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
///TEMPORARY
|
|
///
|
|
public async Task<bool> MigrateQdrantToContentItemsAsync(int siteId, string collectionName)
|
|
{
|
|
using var scope = _serviceScopeFactory.CreateScope();
|
|
var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
|
|
var site = await GetSiteInfoByIdAsync(siteId);
|
|
// 1. Get menu items
|
|
var menuItems = await GetMenuItemsBySiteIdAsync(siteId);
|
|
PointId[] pointIds = new PointId[menuItems.Count];
|
|
for (int i=0; i < menuItems.Count; i++)
|
|
{
|
|
// Ensure SortOrder is set to i+1 (1-based index)
|
|
pointIds[i] = Convert.ToUInt64(menuItems[i].SortOrder);
|
|
}
|
|
|
|
if (!pointIds.Any())
|
|
return false;
|
|
|
|
// 2. Get points from Qdrant
|
|
var qdrantPoints = await _qDrantService.GetPointsFromQdrantAsyncByIntegerPointIds(collectionName, pointIds);
|
|
|
|
// 3. Create new content group
|
|
var contentGroup = new ContentGroup
|
|
{
|
|
Name = "Pages",
|
|
Type = "page",
|
|
SiteInfoId = siteId,
|
|
CreatedAt = DateTime.UtcNow,
|
|
EmbeddingModel = "openai",
|
|
Slug = $"group-{DateTime.UtcNow.Ticks}",
|
|
Version = 0,
|
|
LastUpdated = DateTime.UtcNow
|
|
};
|
|
db.ContentGroups.Add(contentGroup);
|
|
await db.SaveChangesAsync(); // Save to get ID
|
|
|
|
// 4. Create ContentItems from Qdrant points
|
|
foreach (var point in qdrantPoints)
|
|
{
|
|
var item = new ContentItem
|
|
{
|
|
ContentGroupId = contentGroup.Id,
|
|
Title = point.Name,
|
|
Description = point.Description,
|
|
Content = point.Content,
|
|
Language = "en",
|
|
Tags = "",
|
|
IsPublished = true,
|
|
Version = 1,
|
|
CreatedAt = DateTime.UtcNow,
|
|
LastUpdated = DateTime.UtcNow
|
|
};
|
|
|
|
db.ContentItems.Add(item);
|
|
}
|
|
|
|
await db.SaveChangesAsync();
|
|
return true;
|
|
}
|
|
|
|
|
|
|
|
}
|
|
}
|