SeemGen/Services/ContentEditorService.cs

using BLAIzor.Data;
using BLAIzor.Models;
using Microsoft.EntityFrameworkCore;
using Microsoft.Identity.Client;
using BLAIzor.Helpers;
using Qdrant.Client.Grpc;
using System.Collections;

namespace BLAIzor.Services
{


    public class ContentEditorService
    {
        #region Private Fields

        private readonly OpenAIApiService _openAIApiService;
        private readonly QDrantService _qDrantService;
        private readonly HtmlSnippetProcessor _htmlSnippetProcessor;
        private readonly IServiceScopeFactory _serviceScopeFactory;
        private readonly ScopedContentService _scopedContentService;
        private readonly OpenAIEmbeddingService _openAIEmbeddingService;
        private readonly LocalEmbeddingService _localEmbeddingService;
        private readonly ISimpleLogger _logger;


        public static IConfiguration? _configuration;

        #endregion

        #region Constructor

        /// <summary>
        /// Initializes a new instance of the <see cref="ContentEditorService"/> class.
        /// </summary>
        /// <param name="openAIApiService">The OpenAI API service.</param>
        /// <param name="qDrantService">The Qdrant service.</param>
        /// <param name="openAIEmbeddingService">The OpenAI embedding service.</param>
        /// <param name="localEmbeddingService">The local embedding service.</param>
        /// <param name="htmlSnippetProcessor">The HTML snippet processor.</param>
        /// <param name="serviceScopeFactory">The service scope factory for creating database contexts.</param>
        /// <param name="scopedContentService">The scoped content service.</param>
        /// <param name="configuration">The application configuration.</param>
        public ContentEditorService(OpenAIApiService openAIApiService,
            QDrantService qDrantService,
            OpenAIEmbeddingService openAIEmbeddingService,
            LocalEmbeddingService localEmbeddingService,
            HtmlSnippetProcessor htmlSnippetProcessor,
            IServiceScopeFactory serviceScopeFactory,
            ScopedContentService scopedContentService,
            IConfiguration? configuration,
            ISimpleLogger logger)
        {
            _openAIApiService = openAIApiService;
            _qDrantService = qDrantService;
            _openAIEmbeddingService = openAIEmbeddingService;
            _localEmbeddingService = localEmbeddingService;
            _htmlSnippetProcessor = htmlSnippetProcessor;
            _serviceScopeFactory = serviceScopeFactory;
            _scopedContentService = scopedContentService;
            _configuration = configuration;
            _logger = logger;
        }

        #endregion

        #region Private Helper Methods

        /// <summary>
        /// Retrieves the AI embedding service setting from the configuration.
        /// </summary>
        /// <returns>The name of the embedding service (e.g., "local", "openai").</returns>
        private string GetAiEmbeddingSettings() =>
            _configuration?.GetSection("AiSettings")?.GetValue<string>("EmbeddingService") ?? string.Empty;

        /// <summary>
        /// Removes content chunks and their corresponding Qdrant entries by their IDs.
        /// </summary>
        /// <param name="chunkIds">A list of content chunk IDs to remove.</param>
        /// <param name="collectionName">The name of the Qdrant collection.</param>
        /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
        private async Task RemoveChunksAndQdrantEntriesByIdsAsync(List<int> chunkIds, string collectionName)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            var chunks = await db.ContentChunks
                .Where(c => chunkIds.Contains(c.Id))
                .ToListAsync();

            var pointIds = chunks.Select(c => Guid.Parse(c.QdrantPointId)).ToArray();
            await _qDrantService.DeletePointsAsync(pointIds, collectionName);

            db.ContentChunks.RemoveRange(chunks);
            await db.SaveChangesAsync();
        }

        /// <summary>
        /// Generates a unique vector collection name for a given site.
        /// </summary>
        /// <param name="siteInfo">The site information.</param>
        /// <returns>A generated vector collection name.</returns>
        public string GetGeneratedVectorCollectionName(SiteInfo siteInfo)
        {
            var safeName = siteInfo.SiteName?.ToLower().Replace(" ", "_").Replace("-", "_");
            return $"site_{safeName}_{Guid.NewGuid().ToString().Substring(0, 8)}";
        }

        #endregion

        #region MenuItem CRUD Operations

        /// <summary>
        /// Adds a new menu item to the database.
        /// </summary>
        /// <param name="menuItem">The menu item to add.</param>
        /// <returns>The added menu item with its generated ID.</returns>
        public async Task<MenuItem> AddMenuItemAsync(MenuItem menuItem)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var result = await _context.MenuItems.AddAsync(menuItem);
                await _context.SaveChangesAsync();
                return result.Entity;
            }
        }

        /// <summary>
        /// Adds multiple menu items to the database.
        /// </summary>
        /// <param name="menuItems">A list of menu items to add.</param>
        /// <returns>The number of state entries written to the database.</returns>
        public async Task<int> AddMenuItemsAsync(List<MenuItem> menuItems)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                _context.MenuItems.AddRange(menuItems);
                var result = await _context.SaveChangesAsync();
                return result;
            }
        }

        /// <summary>
        /// Retrieves all menu items for a specific site.
        /// </summary>
        /// <param name="siteInfoId">The ID of the site.</param>
        /// <returns>A list of menu items belonging to the specified site.</returns>
        public async Task<List<MenuItem>> GetMenuItemsBySiteIdAsync(int siteInfoId)
        {
            await _logger.InfoAsync($"GetMenuItemsBySiteIdAsync: method called", siteInfoId.ToString());
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var result = await _context.MenuItems
                    .Where(m => m.SiteInfoId == siteInfoId)
                    .ToListAsync();
                return result;
            }
        }

        /// <summary>
        /// Retrieves all top-level menu items for a specific site, including their children.
        /// </summary>
        /// <param name="siteInfoId">The ID of the site.</param>
        /// <returns>A list of top-level menu items with their associated children.</returns>
        public async Task<List<MenuItem>> GetMenuItemsBySiteIdWithChildrenAsync(int siteInfoId)
        {
            await _logger.InfoAsync($"GetMenuItemsBySiteIdWithChildrenAsync: method called", siteInfoId.ToString());
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var result = await _context.MenuItems
                    .Where(m => m.SiteInfoId == siteInfoId && m.ParentId == null)
                    .Include(m => m.Children)
                    .ToListAsync();
                return result;
            }
        }

        /// <summary>
        /// Updates an existing menu item in the database.
        /// </summary>
        /// <param name="menuItem">The menu item with updated information.</param>
        /// <returns>The updated menu item.</returns>
        /// <exception cref="Exception">Thrown if the menu item is not found.</exception>
        public async Task<MenuItem> UpdateMenuItemAsync(MenuItem menuItem)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            var existingMenuItem = await _context.MenuItems.FindAsync(menuItem.Id);
            if (existingMenuItem == null)
            {
                throw new Exception("MenuItem not found.");
            }

            existingMenuItem.Name = menuItem.Name;
            existingMenuItem.Slug = menuItem.Slug;
            existingMenuItem.ParentId = menuItem.ParentId;
            existingMenuItem.StoredHtml = menuItem.StoredHtml;
            existingMenuItem.SortOrder = menuItem.SortOrder;
            existingMenuItem.ShowInMainMenu = menuItem.ShowInMainMenu;

            await _context.SaveChangesAsync();
            return existingMenuItem;
        }

        /// <summary>
        /// Deletes a menu item from the database.
        /// </summary>
        /// <param name="menuItemId">The ID of the menu item to delete.</param>
        /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
        /// <exception cref="Exception">Thrown if the menu item is not found.</exception>
        public async Task DeleteMenuItemAsync(int menuItemId)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var menuItem = await _context.MenuItems.FindAsync(menuItemId);
                if (menuItem == null)
                {
                    throw new Exception("MenuItem not found.");
                }

                _context.MenuItems.Remove(menuItem);
                await _context.SaveChangesAsync();
            }
        }

        #endregion

        #region ContentGroup Operations

        /// <summary>
        /// Retrieves the first ContentGroup associated with the given SiteInfoId.
        /// Returns null if not found.
        /// </summary>
        /// <param name="siteInfoId">The ID of the site information.</param>
        /// <returns>The first ContentGroup found, or null if none exists.</returns>
        public async Task<ContentGroup?> GetContentGroupBySiteInfoIdAsync(int siteInfoId)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var result = await _context.ContentGroups
                .Where(cg => cg.SiteInfoId == siteInfoId)
                .OrderBy(cg => cg.Id)
                .FirstOrDefaultAsync();
                return result;
            }
        }

        /// <summary>
        /// Returns a list of ContentGroups for a site, with optional filtering by type or slug.
        /// </summary>
        /// <param name="siteInfoId">The ID of the site information.</param>
        /// <param name="type">Optional. The type of the content group to filter by.</param>
        /// <param name="slug">Optional. The slug of the content group to filter by.</param>
        /// <returns>A list of matching ContentGroups.</returns>
        public async Task<List<ContentGroup>> GetContentGroupsBySiteInfoIdAsync(int siteInfoId, string? type = null, string? slug = null)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

                var query = _context.ContentGroups
                    .Include(cg => cg.SiteInfo)
                    .Include(cg => cg.Items)
                        .ThenInclude(ci => ci.Chunks)
                    .AsQueryable();

                query = query.Where(cg => cg.SiteInfoId == siteInfoId);

                if (!string.IsNullOrWhiteSpace(type))
                    query = query.Where(cg => cg.Type == type);

                if (!string.IsNullOrWhiteSpace(slug))
                    query = query.Where(cg => cg.Slug == slug);

                return query.ToList();
            }
        }

        /// <summary>
        /// Updates an existing content group in the database.
        /// </summary>
        /// <param name="updatedGroup">The content group with updated information.</param>
        /// <returns>The updated content group.</returns>
        /// <exception cref="Exception">Thrown if the content group is not found.</exception>
        public async Task<ContentGroup> UpdateContentGroupByIdAsync(ContentGroup updatedGroup)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            var existingGroup = await _context.ContentGroups.FindAsync(updatedGroup.Id);
            if (existingGroup == null)
                throw new Exception("ContentGroup not found.");

            existingGroup.Name = updatedGroup.Name;
            existingGroup.Slug = updatedGroup.Slug;
            existingGroup.Type = updatedGroup.Type;
            existingGroup.VectorSize = updatedGroup.VectorSize;
            existingGroup.EmbeddingModel = updatedGroup.EmbeddingModel;
            existingGroup.LastUpdated = DateTime.UtcNow;
            existingGroup.Version = updatedGroup.Version;

            _context.ContentGroups.Update(existingGroup);
            await _context.SaveChangesAsync();

            return existingGroup;
        }

        /// <summary>
        /// Deletes a content group by its ID.
        /// </summary>
        /// <param name="contentGroupId">The ID of the content group to delete.</param>
        /// <returns><c>true</c> if the content group was deleted, <c>false</c> otherwise.</returns>
        public async Task<bool> DeleteContentGroupByIdAsync(int contentGroupId)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            var group = await _context.ContentGroups.FindAsync(contentGroupId);
            if (group == null)
                return false;

            _context.ContentGroups.Remove(group);
            await _context.SaveChangesAsync();

            return true;
        }

        /// <summary>
        /// Creates a new content group in the database.
        /// </summary>
        /// <param name="newGroup">The new content group to create.</param>
        /// <returns>The newly created content group with its generated ID and timestamps.</returns>
        public async Task<ContentGroup> CreateContentGroupAsync(ContentGroup newGroup)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            newGroup.CreatedAt = DateTime.UtcNow;
            newGroup.LastUpdated = DateTime.UtcNow;

            _context.ContentGroups.Add(newGroup);
            await _context.SaveChangesAsync();

            return newGroup;
        }

        /// <summary>
        /// Returns all ContentGroups for a site, with optional filtering by type.
        /// </summary>
        /// <param name="siteInfoId">The ID of the site information.</param>
        /// <param name="type">Optional. The type of the content group to filter by.</param>
        /// <returns>A list of matching ContentGroups.</returns>
        public async Task<List<ContentGroup>> GetAllContentGroupsBySiteInfoIdAsync(int siteInfoId, string? type = null)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var query = _context.ContentGroups
                .Where(cg => cg.SiteInfoId == siteInfoId);

                if (!string.IsNullOrWhiteSpace(type))
                    query = query.Where(cg => cg.Type == type);

                return await query
                    .OrderBy(cg => cg.Name)
                    .ToListAsync();
            }
        }

        #endregion

        #region ContentItem Operations

        /// <summary>
        /// Retrieves a content item by its ID, optionally filtered by type.
        /// </summary>
        /// <param name="contentItemId">The ID of the content item.</param>
        /// <param name="type">Optional. The type of the content item to filter by.</param>
        /// <returns>The matching content item with its chunks, or null if not found.</returns>
        public async Task<ContentItem?> GetContentItemByIdAsync(int contentItemId, string? type = null)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var query = _context.ContentItems
                .Where(cg => cg.Id == contentItemId).Include(ci => ci.Chunks);

                return query.FirstOrDefault();
            }
        }

        /// <summary>
        /// Retrieves a content item by its ID, including related ContentGroup, Chunks, and SiteInfo.
        /// </summary>
        /// <param name="id">The ID of the content item.</param>
        /// <returns>The matching content item, or null if not found.</returns>
        public async Task<ContentItem?> GetContentItemByIdAsync(int id)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
            var query = db.ContentItems.Where(x => x.Id == id).Include(ci => ci.ContentGroup).Include(ci => ci.Chunks).Include(ci => ci.ContentGroup.SiteInfo);
            return query.FirstOrDefault();
        }

        /// <summary>
        /// Creates a new content item, chunks its content, generates embeddings, and inserts them into Qdrant.
        /// </summary>
        /// <param name="item">The content item to create.</param>
        /// <param name="collectionName">The name of the Qdrant collection to insert into.</param>
        /// <returns>The created content item.</returns>
        /// <exception cref="Exception">Thrown if the saved ContentItem cannot be retrieved.</exception>
        public async Task<ContentItem> CreateContentItemAsync(ContentItem item, string collectionName)
        {
            await _logger.InfoAsync($"CreateContentItemAsync: method called", item.Id.ToString());
            using var scope = _serviceScopeFactory.CreateScope();
            var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            item.CreatedAt = DateTime.UtcNow;
            item.LastUpdated = DateTime.UtcNow;

            db.ContentItems.Add(item);
            await db.SaveChangesAsync();

            var fullItem = await db.ContentItems
                .Include(ci => ci.ContentGroup)
                .FirstOrDefaultAsync(ci => ci.Id == item.Id);

            if (fullItem == null)
                throw new Exception("Failed to retrieve saved ContentItem.");

            if (!string.IsNullOrEmpty(item.Content))
            {
                var chunks = ChunkingHelper.SplitStructuredText(item.Content, 3000);
                var vectorizedChunks = await VectorizeChunksWithGuidsAsync(chunks, fullItem, collectionName);

                var chunkEntities = vectorizedChunks.Select((chunk, index) => new ContentChunk
                {
                    ContentItemId = item.Id,
                    QdrantPointId = chunk.UId,
                    ChunkIndex = index,
                    CreatedAt = DateTime.UtcNow
                }).ToList();

                db.ContentChunks.AddRange(chunkEntities);
            }
            await db.SaveChangesAsync();

            return item;
        }

        /// <summary>
        /// Vectorizes a list of content chunks, generates unique GUIDs for each, and inserts them into Qdrant.
        /// </summary>
        /// <param name="chunks">The list of text chunks to vectorize.</param>
        /// <param name="item">The content item associated with these chunks.</param>
        /// <param name="collectionName">The name of the Qdrant collection.</param>
        /// <returns>A list of <see cref="WebPageContent"/> representing the vectorized chunks.</returns>
        public async Task<List<WebPageContent>> VectorizeChunksWithGuidsAsync(List<string> chunks, ContentItem item, string collectionName)
        {
            var result = new List<WebPageContent>();

            foreach (var (chunk, index) in chunks.Select((c, i) => (c, i)))
            {
                var combinedText = chunk;
                float[] embedding = [];

                var embeddingServiceProvider = GetAiEmbeddingSettings();
                if (embeddingServiceProvider == "local")
                {
                    embedding = await _localEmbeddingService.GenerateEmbeddingAsync(combinedText);
                }
                else
                {
                    embedding = await _openAIEmbeddingService.GenerateEmbeddingAsync(combinedText);
                }

                var uid = Guid.NewGuid().ToString();

                result.Add(new WebPageContent
                {
                    Id = Guid.Parse(uid),
                    UId = uid,
                    SiteId = item.ContentGroup.SiteInfoId,
                    Type = "content-item",
                    Name = item.Title,
                    Description = item.Description,
                    Content = chunk,
                    Vectors = embedding,
                    LastUpdated = DateTime.UtcNow
                });
            }

            await _qDrantService.QDrantInsertManyAsync(result, collectionName);
            return result;
        }

        /// <summary>
        /// Vectorizes a list of content chunks and inserts them into Qdrant.
        /// </summary>
        /// <param name="chunks">The list of text chunks to vectorize.</param>
        /// <param name="item">The content item associated with these chunks.</param>
        /// <param name="collectionName">The name of the Qdrant collection.</param>
        /// <returns>A list of <see cref="WebPageContent"/> representing the vectorized chunks.</returns>
        public async Task<List<WebPageContent>> VectorizeChunksAsync(List<string> chunks, ContentItem item, string collectionName)
        {
            var result = new List<WebPageContent>();

            foreach (var (chunk, index) in chunks.Select((c, i) => (c, i)))
            {
                var combinedText = $"{chunk}";
                float[] embedding = [];

                var embeddingServiceProvider = GetAiEmbeddingSettings();
                if (embeddingServiceProvider == "local")
                {
                    embedding = await _localEmbeddingService.GenerateEmbeddingAsync(combinedText);
                }
                else
                {
                    embedding = await _openAIEmbeddingService.GenerateEmbeddingAsync(combinedText);
                }

                var uid = Guid.NewGuid();

                var webChunk = new WebPageContent
                {
                    Id = uid,
                    UId = uid.ToString(),
                    SiteId = item.ContentGroup.SiteInfoId,
                    Type = "content-item",
                    Name = item.Title,
                    Description = item.Description,
                    Content = chunk,
                    Vectors = embedding,
                    LastUpdated = DateTime.UtcNow
                };

                result.Add(webChunk);
            }
            await _qDrantService.QDrantInsertManyAsync(result, collectionName);
            return result;
        }

        /// <summary>
        /// Forces a re-chunking and re-embedding of all content items within a specified content group.
        /// </summary>
        /// <param name="contentGroupId">The ID of the content group to re-chunk.</param>
        /// <param name="collectionName">The name of the Qdrant collection. If null, the existing collection name will be used.</param>
        /// <returns><c>true</c> if the operation was successful, <c>false</c> otherwise.</returns>
        public async Task<bool> ForceRechunkContentGroupAsync(int contentGroupId, string collectionName = null)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            var group = await db.ContentGroups
                .Include(g => g.Items)
                    .ThenInclude(ci => ci.Chunks)
                .Include(g => g.SiteInfo)
                .FirstOrDefaultAsync(g => g.Id == contentGroupId);

            if (group == null)
                return false;

            foreach (var item in group.Items)
            {
                await SaveAndSyncContentItemAsync(item, collectionName, forceRechunk: true);
            }
            return true;
        }

        /// <summary>
        /// Forces the recreation of a Qdrant collection for a given site, including re-chunking and re-embedding all content.
        /// </summary>
        /// <param name="siteInfoId">The ID of the site for which to recreate the collection.</param>
        /// <returns><c>true</c> if the collection was successfully recreated and content re-synced, <c>false</c> otherwise.</returns>
        /// <exception cref="Exception">Thrown if the new Qdrant collection cannot be created.</exception>
        public async Task<bool> ForceRecreateQdrantCollectionAsync(int siteInfoId)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            var site = await db.SiteInfos
                .Include(s => s.ContentGroups)
                    .ThenInclude(g => g.Items)
                        .ThenInclude(i => i.Chunks)
                .FirstOrDefaultAsync(s => s.Id == siteInfoId);

            if (site == null)
                return false;

            string oldCollectionName = site.VectorCollectionName;

            await _qDrantService.DeleteCollectionAsync(oldCollectionName);

            string newCollectionName = GetGeneratedVectorCollectionName(site);

            bool created = await _qDrantService.CreateQdrantCollectionAsync(newCollectionName);
            if (!created)
                throw new Exception($"Failed to create Qdrant collection '{newCollectionName}'.");

            site.VectorCollectionName = newCollectionName;
            await UpdateSiteInfoAsync(site);

            foreach (var group in site.ContentGroups)
            {
                foreach (var item in group.Items.ToList())
                {
                    var dto = new ContentItem
                    {
                        Id = item.Id,
                        Title = item.Title,
                        Description = item.Description,
                        Content = item.Content,
                        Language = item.Language,
                        Tags = item.Tags,
                        IsPublished = item.IsPublished,
                        Version = item.Version,
                        LastUpdated = item.LastUpdated,
                        ContentGroupId = item.ContentGroupId
                    };

                    await SaveAndSyncContentItemAsync(dto, newCollectionName, forceRechunk: true);
                }
            }

            return true;
        }

        /// <summary>
        /// Retrieves all content items belonging to a specific content group.
        /// </summary>
        /// <param name="contentGroupId">The ID of the content group.</param>
        /// <returns>A list of content items in the specified group, ordered by last updated date descending.</returns>
        public async Task<List<ContentItem>> GetContentItemsByGroupIdAsync(int contentGroupId)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                return await context.ContentItems
                    .Where(ci => ci.ContentGroupId == contentGroupId)
                    .OrderByDescending(ci => ci.LastUpdated)
                    .ToListAsync();
            }
        }

        /// <summary>
        /// Retrieves all content items belonging to a specific content group.
        /// </summary>
        /// <param name="contentGroupId">The ID of the content group.</param>
        /// <returns>A list of content items in the specified group, ordered by last updated date descending.</returns>
        public async Task<List<ContentItem>> GetAllContentItemsBySiteIdAsync(int siteId)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

                //var site = await GetSiteInfoByIdAsync(siteId);
                //List<ContentItem> contentList = new List<ContentItem>();
                //foreach (var item in site.ContentGroups)
                //{
                //    var contentItems = await GetContentItemsByGroupIdAsync(item.Id);
                //    contentList.AddRange(contentItems);
                //}
                //return contentList;
                return await context.ContentItems
        .Where(ci => ci.ContentGroup.SiteInfoId == siteId)
        .Include(ci => ci.Chunks)
        .ToListAsync();

            }
        }

        /// <summary>
        /// Creates a new content item in the database.
        /// </summary>
        /// <param name="newItem">The new content item to create.</param>
        /// <returns>The newly created content item with its generated ID and timestamps.</returns>
        public async Task<ContentItem?> CreateContentItemAsync(ContentItem newItem)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            newItem.CreatedAt = DateTime.UtcNow;
            newItem.LastUpdated = DateTime.UtcNow;

            _context.ContentItems.Add(newItem);
            await _context.SaveChangesAsync();

            return newItem;
        }

        /// <summary>
        /// Saves changes to a content item and synchronizes its chunks and embeddings with Qdrant.
        /// If content has changed or re-chunking is forced, old chunks are removed and new ones are created and embedded.
        /// </summary>
        /// <param name="dto">The content item data transfer object with updated information.</param>
        /// <param name="collectionName">The name of the Qdrant collection to sync with.</param>
        /// <param name="forceRechunk">If set to <c>true</c>, forces re-chunking and re-embedding even if content hasn't changed.</param>
        /// <returns>The updated and synchronized content item.</returns>
        /// <exception cref="Exception">Thrown if the content item is not found.</exception>
        public async Task<ContentItem> SaveAndSyncContentItemAsync(ContentItem dto, string collectionName, bool forceRechunk = false)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            var existing = await db.ContentItems
                .Include(ci => ci.Chunks)
                .Include(ci => ci.ContentGroup)
                .FirstOrDefaultAsync(ci => ci.Id == dto.Id);

            if (existing == null)
                throw new Exception("ContentItem not found.");

            bool contentChanged = existing.Content?.Trim() != dto.Content?.Trim();

            existing.Title = dto.Title;
            existing.Description = dto.Description;
            existing.Content = dto.Content;
            existing.Language = dto.Language;
            existing.Tags = dto.Tags;
            existing.IsPublished = dto.IsPublished;
            existing.LastUpdated = dto.LastUpdated;

            if (contentChanged || forceRechunk)
            {
                existing.Version++;

                var chunkIds = existing.Chunks.Select(c => c.Id).ToList();
                if (chunkIds.Any())
                {
                    await RemoveChunksAndQdrantEntriesByIdsAsync(chunkIds, collectionName);

                    foreach (var chunk in existing.Chunks.ToList())
                    {
                        db.Entry(chunk).State = EntityState.Detached;
                    }

                    existing.Chunks.Clear();
                }

                var newChunks = ChunkingHelper.SplitStructuredText(existing.Content, 3000);
                var webChunks = await VectorizeChunksAsync(newChunks, existing, collectionName);

                var chunkEntities = webChunks.Select((w, i) => new ContentChunk
                {
                    ContentItemId = existing.Id,
                    ChunkIndex = i,
                    QdrantPointId = w.Id.Uuid,
                    CreatedAt = DateTime.UtcNow
                }).ToList();

                db.ContentChunks.AddRange(chunkEntities);
            }

            await db.SaveChangesAsync();
            return existing;
        }

        /// <summary>
        /// Groups content items by their content group type.
        /// </summary>
        /// <param name="model">The website content model containing content items.</param>
        /// <returns>A dictionary where the key is the content group type and the value is a list of content item models.</returns>
        public static Dictionary<string, List<ContentItemModel>> GroupContentItemsByType(WebsiteContentModel model)
        {
            return model.ContentItems
                .GroupBy(ci => ci.ContentItem.ContentGroup.Type)
                .ToDictionary(g => g.Key, g => g.ToList());
        }

        /// <summary>
        /// Deletes a content item by its ID.
        /// </summary>
        /// <param name="id">The ID of the content item to delete.</param>
        /// <returns><c>true</c> if the content item was deleted, <c>false</c> otherwise.</returns>
        public async Task<bool> DeleteContentItemByIdAsync(int id)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

            var item = await _context.ContentItems.FindAsync(id);
            if (item != null)
            {
                _context.ContentItems.Remove(item);
                await _context.SaveChangesAsync();
                return true;
            }

            return false;
        }

        /// <summary>
        /// Retrieves the Qdrant point IDs associated with content chunks belonging to a specific content group.
        /// </summary>
        /// <param name="contentGroupId">The ID of the content group.</param>
        /// <returns>An array of Qdrant point IDs (represented as chunk indices).</returns>
        public async Task<int[]> GetPointIdsByContentGroupIdAsync(int contentGroupId)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                return await _context.ContentChunks
                .Where(chunk => chunk.ContentItem.ContentGroupId == contentGroupId)
                .Select(chunk => chunk.ChunkIndex)
                .ToArrayAsync();
            }
        }

        #endregion

        #region SiteInfo Operations

        /// <summary>
        /// Retrieves site information by its ID.
        /// </summary>
        /// <param name="SiteInfoId">The ID of the site information.</param>
        /// <returns>The matching site information, or the first site information if not found.</returns>
        public async Task<SiteInfo> GetSiteInfoByIdAsync(int SiteInfoId)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var result = await _context.SiteInfos.Where(x => x.Id == SiteInfoId).FirstOrDefaultAsync();
                if (result == null)
                {
                    return await _context.SiteInfos.FirstOrDefaultAsync();
                }
                else
                {
                    return result;
                }
            }
        }

        /// <summary>
        /// Retrieves site information by its ID, including associated form definitions.
        /// </summary>
        /// <param name="siteId">The ID of the site.</param>
        /// <returns>The matching site information with form definitions, or a new <see cref="SiteInfo"/> if not found.</returns>
        public async Task<SiteInfo?> GetSiteInfoWithFormsByIdAsync(int siteId)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var result = await _context.SiteInfos.Include(s => s.FormDefinitions).FirstOrDefaultAsync(s => s.Id == siteId);
                if (result == null)
                {
                    return new SiteInfo();
                }
                else
                {
                    return result;
                }
            }
        }

        /// <summary>
        /// Retrieves site information by its name.
        /// </summary>
        /// <param name="siteName">The name of the site.</param>
        /// <returns>The matching site information, or the first site information if not found.</returns>
        public async Task<SiteInfo> GetSiteInfoByNameAsync(string siteName)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var result = await _context.SiteInfos.Where(x => x.SiteName == siteName).FirstOrDefaultAsync();
                if (result == null)
                {
                    return await _context.SiteInfos.FirstOrDefaultAsync();
                }
                else
                {
                    return result;
                }
            }
        }

        /// <summary>
        /// Retrieves site information by its default URL.
        /// </summary>
        /// <param name="url">The default URL of the site.</param>
        /// <returns>The matching site information, or the first site information if not found.</returns>
        public async Task<SiteInfo> GetSiteInfoByUrlAsync(string url)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                var result = await _context.SiteInfos.Where(x => x.DefaultUrl == url).FirstOrDefaultAsync();
                if (result == null)
                {
                    return await _context.SiteInfos.FirstOrDefaultAsync();
                }
                else
                {
                    return result;
                }
            }
        }

        /// <summary>
        /// Updates existing site information in the database.
        /// </summary>
        /// <param name="siteInfo">The site information with updated details.</param>
        /// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
        public async Task UpdateSiteInfoAsync(SiteInfo siteInfo)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                _context.SiteInfos.Update(siteInfo);
                await _context.SaveChangesAsync();
            }
        }

        /// <summary>
        /// Retrieves all sites associated with a specific user.
        /// </summary>
        /// <param name="userId">The ID of the user.</param>
        /// <returns>A list of sites belonging to the specified user.</returns>
        public async Task<List<SiteInfo>> GetUserSitesAsync(string userId)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                return await _context.SiteInfos
                    .Where(s => s.UserId == userId)
                    .ToListAsync();
            }
        }

        /// <summary>
        /// Retrieves all site information records from the database.
        /// </summary>
        /// <returns>A list of all site information records.</returns>
        public async Task<List<SiteInfo>> GetAllSitesAsync()
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
                return await _context.SiteInfos.ToListAsync();
            }
        }

        /// <summary>
        /// Adds new site information to the database and creates a corresponding Qdrant collection.
        /// </summary>
        /// <param name="siteInfo">The site information to add.</param>
        /// <returns>The added site information.</returns>
        /// <exception cref="ArgumentNullException">Thrown if <paramref name="siteInfo"/> is null.</exception>
        /// <exception cref="InvalidOperationException">Thrown if an error occurs during the site creation or Qdrant collection creation.</exception>
        public async Task<SiteInfo> AddSiteInfoAsync(SiteInfo siteInfo)
        {
            using (var scope = _serviceScopeFactory.CreateScope())
            {
                var _context = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();

                if (siteInfo == null)
                    throw new ArgumentNullException(nameof(siteInfo), "SiteInfo cannot be null.");

                try
                {
                    if (string.IsNullOrEmpty(siteInfo.VectorCollectionName))
                    {
                        siteInfo.VectorCollectionName = GetGeneratedVectorCollectionName(siteInfo);
                    }

                    await _context.SiteInfos.AddAsync(siteInfo);
                    var result = await _context.SaveChangesAsync();
                    if (result > 0)
                    {
                        bool checkResult = await _qDrantService.CollectionExistsAsync(siteInfo.VectorCollectionName);
                        if (checkResult)
                        {
                            siteInfo.VectorCollectionName = GetGeneratedVectorCollectionName(siteInfo);
                            _context.SiteInfos.Update(siteInfo);
                            await _context.SaveChangesAsync();
                            bool qresult = await _qDrantService.CreateQdrantCollectionAsync(siteInfo.VectorCollectionName);
                            if (!qresult)
                            {
                                throw new Exception("Failed to create Qdrant collection for the site.");
                            }
                        }
                        else
                        {
                            bool qresult = await _qDrantService.CreateQdrantCollectionAsync(siteInfo.VectorCollectionName);
                            if (!qresult)
                            {
                                throw new Exception("Failed to create Qdrant collection for the site.");
                            }
                        }
                    }
                    return siteInfo;
                }
                catch (Exception ex)
                {
                    throw new InvalidOperationException("An error occurred while adding the site info.", ex);
                }
            }
        }

        #endregion


        ///TEMPORARY
        ///
        public async Task<bool> MigrateQdrantToContentItemsAsync(int siteId, string collectionName)
        {
            using var scope = _serviceScopeFactory.CreateScope();
            var db = scope.ServiceProvider.GetRequiredService<ApplicationDbContext>();
            var site = await GetSiteInfoByIdAsync(siteId);
            // 1. Get menu items
            var menuItems = await GetMenuItemsBySiteIdAsync(siteId);
            PointId[] pointIds = new PointId[menuItems.Count];
            for (int i = 0; i < menuItems.Count; i++)
            {
                // Ensure SortOrder is set to i+1 (1-based index)
                pointIds[i] = Convert.ToUInt64(menuItems[i].SortOrder);
            }

            if (!pointIds.Any())
                return false;

            // 2. Get points from Qdrant
            var qdrantPoints = await _qDrantService.GetPointsFromQdrantAsyncByIntegerPointIds(collectionName, pointIds);

            // 3. Create new content group
            var contentGroup = new ContentGroup
            {
                Name = "Pages",
                Type = "page",
                SiteInfoId = siteId,
                CreatedAt = DateTime.UtcNow,
                EmbeddingModel = "openai",
                Slug = $"group-{DateTime.UtcNow.Ticks}",
                Version = 0,
                LastUpdated = DateTime.UtcNow
            };
            db.ContentGroups.Add(contentGroup);
            await db.SaveChangesAsync(); // Save to get ID

            // 4. Create ContentItems from Qdrant points
            foreach (var point in qdrantPoints)
            {
                var item = new ContentItem
                {
                    ContentGroupId = contentGroup.Id,
                    Title = point.Name,
                    Description = point.Description,
                    Content = point.Content,
                    Language = "en",
                    Tags = "",
                    IsPublished = true,
                    Version = 1,
                    CreatedAt = DateTime.UtcNow,
                    LastUpdated = DateTime.UtcNow
                };

                db.ContentItems.Add(item);
            }

            await db.SaveChangesAsync();
            return true;
        }



    }
}