Improve file list performance.

Lantean.QBTMud/Components/FilesTab.razor.cs

@@ -21,6 +21,9 @@ namespace Lantean.QBTMud.Components
 
         private readonly CancellationTokenSource _timerCancellationToken = new();
         private bool _disposedValue;
+        private static readonly ReadOnlyCollection<ContentItem> EmptyContentItems = new ReadOnlyCollection<ContentItem>(Array.Empty<ContentItem>());
+        private ReadOnlyCollection<ContentItem> _visibleFiles = EmptyContentItems;
+        private bool _filesDirty = true;
 
         private List<PropertyFilterDefinition<ContentItem>>? _filterDefinitions;
         private readonly Dictionary<string, RenderFragment<RowContext<ContentItem>>> _columnRenderFragments = [];
@@ -103,6 +106,7 @@ namespace Lantean.QBTMud.Components
             if (_filterDefinitions is null)
             {
                 Filters = null;
+                MarkFilesDirty();
                 return;
             }
 
@@ -114,11 +118,13 @@ namespace Lantean.QBTMud.Components
             }
 
             Filters = filters;
+            MarkFilesDirty();
         }
 
         protected void RemoveFilter()
         {
             Filters = null;
+            MarkFilesDirty();
         }
 
         public async ValueTask DisposeAsync()
@@ -158,6 +164,7 @@ namespace Lantean.QBTMud.Components
         protected void SearchTextChanged(string value)
         {
             SearchText = value;
+            MarkFilesDirty();
         }
 
         protected Task TableDataContextMenu(TableDataContextMenuEventArgs<ContentItem> eventArgs)
@@ -198,6 +205,7 @@ namespace Lantean.QBTMud.Components
             {
                 while (!_timerCancellationToken.IsCancellationRequested && await timer.WaitForNextTickAsync())
                 {
+                    var hasUpdates = false;
                     if (Active && Hash is not null)
                     {
                         IReadOnlyList<QBitTorrentClient.Models.FileData> files;
@@ -214,17 +222,23 @@ namespace Lantean.QBTMud.Components
                         if (FileList is null)
                         {
                             FileList = DataManager.CreateContentsList(files);
+                            hasUpdates = true;
                         }
                         else
                         {
-                            DataManager.MergeContentsList(files, FileList);
+                            hasUpdates = DataManager.MergeContentsList(files, FileList);
                         }
                     }
 
+                    if (hasUpdates)
+                    {
+                        MarkFilesDirty();
+                        PruneSelectionIfMissing();
                         await InvokeAsync(StateHasChanged);
                     }
                 }
             }
+        }
 
         protected override async Task OnParametersSetAsync()
         {
@@ -247,6 +261,8 @@ namespace Lantean.QBTMud.Components
 
             var contents = await ApiClient.GetTorrentContents(Hash);
             FileList = DataManager.CreateContentsList(contents);
+            MarkFilesDirty();
+            PruneSelectionIfMissing();
 
             var expandedNodes = await LocalStorage.GetItemAsync<HashSet<string>>($"{_expandedNodesStorageKey}.{Hash}");
             if (expandedNodes is not null)
@@ -257,6 +273,8 @@ namespace Lantean.QBTMud.Components
             {
                 ExpandedNodes.Clear();
             }
+
+            MarkFilesDirty();
         }
 
         protected async Task PriorityValueChanged(ContentItem contentItem, Priority priority)
@@ -321,11 +339,13 @@ namespace Lantean.QBTMud.Components
         protected void SortColumnChanged(string sortColumn)
         {
             _sortColumn = sortColumn;
+            MarkFilesDirty();
         }
 
         protected void SortDirectionChanged(SortDirection sortDirection)
         {
             _sortDirection = sortDirection;
+            MarkFilesDirty();
         }
 
         protected void SelectedItemChanged(ContentItem item)
@@ -344,6 +364,7 @@ namespace Lantean.QBTMud.Components
                 ExpandedNodes.Add(contentItem.Name);
             }
 
+            MarkFilesDirty();
             await LocalStorage.SetItemAsync($"{_expandedNodesStorageKey}.{Hash}", ExpandedNodes);
         }
 
@@ -369,44 +390,6 @@ namespace Lantean.QBTMud.Components
             return FileList!.Values.Where(f => f.Name.StartsWith(contentItem.Name + Extensions.DirectorySeparator) && !f.IsFolder);
         }
 
-        private IEnumerable<ContentItem> GetChildren(ContentItem folder)
-        {
-            var childLevel = folder.Level + 1;
-            var prefix = string.Concat(folder.Name, Extensions.DirectorySeparator);
-
-            foreach (var item in FileList!.Values.Where(f => f.Level == childLevel && f.Name.StartsWith(prefix, StringComparison.Ordinal)).OrderByDirection(_sortDirection, GetSortSelector()))
-            {
-                if (item.IsFolder)
-                {
-                    var descendants = GetChildren(item);
-                    // if the filter returns some results then show folder item
-                    if (descendants.Any())
-                    {
-                        yield return item;
-                    }
-
-                    // if the folder is not expanded - don't return children
-                    if (!ExpandedNodes.Contains(item.Name))
-                    {
-                        continue;
-                    }
-
-                    // then show children
-                    foreach (var descendant in descendants)
-                    {
-                        yield return descendant;
-                    }
-                }
-                else
-                {
-                    if (FilterContentItem(item))
-                    {
-                        yield return item;
-                    }
-                }
-            }
-        }
-
         private bool FilterContentItem(ContentItem item)
         {
             if (Filters is not null)
@@ -430,37 +413,130 @@ namespace Lantean.QBTMud.Components
         }
 
         private ReadOnlyCollection<ContentItem> GetFiles()
+        {
+            if (!_filesDirty)
+            {
+                return _visibleFiles;
+            }
+
+            _visibleFiles = BuildVisibleFiles();
+            _filesDirty = false;
+
+            return _visibleFiles;
+        }
+
+        private ReadOnlyCollection<ContentItem> BuildVisibleFiles()
         {
             if (FileList is null || FileList.Values.Count == 0)
             {
-                return new ReadOnlyCollection<ContentItem>([]);
+                return EmptyContentItems;
             }
 
-            var maxLevel = FileList.Values.Max(f => f.Level);
-            // this is a flat file structure
-            if (maxLevel == 0)
+            var lookup = BuildChildrenLookup();
+            if (!lookup.TryGetValue(string.Empty, out var roots))
             {
-                return FileList.Values.Where(FilterContentItem).OrderByDirection(_sortDirection, GetSortSelector()).ToList().AsReadOnly();
+                return EmptyContentItems;
             }
 
-            var list = new List<ContentItem>();
+            var sortSelector = GetSortSelector();
+            var orderedRoots = roots.OrderByDirection(_sortDirection, sortSelector).ToList();
+            var result = new List<ContentItem>(FileList.Values.Count);
 
-            var rootItems = FileList.Values.Where(c => c.Level == 0).OrderByDirection(_sortDirection, GetSortSelector()).ToList();
-            foreach (var item in rootItems)
+            foreach (var item in orderedRoots)
             {
-                list.Add(item);
+                if (item.IsFolder)
+                {
+                    result.Add(item);
 
-                if (item.IsFolder && ExpandedNodes.Contains(item.Name))
+                    if (!ExpandedNodes.Contains(item.Name))
                     {
-                    var descendants = GetChildren(item);
-                    foreach (var descendant in descendants)
+                        continue;
+                    }
+
+                    var descendants = GetVisibleDescendants(item, lookup, sortSelector);
+                    result.AddRange(descendants);
+                }
+                else
                 {
-                        list.Add(descendant);
+                    if (FilterContentItem(item))
+                    {
+                        result.Add(item);
                     }
                 }
             }
 
-            return list.AsReadOnly();
+            return new ReadOnlyCollection<ContentItem>(result);
+        }
+
+        private Dictionary<string, List<ContentItem>> BuildChildrenLookup()
+        {
+            var lookup = new Dictionary<string, List<ContentItem>>(FileList!.Count);
+
+            foreach (var item in FileList!.Values)
+            {
+                var parentPath = item.Level == 0 ? string.Empty : item.Name.GetDirectoryPath();
+                if (!lookup.TryGetValue(parentPath, out var children))
+                {
+                    children = [];
+                    lookup[parentPath] = children;
+                }
+
+                children.Add(item);
+            }
+
+            return lookup;
+        }
+
+        private List<ContentItem> GetVisibleDescendants(ContentItem folder, Dictionary<string, List<ContentItem>> lookup, Func<ContentItem, object?> sortSelector)
+        {
+            if (!lookup.TryGetValue(folder.Name, out var children))
+            {
+                return [];
+            }
+
+            var orderedChildren = children.OrderByDirection(_sortDirection, sortSelector).ToList();
+            var visible = new List<ContentItem>();
+
+            foreach (var child in orderedChildren)
+            {
+                if (child.IsFolder)
+                {
+                    var descendants = GetVisibleDescendants(child, lookup, sortSelector);
+                    if (descendants.Count != 0)
+                    {
+                        visible.Add(child);
+
+                        if (ExpandedNodes.Contains(child.Name))
+                        {
+                            visible.AddRange(descendants);
+                        }
+                    }
+                }
+                else if (FilterContentItem(child))
+                {
+                    visible.Add(child);
+                }
+            }
+
+            return visible;
+        }
+
+        private void MarkFilesDirty()
+        {
+            _filesDirty = true;
+        }
+
+        private void PruneSelectionIfMissing()
+        {
+            if (SelectedItem is not null && (FileList is null || !FileList.ContainsKey(SelectedItem.Name)))
+            {
+                SelectedItem = null;
+            }
+
+            if (ContextMenuItem is not null && (FileList is null || !FileList.ContainsKey(ContextMenuItem.Name)))
+            {
+                ContextMenuItem = null;
+            }
         }
 
         protected async Task DoNotDownloadLessThan100PercentAvailability()

Lantean.QBTMud/Components/UI/DynamicTable.razor.cs

@@ -4,6 +4,7 @@ using Lantean.QBTMud.Models;
 using Microsoft.AspNetCore.Components;
 using Microsoft.AspNetCore.Components.Web;
 using MudBlazor;
+using System;
 
 namespace Lantean.QBTMud.Components.UI
 {
@@ -81,6 +82,8 @@ namespace Lantean.QBTMud.Components.UI
 
         protected HashSet<string> SelectedColumns { get; set; } = [];
 
+        private static readonly IReadOnlyList<ColumnDefinition<T>> EmptyColumns = Array.Empty<ColumnDefinition<T>>();
+
         private Dictionary<string, int?> _columnWidths = [];
 
         private Dictionary<string, int> _columnOrder = [];
@@ -91,6 +94,12 @@ namespace Lantean.QBTMud.Components.UI
 
         private readonly Dictionary<string, TdExtended> _tds = [];
 
+        private IReadOnlyList<ColumnDefinition<T>> _visibleColumns = EmptyColumns;
+
+        private bool _columnsDirty = true;
+
+        private IEnumerable<ColumnDefinition<T>>? _lastColumnDefinitions;
+
         protected override async Task OnInitializedAsync()
         {
             HashSet<string> selectedColumns;
@@ -109,6 +118,13 @@ namespace Lantean.QBTMud.Components.UI
                 SelectedColumns = selectedColumns;
                 await SelectedColumnsChanged.InvokeAsync(SelectedColumns);
             }
+            else
+            {
+                SelectedColumns = selectedColumns;
+            }
+
+            _lastColumnDefinitions = ColumnDefinitions;
+            MarkColumnsDirty();
 
             string? sortColumn;
             SortDirection sortDirection;
@@ -137,11 +153,24 @@ namespace Lantean.QBTMud.Components.UI
                 await SortDirectionChanged.InvokeAsync(_sortDirection);
             }
 
+            MarkColumnsDirty();
+
             var storedColumnsWidths = await LocalStorage.GetItemAsync<Dictionary<string, int?>>(_columnWidthsStorageKey);
             if (storedColumnsWidths is not null)
             {
                 _columnWidths = storedColumnsWidths;
             }
+            MarkColumnsDirty();
+        }
+
+        protected override void OnParametersSet()
+        {
+            base.OnParametersSet();
+            if (!ReferenceEquals(_lastColumnDefinitions, ColumnDefinitions))
+            {
+                _lastColumnDefinitions = ColumnDefinitions;
+                MarkColumnsDirty();
+            }
         }
 
         private IEnumerable<T>? GetOrderedItems()
@@ -165,39 +194,74 @@ namespace Lantean.QBTMud.Components.UI
             return Items.OrderByDirection(_sortDirection, sortSelector);
         }
 
-        protected IEnumerable<ColumnDefinition<T>> GetColumns()
+        protected IReadOnlyList<ColumnDefinition<T>> GetColumns()
         {
-            var filteredColumns = ColumnDefinitions.Where(c => SelectedColumns.Contains(c.Id)).Where(ColumnFilter);
+            if (!_columnsDirty)
+            {
+                return _visibleColumns;
+            }
+
+            _visibleColumns = BuildVisibleColumns();
+            _columnsDirty = false;
+
+            return _visibleColumns;
+        }
+
+        private IReadOnlyList<ColumnDefinition<T>> BuildVisibleColumns()
+        {
+            var filteredColumns = ColumnDefinitions
+                .Where(c => SelectedColumns.Contains(c.Id))
+                .Where(ColumnFilter)
+                .ToList();
+
+            if (filteredColumns.Count == 0)
+            {
+                return EmptyColumns;
+            }
+
+            List<ColumnDefinition<T>> orderedColumns;
             if (_columnOrder.Count == 0)
             {
-                foreach (var column in filteredColumns)
+                orderedColumns = filteredColumns;
+            }
+            else
             {
-                    if (_columnWidths.TryGetValue(column.Id, out var value))
-                    {
-                        column.Width = value;
-                    }
-
-                    yield return column;
-                }
-
-                yield break;
-            }
-
+                var orderLookup = _columnOrder.OrderBy(entry => entry.Value).ToList();
                 var columnDictionary = filteredColumns.ToDictionary(c => c.Id);
-            foreach (var columnId in _columnOrder.OrderBy(c => c.Value).Select(c => c.Key))
+                orderedColumns = new List<ColumnDefinition<T>>(filteredColumns.Count);
+
+                foreach (var (columnId, _) in orderLookup)
                 {
                     if (!columnDictionary.TryGetValue(columnId, out var column))
                     {
                         continue;
                     }
 
+                    orderedColumns.Add(column);
+                }
+
+                if (orderedColumns.Count != filteredColumns.Count)
+                {
+                    var existingIds = new HashSet<string>(orderedColumns.Select(c => c.Id));
+                    foreach (var column in filteredColumns)
+                    {
+                        if (existingIds.Add(column.Id))
+                        {
+                            orderedColumns.Add(column);
+                        }
+                    }
+                }
+            }
+
+            foreach (var column in orderedColumns)
+            {
                 if (_columnWidths.TryGetValue(column.Id, out var value))
                 {
                     column.Width = value;
                 }
-
-                yield return column;
             }
+
+            return orderedColumns;
         }
 
         private async Task SetSort(string columnId, SortDirection sortDirection)
@@ -316,18 +380,21 @@ namespace Lantean.QBTMud.Components.UI
                 SelectedColumns = result.SelectedColumns;
                 await LocalStorage.SetItemAsync(_columnSelectionStorageKey, SelectedColumns);
                 await SelectedColumnsChanged.InvokeAsync(SelectedColumns);
+                MarkColumnsDirty();
             }
 
             if (!DictionaryEqual(_columnWidths, result.ColumnWidths))
             {
                 _columnWidths = result.ColumnWidths;
                 await LocalStorage.SetItemAsync(_columnWidthsStorageKey, _columnWidths);
+                MarkColumnsDirty();
             }
 
             if (!DictionaryEqual(_columnOrder, result.ColumnOrder))
             {
                 _columnOrder = result.ColumnOrder;
                 await LocalStorage.SetItemAsync(_columnOrderStorageKey, _columnOrder);
+                MarkColumnsDirty();
             }
         }
 
@@ -379,6 +446,12 @@ namespace Lantean.QBTMud.Components.UI
             return className;
         }
 
+        private void MarkColumnsDirty()
+        {
+            _columnsDirty = true;
+            _visibleColumns = EmptyColumns;
+        }
+
         private sealed record SortData
         {
             public SortData(string sortColumn, SortDirection sortDirection)

Lantean.QBTMud/Services/DataManager.cs

@@ -745,6 +745,111 @@ namespace Lantean.QBTMud.Services
             return result;
         }
 
+        private static bool UpdateContentItem(ContentItem destination, ContentItem source)
+        {
+            const float floatTolerance = 0.0001f;
+            var changed = false;
+
+            if (destination.Priority != source.Priority)
+            {
+                destination.Priority = source.Priority;
+                changed = true;
+            }
+
+            if (System.Math.Abs(destination.Progress - source.Progress) > floatTolerance)
+            {
+                destination.Progress = source.Progress;
+                changed = true;
+            }
+
+            if (destination.Size != source.Size)
+            {
+                destination.Size = source.Size;
+                changed = true;
+            }
+
+            if (System.Math.Abs(destination.Availability - source.Availability) > floatTolerance)
+            {
+                destination.Availability = source.Availability;
+                changed = true;
+            }
+
+            return changed;
+        }
+
+        private struct DirectoryAccumulator
+        {
+            public long TotalSize { get; private set; }
+
+            private long _activeSize;
+            private double _progressSum;
+            private double _availabilitySum;
+            private Priority? _priority;
+            private bool _mixedPriority;
+
+            public void Add(Priority priority, float progress, long size, float availability)
+            {
+                TotalSize += size;
+
+                if (priority != Priority.DoNotDownload)
+                {
+                    _activeSize += size;
+                    _progressSum += progress * size;
+                    _availabilitySum += availability * size;
+                }
+
+                if (!_priority.HasValue)
+                {
+                    _priority = priority;
+                }
+                else if (_priority.Value != priority)
+                {
+                    _mixedPriority = true;
+                }
+            }
+
+            public Priority ResolvePriority()
+            {
+                if (_mixedPriority)
+                {
+                    return Priority.Mixed;
+                }
+
+                return _priority ?? Priority.Normal;
+            }
+
+            public float ResolveProgress()
+            {
+                if (_activeSize == 0 || TotalSize == 0)
+                {
+                    return 0f;
+                }
+
+                var value = _progressSum / _activeSize;
+                if (value < 0)
+                {
+                    return 0f;
+                }
+
+                if (value > 1)
+                {
+                    return 1f;
+                }
+
+                return (float)value;
+            }
+
+            public float ResolveAvailability()
+            {
+                if (_activeSize == 0 || TotalSize == 0)
+                {
+                    return 0f;
+                }
+
+                return (float)(_availabilitySum / _activeSize);
+            }
+        }
+
         private sealed class ContentTreeNode
         {
             public ContentTreeNode(ContentItem? item, ContentTreeNode? parent)
@@ -1180,24 +1285,120 @@ namespace Lantean.QBTMud.Services
             return original;
         }
 
-        public void MergeContentsList(IReadOnlyList<QBitTorrentClient.Models.FileData> files, Dictionary<string, ContentItem> contents)
+        public bool MergeContentsList(IReadOnlyList<QBitTorrentClient.Models.FileData> files, Dictionary<string, ContentItem> contents)
         {
-            var contentsList = CreateContentsList(files);
+            if (files.Count == 0)
+            {
+                if (contents.Count == 0)
+                {
+                    return false;
+                }
 
-            foreach (var (key, value) in contentsList)
+                contents.Clear();
+                return true;
+            }
+
+            var hasChanges = false;
+            var seenPaths = new HashSet<string>(files.Count * 2);
+            var directoryAccumulators = new Dictionary<string, DirectoryAccumulator>();
+
+            var minExistingIndex = contents.Count == 0
+                ? int.MaxValue
+                : contents.Values.Min(c => c.Index);
+            var minFileIndex = files.Min(f => f.Index);
+            var nextFolderIndex = System.Math.Min(minExistingIndex, minFileIndex) - 1;
+
+            foreach (var file in files)
             {
-                if (contents.TryGetValue(key, out var content))
+                var priority = (Priority)(int)file.Priority;
+                var pathSegments = file.Name.Split(Extensions.DirectorySeparator);
+                var level = pathSegments.Length - 1;
+                var displayName = pathSegments[^1];
+                var filePath = file.Name;
+                seenPaths.Add(filePath);
+
+                if (contents.TryGetValue(filePath, out var existingFile))
                 {
-                    content.Availability = value.Availability;
-                    content.Priority = value.Priority;
-                    content.Progress = value.Progress;
-                    content.Size = value.Size;
+                    var updatedFile = new ContentItem(filePath, displayName, file.Index, priority, file.Progress, file.Size, file.Availability, false, level);
+                    if (UpdateContentItem(existingFile, updatedFile))
+                    {
+                        hasChanges = true;
+                    }
                 }
                 else
                 {
-                    contents[key] = value;
+                    var newFile = new ContentItem(filePath, displayName, file.Index, priority, file.Progress, file.Size, file.Availability, false, level);
+                    contents[filePath] = newFile;
+                    hasChanges = true;
+                }
+
+                string directoryPath = string.Empty;
+                for (var i = 0; i < level; i++)
+                {
+                    var segment = pathSegments[i];
+                    if (segment == ".unwanted")
+                    {
+                        continue;
+                    }
+
+                    directoryPath = string.IsNullOrEmpty(directoryPath)
+                        ? segment
+                        : string.Concat(directoryPath, Extensions.DirectorySeparator, segment);
+
+                    seenPaths.Add(directoryPath);
+
+                    if (!contents.TryGetValue(directoryPath, out var directoryItem))
+                    {
+                        var newDirectory = new ContentItem(directoryPath, segment, nextFolderIndex--, Priority.Normal, 0, 0, 0, true, i);
+                        contents[directoryPath] = newDirectory;
+                        hasChanges = true;
+                    }
+
+                    if (!directoryAccumulators.TryGetValue(directoryPath, out var accumulator))
+                    {
+                        accumulator = new DirectoryAccumulator();
+                    }
+
+                    accumulator.Add(priority, file.Progress, file.Size, file.Availability);
+                    directoryAccumulators[directoryPath] = accumulator;
                 }
             }
+
+            var keysToRemove = contents.Keys.Where(key => !seenPaths.Contains(key)).ToList();
+            if (keysToRemove.Count != 0)
+            {
+                hasChanges = true;
+                foreach (var key in keysToRemove)
+                {
+                    contents.Remove(key);
+                }
+            }
+
+            foreach (var (directoryPath, accumulator) in directoryAccumulators)
+            {
+                if (!contents.TryGetValue(directoryPath, out var directoryItem))
+                {
+                    continue;
+                }
+
+                var updatedDirectory = new ContentItem(
+                    directoryPath,
+                    directoryItem.DisplayName,
+                    directoryItem.Index,
+                    accumulator.ResolvePriority(),
+                    accumulator.ResolveProgress(),
+                    accumulator.TotalSize,
+                    accumulator.ResolveAvailability(),
+                    true,
+                    directoryItem.Level);
+
+                if (UpdateContentItem(directoryItem, updatedDirectory))
+                {
+                    hasChanges = true;
+                }
+            }
+
+            return hasChanges;
         }
 
         public RssList CreateRssList(IReadOnlyDictionary<string, QBitTorrentClient.Models.RssItem> rssItems)

Lantean.QBTMud/Services/IDataManager.cs

@@ -16,7 +16,7 @@ namespace Lantean.QBTMud.Services
 
         Dictionary<string, ContentItem> CreateContentsList(IReadOnlyList<QBitTorrentClient.Models.FileData> files);
 
-        void MergeContentsList(IReadOnlyList<QBitTorrentClient.Models.FileData> files, Dictionary<string, ContentItem> contents);
+        bool MergeContentsList(IReadOnlyList<QBitTorrentClient.Models.FileData> files, Dictionary<string, ContentItem> contents);
 
         QBitTorrentClient.Models.UpdatePreferences MergePreferences(QBitTorrentClient.Models.UpdatePreferences? original, QBitTorrentClient.Models.UpdatePreferences changed);