first commit

2026-07-13 10:04:36 +08:00
commit 577aa128bd
870 changed files with 25849 additions and 0 deletions
--- a/DCIT.Core/Services/AutoExtractService.cs
+++ b/DCIT.Core/Services/AutoExtractService.cs
@@ -0,0 +1,550 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using DCIT.Core.Models;
+
+namespace DCIT.Core.Services
+{
+    public class AutoExtractService
+    {
+        private static readonly Regex TokenRegex = new Regex(
+            @"(?<Email>[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,})|(?<English>[A-Za-z][A-Za-z0-9_\-']*)|(?<Chinese>[\u4E00-\u9FFF]+)",
+            RegexOptions.Compiled);
+
+        private readonly WordTextExtractionService _wordTextExtractionService;
+
+        public AutoExtractService(WordTextExtractionService wordTextExtractionService)
+        {
+            _wordTextExtractionService = wordTextExtractionService;
+        }
+
+        public Task<AutoExtractResult> ExtractAsync(
+            IList<string> selectedFiles,
+            AutoExtractSettings settings,
+            IList<RuleDefinition> existingRules,
+            IProgress<ExtractionProgress> progress,
+            CancellationToken cancellationToken)
+        {
+            return Task.Run(
+                () => Execute(selectedFiles, settings, existingRules, progress, cancellationToken),
+                cancellationToken);
+        }
+
+        private AutoExtractResult Execute(
+            IList<string> selectedFiles,
+            AutoExtractSettings settings,
+            IList<RuleDefinition> existingRules,
+            IProgress<ExtractionProgress> progress,
+            CancellationToken cancellationToken)
+        {
+            var result = new AutoExtractResult();
+            var extractedDocuments = new List<ExtractedDocumentText>();
+
+            if (selectedFiles == null || selectedFiles.Count == 0)
+            {
+                return result;
+            }
+
+            settings = settings ?? new AutoExtractSettings();
+            existingRules = existingRules ?? Array.Empty<RuleDefinition>();
+
+            for (var index = 0; index < selectedFiles.Count; index++)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+                var filePath = selectedFiles[index];
+                progress?.Report(new ExtractionProgress
+                {
+                    Current = index + 1,
+                    Total = selectedFiles.Count,
+                    Message = "正在分析 " + filePath,
+                });
+
+                try
+                {
+                    var extracted = _wordTextExtractionService.ExtractVisibleText(filePath);
+                    if (!string.IsNullOrWhiteSpace(extracted.VisibleText))
+                    {
+                        extractedDocuments.Add(extracted);
+                        result.DocumentsProcessed++;
+                    }
+                    else
+                    {
+                        result.DocumentsSkipped++;
+                        result.Warnings.Add(filePath + " 未提取到可见文本。");
+                    }
+                }
+                catch (Exception ex)
+                {
+                    result.DocumentsSkipped++;
+                    result.Warnings.Add(filePath + " 提取失败: " + ex.Message);
+                }
+            }
+
+            if (extractedDocuments.Count == 0)
+            {
+                return result;
+            }
+
+            var candidateMap = new Dictionary<string, ExtractionCandidate>(StringComparer.Ordinal);
+            var existingSearchTexts = new HashSet<string>(
+                existingRules
+                    .Where(rule => !string.IsNullOrWhiteSpace(rule.SearchText))
+                    .Select(rule => rule.SearchText),
+                StringComparer.Ordinal);
+
+            if (settings.ExtractKeywords)
+            {
+                foreach (var keyword in BuildKeywordCandidates(extractedDocuments, settings))
+                {
+                    AddCandidate(candidateMap, existingSearchTexts, keyword);
+                }
+            }
+
+            if (settings.ExtractHighFrequencyWords)
+            {
+                foreach (var word in BuildHighFrequencyWordCandidates(extractedDocuments, settings))
+                {
+                    AddCandidate(candidateMap, existingSearchTexts, word);
+                }
+            }
+
+            if (settings.ExtractHighFrequencySentences)
+            {
+                foreach (var sentence in BuildSentenceCandidates(extractedDocuments, settings))
+                {
+                    AddCandidate(candidateMap, existingSearchTexts, sentence);
+                }
+            }
+
+            foreach (var candidate in candidateMap.Values.OrderBy(item => item.FirstOccurrenceOrder))
+            {
+                result.Candidates.Add(candidate);
+            }
+
+            return result;
+        }
+
+        private IEnumerable<ExtractionCandidate> BuildKeywordCandidates(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
+        {
+            var tokenObservations = BuildTokenObservations(documents, settings).ToList();
+            if (tokenObservations.Count == 0)
+            {
+                return Enumerable.Empty<ExtractionCandidate>();
+            }
+
+            var candidates = new List<ExtractionCandidate>();
+            if (settings.UseKeywordFrequency)
+            {
+                candidates.AddRange(CreateCandidatesFromRankedEntries(
+                    RankByFrequency(tokenObservations).Take(settings.MaxKeywordCount),
+                    AutoExtractCategory.Keyword,
+                    "关键字(词频)"));
+            }
+
+            if (settings.UseKeywordTfIdf)
+            {
+                candidates.AddRange(CreateCandidatesFromRankedEntries(
+                    RankByTfIdf(tokenObservations).Take(settings.MaxKeywordCount),
+                    AutoExtractCategory.Keyword,
+                    "关键字(TF-IDF)"));
+            }
+
+            if (settings.UseKeywordTextRank)
+            {
+                candidates.AddRange(CreateCandidatesFromRankedEntries(
+                    RankByTextRank(tokenObservations).Take(settings.MaxKeywordCount),
+                    AutoExtractCategory.Keyword,
+                    "关键字(TextRank)"));
+            }
+
+            return candidates;
+        }
+
+        private IEnumerable<ExtractionCandidate> BuildHighFrequencyWordCandidates(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
+        {
+            var tokenObservations = BuildTokenObservations(documents, settings).ToList();
+            return CreateCandidatesFromRankedEntries(
+                RankByFrequency(tokenObservations).Take(settings.MaxWordCount),
+                AutoExtractCategory.HighFrequencyWord,
+                "高频词");
+        }
+
+        private IEnumerable<ExtractionCandidate> BuildSentenceCandidates(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
+        {
+            var sentenceMap = new Dictionary<string, RankedEntry>(StringComparer.Ordinal);
+            var order = 0;
+            var delimiters = new HashSet<char>((settings.SentenceDelimiters ?? string.Empty).ToCharArray());
+
+            foreach (var document in documents)
+            {
+                foreach (var sentence in SplitSentences(document.VisibleText, delimiters))
+                {
+                    var trimmed = sentence.Trim();
+                    if (!IsAllowedSentence(trimmed, settings))
+                    {
+                        continue;
+                    }
+
+                    order++;
+                    RankedEntry entry;
+                    if (!sentenceMap.TryGetValue(trimmed, out entry))
+                    {
+                        entry = new RankedEntry
+                        {
+                            Text = trimmed,
+                            Count = 0,
+                            Score = 0,
+                            FirstDocumentPath = document.FilePath,
+                            FirstOccurrenceOrder = order,
+                        };
+                        sentenceMap[trimmed] = entry;
+                    }
+
+                    entry.Count++;
+                    entry.Score = entry.Count;
+                }
+            }
+
+            return CreateCandidatesFromRankedEntries(
+                sentenceMap.Values
+                    .OrderByDescending(item => item.Count)
+                    .ThenBy(item => item.FirstOccurrenceOrder)
+                    .Take(settings.MaxSentenceCount),
+                AutoExtractCategory.HighFrequencySentence,
+                "高频句");
+        }
+
+        private IEnumerable<TokenObservation> BuildTokenObservations(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
+        {
+            var observations = new List<TokenObservation>();
+            var order = 0;
+
+            for (var documentIndex = 0; documentIndex < documents.Count; documentIndex++)
+            {
+                var document = documents[documentIndex];
+                foreach (Match match in TokenRegex.Matches(document.VisibleText ?? string.Empty))
+                {
+                    if (match.Groups["Email"].Success)
+                    {
+                        if (!settings.ExcludeEmailAddresses)
+                        {
+                            order++;
+                            observations.Add(CreateTokenObservation(match.Value, document.FilePath, documentIndex, order));
+                        }
+
+                        continue;
+                    }
+
+                    if (match.Groups["English"].Success)
+                    {
+                        if (settings.CountEnglishWords && match.Value.Length >= settings.MinEnglishWordLength)
+                        {
+                            order++;
+                            observations.Add(CreateTokenObservation(match.Value, document.FilePath, documentIndex, order));
+                        }
+
+                        continue;
+                    }
+
+                    if (match.Groups["Chinese"].Success && settings.CountChineseWords)
+                    {
+                        foreach (var token in SegmentChinese(match.Value, settings.MinChineseWordLength))
+                        {
+                            if (!ContainsChinese(token))
+                            {
+                                continue;
+                            }
+
+                            order++;
+                            observations.Add(CreateTokenObservation(token, document.FilePath, documentIndex, order));
+                        }
+                    }
+                }
+            }
+
+            return observations;
+        }
+
+        private static IEnumerable<string> SegmentChinese(string text, int minLength)
+        {
+            if (string.IsNullOrWhiteSpace(text))
+            {
+                yield break;
+            }
+
+            var normalized = new string(text.Where(character => character >= 0x4E00 && character <= 0x9FFF).ToArray());
+            if (string.IsNullOrWhiteSpace(normalized))
+            {
+                yield break;
+            }
+
+            var effectiveMinLength = Math.Max(minLength, 1);
+            var maxLength = Math.Min(4, normalized.Length);
+            var emitted = false;
+
+            for (var length = maxLength; length >= effectiveMinLength; length--)
+            {
+                for (var index = 0; index + length <= normalized.Length; index++)
+                {
+                    emitted = true;
+                    yield return normalized.Substring(index, length);
+                }
+            }
+
+            if (!emitted && normalized.Length >= effectiveMinLength)
+            {
+                yield return normalized;
+            }
+        }
+
+        private IEnumerable<RankedEntry> RankByFrequency(IList<TokenObservation> observations)
+        {
+            return observations
+                .GroupBy(item => item.Token, StringComparer.Ordinal)
+                .Select(group => new RankedEntry
+                {
+                    Text = group.Key,
+                    Count = group.Count(),
+                    Score = group.Count(),
+                    FirstDocumentPath = group.OrderBy(item => item.GlobalOrder).First().FilePath,
+                    FirstOccurrenceOrder = group.Min(item => item.GlobalOrder),
+                })
+                .OrderByDescending(item => item.Score)
+                .ThenBy(item => item.FirstOccurrenceOrder)
+                .ToList();
+        }
+
+        private IEnumerable<RankedEntry> RankByTfIdf(IList<TokenObservation> observations)
+        {
+            var documentCount = observations.Select(item => item.DocumentIndex).Distinct().Count();
+            var byToken = observations.GroupBy(item => item.Token, StringComparer.Ordinal);
+            var ranked = new List<RankedEntry>();
+
+            foreach (var tokenGroup in byToken)
+            {
+                var docFrequency = tokenGroup.Select(item => item.DocumentIndex).Distinct().Count();
+                var termFrequency = tokenGroup.Count();
+                var idf = Math.Log((documentCount + 1.0) / (docFrequency + 1.0)) + 1.0;
+                ranked.Add(new RankedEntry
+                {
+                    Text = tokenGroup.Key,
+                    Count = termFrequency,
+                    Score = termFrequency * idf,
+                    FirstDocumentPath = tokenGroup.OrderBy(item => item.GlobalOrder).First().FilePath,
+                    FirstOccurrenceOrder = tokenGroup.Min(item => item.GlobalOrder),
+                });
+            }
+
+            return ranked
+                .OrderByDescending(item => item.Score)
+                .ThenBy(item => item.FirstOccurrenceOrder)
+                .ToList();
+        }
+
+        private IEnumerable<RankedEntry> RankByTextRank(IList<TokenObservation> observations)
+        {
+            var graph = new Dictionary<string, HashSet<string>>(StringComparer.Ordinal);
+            var orderedTokens = observations.OrderBy(item => item.GlobalOrder).ToList();
+            const int windowSize = 4;
+
+            for (var index = 0; index < orderedTokens.Count; index++)
+            {
+                var current = orderedTokens[index].Token;
+                if (!graph.ContainsKey(current))
+                {
+                    graph[current] = new HashSet<string>(StringComparer.Ordinal);
+                }
+
+                for (var offset = 1; offset < windowSize && index + offset < orderedTokens.Count; offset++)
+                {
+                    var neighbor = orderedTokens[index + offset].Token;
+                    if (string.Equals(current, neighbor, StringComparison.Ordinal))
+                    {
+                        continue;
+                    }
+
+                    graph[current].Add(neighbor);
+                    if (!graph.ContainsKey(neighbor))
+                    {
+                        graph[neighbor] = new HashSet<string>(StringComparer.Ordinal);
+                    }
+
+                    graph[neighbor].Add(current);
+                }
+            }
+
+            var scores = graph.Keys.ToDictionary(key => key, key => 1.0, StringComparer.Ordinal);
+            for (var iteration = 0; iteration < 20; iteration++)
+            {
+                var next = new Dictionary<string, double>(StringComparer.Ordinal);
+                foreach (var node in graph.Keys)
+                {
+                    var score = 0.15;
+                    foreach (var neighbor in graph[node])
+                    {
+                        var degree = Math.Max(graph[neighbor].Count, 1);
+                        score += 0.85 * (scores[neighbor] / degree);
+                    }
+
+                    next[node] = score;
+                }
+
+                scores = next;
+            }
+
+            return observations
+                .GroupBy(item => item.Token, StringComparer.Ordinal)
+                .Select(group => new RankedEntry
+                {
+                    Text = group.Key,
+                    Count = group.Count(),
+                    Score = scores.ContainsKey(group.Key) ? scores[group.Key] : 0,
+                    FirstDocumentPath = group.OrderBy(item => item.GlobalOrder).First().FilePath,
+                    FirstOccurrenceOrder = group.Min(item => item.GlobalOrder),
+                })
+                .OrderByDescending(item => item.Score)
+                .ThenBy(item => item.FirstOccurrenceOrder)
+                .ToList();
+        }
+
+        private static IEnumerable<ExtractionCandidate> CreateCandidatesFromRankedEntries(
+            IEnumerable<RankedEntry> entries,
+            AutoExtractCategory category,
+            string categoryDisplay)
+        {
+            return entries.Select(entry => new ExtractionCandidate
+            {
+                PrimaryCategory = category,
+                CategoryDisplay = categoryDisplay,
+                OriginalText = entry.Text,
+                StatisticDisplay = string.Format("次数={0}; 分值={1:F3}", entry.Count, entry.Score),
+                FirstDocumentPath = entry.FirstDocumentPath,
+                FirstOccurrenceOrder = entry.FirstOccurrenceOrder,
+            });
+        }
+
+        private static IEnumerable<string> SplitSentences(string text, HashSet<char> delimiters)
+        {
+            if (string.IsNullOrWhiteSpace(text))
+            {
+                yield break;
+            }
+
+            var buffer = string.Empty;
+            foreach (var character in text)
+            {
+                if (delimiters.Contains(character))
+                {
+                    if (!string.IsNullOrWhiteSpace(buffer))
+                    {
+                        yield return buffer;
+                        buffer = string.Empty;
+                    }
+
+                    continue;
+                }
+
+                buffer += character;
+            }
+
+            if (!string.IsNullOrWhiteSpace(buffer))
+            {
+                yield return buffer;
+            }
+        }
+
+        private static bool IsAllowedSentence(string sentence, AutoExtractSettings settings)
+        {
+            if (string.IsNullOrWhiteSpace(sentence))
+            {
+                return false;
+            }
+
+            var hasChinese = ContainsChinese(sentence);
+            var hasEnglish = sentence.Any(character => (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'));
+
+            if (hasChinese && settings.CountChineseSentences)
+            {
+                return true;
+            }
+
+            if (hasEnglish && settings.CountEnglishSentences)
+            {
+                return true;
+            }
+
+            return !hasChinese && !hasEnglish;
+        }
+
+        private static bool ContainsChinese(string value)
+        {
+            return value.Any(character => character >= 0x4E00 && character <= 0x9FFF);
+        }
+
+        private static TokenObservation CreateTokenObservation(string token, string filePath, int documentIndex, int order)
+        {
+            return new TokenObservation
+            {
+                Token = token,
+                FilePath = filePath,
+                DocumentIndex = documentIndex,
+                GlobalOrder = order,
+            };
+        }
+
+        private static void AddCandidate(
+            IDictionary<string, ExtractionCandidate> candidateMap,
+            ISet<string> existingSearchTexts,
+            ExtractionCandidate candidate)
+        {
+            if (string.IsNullOrWhiteSpace(candidate.OriginalText))
+            {
+                return;
+            }
+
+            if (existingSearchTexts.Contains(candidate.OriginalText))
+            {
+                return;
+            }
+
+            ExtractionCandidate existing;
+            if (!candidateMap.TryGetValue(candidate.OriginalText, out existing))
+            {
+                candidateMap[candidate.OriginalText] = candidate;
+                return;
+            }
+
+            if (existing.CategoryDisplay.IndexOf(candidate.CategoryDisplay, StringComparison.OrdinalIgnoreCase) < 0)
+            {
+                existing.CategoryDisplay += ", " + candidate.CategoryDisplay;
+            }
+        }
+
+        private class TokenObservation
+        {
+            public string Token { get; set; }
+
+            public string FilePath { get; set; }
+
+            public int DocumentIndex { get; set; }
+
+            public int GlobalOrder { get; set; }
+        }
+
+        private class RankedEntry
+        {
+            public string Text { get; set; }
+
+            public int Count { get; set; }
+
+            public double Score { get; set; }
+
+            public string FirstDocumentPath { get; set; }
+
+            public int FirstOccurrenceOrder { get; set; }
+        }
+    }
+}