DCIT/DCIT.Core/Services/AutoExtractService.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using DCIT.Core.Models;

namespace DCIT.Core.Services
{
    public class AutoExtractService
    {
        private static readonly Regex TokenRegex = new Regex(
            @"(?<Email>[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,})|(?<English>[A-Za-z][A-Za-z0-9_\-']*)|(?<Chinese>[\u4E00-\u9FFF]+)",
            RegexOptions.Compiled);

        private readonly WordTextExtractionService _wordTextExtractionService;

        public AutoExtractService(WordTextExtractionService wordTextExtractionService)
        {
            _wordTextExtractionService = wordTextExtractionService;
        }

        public Task<AutoExtractResult> ExtractAsync(
            IList<string> selectedFiles,
            AutoExtractSettings settings,
            IList<RuleDefinition> existingRules,
            IProgress<ExtractionProgress> progress,
            CancellationToken cancellationToken)
        {
            return Task.Run(
                () => Execute(selectedFiles, settings, existingRules, progress, cancellationToken),
                cancellationToken);
        }

        private AutoExtractResult Execute(
            IList<string> selectedFiles,
            AutoExtractSettings settings,
            IList<RuleDefinition> existingRules,
            IProgress<ExtractionProgress> progress,
            CancellationToken cancellationToken)
        {
            var result = new AutoExtractResult();
            var extractedDocuments = new List<ExtractedDocumentText>();

            if (selectedFiles == null || selectedFiles.Count == 0)
            {
                return result;
            }

            settings = settings ?? new AutoExtractSettings();
            existingRules = existingRules ?? Array.Empty<RuleDefinition>();

            for (var index = 0; index < selectedFiles.Count; index++)
            {
                cancellationToken.ThrowIfCancellationRequested();
                var filePath = selectedFiles[index];
                progress?.Report(new ExtractionProgress
                {
                    Current = index + 1,
                    Total = selectedFiles.Count,
                    Message = "正在分析 " + filePath,
                });

                try
                {
                    var extracted = _wordTextExtractionService.ExtractVisibleText(filePath);
                    if (!string.IsNullOrWhiteSpace(extracted.VisibleText))
                    {
                        extractedDocuments.Add(extracted);
                        result.DocumentsProcessed++;
                    }
                    else
                    {
                        result.DocumentsSkipped++;
                        result.Warnings.Add(filePath + " 未提取到可见文本。");
                    }
                }
                catch (Exception ex)
                {
                    result.DocumentsSkipped++;
                    result.Warnings.Add(filePath + " 提取失败: " + ex.Message);
                }
            }

            if (extractedDocuments.Count == 0)
            {
                return result;
            }

            var candidateMap = new Dictionary<string, ExtractionCandidate>(StringComparer.Ordinal);
            var existingSearchTexts = new HashSet<string>(
                existingRules
                    .Where(rule => !string.IsNullOrWhiteSpace(rule.SearchText))
                    .Select(rule => rule.SearchText),
                StringComparer.Ordinal);

            if (settings.ExtractKeywords)
            {
                foreach (var keyword in BuildKeywordCandidates(extractedDocuments, settings))
                {
                    AddCandidate(candidateMap, existingSearchTexts, keyword);
                }
            }

            if (settings.ExtractHighFrequencyWords)
            {
                foreach (var word in BuildHighFrequencyWordCandidates(extractedDocuments, settings))
                {
                    AddCandidate(candidateMap, existingSearchTexts, word);
                }
            }

            if (settings.ExtractHighFrequencySentences)
            {
                foreach (var sentence in BuildSentenceCandidates(extractedDocuments, settings))
                {
                    AddCandidate(candidateMap, existingSearchTexts, sentence);
                }
            }

            foreach (var candidate in candidateMap.Values.OrderBy(item => item.FirstOccurrenceOrder))
            {
                result.Candidates.Add(candidate);
            }

            return result;
        }

        private IEnumerable<ExtractionCandidate> BuildKeywordCandidates(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
        {
            var tokenObservations = BuildTokenObservations(documents, settings).ToList();
            if (tokenObservations.Count == 0)
            {
                return Enumerable.Empty<ExtractionCandidate>();
            }

            var candidates = new List<ExtractionCandidate>();
            if (settings.UseKeywordFrequency)
            {
                candidates.AddRange(CreateCandidatesFromRankedEntries(
                    RankByFrequency(tokenObservations).Take(settings.MaxKeywordCount),
                    AutoExtractCategory.Keyword,
                    "关键字(词频)"));
            }

            if (settings.UseKeywordTfIdf)
            {
                candidates.AddRange(CreateCandidatesFromRankedEntries(
                    RankByTfIdf(tokenObservations).Take(settings.MaxKeywordCount),
                    AutoExtractCategory.Keyword,
                    "关键字(TF-IDF)"));
            }

            if (settings.UseKeywordTextRank)
            {
                candidates.AddRange(CreateCandidatesFromRankedEntries(
                    RankByTextRank(tokenObservations).Take(settings.MaxKeywordCount),
                    AutoExtractCategory.Keyword,
                    "关键字(TextRank)"));
            }

            return candidates;
        }

        private IEnumerable<ExtractionCandidate> BuildHighFrequencyWordCandidates(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
        {
            var tokenObservations = BuildTokenObservations(documents, settings).ToList();
            return CreateCandidatesFromRankedEntries(
                RankByFrequency(tokenObservations).Take(settings.MaxWordCount),
                AutoExtractCategory.HighFrequencyWord,
                "高频词");
        }

        private IEnumerable<ExtractionCandidate> BuildSentenceCandidates(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
        {
            var sentenceMap = new Dictionary<string, RankedEntry>(StringComparer.Ordinal);
            var order = 0;
            var delimiters = new HashSet<char>((settings.SentenceDelimiters ?? string.Empty).ToCharArray());

            foreach (var document in documents)
            {
                foreach (var sentence in SplitSentences(document.VisibleText, delimiters))
                {
                    var trimmed = sentence.Trim();
                    if (!IsAllowedSentence(trimmed, settings))
                    {
                        continue;
                    }

                    order++;
                    RankedEntry entry;
                    if (!sentenceMap.TryGetValue(trimmed, out entry))
                    {
                        entry = new RankedEntry
                        {
                            Text = trimmed,
                            Count = 0,
                            Score = 0,
                            FirstDocumentPath = document.FilePath,
                            FirstOccurrenceOrder = order,
                        };
                        sentenceMap[trimmed] = entry;
                    }

                    entry.Count++;
                    entry.Score = entry.Count;
                }
            }

            return CreateCandidatesFromRankedEntries(
                sentenceMap.Values
                    .OrderByDescending(item => item.Count)
                    .ThenBy(item => item.FirstOccurrenceOrder)
                    .Take(settings.MaxSentenceCount),
                AutoExtractCategory.HighFrequencySentence,
                "高频句");
        }

        private IEnumerable<TokenObservation> BuildTokenObservations(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
        {
            var observations = new List<TokenObservation>();
            var order = 0;

            for (var documentIndex = 0; documentIndex < documents.Count; documentIndex++)
            {
                var document = documents[documentIndex];
                foreach (Match match in TokenRegex.Matches(document.VisibleText ?? string.Empty))
                {
                    if (match.Groups["Email"].Success)
                    {
                        if (!settings.ExcludeEmailAddresses)
                        {
                            order++;
                            observations.Add(CreateTokenObservation(match.Value, document.FilePath, documentIndex, order));
                        }

                        continue;
                    }

                    if (match.Groups["English"].Success)
                    {
                        if (settings.CountEnglishWords && match.Value.Length >= settings.MinEnglishWordLength)
                        {
                            order++;
                            observations.Add(CreateTokenObservation(match.Value, document.FilePath, documentIndex, order));
                        }

                        continue;
                    }

                    if (match.Groups["Chinese"].Success && settings.CountChineseWords)
                    {
                        foreach (var token in SegmentChinese(match.Value, settings.MinChineseWordLength))
                        {
                            if (!ContainsChinese(token))
                            {
                                continue;
                            }

                            order++;
                            observations.Add(CreateTokenObservation(token, document.FilePath, documentIndex, order));
                        }
                    }
                }
            }

            return observations;
        }

        private static IEnumerable<string> SegmentChinese(string text, int minLength)
        {
            if (string.IsNullOrWhiteSpace(text))
            {
                yield break;
            }

            var normalized = new string(text.Where(character => character >= 0x4E00 && character <= 0x9FFF).ToArray());
            if (string.IsNullOrWhiteSpace(normalized))
            {
                yield break;
            }

            var effectiveMinLength = Math.Max(minLength, 1);
            var maxLength = Math.Min(4, normalized.Length);
            var emitted = false;

            for (var length = maxLength; length >= effectiveMinLength; length--)
            {
                for (var index = 0; index + length <= normalized.Length; index++)
                {
                    emitted = true;
                    yield return normalized.Substring(index, length);
                }
            }

            if (!emitted && normalized.Length >= effectiveMinLength)
            {
                yield return normalized;
            }
        }

        private IEnumerable<RankedEntry> RankByFrequency(IList<TokenObservation> observations)
        {
            return observations
                .GroupBy(item => item.Token, StringComparer.Ordinal)
                .Select(group => new RankedEntry
                {
                    Text = group.Key,
                    Count = group.Count(),
                    Score = group.Count(),
                    FirstDocumentPath = group.OrderBy(item => item.GlobalOrder).First().FilePath,
                    FirstOccurrenceOrder = group.Min(item => item.GlobalOrder),
                })
                .OrderByDescending(item => item.Score)
                .ThenBy(item => item.FirstOccurrenceOrder)
                .ToList();
        }

        private IEnumerable<RankedEntry> RankByTfIdf(IList<TokenObservation> observations)
        {
            var documentCount = observations.Select(item => item.DocumentIndex).Distinct().Count();
            var byToken = observations.GroupBy(item => item.Token, StringComparer.Ordinal);
            var ranked = new List<RankedEntry>();

            foreach (var tokenGroup in byToken)
            {
                var docFrequency = tokenGroup.Select(item => item.DocumentIndex).Distinct().Count();
                var termFrequency = tokenGroup.Count();
                var idf = Math.Log((documentCount + 1.0) / (docFrequency + 1.0)) + 1.0;
                ranked.Add(new RankedEntry
                {
                    Text = tokenGroup.Key,
                    Count = termFrequency,
                    Score = termFrequency * idf,
                    FirstDocumentPath = tokenGroup.OrderBy(item => item.GlobalOrder).First().FilePath,
                    FirstOccurrenceOrder = tokenGroup.Min(item => item.GlobalOrder),
                });
            }

            return ranked
                .OrderByDescending(item => item.Score)
                .ThenBy(item => item.FirstOccurrenceOrder)
                .ToList();
        }

        private IEnumerable<RankedEntry> RankByTextRank(IList<TokenObservation> observations)
        {
            var graph = new Dictionary<string, HashSet<string>>(StringComparer.Ordinal);
            var orderedTokens = observations.OrderBy(item => item.GlobalOrder).ToList();
            const int windowSize = 4;

            for (var index = 0; index < orderedTokens.Count; index++)
            {
                var current = orderedTokens[index].Token;
                if (!graph.ContainsKey(current))
                {
                    graph[current] = new HashSet<string>(StringComparer.Ordinal);
                }

                for (var offset = 1; offset < windowSize && index + offset < orderedTokens.Count; offset++)
                {
                    var neighbor = orderedTokens[index + offset].Token;
                    if (string.Equals(current, neighbor, StringComparison.Ordinal))
                    {
                        continue;
                    }

                    graph[current].Add(neighbor);
                    if (!graph.ContainsKey(neighbor))
                    {
                        graph[neighbor] = new HashSet<string>(StringComparer.Ordinal);
                    }

                    graph[neighbor].Add(current);
                }
            }

            var scores = graph.Keys.ToDictionary(key => key, key => 1.0, StringComparer.Ordinal);
            for (var iteration = 0; iteration < 20; iteration++)
            {
                var next = new Dictionary<string, double>(StringComparer.Ordinal);
                foreach (var node in graph.Keys)
                {
                    var score = 0.15;
                    foreach (var neighbor in graph[node])
                    {
                        var degree = Math.Max(graph[neighbor].Count, 1);
                        score += 0.85 * (scores[neighbor] / degree);
                    }

                    next[node] = score;
                }

                scores = next;
            }

            return observations
                .GroupBy(item => item.Token, StringComparer.Ordinal)
                .Select(group => new RankedEntry
                {
                    Text = group.Key,
                    Count = group.Count(),
                    Score = scores.ContainsKey(group.Key) ? scores[group.Key] : 0,
                    FirstDocumentPath = group.OrderBy(item => item.GlobalOrder).First().FilePath,
                    FirstOccurrenceOrder = group.Min(item => item.GlobalOrder),
                })
                .OrderByDescending(item => item.Score)
                .ThenBy(item => item.FirstOccurrenceOrder)
                .ToList();
        }

        private static IEnumerable<ExtractionCandidate> CreateCandidatesFromRankedEntries(
            IEnumerable<RankedEntry> entries,
            AutoExtractCategory category,
            string categoryDisplay)
        {
            return entries.Select(entry => new ExtractionCandidate
            {
                PrimaryCategory = category,
                CategoryDisplay = categoryDisplay,
                OriginalText = entry.Text,
                StatisticDisplay = string.Format("次数={0}; 分值={1:F3}", entry.Count, entry.Score),
                FirstDocumentPath = entry.FirstDocumentPath,
                FirstOccurrenceOrder = entry.FirstOccurrenceOrder,
            });
        }

        private static IEnumerable<string> SplitSentences(string text, HashSet<char> delimiters)
        {
            if (string.IsNullOrWhiteSpace(text))
            {
                yield break;
            }

            var buffer = string.Empty;
            foreach (var character in text)
            {
                if (delimiters.Contains(character))
                {
                    if (!string.IsNullOrWhiteSpace(buffer))
                    {
                        yield return buffer;
                        buffer = string.Empty;
                    }

                    continue;
                }

                buffer += character;
            }

            if (!string.IsNullOrWhiteSpace(buffer))
            {
                yield return buffer;
            }
        }

        private static bool IsAllowedSentence(string sentence, AutoExtractSettings settings)
        {
            if (string.IsNullOrWhiteSpace(sentence))
            {
                return false;
            }

            var hasChinese = ContainsChinese(sentence);
            var hasEnglish = sentence.Any(character => (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'));

            if (hasChinese && settings.CountChineseSentences)
            {
                return true;
            }

            if (hasEnglish && settings.CountEnglishSentences)
            {
                return true;
            }

            return !hasChinese && !hasEnglish;
        }

        private static bool ContainsChinese(string value)
        {
            return value.Any(character => character >= 0x4E00 && character <= 0x9FFF);
        }

        private static TokenObservation CreateTokenObservation(string token, string filePath, int documentIndex, int order)
        {
            return new TokenObservation
            {
                Token = token,
                FilePath = filePath,
                DocumentIndex = documentIndex,
                GlobalOrder = order,
            };
        }

        private static void AddCandidate(
            IDictionary<string, ExtractionCandidate> candidateMap,
            ISet<string> existingSearchTexts,
            ExtractionCandidate candidate)
        {
            if (string.IsNullOrWhiteSpace(candidate.OriginalText))
            {
                return;
            }

            if (existingSearchTexts.Contains(candidate.OriginalText))
            {
                return;
            }

            ExtractionCandidate existing;
            if (!candidateMap.TryGetValue(candidate.OriginalText, out existing))
            {
                candidateMap[candidate.OriginalText] = candidate;
                return;
            }

            if (existing.CategoryDisplay.IndexOf(candidate.CategoryDisplay, StringComparison.OrdinalIgnoreCase) < 0)
            {
                existing.CategoryDisplay += ", " + candidate.CategoryDisplay;
            }
        }

        private class TokenObservation
        {
            public string Token { get; set; }

            public string FilePath { get; set; }

            public int DocumentIndex { get; set; }

            public int GlobalOrder { get; set; }
        }

        private class RankedEntry
        {
            public string Text { get; set; }

            public int Count { get; set; }

            public double Score { get; set; }

            public string FirstDocumentPath { get; set; }

            public int FirstOccurrenceOrder { get; set; }
        }
    }
}