551 lines
20 KiB
C#
551 lines
20 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Text.RegularExpressions;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using DCIT.Core.Models;
|
|
|
|
namespace DCIT.Core.Services
|
|
{
|
|
public class AutoExtractService
|
|
{
|
|
private static readonly Regex TokenRegex = new Regex(
|
|
@"(?<Email>[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,})|(?<English>[A-Za-z][A-Za-z0-9_\-']*)|(?<Chinese>[\u4E00-\u9FFF]+)",
|
|
RegexOptions.Compiled);
|
|
|
|
private readonly WordTextExtractionService _wordTextExtractionService;
|
|
|
|
public AutoExtractService(WordTextExtractionService wordTextExtractionService)
|
|
{
|
|
_wordTextExtractionService = wordTextExtractionService;
|
|
}
|
|
|
|
public Task<AutoExtractResult> ExtractAsync(
|
|
IList<string> selectedFiles,
|
|
AutoExtractSettings settings,
|
|
IList<RuleDefinition> existingRules,
|
|
IProgress<ExtractionProgress> progress,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
return Task.Run(
|
|
() => Execute(selectedFiles, settings, existingRules, progress, cancellationToken),
|
|
cancellationToken);
|
|
}
|
|
|
|
private AutoExtractResult Execute(
|
|
IList<string> selectedFiles,
|
|
AutoExtractSettings settings,
|
|
IList<RuleDefinition> existingRules,
|
|
IProgress<ExtractionProgress> progress,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var result = new AutoExtractResult();
|
|
var extractedDocuments = new List<ExtractedDocumentText>();
|
|
|
|
if (selectedFiles == null || selectedFiles.Count == 0)
|
|
{
|
|
return result;
|
|
}
|
|
|
|
settings = settings ?? new AutoExtractSettings();
|
|
existingRules = existingRules ?? Array.Empty<RuleDefinition>();
|
|
|
|
for (var index = 0; index < selectedFiles.Count; index++)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var filePath = selectedFiles[index];
|
|
progress?.Report(new ExtractionProgress
|
|
{
|
|
Current = index + 1,
|
|
Total = selectedFiles.Count,
|
|
Message = "正在分析 " + filePath,
|
|
});
|
|
|
|
try
|
|
{
|
|
var extracted = _wordTextExtractionService.ExtractVisibleText(filePath);
|
|
if (!string.IsNullOrWhiteSpace(extracted.VisibleText))
|
|
{
|
|
extractedDocuments.Add(extracted);
|
|
result.DocumentsProcessed++;
|
|
}
|
|
else
|
|
{
|
|
result.DocumentsSkipped++;
|
|
result.Warnings.Add(filePath + " 未提取到可见文本。");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
result.DocumentsSkipped++;
|
|
result.Warnings.Add(filePath + " 提取失败: " + ex.Message);
|
|
}
|
|
}
|
|
|
|
if (extractedDocuments.Count == 0)
|
|
{
|
|
return result;
|
|
}
|
|
|
|
var candidateMap = new Dictionary<string, ExtractionCandidate>(StringComparer.Ordinal);
|
|
var existingSearchTexts = new HashSet<string>(
|
|
existingRules
|
|
.Where(rule => !string.IsNullOrWhiteSpace(rule.SearchText))
|
|
.Select(rule => rule.SearchText),
|
|
StringComparer.Ordinal);
|
|
|
|
if (settings.ExtractKeywords)
|
|
{
|
|
foreach (var keyword in BuildKeywordCandidates(extractedDocuments, settings))
|
|
{
|
|
AddCandidate(candidateMap, existingSearchTexts, keyword);
|
|
}
|
|
}
|
|
|
|
if (settings.ExtractHighFrequencyWords)
|
|
{
|
|
foreach (var word in BuildHighFrequencyWordCandidates(extractedDocuments, settings))
|
|
{
|
|
AddCandidate(candidateMap, existingSearchTexts, word);
|
|
}
|
|
}
|
|
|
|
if (settings.ExtractHighFrequencySentences)
|
|
{
|
|
foreach (var sentence in BuildSentenceCandidates(extractedDocuments, settings))
|
|
{
|
|
AddCandidate(candidateMap, existingSearchTexts, sentence);
|
|
}
|
|
}
|
|
|
|
foreach (var candidate in candidateMap.Values.OrderBy(item => item.FirstOccurrenceOrder))
|
|
{
|
|
result.Candidates.Add(candidate);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
private IEnumerable<ExtractionCandidate> BuildKeywordCandidates(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
|
|
{
|
|
var tokenObservations = BuildTokenObservations(documents, settings).ToList();
|
|
if (tokenObservations.Count == 0)
|
|
{
|
|
return Enumerable.Empty<ExtractionCandidate>();
|
|
}
|
|
|
|
var candidates = new List<ExtractionCandidate>();
|
|
if (settings.UseKeywordFrequency)
|
|
{
|
|
candidates.AddRange(CreateCandidatesFromRankedEntries(
|
|
RankByFrequency(tokenObservations).Take(settings.MaxKeywordCount),
|
|
AutoExtractCategory.Keyword,
|
|
"关键字(词频)"));
|
|
}
|
|
|
|
if (settings.UseKeywordTfIdf)
|
|
{
|
|
candidates.AddRange(CreateCandidatesFromRankedEntries(
|
|
RankByTfIdf(tokenObservations).Take(settings.MaxKeywordCount),
|
|
AutoExtractCategory.Keyword,
|
|
"关键字(TF-IDF)"));
|
|
}
|
|
|
|
if (settings.UseKeywordTextRank)
|
|
{
|
|
candidates.AddRange(CreateCandidatesFromRankedEntries(
|
|
RankByTextRank(tokenObservations).Take(settings.MaxKeywordCount),
|
|
AutoExtractCategory.Keyword,
|
|
"关键字(TextRank)"));
|
|
}
|
|
|
|
return candidates;
|
|
}
|
|
|
|
private IEnumerable<ExtractionCandidate> BuildHighFrequencyWordCandidates(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
|
|
{
|
|
var tokenObservations = BuildTokenObservations(documents, settings).ToList();
|
|
return CreateCandidatesFromRankedEntries(
|
|
RankByFrequency(tokenObservations).Take(settings.MaxWordCount),
|
|
AutoExtractCategory.HighFrequencyWord,
|
|
"高频词");
|
|
}
|
|
|
|
private IEnumerable<ExtractionCandidate> BuildSentenceCandidates(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
|
|
{
|
|
var sentenceMap = new Dictionary<string, RankedEntry>(StringComparer.Ordinal);
|
|
var order = 0;
|
|
var delimiters = new HashSet<char>((settings.SentenceDelimiters ?? string.Empty).ToCharArray());
|
|
|
|
foreach (var document in documents)
|
|
{
|
|
foreach (var sentence in SplitSentences(document.VisibleText, delimiters))
|
|
{
|
|
var trimmed = sentence.Trim();
|
|
if (!IsAllowedSentence(trimmed, settings))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
order++;
|
|
RankedEntry entry;
|
|
if (!sentenceMap.TryGetValue(trimmed, out entry))
|
|
{
|
|
entry = new RankedEntry
|
|
{
|
|
Text = trimmed,
|
|
Count = 0,
|
|
Score = 0,
|
|
FirstDocumentPath = document.FilePath,
|
|
FirstOccurrenceOrder = order,
|
|
};
|
|
sentenceMap[trimmed] = entry;
|
|
}
|
|
|
|
entry.Count++;
|
|
entry.Score = entry.Count;
|
|
}
|
|
}
|
|
|
|
return CreateCandidatesFromRankedEntries(
|
|
sentenceMap.Values
|
|
.OrderByDescending(item => item.Count)
|
|
.ThenBy(item => item.FirstOccurrenceOrder)
|
|
.Take(settings.MaxSentenceCount),
|
|
AutoExtractCategory.HighFrequencySentence,
|
|
"高频句");
|
|
}
|
|
|
|
private IEnumerable<TokenObservation> BuildTokenObservations(IList<ExtractedDocumentText> documents, AutoExtractSettings settings)
|
|
{
|
|
var observations = new List<TokenObservation>();
|
|
var order = 0;
|
|
|
|
for (var documentIndex = 0; documentIndex < documents.Count; documentIndex++)
|
|
{
|
|
var document = documents[documentIndex];
|
|
foreach (Match match in TokenRegex.Matches(document.VisibleText ?? string.Empty))
|
|
{
|
|
if (match.Groups["Email"].Success)
|
|
{
|
|
if (!settings.ExcludeEmailAddresses)
|
|
{
|
|
order++;
|
|
observations.Add(CreateTokenObservation(match.Value, document.FilePath, documentIndex, order));
|
|
}
|
|
|
|
continue;
|
|
}
|
|
|
|
if (match.Groups["English"].Success)
|
|
{
|
|
if (settings.CountEnglishWords && match.Value.Length >= settings.MinEnglishWordLength)
|
|
{
|
|
order++;
|
|
observations.Add(CreateTokenObservation(match.Value, document.FilePath, documentIndex, order));
|
|
}
|
|
|
|
continue;
|
|
}
|
|
|
|
if (match.Groups["Chinese"].Success && settings.CountChineseWords)
|
|
{
|
|
foreach (var token in SegmentChinese(match.Value, settings.MinChineseWordLength))
|
|
{
|
|
if (!ContainsChinese(token))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
order++;
|
|
observations.Add(CreateTokenObservation(token, document.FilePath, documentIndex, order));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return observations;
|
|
}
|
|
|
|
private static IEnumerable<string> SegmentChinese(string text, int minLength)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(text))
|
|
{
|
|
yield break;
|
|
}
|
|
|
|
var normalized = new string(text.Where(character => character >= 0x4E00 && character <= 0x9FFF).ToArray());
|
|
if (string.IsNullOrWhiteSpace(normalized))
|
|
{
|
|
yield break;
|
|
}
|
|
|
|
var effectiveMinLength = Math.Max(minLength, 1);
|
|
var maxLength = Math.Min(4, normalized.Length);
|
|
var emitted = false;
|
|
|
|
for (var length = maxLength; length >= effectiveMinLength; length--)
|
|
{
|
|
for (var index = 0; index + length <= normalized.Length; index++)
|
|
{
|
|
emitted = true;
|
|
yield return normalized.Substring(index, length);
|
|
}
|
|
}
|
|
|
|
if (!emitted && normalized.Length >= effectiveMinLength)
|
|
{
|
|
yield return normalized;
|
|
}
|
|
}
|
|
|
|
private IEnumerable<RankedEntry> RankByFrequency(IList<TokenObservation> observations)
|
|
{
|
|
return observations
|
|
.GroupBy(item => item.Token, StringComparer.Ordinal)
|
|
.Select(group => new RankedEntry
|
|
{
|
|
Text = group.Key,
|
|
Count = group.Count(),
|
|
Score = group.Count(),
|
|
FirstDocumentPath = group.OrderBy(item => item.GlobalOrder).First().FilePath,
|
|
FirstOccurrenceOrder = group.Min(item => item.GlobalOrder),
|
|
})
|
|
.OrderByDescending(item => item.Score)
|
|
.ThenBy(item => item.FirstOccurrenceOrder)
|
|
.ToList();
|
|
}
|
|
|
|
private IEnumerable<RankedEntry> RankByTfIdf(IList<TokenObservation> observations)
|
|
{
|
|
var documentCount = observations.Select(item => item.DocumentIndex).Distinct().Count();
|
|
var byToken = observations.GroupBy(item => item.Token, StringComparer.Ordinal);
|
|
var ranked = new List<RankedEntry>();
|
|
|
|
foreach (var tokenGroup in byToken)
|
|
{
|
|
var docFrequency = tokenGroup.Select(item => item.DocumentIndex).Distinct().Count();
|
|
var termFrequency = tokenGroup.Count();
|
|
var idf = Math.Log((documentCount + 1.0) / (docFrequency + 1.0)) + 1.0;
|
|
ranked.Add(new RankedEntry
|
|
{
|
|
Text = tokenGroup.Key,
|
|
Count = termFrequency,
|
|
Score = termFrequency * idf,
|
|
FirstDocumentPath = tokenGroup.OrderBy(item => item.GlobalOrder).First().FilePath,
|
|
FirstOccurrenceOrder = tokenGroup.Min(item => item.GlobalOrder),
|
|
});
|
|
}
|
|
|
|
return ranked
|
|
.OrderByDescending(item => item.Score)
|
|
.ThenBy(item => item.FirstOccurrenceOrder)
|
|
.ToList();
|
|
}
|
|
|
|
private IEnumerable<RankedEntry> RankByTextRank(IList<TokenObservation> observations)
|
|
{
|
|
var graph = new Dictionary<string, HashSet<string>>(StringComparer.Ordinal);
|
|
var orderedTokens = observations.OrderBy(item => item.GlobalOrder).ToList();
|
|
const int windowSize = 4;
|
|
|
|
for (var index = 0; index < orderedTokens.Count; index++)
|
|
{
|
|
var current = orderedTokens[index].Token;
|
|
if (!graph.ContainsKey(current))
|
|
{
|
|
graph[current] = new HashSet<string>(StringComparer.Ordinal);
|
|
}
|
|
|
|
for (var offset = 1; offset < windowSize && index + offset < orderedTokens.Count; offset++)
|
|
{
|
|
var neighbor = orderedTokens[index + offset].Token;
|
|
if (string.Equals(current, neighbor, StringComparison.Ordinal))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
graph[current].Add(neighbor);
|
|
if (!graph.ContainsKey(neighbor))
|
|
{
|
|
graph[neighbor] = new HashSet<string>(StringComparer.Ordinal);
|
|
}
|
|
|
|
graph[neighbor].Add(current);
|
|
}
|
|
}
|
|
|
|
var scores = graph.Keys.ToDictionary(key => key, key => 1.0, StringComparer.Ordinal);
|
|
for (var iteration = 0; iteration < 20; iteration++)
|
|
{
|
|
var next = new Dictionary<string, double>(StringComparer.Ordinal);
|
|
foreach (var node in graph.Keys)
|
|
{
|
|
var score = 0.15;
|
|
foreach (var neighbor in graph[node])
|
|
{
|
|
var degree = Math.Max(graph[neighbor].Count, 1);
|
|
score += 0.85 * (scores[neighbor] / degree);
|
|
}
|
|
|
|
next[node] = score;
|
|
}
|
|
|
|
scores = next;
|
|
}
|
|
|
|
return observations
|
|
.GroupBy(item => item.Token, StringComparer.Ordinal)
|
|
.Select(group => new RankedEntry
|
|
{
|
|
Text = group.Key,
|
|
Count = group.Count(),
|
|
Score = scores.ContainsKey(group.Key) ? scores[group.Key] : 0,
|
|
FirstDocumentPath = group.OrderBy(item => item.GlobalOrder).First().FilePath,
|
|
FirstOccurrenceOrder = group.Min(item => item.GlobalOrder),
|
|
})
|
|
.OrderByDescending(item => item.Score)
|
|
.ThenBy(item => item.FirstOccurrenceOrder)
|
|
.ToList();
|
|
}
|
|
|
|
private static IEnumerable<ExtractionCandidate> CreateCandidatesFromRankedEntries(
|
|
IEnumerable<RankedEntry> entries,
|
|
AutoExtractCategory category,
|
|
string categoryDisplay)
|
|
{
|
|
return entries.Select(entry => new ExtractionCandidate
|
|
{
|
|
PrimaryCategory = category,
|
|
CategoryDisplay = categoryDisplay,
|
|
OriginalText = entry.Text,
|
|
StatisticDisplay = string.Format("次数={0}; 分值={1:F3}", entry.Count, entry.Score),
|
|
FirstDocumentPath = entry.FirstDocumentPath,
|
|
FirstOccurrenceOrder = entry.FirstOccurrenceOrder,
|
|
});
|
|
}
|
|
|
|
private static IEnumerable<string> SplitSentences(string text, HashSet<char> delimiters)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(text))
|
|
{
|
|
yield break;
|
|
}
|
|
|
|
var buffer = string.Empty;
|
|
foreach (var character in text)
|
|
{
|
|
if (delimiters.Contains(character))
|
|
{
|
|
if (!string.IsNullOrWhiteSpace(buffer))
|
|
{
|
|
yield return buffer;
|
|
buffer = string.Empty;
|
|
}
|
|
|
|
continue;
|
|
}
|
|
|
|
buffer += character;
|
|
}
|
|
|
|
if (!string.IsNullOrWhiteSpace(buffer))
|
|
{
|
|
yield return buffer;
|
|
}
|
|
}
|
|
|
|
private static bool IsAllowedSentence(string sentence, AutoExtractSettings settings)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(sentence))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
var hasChinese = ContainsChinese(sentence);
|
|
var hasEnglish = sentence.Any(character => (character >= 'A' && character <= 'Z') || (character >= 'a' && character <= 'z'));
|
|
|
|
if (hasChinese && settings.CountChineseSentences)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
if (hasEnglish && settings.CountEnglishSentences)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
return !hasChinese && !hasEnglish;
|
|
}
|
|
|
|
private static bool ContainsChinese(string value)
|
|
{
|
|
return value.Any(character => character >= 0x4E00 && character <= 0x9FFF);
|
|
}
|
|
|
|
private static TokenObservation CreateTokenObservation(string token, string filePath, int documentIndex, int order)
|
|
{
|
|
return new TokenObservation
|
|
{
|
|
Token = token,
|
|
FilePath = filePath,
|
|
DocumentIndex = documentIndex,
|
|
GlobalOrder = order,
|
|
};
|
|
}
|
|
|
|
private static void AddCandidate(
|
|
IDictionary<string, ExtractionCandidate> candidateMap,
|
|
ISet<string> existingSearchTexts,
|
|
ExtractionCandidate candidate)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(candidate.OriginalText))
|
|
{
|
|
return;
|
|
}
|
|
|
|
if (existingSearchTexts.Contains(candidate.OriginalText))
|
|
{
|
|
return;
|
|
}
|
|
|
|
ExtractionCandidate existing;
|
|
if (!candidateMap.TryGetValue(candidate.OriginalText, out existing))
|
|
{
|
|
candidateMap[candidate.OriginalText] = candidate;
|
|
return;
|
|
}
|
|
|
|
if (existing.CategoryDisplay.IndexOf(candidate.CategoryDisplay, StringComparison.OrdinalIgnoreCase) < 0)
|
|
{
|
|
existing.CategoryDisplay += ", " + candidate.CategoryDisplay;
|
|
}
|
|
}
|
|
|
|
private class TokenObservation
|
|
{
|
|
public string Token { get; set; }
|
|
|
|
public string FilePath { get; set; }
|
|
|
|
public int DocumentIndex { get; set; }
|
|
|
|
public int GlobalOrder { get; set; }
|
|
}
|
|
|
|
private class RankedEntry
|
|
{
|
|
public string Text { get; set; }
|
|
|
|
public int Count { get; set; }
|
|
|
|
public double Score { get; set; }
|
|
|
|
public string FirstDocumentPath { get; set; }
|
|
|
|
public int FirstOccurrenceOrder { get; set; }
|
|
}
|
|
}
|
|
}
|