377 lines
14 KiB
C#
377 lines
14 KiB
C#
|
using System;
|
||
|
|
using System.Collections.Generic;
|
||
|
|
using System.Linq;
|
||
|
|
using System.Text;
|
||
|
|
using System.Text.RegularExpressions;
|
||
|
|
using DCIT.Core.Models;
|
||
|
|
using Newtonsoft.Json;
|
||
|
|
|
||
|
|
namespace DCIT.Core.Services
|
||
|
|
{
|
||
|
|
public class RuleService
|
||
|
|
{
|
||
|
|
private static readonly TimeSpan RegexValidationTimeout = TimeSpan.FromMilliseconds(250);
|
||
|
|
|
||
|
|
public RuleFile Load(string ruleFilePath)
|
||
|
|
{
|
||
|
|
if (string.IsNullOrWhiteSpace(ruleFilePath) || !System.IO.File.Exists(ruleFilePath))
|
||
|
|
{
|
||
|
|
return new RuleFile();
|
||
|
|
}
|
||
|
|
|
||
|
|
var content = System.IO.File.ReadAllText(ruleFilePath, Encoding.UTF8);
|
||
|
|
return JsonConvert.DeserializeObject<RuleFile>(content) ?? new RuleFile();
|
||
|
|
}
|
||
|
|
|
||
|
|
public void Save(string ruleFilePath, RuleFile ruleFile)
|
||
|
|
{
|
||
|
|
var report = Validate(ruleFile);
|
||
|
|
if (report.HasErrors)
|
||
|
|
{
|
||
|
|
var detail = string.Join(Environment.NewLine, report.Issues.Select(FormatIssue));
|
||
|
|
throw new InvalidOperationException(detail);
|
||
|
|
}
|
||
|
|
|
||
|
|
var content = JsonConvert.SerializeObject(ruleFile, Formatting.Indented);
|
||
|
|
System.IO.File.WriteAllText(ruleFilePath, content, new UTF8Encoding(false));
|
||
|
|
}
|
||
|
|
|
||
|
|
public RuleValidationReport Validate(RuleFile ruleFile)
|
||
|
|
{
|
||
|
|
ruleFile = ruleFile ?? new RuleFile();
|
||
|
|
|
||
|
|
var report = new RuleValidationReport();
|
||
|
|
var idSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||
|
|
var validatedRules = new List<ValidatedRuleContext>();
|
||
|
|
|
||
|
|
for (var index = 0; index < ruleFile.Rules.Count; index++)
|
||
|
|
{
|
||
|
|
var rule = ruleFile.Rules[index] ?? new RuleDefinition();
|
||
|
|
if (string.IsNullOrWhiteSpace(rule.Id))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "规则 ID 不能为空。"));
|
||
|
|
}
|
||
|
|
else if (!idSet.Add(rule.Id.Trim()))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "规则 ID 重复。"));
|
||
|
|
}
|
||
|
|
|
||
|
|
if (string.IsNullOrWhiteSpace(rule.Name))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Warning, index, rule, "规则名称为空。"));
|
||
|
|
}
|
||
|
|
|
||
|
|
if (string.IsNullOrWhiteSpace(rule.SearchText))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "原文本或表达式不能为空。"));
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
Regex compiledRegex = null;
|
||
|
|
if (rule.MatchMode == RuleMatchMode.RegularExpression)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
compiledRegex = new Regex(rule.SearchText, GetRegexOptions(rule), RegexValidationTimeout);
|
||
|
|
ValidateRegexRule(report, index, rule, compiledRegex);
|
||
|
|
}
|
||
|
|
catch (Exception ex)
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "正则表达式非法: " + ex.Message));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (rule.MatchMode == RuleMatchMode.PlainText
|
||
|
|
&& !string.IsNullOrEmpty(rule.ReplacementText)
|
||
|
|
&& rule.ReplacementText.IndexOf(rule.SearchText, rule.IsCaseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase) >= 0)
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Warning, index, rule, "替换文本会再次命中原文本，请确认是否符合预期。"));
|
||
|
|
}
|
||
|
|
|
||
|
|
if (rule.MatchMode == RuleMatchMode.RegularExpression
|
||
|
|
&& compiledRegex != null
|
||
|
|
&& !string.IsNullOrEmpty(rule.ReplacementText)
|
||
|
|
&& IsRegexReplacementSelfMatching(compiledRegex, rule.ReplacementText))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Warning, index, rule, "替换模板会再次命中当前规则，请确认是否符合预期。"));
|
||
|
|
}
|
||
|
|
|
||
|
|
var current = new ValidatedRuleContext(index, rule, compiledRegex);
|
||
|
|
foreach (var previous in validatedRules)
|
||
|
|
{
|
||
|
|
AddCompatibilityIssues(report, previous, current);
|
||
|
|
}
|
||
|
|
|
||
|
|
validatedRules.Add(current);
|
||
|
|
}
|
||
|
|
|
||
|
|
return report;
|
||
|
|
}
|
||
|
|
|
||
|
|
public RuleDefinition CreateAutoExtractRule(ExtractionCandidate candidate, IList<RuleDefinition> existingRules)
|
||
|
|
{
|
||
|
|
var prefix = GetPrefix(candidate.PrimaryCategory);
|
||
|
|
var namePrefix = GetNamePrefix(candidate.PrimaryCategory);
|
||
|
|
var nextIndex = 1;
|
||
|
|
var existingIds = new HashSet<string>(existingRules.Select(rule => rule.Id ?? string.Empty), StringComparer.OrdinalIgnoreCase);
|
||
|
|
var existingNames = new HashSet<string>(existingRules.Select(rule => rule.Name ?? string.Empty), StringComparer.OrdinalIgnoreCase);
|
||
|
|
|
||
|
|
string id;
|
||
|
|
do
|
||
|
|
{
|
||
|
|
id = string.Format("{0}{1:0000}", prefix, nextIndex++);
|
||
|
|
}
|
||
|
|
while (existingIds.Contains(id));
|
||
|
|
|
||
|
|
nextIndex = 1;
|
||
|
|
string name;
|
||
|
|
do
|
||
|
|
{
|
||
|
|
name = string.Format("{0}-{1:0000}", namePrefix, nextIndex++);
|
||
|
|
}
|
||
|
|
while (existingNames.Contains(name));
|
||
|
|
|
||
|
|
return new RuleDefinition
|
||
|
|
{
|
||
|
|
Id = id,
|
||
|
|
Name = name,
|
||
|
|
IsEnabled = true,
|
||
|
|
MatchMode = RuleMatchMode.PlainText,
|
||
|
|
IsCaseSensitive = false,
|
||
|
|
IsWholeWord = false,
|
||
|
|
SearchText = candidate.OriginalText,
|
||
|
|
ReplacementText = string.Empty,
|
||
|
|
Note = string.Format(
|
||
|
|
"自动提取; 类型={0}; 统计={1}; 首次文档={2}; 时间={3:yyyy-MM-dd HH:mm:ss}",
|
||
|
|
candidate.CategoryDisplay,
|
||
|
|
candidate.StatisticDisplay,
|
||
|
|
candidate.FirstDocumentPath,
|
||
|
|
DateTime.Now),
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
public string FormatIssue(RuleValidationIssue issue)
|
||
|
|
{
|
||
|
|
return string.Format(
|
||
|
|
"[{0}] 规则#{1} ({2}/{3}) {4}",
|
||
|
|
issue.Severity,
|
||
|
|
issue.Index + 1,
|
||
|
|
issue.RuleId ?? "-",
|
||
|
|
issue.RuleName ?? "-",
|
||
|
|
issue.Message);
|
||
|
|
}
|
||
|
|
|
||
|
|
private static RuleValidationIssue CreateIssue(RuleValidationSeverity severity, int index, RuleDefinition rule, string message)
|
||
|
|
{
|
||
|
|
return new RuleValidationIssue
|
||
|
|
{
|
||
|
|
Severity = severity,
|
||
|
|
Index = index,
|
||
|
|
RuleId = rule.Id,
|
||
|
|
RuleName = rule.Name,
|
||
|
|
Message = message,
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
private static RegexOptions GetRegexOptions(RuleDefinition rule)
|
||
|
|
{
|
||
|
|
var options = RegexOptions.CultureInvariant;
|
||
|
|
if (!rule.IsCaseSensitive)
|
||
|
|
{
|
||
|
|
options |= RegexOptions.IgnoreCase;
|
||
|
|
}
|
||
|
|
|
||
|
|
return options;
|
||
|
|
}
|
||
|
|
|
||
|
|
private static void ValidateRegexRule(RuleValidationReport report, int index, RuleDefinition rule, Regex regex)
|
||
|
|
{
|
||
|
|
if (MatchesEmptyText(regex))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "正则表达式可匹配零长度文本，执行时可能导致死循环。"));
|
||
|
|
}
|
||
|
|
|
||
|
|
try
|
||
|
|
{
|
||
|
|
regex.Replace("validation", rule.ReplacementText ?? string.Empty, 1);
|
||
|
|
}
|
||
|
|
catch (Exception ex)
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "替换模板非法: " + ex.Message));
|
||
|
|
}
|
||
|
|
|
||
|
|
if (HasNestedQuantifierRisk(rule.SearchText))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Warning, index, rule, "正则表达式存在嵌套量词风险，可能导致性能急剧下降。"));
|
||
|
|
}
|
||
|
|
|
||
|
|
if (MayTimeoutDuringValidation(regex))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(RuleValidationSeverity.Warning, index, rule, "正则表达式验证探测时出现超时风险，建议简化表达式。"));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private static bool MatchesEmptyText(Regex regex)
|
||
|
|
{
|
||
|
|
var emptyMatch = regex.Match(string.Empty);
|
||
|
|
if (emptyMatch.Success && emptyMatch.Length == 0)
|
||
|
|
{
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
|
||
|
|
var sampleMatch = regex.Match("A");
|
||
|
|
return sampleMatch.Success && sampleMatch.Length == 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
private static bool IsRegexReplacementSelfMatching(Regex regex, string replacementText)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
return regex.Match(replacementText ?? string.Empty).Success;
|
||
|
|
}
|
||
|
|
catch
|
||
|
|
{
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private static bool MayTimeoutDuringValidation(Regex regex)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
_ = regex.IsMatch(new string('a', 4096) + "!");
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
catch (RegexMatchTimeoutException)
|
||
|
|
{
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private static bool HasNestedQuantifierRisk(string pattern)
|
||
|
|
{
|
||
|
|
if (string.IsNullOrWhiteSpace(pattern))
|
||
|
|
{
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
return Regex.IsMatch(
|
||
|
|
pattern,
|
||
|
|
@"\((?:[^()\\]|\\.)+[+*](?:[^()\\]|\\.)*\)[+*{]",
|
||
|
|
RegexOptions.CultureInvariant);
|
||
|
|
}
|
||
|
|
|
||
|
|
private static void AddCompatibilityIssues(RuleValidationReport report, ValidatedRuleContext previous, ValidatedRuleContext current)
|
||
|
|
{
|
||
|
|
if (previous.Rule.Target != current.Rule.Target)
|
||
|
|
{
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (IsEquivalentSearchText(previous.Rule.SearchText, current.Rule.SearchText))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(
|
||
|
|
RuleValidationSeverity.Warning,
|
||
|
|
current.Index,
|
||
|
|
current.Rule,
|
||
|
|
string.Format("当前规则与 #{0} 使用相同原文本或表达式，请确认顺序影响。", previous.Index + 1)));
|
||
|
|
|
||
|
|
if (HaveDifferentMatchingOptions(previous.Rule, current.Rule))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(
|
||
|
|
RuleValidationSeverity.Warning,
|
||
|
|
current.Index,
|
||
|
|
current.Rule,
|
||
|
|
string.Format("当前规则与 #{0} 使用相同原文本但匹配选项或替换文本不同，继续执行结果可能不稳定。", previous.Index + 1)));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (previous.Rule.MatchMode == RuleMatchMode.PlainText
|
||
|
|
&& current.Rule.MatchMode == RuleMatchMode.PlainText
|
||
|
|
&& HasPlainTextOverlap(previous.Rule, current.Rule))
|
||
|
|
{
|
||
|
|
report.Issues.Add(CreateIssue(
|
||
|
|
RuleValidationSeverity.Warning,
|
||
|
|
current.Index,
|
||
|
|
current.Rule,
|
||
|
|
string.Format("当前规则与 #{0} 存在重叠命中风险，请确认执行顺序。", previous.Index + 1)));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private static bool IsEquivalentSearchText(string left, string right)
|
||
|
|
{
|
||
|
|
return string.Equals(left ?? string.Empty, right ?? string.Empty, StringComparison.OrdinalIgnoreCase);
|
||
|
|
}
|
||
|
|
|
||
|
|
private static bool HaveDifferentMatchingOptions(RuleDefinition left, RuleDefinition right)
|
||
|
|
{
|
||
|
|
return left.MatchMode != right.MatchMode
|
||
|
|
|| left.IsCaseSensitive != right.IsCaseSensitive
|
||
|
|
|| left.IsWholeWord != right.IsWholeWord
|
||
|
|
|| !string.Equals(left.ReplacementText ?? string.Empty, right.ReplacementText ?? string.Empty, StringComparison.Ordinal);
|
||
|
|
}
|
||
|
|
|
||
|
|
private static bool HasPlainTextOverlap(RuleDefinition left, RuleDefinition right)
|
||
|
|
{
|
||
|
|
if (string.IsNullOrEmpty(left.SearchText) || string.IsNullOrEmpty(right.SearchText))
|
||
|
|
{
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (left.IsWholeWord && right.IsWholeWord)
|
||
|
|
{
|
||
|
|
return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
var comparison = left.IsCaseSensitive && right.IsCaseSensitive
|
||
|
|
? StringComparison.Ordinal
|
||
|
|
: StringComparison.OrdinalIgnoreCase;
|
||
|
|
|
||
|
|
return left.SearchText.IndexOf(right.SearchText, comparison) >= 0
|
||
|
|
|| right.SearchText.IndexOf(left.SearchText, comparison) >= 0;
|
||
|
|
}
|
||
|
|
|
||
|
|
private static string GetPrefix(AutoExtractCategory category)
|
||
|
|
{
|
||
|
|
switch (category)
|
||
|
|
{
|
||
|
|
case AutoExtractCategory.Keyword:
|
||
|
|
return "AK";
|
||
|
|
case AutoExtractCategory.HighFrequencyWord:
|
||
|
|
return "AW";
|
||
|
|
default:
|
||
|
|
return "AS";
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private static string GetNamePrefix(AutoExtractCategory category)
|
||
|
|
{
|
||
|
|
switch (category)
|
||
|
|
{
|
||
|
|
case AutoExtractCategory.Keyword:
|
||
|
|
return "AUTO-KW";
|
||
|
|
case AutoExtractCategory.HighFrequencyWord:
|
||
|
|
return "AUTO-WORD";
|
||
|
|
default:
|
||
|
|
return "AUTO-SENT";
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private sealed class ValidatedRuleContext
|
||
|
|
{
|
||
|
|
public ValidatedRuleContext(int index, RuleDefinition rule, Regex compiledRegex)
|
||
|
|
{
|
||
|
|
Index = index;
|
||
|
|
Rule = rule;
|
||
|
|
CompiledRegex = compiledRegex;
|
||
|
|
}
|
||
|
|
|
||
|
|
public int Index { get; private set; }
|
||
|
|
|
||
|
|
public RuleDefinition Rule { get; private set; }
|
||
|
|
|
||
|
|
public Regex CompiledRegex { get; private set; }
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|