first commit
This commit is contained in:
lihansani
376
DCIT.Core/Services/RuleService.cs
Normal file
376
DCIT.Core/Services/RuleService.cs
Normal file
@@ -0,0 +1,376 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using DCIT.Core.Models;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace DCIT.Core.Services
|
||||
{
|
||||
public class RuleService
|
||||
{
|
||||
private static readonly TimeSpan RegexValidationTimeout = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
public RuleFile Load(string ruleFilePath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(ruleFilePath) || !System.IO.File.Exists(ruleFilePath))
|
||||
{
|
||||
return new RuleFile();
|
||||
}
|
||||
|
||||
var content = System.IO.File.ReadAllText(ruleFilePath, Encoding.UTF8);
|
||||
return JsonConvert.DeserializeObject<RuleFile>(content) ?? new RuleFile();
|
||||
}
|
||||
|
||||
public void Save(string ruleFilePath, RuleFile ruleFile)
|
||||
{
|
||||
var report = Validate(ruleFile);
|
||||
if (report.HasErrors)
|
||||
{
|
||||
var detail = string.Join(Environment.NewLine, report.Issues.Select(FormatIssue));
|
||||
throw new InvalidOperationException(detail);
|
||||
}
|
||||
|
||||
var content = JsonConvert.SerializeObject(ruleFile, Formatting.Indented);
|
||||
System.IO.File.WriteAllText(ruleFilePath, content, new UTF8Encoding(false));
|
||||
}
|
||||
|
||||
public RuleValidationReport Validate(RuleFile ruleFile)
|
||||
{
|
||||
ruleFile = ruleFile ?? new RuleFile();
|
||||
|
||||
var report = new RuleValidationReport();
|
||||
var idSet = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var validatedRules = new List<ValidatedRuleContext>();
|
||||
|
||||
for (var index = 0; index < ruleFile.Rules.Count; index++)
|
||||
{
|
||||
var rule = ruleFile.Rules[index] ?? new RuleDefinition();
|
||||
if (string.IsNullOrWhiteSpace(rule.Id))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "规则 ID 不能为空。"));
|
||||
}
|
||||
else if (!idSet.Add(rule.Id.Trim()))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "规则 ID 重复。"));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(rule.Name))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Warning, index, rule, "规则名称为空。"));
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(rule.SearchText))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "原文本或表达式不能为空。"));
|
||||
continue;
|
||||
}
|
||||
|
||||
Regex compiledRegex = null;
|
||||
if (rule.MatchMode == RuleMatchMode.RegularExpression)
|
||||
{
|
||||
try
|
||||
{
|
||||
compiledRegex = new Regex(rule.SearchText, GetRegexOptions(rule), RegexValidationTimeout);
|
||||
ValidateRegexRule(report, index, rule, compiledRegex);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "正则表达式非法: " + ex.Message));
|
||||
}
|
||||
}
|
||||
|
||||
if (rule.MatchMode == RuleMatchMode.PlainText
|
||||
&& !string.IsNullOrEmpty(rule.ReplacementText)
|
||||
&& rule.ReplacementText.IndexOf(rule.SearchText, rule.IsCaseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Warning, index, rule, "替换文本会再次命中原文本，请确认是否符合预期。"));
|
||||
}
|
||||
|
||||
if (rule.MatchMode == RuleMatchMode.RegularExpression
|
||||
&& compiledRegex != null
|
||||
&& !string.IsNullOrEmpty(rule.ReplacementText)
|
||||
&& IsRegexReplacementSelfMatching(compiledRegex, rule.ReplacementText))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Warning, index, rule, "替换模板会再次命中当前规则，请确认是否符合预期。"));
|
||||
}
|
||||
|
||||
var current = new ValidatedRuleContext(index, rule, compiledRegex);
|
||||
foreach (var previous in validatedRules)
|
||||
{
|
||||
AddCompatibilityIssues(report, previous, current);
|
||||
}
|
||||
|
||||
validatedRules.Add(current);
|
||||
}
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
public RuleDefinition CreateAutoExtractRule(ExtractionCandidate candidate, IList<RuleDefinition> existingRules)
|
||||
{
|
||||
var prefix = GetPrefix(candidate.PrimaryCategory);
|
||||
var namePrefix = GetNamePrefix(candidate.PrimaryCategory);
|
||||
var nextIndex = 1;
|
||||
var existingIds = new HashSet<string>(existingRules.Select(rule => rule.Id ?? string.Empty), StringComparer.OrdinalIgnoreCase);
|
||||
var existingNames = new HashSet<string>(existingRules.Select(rule => rule.Name ?? string.Empty), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
string id;
|
||||
do
|
||||
{
|
||||
id = string.Format("{0}{1:0000}", prefix, nextIndex++);
|
||||
}
|
||||
while (existingIds.Contains(id));
|
||||
|
||||
nextIndex = 1;
|
||||
string name;
|
||||
do
|
||||
{
|
||||
name = string.Format("{0}-{1:0000}", namePrefix, nextIndex++);
|
||||
}
|
||||
while (existingNames.Contains(name));
|
||||
|
||||
return new RuleDefinition
|
||||
{
|
||||
Id = id,
|
||||
Name = name,
|
||||
IsEnabled = true,
|
||||
MatchMode = RuleMatchMode.PlainText,
|
||||
IsCaseSensitive = false,
|
||||
IsWholeWord = false,
|
||||
SearchText = candidate.OriginalText,
|
||||
ReplacementText = string.Empty,
|
||||
Note = string.Format(
|
||||
"自动提取; 类型={0}; 统计={1}; 首次文档={2}; 时间={3:yyyy-MM-dd HH:mm:ss}",
|
||||
candidate.CategoryDisplay,
|
||||
candidate.StatisticDisplay,
|
||||
candidate.FirstDocumentPath,
|
||||
DateTime.Now),
|
||||
};
|
||||
}
|
||||
|
||||
public string FormatIssue(RuleValidationIssue issue)
|
||||
{
|
||||
return string.Format(
|
||||
"[{0}] 规则#{1} ({2}/{3}) {4}",
|
||||
issue.Severity,
|
||||
issue.Index + 1,
|
||||
issue.RuleId ?? "-",
|
||||
issue.RuleName ?? "-",
|
||||
issue.Message);
|
||||
}
|
||||
|
||||
private static RuleValidationIssue CreateIssue(RuleValidationSeverity severity, int index, RuleDefinition rule, string message)
|
||||
{
|
||||
return new RuleValidationIssue
|
||||
{
|
||||
Severity = severity,
|
||||
Index = index,
|
||||
RuleId = rule.Id,
|
||||
RuleName = rule.Name,
|
||||
Message = message,
|
||||
};
|
||||
}
|
||||
|
||||
private static RegexOptions GetRegexOptions(RuleDefinition rule)
|
||||
{
|
||||
var options = RegexOptions.CultureInvariant;
|
||||
if (!rule.IsCaseSensitive)
|
||||
{
|
||||
options |= RegexOptions.IgnoreCase;
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
private static void ValidateRegexRule(RuleValidationReport report, int index, RuleDefinition rule, Regex regex)
|
||||
{
|
||||
if (MatchesEmptyText(regex))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "正则表达式可匹配零长度文本，执行时可能导致死循环。"));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
regex.Replace("validation", rule.ReplacementText ?? string.Empty, 1);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Error, index, rule, "替换模板非法: " + ex.Message));
|
||||
}
|
||||
|
||||
if (HasNestedQuantifierRisk(rule.SearchText))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Warning, index, rule, "正则表达式存在嵌套量词风险，可能导致性能急剧下降。"));
|
||||
}
|
||||
|
||||
if (MayTimeoutDuringValidation(regex))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(RuleValidationSeverity.Warning, index, rule, "正则表达式验证探测时出现超时风险，建议简化表达式。"));
|
||||
}
|
||||
}
|
||||
|
||||
private static bool MatchesEmptyText(Regex regex)
|
||||
{
|
||||
var emptyMatch = regex.Match(string.Empty);
|
||||
if (emptyMatch.Success && emptyMatch.Length == 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var sampleMatch = regex.Match("A");
|
||||
return sampleMatch.Success && sampleMatch.Length == 0;
|
||||
}
|
||||
|
||||
private static bool IsRegexReplacementSelfMatching(Regex regex, string replacementText)
|
||||
{
|
||||
try
|
||||
{
|
||||
return regex.Match(replacementText ?? string.Empty).Success;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool MayTimeoutDuringValidation(Regex regex)
|
||||
{
|
||||
try
|
||||
{
|
||||
_ = regex.IsMatch(new string('a', 4096) + "!");
|
||||
return false;
|
||||
}
|
||||
catch (RegexMatchTimeoutException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool HasNestedQuantifierRisk(string pattern)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(pattern))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return Regex.IsMatch(
|
||||
pattern,
|
||||
@"\((?:[^()\\]|\\.)+[+*](?:[^()\\]|\\.)*\)[+*{]",
|
||||
RegexOptions.CultureInvariant);
|
||||
}
|
||||
|
||||
private static void AddCompatibilityIssues(RuleValidationReport report, ValidatedRuleContext previous, ValidatedRuleContext current)
|
||||
{
|
||||
if (previous.Rule.Target != current.Rule.Target)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (IsEquivalentSearchText(previous.Rule.SearchText, current.Rule.SearchText))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(
|
||||
RuleValidationSeverity.Warning,
|
||||
current.Index,
|
||||
current.Rule,
|
||||
string.Format("当前规则与 #{0} 使用相同原文本或表达式，请确认顺序影响。", previous.Index + 1)));
|
||||
|
||||
if (HaveDifferentMatchingOptions(previous.Rule, current.Rule))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(
|
||||
RuleValidationSeverity.Warning,
|
||||
current.Index,
|
||||
current.Rule,
|
||||
string.Format("当前规则与 #{0} 使用相同原文本但匹配选项或替换文本不同，继续执行结果可能不稳定。", previous.Index + 1)));
|
||||
}
|
||||
}
|
||||
|
||||
if (previous.Rule.MatchMode == RuleMatchMode.PlainText
|
||||
&& current.Rule.MatchMode == RuleMatchMode.PlainText
|
||||
&& HasPlainTextOverlap(previous.Rule, current.Rule))
|
||||
{
|
||||
report.Issues.Add(CreateIssue(
|
||||
RuleValidationSeverity.Warning,
|
||||
current.Index,
|
||||
current.Rule,
|
||||
string.Format("当前规则与 #{0} 存在重叠命中风险，请确认执行顺序。", previous.Index + 1)));
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsEquivalentSearchText(string left, string right)
|
||||
{
|
||||
return string.Equals(left ?? string.Empty, right ?? string.Empty, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static bool HaveDifferentMatchingOptions(RuleDefinition left, RuleDefinition right)
|
||||
{
|
||||
return left.MatchMode != right.MatchMode
|
||||
|| left.IsCaseSensitive != right.IsCaseSensitive
|
||||
|| left.IsWholeWord != right.IsWholeWord
|
||||
|| !string.Equals(left.ReplacementText ?? string.Empty, right.ReplacementText ?? string.Empty, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
private static bool HasPlainTextOverlap(RuleDefinition left, RuleDefinition right)
|
||||
{
|
||||
if (string.IsNullOrEmpty(left.SearchText) || string.IsNullOrEmpty(right.SearchText))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (left.IsWholeWord && right.IsWholeWord)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var comparison = left.IsCaseSensitive && right.IsCaseSensitive
|
||||
? StringComparison.Ordinal
|
||||
: StringComparison.OrdinalIgnoreCase;
|
||||
|
||||
return left.SearchText.IndexOf(right.SearchText, comparison) >= 0
|
||||
|| right.SearchText.IndexOf(left.SearchText, comparison) >= 0;
|
||||
}
|
||||
|
||||
private static string GetPrefix(AutoExtractCategory category)
|
||||
{
|
||||
switch (category)
|
||||
{
|
||||
case AutoExtractCategory.Keyword:
|
||||
return "AK";
|
||||
case AutoExtractCategory.HighFrequencyWord:
|
||||
return "AW";
|
||||
default:
|
||||
return "AS";
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetNamePrefix(AutoExtractCategory category)
|
||||
{
|
||||
switch (category)
|
||||
{
|
||||
case AutoExtractCategory.Keyword:
|
||||
return "AUTO-KW";
|
||||
case AutoExtractCategory.HighFrequencyWord:
|
||||
return "AUTO-WORD";
|
||||
default:
|
||||
return "AUTO-SENT";
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class ValidatedRuleContext
|
||||
{
|
||||
public ValidatedRuleContext(int index, RuleDefinition rule, Regex compiledRegex)
|
||||
{
|
||||
Index = index;
|
||||
Rule = rule;
|
||||
CompiledRegex = compiledRegex;
|
||||
}
|
||||
|
||||
public int Index { get; private set; }
|
||||
|
||||
public RuleDefinition Rule { get; private set; }
|
||||
|
||||
public Regex CompiledRegex { get; private set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user