..
This commit is contained in:
那些花儿
@@ -0,0 +1,273 @@
|
||||
package com.budwk.app.base.utils;
|
||||
|
||||
import cn.hutool.core.util.ArrayUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import org.nutz.lang.util.NutMap;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* 文本相似度计算工具类
|
||||
*/
|
||||
public class TextSimilarityUtil {
|
||||
|
||||
/**
|
||||
* 提案与关键词的相似度
|
||||
*
|
||||
* @param proposal 提案数据
|
||||
* @param keywords 关键词
|
||||
* @param matchScope 匹配范围
|
||||
* @return 相似度百分比 (0-100)
|
||||
*/
|
||||
public static double calculateProposalSimilarity(NutMap proposal, String keywords, String[] matchScope) {
|
||||
String title = proposal.getString("name", "");
|
||||
String brief = proposal.getString("brief", "");
|
||||
String measures = proposal.getString("measures", "");
|
||||
|
||||
StringBuilder fullText = new StringBuilder();
|
||||
if (ArrayUtil.isEmpty(matchScope)) {
|
||||
fullText.append(title);
|
||||
fullText.append(" ");
|
||||
fullText.append(brief);
|
||||
fullText.append(" ");
|
||||
fullText.append(measures);
|
||||
} else {
|
||||
for (String scope : matchScope) {
|
||||
String value = removeHtmlTags(proposal.getString(scope, ""));
|
||||
fullText.append(value);
|
||||
fullText.append(" ");
|
||||
}
|
||||
}
|
||||
|
||||
// 计算关键词匹配度
|
||||
return calculateTextSimilarity(fullText.toString(), keywords);
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算两个文本的相似度
|
||||
*
|
||||
* @param text1 文本1
|
||||
* @param text2 文本2（关键词）
|
||||
* @return 相似度百分比 (0-100)
|
||||
*/
|
||||
public static double calculateTextSimilarity(String text1, String text2) {
|
||||
if (StrUtil.isBlank(text1) || StrUtil.isBlank(text2)) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
text1 = text1.toLowerCase().trim();
|
||||
text2 = text2.toLowerCase().trim();
|
||||
|
||||
// 使用多种算法综合计算相似度
|
||||
double keywordSimilarity = calculateKeywordSimilarity(text1, text2);
|
||||
double jaccardSimilarity = calculateJaccardSimilarity(text1, text2);
|
||||
|
||||
// 加权平均
|
||||
return (keywordSimilarity * 0.7 + jaccardSimilarity * 0.3);
|
||||
}
|
||||
|
||||
/**
|
||||
* 基于关键词匹配的相似度计算
|
||||
*/
|
||||
private static double calculateKeywordSimilarity(String text1, String text2) {
|
||||
// 分词处理
|
||||
String[] keywords = text2.split("[\\s\\p{Punct}]+");
|
||||
int matchCount = 0;
|
||||
int totalKeywords = 0;
|
||||
|
||||
for (String keyword : keywords) {
|
||||
keyword = keyword.trim();
|
||||
if (!keyword.isEmpty()) {
|
||||
totalKeywords++;
|
||||
if (text1.contains(keyword)) {
|
||||
matchCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (totalKeywords == 0) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
// 计算匹配率
|
||||
double matchRate = (double) matchCount / totalKeywords;
|
||||
|
||||
// 考虑文本长度因素，避免短文本获得过高分数
|
||||
double lengthFactor = Math.min(text2.length() / (double) Math.max(text1.length(), 1), 1.0);
|
||||
lengthFactor = Math.max(lengthFactor, 0.3); // 最小权重0.3
|
||||
|
||||
return matchRate * lengthFactor * 100;
|
||||
}
|
||||
|
||||
/**
|
||||
* Jaccard相似度计算
|
||||
* 基于集合的交集与并集比值
|
||||
*/
|
||||
private static double calculateJaccardSimilarity(String text1, String text2) {
|
||||
Set<String> set1 = new HashSet<>(Arrays.asList(text1.split("[\\s\\p{Punct}]+")));
|
||||
Set<String> set2 = new HashSet<>(Arrays.asList(text2.split("[\\s\\p{Punct}]+")));
|
||||
|
||||
// 移除空字符串
|
||||
set1.removeIf(String::isEmpty);
|
||||
set2.removeIf(String::isEmpty);
|
||||
|
||||
if (set1.isEmpty() && set2.isEmpty()) {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
// 计算交集
|
||||
Set<String> intersection = new HashSet<>(set1);
|
||||
intersection.retainAll(set2);
|
||||
|
||||
// 计算并集
|
||||
Set<String> union = new HashSet<>(set1);
|
||||
union.addAll(set2);
|
||||
|
||||
return union.isEmpty() ? 0.0 : (double) intersection.size() / union.size() * 100;
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取相似内容片段
|
||||
*
|
||||
* @param proposal 提案数据
|
||||
* @param keywords 关键词
|
||||
* @return 相似内容片段
|
||||
*/
|
||||
public static String extractSimilarContent(NutMap proposal, String keywords) {
|
||||
String brief = removeHtmlTags(proposal.getString("brief", ""));
|
||||
String measures = removeHtmlTags(proposal.getString("measures", ""));
|
||||
String title = proposal.getString("name", "");
|
||||
|
||||
// 优先从标题中查找
|
||||
if (containsKeywords(title, keywords)) {
|
||||
return title;
|
||||
}
|
||||
|
||||
// 查找包含关键词的句子
|
||||
String[] sentences = (brief + " " + measures).split("[。！？\\.\\!\\?]");
|
||||
|
||||
for (String sentence : sentences) {
|
||||
sentence = sentence.trim();
|
||||
if (!sentence.isEmpty() && containsKeywords(sentence, keywords)) {
|
||||
return truncateText(sentence, 150);
|
||||
}
|
||||
}
|
||||
|
||||
// 如果没找到，返回摘要的前150个字符
|
||||
String fallback = !brief.isEmpty() ? brief : measures;
|
||||
return truncateText(fallback, 150);
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查文本是否包含关键词
|
||||
*/
|
||||
public static boolean containsKeywords(String text, String keywords) {
|
||||
if (StrUtil.isBlank(text) || StrUtil.isBlank(keywords)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
String[] keywordArray = keywords.toLowerCase().split("[\\s\\p{Punct}]+");
|
||||
String lowerText = text.toLowerCase();
|
||||
|
||||
for (String keyword : keywordArray) {
|
||||
keyword = keyword.trim();
|
||||
if (!keyword.isEmpty() && lowerText.contains(keyword)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 移除HTML标签
|
||||
*/
|
||||
public static String removeHtmlTags(String html) {
|
||||
if (StrUtil.isBlank(html)) {
|
||||
return "";
|
||||
}
|
||||
return html.replaceAll("<[^>]+>", "")
|
||||
.replaceAll(" ", " ")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll("&", "&")
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* 截断文本到指定长度
|
||||
*/
|
||||
public static String truncateText(String text, int maxLength) {
|
||||
if (StrUtil.isBlank(text)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
text = text.trim();
|
||||
if (text.length() <= maxLength) {
|
||||
return text;
|
||||
}
|
||||
|
||||
return text.substring(0, maxLength) + "...";
|
||||
}
|
||||
|
||||
/**
|
||||
* 高亮关键词
|
||||
*
|
||||
* @param text 原文本
|
||||
* @param keywords 关键词
|
||||
* @return 高亮后的HTML文本
|
||||
*/
|
||||
public static String highlightKeywords(String text, String keywords) {
|
||||
if (StrUtil.isBlank(text) || StrUtil.isBlank(keywords)) {
|
||||
return text;
|
||||
}
|
||||
|
||||
String[] keywordArray = keywords.split("[\\s\\p{Punct}]+");
|
||||
String result = text;
|
||||
|
||||
for (String keyword : keywordArray) {
|
||||
keyword = keyword.trim();
|
||||
if (!keyword.isEmpty()) {
|
||||
// 使用正则表达式进行大小写不敏感的替换
|
||||
result = result.replaceAll("(?i)(" + escapeRegex(keyword) + ")",
|
||||
"<span class=\"highlight\">$1</span>");
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* 转义正则表达式特殊字符
|
||||
*/
|
||||
private static String escapeRegex(String text) {
|
||||
return text.replaceAll("([\\\\\\[\\]{}()*+?.^$|])", "\\\\$1");
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算文本的词频
|
||||
*
|
||||
* @param text 文本
|
||||
* @return 词频统计Map
|
||||
*/
|
||||
public static NutMap calculateWordFrequency(String text) {
|
||||
NutMap frequency = new NutMap();
|
||||
|
||||
if (StrUtil.isBlank(text)) {
|
||||
return frequency;
|
||||
}
|
||||
|
||||
String[] words = text.toLowerCase().split("[\\s\\p{Punct}]+");
|
||||
|
||||
for (String word : words) {
|
||||
word = word.trim();
|
||||
if (word.length() > 1) { // 忽略单字符
|
||||
frequency.put(word, frequency.getInt(word, 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return frequency;
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user