Java實現DFA算法進行敏感詞過濾
封裝工具類如下：
使用前需對敏感詞庫進行初始化： SensitiveWordUtil.init(sensitiveWordSet);
package cn.swfilter.util;
import java.util.*;
/**
-
敏感詞處理工具 - DFA算法實現
-
@author sam
-
@since 2017/9/4
*/
public class SensitiveWordUtil {
/**
- 敏感詞匹配規則
*/
public static final int MinMatchTYpe = 1; //最小匹配規則，如：敏感詞庫["中國","中國人"]，語句："我是中國人"，匹配結果：我是[中國]人
public static final int MaxMatchType = 2; //最大匹配規則，如：敏感詞庫["中國","中國人"]，語句："我是中國人"，匹配結果：我是[中國人]
/**
- 敏感詞集合
*/
public static HashMap sensitiveWordMap;
/**
- 初始化敏感詞庫，構建DFA算法模型
- @param sensitiveWordSet 敏感詞庫
*/
public static synchronized void init(Set<String> sensitiveWordSet) {
initSensitiveWordMap(sensitiveWordSet);
}
/**
-
初始化敏感詞庫，構建DFA算法模型
-
@param sensitiveWordSet 敏感詞庫
*/
private static void initSensitiveWordMap(Set<String> sensitiveWordSet) {
//初始化敏感詞容器，減少擴容操作
sensitiveWordMap = new HashMap(sensitiveWordSet.size());
String key;
Map nowMap;
Map<String, String> newWorMap;
//迭代sensitiveWordSet
Iterator<String> iterator = sensitiveWordSet.iterator();
while (iterator.hasNext()) {
//關鍵字
key = iterator.next();
nowMap = sensitiveWordMap;
for (int i = 0; i < key.length(); i++) {
//轉換成char型
char keyChar = key.charAt(i);
//庫中獲取關鍵字
Object wordMap = nowMap.get(keyChar);
//如果存在該key，直接賦值，用於下一個循環獲取
if (wordMap != null) {
nowMap = (Map) wordMap;
} else {
//不存在則，則構建一個map，同時將isEnd設置為0，因為他不是最后一個
newWorMap = new HashMap<>();
//不是最后一個
newWorMap.put("isEnd", "0");
nowMap.put(keyChar, newWorMap);
nowMap = newWorMap;
}
<span class="hljs-keyword">if</span> (i == key.length() - <span class="hljs-number">1</span>) {
<span class="hljs-comment">//最后一個</span>
nowMap.put(<span class="hljs-string">"isEnd"</span>, <span class="hljs-string">"1"</span>);
}
}
}
}
/**
- 判斷文字是否包含敏感字符
- @param txt 文字
- @param matchType 匹配規則 1：最小匹配規則，2：最大匹配規則
- @return 若包含返回true，否則返回false
*/
public static boolean contains(String txt, int matchType) {
boolean flag = false;
for (int i = 0; i < txt.length(); i++) {
int matchFlag = checkSensitiveWord(txt, i, matchType); //判斷是否包含敏感字符
if (matchFlag > 0) { //大於0存在，返回true
flag = true;
}
}
return flag;
}
/**
- 判斷文字是否包含敏感字符
- @param txt 文字
- @return 若包含返回true，否則返回false
*/
public static boolean contains(String txt) {
return contains(txt, MaxMatchType);
}
/**
-
獲取文字中的敏感詞
-
@param txt 文字
-
@param matchType 匹配規則 1：最小匹配規則，2：最大匹配規則
-
@return
*/
public static Set<String> getSensitiveWord(String txt, int matchType) {
Set<String> sensitiveWordList = new HashSet<>();
for (int i = 0; i < txt.length(); i++) {
//判斷是否包含敏感字符
int length = checkSensitiveWord(txt, i, matchType);
if (length > 0) {//存在,加入list中
sensitiveWordList.add(txt.substring(i, i + length));
i = i + length - 1;//減1的原因，是因為for會自增
}
}
return sensitiveWordList;
}
/**
- 獲取文字中的敏感詞
- @param txt 文字
- @return
*/
public static Set<String> getSensitiveWord(String txt) {
return getSensitiveWord(txt, MaxMatchType);
}
/**
-
替換敏感字字符
-
@param txt 文本
-
@param replaceChar 替換的字符，匹配的敏感詞以字符逐個替換，如 語句：我愛中國人 敏感詞：中國人，替換字符：， 替換結果：我愛**
-
@param matchType 敏感詞匹配規則
-
@return
*/
public static String replaceSensitiveWord(String txt, char replaceChar, int matchType) {
String resultTxt = txt;
//獲取所有的敏感詞
Set<String> set = getSensitiveWord(txt, matchType);
Iterator<String> iterator = set.iterator();
String word;
String replaceString;
while (iterator.hasNext()) {
word = iterator.next();
replaceString = getReplaceChars(replaceChar, word.length());
resultTxt = resultTxt.replaceAll(word, replaceString);
}
return resultTxt;
}
/**
- 替換敏感字字符
- @param txt 文本
- @param replaceChar 替換的字符，匹配的敏感詞以字符逐個替換，如 語句：我愛中國人 敏感詞：中國人，替換字符：， 替換結果：我愛**
- @return
*/
public static String replaceSensitiveWord(String txt, char replaceChar) {
return replaceSensitiveWord(txt, replaceChar, MaxMatchType);
}
/**
-
替換敏感字字符
-
@param txt 文本
-
@param replaceStr 替換的字符串，匹配的敏感詞以字符逐個替換，如 語句：我愛中國人 敏感詞：中國人，替換字符串：[屏蔽]，替換結果：我愛[屏蔽]
-
@param matchType 敏感詞匹配規則
-
@return
*/
public static String replaceSensitiveWord(String txt, String replaceStr, int matchType) {
String resultTxt = txt;
//獲取所有的敏感詞
Set<String> set = getSensitiveWord(txt, matchType);
Iterator<String> iterator = set.iterator();
String word;
while (iterator.hasNext()) {
word = iterator.next();
resultTxt = resultTxt.replaceAll(word, replaceStr);
}
return resultTxt;
}
/**
- 替換敏感字字符
- @param txt 文本
- @param replaceStr 替換的字符串，匹配的敏感詞以字符逐個替換，如 語句：我愛中國人 敏感詞：中國人，替換字符串：[屏蔽]，替換結果：我愛[屏蔽]
- @return
*/
public static String replaceSensitiveWord(String txt, String replaceStr) {
return replaceSensitiveWord(txt, replaceStr, MaxMatchType);
}
/**
-
獲取替換字符串
-
@param replaceChar
-
@param length
-
@return
*/
private static String getReplaceChars(char replaceChar, int length) {
String resultReplace = String.valueOf(replaceChar);
for (int i = 1; i < length; i++) {
resultReplace += replaceChar;
}
return resultReplace;
}
/**
- 檢查文字中是否包含敏感字符，檢查規則如下：<br>
- @param txt
- @param beginIndex
- @param matchType
- @return 如果存在，則返回敏感詞字符的長度，不存在返回0
*/
private static int checkSensitiveWord(String txt, int beginIndex, int matchType) {
//敏感詞結束標識位：用於敏感詞只有1位的情況
boolean flag = false;
//匹配標識數默認為0
int matchFlag = 0;
char word;
Map nowMap = sensitiveWordMap;
for (int i = beginIndex; i < txt.length(); i++) {
word = txt.charAt(i);
//獲取指定key
nowMap = (Map) nowMap.get(word);
if (nowMap != null) {//存在，則判斷是否為最后一個
//找到相應key，匹配標識+1
matchFlag++;
//如果為最后一個匹配規則,結束循環，返回匹配標識數
if ("1".equals(nowMap.get("isEnd"))) {
//結束標志位為true
flag = true;
//最小規則，直接返回,最大規則還需繼續查找
if (MinMatchTYpe == matchType) {
break;
}
}
} else {//不存在，直接返回
break;
}
}
if (matchFlag < 2 || !flag) {//長度必須大於等於1，為詞
matchFlag = 0;
}
return matchFlag;
}
public static void main(String[] args) {
Set<String> sensitiveWordSet = <span class="hljs-keyword">new</span> HashSet<>();
sensitiveWordSet.<span class="hljs-keyword">add</span>(<span class="hljs-string">"太多"</span>);
sensitiveWordSet.<span class="hljs-keyword">add</span>(<span class="hljs-string">"愛戀"</span>);
sensitiveWordSet.<span class="hljs-keyword">add</span>(<span class="hljs-string">"靜靜"</span>);
sensitiveWordSet.<span class="hljs-keyword">add</span>(<span class="hljs-string">"哈哈"</span>);
sensitiveWordSet.<span class="hljs-keyword">add</span>(<span class="hljs-string">"啦啦"</span>);
sensitiveWordSet.<span class="hljs-keyword">add</span>(<span class="hljs-string">"感動"</span>);
sensitiveWordSet.<span class="hljs-keyword">add</span>(<span class="hljs-string">"發呆"</span>);
<span class="hljs-comment">//初始化敏感詞庫</span>
SensitiveWordUtil.init(sensitiveWordSet);
System.<span class="hljs-keyword">out</span>.println(<span class="hljs-string">"敏感詞的數量："</span> + SensitiveWordUtil.sensitiveWordMap.size());
String <span class="hljs-keyword">string</span> = <span class="hljs-string">"太多的傷感情懷也許只局限於飼養基地 熒幕中的情節。"</span>
+ <span class="hljs-string">"然后我們的扮演的角色就是跟隨着主人公的喜紅客聯盟 怒哀樂而過於牽強的把自己的情感也附加於銀幕情節中，然后感動就流淚，"</span>
+ <span class="hljs-string">"難過就躺在某一個人的懷里盡情的闡述心扉或者手機卡復制器一個賤人一杯紅酒一部電影在夜 深人靜的晚上，關上電話靜靜的發呆着。"</span>;
System.<span class="hljs-keyword">out</span>.println(<span class="hljs-string">"待檢測語句字數："</span> + <span class="hljs-keyword">string</span>.length());
<span class="hljs-comment">//是否含有關鍵字</span>
boolean result = SensitiveWordUtil.contains(<span class="hljs-keyword">string</span>);
System.<span class="hljs-keyword">out</span>.println(result);
result = SensitiveWordUtil.contains(<span class="hljs-keyword">string</span>, SensitiveWordUtil.MinMatchTYpe);
System.<span class="hljs-keyword">out</span>.println(result);
<span class="hljs-comment">//獲取語句中的敏感詞</span>
Set<String> <span class="hljs-keyword">set</span> = SensitiveWordUtil.getSensitiveWord(<span class="hljs-keyword">string</span>);
System.<span class="hljs-keyword">out</span>.println(<span class="hljs-string">"語句中包含敏感詞的個數為："</span> + <span class="hljs-keyword">set</span>.size() + <span class="hljs-string">"。包含："</span> + <span class="hljs-keyword">set</span>);
<span class="hljs-keyword">set</span> = SensitiveWordUtil.getSensitiveWord(<span class="hljs-keyword">string</span>, SensitiveWordUtil.MinMatchTYpe);
System.<span class="hljs-keyword">out</span>.println(<span class="hljs-string">"語句中包含敏感詞的個數為："</span> + <span class="hljs-keyword">set</span>.size() + <span class="hljs-string">"。包含："</span> + <span class="hljs-keyword">set</span>);
<span class="hljs-comment">//替換語句中的敏感詞</span>
String filterStr = SensitiveWordUtil.replaceSensitiveWord(<span class="hljs-keyword">string</span>, <span class="hljs-string">'*'</span>);
System.<span class="hljs-keyword">out</span>.println(filterStr);
filterStr = SensitiveWordUtil.replaceSensitiveWord(<span class="hljs-keyword">string</span>, <span class="hljs-string">'*'</span>, SensitiveWordUtil.MinMatchTYpe);
System.<span class="hljs-keyword">out</span>.println(filterStr);
String filterStr2 = SensitiveWordUtil.replaceSensitiveWord(<span class="hljs-keyword">string</span>, <span class="hljs-string">"[*敏感詞*]"</span>);
System.<span class="hljs-keyword">out</span>.println(filterStr2);
filterStr2 = SensitiveWordUtil.replaceSensitiveWord(<span class="hljs-keyword">string</span>, <span class="hljs-string">"[*敏感詞*]"</span>, SensitiveWordUtil.MinMatchTYpe);
System.<span class="hljs-keyword">out</span>.println(filterStr2);
}
}
參考：Java實現敏感詞過濾
附敏感詞庫：鏈接: https://pan.baidu.com/s/1bBrbtk 密碼: e4w6
<div class="ad-wrap" style="margin-top: 12px;">
<div id="blog-title-ad" data-traceid="blog_ali" data-tracepid="blogdown" style="">
</div>
</div>