Skip to content
项目
群组
代码片段
帮助
当前项目
正在载入...
登录 / 注册
切换导航面板
E
event
概览
概览
详情
活动
周期分析
版本库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
统计图
问题
0
议题
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
CI / CD
CI / CD
流水线
作业
日程表
图表
维基
Wiki
代码片段
代码片段
成员
成员
折叠边栏
关闭边栏
活动
图像
聊天
创建新问题
作业
提交
问题看板
Open sidebar
陈世强
event
Commits
c311ac86
提交
c311ac86
authored
4月 28, 2025
作者:
yanxin
浏览文件
操作
浏览文件
下载
差异文件
Merge remote-tracking branch 'origin/event_fusion' into event_fusion
# Conflicts: # src/main/java/com/zzsn/event/es/EsService.java
上级
c78bddff
9badde3e
显示空白字符变更
内嵌
并排
正在显示
27 个修改的文件
包含
1049 行增加
和
81 行删除
+1049
-81
Constants.java
src/main/java/com/zzsn/event/constant/Constants.java
+2
-0
SubjectManageController.java
...va/com/zzsn/event/controller/SubjectManageController.java
+12
-0
ClbSubjectHitwordsMapController.java
...event/controller/biz/ClbSubjectHitwordsMapController.java
+17
-6
InformationController.java
...m/zzsn/event/controller/common/InformationController.java
+171
-7
RelationBindController.java
.../zzsn/event/controller/common/RelationBindController.java
+26
-1
StatisticalAnalysisController.java
...vent/controller/common/StatisticalAnalysisController.java
+3
-0
InfoSource.java
src/main/java/com/zzsn/event/entity/InfoSource.java
+4
-0
Subject.java
src/main/java/com/zzsn/event/entity/Subject.java
+3
-0
EsService.java
src/main/java/com/zzsn/event/es/EsService.java
+167
-30
RemoteProjectService.java
...n/java/com/zzsn/event/feign/api/RemoteProjectService.java
+24
-0
ClbSubjectHitwordsMapMapper.java
...va/com/zzsn/event/mapper/ClbSubjectHitwordsMapMapper.java
+4
-0
ClbSubjectHitwordsMapMapper.xml
...com/zzsn/event/mapper/xml/ClbSubjectHitwordsMapMapper.xml
+5
-0
SubjectMapper.xml
src/main/java/com/zzsn/event/mapper/xml/SubjectMapper.xml
+1
-1
ClbSubjectHitwordsMapService.java
.../com/zzsn/event/service/ClbSubjectHitwordsMapService.java
+3
-0
InformationService.java
src/main/java/com/zzsn/event/service/InformationService.java
+81
-0
SubjectService.java
src/main/java/com/zzsn/event/service/SubjectService.java
+3
-0
ClbSubjectHitwordsMapServiceImpl.java
.../event/service/impl/ClbSubjectHitwordsMapServiceImpl.java
+48
-1
InformationServiceImpl.java
...a/com/zzsn/event/service/impl/InformationServiceImpl.java
+209
-5
SubjectServiceImpl.java
.../java/com/zzsn/event/service/impl/SubjectServiceImpl.java
+51
-0
EsOpUtil.java
src/main/java/com/zzsn/event/util/EsOpUtil.java
+144
-19
HanlpUtil.java
src/main/java/com/zzsn/event/util/HanlpUtil.java
+1
-1
InfoDataSearchCondition.java
src/main/java/com/zzsn/event/vo/InfoDataSearchCondition.java
+32
-9
InfoSourceCondition.java
src/main/java/com/zzsn/event/vo/InfoSourceCondition.java
+1
-1
SubjectPageVO.java
src/main/java/com/zzsn/event/vo/SubjectPageVO.java
+1
-0
DisplayInfo.java
src/main/java/com/zzsn/event/vo/es/DisplayInfo.java
+4
-0
SortField.java
src/main/java/com/zzsn/event/vo/es/SortField.java
+30
-0
SpecialInformation.java
src/main/java/com/zzsn/event/vo/es/SpecialInformation.java
+2
-0
没有找到文件。
src/main/java/com/zzsn/event/constant/Constants.java
浏览文件 @
c311ac86
...
@@ -107,4 +107,6 @@ public class Constants {
...
@@ -107,4 +107,6 @@ public class Constants {
//obs 文件浏览地址 前部分
//obs 文件浏览地址 前部分
public
static
final
String
OBS_FILE_PATH_URL_PREFIX
=
"http://obs.ciglobal.cn/"
;
public
static
final
String
OBS_FILE_PATH_URL_PREFIX
=
"http://obs.ciglobal.cn/"
;
/**专题模型处理中断缓存*/
public
static
final
String
SUBJECT_MODEL_DETAIL_BREAK
=
"SUBJECT_MODEL_DETAIL_BREAK::"
;
}
}
src/main/java/com/zzsn/event/controller/SubjectManageController.java
浏览文件 @
c311ac86
...
@@ -860,6 +860,18 @@ public class SubjectManageController {
...
@@ -860,6 +860,18 @@ public class SubjectManageController {
List
<
InfoSourceLabelVO
>
bindList
=
clbLabelService
.
bindInfoSourceLabelInfo
(
subjectId
);
List
<
InfoSourceLabelVO
>
bindList
=
clbLabelService
.
bindInfoSourceLabelInfo
(
subjectId
);
return
Result
.
OK
(
bindList
);
return
Result
.
OK
(
bindList
);
}
}
/**
* 更新subject的数据范围
* @param subjectId 专题id
*/
@GetMapping
(
"/subjectUpdateScope"
)
public
Result
<?>
subjectUpdateScope
(
@RequestParam
(
name
=
"subjectId"
)
String
subjectId
,
@RequestParam
(
name
=
"type"
)
String
type
,
@RequestParam
(
name
=
"scop"
,
defaultValue
=
""
)
String
scop
)
{
Subject
subject
=
subjectService
.
subjectUpdateScope
(
subjectId
,
type
,
scop
);
return
Result
.
OK
(
subject
);
}
/**
/**
* 绑定信息源标签数据
* 绑定信息源标签数据
...
...
src/main/java/com/zzsn/event/controller/biz/ClbSubjectHitwordsMapController.java
浏览文件 @
c311ac86
...
@@ -3,26 +3,23 @@ package com.zzsn.event.controller.biz;
...
@@ -3,26 +3,23 @@ package com.zzsn.event.controller.biz;
import
cn.hutool.core.collection.CollectionUtil
;
import
cn.hutool.core.collection.CollectionUtil
;
import
cn.hutool.core.util.StrUtil
;
import
cn.hutool.core.util.StrUtil
;
import
cn.hutool.json.JSONObject
;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.baomidou.mybatisplus.core.conditions.query.QueryWrapper
;
import
com.baomidou.mybatisplus.core.metadata.IPage
;
import
com.baomidou.mybatisplus.core.metadata.IPage
;
import
com.baomidou.mybatisplus.core.toolkit.Wrappers
;
import
com.baomidou.mybatisplus.core.toolkit.Wrappers
;
import
com.baomidou.mybatisplus.extension.plugins.pagination.Page
;
import
com.baomidou.mybatisplus.extension.plugins.pagination.Page
;
import
com.zzsn.event.constant.Result
;
import
com.zzsn.event.constant.Result
;
import
com.zzsn.event.entity.ClbSubjectHitwordsMap
;
import
com.zzsn.event.entity.ClbSubjectHitwordsMap
;
import
com.zzsn.event.feign.api.RemoteProjectService
;
import
com.zzsn.event.service.ClbSubjectHitwordsMapService
;
import
com.zzsn.event.service.ClbSubjectHitwordsMapService
;
import
com.zzsn.event.vo.InfoDataSearchCondition
;
import
io.swagger.annotations.ApiOperation
;
import
io.swagger.annotations.ApiOperation
;
import
lombok.extern.slf4j.Slf4j
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.web.bind.annotation.*
;
import
org.springframework.web.bind.annotation.*
;
import
org.springframework.web.servlet.ModelAndView
;
import
javax.servlet.http.HttpServletRequest
;
import
javax.servlet.http.HttpServletRequest
;
import
javax.servlet.http.HttpServletResponse
;
import
java.util.Arrays
;
import
java.util.Arrays
;
import
java.util.List
;
import
java.util.List
;
import
java.util.Map
;
import
java.util.stream.Collectors
;
/**
/**
* <p>
* <p>
...
@@ -33,10 +30,13 @@ import java.util.stream.Collectors;
...
@@ -33,10 +30,13 @@ import java.util.stream.Collectors;
* @since 2025-04-23
* @since 2025-04-23
*/
*/
@RestController
@RestController
@Slf4j
@RequestMapping
(
"/subject/hitwordsMap"
)
@RequestMapping
(
"/subject/hitwordsMap"
)
public
class
ClbSubjectHitwordsMapController
{
public
class
ClbSubjectHitwordsMapController
{
@Autowired
@Autowired
private
ClbSubjectHitwordsMapService
clbSubjectHitwordsMapService
;
private
ClbSubjectHitwordsMapService
clbSubjectHitwordsMapService
;
@Autowired
private
RemoteProjectService
remoteProjectService
;
/**
/**
* 分页列表查询
* 分页列表查询
...
@@ -106,6 +106,17 @@ public class ClbSubjectHitwordsMapController {
...
@@ -106,6 +106,17 @@ public class ClbSubjectHitwordsMapController {
}
}
clbSubjectHitwordsMapService
.
save
(
clbSubjectHitwordsMap
);
clbSubjectHitwordsMapService
.
save
(
clbSubjectHitwordsMap
);
InfoDataSearchCondition
searchCondition
=
new
InfoDataSearchCondition
();
searchCondition
.
setSubjectId
(
clbSubjectHitwordsMap
.
getSubjectId
());
searchCondition
.
setStatus
(
0
);
searchCondition
.
setIsSubject
(
"1"
);
searchCondition
.
setCategory
(
1
);
searchCondition
.
setFlowId
(
"1914980469431201794"
);
//手动中断正在执行的词频历史处理任务
clbSubjectHitwordsMapService
.
stopTask
(
searchCondition
);
String
s
=
remoteProjectService
.
doTaskByFlow
(
searchCondition
);
log
.
info
(
"调用流程编排接口返回结果:{}"
,
s
);
return
Result
.
OK
(
clbSubjectHitwordsMap
);
return
Result
.
OK
(
clbSubjectHitwordsMap
);
}
}
...
...
src/main/java/com/zzsn/event/controller/common/InformationController.java
浏览文件 @
c311ac86
...
@@ -13,11 +13,11 @@ import com.zzsn.event.enums.LabelTypeEnum;
...
@@ -13,11 +13,11 @@ import com.zzsn.event.enums.LabelTypeEnum;
import
com.zzsn.event.es.EsService
;
import
com.zzsn.event.es.EsService
;
import
com.zzsn.event.service.*
;
import
com.zzsn.event.service.*
;
import
com.zzsn.event.util.PythonUtil
;
import
com.zzsn.event.util.PythonUtil
;
import
com.zzsn.event.util.tree.Node
;
import
com.zzsn.event.util.user.UserUtil
;
import
com.zzsn.event.util.user.UserUtil
;
import
com.zzsn.event.util.user.UserVo
;
import
com.zzsn.event.util.user.UserVo
;
import
com.zzsn.event.vo.*
;
import
com.zzsn.event.vo.*
;
import
com.zzsn.event.vo.es.DisplayInfo
;
import
com.zzsn.event.vo.es.DisplayInfo
;
import
com.zzsn.event.vo.es.Label
;
import
com.zzsn.event.vo.es.SpecialInformation
;
import
com.zzsn.event.vo.es.SpecialInformation
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
...
@@ -211,6 +211,7 @@ public class InformationController {
...
@@ -211,6 +211,7 @@ public class InformationController {
IPage
<
DisplayInfo
>
pageList
=
informationService
.
subjectPageList
(
searchCondition
);
IPage
<
DisplayInfo
>
pageList
=
informationService
.
subjectPageList
(
searchCondition
);
return
Result
.
OK
(
pageList
);
return
Result
.
OK
(
pageList
);
}
}
/**
/**
* 事件对应专题库的资讯分页列表
* 事件对应专题库的资讯分页列表
*
*
...
@@ -270,6 +271,7 @@ public class InformationController {
...
@@ -270,6 +271,7 @@ public class InformationController {
/**
/**
* 删除指定标签
* 删除指定标签
*
* @param id
* @param id
* @param relationId
* @param relationId
* @param index
* @param index
...
@@ -279,9 +281,10 @@ public class InformationController {
...
@@ -279,9 +281,10 @@ public class InformationController {
public
Result
<?>
delLabel
(
@RequestParam
(
"id"
)
String
id
,
public
Result
<?>
delLabel
(
@RequestParam
(
"id"
)
String
id
,
@RequestParam
(
"relationId"
)
String
relationId
,
@RequestParam
(
"relationId"
)
String
relationId
,
@RequestParam
(
"index"
)
String
index
)
{
@RequestParam
(
"index"
)
String
index
)
{
informationService
.
removeLabelById
(
index
,
id
,
relationId
);
informationService
.
removeLabelById
(
index
,
id
,
relationId
);
return
Result
.
OK
(
"删除成功"
);
return
Result
.
OK
(
"删除成功"
);
}
}
/**
/**
* 批量删除标签
* 批量删除标签
*
*
...
@@ -303,7 +306,6 @@ public class InformationController {
...
@@ -303,7 +306,6 @@ public class InformationController {
}
}
/**
/**
* 通过id查询(专题库)
* 通过id查询(专题库)
*
*
...
@@ -394,11 +396,11 @@ public class InformationController {
...
@@ -394,11 +396,11 @@ public class InformationController {
*/
*/
@PostMapping
(
value
=
"/modifyLabel"
)
@PostMapping
(
value
=
"/modifyLabel"
)
public
Result
<?>
modifyLabel
(
@RequestBody
DataBindLabelFrom
bindLabelFrom
)
{
public
Result
<?>
modifyLabel
(
@RequestBody
DataBindLabelFrom
bindLabelFrom
)
{
if
(
bindLabelFrom
.
getBindList
()==
null
||
bindLabelFrom
.
getBindList
().
isEmpty
())
{
if
(
bindLabelFrom
.
getBindList
()
==
null
||
bindLabelFrom
.
getBindList
().
isEmpty
())
{
List
<
DictVO
>
boundList
=
subjectDictMapService
.
boundList
(
bindLabelFrom
.
getSubjectId
());
List
<
DictVO
>
boundList
=
subjectDictMapService
.
boundList
(
bindLabelFrom
.
getSubjectId
());
bindLabelFrom
.
setBindList
(
boundList
);
bindLabelFrom
.
setBindList
(
boundList
);
}
}
if
(
bindLabelFrom
.
getBindList
()==
null
||
bindLabelFrom
.
getBindList
().
isEmpty
())
{
if
(
bindLabelFrom
.
getBindList
()
==
null
||
bindLabelFrom
.
getBindList
().
isEmpty
())
{
return
Result
.
FAIL
(
"专题未绑定标签,不可编辑"
);
return
Result
.
FAIL
(
"专题未绑定标签,不可编辑"
);
}
}
informationService
.
modifyLabel
(
bindLabelFrom
);
informationService
.
modifyLabel
(
bindLabelFrom
);
...
@@ -503,7 +505,7 @@ public class InformationController {
...
@@ -503,7 +505,7 @@ public class InformationController {
* @date 2025/2/20
* @date 2025/2/20
*/
*/
@PostMapping
(
"/removeByCondition"
)
@PostMapping
(
"/removeByCondition"
)
public
Result
<?>
removeByCondition
(
@RequestBody
JSONObject
params
){
public
Result
<?>
removeByCondition
(
@RequestBody
JSONObject
params
)
{
String
subjectId
=
params
.
getString
(
"subjectId"
);
String
subjectId
=
params
.
getString
(
"subjectId"
);
if
(
StringUtils
.
isEmpty
(
subjectId
))
{
if
(
StringUtils
.
isEmpty
(
subjectId
))
{
return
Result
.
FAIL
(
"专题id不能为空"
);
return
Result
.
FAIL
(
"专题id不能为空"
);
...
@@ -524,7 +526,7 @@ public class InformationController {
...
@@ -524,7 +526,7 @@ public class InformationController {
* @date 2025/2/20
* @date 2025/2/20
*/
*/
@PostMapping
(
"/supplyByCondition"
)
@PostMapping
(
"/supplyByCondition"
)
public
Result
<?>
supplyByCondition
(
@RequestBody
JSONObject
params
){
public
Result
<?>
supplyByCondition
(
@RequestBody
JSONObject
params
)
{
String
subjectId
=
params
.
getString
(
"subjectId"
);
String
subjectId
=
params
.
getString
(
"subjectId"
);
if
(
StringUtils
.
isEmpty
(
subjectId
))
{
if
(
StringUtils
.
isEmpty
(
subjectId
))
{
return
Result
.
FAIL
(
"专题id不能为空"
);
return
Result
.
FAIL
(
"专题id不能为空"
);
...
@@ -536,4 +538,166 @@ public class InformationController {
...
@@ -536,4 +538,166 @@ public class InformationController {
informationService
.
supplyByCondition
(
subjectId
,
JSONArray
.
parseArray
(
themeIds
.
toJSONString
(),
String
.
class
));
informationService
.
supplyByCondition
(
subjectId
,
JSONArray
.
parseArray
(
themeIds
.
toJSONString
(),
String
.
class
));
return
Result
.
OK
();
return
Result
.
OK
();
}
}
/**
* 批量删除关键词
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping
(
"/removeWord"
)
public
Result
<?>
removeWordLabel
(
@RequestBody
InfoDataSearchCondition
searchCondition
)
{
List
<
String
>
keywordList
=
searchCondition
.
getKeywordList
();
if
(
CollectionUtils
.
isEmpty
(
keywordList
))
{
return
Result
.
FAIL
(
"关键词不能为空"
);
}
CompletableFuture
.
runAsync
(()
->
informationService
.
removeWordLabel
(
searchCondition
));
return
Result
.
OK
(
"关键词删除中。。。"
);
}
/**
* 批量添加到精选
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping
(
"/addToFavorites"
)
public
Result
<?>
addToFavorites
(
@RequestBody
InfoDataSearchCondition
searchCondition
)
{
String
subjectId
=
searchCondition
.
getSubjectId
();
List
<
String
>
ids
=
searchCondition
.
getIds
();
if
(
StringUtils
.
isEmpty
(
subjectId
)
&&
CollectionUtils
.
isEmpty
(
ids
))
{
return
Result
.
FAIL
(
"专题id和资讯id集合不能同时为空"
);
}
informationService
.
addToFavorites
(
searchCondition
);
return
Result
.
OK
();
}
/**
* 批量添加到待定
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping
(
"/addToPend"
)
public
Result
<?>
addToPend
(
@RequestBody
InfoDataSearchCondition
searchCondition
)
{
String
subjectId
=
searchCondition
.
getSubjectId
();
List
<
String
>
ids
=
searchCondition
.
getIds
();
if
(
StringUtils
.
isEmpty
(
subjectId
)
&&
CollectionUtils
.
isEmpty
(
ids
))
{
return
Result
.
FAIL
(
"专题id和资讯id集合不能同时为空"
);
}
informationService
.
addToPend
(
searchCondition
);
return
Result
.
OK
();
}
/**
* 批量添加到移除
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping
(
"/addToRemove"
)
public
Result
<?>
addToRemove
(
@RequestBody
InfoDataSearchCondition
searchCondition
)
{
String
subjectId
=
searchCondition
.
getSubjectId
();
List
<
String
>
ids
=
searchCondition
.
getIds
();
if
(
StringUtils
.
isEmpty
(
subjectId
)
&&
CollectionUtils
.
isEmpty
(
ids
))
{
return
Result
.
FAIL
(
"专题id和资讯id集合不能同时为空"
);
}
informationService
.
addToRemove
(
searchCondition
);
return
Result
.
OK
();
}
/**
* 初始化数据
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping
(
"/initialData"
)
public
Result
<?>
initialData
(
@RequestBody
InfoDataSearchCondition
searchCondition
)
{
String
subjectId
=
searchCondition
.
getSubjectId
();
if
(
StringUtils
.
isEmpty
(
subjectId
))
{
return
Result
.
FAIL
(
"专题id不能为空"
);
}
informationService
.
initialData
(
searchCondition
);
return
Result
.
OK
();
}
//@PostMapping("/searchForReplaceList")
public
Result
<?>
searchForReplaceList
(
@RequestBody
InfoDataSearchCondition
searchCondition
)
{
IPage
<
SpecialInformation
>
page
=
informationService
.
searchForReplaceList
(
searchCondition
);
return
Result
.
OK
(
page
);
}
/**
* 字符串替换
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping
(
"/charReplace"
)
public
Result
<?>
charReplace
(
@RequestBody
InfoDataSearchCondition
searchCondition
)
{
String
subjectId
=
searchCondition
.
getSubjectId
();
if
(
StringUtils
.
isEmpty
(
subjectId
))
{
return
Result
.
FAIL
(
"专题id不能为空"
);
}
List
<
SearchWordVO
>
searchWordList
=
searchCondition
.
getSearchWordList
();
if
(
CollectionUtils
.
isEmpty
(
searchWordList
))
{
return
Result
.
FAIL
(
"检索词不能为空"
);
}
informationService
.
charReplace
(
searchCondition
);
return
Result
.
OK
();
}
/**
* 资讯批量打标
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping
(
"/markTag"
)
public
Result
<?>
markTag
(
@RequestBody
InfoDataSearchCondition
searchCondition
)
{
String
subjectId
=
searchCondition
.
getSubjectId
();
List
<
String
>
ids
=
searchCondition
.
getIds
();
if
(
StringUtils
.
isEmpty
(
subjectId
)
&&
CollectionUtils
.
isEmpty
(
ids
))
{
return
Result
.
FAIL
(
"专题id和资讯id集合不能同时为空"
);
}
List
<
Label
>
markTags
=
searchCondition
.
getMarkTags
();
if
(
CollectionUtils
.
isEmpty
(
markTags
))
{
return
Result
.
FAIL
(
"标签不能为空"
);
}
informationService
.
markTag
(
searchCondition
);
return
Result
.
OK
();
}
/**
* 资讯批量删除标签
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping
(
"/removeTag"
)
public
Result
<?>
removeTag
(
@RequestBody
InfoDataSearchCondition
searchCondition
)
{
String
subjectId
=
searchCondition
.
getSubjectId
();
List
<
String
>
ids
=
searchCondition
.
getIds
();
if
(
StringUtils
.
isEmpty
(
subjectId
)
&&
CollectionUtils
.
isEmpty
(
ids
))
{
return
Result
.
FAIL
(
"专题id和资讯id集合不能同时为空"
);
}
List
<
Label
>
markTags
=
searchCondition
.
getMarkTags
();
if
(
CollectionUtils
.
isEmpty
(
markTags
))
{
return
Result
.
FAIL
(
"标签不能为空"
);
}
informationService
.
removeTag
(
searchCondition
);
return
Result
.
OK
();
}
}
}
src/main/java/com/zzsn/event/controller/common/RelationBindController.java
浏览文件 @
c311ac86
...
@@ -7,9 +7,11 @@ import com.alibaba.fastjson2.JSONObject;
...
@@ -7,9 +7,11 @@ import com.alibaba.fastjson2.JSONObject;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper
;
import
com.baomidou.mybatisplus.core.metadata.IPage
;
import
com.baomidou.mybatisplus.core.metadata.IPage
;
import
com.baomidou.mybatisplus.core.toolkit.Wrappers
;
import
com.baomidou.mybatisplus.core.toolkit.Wrappers
;
import
com.baomidou.mybatisplus.extension.plugins.pagination.Page
;
import
com.zzsn.event.constant.Result
;
import
com.zzsn.event.constant.Result
;
import
com.zzsn.event.entity.InfoSource
;
import
com.zzsn.event.entity.InfoSource
;
import
com.zzsn.event.entity.InfoSourceGroup
;
import
com.zzsn.event.entity.InfoSourceGroup
;
import
com.zzsn.event.entity.Subject
;
import
com.zzsn.event.service.*
;
import
com.zzsn.event.service.*
;
import
com.zzsn.event.util.tree.Node
;
import
com.zzsn.event.util.tree.Node
;
import
com.zzsn.event.vo.*
;
import
com.zzsn.event.vo.*
;
...
@@ -116,7 +118,30 @@ public class RelationBindController {
...
@@ -116,7 +118,30 @@ public class RelationBindController {
}
}
subjectIdList
.
add
(
subjectId
);
subjectIdList
.
add
(
subjectId
);
}
}
IPage
<
InfoSourceVo
>
pageList
=
infoSourceService
.
bindInfoSourcePageList
(
infoSourceCondition
,
subjectIdList
,
pageNo
,
pageSize
);
//查询
IPage
<
InfoSourceVo
>
pageList
=
null
;
if
(
CollectionUtil
.
isNotEmpty
(
subjectIdList
)
&&
subjectIdList
.
size
()
==
1
&&
subjectId
.
equals
(
subjectIdList
.
get
(
0
)))
{
Subject
byId
=
subjectService
.
getById
(
subjectId
);
if
(
ObjectUtil
.
isNotNull
(
byId
)
&&
StringUtils
.
isNotEmpty
(
byId
.
getDataScope
())
&&
byId
.
getDataScope
().
contains
(
"1"
))
{
Page
<
InfoSource
>
page
=
new
Page
<>(
pageNo
,
pageSize
);
return
Result
.
OK
(
infoSourceService
.
page
(
page
,
Wrappers
.<
InfoSource
>
lambdaQuery
()
.
eq
(
ObjectUtil
.
isNotNull
(
infoSourceCondition
.
getCrawlType
()),
InfoSource:
:
getCrawlType
,
infoSourceCondition
.
getCrawlType
())
.
eq
(
StrUtil
.
isNotBlank
(
infoSourceCondition
.
getStatus
()),
InfoSource:
:
getStatus
,
infoSourceCondition
.
getStatus
())
.
like
(
StrUtil
.
isNotBlank
(
infoSourceCondition
.
getWebSiteName
()),
InfoSource:
:
getWebSiteName
,
infoSourceCondition
.
getWebSiteName
())
.
like
(
StrUtil
.
isNotBlank
(
infoSourceCondition
.
getSiteName
()),
InfoSource:
:
getSiteName
,
infoSourceCondition
.
getSiteName
())
.
like
(
StrUtil
.
isNotBlank
(
infoSourceCondition
.
getSiteUri
()),
InfoSource:
:
getSiteUri
,
infoSourceCondition
.
getSiteUri
())
.
orderByDesc
(
InfoSource:
:
getCreateTime
)
));
}
else
{
pageList
=
infoSourceService
.
bindInfoSourcePageList
(
infoSourceCondition
,
subjectIdList
,
pageNo
,
pageSize
);
}
}
else
{
pageList
=
infoSourceService
.
bindInfoSourcePageList
(
infoSourceCondition
,
subjectIdList
,
pageNo
,
pageSize
);
}
return
Result
.
OK
(
pageList
);
return
Result
.
OK
(
pageList
);
}
}
...
...
src/main/java/com/zzsn/event/controller/common/StatisticalAnalysisController.java
浏览文件 @
c311ac86
...
@@ -64,6 +64,9 @@ public class StatisticalAnalysisController {
...
@@ -64,6 +64,9 @@ public class StatisticalAnalysisController {
List
<
CountVO
>
countList
=
new
ArrayList
<>();
List
<
CountVO
>
countList
=
new
ArrayList
<>();
int
totalCount
;
int
totalCount
;
List
<
Label
>
bindLabelList
=
bindLabelList
(
searchCondition
.
getSubjectId
());
List
<
Label
>
bindLabelList
=
bindLabelList
(
searchCondition
.
getSubjectId
());
if
(
CollectionUtils
.
isEmpty
(
bindLabelList
))
{
return
Result
.
FAIL
(
"该专题未绑定信息源标签,无法分析"
);
}
Map
<
String
,
Label
>
labelMap
=
bindLabelList
.
stream
().
collect
(
Collectors
.
toMap
(
label
->
label
.
getLabelMark
()
+
"-"
+
label
.
getRelationId
(),
label
->
label
));
Map
<
String
,
Label
>
labelMap
=
bindLabelList
.
stream
().
collect
(
Collectors
.
toMap
(
label
->
label
.
getLabelMark
()
+
"-"
+
label
.
getRelationId
(),
label
->
label
));
if
(
CollectionUtils
.
isEmpty
(
searchCondition
.
getIds
())
&&
searchCondition
.
getNum
()
==
null
)
{
if
(
CollectionUtils
.
isEmpty
(
searchCondition
.
getIds
())
&&
searchCondition
.
getNum
()
==
null
)
{
List
<
String
>
includeValues
=
new
ArrayList
<>();
List
<
String
>
includeValues
=
new
ArrayList
<>();
...
...
src/main/java/com/zzsn/event/entity/InfoSource.java
浏览文件 @
c311ac86
...
@@ -69,4 +69,8 @@ public class InfoSource implements Serializable {
...
@@ -69,4 +69,8 @@ public class InfoSource implements Serializable {
/**原创来源(中文逗号隔开)*/
/**原创来源(中文逗号隔开)*/
private
String
originalSource
;
private
String
originalSource
;
/**爬虫类别*/
private
String
crawlType
;
/**爬虫类别*/
private
String
status
;
}
}
src/main/java/com/zzsn/event/entity/Subject.java
浏览文件 @
c311ac86
...
@@ -114,5 +114,8 @@ public class Subject implements Serializable {
...
@@ -114,5 +114,8 @@ public class Subject implements Serializable {
/**专题分类id*/
/**专题分类id*/
@TableField
(
exist
=
false
)
@TableField
(
exist
=
false
)
private
String
typeId
;
private
String
typeId
;
/**数据范围(是否是全库) - 采集库全库-1,企业库全库-2,政策库全库-3*/
@TableField
(
updateStrategy
=
FieldStrategy
.
IGNORED
)
// 忽略更新策略
private
String
dataScope
;
}
}
src/main/java/com/zzsn/event/es/EsService.java
浏览文件 @
c311ac86
...
@@ -36,6 +36,7 @@ import org.elasticsearch.action.update.UpdateRequest;
...
@@ -36,6 +36,7 @@ import org.elasticsearch.action.update.UpdateRequest;
import
org.elasticsearch.client.HttpAsyncResponseConsumerFactory
;
import
org.elasticsearch.client.HttpAsyncResponseConsumerFactory
;
import
org.elasticsearch.client.RequestOptions
;
import
org.elasticsearch.client.RequestOptions
;
import
org.elasticsearch.client.RestHighLevelClient
;
import
org.elasticsearch.client.RestHighLevelClient
;
import
org.elasticsearch.common.text.Text
;
import
org.elasticsearch.index.query.*
;
import
org.elasticsearch.index.query.*
;
import
org.elasticsearch.search.SearchHit
;
import
org.elasticsearch.search.SearchHit
;
import
org.elasticsearch.search.SearchHits
;
import
org.elasticsearch.search.SearchHits
;
...
@@ -56,8 +57,9 @@ import org.elasticsearch.search.aggregations.metrics.Cardinality;
...
@@ -56,8 +57,9 @@ import org.elasticsearch.search.aggregations.metrics.Cardinality;
import
org.elasticsearch.search.builder.SearchSourceBuilder
;
import
org.elasticsearch.search.builder.SearchSourceBuilder
;
import
org.elasticsearch.search.collapse.CollapseBuilder
;
import
org.elasticsearch.search.collapse.CollapseBuilder
;
import
org.elasticsearch.search.fetch.subphase.FetchSourceContext
;
import
org.elasticsearch.search.fetch.subphase.FetchSourceContext
;
import
org.elasticsearch.search.sort.SortBuilders
;
import
org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder
;
import
org.elasticsearch.search.sort.SortOrder
;
import
org.elasticsearch.search.fetch.subphase.highlight.HighlightField
;
import
org.elasticsearch.search.sort.*
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.stereotype.Service
;
import
org.springframework.stereotype.Service
;
...
@@ -620,20 +622,20 @@ public class EsService {
...
@@ -620,20 +622,20 @@ public class EsService {
if
(
"1"
.
equals
(
eventDataCondition
.
getIsSubject
()))
{
if
(
"1"
.
equals
(
eventDataCondition
.
getIsSubject
()))
{
Subject
byId
=
subjectService
.
getById
(
eventDataCondition
.
getSubjectId
());
Subject
byId
=
subjectService
.
getById
(
eventDataCondition
.
getSubjectId
());
//更新开始、结束时间
//更新开始、结束时间
if
(
byId
.
getTimeEnable
()!=
null
)
{
if
(
byId
.
getTimeEnable
()
!=
null
)
{
String
timeEnable
=
DateUtil
.
format
(
byId
.
getTimeEnable
(),
"yyyy-MM-dd"
);
String
timeEnable
=
DateUtil
.
format
(
byId
.
getTimeEnable
(),
"yyyy-MM-dd"
);
if
(
StringUtils
.
isEmpty
(
eventDataCondition
.
getStartTime
())
||
eventDataCondition
.
getStartTime
().
compareTo
(
timeEnable
)<
0
)
{
if
(
StringUtils
.
isEmpty
(
eventDataCondition
.
getStartTime
())
||
eventDataCondition
.
getStartTime
().
compareTo
(
timeEnable
)
<
0
)
{
eventDataCondition
.
setStartTime
(
timeEnable
);
eventDataCondition
.
setStartTime
(
timeEnable
);
}
}
}
}
if
(
byId
.
getTimeDisable
()!=
null
)
{
if
(
byId
.
getTimeDisable
()
!=
null
)
{
String
timeDisable
=
DateUtil
.
format
(
byId
.
getTimeDisable
(),
"yyyy-MM-dd"
);
String
timeDisable
=
DateUtil
.
format
(
byId
.
getTimeDisable
(),
"yyyy-MM-dd"
);
if
(
StringUtils
.
isEmpty
(
eventDataCondition
.
getEndTime
())
||
eventDataCondition
.
getEndTime
().
compareTo
(
timeDisable
)>
0
)
{
if
(
StringUtils
.
isEmpty
(
eventDataCondition
.
getEndTime
())
||
eventDataCondition
.
getEndTime
().
compareTo
(
timeDisable
)
>
0
)
{
eventDataCondition
.
setEndTime
(
timeDisable
);
eventDataCondition
.
setEndTime
(
timeDisable
);
}
}
}
}
//以专题指定的索引为准
//以专题指定的索引为准
if
(
StringUtils
.
isNotEmpty
(
byId
.
getEsIndex
()))
{
if
(
StringUtils
.
isNotEmpty
(
byId
.
getEsIndex
()))
{
index
=
byId
.
getEsIndex
();
index
=
byId
.
getEsIndex
();
}
}
}
}
...
@@ -774,9 +776,7 @@ public class EsService {
...
@@ -774,9 +776,7 @@ public class EsService {
* @date 2024/5/6
* @date 2024/5/6
*/
*/
public
IPage
<
SpecialInformation
>
pageListByCondition
(
InfoDataSearchCondition
searchCondition
,
List
<
String
>
subjectIdList
)
throws
IOException
{
public
IPage
<
SpecialInformation
>
pageListByCondition
(
InfoDataSearchCondition
searchCondition
,
List
<
String
>
subjectIdList
)
throws
IOException
{
String
minDate
=
subjectService
.
getMinCreateTime
(
subjectIdList
);
String
minDate
=
subjectService
.
getMinCreateTime
(
subjectIdList
);
// Subject byId = subjectService(searchCondition.getSubjectId());
String
[]
indexs
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minDate
);
String
[]
indexs
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minDate
);
SearchRequest
searchRequest
=
new
SearchRequest
(
indexs
);
SearchRequest
searchRequest
=
new
SearchRequest
(
indexs
);
SearchSourceBuilder
searchSourceBuilder
=
new
SearchSourceBuilder
();
SearchSourceBuilder
searchSourceBuilder
=
new
SearchSourceBuilder
();
...
@@ -800,36 +800,62 @@ public class EsService {
...
@@ -800,36 +800,62 @@ public class EsService {
if
(
searchCondition
.
getTopSortValid
()
==
1
)
{
if
(
searchCondition
.
getTopSortValid
()
==
1
)
{
searchSourceBuilder
.
sort
(
"topNum"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"topNum"
,
SortOrder
.
DESC
);
}
}
switch
(
column
)
{
if
(
StringUtils
.
isEmpty
(
column
))
{
case
"score"
:
if
(
order
.
equals
(
"asc"
))
{
searchSourceBuilder
.
sort
(
"score"
,
SortOrder
.
ASC
);
searchSourceBuilder
.
sort
(
"publishDate"
,
SortOrder
.
ASC
);
}
else
if
(
order
.
equals
(
"desc"
))
{
searchSourceBuilder
.
sort
(
"score"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"publishDate"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"publishDate"
,
SortOrder
.
DESC
);
}
else
{
SortOrder
sortOrder
=
SortOrder
.
DESC
;
if
(
order
.
equals
(
"asc"
))
{
sortOrder
=
SortOrder
.
ASC
;
}
}
switch
(
column
)
{
case
"score"
:
searchSourceBuilder
.
sort
(
"score"
,
sortOrder
);
searchSourceBuilder
.
sort
(
"publishDate"
,
sortOrder
);
break
;
break
;
case
"publishDate"
:
case
"publishDate"
:
if
(
order
.
equals
(
"desc"
))
{
searchSourceBuilder
.
sort
(
"publishDate"
,
sortOrder
);
searchSourceBuilder
.
sort
(
"publishDate"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"score"
,
sortOrder
);
searchSourceBuilder
.
sort
(
"score"
,
SortOrder
.
DESC
);
}
else
if
(
order
.
equals
(
"asc"
))
{
searchSourceBuilder
.
sort
(
"publishDate"
,
SortOrder
.
ASC
);
searchSourceBuilder
.
sort
(
"score"
,
SortOrder
.
ASC
);
}
break
;
break
;
case
"yqgzzk"
:
case
"yqgzzk"
:
searchSourceBuilder
.
sort
(
"dataFrom"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"dataFrom"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"topNum"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"topNum"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"score"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"score"
,
SortOrder
.
DESC
);
break
;
break
;
case
"title"
:
case
"origin"
:
searchSourceBuilder
.
sort
(
column
+
".keyword"
,
sortOrder
);
break
;
case
"summary"
:
case
"content"
:
NestedSortBuilder
nestedSortBuilder
=
new
NestedSortBuilder
(
"sortField"
);
nestedSortBuilder
.
setFilter
(
QueryBuilders
.
termQuery
(
"sortField.fieldType"
,
column
));
FieldSortBuilder
fieldSortBuilder
=
SortBuilders
.
fieldSort
(
"sortField.fieldKeyword"
)
.
order
(
sortOrder
)
.
sortMode
(
SortMode
.
MAX
)
.
setNestedSort
(
nestedSortBuilder
);
searchSourceBuilder
.
sort
(
fieldSortBuilder
);
break
;
case
"labels"
:
NestedSortBuilder
nestedLabelsSortBuilder
=
new
NestedSortBuilder
(
"labels"
);
FieldSortBuilder
labelSortBuilder
=
SortBuilders
.
fieldSort
(
"labels.relationId"
)
.
order
(
sortOrder
)
.
sortMode
(
SortMode
.
MAX
)
.
setNestedSort
(
nestedLabelsSortBuilder
);
searchSourceBuilder
.
sort
(
labelSortBuilder
);
break
;
default
:
default
:
searchSourceBuilder
.
sort
(
"publishDate"
,
SortOrder
.
DESC
);
NestedSortBuilder
nestedWordSortBuilder
=
new
NestedSortBuilder
(
"sortField"
);
nestedWordSortBuilder
.
setFilter
(
QueryBuilders
.
termsQuery
(
"sortField.fieldType"
,
Arrays
.
asList
(
column
.
split
(
","
))));
FieldSortBuilder
fieldWordSortBuilder
=
SortBuilders
.
fieldSort
(
"sortField.fieldLong"
)
.
order
(
sortOrder
)
.
sortMode
(
SortMode
.
SUM
)
.
setNestedSort
(
nestedWordSortBuilder
);
searchSourceBuilder
.
sort
(
fieldWordSortBuilder
);
break
;
break
;
}
}
}
//构建es查询条件
//构建es查询条件
BoolQueryBuilder
boolQuery
=
null
;
BoolQueryBuilder
boolQuery
;
//判断ids字段是否为空,若不为空,则表示按照id勾选
//判断ids字段是否为空,若不为空,则表示按照id勾选
List
<
String
>
ids
=
searchCondition
.
getIds
();
List
<
String
>
ids
=
searchCondition
.
getIds
();
if
(
CollectionUtils
.
isNotEmpty
(
ids
))
{
if
(
CollectionUtils
.
isNotEmpty
(
ids
))
{
...
@@ -839,7 +865,7 @@ public class EsService {
...
@@ -839,7 +865,7 @@ public class EsService {
if
(
CollectionUtils
.
isNotEmpty
(
sourceIdList
))
{
if
(
CollectionUtils
.
isNotEmpty
(
sourceIdList
))
{
boolQuery
.
must
(
QueryBuilders
.
termsQuery
(
"sid.keyword"
,
sourceIdList
));
boolQuery
.
must
(
QueryBuilders
.
termsQuery
(
"sid.keyword"
,
sourceIdList
));
}
}
}
else
{
}
else
{
boolQuery
=
buildQuery
(
searchCondition
,
subjectIdList
);
boolQuery
=
buildQuery
(
searchCondition
,
subjectIdList
);
}
}
searchSourceBuilder
.
query
(
boolQuery
);
searchSourceBuilder
.
query
(
boolQuery
);
...
@@ -1213,6 +1239,9 @@ public class EsService {
...
@@ -1213,6 +1239,9 @@ public class EsService {
SearchSourceBuilder
searchSourceBuilder
=
new
SearchSourceBuilder
();
SearchSourceBuilder
searchSourceBuilder
=
new
SearchSourceBuilder
();
String
column
=
searchCondition
.
getColumn
();
String
column
=
searchCondition
.
getColumn
();
String
order
=
searchCondition
.
getOrder
();
String
order
=
searchCondition
.
getOrder
();
if
(
StringUtils
.
isEmpty
(
column
))
{
searchSourceBuilder
.
sort
(
"publishDate"
,
SortOrder
.
DESC
);
}
else
{
switch
(
column
)
{
switch
(
column
)
{
case
"topNum"
:
case
"topNum"
:
searchSourceBuilder
.
sort
(
"topNum"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"topNum"
,
SortOrder
.
DESC
);
...
@@ -1235,6 +1264,7 @@ public class EsService {
...
@@ -1235,6 +1264,7 @@ public class EsService {
searchSourceBuilder
.
sort
(
"publishDate"
,
SortOrder
.
DESC
);
searchSourceBuilder
.
sort
(
"publishDate"
,
SortOrder
.
DESC
);
break
;
break
;
}
}
}
String
[]
fetchFields
=
searchCondition
.
getFetchFields
();
String
[]
fetchFields
=
searchCondition
.
getFetchFields
();
if
(
fetchFields
!=
null
&&
fetchFields
.
length
>
0
)
{
if
(
fetchFields
!=
null
&&
fetchFields
.
length
>
0
)
{
searchSourceBuilder
.
fetchSource
(
fetchFields
,
null
);
searchSourceBuilder
.
fetchSource
(
fetchFields
,
null
);
...
@@ -1914,6 +1944,76 @@ public class EsService {
...
@@ -1914,6 +1944,76 @@ public class EsService {
return
list
;
return
list
;
}
}
public
IPage
<
SpecialInformation
>
searchForReplaceList
(
InfoDataSearchCondition
searchCondition
)
{
List
<
String
>
subjectIds
=
new
ArrayList
<>();
subjectIds
.
add
(
searchCondition
.
getSubjectId
());
String
minCreateTime
=
subjectService
.
getMinCreateTime
(
subjectIds
);
String
[]
indexArr
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minCreateTime
);
SearchRequest
searchRequest
=
new
SearchRequest
(
indexArr
);
SearchSourceBuilder
searchSourceBuilder
=
new
SearchSourceBuilder
();
searchSourceBuilder
.
trackTotalHits
(
true
);
String
[]
fetchFields
=
new
String
[]{
"id"
,
"title"
,
"origin"
,
"content"
};
searchSourceBuilder
.
fetchSource
(
fetchFields
,
null
);
//设置高亮
HighlightBuilder
highlightBuilder
=
new
HighlightBuilder
();
highlightBuilder
.
preTags
(
"<font style = 'color:red;'>"
);
highlightBuilder
.
postTags
(
"</font>"
);
List
<
HighlightBuilder
.
Field
>
fields
=
highlightBuilder
.
fields
();
fields
.
add
(
new
HighlightBuilder
.
Field
(
"title"
).
numOfFragments
(
0
));
fields
.
add
(
new
HighlightBuilder
.
Field
(
"origin"
).
numOfFragments
(
0
));
fields
.
add
(
new
HighlightBuilder
.
Field
(
"content"
).
numOfFragments
(
0
));
searchSourceBuilder
.
highlighter
(
highlightBuilder
);
//设置分页参数
Integer
pageNo
=
searchCondition
.
getPageNo
();
Integer
pageSize
=
searchCondition
.
getPageSize
();
searchSourceBuilder
.
from
((
pageNo
-
1
)
*
pageSize
);
searchSourceBuilder
.
size
(
pageSize
);
BoolQueryBuilder
boolQuery
=
buildQuery
(
searchCondition
,
subjectIds
);
searchSourceBuilder
.
query
(
boolQuery
);
searchRequest
.
source
(
searchSourceBuilder
);
IPage
<
SpecialInformation
>
pageData
=
new
Page
<>();
try
{
SearchResponse
searchResponse
=
client
.
search
(
searchRequest
,
RequestOptions
.
DEFAULT
);
SearchHit
[]
searchHits
=
searchResponse
.
getHits
().
getHits
();
List
<
SpecialInformation
>
list
=
new
ArrayList
<>();
for
(
SearchHit
hit
:
searchHits
)
{
String
queryInfo
=
hit
.
getSourceAsString
();
SpecialInformation
info
=
JSONUtil
.
toBean
(
queryInfo
,
SpecialInformation
.
class
);
Map
<
String
,
HighlightField
>
highlightFields
=
hit
.
getHighlightFields
();
if
(
highlightFields
.
containsKey
(
"title"
))
{
Text
[]
titles
=
highlightFields
.
get
(
"title"
).
getFragments
();
StringBuilder
titleHighlight
=
new
StringBuilder
();
for
(
Text
title
:
titles
)
{
titleHighlight
.
append
(
title
.
string
());
}
info
.
setTitle
(
titleHighlight
.
toString
());
}
if
(
highlightFields
.
containsKey
(
"origin"
))
{
Text
[]
origins
=
highlightFields
.
get
(
"origin"
).
getFragments
();
StringBuilder
originHighlight
=
new
StringBuilder
();
for
(
Text
origin
:
origins
)
{
originHighlight
.
append
(
origin
.
string
());
}
info
.
setOrigin
(
originHighlight
.
toString
());
}
if
(
highlightFields
.
containsKey
(
"content"
))
{
Text
[]
contents
=
highlightFields
.
get
(
"content"
).
getFragments
();
StringBuilder
contentHighlight
=
new
StringBuilder
();
for
(
Text
content
:
contents
)
{
contentHighlight
.
append
(
content
.
string
());
}
info
.
setContent
(
contentHighlight
.
toString
());
}
list
.
add
(
info
);
}
pageData
=
new
Page
<>(
pageNo
,
pageSize
,
searchResponse
.
getHits
().
getTotalHits
().
value
);
pageData
.
setRecords
(
list
);
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
return
pageData
;
}
/**
/**
* term方式聚合分组
* term方式聚合分组
*
*
...
@@ -2064,7 +2164,23 @@ public class EsService {
...
@@ -2064,7 +2164,23 @@ public class EsService {
nestedBoolQueryBuilder
.
should
(
QueryBuilders
.
nestedQuery
(
"labels"
,
relationNameQuery
,
ScoreMode
.
None
));
nestedBoolQueryBuilder
.
should
(
QueryBuilders
.
nestedQuery
(
"labels"
,
relationNameQuery
,
ScoreMode
.
None
));
boolQuery
.
must
(
nestedBoolQueryBuilder
);
boolQuery
.
must
(
nestedBoolQueryBuilder
);
}
}
//资讯状态-研究中心(0-全部;1-模型推荐;2-精选;3-待定;4-移除),和checkStatus、deleteFlag、isFreeCheck互斥
Integer
status
=
searchCondition
.
getStatus
();
if
(
status
!=
null
)
{
if
(
status
==
1
)
{
boolQuery
.
must
(
QueryBuilders
.
termQuery
(
"checkStatus"
,
0
));
boolQuery
.
must
(
QueryBuilders
.
termQuery
(
"isFreeCheck"
,
1
));
boolQuery
.
mustNot
(
QueryBuilders
.
termQuery
(
"deleteFlag"
,
1
));
}
else
if
(
status
==
2
)
{
boolQuery
.
must
(
QueryBuilders
.
termQuery
(
"checkStatus"
,
1
));
boolQuery
.
mustNot
(
QueryBuilders
.
termQuery
(
"deleteFlag"
,
1
));
}
else
if
(
status
==
3
)
{
boolQuery
.
must
(
QueryBuilders
.
termQuery
(
"checkStatus"
,
3
));
boolQuery
.
mustNot
(
QueryBuilders
.
termQuery
(
"deleteFlag"
,
1
));
}
else
if
(
status
==
4
)
{
boolQuery
.
must
(
QueryBuilders
.
termQuery
(
"deleteFlag"
,
1
));
}
}
else
{
Integer
checkStatus
=
searchCondition
.
getCheckStatus
();
Integer
checkStatus
=
searchCondition
.
getCheckStatus
();
Integer
deleteFlag
=
searchCondition
.
getDeleteFlag
();
Integer
deleteFlag
=
searchCondition
.
getDeleteFlag
();
if
(
checkStatus
!=
null
)
{
if
(
checkStatus
!=
null
)
{
...
@@ -2093,6 +2209,7 @@ public class EsService {
...
@@ -2093,6 +2209,7 @@ public class EsService {
if
(
searchCondition
.
getIsFreeCheck
()
!=
null
)
{
if
(
searchCondition
.
getIsFreeCheck
()
!=
null
)
{
boolQuery
.
must
(
QueryBuilders
.
termQuery
(
"isFreeCheck"
,
1
));
boolQuery
.
must
(
QueryBuilders
.
termQuery
(
"isFreeCheck"
,
1
));
}
}
}
if
(
StringUtils
.
isNotBlank
(
searchCondition
.
getVideo
()))
{
if
(
StringUtils
.
isNotBlank
(
searchCondition
.
getVideo
()))
{
boolQuery
.
must
(
QueryBuilders
.
matchQuery
(
"type"
,
"video"
));
boolQuery
.
must
(
QueryBuilders
.
matchQuery
(
"type"
,
"video"
));
}
else
{
}
else
{
...
@@ -2115,6 +2232,19 @@ public class EsService {
...
@@ -2115,6 +2232,19 @@ public class EsService {
if
(
maxScore
!=
null
)
{
if
(
maxScore
!=
null
)
{
boolQuery
.
filter
(
QueryBuilders
.
rangeQuery
(
"score"
).
lte
(
maxScore
));
boolQuery
.
filter
(
QueryBuilders
.
rangeQuery
(
"score"
).
lte
(
maxScore
));
}
}
//正文长度筛选
Integer
minContentLength
=
searchCondition
.
getMinContentLength
();
if
(
minContentLength
!=
null
)
{
NestedQueryBuilder
nestedQueryBuilder
=
QueryBuilders
.
nestedQuery
(
"sortField"
,
QueryBuilders
.
rangeQuery
(
"sortField.fieldLong"
).
gte
(
minContentLength
),
ScoreMode
.
None
);
boolQuery
.
filter
(
nestedQueryBuilder
);
}
Integer
maxContentLength
=
searchCondition
.
getMaxContentLength
();
if
(
maxContentLength
!=
null
)
{
NestedQueryBuilder
nestedQueryBuilder
=
QueryBuilders
.
nestedQuery
(
"sortField"
,
QueryBuilders
.
rangeQuery
(
"sortField.fieldLong"
).
lte
(
maxContentLength
),
ScoreMode
.
None
);
boolQuery
.
filter
(
nestedQueryBuilder
);
}
//时间过滤筛选-前端传参
//时间过滤筛选-前端传参
if
(
StringUtils
.
isEmpty
(
searchCondition
.
getStartTime
())
&&
StringUtils
.
isEmpty
(
searchCondition
.
getEndTime
())){
if
(
StringUtils
.
isEmpty
(
searchCondition
.
getStartTime
())
&&
StringUtils
.
isEmpty
(
searchCondition
.
getEndTime
())){
BoolQueryBuilder
publishDateQueryBuilder
=
QueryBuilders
.
boolQuery
();
BoolQueryBuilder
publishDateQueryBuilder
=
QueryBuilders
.
boolQuery
();
...
@@ -2146,7 +2276,7 @@ public class EsService {
...
@@ -2146,7 +2276,7 @@ public class EsService {
for
(
String
items
:
split
)
{
for
(
String
items
:
split
)
{
List
<
String
>
ids
=
Arrays
.
asList
(
items
.
split
(
","
));
List
<
String
>
ids
=
Arrays
.
asList
(
items
.
split
(
","
));
NestedQueryBuilder
nestedQueryBuilder
=
QueryBuilders
NestedQueryBuilder
nestedQueryBuilder
=
QueryBuilders
.
nestedQuery
(
"labels"
,
QueryBuilders
.
termsQuery
(
"labels.relationId"
,
ids
),
ScoreMode
.
None
);
.
nestedQuery
(
"labels"
,
QueryBuilders
.
termsQuery
(
"labels.relationId"
,
ids
),
ScoreMode
.
None
);
boolQuery
.
must
(
nestedQueryBuilder
);
boolQuery
.
must
(
nestedQueryBuilder
);
}
}
}
}
...
@@ -2192,6 +2322,11 @@ public class EsService {
...
@@ -2192,6 +2322,11 @@ public class EsService {
if
(
StringUtils
.
isNotBlank
(
sourceId
))
{
if
(
StringUtils
.
isNotBlank
(
sourceId
))
{
boolQuery
.
must
(
QueryBuilders
.
termQuery
(
"sid.keyword"
,
sourceId
));
boolQuery
.
must
(
QueryBuilders
.
termQuery
(
"sid.keyword"
,
sourceId
));
}
}
//待删除的关键词列表
if
(
CollectionUtils
.
isNotEmpty
(
searchCondition
.
getKeywordList
()))
{
TermsQueryBuilder
sortFieldQuery
=
QueryBuilders
.
termsQuery
(
"sortField.fieldType"
,
searchCondition
.
getKeywordList
());
boolQuery
.
must
(
QueryBuilders
.
nestedQuery
(
"sortField"
,
sortFieldQuery
,
ScoreMode
.
None
));
}
return
boolQuery
;
return
boolQuery
;
}
}
...
@@ -2608,6 +2743,7 @@ public class EsService {
...
@@ -2608,6 +2743,7 @@ public class EsService {
/**
/**
* 根据id查询标签列表
* 根据id查询标签列表
*
* @param index
* @param index
* @param id
* @param id
* @return
* @return
...
@@ -2615,7 +2751,7 @@ public class EsService {
...
@@ -2615,7 +2751,7 @@ public class EsService {
public
List
<
Label
>
getLabelsById
(
String
index
,
String
id
)
{
public
List
<
Label
>
getLabelsById
(
String
index
,
String
id
)
{
try
{
try
{
GetRequest
request
=
new
GetRequest
(
index
,
id
);
GetRequest
request
=
new
GetRequest
(
index
,
id
);
request
.
fetchSourceContext
(
new
FetchSourceContext
(
true
,
new
String
[]{
"labels"
,
"title"
},
null
));
request
.
fetchSourceContext
(
new
FetchSourceContext
(
true
,
new
String
[]{
"labels"
,
"title"
},
null
));
request
.
realtime
(
true
);
request
.
realtime
(
true
);
GetResponse
documentFields
=
client
.
get
(
request
,
RequestOptions
.
DEFAULT
);
GetResponse
documentFields
=
client
.
get
(
request
,
RequestOptions
.
DEFAULT
);
String
labelsStr
=
JSONArray
.
toJSONString
(
documentFields
.
getSourceAsMap
().
get
(
"labels"
));
String
labelsStr
=
JSONArray
.
toJSONString
(
documentFields
.
getSourceAsMap
().
get
(
"labels"
));
...
@@ -2628,6 +2764,7 @@ public class EsService {
...
@@ -2628,6 +2764,7 @@ public class EsService {
/**
/**
* 根据id更新标签列表
* 根据id更新标签列表
*
* @param index
* @param index
* @param id
* @param id
* @param labels
* @param labels
...
@@ -2635,6 +2772,6 @@ public class EsService {
...
@@ -2635,6 +2772,6 @@ public class EsService {
public
void
updateLabelsById
(
String
index
,
String
id
,
List
<
Label
>
labels
)
{
public
void
updateLabelsById
(
String
index
,
String
id
,
List
<
Label
>
labels
)
{
JSONObject
jsonObject
=
new
JSONObject
();
JSONObject
jsonObject
=
new
JSONObject
();
jsonObject
.
put
(
"labels"
,
labels
);
jsonObject
.
put
(
"labels"
,
labels
);
esOpUtil
.
docUpdateById
(
index
,
id
,
jsonObject
.
toString
());
esOpUtil
.
docUpdateById
(
index
,
id
,
jsonObject
.
toString
());
}
}
}
}
src/main/java/com/zzsn/event/feign/api/RemoteProjectService.java
0 → 100644
浏览文件 @
c311ac86
package
com
.
zzsn
.
event
.
feign
.
api
;
import
com.zzsn.event.vo.InfoDataSearchCondition
;
import
org.springframework.cloud.openfeign.FeignClient
;
import
org.springframework.http.HttpHeaders
;
import
org.springframework.web.bind.annotation.*
;
//feign调用
@FeignClient
(
value
=
"service-project"
)
public
interface
RemoteProjectService
{
/**
* 清除专题缓存
*
* @param
* @author lkg
* @date 2025/2/7
*/
@PostMapping
(
"/manage/subjectModel/doTaskByFlow"
)
String
doTaskByFlow
(
@RequestBody
InfoDataSearchCondition
task
);
}
src/main/java/com/zzsn/event/mapper/ClbSubjectHitwordsMapMapper.java
浏览文件 @
c311ac86
...
@@ -3,6 +3,9 @@ package com.zzsn.event.mapper;
...
@@ -3,6 +3,9 @@ package com.zzsn.event.mapper;
import
com.baomidou.mybatisplus.core.mapper.BaseMapper
;
import
com.baomidou.mybatisplus.core.mapper.BaseMapper
;
import
com.zzsn.event.entity.ClbSubjectHitwordsMap
;
import
com.zzsn.event.entity.ClbSubjectHitwordsMap
;
import
org.apache.ibatis.annotations.Mapper
;
import
org.apache.ibatis.annotations.Mapper
;
import
org.apache.ibatis.annotations.Param
;
import
java.util.List
;
/**
/**
* <p>
* <p>
...
@@ -15,4 +18,5 @@ import org.apache.ibatis.annotations.Mapper;
...
@@ -15,4 +18,5 @@ import org.apache.ibatis.annotations.Mapper;
@Mapper
@Mapper
public
interface
ClbSubjectHitwordsMapMapper
extends
BaseMapper
<
ClbSubjectHitwordsMap
>
{
public
interface
ClbSubjectHitwordsMapMapper
extends
BaseMapper
<
ClbSubjectHitwordsMap
>
{
List
<
String
>
detail
(
@Param
(
"flowId"
)
String
flowId
,
@Param
(
"subjectId"
)
String
subjectId
);
}
}
src/main/java/com/zzsn/event/mapper/xml/ClbSubjectHitwordsMapMapper.xml
浏览文件 @
c311ac86
...
@@ -15,4 +15,9 @@
...
@@ -15,4 +15,9 @@
<result
column=
"del_flag"
property=
"delFlag"
/>
<result
column=
"del_flag"
property=
"delFlag"
/>
</resultMap>
</resultMap>
<select
id=
"detail"
resultType=
"String"
>
SELECT id FROM clb_subject_model_detail_task WHERE subject_id = #{subjectId} AND flow_id = #{flowId} AND task_status in (0,1);
</select>
</mapper>
</mapper>
src/main/java/com/zzsn/event/mapper/xml/SubjectMapper.xml
浏览文件 @
c311ac86
...
@@ -45,7 +45,7 @@
...
@@ -45,7 +45,7 @@
</select>
</select>
<select
id=
"pageList_new"
resultType=
"com.zzsn.event.vo.SubjectPageVO"
>
<select
id=
"pageList_new"
resultType=
"com.zzsn.event.vo.SubjectPageVO"
>
SELECT d.id,d.subject_name,d.status,
SELECT d.id,d.subject_name,d.status,
d.data_scope,
s.total_num,s.un_check_num,s.bind_source_num,s.bind_keyword_num,s.latest_data_date,
s.total_num,s.un_check_num,s.bind_source_num,s.bind_keyword_num,s.latest_data_date,
c.type_name as subjectTypeName, g.project_name as projectName
c.type_name as subjectTypeName, g.project_name as projectName
from subject d
from subject d
...
...
src/main/java/com/zzsn/event/service/ClbSubjectHitwordsMapService.java
浏览文件 @
c311ac86
...
@@ -2,6 +2,7 @@ package com.zzsn.event.service;
...
@@ -2,6 +2,7 @@ package com.zzsn.event.service;
import
com.baomidou.mybatisplus.extension.service.IService
;
import
com.baomidou.mybatisplus.extension.service.IService
;
import
com.zzsn.event.entity.ClbSubjectHitwordsMap
;
import
com.zzsn.event.entity.ClbSubjectHitwordsMap
;
import
com.zzsn.event.vo.InfoDataSearchCondition
;
import
java.util.List
;
import
java.util.List
;
...
@@ -16,4 +17,6 @@ import java.util.List;
...
@@ -16,4 +17,6 @@ import java.util.List;
public
interface
ClbSubjectHitwordsMapService
extends
IService
<
ClbSubjectHitwordsMap
>
{
public
interface
ClbSubjectHitwordsMapService
extends
IService
<
ClbSubjectHitwordsMap
>
{
void
delEs
(
List
<
ClbSubjectHitwordsMap
>
clbSubjectHitwordsMaps
);
void
delEs
(
List
<
ClbSubjectHitwordsMap
>
clbSubjectHitwordsMaps
);
void
stopTask
(
InfoDataSearchCondition
searchCondition
);
}
}
src/main/java/com/zzsn/event/service/InformationService.java
浏览文件 @
c311ac86
...
@@ -261,4 +261,85 @@ public interface InformationService {
...
@@ -261,4 +261,85 @@ public interface InformationService {
List
<
SpecialInformation
>
informationAllList
(
InfoDataSearchCondition
searchCondition
);
List
<
SpecialInformation
>
informationAllList
(
InfoDataSearchCondition
searchCondition
);
void
removeLabelById
(
String
index
,
String
id
,
String
relationId
);
void
removeLabelById
(
String
index
,
String
id
,
String
relationId
);
/**
* 删除关键词标签
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/22
*/
void
removeWordLabel
(
InfoDataSearchCondition
searchCondition
);
/**
* 批量添加到精选
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void
addToFavorites
(
InfoDataSearchCondition
searchCondition
);
/**
* 批量添加到精选
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void
addToPend
(
InfoDataSearchCondition
searchCondition
);
/**
* 批量添加到精选
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void
addToRemove
(
InfoDataSearchCondition
searchCondition
);
/**
* 初始化数据(数据状态全部重置为待审核、未删除)
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void
initialData
(
InfoDataSearchCondition
searchCondition
);
/**
* 查找替换信息列表
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/24
*/
IPage
<
SpecialInformation
>
searchForReplaceList
(
InfoDataSearchCondition
searchCondition
);
/**
* 字符串替换
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void
charReplace
(
InfoDataSearchCondition
searchCondition
);
/**
* 打标签
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void
markTag
(
InfoDataSearchCondition
searchCondition
);
/**
* 打标签
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void
removeTag
(
InfoDataSearchCondition
searchCondition
);
}
}
src/main/java/com/zzsn/event/service/SubjectService.java
浏览文件 @
c311ac86
...
@@ -202,4 +202,7 @@ public interface SubjectService extends IService<Subject> {
...
@@ -202,4 +202,7 @@ public interface SubjectService extends IService<Subject> {
void
deleteBindNew
(
SubjectPage
subjectPage
);
void
deleteBindNew
(
SubjectPage
subjectPage
);
String
getMinCreateTime
(
List
<
String
>
subjectIdList
);
String
getMinCreateTime
(
List
<
String
>
subjectIdList
);
Subject
subjectUpdateScope
(
String
subjectId
,
String
type
,
String
scop
);
}
}
src/main/java/com/zzsn/event/service/impl/ClbSubjectHitwordsMapServiceImpl.java
浏览文件 @
c311ac86
...
@@ -2,14 +2,23 @@ package com.zzsn.event.service.impl;
...
@@ -2,14 +2,23 @@ package com.zzsn.event.service.impl;
import
cn.hutool.core.collection.CollectionUtil
;
import
cn.hutool.core.collection.CollectionUtil
;
import
cn.hutool.core.util.StrUtil
;
import
com.baomidou.mybatisplus.extension.service.impl.ServiceImpl
;
import
com.baomidou.mybatisplus.extension.service.impl.ServiceImpl
;
import
com.zzsn.event.constant.Constants
;
import
com.zzsn.event.entity.ClbSubjectHitwordsMap
;
import
com.zzsn.event.entity.ClbSubjectHitwordsMap
;
import
com.zzsn.event.mapper.ClbSubjectHitwordsMapMapper
;
import
com.zzsn.event.mapper.ClbSubjectHitwordsMapMapper
;
import
com.zzsn.event.service.ClbSubjectHitwordsMapService
;
import
com.zzsn.event.service.ClbSubjectHitwordsMapService
;
import
com.zzsn.event.service.InformationService
;
import
com.zzsn.event.util.RedisUtil
;
import
com.zzsn.event.vo.InfoDataSearchCondition
;
import
lombok.extern.slf4j.Slf4j
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.stereotype.Service
;
import
org.springframework.stereotype.Service
;
import
org.springframework.util.CollectionUtils
;
import
org.springframework.util.CollectionUtils
;
import
java.util.ArrayList
;
import
java.util.List
;
import
java.util.List
;
import
java.util.concurrent.CompletableFuture
;
/**
/**
* <p>
* <p>
...
@@ -20,14 +29,52 @@ import java.util.List;
...
@@ -20,14 +29,52 @@ import java.util.List;
* @since 2025-04-23
* @since 2025-04-23
*/
*/
@Service
@Service
@Slf4j
public
class
ClbSubjectHitwordsMapServiceImpl
extends
ServiceImpl
<
ClbSubjectHitwordsMapMapper
,
ClbSubjectHitwordsMap
>
implements
ClbSubjectHitwordsMapService
{
public
class
ClbSubjectHitwordsMapServiceImpl
extends
ServiceImpl
<
ClbSubjectHitwordsMapMapper
,
ClbSubjectHitwordsMap
>
implements
ClbSubjectHitwordsMapService
{
@Autowired
private
InformationService
informationService
;
@Autowired
RedisUtil
redisUtil
;
@Override
@Override
public
void
delEs
(
List
<
ClbSubjectHitwordsMap
>
clbSubjectHitwordsMaps
)
{
public
void
delEs
(
List
<
ClbSubjectHitwordsMap
>
clbSubjectHitwordsMaps
)
{
//TODO
if
(
CollectionUtils
.
isEmpty
(
clbSubjectHitwordsMaps
))
{
return
;
}
String
subjectId
=
clbSubjectHitwordsMaps
.
get
(
0
).
getSubjectId
();
InfoDataSearchCondition
searchCondition
=
new
InfoDataSearchCondition
();
searchCondition
.
setSubjectId
(
subjectId
);
searchCondition
.
setStatus
(
0
);
searchCondition
.
setKeywordList
(
new
ArrayList
<>());
if
(
CollectionUtil
.
isNotEmpty
(
clbSubjectHitwordsMaps
))
{
if
(
CollectionUtil
.
isNotEmpty
(
clbSubjectHitwordsMaps
))
{
for
(
ClbSubjectHitwordsMap
clbSubjectHitwordsMap
:
clbSubjectHitwordsMaps
)
{
if
(
StrUtil
.
isNotBlank
(
clbSubjectHitwordsMap
.
getWordName
()))
{
searchCondition
.
getKeywordList
().
add
(
clbSubjectHitwordsMap
.
getWordName
());
}
}
}
}
if
(
CollectionUtil
.
isEmpty
(
searchCondition
.
getKeywordList
()))
{
log
.
info
(
"命中词为空,不删除es"
);
return
;
}
log
.
info
(
"delEs:{}"
,
searchCondition
);
CompletableFuture
.
runAsync
(()->
informationService
.
removeWordLabel
(
searchCondition
));
}
@Override
public
void
stopTask
(
InfoDataSearchCondition
searchCondition
)
{
String
flowId
=
searchCondition
.
getFlowId
();
String
subjectId
=
searchCondition
.
getSubjectId
();
List
<
String
>
taskIds
=
baseMapper
.
detail
(
flowId
,
subjectId
);
if
(
CollectionUtil
.
isNotEmpty
(
taskIds
))
{
taskIds
.
forEach
(
taskId
->
{
redisUtil
.
hset
(
Constants
.
SUBJECT_MODEL_DETAIL_BREAK
+
subjectId
,
taskId
,
"1"
);
});
}
}
}
}
}
src/main/java/com/zzsn/event/service/impl/InformationServiceImpl.java
浏览文件 @
c311ac86
...
@@ -30,6 +30,10 @@ import lombok.extern.slf4j.Slf4j;
...
@@ -30,6 +30,10 @@ import lombok.extern.slf4j.Slf4j;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.collections4.CollectionUtils
;
import
org.apache.commons.lang3.ObjectUtils
;
import
org.apache.commons.lang3.ObjectUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.commons.lang3.StringUtils
;
import
org.apache.lucene.search.join.ScoreMode
;
import
org.elasticsearch.index.query.BoolQueryBuilder
;
import
org.elasticsearch.index.query.QueryBuilders
;
import
org.elasticsearch.index.query.TermsQueryBuilder
;
import
org.jsoup.Jsoup
;
import
org.jsoup.Jsoup
;
import
org.springframework.beans.BeanUtils
;
import
org.springframework.beans.BeanUtils
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Autowired
;
...
@@ -430,6 +434,7 @@ public class InformationServiceImpl implements InformationService {
...
@@ -430,6 +434,7 @@ public class InformationServiceImpl implements InformationService {
labels
.
add
(
dataSet
);
labels
.
add
(
dataSet
);
}
}
}
else
{
}
else
{
labels
=
new
ArrayList
<>();
labels
.
add
(
dataSet
);
labels
.
add
(
dataSet
);
}
}
information
.
setLabels
(
labels
);
information
.
setLabels
(
labels
);
...
@@ -659,9 +664,9 @@ public class InformationServiceImpl implements InformationService {
...
@@ -659,9 +664,9 @@ public class InformationServiceImpl implements InformationService {
List
<
DictVO
>
boundList
=
dataBindLabelFrom
.
getBindList
();
List
<
DictVO
>
boundList
=
dataBindLabelFrom
.
getBindList
();
List
<
String
>
labelMarks
=
new
ArrayList
<>();
List
<
String
>
labelMarks
=
new
ArrayList
<>();
for
(
DictVO
dictVO
:
boundList
)
{
for
(
DictVO
dictVO
:
boundList
)
{
if
(
StringUtils
.
isNotEmpty
(
dictVO
.
getCode
()))
{
if
(
StringUtils
.
isNotEmpty
(
dictVO
.
getCode
()))
{
labelMarks
.
add
(
dictVO
.
getCode
());
labelMarks
.
add
(
dictVO
.
getCode
());
}
else
{
}
else
{
labelMarks
.
add
(
dictVO
.
getLabelMark
());
labelMarks
.
add
(
dictVO
.
getLabelMark
());
}
}
}
}
...
@@ -1139,12 +1144,211 @@ public class InformationServiceImpl implements InformationService {
...
@@ -1139,12 +1144,211 @@ public class InformationServiceImpl implements InformationService {
@Override
@Override
public
void
removeLabelById
(
String
index
,
String
id
,
String
relationId
)
{
public
void
removeLabelById
(
String
index
,
String
id
,
String
relationId
)
{
List
<
Label
>
labels
=
esService
.
getLabelsById
(
index
,
id
);
List
<
Label
>
labels
=
esService
.
getLabelsById
(
index
,
id
);
if
(
labels
==
null
)
{
if
(
labels
==
null
)
{
return
;
return
;
}
}
labels
.
removeIf
(
e
->
e
.
getRelationId
().
equals
(
relationId
));
labels
.
removeIf
(
e
->
e
.
getRelationId
().
equals
(
relationId
));
esService
.
updateLabelsById
(
index
,
id
,
labels
);
esService
.
updateLabelsById
(
index
,
id
,
labels
);
}
@Override
public
void
removeWordLabel
(
InfoDataSearchCondition
searchCondition
)
{
List
<
String
>
subjectIdList
=
new
ArrayList
<>();
subjectIdList
.
add
(
searchCondition
.
getSubjectId
());
String
minCreateTime
=
subjectService
.
getMinCreateTime
(
subjectIdList
);
String
[]
indexArr
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minCreateTime
);
BoolQueryBuilder
buildQuery
=
esService
.
buildQuery
(
searchCondition
,
subjectIdList
);
buildQuery
.
must
(
QueryBuilders
.
nestedQuery
(
"sortField"
,
QueryBuilders
.
termsQuery
(
"sortField.fieldType"
,
searchCondition
.
getKeywordList
()),
ScoreMode
.
None
));
List
<
String
>
keywordList
=
searchCondition
.
getKeywordList
();
for
(
String
word
:
keywordList
)
{
Map
<
String
,
Object
>
deleteParams
=
new
HashMap
<>();
deleteParams
.
put
(
"fieldType"
,
word
);
long
start
=
System
.
currentTimeMillis
();
esOpUtil
.
batchNestedDeleteScript
(
indexArr
,
buildQuery
,
deleteParams
,
"sortField"
);
log
.
info
(
"删除关键词-{},耗时:{}"
,
word
,
System
.
currentTimeMillis
()-
start
);
try
{
Thread
.
sleep
(
1000
);
}
catch
(
InterruptedException
e
)
{
e
.
printStackTrace
();
}
}
}
@Override
public
void
addToFavorites
(
InfoDataSearchCondition
searchCondition
)
{
List
<
String
>
subjectIdList
=
new
ArrayList
<>();
subjectIdList
.
add
(
searchCondition
.
getSubjectId
());
String
minCreateTime
=
subjectService
.
getMinCreateTime
(
subjectIdList
);
String
[]
indexArr
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minCreateTime
);
List
<
String
>
ids
=
searchCondition
.
getIds
();
BoolQueryBuilder
buildQuery
;
if
(
CollectionUtils
.
isEmpty
(
ids
))
{
buildQuery
=
esService
.
buildQuery
(
searchCondition
,
subjectIdList
);
}
else
{
buildQuery
=
QueryBuilders
.
boolQuery
().
must
(
QueryBuilders
.
termsQuery
(
"id"
,
ids
));
}
Map
<
String
,
Object
>
modifyParams
=
new
HashMap
<>();
modifyParams
.
put
(
"checkStatus"
,
1
);
modifyParams
.
put
(
"deleteFlag"
,
0
);
long
start
=
System
.
currentTimeMillis
();
esOpUtil
.
batchUpdateScript
(
indexArr
,
buildQuery
,
modifyParams
);
log
.
info
(
"添加到精选,耗时:{}"
,
System
.
currentTimeMillis
()-
start
);
}
@Override
public
void
addToPend
(
InfoDataSearchCondition
searchCondition
)
{
List
<
String
>
subjectIdList
=
new
ArrayList
<>();
subjectIdList
.
add
(
searchCondition
.
getSubjectId
());
String
minCreateTime
=
subjectService
.
getMinCreateTime
(
subjectIdList
);
String
[]
indexArr
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minCreateTime
);
List
<
String
>
ids
=
searchCondition
.
getIds
();
BoolQueryBuilder
buildQuery
;
if
(
CollectionUtils
.
isEmpty
(
ids
))
{
buildQuery
=
esService
.
buildQuery
(
searchCondition
,
subjectIdList
);
}
else
{
buildQuery
=
QueryBuilders
.
boolQuery
().
must
(
QueryBuilders
.
termsQuery
(
"id"
,
ids
));
}
Map
<
String
,
Object
>
modifyParams
=
new
HashMap
<>();
modifyParams
.
put
(
"checkStatus"
,
3
);
modifyParams
.
put
(
"deleteFlag"
,
0
);
long
start
=
System
.
currentTimeMillis
();
esOpUtil
.
batchUpdateScript
(
indexArr
,
buildQuery
,
modifyParams
);
log
.
info
(
"添加到待定,耗时:{}"
,
System
.
currentTimeMillis
()-
start
);
}
@Override
public
void
addToRemove
(
InfoDataSearchCondition
searchCondition
)
{
List
<
String
>
subjectIdList
=
new
ArrayList
<>();
subjectIdList
.
add
(
searchCondition
.
getSubjectId
());
String
minCreateTime
=
subjectService
.
getMinCreateTime
(
subjectIdList
);
String
[]
indexArr
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minCreateTime
);
List
<
String
>
ids
=
searchCondition
.
getIds
();
BoolQueryBuilder
buildQuery
;
if
(
CollectionUtils
.
isEmpty
(
ids
))
{
buildQuery
=
esService
.
buildQuery
(
searchCondition
,
subjectIdList
);
}
else
{
buildQuery
=
QueryBuilders
.
boolQuery
().
must
(
QueryBuilders
.
termsQuery
(
"id"
,
ids
));
}
Map
<
String
,
Object
>
modifyParams
=
new
HashMap
<>();
modifyParams
.
put
(
"deleteFlag"
,
1
);
long
start
=
System
.
currentTimeMillis
();
esOpUtil
.
batchUpdateScript
(
indexArr
,
buildQuery
,
modifyParams
);
log
.
info
(
"添加到删除,耗时:{}"
,
System
.
currentTimeMillis
()-
start
);
}
@Override
public
void
initialData
(
InfoDataSearchCondition
searchCondition
)
{
List
<
String
>
subjectIdList
=
new
ArrayList
<>();
subjectIdList
.
add
(
searchCondition
.
getSubjectId
());
String
minCreateTime
=
subjectService
.
getMinCreateTime
(
subjectIdList
);
String
[]
indexArr
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minCreateTime
);
BoolQueryBuilder
buildQuery
=
esService
.
buildQuery
(
searchCondition
,
subjectIdList
);
Map
<
String
,
Object
>
modifyParams
=
new
HashMap
<>();
modifyParams
.
put
(
"checkStatus"
,
0
);
modifyParams
.
put
(
"deleteFlag"
,
0
);
long
start
=
System
.
currentTimeMillis
();
esOpUtil
.
batchUpdateScript
(
indexArr
,
buildQuery
,
modifyParams
);
log
.
info
(
"初始化数据,耗时:{}"
,
System
.
currentTimeMillis
()-
start
);
}
@Override
public
IPage
<
SpecialInformation
>
searchForReplaceList
(
InfoDataSearchCondition
searchCondition
)
{
return
esService
.
searchForReplaceList
(
searchCondition
);
}
@Override
public
void
charReplace
(
InfoDataSearchCondition
searchCondition
)
{
SearchWordVO
searchWordVO
=
searchCondition
.
getSearchWordList
().
get
(
0
);
Integer
searchScope
=
searchWordVO
.
getSearchScope
();
String
oldWord
=
searchWordVO
.
getSearchInfo
();
String
newWord
=
searchCondition
.
getReplaceWord
();
if
(
StringUtils
.
isEmpty
(
newWord
))
{
newWord
=
""
;
}
List
<
String
>
modifyFields
=
new
ArrayList
<>();
if
(
searchScope
==
1
)
{
modifyFields
.
add
(
"title"
);
}
else
if
(
searchScope
==
2
)
{
modifyFields
.
add
(
"content"
);
modifyFields
.
add
(
"contentWithTag"
);
}
else
if
(
searchScope
==
3
)
{
modifyFields
.
add
(
"title"
);
modifyFields
.
add
(
"content"
);
modifyFields
.
add
(
"contentWithTag"
);
}
else
if
(
searchScope
==
4
)
{
modifyFields
.
add
(
"origin"
);
}
List
<
String
>
subjectIdList
=
new
ArrayList
<>();
subjectIdList
.
add
(
searchCondition
.
getSubjectId
());
String
minCreateTime
=
subjectService
.
getMinCreateTime
(
subjectIdList
);
String
[]
indexArr
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minCreateTime
);
BoolQueryBuilder
buildQuery
=
esService
.
buildQuery
(
searchCondition
,
subjectIdList
);
long
start
=
System
.
currentTimeMillis
();
esOpUtil
.
batchReplaceScript
(
indexArr
,
buildQuery
,
modifyFields
,
oldWord
,
newWord
);
log
.
info
(
"字符串替换-{},耗时:{}"
,
oldWord
,
System
.
currentTimeMillis
()-
start
);
}
@Override
public
void
markTag
(
InfoDataSearchCondition
searchCondition
)
{
List
<
String
>
subjectIdList
=
new
ArrayList
<>();
subjectIdList
.
add
(
searchCondition
.
getSubjectId
());
String
minCreateTime
=
subjectService
.
getMinCreateTime
(
subjectIdList
);
String
[]
indexArr
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minCreateTime
);
List
<
String
>
ids
=
searchCondition
.
getIds
();
BoolQueryBuilder
buildQuery
;
if
(
CollectionUtils
.
isEmpty
(
ids
))
{
buildQuery
=
esService
.
buildQuery
(
searchCondition
,
subjectIdList
);
}
else
{
buildQuery
=
QueryBuilders
.
boolQuery
().
must
(
QueryBuilders
.
termsQuery
(
"id"
,
ids
));
}
Map
<
String
,
Object
>
addParams
=
new
HashMap
<>();
for
(
Label
markTag
:
searchCondition
.
getMarkTags
())
{
Map
<
String
,
String
>
addTag
=
ObjectUtil
.
objectToMap
(
markTag
);
addParams
.
put
(
"labels"
,
addTag
);
long
start
=
System
.
currentTimeMillis
();
esOpUtil
.
batchNestedAddScript
(
indexArr
,
buildQuery
,
addParams
,
"labels"
,
"relationId"
);
log
.
info
(
"打标-{},耗时:{}"
,
markTag
.
getRelationName
(),
System
.
currentTimeMillis
()-
start
);
try
{
Thread
.
sleep
(
1000
);
}
catch
(
InterruptedException
e
)
{
e
.
printStackTrace
();
}
}
}
@Override
public
void
removeTag
(
InfoDataSearchCondition
searchCondition
)
{
List
<
String
>
subjectIdList
=
new
ArrayList
<>();
subjectIdList
.
add
(
searchCondition
.
getSubjectId
());
String
minCreateTime
=
subjectService
.
getMinCreateTime
(
subjectIdList
);
String
[]
indexArr
=
EsIndexUtil
.
getIndexIntervalYearStr
(
Constants
.
SUBJECT_INDEX
,
minCreateTime
);
List
<
String
>
ids
=
searchCondition
.
getIds
();
BoolQueryBuilder
buildQuery
;
if
(
CollectionUtils
.
isEmpty
(
ids
))
{
buildQuery
=
esService
.
buildQuery
(
searchCondition
,
subjectIdList
);
}
else
{
buildQuery
=
QueryBuilders
.
boolQuery
().
must
(
QueryBuilders
.
termsQuery
(
"id"
,
ids
));
}
List
<
Label
>
markTags
=
searchCondition
.
getMarkTags
();
List
<
String
>
tagIds
=
markTags
.
stream
().
map
(
Label:
:
getRelationId
).
collect
(
Collectors
.
toList
());
buildQuery
.
must
(
QueryBuilders
.
nestedQuery
(
"labels"
,
QueryBuilders
.
termsQuery
(
"labels.relationId"
,
tagIds
),
ScoreMode
.
None
));
for
(
Label
markTag
:
markTags
)
{
Map
<
String
,
Object
>
deleteParams
=
new
HashMap
<>();
deleteParams
.
put
(
"relationId"
,
markTag
.
getRelationId
());
long
start
=
System
.
currentTimeMillis
();
esOpUtil
.
batchNestedDeleteScript
(
indexArr
,
buildQuery
,
deleteParams
,
"labels"
);
log
.
info
(
"删除标签-{},耗时:{}"
,
markTag
.
getRelationName
(),
System
.
currentTimeMillis
()-
start
);
try
{
Thread
.
sleep
(
1000
);
}
catch
(
InterruptedException
e
)
{
e
.
printStackTrace
();
}
}
}
}
private
void
formatLabel
(
List
<
LabelModelVo
>
labelModelVos
,
DisplayInfo
info
)
{
private
void
formatLabel
(
List
<
LabelModelVo
>
labelModelVos
,
DisplayInfo
info
)
{
...
...
src/main/java/com/zzsn/event/service/impl/SubjectServiceImpl.java
浏览文件 @
c311ac86
...
@@ -184,6 +184,16 @@ public class SubjectServiceImpl extends ServiceImpl<SubjectMapper, Subject> impl
...
@@ -184,6 +184,16 @@ public class SubjectServiceImpl extends ServiceImpl<SubjectMapper, Subject> impl
Map
<
String
,
List
<
ClbModelArrange
>>
collect
=
clbModelArranges
.
stream
().
collect
(
Collectors
.
groupingBy
(
ClbModelArrange:
:
getSubjectId
));
Map
<
String
,
List
<
ClbModelArrange
>>
collect
=
clbModelArranges
.
stream
().
collect
(
Collectors
.
groupingBy
(
ClbModelArrange:
:
getSubjectId
));
records
.
forEach
(
e
->
e
.
setClbModelArranges
(
collect
.
get
(
e
.
getId
())));
records
.
forEach
(
e
->
e
.
setClbModelArranges
(
collect
.
get
(
e
.
getId
())));
}
}
Optional
<
SubjectPageVO
>
any
=
records
.
stream
().
filter
(
f
->
StrUtil
.
isNotBlank
(
f
.
getDataScope
())).
findAny
();
if
(
any
.
isPresent
())
{
int
count
=
infoSourceService
.
count
();
records
.
forEach
(
e
->
{
if
(
StrUtil
.
isNotBlank
(
e
.
getDataScope
())
&&
e
.
getDataScope
().
contains
(
"1"
))
{
e
.
setBindSourceNum
(
count
);
}
});
}
}
}
return
pageList
;
return
pageList
;
}
}
...
@@ -905,6 +915,47 @@ public class SubjectServiceImpl extends ServiceImpl<SubjectMapper, Subject> impl
...
@@ -905,6 +915,47 @@ public class SubjectServiceImpl extends ServiceImpl<SubjectMapper, Subject> impl
return
baseMapper
.
getMinCreateTime
(
subjectIdList
);
return
baseMapper
.
getMinCreateTime
(
subjectIdList
);
}
}
@Override
public
Subject
subjectUpdateScope
(
String
subjectId
,
String
type
,
String
scop
)
{
Subject
byId
=
super
.
getById
(
subjectId
);
if
(
ObjectUtil
.
isNotNull
(
byId
)){
String
dataScope
=
byId
.
getDataScope
();
if
(
StrUtil
.
isNotBlank
(
dataScope
)){
List
<
String
>
scopeList
=
CollectionUtil
.
newArrayList
(
dataScope
.
split
(
","
));
if
(
StrUtil
.
equals
(
type
,
"1"
)
&&
"1"
.
equals
(
scop
)){
scopeList
.
add
(
"1"
);
}
else
if
(
StrUtil
.
equals
(
type
,
"1"
)
&&
"0"
.
equals
(
scop
)){
scopeList
.
remove
(
"1"
);
}
if
(
StrUtil
.
equals
(
type
,
"2"
)
&&
"1"
.
equals
(
scop
)){
scopeList
.
add
(
"2"
);
}
else
if
(
StrUtil
.
equals
(
type
,
"2"
)
&&
"0"
.
equals
(
scop
)){
scopeList
.
remove
(
"2"
);
}
if
(
StrUtil
.
equals
(
type
,
"3"
)
&&
"1"
.
equals
(
scop
)){
scopeList
.
add
(
"3"
);
}
else
if
(
StrUtil
.
equals
(
type
,
"3"
)
&&
"0"
.
equals
(
scop
)){
scopeList
.
remove
(
"3"
);
}
if
(
CollectionUtil
.
isNotEmpty
(
scopeList
))
{
scopeList
=
scopeList
.
stream
().
distinct
().
collect
(
Collectors
.
toList
());
String
join
=
StrUtil
.
join
(
","
,
scopeList
);
byId
.
setDataScope
(
join
);
}
else
{
byId
.
setDataScope
(
null
);
}
this
.
updateById
(
byId
);
}
else
{
if
(
"1"
.
equals
(
scop
))
{
byId
.
setDataScope
(
scop
);
}
this
.
updateById
(
byId
);
}
}
return
byId
;
}
private
void
unBindInfoSourceGroup
(
SubjectPage
subjectPage
)
{
private
void
unBindInfoSourceGroup
(
SubjectPage
subjectPage
)
{
String
sourceBindType
=
subjectPage
.
getSourceBindType
();
String
sourceBindType
=
subjectPage
.
getSourceBindType
();
...
...
src/main/java/com/zzsn/event/util/EsOpUtil.java
浏览文件 @
c311ac86
...
@@ -41,6 +41,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
...
@@ -41,6 +41,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import
org.elasticsearch.common.xcontent.XContentType
;
import
org.elasticsearch.common.xcontent.XContentType
;
import
org.elasticsearch.index.query.*
;
import
org.elasticsearch.index.query.*
;
import
org.elasticsearch.index.reindex.BulkByScrollResponse
;
import
org.elasticsearch.index.reindex.BulkByScrollResponse
;
import
org.elasticsearch.index.reindex.BulkByScrollTask
;
import
org.elasticsearch.index.reindex.DeleteByQueryRequest
;
import
org.elasticsearch.index.reindex.DeleteByQueryRequest
;
import
org.elasticsearch.index.reindex.UpdateByQueryRequest
;
import
org.elasticsearch.index.reindex.UpdateByQueryRequest
;
import
org.elasticsearch.rest.RestStatus
;
import
org.elasticsearch.rest.RestStatus
;
...
@@ -60,9 +61,11 @@ import org.elasticsearch.search.sort.SortOrder;
...
@@ -60,9 +61,11 @@ import org.elasticsearch.search.sort.SortOrder;
import
org.springframework.stereotype.Component
;
import
org.springframework.stereotype.Component
;
import
javax.annotation.Resource
;
import
javax.annotation.Resource
;
import
javax.validation.constraints.NotNull
;
import
java.io.IOException
;
import
java.io.IOException
;
import
java.lang.reflect.Field
;
import
java.lang.reflect.Field
;
import
java.util.*
;
import
java.util.*
;
import
java.util.stream.Collectors
;
/**
/**
* Es操作相关工具栏
* Es操作相关工具栏
...
@@ -498,7 +501,7 @@ public class EsOpUtil<T> {
...
@@ -498,7 +501,7 @@ public class EsOpUtil<T> {
}
}
public
void
docUpdateBulk
(
String
index
,
List
<
SpecialInformation
>
dataList
){
public
void
docUpdateBulk
(
String
index
,
List
<
SpecialInformation
>
dataList
)
{
BulkRequest
bulkRequest
=
new
BulkRequest
();
BulkRequest
bulkRequest
=
new
BulkRequest
();
for
(
SpecialInformation
information
:
dataList
)
{
for
(
SpecialInformation
information
:
dataList
)
{
UpdateRequest
request
=
new
UpdateRequest
(
index
,
information
.
getId
());
UpdateRequest
request
=
new
UpdateRequest
(
index
,
information
.
getId
());
...
@@ -534,9 +537,9 @@ public class EsOpUtil<T> {
...
@@ -534,9 +537,9 @@ public class EsOpUtil<T> {
* @author lkg
* @author lkg
* @date 2024/12/19
* @date 2024/12/19
*/
*/
public
void
docRemoveBulk
(
List
<
Map
<
String
,
Object
>>
removeList
){
public
void
docRemoveBulk
(
List
<
Map
<
String
,
Object
>>
removeList
)
{
BulkRequest
bulkRequest
=
new
BulkRequest
();
BulkRequest
bulkRequest
=
new
BulkRequest
();
for
(
Map
<
String
,
Object
>
m
:
removeList
)
{
for
(
Map
<
String
,
Object
>
m
:
removeList
)
{
String
index
=
m
.
get
(
"index"
).
toString
();
String
index
=
m
.
get
(
"index"
).
toString
();
String
id
=
m
.
get
(
"id"
).
toString
();
String
id
=
m
.
get
(
"id"
).
toString
();
DeleteRequest
deleteRequest
=
new
DeleteRequest
(
index
,
id
);
DeleteRequest
deleteRequest
=
new
DeleteRequest
(
index
,
id
);
...
@@ -546,7 +549,7 @@ public class EsOpUtil<T> {
...
@@ -546,7 +549,7 @@ public class EsOpUtil<T> {
try
{
try
{
client
.
bulk
(
bulkRequest
,
RequestOptions
.
DEFAULT
);
client
.
bulk
(
bulkRequest
,
RequestOptions
.
DEFAULT
);
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
log
.
error
(
"数据删除失败!"
,
e
);
log
.
error
(
"数据删除失败!"
,
e
);
}
}
}
}
...
@@ -557,7 +560,7 @@ public class EsOpUtil<T> {
...
@@ -557,7 +560,7 @@ public class EsOpUtil<T> {
* @author lkg
* @author lkg
* @date 2024/12/19
* @date 2024/12/19
*/
*/
public
void
docRemoveBulk
(
Map
<
String
,
List
<
SpecialInformation
>>
removeMap
){
public
void
docRemoveBulk
(
Map
<
String
,
List
<
SpecialInformation
>>
removeMap
)
{
BulkRequest
bulkRequest
=
new
BulkRequest
();
BulkRequest
bulkRequest
=
new
BulkRequest
();
for
(
Map
.
Entry
<
String
,
List
<
SpecialInformation
>>
entry
:
removeMap
.
entrySet
())
{
for
(
Map
.
Entry
<
String
,
List
<
SpecialInformation
>>
entry
:
removeMap
.
entrySet
())
{
String
index
=
entry
.
getKey
();
String
index
=
entry
.
getKey
();
...
@@ -570,7 +573,7 @@ public class EsOpUtil<T> {
...
@@ -570,7 +573,7 @@ public class EsOpUtil<T> {
try
{
try
{
client
.
bulk
(
bulkRequest
,
RequestOptions
.
DEFAULT
);
client
.
bulk
(
bulkRequest
,
RequestOptions
.
DEFAULT
);
}
catch
(
Exception
e
)
{
}
catch
(
Exception
e
)
{
log
.
error
(
"数据删除失败!"
,
e
);
log
.
error
(
"数据删除失败!"
,
e
);
}
}
}
}
...
@@ -667,6 +670,8 @@ public class EsOpUtil<T> {
...
@@ -667,6 +670,8 @@ public class EsOpUtil<T> {
}
}
/**
/**
* 批量更新(参数deleteParams 中的key要和es中的字段保持一致)
* <p>
* 批量更新操作,根据指定的查询条件和多个字段的映射关系,更新符合条件的文档的多个字段的值。
* 批量更新操作,根据指定的查询条件和多个字段的映射关系,更新符合条件的文档的多个字段的值。
* 方法名:updataMoreColumBatchByQuery,表示批量更新文档的多个字段的方法。
* 方法名:updataMoreColumBatchByQuery,表示批量更新文档的多个字段的方法。
* 参数:index,指定要更新的索引名称。
* 参数:index,指定要更新的索引名称。
...
@@ -679,22 +684,142 @@ public class EsOpUtil<T> {
...
@@ -679,22 +684,142 @@ public class EsOpUtil<T> {
* 设置更新脚本,使用setScript方法,传入Script对象。脚本使用painless语言,通过ctx._source.字段名 = 字段值的方式来更新文档的多个字段的值。
* 设置更新脚本,使用setScript方法,传入Script对象。脚本使用painless语言,通过ctx._source.字段名 = 字段值的方式来更新文档的多个字段的值。
* 调用client.updateByQuery方法,传入UpdateByQueryRequest对象和默认的RequestOptions,执行批量更新操作,并返回BulkByScrollResponse对象。
* 调用client.updateByQuery方法,传入UpdateByQueryRequest对象和默认的RequestOptions,执行批量更新操作,并返回BulkByScrollResponse对象。
* Collections.emptyMap()是setScript脚本的参数,此方法没有设置其他参数,使用一个空的map
* Collections.emptyMap()是setScript脚本的参数,此方法没有设置其他参数,使用一个空的map
*
* @param index 索引
* @param boolQuery es查询条件
* @param modifyParams 更新参数
* @author lkg
* @date 2025/4/22
*/
*/
public
void
updateMoreColumBatchByQuery
(
String
index
,
BoolQueryBuilder
boolQuery
,
Map
<
String
,
String
>
modifyColumValue
)
throws
IOException
,
InterruptedException
{
public
void
batchUpdateScript
(
String
[]
index
,
BoolQueryBuilder
boolQuery
,
Map
<
String
,
Object
>
modifyParams
)
{
UpdateByQueryRequest
request
=
new
UpdateByQueryRequest
(
index
);
UpdateByQueryRequest
request
=
new
UpdateByQueryRequest
(
index
);
request
.
setQuery
(
boolQuery
);
request
.
setQuery
(
boolQuery
);
request
.
setScript
(
new
Script
(
ScriptType
.
INLINE
,
"painless"
,
getIdOrCode
(
modifyColumValue
),
Collections
.
emptyMap
()));
StringBuffer
script
=
new
StringBuffer
();
modifyParams
.
forEach
((
colum
,
value
)
->
{
if
(
value
instanceof
String
)
{
script
.
append
(
"ctx._source."
).
append
(
colum
).
append
(
" = '"
).
append
(
value
).
append
(
"';"
);
}
else
{
script
.
append
(
"ctx._source."
).
append
(
colum
).
append
(
" = "
).
append
(
value
).
append
(
";"
);
}
});
request
.
setScript
(
new
Script
(
ScriptType
.
INLINE
,
"painless"
,
script
.
toString
(),
Collections
.
emptyMap
()));
//可跳过版本冲突的文档
request
.
setConflicts
(
"proceed"
);
try
{
BulkByScrollResponse
response
=
client
.
updateByQuery
(
request
,
RequestOptions
.
DEFAULT
);
BulkByScrollResponse
response
=
client
.
updateByQuery
(
request
,
RequestOptions
.
DEFAULT
);
long
versionConflicts
=
response
.
getVersionConflicts
();
long
updated
=
response
.
getUpdated
();
long
updated
=
response
.
getUpdated
();
log
.
info
(
"更新条数{}"
,
updated
);
log
.
info
(
"更新条数:{},冲突条数{}"
,
updated
,
versionConflicts
);
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
}
private
String
getIdOrCode
(
Map
<
String
,
String
>
columValue
)
{
/**
StringBuffer
script
=
new
StringBuffer
();
* 批量替换字符串(更新的一种)
columValue
.
forEach
((
colum
,
value
)
->
{
*
script
.
append
(
"ctx._source."
).
append
(
colum
).
append
(
" = '"
).
append
(
value
).
append
(
"';"
);
* @param index 索引
* @param boolQuery es查询条件
* @param modifyField 编辑的字段集合
* @param oldValue 旧字符串
* @param newValue 新字符串
* @author lkg
* @date 2025/4/22
*/
public
void
batchReplaceScript
(
String
[]
index
,
BoolQueryBuilder
boolQuery
,
List
<
String
>
modifyField
,
String
oldValue
,
@NotNull
String
newValue
)
{
UpdateByQueryRequest
request
=
new
UpdateByQueryRequest
(
index
);
request
.
setQuery
(
boolQuery
);
String
script
=
modifyField
.
stream
()
.
map
(
field
->
String
.
format
(
"if(ctx._source.%s != null) { ctx._source.%s = ctx._source.%s.replace('%s', '%s'); }"
,
field
,
field
,
field
,
oldValue
,
newValue
))
.
collect
(
Collectors
.
joining
(
""
));
request
.
setScript
(
new
Script
(
ScriptType
.
INLINE
,
"painless"
,
script
,
Collections
.
emptyMap
()));
//可跳过版本冲突的文档
request
.
setConflicts
(
"proceed"
);
try
{
BulkByScrollResponse
response
=
client
.
updateByQuery
(
request
,
RequestOptions
.
DEFAULT
);
long
versionConflicts
=
response
.
getVersionConflicts
();
long
updated
=
response
.
getUpdated
();
log
.
info
(
"更新条数:{},冲突条数{}"
,
updated
,
versionConflicts
);
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
/**
* 批量删除nest字段中的数据
* <p>
* 参数deleteParams 中的key要和es中的字段保持一致
*
* @param index 索引
* @param boolQuery es查询条件
* @param deleteParams 删除条件
* @param nestField nest字段
* @author lkg
* @date 2025/4/22
*/
public
void
batchNestedDeleteScript
(
String
[]
index
,
BoolQueryBuilder
boolQuery
,
Map
<
String
,
Object
>
deleteParams
,
String
nestField
)
{
UpdateByQueryRequest
request
=
new
UpdateByQueryRequest
(
index
);
request
.
setQuery
(
boolQuery
);
//Painless 脚本:从 nested 数组中删除匹配项
String
scriptSource
=
"ctx._source."
+
nestField
+
".removeIf(item -> "
;
StringBuilder
paramString
=
new
StringBuilder
();
deleteParams
.
forEach
((
colum
,
value
)
->
{
if
(
value
instanceof
String
)
{
paramString
.
append
(
" && "
).
append
(
"item."
).
append
(
colum
).
append
(
" == params."
).
append
(
colum
);
}
else
{
paramString
.
append
(
" && "
).
append
(
"item."
).
append
(
colum
).
append
(
" == params."
).
append
(
colum
);
}
});
});
return
script
.
toString
();
String
substring
=
paramString
.
substring
(
4
);
scriptSource
=
scriptSource
+
substring
+
")"
;
Script
script
=
new
Script
(
ScriptType
.
INLINE
,
"painless"
,
scriptSource
,
deleteParams
);
request
.
setScript
(
script
);
//可跳过版本冲突的文档
request
.
setConflicts
(
"proceed"
);
try
{
BulkByScrollResponse
response
=
client
.
updateByQuery
(
request
,
RequestOptions
.
DEFAULT
);
long
versionConflicts
=
response
.
getVersionConflicts
();
long
updated
=
response
.
getUpdated
();
log
.
info
(
"更新条数:{},冲突条数{}"
,
updated
,
versionConflicts
);
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
/**
* 批量新增nest字段中的数据
* <p>
* 参数addParams 中的key要和es中的字段保持一致
*
* @param index 索引
* @param boolQuery es查询条件
* @param addParams 添加的数据
* @param nestField nest字段
* @param uniqueField 唯一字段
* @author lkg
* @date 2025/4/22
*/
public
void
batchNestedAddScript
(
String
[]
index
,
BoolQueryBuilder
boolQuery
,
Map
<
String
,
Object
>
addParams
,
String
nestField
,
String
uniqueField
)
{
UpdateByQueryRequest
request
=
new
UpdateByQueryRequest
(
index
);
request
.
setQuery
(
boolQuery
);
//Painless 脚本:从 nested 数组中删除匹配项
String
scriptString
=
"if(ctx._source.NEST_FIELD==null){List ls=new ArrayList();ls.add(params.NEST_FIELD);ctx._source.NEST_FIELD=ls;}else{List olds=new ArrayList();ctx._source.NEST_FIELD.forEach(item->{olds.add(item.UNIQUE_FIELD);});if(!olds.contains(params.NEST_FIELD.UNIQUE_FIELD)){ctx._source.NEST_FIELD.add(params.NEST_FIELD);}}"
;
String
scriptSource
=
scriptString
.
replace
(
"NEST_FIELD"
,
nestField
).
replace
(
"UNIQUE_FIELD"
,
uniqueField
);
Script
script
=
new
Script
(
ScriptType
.
INLINE
,
"painless"
,
scriptSource
,
addParams
);
request
.
setScript
(
script
);
//可跳过版本冲突的文档
request
.
setConflicts
(
"proceed"
);
try
{
BulkByScrollResponse
response
=
client
.
updateByQuery
(
request
,
RequestOptions
.
DEFAULT
);
long
versionConflicts
=
response
.
getVersionConflicts
();
long
updated
=
response
.
getUpdated
();
log
.
info
(
"更新条数:{},冲突条数{}"
,
updated
,
versionConflicts
);
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
}
}
}
...
@@ -762,13 +887,13 @@ public class EsOpUtil<T> {
...
@@ -762,13 +887,13 @@ public class EsOpUtil<T> {
}
}
// 执行更新请求
// 执行更新请求
try
{
try
{
UpdateResponse
response
=
client
.
update
(
createUpdateRequest
(
index
,
id
,
args
,
true
),
RequestOptions
.
DEFAULT
);
UpdateResponse
response
=
client
.
update
(
createUpdateRequest
(
index
,
id
,
args
,
true
),
RequestOptions
.
DEFAULT
);
RestStatus
status
=
response
.
status
();
RestStatus
status
=
response
.
status
();
if
(
status
.
getStatus
()
!=
200
)
{
if
(
status
.
getStatus
()
!=
200
)
{
log
.
info
(
"{},更新失败"
,
id
);
log
.
info
(
"{},更新失败"
,
id
);
}
}
}
catch
(
IOException
e
)
{
}
catch
(
IOException
e
)
{
log
.
info
(
"{},更新异常"
,
id
);
log
.
info
(
"{},更新异常"
,
id
);
}
}
}
}
...
@@ -788,7 +913,7 @@ public class EsOpUtil<T> {
...
@@ -788,7 +913,7 @@ public class EsOpUtil<T> {
BulkRequest
bulkRequest
=
new
BulkRequest
();
BulkRequest
bulkRequest
=
new
BulkRequest
();
args
.
forEach
((
id
,
args1
)
->
{
args
.
forEach
((
id
,
args1
)
->
{
try
{
try
{
bulkRequest
.
add
(
createUpdateRequest
(
index
,
id
,
args1
,
false
));
bulkRequest
.
add
(
createUpdateRequest
(
index
,
id
,
args1
,
false
));
}
catch
(
IOException
e
)
{
}
catch
(
IOException
e
)
{
e
.
printStackTrace
();
e
.
printStackTrace
();
}
}
...
@@ -817,7 +942,7 @@ public class EsOpUtil<T> {
...
@@ -817,7 +942,7 @@ public class EsOpUtil<T> {
* 将contentBuilder设置为更新请求的内容。
* 将contentBuilder设置为更新请求的内容。
* 返回更新请求UpdateRequest对象。
* 返回更新请求UpdateRequest对象。
*/
*/
private
UpdateRequest
createUpdateRequest
(
String
index
,
String
documentId
,
Map
<
String
,
Object
>
args
,
Boolean
refreshPolicy
)
throws
IOException
{
private
UpdateRequest
createUpdateRequest
(
String
index
,
String
documentId
,
Map
<
String
,
Object
>
args
,
Boolean
refreshPolicy
)
throws
IOException
{
UpdateRequest
request
=
new
UpdateRequest
(
index
,
documentId
);
UpdateRequest
request
=
new
UpdateRequest
(
index
,
documentId
);
if
(
Boolean
.
TRUE
.
equals
(
refreshPolicy
))
{
if
(
Boolean
.
TRUE
.
equals
(
refreshPolicy
))
{
//刷新策略,立即刷新
//刷新策略,立即刷新
...
...
src/main/java/com/zzsn/event/util/HanlpUtil.java
浏览文件 @
c311ac86
...
@@ -114,7 +114,7 @@ public class HanlpUtil {
...
@@ -114,7 +114,7 @@ public class HanlpUtil {
* @创建时间 2020/8/27 19:56
* @创建时间 2020/8/27 19:56
* @Version 1.0
* @Version 1.0
*/
*/
p
rivate
static
int
countKeyWordInContent
(
String
keyword
,
String
srcContent
){
p
ublic
static
int
countKeyWordInContent
(
String
keyword
,
String
srcContent
){
if
(
keyword
==
null
||
keyword
.
trim
().
equals
(
""
)){
if
(
keyword
==
null
||
keyword
.
trim
().
equals
(
""
)){
return
0
;
return
0
;
}
}
...
...
src/main/java/com/zzsn/event/vo/InfoDataSearchCondition.java
浏览文件 @
c311ac86
...
@@ -50,11 +50,17 @@ public class InfoDataSearchCondition {
...
@@ -50,11 +50,17 @@ public class InfoDataSearchCondition {
/**原创性(0-非原创;1-原创;2-疑似)*/
/**原创性(0-非原创;1-原创;2-疑似)*/
private
String
originality
;
private
String
originality
;
//资讯状态-研究中心(0-全部;1-模型推荐;2-精选;3-待定;4-移除),和checkStatus、deleteFlag、isFreeCheck互斥
private
Integer
status
;
//审核操作(0:未审核 1:审核通过 2:审核未通过 3:暂定 默认值为0)
//审核操作(0:未审核 1:审核通过 2:审核未通过 3:暂定 默认值为0)
private
Integer
checkStatus
;
private
Integer
checkStatus
;
//删除标记(1:删除;0:未删除)
//删除标记(1:删除;0:未删除)
private
Integer
deleteFlag
=
0
;
private
Integer
deleteFlag
;
//是否免审核(1-是;0-否)
private
Integer
isFreeCheck
;
//关联标签名称
//关联标签名称
private
String
labelName
;
private
String
labelName
;
...
@@ -70,7 +76,7 @@ public class InfoDataSearchCondition {
...
@@ -70,7 +76,7 @@ public class InfoDataSearchCondition {
//爬虫类型
//爬虫类型
private
String
crawler
;
private
String
crawler
;
//组合标签查询(不同类标签之间是与的关系,同一类标签之间是或的关系),示例: "a
,b;c,d;e,f
"
//组合标签查询(不同类标签之间是与的关系,同一类标签之间是或的关系),示例: "a
1,a2;c1,c2
"
private
String
composeSearchLabelIds
;
private
String
composeSearchLabelIds
;
/**----企业类标签筛选----**/
/**----企业类标签筛选----**/
...
@@ -104,21 +110,22 @@ public class InfoDataSearchCondition {
...
@@ -104,21 +110,22 @@ public class InfoDataSearchCondition {
//待删除的标签id(多个用逗号隔开)-研究中心
//待删除的标签id(多个用逗号隔开)-研究中心
private
String
removeRelationId
;
private
String
removeRelationId
;
//关键词信息;推荐信息源时使用-研究中心
//关键词信息;推荐信息源时使用-研究中心
private
List
<
String
>
wordsList
;
private
List
<
String
>
wordsList
;
//是否免审核(1-是;0-否)
private
Integer
isFreeCheck
;
//得分范围-研究中心
//得分范围-研究中心
//最小得分
//最小得分
private
Integer
minScore
;
private
Integer
minScore
;
//最大得分
//最大得分
private
Integer
maxScore
;
private
Integer
maxScore
;
//正文长度-研究中心
//最小长度
private
Integer
minContentLength
;
//最大长度
private
Integer
maxContentLength
;
//es查询字段数组
//es查询字段数组
private
String
[]
fetchFields
;
private
String
[]
fetchFields
;
//排除字段数组
//排除字段数组
...
@@ -126,7 +133,7 @@ public class InfoDataSearchCondition {
...
@@ -126,7 +133,7 @@ public class InfoDataSearchCondition {
//排序参数
//排序参数
//排序字段
//排序字段
private
String
column
=
"publishDate"
;
private
String
column
;
//排序方式 asc/desc
//排序方式 asc/desc
private
String
order
=
"desc"
;
private
String
order
=
"desc"
;
//置顶排序是否起效(1-是;0-否)
//置顶排序是否起效(1-是;0-否)
...
@@ -166,7 +173,23 @@ public class InfoDataSearchCondition {
...
@@ -166,7 +173,23 @@ public class InfoDataSearchCondition {
/*------添加至精选时,主题列表---start-------------------*/
/*------添加至精选时,主题列表---start-------------------*/
//聚合分组类型-按日期集合分析时使用
private
List
<
Label
>
themeList
;
private
List
<
Label
>
themeList
;
/*------添加至精选时,主题列表---end-------------------*/
/*------添加至精选时,主题列表---end-------------------*/
/*------待删除的关键词列表---start-------------------*/
private
List
<
String
>
keywordList
;
/*------待删除的关键词列表---end-------------------*/
/*-----字符串替换---start-------------------*/
private
String
replaceWord
;
/*------字符串替换---end-------------------*/
/*-----打标签---start-------------------*/
private
List
<
Label
>
markTags
;
/*------打标签---end-------------------*/
/**
* 本次任务使用的流程id
*/
private
String
flowId
;
}
}
src/main/java/com/zzsn/event/vo/InfoSourceCondition.java
浏览文件 @
c311ac86
...
@@ -30,7 +30,7 @@ public class InfoSourceCondition {
...
@@ -30,7 +30,7 @@ public class InfoSourceCondition {
/**
/**
* 爬虫类别
* 爬虫类别
*/
*/
private
int
crawlType
;
private
Integer
crawlType
;
/**
/**
* 专题id/事件id-获取专题绑定信息源信息时使用
* 专题id/事件id-获取专题绑定信息源信息时使用
...
...
src/main/java/com/zzsn/event/vo/SubjectPageVO.java
浏览文件 @
c311ac86
...
@@ -27,6 +27,7 @@ public class SubjectPageVO {
...
@@ -27,6 +27,7 @@ public class SubjectPageVO {
private
String
subjectTypeName
;
private
String
subjectTypeName
;
/**专题所属项目名称*/
/**专题所属项目名称*/
private
String
projectName
;
private
String
projectName
;
private
String
dataScope
;
/**信息总数量*/
/**信息总数量*/
private
Integer
totalNum
;
private
Integer
totalNum
;
...
...
src/main/java/com/zzsn/event/vo/es/DisplayInfo.java
浏览文件 @
c311ac86
...
@@ -44,6 +44,8 @@ public class DisplayInfo {
...
@@ -44,6 +44,8 @@ public class DisplayInfo {
private
String
summaryRaw
;
private
String
summaryRaw
;
//关键词
//关键词
private
String
keyWords
;
private
String
keyWords
;
//命中词列表
private
List
<
String
>
keyWordsList
;
//标题
//标题
private
String
title
;
private
String
title
;
private
String
titleRaw
;
private
String
titleRaw
;
...
@@ -54,6 +56,8 @@ public class DisplayInfo {
...
@@ -54,6 +56,8 @@ public class DisplayInfo {
private
String
type
;
private
String
type
;
//标签信息
//标签信息
private
List
<
Label
>
labels
;
private
List
<
Label
>
labels
;
//排序字段信息
private
List
<
SortField
>
sortField
;
//模型打分信息
//模型打分信息
private
List
<
ModelScore
>
modelScores
;
private
List
<
ModelScore
>
modelScores
;
//视频下载链接
//视频下载链接
...
...
src/main/java/com/zzsn/event/vo/es/SortField.java
0 → 100644
浏览文件 @
c311ac86
package
com
.
zzsn
.
event
.
vo
.
es
;
import
lombok.Data
;
/**
* 排序字段
*
* @author lkg
* @date 2025/4/22
*/
@Data
public
class
SortField
{
//字段 (摘要、内容、标签、词频统计)
private
String
fieldType
;
//字段排序值
private
String
fieldKeyword
;
//词频
private
Long
fieldLong
;
public
SortField
(
String
fieldType
,
Long
fieldLong
)
{
this
.
fieldType
=
fieldType
;
this
.
fieldLong
=
fieldLong
;
}
public
SortField
(
String
fieldType
,
String
fieldKeyword
)
{
this
.
fieldType
=
fieldType
;
this
.
fieldKeyword
=
fieldKeyword
;
}
}
src/main/java/com/zzsn/event/vo/es/SpecialInformation.java
浏览文件 @
c311ac86
...
@@ -53,6 +53,8 @@ public class SpecialInformation {
...
@@ -53,6 +53,8 @@ public class SpecialInformation {
private
String
type
;
private
String
type
;
//标签信息
//标签信息
private
List
<
Label
>
labels
;
private
List
<
Label
>
labels
;
//排序字段信息
private
List
<
SortField
>
sortField
;
//模型打分信息
//模型打分信息
private
List
<
ModelScore
>
modelScores
;
private
List
<
ModelScore
>
modelScores
;
//视频下载链接
//视频下载链接
...
...
编写
预览
Markdown
格式
0%
重试
或
添加新文件
添加附件
取消
您添加了
0
人
到此讨论。请谨慎行事。
请先完成此评论的编辑!
取消
请
注册
或者
登录
后发表评论