提交 c311ac86 作者: yanxin

Merge remote-tracking branch 'origin/event_fusion' into event_fusion

# Conflicts:
#	src/main/java/com/zzsn/event/es/EsService.java
...@@ -107,4 +107,6 @@ public class Constants { ...@@ -107,4 +107,6 @@ public class Constants {
//obs 文件浏览地址 前部分 //obs 文件浏览地址 前部分
public static final String OBS_FILE_PATH_URL_PREFIX = "http://obs.ciglobal.cn/"; public static final String OBS_FILE_PATH_URL_PREFIX = "http://obs.ciglobal.cn/";
/**专题模型处理中断缓存*/
public static final String SUBJECT_MODEL_DETAIL_BREAK = "SUBJECT_MODEL_DETAIL_BREAK::";
} }
...@@ -860,6 +860,18 @@ public class SubjectManageController { ...@@ -860,6 +860,18 @@ public class SubjectManageController {
List<InfoSourceLabelVO> bindList = clbLabelService.bindInfoSourceLabelInfo(subjectId); List<InfoSourceLabelVO> bindList = clbLabelService.bindInfoSourceLabelInfo(subjectId);
return Result.OK(bindList); return Result.OK(bindList);
} }
/**
* 更新subject的数据范围
* @param subjectId 专题id
*/
@GetMapping("/subjectUpdateScope")
public Result<?> subjectUpdateScope(@RequestParam(name = "subjectId") String subjectId,
@RequestParam(name = "type") String type,
@RequestParam(name = "scop" , defaultValue = "") String scop
) {
Subject subject = subjectService.subjectUpdateScope(subjectId,type,scop);
return Result.OK(subject);
}
/** /**
* 绑定信息源标签数据 * 绑定信息源标签数据
......
...@@ -3,26 +3,23 @@ package com.zzsn.event.controller.biz; ...@@ -3,26 +3,23 @@ package com.zzsn.event.controller.biz;
import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.zzsn.event.constant.Result; import com.zzsn.event.constant.Result;
import com.zzsn.event.entity.ClbSubjectHitwordsMap; import com.zzsn.event.entity.ClbSubjectHitwordsMap;
import com.zzsn.event.feign.api.RemoteProjectService;
import com.zzsn.event.service.ClbSubjectHitwordsMapService; import com.zzsn.event.service.ClbSubjectHitwordsMapService;
import com.zzsn.event.vo.InfoDataSearchCondition;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import org.springframework.web.servlet.ModelAndView;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/** /**
* <p> * <p>
...@@ -33,10 +30,13 @@ import java.util.stream.Collectors; ...@@ -33,10 +30,13 @@ import java.util.stream.Collectors;
* @since 2025-04-23 * @since 2025-04-23
*/ */
@RestController @RestController
@Slf4j
@RequestMapping("/subject/hitwordsMap") @RequestMapping("/subject/hitwordsMap")
public class ClbSubjectHitwordsMapController { public class ClbSubjectHitwordsMapController {
@Autowired @Autowired
private ClbSubjectHitwordsMapService clbSubjectHitwordsMapService; private ClbSubjectHitwordsMapService clbSubjectHitwordsMapService;
@Autowired
private RemoteProjectService remoteProjectService;
/** /**
* 分页列表查询 * 分页列表查询
...@@ -106,6 +106,17 @@ public class ClbSubjectHitwordsMapController { ...@@ -106,6 +106,17 @@ public class ClbSubjectHitwordsMapController {
} }
clbSubjectHitwordsMapService.save(clbSubjectHitwordsMap); clbSubjectHitwordsMapService.save(clbSubjectHitwordsMap);
InfoDataSearchCondition searchCondition = new InfoDataSearchCondition();
searchCondition.setSubjectId(clbSubjectHitwordsMap.getSubjectId());
searchCondition.setStatus(0);
searchCondition.setIsSubject("1");
searchCondition.setCategory(1);
searchCondition.setFlowId("1914980469431201794");
//手动中断正在执行的词频历史处理任务
clbSubjectHitwordsMapService.stopTask(searchCondition);
String s = remoteProjectService.doTaskByFlow(searchCondition);
log.info("调用流程编排接口返回结果:{}",s);
return Result.OK(clbSubjectHitwordsMap); return Result.OK(clbSubjectHitwordsMap);
} }
......
...@@ -13,11 +13,11 @@ import com.zzsn.event.enums.LabelTypeEnum; ...@@ -13,11 +13,11 @@ import com.zzsn.event.enums.LabelTypeEnum;
import com.zzsn.event.es.EsService; import com.zzsn.event.es.EsService;
import com.zzsn.event.service.*; import com.zzsn.event.service.*;
import com.zzsn.event.util.PythonUtil; import com.zzsn.event.util.PythonUtil;
import com.zzsn.event.util.tree.Node;
import com.zzsn.event.util.user.UserUtil; import com.zzsn.event.util.user.UserUtil;
import com.zzsn.event.util.user.UserVo; import com.zzsn.event.util.user.UserVo;
import com.zzsn.event.vo.*; import com.zzsn.event.vo.*;
import com.zzsn.event.vo.es.DisplayInfo; import com.zzsn.event.vo.es.DisplayInfo;
import com.zzsn.event.vo.es.Label;
import com.zzsn.event.vo.es.SpecialInformation; import com.zzsn.event.vo.es.SpecialInformation;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
...@@ -211,6 +211,7 @@ public class InformationController { ...@@ -211,6 +211,7 @@ public class InformationController {
IPage<DisplayInfo> pageList = informationService.subjectPageList(searchCondition); IPage<DisplayInfo> pageList = informationService.subjectPageList(searchCondition);
return Result.OK(pageList); return Result.OK(pageList);
} }
/** /**
* 事件对应专题库的资讯分页列表 * 事件对应专题库的资讯分页列表
* *
...@@ -270,6 +271,7 @@ public class InformationController { ...@@ -270,6 +271,7 @@ public class InformationController {
/** /**
* 删除指定标签 * 删除指定标签
*
* @param id * @param id
* @param relationId * @param relationId
* @param index * @param index
...@@ -279,9 +281,10 @@ public class InformationController { ...@@ -279,9 +281,10 @@ public class InformationController {
public Result<?> delLabel(@RequestParam("id") String id, public Result<?> delLabel(@RequestParam("id") String id,
@RequestParam("relationId") String relationId, @RequestParam("relationId") String relationId,
@RequestParam("index") String index) { @RequestParam("index") String index) {
informationService.removeLabelById(index,id,relationId); informationService.removeLabelById(index, id, relationId);
return Result.OK("删除成功"); return Result.OK("删除成功");
} }
/** /**
* 批量删除标签 * 批量删除标签
* *
...@@ -303,7 +306,6 @@ public class InformationController { ...@@ -303,7 +306,6 @@ public class InformationController {
} }
/** /**
* 通过id查询(专题库) * 通过id查询(专题库)
* *
...@@ -394,11 +396,11 @@ public class InformationController { ...@@ -394,11 +396,11 @@ public class InformationController {
*/ */
@PostMapping(value = "/modifyLabel") @PostMapping(value = "/modifyLabel")
public Result<?> modifyLabel(@RequestBody DataBindLabelFrom bindLabelFrom) { public Result<?> modifyLabel(@RequestBody DataBindLabelFrom bindLabelFrom) {
if(bindLabelFrom.getBindList()==null || bindLabelFrom.getBindList().isEmpty()){ if (bindLabelFrom.getBindList() == null || bindLabelFrom.getBindList().isEmpty()) {
List<DictVO> boundList = subjectDictMapService.boundList(bindLabelFrom.getSubjectId()); List<DictVO> boundList = subjectDictMapService.boundList(bindLabelFrom.getSubjectId());
bindLabelFrom.setBindList(boundList); bindLabelFrom.setBindList(boundList);
} }
if(bindLabelFrom.getBindList()==null || bindLabelFrom.getBindList().isEmpty()){ if (bindLabelFrom.getBindList() == null || bindLabelFrom.getBindList().isEmpty()) {
return Result.FAIL("专题未绑定标签,不可编辑"); return Result.FAIL("专题未绑定标签,不可编辑");
} }
informationService.modifyLabel(bindLabelFrom); informationService.modifyLabel(bindLabelFrom);
...@@ -503,7 +505,7 @@ public class InformationController { ...@@ -503,7 +505,7 @@ public class InformationController {
* @date 2025/2/20 * @date 2025/2/20
*/ */
@PostMapping("/removeByCondition") @PostMapping("/removeByCondition")
public Result<?> removeByCondition(@RequestBody JSONObject params){ public Result<?> removeByCondition(@RequestBody JSONObject params) {
String subjectId = params.getString("subjectId"); String subjectId = params.getString("subjectId");
if (StringUtils.isEmpty(subjectId)) { if (StringUtils.isEmpty(subjectId)) {
return Result.FAIL("专题id不能为空"); return Result.FAIL("专题id不能为空");
...@@ -524,7 +526,7 @@ public class InformationController { ...@@ -524,7 +526,7 @@ public class InformationController {
* @date 2025/2/20 * @date 2025/2/20
*/ */
@PostMapping("/supplyByCondition") @PostMapping("/supplyByCondition")
public Result<?> supplyByCondition(@RequestBody JSONObject params){ public Result<?> supplyByCondition(@RequestBody JSONObject params) {
String subjectId = params.getString("subjectId"); String subjectId = params.getString("subjectId");
if (StringUtils.isEmpty(subjectId)) { if (StringUtils.isEmpty(subjectId)) {
return Result.FAIL("专题id不能为空"); return Result.FAIL("专题id不能为空");
...@@ -536,4 +538,166 @@ public class InformationController { ...@@ -536,4 +538,166 @@ public class InformationController {
informationService.supplyByCondition(subjectId, JSONArray.parseArray(themeIds.toJSONString(), String.class)); informationService.supplyByCondition(subjectId, JSONArray.parseArray(themeIds.toJSONString(), String.class));
return Result.OK(); return Result.OK();
} }
/**
* 批量删除关键词
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping("/removeWord")
public Result<?> removeWordLabel(@RequestBody InfoDataSearchCondition searchCondition) {
List<String> keywordList = searchCondition.getKeywordList();
if (CollectionUtils.isEmpty(keywordList)) {
return Result.FAIL("关键词不能为空");
}
CompletableFuture.runAsync(() -> informationService.removeWordLabel(searchCondition));
return Result.OK("关键词删除中。。。");
}
/**
* 批量添加到精选
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping("/addToFavorites")
public Result<?> addToFavorites(@RequestBody InfoDataSearchCondition searchCondition) {
String subjectId = searchCondition.getSubjectId();
List<String> ids = searchCondition.getIds();
if (StringUtils.isEmpty(subjectId) && CollectionUtils.isEmpty(ids)) {
return Result.FAIL("专题id和资讯id集合不能同时为空");
}
informationService.addToFavorites(searchCondition);
return Result.OK();
}
/**
* 批量添加到待定
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping("/addToPend")
public Result<?> addToPend(@RequestBody InfoDataSearchCondition searchCondition) {
String subjectId = searchCondition.getSubjectId();
List<String> ids = searchCondition.getIds();
if (StringUtils.isEmpty(subjectId) && CollectionUtils.isEmpty(ids)) {
return Result.FAIL("专题id和资讯id集合不能同时为空");
}
informationService.addToPend(searchCondition);
return Result.OK();
}
/**
* 批量添加到移除
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping("/addToRemove")
public Result<?> addToRemove(@RequestBody InfoDataSearchCondition searchCondition) {
String subjectId = searchCondition.getSubjectId();
List<String> ids = searchCondition.getIds();
if (StringUtils.isEmpty(subjectId) && CollectionUtils.isEmpty(ids)) {
return Result.FAIL("专题id和资讯id集合不能同时为空");
}
informationService.addToRemove(searchCondition);
return Result.OK();
}
/**
* 初始化数据
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping("/initialData")
public Result<?> initialData(@RequestBody InfoDataSearchCondition searchCondition) {
String subjectId = searchCondition.getSubjectId();
if (StringUtils.isEmpty(subjectId)) {
return Result.FAIL("专题id不能为空");
}
informationService.initialData(searchCondition);
return Result.OK();
}
//@PostMapping("/searchForReplaceList")
public Result<?> searchForReplaceList(@RequestBody InfoDataSearchCondition searchCondition) {
IPage<SpecialInformation> page = informationService.searchForReplaceList(searchCondition);
return Result.OK(page);
}
/**
* 字符串替换
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping("/charReplace")
public Result<?> charReplace(@RequestBody InfoDataSearchCondition searchCondition) {
String subjectId = searchCondition.getSubjectId();
if (StringUtils.isEmpty(subjectId)) {
return Result.FAIL("专题id不能为空");
}
List<SearchWordVO> searchWordList = searchCondition.getSearchWordList();
if (CollectionUtils.isEmpty(searchWordList)) {
return Result.FAIL("检索词不能为空");
}
informationService.charReplace(searchCondition);
return Result.OK();
}
/**
* 资讯批量打标
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping("/markTag")
public Result<?> markTag(@RequestBody InfoDataSearchCondition searchCondition) {
String subjectId = searchCondition.getSubjectId();
List<String> ids = searchCondition.getIds();
if (StringUtils.isEmpty(subjectId) && CollectionUtils.isEmpty(ids)) {
return Result.FAIL("专题id和资讯id集合不能同时为空");
}
List<Label> markTags = searchCondition.getMarkTags();
if (CollectionUtils.isEmpty(markTags)) {
return Result.FAIL("标签不能为空");
}
informationService.markTag(searchCondition);
return Result.OK();
}
/**
* 资讯批量删除标签
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/23
*/
@PostMapping("/removeTag")
public Result<?> removeTag(@RequestBody InfoDataSearchCondition searchCondition) {
String subjectId = searchCondition.getSubjectId();
List<String> ids = searchCondition.getIds();
if (StringUtils.isEmpty(subjectId) && CollectionUtils.isEmpty(ids)) {
return Result.FAIL("专题id和资讯id集合不能同时为空");
}
List<Label> markTags = searchCondition.getMarkTags();
if (CollectionUtils.isEmpty(markTags)) {
return Result.FAIL("标签不能为空");
}
informationService.removeTag(searchCondition);
return Result.OK();
}
} }
...@@ -7,9 +7,11 @@ import com.alibaba.fastjson2.JSONObject; ...@@ -7,9 +7,11 @@ import com.alibaba.fastjson2.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.Wrappers; import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.zzsn.event.constant.Result; import com.zzsn.event.constant.Result;
import com.zzsn.event.entity.InfoSource; import com.zzsn.event.entity.InfoSource;
import com.zzsn.event.entity.InfoSourceGroup; import com.zzsn.event.entity.InfoSourceGroup;
import com.zzsn.event.entity.Subject;
import com.zzsn.event.service.*; import com.zzsn.event.service.*;
import com.zzsn.event.util.tree.Node; import com.zzsn.event.util.tree.Node;
import com.zzsn.event.vo.*; import com.zzsn.event.vo.*;
...@@ -116,7 +118,30 @@ public class RelationBindController { ...@@ -116,7 +118,30 @@ public class RelationBindController {
} }
subjectIdList.add(subjectId); subjectIdList.add(subjectId);
} }
IPage<InfoSourceVo> pageList = infoSourceService.bindInfoSourcePageList(infoSourceCondition, subjectIdList, pageNo, pageSize); //查询
IPage<InfoSourceVo> pageList = null;
if (CollectionUtil.isNotEmpty(subjectIdList) && subjectIdList.size() == 1 && subjectId.equals(subjectIdList.get(0))) {
Subject byId = subjectService.getById(subjectId);
if (ObjectUtil.isNotNull(byId) && StringUtils.isNotEmpty(byId.getDataScope()) && byId.getDataScope().contains("1")) {
Page<InfoSource> page = new Page<>(pageNo, pageSize);
return Result.OK(infoSourceService.page(page,
Wrappers.<InfoSource>lambdaQuery()
.eq(ObjectUtil.isNotNull(infoSourceCondition.getCrawlType()),InfoSource::getCrawlType, infoSourceCondition.getCrawlType())
.eq(StrUtil.isNotBlank(infoSourceCondition.getStatus()),InfoSource::getStatus, infoSourceCondition.getStatus())
.like(StrUtil.isNotBlank(infoSourceCondition.getWebSiteName()),InfoSource::getWebSiteName, infoSourceCondition.getWebSiteName())
.like(StrUtil.isNotBlank(infoSourceCondition.getSiteName()),InfoSource::getSiteName, infoSourceCondition.getSiteName())
.like(StrUtil.isNotBlank(infoSourceCondition.getSiteUri()),InfoSource::getSiteUri, infoSourceCondition.getSiteUri())
.orderByDesc(InfoSource::getCreateTime)
));
}else {
pageList = infoSourceService.bindInfoSourcePageList(infoSourceCondition, subjectIdList, pageNo, pageSize);
}
}else {
pageList = infoSourceService.bindInfoSourcePageList(infoSourceCondition, subjectIdList, pageNo, pageSize);
}
return Result.OK(pageList); return Result.OK(pageList);
} }
......
...@@ -64,6 +64,9 @@ public class StatisticalAnalysisController { ...@@ -64,6 +64,9 @@ public class StatisticalAnalysisController {
List<CountVO> countList = new ArrayList<>(); List<CountVO> countList = new ArrayList<>();
int totalCount; int totalCount;
List<Label> bindLabelList = bindLabelList(searchCondition.getSubjectId()); List<Label> bindLabelList = bindLabelList(searchCondition.getSubjectId());
if (CollectionUtils.isEmpty(bindLabelList)) {
return Result.FAIL("该专题未绑定信息源标签,无法分析");
}
Map<String, Label> labelMap = bindLabelList.stream().collect(Collectors.toMap(label -> label.getLabelMark() + "-" + label.getRelationId(), label -> label)); Map<String, Label> labelMap = bindLabelList.stream().collect(Collectors.toMap(label -> label.getLabelMark() + "-" + label.getRelationId(), label -> label));
if (CollectionUtils.isEmpty(searchCondition.getIds()) && searchCondition.getNum() == null) { if (CollectionUtils.isEmpty(searchCondition.getIds()) && searchCondition.getNum() == null) {
List<String> includeValues = new ArrayList<>(); List<String> includeValues = new ArrayList<>();
......
...@@ -69,4 +69,8 @@ public class InfoSource implements Serializable { ...@@ -69,4 +69,8 @@ public class InfoSource implements Serializable {
/**原创来源(中文逗号隔开)*/ /**原创来源(中文逗号隔开)*/
private String originalSource; private String originalSource;
/**爬虫类别*/
private String crawlType;
/**爬虫类别*/
private String status;
} }
...@@ -114,5 +114,8 @@ public class Subject implements Serializable { ...@@ -114,5 +114,8 @@ public class Subject implements Serializable {
/**专题分类id*/ /**专题分类id*/
@TableField(exist = false) @TableField(exist = false)
private String typeId; private String typeId;
/**数据范围(是否是全库) - 采集库全库-1,企业库全库-2,政策库全库-3*/
@TableField(updateStrategy = FieldStrategy.IGNORED) // 忽略更新策略
private String dataScope;
} }
...@@ -36,6 +36,7 @@ import org.elasticsearch.action.update.UpdateRequest; ...@@ -36,6 +36,7 @@ import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; import org.elasticsearch.client.HttpAsyncResponseConsumerFactory;
import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.*;
import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.SearchHits;
...@@ -56,8 +57,9 @@ import org.elasticsearch.search.aggregations.metrics.Cardinality; ...@@ -56,8 +57,9 @@ import org.elasticsearch.search.aggregations.metrics.Cardinality;
import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.collapse.CollapseBuilder;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.*;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
...@@ -620,20 +622,20 @@ public class EsService { ...@@ -620,20 +622,20 @@ public class EsService {
if ("1".equals(eventDataCondition.getIsSubject())) { if ("1".equals(eventDataCondition.getIsSubject())) {
Subject byId = subjectService.getById(eventDataCondition.getSubjectId()); Subject byId = subjectService.getById(eventDataCondition.getSubjectId());
//更新开始、结束时间 //更新开始、结束时间
if(byId.getTimeEnable()!=null){ if (byId.getTimeEnable() != null) {
String timeEnable = DateUtil.format(byId.getTimeEnable(), "yyyy-MM-dd"); String timeEnable = DateUtil.format(byId.getTimeEnable(), "yyyy-MM-dd");
if(StringUtils.isEmpty(eventDataCondition.getStartTime()) || eventDataCondition.getStartTime().compareTo(timeEnable)<0){ if (StringUtils.isEmpty(eventDataCondition.getStartTime()) || eventDataCondition.getStartTime().compareTo(timeEnable) < 0) {
eventDataCondition.setStartTime(timeEnable); eventDataCondition.setStartTime(timeEnable);
} }
} }
if(byId.getTimeDisable()!=null){ if (byId.getTimeDisable() != null) {
String timeDisable = DateUtil.format(byId.getTimeDisable(), "yyyy-MM-dd"); String timeDisable = DateUtil.format(byId.getTimeDisable(), "yyyy-MM-dd");
if(StringUtils.isEmpty(eventDataCondition.getEndTime()) || eventDataCondition.getEndTime().compareTo(timeDisable)>0){ if (StringUtils.isEmpty(eventDataCondition.getEndTime()) || eventDataCondition.getEndTime().compareTo(timeDisable) > 0) {
eventDataCondition.setEndTime(timeDisable); eventDataCondition.setEndTime(timeDisable);
} }
} }
//以专题指定的索引为准 //以专题指定的索引为准
if(StringUtils.isNotEmpty(byId.getEsIndex())){ if (StringUtils.isNotEmpty(byId.getEsIndex())) {
index = byId.getEsIndex(); index = byId.getEsIndex();
} }
} }
...@@ -774,9 +776,7 @@ public class EsService { ...@@ -774,9 +776,7 @@ public class EsService {
* @date 2024/5/6 * @date 2024/5/6
*/ */
public IPage<SpecialInformation> pageListByCondition(InfoDataSearchCondition searchCondition, List<String> subjectIdList) throws IOException { public IPage<SpecialInformation> pageListByCondition(InfoDataSearchCondition searchCondition, List<String> subjectIdList) throws IOException {
String minDate = subjectService.getMinCreateTime(subjectIdList); String minDate = subjectService.getMinCreateTime(subjectIdList);
// Subject byId = subjectService(searchCondition.getSubjectId());
String[] indexs = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minDate); String[] indexs = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minDate);
SearchRequest searchRequest = new SearchRequest(indexs); SearchRequest searchRequest = new SearchRequest(indexs);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
...@@ -800,36 +800,62 @@ public class EsService { ...@@ -800,36 +800,62 @@ public class EsService {
if (searchCondition.getTopSortValid() == 1) { if (searchCondition.getTopSortValid() == 1) {
searchSourceBuilder.sort("topNum", SortOrder.DESC); searchSourceBuilder.sort("topNum", SortOrder.DESC);
} }
switch (column) { if (StringUtils.isEmpty(column)) {
case "score":
if (order.equals("asc")) {
searchSourceBuilder.sort("score", SortOrder.ASC);
searchSourceBuilder.sort("publishDate", SortOrder.ASC);
} else if (order.equals("desc")) {
searchSourceBuilder.sort("score", SortOrder.DESC);
searchSourceBuilder.sort("publishDate", SortOrder.DESC); searchSourceBuilder.sort("publishDate", SortOrder.DESC);
} else {
SortOrder sortOrder = SortOrder.DESC;
if (order.equals("asc")) {
sortOrder = SortOrder.ASC;
} }
switch (column) {
case "score":
searchSourceBuilder.sort("score", sortOrder);
searchSourceBuilder.sort("publishDate", sortOrder);
break; break;
case "publishDate": case "publishDate":
if (order.equals("desc")) { searchSourceBuilder.sort("publishDate", sortOrder);
searchSourceBuilder.sort("publishDate", SortOrder.DESC); searchSourceBuilder.sort("score", sortOrder);
searchSourceBuilder.sort("score", SortOrder.DESC);
} else if (order.equals("asc")) {
searchSourceBuilder.sort("publishDate", SortOrder.ASC);
searchSourceBuilder.sort("score", SortOrder.ASC);
}
break; break;
case "yqgzzk": case "yqgzzk":
searchSourceBuilder.sort("dataFrom", SortOrder.DESC); searchSourceBuilder.sort("dataFrom", SortOrder.DESC);
searchSourceBuilder.sort("topNum", SortOrder.DESC); searchSourceBuilder.sort("topNum", SortOrder.DESC);
searchSourceBuilder.sort("score", SortOrder.DESC); searchSourceBuilder.sort("score", SortOrder.DESC);
break; break;
case "title":
case "origin":
searchSourceBuilder.sort(column + ".keyword", sortOrder);
break;
case "summary":
case "content":
NestedSortBuilder nestedSortBuilder = new NestedSortBuilder("sortField");
nestedSortBuilder.setFilter(QueryBuilders.termQuery("sortField.fieldType", column));
FieldSortBuilder fieldSortBuilder = SortBuilders.fieldSort("sortField.fieldKeyword")
.order(sortOrder)
.sortMode(SortMode.MAX)
.setNestedSort(nestedSortBuilder);
searchSourceBuilder.sort(fieldSortBuilder);
break;
case "labels":
NestedSortBuilder nestedLabelsSortBuilder = new NestedSortBuilder("labels");
FieldSortBuilder labelSortBuilder = SortBuilders.fieldSort("labels.relationId")
.order(sortOrder)
.sortMode(SortMode.MAX)
.setNestedSort(nestedLabelsSortBuilder);
searchSourceBuilder.sort(labelSortBuilder);
break;
default: default:
searchSourceBuilder.sort("publishDate", SortOrder.DESC); NestedSortBuilder nestedWordSortBuilder = new NestedSortBuilder("sortField");
nestedWordSortBuilder.setFilter(QueryBuilders.termsQuery("sortField.fieldType", Arrays.asList(column.split(","))));
FieldSortBuilder fieldWordSortBuilder = SortBuilders.fieldSort("sortField.fieldLong")
.order(sortOrder)
.sortMode(SortMode.SUM)
.setNestedSort(nestedWordSortBuilder);
searchSourceBuilder.sort(fieldWordSortBuilder);
break; break;
} }
}
//构建es查询条件 //构建es查询条件
BoolQueryBuilder boolQuery = null; BoolQueryBuilder boolQuery;
//判断ids字段是否为空,若不为空,则表示按照id勾选 //判断ids字段是否为空,若不为空,则表示按照id勾选
List<String> ids = searchCondition.getIds(); List<String> ids = searchCondition.getIds();
if (CollectionUtils.isNotEmpty(ids)) { if (CollectionUtils.isNotEmpty(ids)) {
...@@ -839,7 +865,7 @@ public class EsService { ...@@ -839,7 +865,7 @@ public class EsService {
if (CollectionUtils.isNotEmpty(sourceIdList)) { if (CollectionUtils.isNotEmpty(sourceIdList)) {
boolQuery.must(QueryBuilders.termsQuery("sid.keyword", sourceIdList)); boolQuery.must(QueryBuilders.termsQuery("sid.keyword", sourceIdList));
} }
}else { } else {
boolQuery = buildQuery(searchCondition, subjectIdList); boolQuery = buildQuery(searchCondition, subjectIdList);
} }
searchSourceBuilder.query(boolQuery); searchSourceBuilder.query(boolQuery);
...@@ -1213,6 +1239,9 @@ public class EsService { ...@@ -1213,6 +1239,9 @@ public class EsService {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
String column = searchCondition.getColumn(); String column = searchCondition.getColumn();
String order = searchCondition.getOrder(); String order = searchCondition.getOrder();
if (StringUtils.isEmpty(column)) {
searchSourceBuilder.sort("publishDate", SortOrder.DESC);
} else {
switch (column) { switch (column) {
case "topNum": case "topNum":
searchSourceBuilder.sort("topNum", SortOrder.DESC); searchSourceBuilder.sort("topNum", SortOrder.DESC);
...@@ -1235,6 +1264,7 @@ public class EsService { ...@@ -1235,6 +1264,7 @@ public class EsService {
searchSourceBuilder.sort("publishDate", SortOrder.DESC); searchSourceBuilder.sort("publishDate", SortOrder.DESC);
break; break;
} }
}
String[] fetchFields = searchCondition.getFetchFields(); String[] fetchFields = searchCondition.getFetchFields();
if (fetchFields != null && fetchFields.length > 0) { if (fetchFields != null && fetchFields.length > 0) {
searchSourceBuilder.fetchSource(fetchFields, null); searchSourceBuilder.fetchSource(fetchFields, null);
...@@ -1914,6 +1944,76 @@ public class EsService { ...@@ -1914,6 +1944,76 @@ public class EsService {
return list; return list;
} }
public IPage<SpecialInformation> searchForReplaceList(InfoDataSearchCondition searchCondition) {
List<String> subjectIds = new ArrayList<>();
subjectIds.add(searchCondition.getSubjectId());
String minCreateTime = subjectService.getMinCreateTime(subjectIds);
String[] indexArr = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minCreateTime);
SearchRequest searchRequest = new SearchRequest(indexArr);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.trackTotalHits(true);
String[] fetchFields = new String[]{"id", "title", "origin", "content"};
searchSourceBuilder.fetchSource(fetchFields, null);
//设置高亮
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.preTags("<font style = 'color:red;'>");
highlightBuilder.postTags("</font>");
List<HighlightBuilder.Field> fields = highlightBuilder.fields();
fields.add(new HighlightBuilder.Field("title").numOfFragments(0));
fields.add(new HighlightBuilder.Field("origin").numOfFragments(0));
fields.add(new HighlightBuilder.Field("content").numOfFragments(0));
searchSourceBuilder.highlighter(highlightBuilder);
//设置分页参数
Integer pageNo = searchCondition.getPageNo();
Integer pageSize = searchCondition.getPageSize();
searchSourceBuilder.from((pageNo - 1) * pageSize);
searchSourceBuilder.size(pageSize);
BoolQueryBuilder boolQuery = buildQuery(searchCondition, subjectIds);
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
IPage<SpecialInformation> pageData = new Page<>();
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] searchHits = searchResponse.getHits().getHits();
List<SpecialInformation> list = new ArrayList<>();
for (SearchHit hit : searchHits) {
String queryInfo = hit.getSourceAsString();
SpecialInformation info = JSONUtil.toBean(queryInfo, SpecialInformation.class);
Map<String, HighlightField> highlightFields = hit.getHighlightFields();
if (highlightFields.containsKey("title")) {
Text[] titles = highlightFields.get("title").getFragments();
StringBuilder titleHighlight = new StringBuilder();
for (Text title : titles) {
titleHighlight.append(title.string());
}
info.setTitle(titleHighlight.toString());
}
if (highlightFields.containsKey("origin")) {
Text[] origins = highlightFields.get("origin").getFragments();
StringBuilder originHighlight = new StringBuilder();
for (Text origin : origins) {
originHighlight.append(origin.string());
}
info.setOrigin(originHighlight.toString());
}
if (highlightFields.containsKey("content")) {
Text[] contents = highlightFields.get("content").getFragments();
StringBuilder contentHighlight = new StringBuilder();
for (Text content : contents) {
contentHighlight.append(content.string());
}
info.setContent(contentHighlight.toString());
}
list.add(info);
}
pageData = new Page<>(pageNo, pageSize, searchResponse.getHits().getTotalHits().value);
pageData.setRecords(list);
} catch (IOException e) {
e.printStackTrace();
}
return pageData;
}
/** /**
* term方式聚合分组 * term方式聚合分组
* *
...@@ -2064,7 +2164,23 @@ public class EsService { ...@@ -2064,7 +2164,23 @@ public class EsService {
nestedBoolQueryBuilder.should(QueryBuilders.nestedQuery("labels", relationNameQuery, ScoreMode.None)); nestedBoolQueryBuilder.should(QueryBuilders.nestedQuery("labels", relationNameQuery, ScoreMode.None));
boolQuery.must(nestedBoolQueryBuilder); boolQuery.must(nestedBoolQueryBuilder);
} }
//资讯状态-研究中心(0-全部;1-模型推荐;2-精选;3-待定;4-移除),和checkStatus、deleteFlag、isFreeCheck互斥
Integer status = searchCondition.getStatus();
if (status != null) {
if (status == 1) {
boolQuery.must(QueryBuilders.termQuery("checkStatus", 0));
boolQuery.must(QueryBuilders.termQuery("isFreeCheck", 1));
boolQuery.mustNot(QueryBuilders.termQuery("deleteFlag", 1));
} else if (status == 2) {
boolQuery.must(QueryBuilders.termQuery("checkStatus", 1));
boolQuery.mustNot(QueryBuilders.termQuery("deleteFlag", 1));
} else if (status == 3) {
boolQuery.must(QueryBuilders.termQuery("checkStatus", 3));
boolQuery.mustNot(QueryBuilders.termQuery("deleteFlag", 1));
} else if (status == 4) {
boolQuery.must(QueryBuilders.termQuery("deleteFlag", 1));
}
} else {
Integer checkStatus = searchCondition.getCheckStatus(); Integer checkStatus = searchCondition.getCheckStatus();
Integer deleteFlag = searchCondition.getDeleteFlag(); Integer deleteFlag = searchCondition.getDeleteFlag();
if (checkStatus != null) { if (checkStatus != null) {
...@@ -2093,6 +2209,7 @@ public class EsService { ...@@ -2093,6 +2209,7 @@ public class EsService {
if (searchCondition.getIsFreeCheck() != null) { if (searchCondition.getIsFreeCheck() != null) {
boolQuery.must(QueryBuilders.termQuery("isFreeCheck", 1)); boolQuery.must(QueryBuilders.termQuery("isFreeCheck", 1));
} }
}
if (StringUtils.isNotBlank(searchCondition.getVideo())) { if (StringUtils.isNotBlank(searchCondition.getVideo())) {
boolQuery.must(QueryBuilders.matchQuery("type", "video")); boolQuery.must(QueryBuilders.matchQuery("type", "video"));
} else { } else {
...@@ -2115,6 +2232,19 @@ public class EsService { ...@@ -2115,6 +2232,19 @@ public class EsService {
if (maxScore != null) { if (maxScore != null) {
boolQuery.filter(QueryBuilders.rangeQuery("score").lte(maxScore)); boolQuery.filter(QueryBuilders.rangeQuery("score").lte(maxScore));
} }
//正文长度筛选
Integer minContentLength = searchCondition.getMinContentLength();
if (minContentLength != null) {
NestedQueryBuilder nestedQueryBuilder = QueryBuilders
.nestedQuery("sortField", QueryBuilders.rangeQuery("sortField.fieldLong").gte(minContentLength), ScoreMode.None);
boolQuery.filter(nestedQueryBuilder);
}
Integer maxContentLength = searchCondition.getMaxContentLength();
if (maxContentLength != null) {
NestedQueryBuilder nestedQueryBuilder = QueryBuilders
.nestedQuery("sortField", QueryBuilders.rangeQuery("sortField.fieldLong").lte(maxContentLength), ScoreMode.None);
boolQuery.filter(nestedQueryBuilder);
}
//时间过滤筛选-前端传参 //时间过滤筛选-前端传参
if(StringUtils.isEmpty(searchCondition.getStartTime()) && StringUtils.isEmpty(searchCondition.getEndTime())){ if(StringUtils.isEmpty(searchCondition.getStartTime()) && StringUtils.isEmpty(searchCondition.getEndTime())){
BoolQueryBuilder publishDateQueryBuilder = QueryBuilders.boolQuery(); BoolQueryBuilder publishDateQueryBuilder = QueryBuilders.boolQuery();
...@@ -2146,7 +2276,7 @@ public class EsService { ...@@ -2146,7 +2276,7 @@ public class EsService {
for (String items : split) { for (String items : split) {
List<String> ids = Arrays.asList(items.split(",")); List<String> ids = Arrays.asList(items.split(","));
NestedQueryBuilder nestedQueryBuilder = QueryBuilders NestedQueryBuilder nestedQueryBuilder = QueryBuilders
.nestedQuery("labels", QueryBuilders.termsQuery("labels.relationId",ids), ScoreMode.None); .nestedQuery("labels", QueryBuilders.termsQuery("labels.relationId", ids), ScoreMode.None);
boolQuery.must(nestedQueryBuilder); boolQuery.must(nestedQueryBuilder);
} }
} }
...@@ -2192,6 +2322,11 @@ public class EsService { ...@@ -2192,6 +2322,11 @@ public class EsService {
if (StringUtils.isNotBlank(sourceId)) { if (StringUtils.isNotBlank(sourceId)) {
boolQuery.must(QueryBuilders.termQuery("sid.keyword", sourceId)); boolQuery.must(QueryBuilders.termQuery("sid.keyword", sourceId));
} }
//待删除的关键词列表
if (CollectionUtils.isNotEmpty(searchCondition.getKeywordList())) {
TermsQueryBuilder sortFieldQuery = QueryBuilders.termsQuery("sortField.fieldType", searchCondition.getKeywordList());
boolQuery.must(QueryBuilders.nestedQuery("sortField", sortFieldQuery, ScoreMode.None));
}
return boolQuery; return boolQuery;
} }
...@@ -2608,6 +2743,7 @@ public class EsService { ...@@ -2608,6 +2743,7 @@ public class EsService {
/** /**
* 根据id查询标签列表 * 根据id查询标签列表
*
* @param index * @param index
* @param id * @param id
* @return * @return
...@@ -2615,7 +2751,7 @@ public class EsService { ...@@ -2615,7 +2751,7 @@ public class EsService {
public List<Label> getLabelsById(String index, String id) { public List<Label> getLabelsById(String index, String id) {
try { try {
GetRequest request = new GetRequest(index, id); GetRequest request = new GetRequest(index, id);
request.fetchSourceContext(new FetchSourceContext(true, new String[]{"labels","title"}, null)); request.fetchSourceContext(new FetchSourceContext(true, new String[]{"labels", "title"}, null));
request.realtime(true); request.realtime(true);
GetResponse documentFields = client.get(request, RequestOptions.DEFAULT); GetResponse documentFields = client.get(request, RequestOptions.DEFAULT);
String labelsStr = JSONArray.toJSONString(documentFields.getSourceAsMap().get("labels")); String labelsStr = JSONArray.toJSONString(documentFields.getSourceAsMap().get("labels"));
...@@ -2628,6 +2764,7 @@ public class EsService { ...@@ -2628,6 +2764,7 @@ public class EsService {
/** /**
* 根据id更新标签列表 * 根据id更新标签列表
*
* @param index * @param index
* @param id * @param id
* @param labels * @param labels
...@@ -2635,6 +2772,6 @@ public class EsService { ...@@ -2635,6 +2772,6 @@ public class EsService {
public void updateLabelsById(String index, String id, List<Label> labels) { public void updateLabelsById(String index, String id, List<Label> labels) {
JSONObject jsonObject = new JSONObject(); JSONObject jsonObject = new JSONObject();
jsonObject.put("labels", labels); jsonObject.put("labels", labels);
esOpUtil.docUpdateById(index,id,jsonObject.toString()); esOpUtil.docUpdateById(index, id, jsonObject.toString());
} }
} }
package com.zzsn.event.feign.api;
import com.zzsn.event.vo.InfoDataSearchCondition;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.http.HttpHeaders;
import org.springframework.web.bind.annotation.*;
//feign调用
@FeignClient(value = "service-project")
public interface RemoteProjectService {
/**
* 清除专题缓存
*
* @param
* @author lkg
* @date 2025/2/7
*/
@PostMapping("/manage/subjectModel/doTaskByFlow")
String doTaskByFlow(@RequestBody InfoDataSearchCondition task);
}
...@@ -3,6 +3,9 @@ package com.zzsn.event.mapper; ...@@ -3,6 +3,9 @@ package com.zzsn.event.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.event.entity.ClbSubjectHitwordsMap; import com.zzsn.event.entity.ClbSubjectHitwordsMap;
import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
/** /**
* <p> * <p>
...@@ -15,4 +18,5 @@ import org.apache.ibatis.annotations.Mapper; ...@@ -15,4 +18,5 @@ import org.apache.ibatis.annotations.Mapper;
@Mapper @Mapper
public interface ClbSubjectHitwordsMapMapper extends BaseMapper<ClbSubjectHitwordsMap> { public interface ClbSubjectHitwordsMapMapper extends BaseMapper<ClbSubjectHitwordsMap> {
List<String> detail(@Param("flowId") String flowId, @Param("subjectId") String subjectId);
} }
...@@ -15,4 +15,9 @@ ...@@ -15,4 +15,9 @@
<result column="del_flag" property="delFlag" /> <result column="del_flag" property="delFlag" />
</resultMap> </resultMap>
<select id="detail" resultType="String">
SELECT id FROM clb_subject_model_detail_task WHERE subject_id = #{subjectId} AND flow_id = #{flowId} AND task_status in (0,1);
</select>
</mapper> </mapper>
...@@ -45,7 +45,7 @@ ...@@ -45,7 +45,7 @@
</select> </select>
<select id="pageList_new" resultType="com.zzsn.event.vo.SubjectPageVO"> <select id="pageList_new" resultType="com.zzsn.event.vo.SubjectPageVO">
SELECT d.id,d.subject_name,d.status, SELECT d.id,d.subject_name,d.status,d.data_scope,
s.total_num,s.un_check_num,s.bind_source_num,s.bind_keyword_num,s.latest_data_date, s.total_num,s.un_check_num,s.bind_source_num,s.bind_keyword_num,s.latest_data_date,
c.type_name as subjectTypeName, g.project_name as projectName c.type_name as subjectTypeName, g.project_name as projectName
from subject d from subject d
......
...@@ -2,6 +2,7 @@ package com.zzsn.event.service; ...@@ -2,6 +2,7 @@ package com.zzsn.event.service;
import com.baomidou.mybatisplus.extension.service.IService; import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.event.entity.ClbSubjectHitwordsMap; import com.zzsn.event.entity.ClbSubjectHitwordsMap;
import com.zzsn.event.vo.InfoDataSearchCondition;
import java.util.List; import java.util.List;
...@@ -16,4 +17,6 @@ import java.util.List; ...@@ -16,4 +17,6 @@ import java.util.List;
public interface ClbSubjectHitwordsMapService extends IService<ClbSubjectHitwordsMap> { public interface ClbSubjectHitwordsMapService extends IService<ClbSubjectHitwordsMap> {
void delEs(List<ClbSubjectHitwordsMap> clbSubjectHitwordsMaps); void delEs(List<ClbSubjectHitwordsMap> clbSubjectHitwordsMaps);
void stopTask(InfoDataSearchCondition searchCondition);
} }
...@@ -261,4 +261,85 @@ public interface InformationService { ...@@ -261,4 +261,85 @@ public interface InformationService {
List<SpecialInformation> informationAllList(InfoDataSearchCondition searchCondition); List<SpecialInformation> informationAllList(InfoDataSearchCondition searchCondition);
void removeLabelById(String index, String id, String relationId); void removeLabelById(String index, String id, String relationId);
/**
* 删除关键词标签
*
* @param searchCondition 检索条件
* @author lkg
* @date 2025/4/22
*/
void removeWordLabel(InfoDataSearchCondition searchCondition);
/**
* 批量添加到精选
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void addToFavorites(InfoDataSearchCondition searchCondition);
/**
* 批量添加到精选
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void addToPend(InfoDataSearchCondition searchCondition);
/**
* 批量添加到精选
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void addToRemove(InfoDataSearchCondition searchCondition);
/**
* 初始化数据(数据状态全部重置为待审核、未删除)
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void initialData(InfoDataSearchCondition searchCondition);
/**
* 查找替换信息列表
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/24
*/
IPage<SpecialInformation> searchForReplaceList(InfoDataSearchCondition searchCondition);
/**
* 字符串替换
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void charReplace(InfoDataSearchCondition searchCondition);
/**
* 打标签
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void markTag(InfoDataSearchCondition searchCondition);
/**
* 打标签
*
* @param searchCondition 资讯检索条件
* @author lkg
* @date 2025/4/23
*/
void removeTag(InfoDataSearchCondition searchCondition);
} }
...@@ -202,4 +202,7 @@ public interface SubjectService extends IService<Subject> { ...@@ -202,4 +202,7 @@ public interface SubjectService extends IService<Subject> {
void deleteBindNew(SubjectPage subjectPage); void deleteBindNew(SubjectPage subjectPage);
String getMinCreateTime(List<String> subjectIdList); String getMinCreateTime(List<String> subjectIdList);
Subject subjectUpdateScope(String subjectId, String type, String scop);
} }
...@@ -2,14 +2,23 @@ package com.zzsn.event.service.impl; ...@@ -2,14 +2,23 @@ package com.zzsn.event.service.impl;
import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.event.constant.Constants;
import com.zzsn.event.entity.ClbSubjectHitwordsMap; import com.zzsn.event.entity.ClbSubjectHitwordsMap;
import com.zzsn.event.mapper.ClbSubjectHitwordsMapMapper; import com.zzsn.event.mapper.ClbSubjectHitwordsMapMapper;
import com.zzsn.event.service.ClbSubjectHitwordsMapService; import com.zzsn.event.service.ClbSubjectHitwordsMapService;
import com.zzsn.event.service.InformationService;
import com.zzsn.event.util.RedisUtil;
import com.zzsn.event.vo.InfoDataSearchCondition;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils; import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.CompletableFuture;
/** /**
* <p> * <p>
...@@ -20,14 +29,52 @@ import java.util.List; ...@@ -20,14 +29,52 @@ import java.util.List;
* @since 2025-04-23 * @since 2025-04-23
*/ */
@Service @Service
@Slf4j
public class ClbSubjectHitwordsMapServiceImpl extends ServiceImpl<ClbSubjectHitwordsMapMapper, ClbSubjectHitwordsMap> implements ClbSubjectHitwordsMapService { public class ClbSubjectHitwordsMapServiceImpl extends ServiceImpl<ClbSubjectHitwordsMapMapper, ClbSubjectHitwordsMap> implements ClbSubjectHitwordsMapService {
@Autowired
private InformationService informationService;
@Autowired
RedisUtil redisUtil;
@Override @Override
public void delEs(List<ClbSubjectHitwordsMap> clbSubjectHitwordsMaps) { public void delEs(List<ClbSubjectHitwordsMap> clbSubjectHitwordsMaps) {
//TODO if (CollectionUtils.isEmpty(clbSubjectHitwordsMaps)) {
return;
}
String subjectId = clbSubjectHitwordsMaps.get(0).getSubjectId();
InfoDataSearchCondition searchCondition = new InfoDataSearchCondition();
searchCondition.setSubjectId(subjectId);
searchCondition.setStatus(0);
searchCondition.setKeywordList(new ArrayList<>());
if (CollectionUtil.isNotEmpty(clbSubjectHitwordsMaps)) { if (CollectionUtil.isNotEmpty(clbSubjectHitwordsMaps)) {
for (ClbSubjectHitwordsMap clbSubjectHitwordsMap : clbSubjectHitwordsMaps) {
if (StrUtil.isNotBlank(clbSubjectHitwordsMap.getWordName())) {
searchCondition.getKeywordList().add(clbSubjectHitwordsMap.getWordName());
}
}
} }
if (CollectionUtil.isEmpty(searchCondition.getKeywordList())) {
log.info("命中词为空,不删除es");
return;
}
log.info("delEs:{}",searchCondition);
CompletableFuture.runAsync(()-> informationService.removeWordLabel(searchCondition));
}
@Override
public void stopTask(InfoDataSearchCondition searchCondition) {
String flowId = searchCondition.getFlowId();
String subjectId = searchCondition.getSubjectId();
List<String> taskIds = baseMapper.detail(flowId,subjectId);
if (CollectionUtil.isNotEmpty(taskIds)) {
taskIds.forEach(taskId -> {
redisUtil.hset(Constants.SUBJECT_MODEL_DETAIL_BREAK+subjectId,taskId,"1");
});
}
} }
} }
...@@ -30,6 +30,10 @@ import lombok.extern.slf4j.Slf4j; ...@@ -30,6 +30,10 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermsQueryBuilder;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.springframework.beans.BeanUtils; import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
...@@ -430,6 +434,7 @@ public class InformationServiceImpl implements InformationService { ...@@ -430,6 +434,7 @@ public class InformationServiceImpl implements InformationService {
labels.add(dataSet); labels.add(dataSet);
} }
} else { } else {
labels = new ArrayList<>();
labels.add(dataSet); labels.add(dataSet);
} }
information.setLabels(labels); information.setLabels(labels);
...@@ -659,9 +664,9 @@ public class InformationServiceImpl implements InformationService { ...@@ -659,9 +664,9 @@ public class InformationServiceImpl implements InformationService {
List<DictVO> boundList = dataBindLabelFrom.getBindList(); List<DictVO> boundList = dataBindLabelFrom.getBindList();
List<String> labelMarks = new ArrayList<>(); List<String> labelMarks = new ArrayList<>();
for (DictVO dictVO : boundList) { for (DictVO dictVO : boundList) {
if(StringUtils.isNotEmpty(dictVO.getCode())){ if (StringUtils.isNotEmpty(dictVO.getCode())) {
labelMarks.add(dictVO.getCode()); labelMarks.add(dictVO.getCode());
}else{ } else {
labelMarks.add(dictVO.getLabelMark()); labelMarks.add(dictVO.getLabelMark());
} }
} }
...@@ -1139,12 +1144,211 @@ public class InformationServiceImpl implements InformationService { ...@@ -1139,12 +1144,211 @@ public class InformationServiceImpl implements InformationService {
@Override @Override
public void removeLabelById(String index, String id, String relationId) { public void removeLabelById(String index, String id, String relationId) {
List<Label> labels = esService.getLabelsById(index,id); List<Label> labels = esService.getLabelsById(index, id);
if(labels == null){ if (labels == null) {
return; return;
} }
labels.removeIf(e -> e.getRelationId().equals(relationId)); labels.removeIf(e -> e.getRelationId().equals(relationId));
esService.updateLabelsById(index,id,labels); esService.updateLabelsById(index, id, labels);
}
@Override
public void removeWordLabel(InfoDataSearchCondition searchCondition) {
List<String> subjectIdList = new ArrayList<>();
subjectIdList.add(searchCondition.getSubjectId());
String minCreateTime = subjectService.getMinCreateTime(subjectIdList);
String[] indexArr = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minCreateTime);
BoolQueryBuilder buildQuery = esService.buildQuery(searchCondition, subjectIdList);
buildQuery.must(QueryBuilders.nestedQuery("sortField",
QueryBuilders.termsQuery("sortField.fieldType", searchCondition.getKeywordList()),
ScoreMode.None));
List<String> keywordList = searchCondition.getKeywordList();
for (String word : keywordList) {
Map<String, Object> deleteParams = new HashMap<>();
deleteParams.put("fieldType", word);
long start = System.currentTimeMillis();
esOpUtil.batchNestedDeleteScript(indexArr, buildQuery, deleteParams, "sortField");
log.info("删除关键词-{},耗时:{}",word, System.currentTimeMillis()-start);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
@Override
public void addToFavorites(InfoDataSearchCondition searchCondition) {
List<String> subjectIdList = new ArrayList<>();
subjectIdList.add(searchCondition.getSubjectId());
String minCreateTime = subjectService.getMinCreateTime(subjectIdList);
String[] indexArr = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minCreateTime);
List<String> ids = searchCondition.getIds();
BoolQueryBuilder buildQuery;
if (CollectionUtils.isEmpty(ids)) {
buildQuery = esService.buildQuery(searchCondition, subjectIdList);
} else {
buildQuery = QueryBuilders.boolQuery().must(QueryBuilders.termsQuery("id", ids));
}
Map<String, Object> modifyParams = new HashMap<>();
modifyParams.put("checkStatus", 1);
modifyParams.put("deleteFlag",0);
long start = System.currentTimeMillis();
esOpUtil.batchUpdateScript(indexArr, buildQuery, modifyParams);
log.info("添加到精选,耗时:{}", System.currentTimeMillis()-start);
}
@Override
public void addToPend(InfoDataSearchCondition searchCondition) {
List<String> subjectIdList = new ArrayList<>();
subjectIdList.add(searchCondition.getSubjectId());
String minCreateTime = subjectService.getMinCreateTime(subjectIdList);
String[] indexArr = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minCreateTime);
List<String> ids = searchCondition.getIds();
BoolQueryBuilder buildQuery;
if (CollectionUtils.isEmpty(ids)) {
buildQuery = esService.buildQuery(searchCondition, subjectIdList);
} else {
buildQuery = QueryBuilders.boolQuery().must(QueryBuilders.termsQuery("id", ids));
}
Map<String, Object> modifyParams = new HashMap<>();
modifyParams.put("checkStatus",3);
modifyParams.put("deleteFlag",0);
long start = System.currentTimeMillis();
esOpUtil.batchUpdateScript(indexArr, buildQuery, modifyParams);
log.info("添加到待定,耗时:{}", System.currentTimeMillis()-start);
}
@Override
public void addToRemove(InfoDataSearchCondition searchCondition) {
List<String> subjectIdList = new ArrayList<>();
subjectIdList.add(searchCondition.getSubjectId());
String minCreateTime = subjectService.getMinCreateTime(subjectIdList);
String[] indexArr = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minCreateTime);
List<String> ids = searchCondition.getIds();
BoolQueryBuilder buildQuery;
if (CollectionUtils.isEmpty(ids)) {
buildQuery = esService.buildQuery(searchCondition, subjectIdList);
} else {
buildQuery = QueryBuilders.boolQuery().must(QueryBuilders.termsQuery("id", ids));
}
Map<String, Object> modifyParams = new HashMap<>();
modifyParams.put("deleteFlag",1);
long start = System.currentTimeMillis();
esOpUtil.batchUpdateScript(indexArr, buildQuery, modifyParams);
log.info("添加到删除,耗时:{}", System.currentTimeMillis()-start);
}
@Override
public void initialData(InfoDataSearchCondition searchCondition) {
List<String> subjectIdList = new ArrayList<>();
subjectIdList.add(searchCondition.getSubjectId());
String minCreateTime = subjectService.getMinCreateTime(subjectIdList);
String[] indexArr = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minCreateTime);
BoolQueryBuilder buildQuery = esService.buildQuery(searchCondition, subjectIdList);
Map<String, Object> modifyParams = new HashMap<>();
modifyParams.put("checkStatus",0);
modifyParams.put("deleteFlag",0);
long start = System.currentTimeMillis();
esOpUtil.batchUpdateScript(indexArr, buildQuery, modifyParams);
log.info("初始化数据,耗时:{}", System.currentTimeMillis()-start);
}
@Override
public IPage<SpecialInformation> searchForReplaceList(InfoDataSearchCondition searchCondition) {
return esService.searchForReplaceList(searchCondition);
}
@Override
public void charReplace(InfoDataSearchCondition searchCondition) {
SearchWordVO searchWordVO = searchCondition.getSearchWordList().get(0);
Integer searchScope = searchWordVO.getSearchScope();
String oldWord = searchWordVO.getSearchInfo();
String newWord = searchCondition.getReplaceWord();
if(StringUtils.isEmpty(newWord)) {
newWord = "";
}
List<String> modifyFields = new ArrayList<>();
if (searchScope == 1) {
modifyFields.add("title");
} else if (searchScope == 2) {
modifyFields.add("content");
modifyFields.add("contentWithTag");
} else if (searchScope == 3) {
modifyFields.add("title");
modifyFields.add("content");
modifyFields.add("contentWithTag");
} else if (searchScope == 4) {
modifyFields.add("origin");
}
List<String> subjectIdList = new ArrayList<>();
subjectIdList.add(searchCondition.getSubjectId());
String minCreateTime = subjectService.getMinCreateTime(subjectIdList);
String[] indexArr = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minCreateTime);
BoolQueryBuilder buildQuery = esService.buildQuery(searchCondition, subjectIdList);
long start = System.currentTimeMillis();
esOpUtil.batchReplaceScript(indexArr, buildQuery, modifyFields, oldWord, newWord);
log.info("字符串替换-{},耗时:{}", oldWord,System.currentTimeMillis()-start);
}
@Override
public void markTag(InfoDataSearchCondition searchCondition) {
List<String> subjectIdList = new ArrayList<>();
subjectIdList.add(searchCondition.getSubjectId());
String minCreateTime = subjectService.getMinCreateTime(subjectIdList);
String[] indexArr = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minCreateTime);
List<String> ids = searchCondition.getIds();
BoolQueryBuilder buildQuery;
if (CollectionUtils.isEmpty(ids)) {
buildQuery = esService.buildQuery(searchCondition, subjectIdList);
} else {
buildQuery = QueryBuilders.boolQuery().must(QueryBuilders.termsQuery("id", ids));
}
Map<String, Object> addParams = new HashMap<>();
for (Label markTag : searchCondition.getMarkTags()) {
Map<String, String> addTag = ObjectUtil.objectToMap(markTag);
addParams.put("labels",addTag);
long start = System.currentTimeMillis();
esOpUtil.batchNestedAddScript(indexArr, buildQuery, addParams,"labels","relationId");
log.info("打标-{},耗时:{}", markTag.getRelationName(),System.currentTimeMillis()-start);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
@Override
public void removeTag(InfoDataSearchCondition searchCondition) {
List<String> subjectIdList = new ArrayList<>();
subjectIdList.add(searchCondition.getSubjectId());
String minCreateTime = subjectService.getMinCreateTime(subjectIdList);
String[] indexArr = EsIndexUtil.getIndexIntervalYearStr(Constants.SUBJECT_INDEX, minCreateTime);
List<String> ids = searchCondition.getIds();
BoolQueryBuilder buildQuery;
if (CollectionUtils.isEmpty(ids)) {
buildQuery = esService.buildQuery(searchCondition, subjectIdList);
} else {
buildQuery = QueryBuilders.boolQuery().must(QueryBuilders.termsQuery("id", ids));
}
List<Label> markTags = searchCondition.getMarkTags();
List<String> tagIds = markTags.stream().map(Label::getRelationId).collect(Collectors.toList());
buildQuery.must(QueryBuilders.nestedQuery("labels",
QueryBuilders.termsQuery("labels.relationId", tagIds),
ScoreMode.None));
for (Label markTag : markTags) {
Map<String, Object> deleteParams = new HashMap<>();
deleteParams.put("relationId", markTag.getRelationId());
long start = System.currentTimeMillis();
esOpUtil.batchNestedDeleteScript(indexArr, buildQuery, deleteParams, "labels");
log.info("删除标签-{},耗时:{}", markTag.getRelationName(),System.currentTimeMillis()-start);
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
} }
private void formatLabel(List<LabelModelVo> labelModelVos, DisplayInfo info) { private void formatLabel(List<LabelModelVo> labelModelVos, DisplayInfo info) {
......
...@@ -184,6 +184,16 @@ public class SubjectServiceImpl extends ServiceImpl<SubjectMapper, Subject> impl ...@@ -184,6 +184,16 @@ public class SubjectServiceImpl extends ServiceImpl<SubjectMapper, Subject> impl
Map<String, List<ClbModelArrange>> collect = clbModelArranges.stream().collect(Collectors.groupingBy(ClbModelArrange::getSubjectId)); Map<String, List<ClbModelArrange>> collect = clbModelArranges.stream().collect(Collectors.groupingBy(ClbModelArrange::getSubjectId));
records.forEach(e -> e.setClbModelArranges(collect.get(e.getId()))); records.forEach(e -> e.setClbModelArranges(collect.get(e.getId())));
} }
Optional<SubjectPageVO> any = records.stream().filter(f -> StrUtil.isNotBlank(f.getDataScope())).findAny();
if (any.isPresent()) {
int count = infoSourceService.count();
records.forEach(e -> {
if (StrUtil.isNotBlank(e.getDataScope()) && e.getDataScope().contains("1")) {
e.setBindSourceNum(count);
}
});
}
} }
return pageList; return pageList;
} }
...@@ -905,6 +915,47 @@ public class SubjectServiceImpl extends ServiceImpl<SubjectMapper, Subject> impl ...@@ -905,6 +915,47 @@ public class SubjectServiceImpl extends ServiceImpl<SubjectMapper, Subject> impl
return baseMapper.getMinCreateTime(subjectIdList); return baseMapper.getMinCreateTime(subjectIdList);
} }
@Override
public Subject subjectUpdateScope(String subjectId, String type, String scop) {
Subject byId = super.getById(subjectId);
if (ObjectUtil.isNotNull(byId)){
String dataScope = byId.getDataScope();
if (StrUtil.isNotBlank(dataScope)){
List<String> scopeList = CollectionUtil.newArrayList(dataScope.split(","));
if (StrUtil.equals(type,"1") && "1".equals(scop)){
scopeList.add("1");
}else if (StrUtil.equals(type,"1") && "0".equals(scop)){
scopeList.remove("1");
}
if (StrUtil.equals(type,"2") && "1".equals(scop)){
scopeList.add("2");
}else if (StrUtil.equals(type,"2") && "0".equals(scop)){
scopeList.remove("2");
}
if (StrUtil.equals(type,"3") && "1".equals(scop)){
scopeList.add("3");
}else if (StrUtil.equals(type,"3") && "0".equals(scop)){
scopeList.remove("3");
}
if (CollectionUtil.isNotEmpty(scopeList)) {
scopeList = scopeList.stream().distinct().collect(Collectors.toList());
String join = StrUtil.join(",", scopeList);
byId.setDataScope(join);
}else {
byId.setDataScope(null);
}
this.updateById(byId);
}else {
if ("1".equals(scop)) {
byId.setDataScope(scop);
}
this.updateById(byId);
}
}
return byId;
}
private void unBindInfoSourceGroup(SubjectPage subjectPage) { private void unBindInfoSourceGroup(SubjectPage subjectPage) {
String sourceBindType = subjectPage.getSourceBindType(); String sourceBindType = subjectPage.getSourceBindType();
......
...@@ -41,6 +41,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; ...@@ -41,6 +41,7 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.*;
import org.elasticsearch.index.reindex.BulkByScrollResponse; import org.elasticsearch.index.reindex.BulkByScrollResponse;
import org.elasticsearch.index.reindex.BulkByScrollTask;
import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.index.reindex.UpdateByQueryRequest; import org.elasticsearch.index.reindex.UpdateByQueryRequest;
import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestStatus;
...@@ -60,9 +61,11 @@ import org.elasticsearch.search.sort.SortOrder; ...@@ -60,9 +61,11 @@ import org.elasticsearch.search.sort.SortOrder;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import javax.annotation.Resource; import javax.annotation.Resource;
import javax.validation.constraints.NotNull;
import java.io.IOException; import java.io.IOException;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
/** /**
* Es操作相关工具栏 * Es操作相关工具栏
...@@ -498,7 +501,7 @@ public class EsOpUtil<T> { ...@@ -498,7 +501,7 @@ public class EsOpUtil<T> {
} }
public void docUpdateBulk(String index, List<SpecialInformation> dataList){ public void docUpdateBulk(String index, List<SpecialInformation> dataList) {
BulkRequest bulkRequest = new BulkRequest(); BulkRequest bulkRequest = new BulkRequest();
for (SpecialInformation information : dataList) { for (SpecialInformation information : dataList) {
UpdateRequest request = new UpdateRequest(index, information.getId()); UpdateRequest request = new UpdateRequest(index, information.getId());
...@@ -534,9 +537,9 @@ public class EsOpUtil<T> { ...@@ -534,9 +537,9 @@ public class EsOpUtil<T> {
* @author lkg * @author lkg
* @date 2024/12/19 * @date 2024/12/19
*/ */
public void docRemoveBulk(List<Map<String, Object>> removeList){ public void docRemoveBulk(List<Map<String, Object>> removeList) {
BulkRequest bulkRequest = new BulkRequest(); BulkRequest bulkRequest = new BulkRequest();
for (Map<String,Object> m : removeList) { for (Map<String, Object> m : removeList) {
String index = m.get("index").toString(); String index = m.get("index").toString();
String id = m.get("id").toString(); String id = m.get("id").toString();
DeleteRequest deleteRequest = new DeleteRequest(index, id); DeleteRequest deleteRequest = new DeleteRequest(index, id);
...@@ -546,7 +549,7 @@ public class EsOpUtil<T> { ...@@ -546,7 +549,7 @@ public class EsOpUtil<T> {
try { try {
client.bulk(bulkRequest, RequestOptions.DEFAULT); client.bulk(bulkRequest, RequestOptions.DEFAULT);
} catch (Exception e) { } catch (Exception e) {
log.error("数据删除失败!",e); log.error("数据删除失败!", e);
} }
} }
...@@ -557,7 +560,7 @@ public class EsOpUtil<T> { ...@@ -557,7 +560,7 @@ public class EsOpUtil<T> {
* @author lkg * @author lkg
* @date 2024/12/19 * @date 2024/12/19
*/ */
public void docRemoveBulk(Map<String, List<SpecialInformation>> removeMap){ public void docRemoveBulk(Map<String, List<SpecialInformation>> removeMap) {
BulkRequest bulkRequest = new BulkRequest(); BulkRequest bulkRequest = new BulkRequest();
for (Map.Entry<String, List<SpecialInformation>> entry : removeMap.entrySet()) { for (Map.Entry<String, List<SpecialInformation>> entry : removeMap.entrySet()) {
String index = entry.getKey(); String index = entry.getKey();
...@@ -570,7 +573,7 @@ public class EsOpUtil<T> { ...@@ -570,7 +573,7 @@ public class EsOpUtil<T> {
try { try {
client.bulk(bulkRequest, RequestOptions.DEFAULT); client.bulk(bulkRequest, RequestOptions.DEFAULT);
} catch (Exception e) { } catch (Exception e) {
log.error("数据删除失败!",e); log.error("数据删除失败!", e);
} }
} }
...@@ -667,6 +670,8 @@ public class EsOpUtil<T> { ...@@ -667,6 +670,8 @@ public class EsOpUtil<T> {
} }
/** /**
* 批量更新(参数deleteParams 中的key要和es中的字段保持一致)
* <p>
* 批量更新操作,根据指定的查询条件和多个字段的映射关系,更新符合条件的文档的多个字段的值。 * 批量更新操作,根据指定的查询条件和多个字段的映射关系,更新符合条件的文档的多个字段的值。
* 方法名:updataMoreColumBatchByQuery,表示批量更新文档的多个字段的方法。 * 方法名:updataMoreColumBatchByQuery,表示批量更新文档的多个字段的方法。
* 参数:index,指定要更新的索引名称。 * 参数:index,指定要更新的索引名称。
...@@ -679,22 +684,142 @@ public class EsOpUtil<T> { ...@@ -679,22 +684,142 @@ public class EsOpUtil<T> {
* 设置更新脚本,使用setScript方法,传入Script对象。脚本使用painless语言,通过ctx._source.字段名 = 字段值的方式来更新文档的多个字段的值。 * 设置更新脚本,使用setScript方法,传入Script对象。脚本使用painless语言,通过ctx._source.字段名 = 字段值的方式来更新文档的多个字段的值。
* 调用client.updateByQuery方法,传入UpdateByQueryRequest对象和默认的RequestOptions,执行批量更新操作,并返回BulkByScrollResponse对象。 * 调用client.updateByQuery方法,传入UpdateByQueryRequest对象和默认的RequestOptions,执行批量更新操作,并返回BulkByScrollResponse对象。
* Collections.emptyMap()是setScript脚本的参数,此方法没有设置其他参数,使用一个空的map * Collections.emptyMap()是setScript脚本的参数,此方法没有设置其他参数,使用一个空的map
*
* @param index 索引
* @param boolQuery es查询条件
* @param modifyParams 更新参数
* @author lkg
* @date 2025/4/22
*/ */
public void updateMoreColumBatchByQuery(String index, BoolQueryBuilder boolQuery, Map<String, String> modifyColumValue) throws IOException, InterruptedException { public void batchUpdateScript(String[] index, BoolQueryBuilder boolQuery, Map<String, Object> modifyParams) {
UpdateByQueryRequest request = new UpdateByQueryRequest(index); UpdateByQueryRequest request = new UpdateByQueryRequest(index);
request.setQuery(boolQuery); request.setQuery(boolQuery);
request.setScript(new Script(ScriptType.INLINE, "painless", getIdOrCode(modifyColumValue), Collections.emptyMap())); StringBuffer script = new StringBuffer();
modifyParams.forEach((colum, value) -> {
if (value instanceof String) {
script.append("ctx._source.").append(colum).append(" = '").append(value).append("';");
} else {
script.append("ctx._source.").append(colum).append(" = ").append(value).append(";");
}
});
request.setScript(new Script(ScriptType.INLINE, "painless", script.toString(), Collections.emptyMap()));
//可跳过版本冲突的文档
request.setConflicts("proceed");
try {
BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT); BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
long versionConflicts = response.getVersionConflicts();
long updated = response.getUpdated(); long updated = response.getUpdated();
log.info("更新条数{}", updated); log.info("更新条数:{},冲突条数{}", updated,versionConflicts);
} catch (IOException e) {
e.printStackTrace();
}
} }
private String getIdOrCode(Map<String, String> columValue) { /**
StringBuffer script = new StringBuffer(); * 批量替换字符串(更新的一种)
columValue.forEach((colum, value) -> { *
script.append("ctx._source.").append(colum).append(" = '").append(value).append("';"); * @param index 索引
* @param boolQuery es查询条件
* @param modifyField 编辑的字段集合
* @param oldValue 旧字符串
* @param newValue 新字符串
* @author lkg
* @date 2025/4/22
*/
public void batchReplaceScript(String[] index, BoolQueryBuilder boolQuery, List<String> modifyField, String oldValue, @NotNull String newValue) {
UpdateByQueryRequest request = new UpdateByQueryRequest(index);
request.setQuery(boolQuery);
String script = modifyField.stream()
.map(field -> String.format(
"if(ctx._source.%s != null) { ctx._source.%s = ctx._source.%s.replace('%s', '%s'); }",
field, field, field, oldValue, newValue))
.collect(Collectors.joining(""));
request.setScript(new Script(ScriptType.INLINE, "painless", script, Collections.emptyMap()));
//可跳过版本冲突的文档
request.setConflicts("proceed");
try {
BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
long versionConflicts = response.getVersionConflicts();
long updated = response.getUpdated();
log.info("更新条数:{},冲突条数{}", updated,versionConflicts);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 批量删除nest字段中的数据
* <p>
* 参数deleteParams 中的key要和es中的字段保持一致
*
* @param index 索引
* @param boolQuery es查询条件
* @param deleteParams 删除条件
* @param nestField nest字段
* @author lkg
* @date 2025/4/22
*/
public void batchNestedDeleteScript(String[] index, BoolQueryBuilder boolQuery, Map<String, Object> deleteParams, String nestField) {
UpdateByQueryRequest request = new UpdateByQueryRequest(index);
request.setQuery(boolQuery);
//Painless 脚本:从 nested 数组中删除匹配项
String scriptSource = "ctx._source." + nestField + ".removeIf(item -> ";
StringBuilder paramString = new StringBuilder();
deleteParams.forEach((colum, value) -> {
if (value instanceof String) {
paramString.append(" && ").append("item.").append(colum).append(" == params.").append(colum);
} else {
paramString.append(" && ").append("item.").append(colum).append(" == params.").append(colum);
}
}); });
return script.toString(); String substring = paramString.substring(4);
scriptSource = scriptSource + substring + ")";
Script script = new Script(ScriptType.INLINE, "painless", scriptSource, deleteParams);
request.setScript(script);
//可跳过版本冲突的文档
request.setConflicts("proceed");
try {
BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
long versionConflicts = response.getVersionConflicts();
long updated = response.getUpdated();
log.info("更新条数:{},冲突条数{}", updated,versionConflicts);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 批量新增nest字段中的数据
* <p>
* 参数addParams 中的key要和es中的字段保持一致
*
* @param index 索引
* @param boolQuery es查询条件
* @param addParams 添加的数据
* @param nestField nest字段
* @param uniqueField 唯一字段
* @author lkg
* @date 2025/4/22
*/
public void batchNestedAddScript(String[] index, BoolQueryBuilder boolQuery, Map<String, Object> addParams, String nestField, String uniqueField) {
UpdateByQueryRequest request = new UpdateByQueryRequest(index);
request.setQuery(boolQuery);
//Painless 脚本:从 nested 数组中删除匹配项
String scriptString = "if(ctx._source.NEST_FIELD==null){List ls=new ArrayList();ls.add(params.NEST_FIELD);ctx._source.NEST_FIELD=ls;}else{List olds=new ArrayList();ctx._source.NEST_FIELD.forEach(item->{olds.add(item.UNIQUE_FIELD);});if(!olds.contains(params.NEST_FIELD.UNIQUE_FIELD)){ctx._source.NEST_FIELD.add(params.NEST_FIELD);}}";
String scriptSource = scriptString.replace("NEST_FIELD", nestField).replace("UNIQUE_FIELD", uniqueField);
Script script = new Script(ScriptType.INLINE, "painless", scriptSource, addParams);
request.setScript(script);
//可跳过版本冲突的文档
request.setConflicts("proceed");
try {
BulkByScrollResponse response = client.updateByQuery(request, RequestOptions.DEFAULT);
long versionConflicts = response.getVersionConflicts();
long updated = response.getUpdated();
log.info("更新条数:{},冲突条数{}", updated,versionConflicts);
} catch (IOException e) {
e.printStackTrace();
}
} }
...@@ -762,13 +887,13 @@ public class EsOpUtil<T> { ...@@ -762,13 +887,13 @@ public class EsOpUtil<T> {
} }
// 执行更新请求 // 执行更新请求
try { try {
UpdateResponse response = client.update(createUpdateRequest(index, id, args,true), RequestOptions.DEFAULT); UpdateResponse response = client.update(createUpdateRequest(index, id, args, true), RequestOptions.DEFAULT);
RestStatus status = response.status(); RestStatus status = response.status();
if (status.getStatus() != 200) { if (status.getStatus() != 200) {
log.info("{},更新失败",id); log.info("{},更新失败", id);
} }
} catch (IOException e) { } catch (IOException e) {
log.info("{},更新异常",id); log.info("{},更新异常", id);
} }
} }
...@@ -788,7 +913,7 @@ public class EsOpUtil<T> { ...@@ -788,7 +913,7 @@ public class EsOpUtil<T> {
BulkRequest bulkRequest = new BulkRequest(); BulkRequest bulkRequest = new BulkRequest();
args.forEach((id, args1) -> { args.forEach((id, args1) -> {
try { try {
bulkRequest.add(createUpdateRequest(index, id, args1,false)); bulkRequest.add(createUpdateRequest(index, id, args1, false));
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
...@@ -817,7 +942,7 @@ public class EsOpUtil<T> { ...@@ -817,7 +942,7 @@ public class EsOpUtil<T> {
* 将contentBuilder设置为更新请求的内容。 * 将contentBuilder设置为更新请求的内容。
* 返回更新请求UpdateRequest对象。 * 返回更新请求UpdateRequest对象。
*/ */
private UpdateRequest createUpdateRequest(String index, String documentId, Map<String, Object> args,Boolean refreshPolicy) throws IOException { private UpdateRequest createUpdateRequest(String index, String documentId, Map<String, Object> args, Boolean refreshPolicy) throws IOException {
UpdateRequest request = new UpdateRequest(index, documentId); UpdateRequest request = new UpdateRequest(index, documentId);
if (Boolean.TRUE.equals(refreshPolicy)) { if (Boolean.TRUE.equals(refreshPolicy)) {
//刷新策略,立即刷新 //刷新策略,立即刷新
......
...@@ -114,7 +114,7 @@ public class HanlpUtil { ...@@ -114,7 +114,7 @@ public class HanlpUtil {
* @创建时间 2020/8/27 19:56 * @创建时间 2020/8/27 19:56
* @Version 1.0 * @Version 1.0
*/ */
private static int countKeyWordInContent(String keyword, String srcContent){ public static int countKeyWordInContent(String keyword, String srcContent){
if(keyword==null ||keyword.trim().equals("")){ if(keyword==null ||keyword.trim().equals("")){
return 0; return 0;
} }
......
...@@ -50,11 +50,17 @@ public class InfoDataSearchCondition { ...@@ -50,11 +50,17 @@ public class InfoDataSearchCondition {
/**原创性(0-非原创;1-原创;2-疑似)*/ /**原创性(0-非原创;1-原创;2-疑似)*/
private String originality; private String originality;
//资讯状态-研究中心(0-全部;1-模型推荐;2-精选;3-待定;4-移除),和checkStatus、deleteFlag、isFreeCheck互斥
private Integer status;
//审核操作(0:未审核 1:审核通过 2:审核未通过 3:暂定 默认值为0) //审核操作(0:未审核 1:审核通过 2:审核未通过 3:暂定 默认值为0)
private Integer checkStatus; private Integer checkStatus;
//删除标记(1:删除;0:未删除) //删除标记(1:删除;0:未删除)
private Integer deleteFlag = 0; private Integer deleteFlag;
//是否免审核(1-是;0-否)
private Integer isFreeCheck;
//关联标签名称 //关联标签名称
private String labelName; private String labelName;
...@@ -70,7 +76,7 @@ public class InfoDataSearchCondition { ...@@ -70,7 +76,7 @@ public class InfoDataSearchCondition {
//爬虫类型 //爬虫类型
private String crawler; private String crawler;
//组合标签查询(不同类标签之间是与的关系,同一类标签之间是或的关系),示例: "a,b;c,d;e,f" //组合标签查询(不同类标签之间是与的关系,同一类标签之间是或的关系),示例: "a1,a2;c1,c2"
private String composeSearchLabelIds; private String composeSearchLabelIds;
/**----企业类标签筛选----**/ /**----企业类标签筛选----**/
...@@ -104,21 +110,22 @@ public class InfoDataSearchCondition { ...@@ -104,21 +110,22 @@ public class InfoDataSearchCondition {
//待删除的标签id(多个用逗号隔开)-研究中心 //待删除的标签id(多个用逗号隔开)-研究中心
private String removeRelationId; private String removeRelationId;
//关键词信息;推荐信息源时使用-研究中心 //关键词信息;推荐信息源时使用-研究中心
private List<String> wordsList; private List<String> wordsList;
//是否免审核(1-是;0-否)
private Integer isFreeCheck;
//得分范围-研究中心 //得分范围-研究中心
//最小得分 //最小得分
private Integer minScore; private Integer minScore;
//最大得分 //最大得分
private Integer maxScore; private Integer maxScore;
//正文长度-研究中心
//最小长度
private Integer minContentLength;
//最大长度
private Integer maxContentLength;
//es查询字段数组 //es查询字段数组
private String[] fetchFields; private String[] fetchFields;
//排除字段数组 //排除字段数组
...@@ -126,7 +133,7 @@ public class InfoDataSearchCondition { ...@@ -126,7 +133,7 @@ public class InfoDataSearchCondition {
//排序参数 //排序参数
//排序字段 //排序字段
private String column = "publishDate"; private String column;
//排序方式 asc/desc //排序方式 asc/desc
private String order = "desc"; private String order = "desc";
//置顶排序是否起效(1-是;0-否) //置顶排序是否起效(1-是;0-否)
...@@ -166,7 +173,23 @@ public class InfoDataSearchCondition { ...@@ -166,7 +173,23 @@ public class InfoDataSearchCondition {
/*------添加至精选时,主题列表---start-------------------*/ /*------添加至精选时,主题列表---start-------------------*/
//聚合分组类型-按日期集合分析时使用
private List<Label> themeList; private List<Label> themeList;
/*------添加至精选时,主题列表---end-------------------*/ /*------添加至精选时,主题列表---end-------------------*/
/*------待删除的关键词列表---start-------------------*/
private List<String> keywordList;
/*------待删除的关键词列表---end-------------------*/
/*-----字符串替换---start-------------------*/
private String replaceWord;
/*------字符串替换---end-------------------*/
/*-----打标签---start-------------------*/
private List<Label> markTags;
/*------打标签---end-------------------*/
/**
* 本次任务使用的流程id
*/
private String flowId;
} }
...@@ -30,7 +30,7 @@ public class InfoSourceCondition { ...@@ -30,7 +30,7 @@ public class InfoSourceCondition {
/** /**
* 爬虫类别 * 爬虫类别
*/ */
private int crawlType; private Integer crawlType;
/** /**
* 专题id/事件id-获取专题绑定信息源信息时使用 * 专题id/事件id-获取专题绑定信息源信息时使用
......
...@@ -27,6 +27,7 @@ public class SubjectPageVO { ...@@ -27,6 +27,7 @@ public class SubjectPageVO {
private String subjectTypeName; private String subjectTypeName;
/**专题所属项目名称*/ /**专题所属项目名称*/
private String projectName; private String projectName;
private String dataScope;
/**信息总数量*/ /**信息总数量*/
private Integer totalNum; private Integer totalNum;
......
...@@ -44,6 +44,8 @@ public class DisplayInfo { ...@@ -44,6 +44,8 @@ public class DisplayInfo {
private String summaryRaw; private String summaryRaw;
//关键词 //关键词
private String keyWords; private String keyWords;
//命中词列表
private List<String> keyWordsList;
//标题 //标题
private String title; private String title;
private String titleRaw; private String titleRaw;
...@@ -54,6 +56,8 @@ public class DisplayInfo { ...@@ -54,6 +56,8 @@ public class DisplayInfo {
private String type; private String type;
//标签信息 //标签信息
private List<Label> labels; private List<Label> labels;
//排序字段信息
private List<SortField> sortField;
//模型打分信息 //模型打分信息
private List<ModelScore> modelScores; private List<ModelScore> modelScores;
//视频下载链接 //视频下载链接
......
package com.zzsn.event.vo.es;
import lombok.Data;
/**
* 排序字段
*
* @author lkg
* @date 2025/4/22
*/
@Data
public class SortField {
//字段 (摘要、内容、标签、词频统计)
private String fieldType;
//字段排序值
private String fieldKeyword;
//词频
private Long fieldLong;
public SortField(String fieldType, Long fieldLong) {
this.fieldType = fieldType;
this.fieldLong = fieldLong;
}
public SortField(String fieldType, String fieldKeyword) {
this.fieldType = fieldType;
this.fieldKeyword = fieldKeyword;
}
}
...@@ -53,6 +53,8 @@ public class SpecialInformation { ...@@ -53,6 +53,8 @@ public class SpecialInformation {
private String type; private String type;
//标签信息 //标签信息
private List<Label> labels; private List<Label> labels;
//排序字段信息
private List<SortField> sortField;
//模型打分信息 //模型打分信息
private List<ModelScore> modelScores; private List<ModelScore> modelScores;
//视频下载链接 //视频下载链接
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论