提交 ba220925 作者: 925993793@qq.com

事件服务和平台事件合并

上级 54e28629
......@@ -12,6 +12,7 @@ import com.zzsn.event.constant.Result;
import com.zzsn.event.entity.EventAnalysisReport;
import com.zzsn.event.entity.LabelEntity;
import com.zzsn.event.entity.SubjectAnalysis;
import com.zzsn.event.es.EsService;
import com.zzsn.event.service.*;
import com.zzsn.event.util.CalculateUtil;
import com.zzsn.event.util.HttpUtil;
......
......@@ -3,7 +3,7 @@ package com.zzsn.event.controller;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.zzsn.event.constant.Constants;
import com.zzsn.event.constant.Result;
import com.zzsn.event.service.EsService;
import com.zzsn.event.es.EsService;
import com.zzsn.event.service.IEventService;
import com.zzsn.event.vo.EventExcelVO;
import com.zzsn.event.vo.EventFrontVO;
......@@ -41,7 +41,7 @@ public class EventDataController {
/**
* 分页列表-门户
* 事件分页列表-门户
*
* @param eventName 事件名称
* @param eventType 事件分类id
......@@ -131,7 +131,7 @@ public class EventDataController {
*/
@ApiOperation(value = "单篇文章热词", notes = "单篇文章热词")
@GetMapping(value = "/hotWords")
public Result<?> articleList(@RequestParam(value = "index",required = false) String index, @RequestParam("id") String id,
public Result<?> articleList(@RequestParam(value = "index", required = false) String index, @RequestParam("id") String id,
@RequestParam(name = "number", defaultValue = "200") Integer number) {
if (StringUtils.isEmpty(index)) {
index = Constants.SUBJECT_INDEX;
......@@ -159,5 +159,4 @@ public class EventDataController {
List<SubjectDataVo> recommendList = esService.queryRecommendList(subjectId, id, title, pageNo, pageSize);
return Result.OK(recommendList);
}
}
......@@ -3,7 +3,7 @@ package com.zzsn.event.controller;
import com.aspose.words.Document;
import com.aspose.words.SaveFormat;
import com.zzsn.event.config.FreeMarkerConfiguration;
import com.zzsn.event.service.EsService;
import com.zzsn.event.es.EsService;
import com.zzsn.event.service.IEventService;
import com.zzsn.event.util.DateUtil;
import com.zzsn.event.util.ExcelExportUtil;
......
......@@ -11,7 +11,6 @@ import com.zzsn.event.service.*;
import com.zzsn.event.util.HttpUtil;
import com.zzsn.event.util.ObjectUtil;
import com.zzsn.event.util.tree.Node;
import com.zzsn.event.util.tree.TreeUtil;
import com.zzsn.event.util.user.UserUtil;
import com.zzsn.event.util.user.UserVo;
import com.zzsn.event.vo.*;
......@@ -20,14 +19,11 @@ import com.zzsn.event.xxljob.service.IXxlJobInfoService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.internal.StringUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.*;
import java.util.*;
import java.util.stream.Collectors;
/**
* 事件后台管理
......@@ -53,8 +49,7 @@ public class EventManageController {
private ISubjectInfoSourceMapService subjectInfoSourceMapService;
@Autowired
private EventRegionMapService eventRegionMapService;
@Autowired
private ISubjectTypeService subjectTypeService;
@Autowired
private ICustomerDataPermissionMapService customerDataPermissionMapService;
@Autowired
......@@ -91,108 +86,9 @@ public class EventManageController {
return Result.OK(pageList);
}
/**
* 事件页左侧树
*
* @author lkg
* @date 2024/4/29
*/
@GetMapping("/leftTypeTree")
public Result<?> leftTree() {
//获取当前登录用户
UserVo currentUser = UserUtil.getLoginUser();
Integer category = currentUser.getCategory();
String userId = null;
String customerId = null;
if (category.equals(Constants.COMMON_USER)) {
userId = currentUser.getUserId();
} else if (category.equals(Constants.ADMIN_USER)) {
customerId = currentUser.getCustomerId();
}
List<SubjectTreeVO> tree = subjectTypeService.subjectAndTypeTree(userId, customerId);
return Result.OK(tree);
}
/**
* 事件页左侧树
*
* @author lkg
* @date 2024/4/29
*/
@GetMapping("/leftCustomerTree")
public Result<?> leftCustomerTree() {
//获取当前登录用户
UserVo currentUser = UserUtil.getLoginUser();
Integer category = currentUser.getCategory();
String userId = null;
String customerId = null;
if (category.equals(Constants.COMMON_USER)) {
userId = currentUser.getUserId();
} else if (category.equals(Constants.ADMIN_USER)) {
customerId = currentUser.getCustomerId();
}
List<SubjectTreeVO> tree = subjectTypeService.subjectAndCustomerTree(userId,customerId);
return Result.OK(tree);
}
/**
* 分页列表-新平台管理
*
* @param subjectCondition 筛选条件
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2024/4/28
*/
@GetMapping("/newPlatPageList")
public Result<?> newPlatPageList(SubjectCondition subjectCondition,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) {
//获取当前登录用户
UserVo currentUser = UserUtil.getLoginUser();
Integer category = currentUser.getCategory();
if (category.equals(Constants.COMMON_USER)) {
subjectCondition.setUserId(currentUser.getUserId());
} else if (category.equals(Constants.ADMIN_USER)) {
subjectCondition.setCustomerId(currentUser.getCustomerId());
}
IPage<EventNewPlatVO> pageList = eventService.newPlatPageList(subjectCondition, pageNo, pageSize);
return Result.OK(pageList);
}
/**
* 分页列表-新平台管理
*
* @param subjectCondition 筛选条件
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2024/4/28
*/
@GetMapping("/newPlatCustomerPageList")
public Result<?> newPlatCustomerPageList(SubjectCondition subjectCondition,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) {
//获取当前登录用户
UserVo currentUser = UserUtil.getLoginUser();
Integer category = currentUser.getCategory();
if (category.equals(Constants.COMMON_USER)) {
subjectCondition.setUserId(currentUser.getUserId());
subjectCondition.setCustomerId(currentUser.getCustomerId());
} else if (category.equals(Constants.ADMIN_USER)) {
if (StringUtils.isEmpty(subjectCondition.getCustomerId()) || "0".equals(subjectCondition.getCustomerId())) {
subjectCondition.setCustomerId(currentUser.getCustomerId());
}
}
if ("0".equals(subjectCondition.getCustomerId())) {
subjectCondition.setCustomerId(null);
}
IPage<EventNewPlatVO> pageList = eventService.newPlatCustomerPageList(subjectCondition, pageNo, pageSize);
return Result.OK(pageList);
}
/**
* 地域信息-树型结构
*
* @param type 类别(1-国际;2-国内)
......
package com.zzsn.event.controller;
package com.zzsn.event.controller.plat;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
......@@ -32,7 +32,7 @@ import java.util.List;
@Slf4j
@Api(tags = "-事件分类")
@RestController
@RequestMapping("/manage/subjectType")
@RequestMapping("/plat/manage/subjectType")
public class EventTypeController {
@Autowired
......
package com.zzsn.event.controller.plat;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.zzsn.event.constant.Constants;
import com.zzsn.event.constant.Result;
import com.zzsn.event.entity.SubjectInfoSourceMap;
import com.zzsn.event.es.EsService;
import com.zzsn.event.service.IEventService;
import com.zzsn.event.service.IInfoSourceService;
import com.zzsn.event.service.ISubjectInfoSourceMapService;
import com.zzsn.event.service.ISubjectTypeService;
import com.zzsn.event.util.DateUtil;
import com.zzsn.event.util.ExcelExportUtil;
import com.zzsn.event.util.user.UserUtil;
import com.zzsn.event.util.user.UserVo;
import com.zzsn.event.vo.*;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletResponse;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* 平台事件管理页
*
* @author lkg
* @date 2024/5/7
*/
@Slf4j
@Api(tags = "平台事件管理")
@RestController
@RequestMapping("/plat/manage")
public class PlatEventManageController {
@Autowired
private IEventService eventService;
@Autowired
private ISubjectTypeService subjectTypeService;
@Autowired
private IInfoSourceService infoSourceService;
@Autowired
private ISubjectInfoSourceMapService subjectInfoSourceMapService;
@Autowired
private EsService esService;
/**
* 事件页左侧树
*
* @author lkg
* @date 2024/4/29
*/
@GetMapping("/leftTypeTree")
public Result<?> leftTree() {
//获取当前登录用户
UserVo currentUser = UserUtil.getLoginUser();
Integer category = currentUser.getCategory();
String userId = null;
String customerId = null;
if (category.equals(Constants.COMMON_USER)) {
userId = currentUser.getUserId();
} else if (category.equals(Constants.ADMIN_USER)) {
customerId = currentUser.getCustomerId();
}
List<SubjectTreeVO> tree = subjectTypeService.subjectAndTypeTree(userId, customerId);
return Result.OK(tree);
}
/**
* 事件页左侧树
*
* @author lkg
* @date 2024/4/29
*/
@GetMapping("/leftCustomerTree")
public Result<?> leftCustomerTree() {
//获取当前登录用户
UserVo currentUser = UserUtil.getLoginUser();
Integer category = currentUser.getCategory();
String userId = null;
String customerId = null;
if (category.equals(Constants.COMMON_USER)) {
userId = currentUser.getUserId();
} else if (category.equals(Constants.ADMIN_USER)) {
customerId = currentUser.getCustomerId();
}
List<SubjectTreeVO> tree = subjectTypeService.subjectAndCustomerTree(userId, customerId);
return Result.OK(tree);
}
/**
* 分页列表-新平台管理
*
* @param subjectCondition 筛选条件
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2024/4/28
*/
@GetMapping("/newPlatPageList")
public Result<?> newPlatPageList(SubjectCondition subjectCondition,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) {
//获取当前登录用户
UserVo currentUser = UserUtil.getLoginUser();
Integer category = currentUser.getCategory();
if (category.equals(Constants.COMMON_USER)) {
subjectCondition.setUserId(currentUser.getUserId());
} else if (category.equals(Constants.ADMIN_USER)) {
subjectCondition.setCustomerId(currentUser.getCustomerId());
}
IPage<EventNewPlatVO> pageList = eventService.newPlatPageList(subjectCondition, pageNo, pageSize);
return Result.OK(pageList);
}
/**
* 分页列表-新平台管理
*
* @param subjectCondition 筛选条件
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2024/4/28
*/
@GetMapping("/newPlatCustomerPageList")
public Result<?> newPlatCustomerPageList(SubjectCondition subjectCondition,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) {
//获取当前登录用户
UserVo currentUser = UserUtil.getLoginUser();
Integer category = currentUser.getCategory();
if (category.equals(Constants.COMMON_USER)) {
subjectCondition.setUserId(currentUser.getUserId());
subjectCondition.setCustomerId(currentUser.getCustomerId());
} else if (category.equals(Constants.ADMIN_USER)) {
if (StringUtils.isEmpty(subjectCondition.getCustomerId()) || "0".equals(subjectCondition.getCustomerId())) {
subjectCondition.setCustomerId(currentUser.getCustomerId());
}
}
if ("0".equals(subjectCondition.getCustomerId())) {
subjectCondition.setCustomerId(null);
}
IPage<EventNewPlatVO> pageList = eventService.newPlatCustomerPageList(subjectCondition, pageNo, pageSize);
return Result.OK(pageList);
}
/**
* 专题绑定的信息源
*
* @param infoSourceCondition 筛选条件
* @param subjectId 专题id/专题分类id
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2024/5/7
*/
@GetMapping(value = "/bindInfoSourceList")
public Result<?> bindInfoSourceList(InfoSourceCondition infoSourceCondition,
@RequestParam(name = "subjectId", defaultValue = "0") String subjectId,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) {
List<String> subjectIdList = new ArrayList<>();
if (!"0".equals(subjectId)) {
UserVo currentUser = UserUtil.getLoginUser();
String userId = null;
if (currentUser.getCategory().equals(Constants.COMMON_USER)) {
userId = currentUser.getUserId();
}
//查询类别id的所有明细id
if (StringUtils.isNotEmpty(subjectId)) {
List<String> typeIds = subjectTypeService.belowIdList(subjectId, userId, null);
subjectIdList = eventService.selectSubjectByTypeIds(userId, typeIds);
}
subjectIdList.add(subjectId);
}
IPage<InfoSourceVo> pageList = infoSourceService.bindInfoSourceList(infoSourceCondition, subjectIdList, pageNo, pageSize);
return Result.OK(pageList);
}
/**
* 专题直接绑定信息源-批量
*
* @param directBindInfoVO 参数封装
* @author lkg
* @date 2024/4/24
*/
@PostMapping("/directBindInfoSource")
public Result<?> directBindInfoSource(@RequestBody DirectBindInfoVO directBindInfoVO) {
eventService.directBindInfoSource(directBindInfoVO);
return Result.OK();
}
/**
* 专题直接解绑信息源-批量
*
* @param directBindInfoVO 参数封装
* @author lkg
* @date 2024/4/24
*/
@PostMapping("/directRemoveInfoSource")
public Result<?> directRemoveInfoSource(@RequestBody DirectBindInfoVO directBindInfoVO) {
eventService.directRemoveInfoSource(directBindInfoVO);
return Result.OK();
}
/**
* 专题直接屏蔽的信息源列表(不包括专题屏蔽的信息源组下的信息源)
*
* @param infoSourceCondition 筛选条件
* @param subjectId 专题id
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2024/4/30
*/
@GetMapping("/unBindInfoSourceList")
public Result<?> unBindInfoSourceList(InfoSourceCondition infoSourceCondition,
@RequestParam(name = "subjectId", defaultValue = "0") String subjectId,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize) {
IPage<InfoSourceVo> pageList = infoSourceService.unBindSourcePageList(infoSourceCondition, subjectId, pageNo, pageSize);
return Result.OK(pageList);
}
/**
* 删除专题下已屏蔽的信息源(即恢复绑定关系)
*
* @param subjectId 专题id
* @param sourceId 信息源id
* @author lkg
* @date 2024/4/30
*/
@GetMapping("/removeUnBindInfoSource")
public Result<?> removeUnBindInfoSource(@RequestParam String subjectId, @RequestParam String sourceId) {
eventService.removeUnBindInfoSource(subjectId,sourceId);
return Result.OK();
}
/**
* 专题关联信息源采集量统计-导出excel
*
* @param subjectId 专题id/专题分类id
* @param startDate 开始时间
* @param endDate 结束时间
* @author lkg
*/
@GetMapping("/exportStatisticsExcel")
public void exportStatisticsExcel(@RequestParam(name = "subjectId") String subjectId,
@RequestParam(name = "startDate") String startDate,
@RequestParam(name = "endDate") String endDate,
HttpServletResponse response) {
List<String> subjectIdList = new ArrayList<>();
try {
UserVo currentUser = UserUtil.getLoginUser();
String userId = null;
if (currentUser.getCategory().equals(Constants.COMMON_USER)) {
userId = currentUser.getUserId();
}
//查询类别id的所有明细id
if (StringUtils.isNotEmpty(subjectId) && !"0".equals(subjectId)) {
List<String> typeIds = subjectTypeService.belowIdList(subjectId, userId, null);
subjectIdList = eventService.selectSubjectByTypeIds(userId, typeIds);
}
subjectIdList.add(subjectId);
List<List<String>> statistics = eventService.subjectStatistics(subjectIdList, startDate, endDate);
List<String> dateList = DateUtil.betweenDate(startDate, endDate);
List<String> headers = Stream.of("信息源id", "网站名称", "栏目名称", "栏目地址", "时间段内采集总量").collect(Collectors.toList());
headers.addAll(dateList);
XSSFWorkbook workbook = new XSSFWorkbook();
ExcelExportUtil.exportExcelData(workbook, 0, headers, statistics, "信息源采集信息汇总");
String name = "statisticsExcel.xls";
setResponseHeader(response, name);
ServletOutputStream outputStream = response.getOutputStream();
workbook.write(outputStream);
outputStream.flush();
outputStream.close();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 事件对应专题库的资讯分页列表
*
* @param eventDataCondition 筛选条件
* @param sourceId 信息源id
* @param pageNo 当前页
* @param pageSize 返回条数
* @param column 排序字段
* @param order 排序方式
* @param isAll 是否全部
* @param video 是否视频
* @param crawler 采集器
* @param isSubject 是否专题
* @param source 来源(类别/客户)
* @author lkg
* @date 2024/5/6
*/
@GetMapping("/collectPageList")
public Result<?> collectPageList(EventDataCondition eventDataCondition,
@RequestParam(name = "sourceId", defaultValue = "") String sourceId,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
@RequestParam(name = "column", defaultValue = "common") String column,
@RequestParam(name = "order", defaultValue = "desc") String order,
@RequestParam(name = "isAll", defaultValue = "0") String isAll,
@RequestParam(name = "video", defaultValue = "") String video,
@RequestParam(name = "crawler", defaultValue = "") String crawler,
@RequestParam(name = "isSubject", defaultValue = "0") String isSubject,
@RequestParam(name = "source", defaultValue = "type") String source) {
UserVo currentUser = UserUtil.getLoginUser();
String userId = null;
String tenantId = currentUser.getCustomerId();
if (currentUser.getCategory().equals(Constants.COMMON_USER)) {
userId = currentUser.getUserId();
}
IPage<EventDataVO> pageList = esService.collectPageList(eventDataCondition, sourceId, pageNo, pageSize, column, order, isAll, video, crawler, userId, tenantId, isSubject, source);
return Result.OK(pageList);
}
/**
* 件对应专题库的资讯分页列表
*
* @param subjectInfoVo 筛选条件
* @param userId 用户id
* @param video 是否视频
* @param pageNo 当前页
* @param isCustomer 是否客户
* @param pageSize 返回条数
* @param column 排序字段
* @param order 排序方式
* @param crawler 采集器
* @param isSubject 是否换题
* @param labelIds 关联的标签id
* @param labelTypeIds 企业标签id
* @param relationIds 企业信用代码
* @param sourceId 信息源id
* @author lkg
* @date 2024/5/6
*/
@GetMapping("/subjectPageList")
public Result<?> subjectPageList(EventDataCondition subjectInfoVo,
@RequestParam(name = "userId", defaultValue = "") String userId,
@RequestParam(name = "video", defaultValue = "") String video,
@RequestParam(name = "pageNo", defaultValue = "1") Integer pageNo,
@RequestParam(name = "isCustomer", defaultValue = "0") Integer isCustomer,
@RequestParam(name = "pageSize", defaultValue = "10") Integer pageSize,
@RequestParam(name = "column", defaultValue = "common") String column,
@RequestParam(name = "order", defaultValue = "desc") String order,
@RequestParam(name = "crawler", defaultValue = "") String crawler,
@RequestParam(name = "isSubject", defaultValue = "1") String isSubject,
@RequestParam(name = "labelIds", required = false) String labelIds,
@RequestParam(name = "labelTypeIds", required = false) String labelTypeIds,
@RequestParam(name = "relationIds", required = false) String relationIds,
@RequestParam(name = "sourceId", required = false) String sourceId) throws Exception {
List<String> socialCreditCodeList = new ArrayList<>();
if (StringUtils.isNotEmpty(relationIds)) {
socialCreditCodeList = Arrays.asList(relationIds.split(","));
} else if (StringUtils.isNotEmpty(labelTypeIds)) {
socialCreditCodeList = eventService.codesByLabels(Arrays.asList(labelTypeIds.split(",")));
}
IPage<DisplayInfo> pageList = esService.subjectPageList(userId, subjectInfoVo, video, pageNo, pageSize, column, order, crawler, isSubject, labelIds, socialCreditCodeList, sourceId, isCustomer);
return Result.OK(pageList);
}
/**
* 专题绑定的关键词组-分页列表
*
* @param id 题id/专题分类id
* @author lkg
* @date 2024/5/7
*/
@GetMapping(value = "/bindKeyWordsList")
public Result<?> bindKeyWordsList(@RequestParam String id,
@RequestParam(required = false) String groupName,
@RequestParam(required = false) String wordName,
@RequestParam(defaultValue = "1") Integer pageNo,
@RequestParam(defaultValue = "10") Integer pageSize) {
List<String> subjectIdList = new ArrayList<>();
try {
UserVo currentUser = UserUtil.getLoginUser();
String userId = null;
if (currentUser.getCategory().equals(Constants.COMMON_USER)) {
userId = currentUser.getUserId();
}
if (StringUtils.isNotEmpty(id) && !"0".equals(id)) {
List<String> typeIds = subjectTypeService.belowIdList(id, userId, null);
subjectIdList = eventService.selectSubjectByTypeIds(userId, typeIds);
}
subjectIdList.add(id);
} catch (Exception e) {
e.printStackTrace();
}
IPage<KeyWordsPage> page = eventService.bindKeyWordsList(subjectIdList, groupName, wordName, pageNo, pageSize);
return Result.OK(page);
}
private void setResponseHeader(HttpServletResponse response, String name) {
try {
try {
name = new String(name.getBytes(), "ISO8859-1");
} catch (Exception e) {
e.printStackTrace();
}
response.setContentType("application/octet-stream;charset=ISO8859-1");
response.setHeader("Content-Disposition", "attachment;filename=" + name);
response.setHeader("Pargam", "no-cache");
response.setHeader("Cache-Control", "no-cache");
} catch (Exception e) {
e.printStackTrace();
}
}
}
package com.zzsn.event.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.experimental.Accessors;
import org.jeecgframework.poi.excel.annotation.Excel;
import java.io.Serializable;
/**
* @Description: 文章收藏映射表
* @Author: jeecg-boot
* @Date: 2022-05-17
* @Version: V1.0
*/
@Data
@TableName("collection_map")
@Accessors(chain = true)
@EqualsAndHashCode(callSuper = false)
@ApiModel(value="collection_map对象", description="文章收藏映射表")
public class CollectionMap implements Serializable {
private static final long serialVersionUID = 1L;
/**主键*/
@TableId(type = IdType.ASSIGN_ID)
@ApiModelProperty(value = "主键")
private String id;
/**用户id*/
@Excel(name = "用户id", width = 15)
@ApiModelProperty(value = "用户id")
private String userId;
/**资讯id*/
@Excel(name = "资讯id", width = 15)
@ApiModelProperty(value = "资讯id")
private String articleId;
}
package com.zzsn.event.enums;
/**
* @author zs
* @Description 信息源/信息源组 枚举
* @since 2021/12/9
*/
public enum BindTypeEnum {
INFO_SOURCE(1,"绑定的信息源"),
INFO_SOURCE_GROUP(2,"绑定的信息源组"),
EXCLUDE_INFO_SOURCE(3,"排除的信息源"),
EXCLUDE_INFO_SOURCE_GROUP(4,"排除的信息源组"),
DIRECTIONA_INFO_SOURCE_GROUP(5,"定向的信息源组");
private Integer value;
private String des;
BindTypeEnum(int value, String des) {
this.value = value;
this.des = des;
}
public Integer getvalue() {
return value;
}
public String getDes() {
return des;
}
}
package com.zzsn.event.service;
package com.zzsn.event.es;
import cn.hutool.core.map.MapUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson2.JSON;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.CollectionUtils;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.zzsn.event.constant.Constants;
import com.zzsn.event.entity.CollectionMap;
import com.zzsn.event.service.ICollectionMapService;
import com.zzsn.event.service.IEventService;
import com.zzsn.event.service.ISubjectInfoSourceMapService;
import com.zzsn.event.service.ISubjectTypeService;
import com.zzsn.event.util.DateUtil;
import com.zzsn.event.util.EsDateUtil;
import com.zzsn.event.vo.RepeatHold;
import com.zzsn.event.vo.SubjectDataVo;
import com.zzsn.event.util.user.UserUtil;
import com.zzsn.event.util.user.UserVo;
import com.zzsn.event.vo.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
......@@ -19,11 +29,16 @@ import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.bucket.histogram.ParsedDateHistogram;
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
......@@ -34,13 +49,12 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
/**
* @author lkg
......@@ -53,6 +67,14 @@ public class EsService {
@Resource
private RestHighLevelClient client;
@Autowired
private ISubjectTypeService iSubjectTypeService;
@Autowired
private IEventService eventService;
@Autowired
private ISubjectInfoSourceMapService iSubjectInfoSourceMapService;
@Autowired
private ICollectionMapService collectionMapService;
/**
......@@ -551,6 +573,509 @@ public class EsService {
return list;
}
/**
* 事件对应采集库的资讯分页列表
*
* @param eventDataCondition
* @param sourceId
* @param offset
* @param pageSize
* @param column
* @param order
* @param isAll
* @param video
* @param crawler
* @param userId
* @param tenantId
* @param isSubject
* @param source
* @author lkg
* @date 2024/5/6
*/
public IPage<EventDataVO> collectPageList(EventDataCondition eventDataCondition, String sourceId, int offset, int pageSize, String column,
String order, String isAll, String video, String crawler, String userId, String tenantId, String isSubject, String source){
SearchRequest searchRequest = new SearchRequest(Constants.COLLECT_INDEX);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//设置分页参数
searchSourceBuilder.size(pageSize);
searchSourceBuilder.from((offset - 1) * pageSize);
//默认按照时间倒序排列
if (column.equals("publishDate")) {
if (order.equals("desc")) {
searchSourceBuilder.sort("publishDate", SortOrder.DESC);
} else if (order.equals("asc")) {
searchSourceBuilder.sort("publishDate", SortOrder.ASC);
}
}
//默认最大数量是10000,设置为true后,显示准确数量
searchSourceBuilder.trackTotalHits(true);
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
if ("1".equals(isAll)) {
List<String> subjectIdList = new ArrayList<>();
//判断是否为专题
if (!"1".equals(isSubject)) {
//不是专题
if ("subjectType".equals(source)) {
//查询类别id的所有明细id
List<String> typeIds = new ArrayList<>();
if (StringUtils.isNotEmpty(eventDataCondition.getSubjectId()) && !"0".equals(eventDataCondition.getSubjectId())) {
typeIds = iSubjectTypeService.belowIdList(eventDataCondition.getSubjectId(),userId,null);
}
subjectIdList = eventService.selectSubjectByTypeIds(userId, typeIds);
} else if ("customer".equals(source)) {
//查询该用户可以查看当前客户下面的专题
if (!StringUtils.isEmpty(eventDataCondition.getSubjectId()) && !"0".equals(eventDataCondition.getSubjectId()) && StringUtils.isEmpty(userId)) {
subjectIdList = eventService.selectSubjectWithCustomer(userId, null);
} else if (StringUtils.isNotBlank(userId)) {
subjectIdList = eventService.selectSubjectWithCustomer(userId, null);
} else {
subjectIdList = eventService.selectSubjectWithCustomer(null, tenantId);
}
}
} else {
subjectIdList.add(eventDataCondition.getSubjectId());
}
//事件真正绑定的信息源id集合
List<String> infoSourceIdList = new ArrayList<>();
if (CollectionUtils.isNotEmpty(subjectIdList)) {
infoSourceIdList = iSubjectInfoSourceMapService.subjectRealBindInfoSourceList(subjectIdList);
//查询专题绑定的关键词组
List<String> wordsIdList = eventService.bindKeyWordsIdList(subjectIdList);
infoSourceIdList.addAll(wordsIdList);
}
//装配信息源的条件
if (CollectionUtils.isNotEmpty(infoSourceIdList)) {
BoolQueryBuilder childQuery = QueryBuilders.boolQuery();
childQuery.should(QueryBuilders.termsQuery("sid", infoSourceIdList));
boolQuery.must(childQuery);
} else {
return new Page<>();
}
} else {
boolQuery.must(QueryBuilders.matchQuery("sid", sourceId));
}
if (StringUtils.isNotEmpty(eventDataCondition.getOrigin())) {
boolQuery.must(QueryBuilders.matchPhraseQuery("origin", eventDataCondition.getOrigin()));
}
if (StringUtils.isNotEmpty(eventDataCondition.getSearch())) {
boolQuery.must(QueryBuilders.multiMatchQuery(eventDataCondition.getSearch(), "title", "content"));
}
if (StringUtils.isNotEmpty(eventDataCondition.getContent())) {
boolQuery.must(QueryBuilders.matchPhraseQuery("content", eventDataCondition.getContent()));
}
if (StringUtils.isNotEmpty(eventDataCondition.getTitle())) {
boolQuery.must(QueryBuilders.matchPhraseQuery("title", eventDataCondition.getTitle()));
}
if (StringUtils.isNotEmpty(eventDataCondition.getSourceAddress())) {
boolQuery.must(QueryBuilders.termQuery("sourceAddress", eventDataCondition.getSourceAddress()));
}
if (StringUtils.isNotEmpty(eventDataCondition.getId())) {
boolQuery.must(QueryBuilders.termQuery("id", eventDataCondition.getId()));
}
if (StringUtils.isNotBlank(video)) {
boolQuery.must(QueryBuilders.matchQuery("type", "video"));
} else {
boolQuery.mustNot(QueryBuilders.matchQuery("type", "video"));
}
if (StringUtils.isNotBlank(crawler)) {
boolQuery.must(QueryBuilders.matchQuery("source", crawler));
}
//时间过滤筛选
if (StringUtils.isNotBlank(eventDataCondition.getStartTime())) {
boolQuery.filter(QueryBuilders.rangeQuery("publishDate").gte(EsDateUtil.esFieldDateFormat(eventDataCondition.getStartTime())));
}
if (StringUtils.isNotBlank(eventDataCondition.getEndTime())) {
boolQuery.filter(QueryBuilders.rangeQuery("publishDate").lte(EsDateUtil.esFieldDateFormat(eventDataCondition.getEndTime())));
} else {
boolQuery.filter(QueryBuilders.rangeQuery("publishDate").lte(EsDateUtil.esFieldDateFormat(DateUtil.dateToString(new Date()))));
}
if (eventDataCondition.getLabelName()!=null) {
BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
TermQueryBuilder relationIdQuery = QueryBuilders.termQuery("labels.relationName", eventDataCondition.getLabelName());
nestedBoolQueryBuilder.should(QueryBuilders.nestedQuery("labels", relationIdQuery, ScoreMode.None));
boolQuery.must(nestedBoolQueryBuilder);
}
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
IPage<EventDataVO> pageData = new Page<>();
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] searchHits = searchResponse.getHits().getHits();
List<EventDataVO> list = new ArrayList<>();
for (SearchHit hit : searchHits) {
String queryInfo = hit.getSourceAsString();
EventDataVO info = JSON.parseObject(queryInfo, EventDataVO.class);
info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
String hitIndex = hit.getIndex();
info.setIndex(hitIndex);
list.add(info);
}
pageData = new Page<>(offset, pageSize, searchResponse.getHits().getTotalHits().value);
pageData.setRecords(list);
} catch (Exception e) {
e.printStackTrace();
}
return pageData;
}
/**
* 事件对应专题库的资讯分页列表
*
* @param userId
* @param subjectInfo
* @param video
* @param offset
* @param pageSize
* @param column
* @param order
* @param crawler
* @param isSubject
* @param labelIds
* @param socialCreditCodeList
* @param sourceId
* @param isCustomer
* @author lkg
* @date 2024/5/6
*/
public IPage<DisplayInfo> subjectPageList(String userId, EventDataCondition subjectInfo, String video, int offset, int pageSize,
String column, String order, String crawler, String isSubject,
String labelIds, List<String> socialCreditCodeList, String sourceId, Integer isCustomer) throws Exception {
SearchRequest searchRequest = new SearchRequest(Constants.SUBJECT_INDEX);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//设置分页参数
searchSourceBuilder.size(pageSize);
searchSourceBuilder.from((offset - 1) * pageSize);
//默认按照置顶以及时间倒序排列
//根据topNum正序查找,查询置顶数据
searchSourceBuilder.sort("topNum", SortOrder.DESC);
if (column.equals("score")) {
if (order.equals("asc")) {
searchSourceBuilder.sort("score", SortOrder.ASC);
searchSourceBuilder.sort("publishDate", SortOrder.ASC);
} else if (order.equals("desc")) {
searchSourceBuilder.sort("score", SortOrder.DESC);
searchSourceBuilder.sort("publishDate", SortOrder.DESC);
}
} else if (column.equals("publishDate")) {
if (order.equals("desc")) {
searchSourceBuilder.sort("publishDate", SortOrder.DESC);
searchSourceBuilder.sort("score", SortOrder.DESC);
} else if (order.equals("asc")) {
searchSourceBuilder.sort("publishDate", SortOrder.ASC);
searchSourceBuilder.sort("score", SortOrder.ASC);
}
}
//默认最大数量是10000,设置为true后,显示准确数量
searchSourceBuilder.trackTotalHits(true);
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
List<String> subjectIdList = new ArrayList<>();
//判断是否是专题
if ("1".equals(isSubject)) {
if (StringUtils.isNotEmpty(subjectInfo.getSubjectId())) {
subjectIdList.add(subjectInfo.getSubjectId());
}
} else {
//该id其实是专题类别id
//查询类别id的所有明细id
List<String> typeIds = new ArrayList<>();
String subjectTypeId = subjectInfo.getSubjectId();
//获取当前登录用户
UserVo currentUser = UserUtil.getLoginUser();
if (isCustomer == 1) {
subjectIdList = eventService.selectSubjectWithCustomer(null, null);
} else {
if (StringUtils.isNotEmpty(subjectTypeId) && !"0".equals(subjectTypeId)) {
typeIds = iSubjectTypeService.belowIdList(subjectTypeId,userId,null);
}
//判断是否是超级用户
if (currentUser != null && currentUser.getCategory() == 1) {
subjectIdList = eventService.selectSubjectByTypeIds(null, typeIds);
} else {
subjectIdList = eventService.selectSubjectByTypeIds(userId, typeIds);
}
}
}
if (CollectionUtils.isNotEmpty(subjectIdList)) {
boolQuery.must(QueryBuilders.termsQuery("subjectId", subjectIdList));
} else {
return new Page<>();
}
if (StringUtils.isNotEmpty(subjectInfo.getSearch())) {
boolQuery.must(QueryBuilders.multiMatchQuery(subjectInfo.getSearch(), "title", "content"));
}
if (StringUtils.isNotEmpty(subjectInfo.getContent())) {
boolQuery.must(QueryBuilders.matchPhraseQuery("content", subjectInfo.getContent()));
}
if (StringUtils.isNotEmpty(subjectInfo.getTitle())) {
boolQuery.must(QueryBuilders.matchPhraseQuery("title", subjectInfo.getTitle()));
}
if (StringUtils.isNotEmpty(subjectInfo.getOrigin())) {
boolQuery.must(QueryBuilders.matchPhraseQuery("origin", subjectInfo.getOrigin()));
}
if (StringUtils.isNotEmpty(subjectInfo.getSourceAddress())) {
boolQuery.must(QueryBuilders.termQuery("sourceAddress", subjectInfo.getSourceAddress()));
}
if (StringUtils.isNotEmpty(subjectInfo.getId())) {
boolQuery.must(QueryBuilders.termQuery("id", subjectInfo.getId()));
}
if (subjectInfo.getCheckStatusList() != null && subjectInfo.getCheckStatusList().size() > 0) {
List<Integer> list;
list = subjectInfo.getCheckStatusList();
if (!(list.size() == 1 && list.contains(4))) {
boolQuery.must(QueryBuilders.termsQuery("checkStatus", subjectInfo.getCheckStatusList()));
}
}
if (StringUtils.isNotBlank(video)) {
boolQuery.must(QueryBuilders.matchQuery("type", "video"));
} else {
boolQuery.mustNot(QueryBuilders.matchQuery("type", "video"));
}
//只查询主条目
boolQuery.mustNot(QueryBuilders.matchQuery("flag", "0"));
//删除状态查询
if (subjectInfo.getDeleteFlag() != 0) {
boolQuery.must(QueryBuilders.matchQuery("deleteFlag", "1"));
} else {
boolQuery.mustNot(QueryBuilders.matchQuery("deleteFlag", "1"));
}
//专题库类别筛选
if (subjectInfo.getClassificationType() != null) {
boolQuery.must(QueryBuilders.termQuery("classificationType", subjectInfo.getClassificationType()));
}
//正负面
if (subjectInfo.getOrientation() != null) {
boolQuery.must(QueryBuilders.termQuery("orientation", subjectInfo.getOrientation()));
}
//时间过滤筛选
if (StringUtils.isNotBlank(subjectInfo.getStartTime())) {
boolQuery.filter(QueryBuilders.rangeQuery("publishDate").gte(EsDateUtil.esFieldDateFormat(subjectInfo.getStartTime())));
}
if (StringUtils.isNotBlank(subjectInfo.getEndTime())) {
boolQuery.filter(QueryBuilders.rangeQuery("publishDate").lte(EsDateUtil.esFieldDateFormat(subjectInfo.getEndTime())));
} else {
boolQuery.filter(QueryBuilders.rangeQuery("publishDate").lte(EsDateUtil.esFieldDateFormat(DateUtil.dateToString(new Date()))));
}
if (StringUtils.isNotBlank(crawler)) {
boolQuery.must(QueryBuilders.matchQuery("source", crawler));
}
Set<String> relationIds = new HashSet<>();
if (StringUtils.isNotEmpty(labelIds)) {
relationIds.add(labelIds);
}
if (CollectionUtils.isNotEmpty(socialCreditCodeList)) {
relationIds.addAll(socialCreditCodeList);
}
if (CollectionUtils.isNotEmpty(relationIds)) {
BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
for (String relationId : relationIds) {
TermQueryBuilder relationIdQuery = QueryBuilders.termQuery("labels.relationId", relationId);
nestedBoolQueryBuilder.should(QueryBuilders.nestedQuery("labels", relationIdQuery, ScoreMode.None));
}
boolQuery.must(nestedBoolQueryBuilder);
}
if (StringUtils.isNotEmpty(subjectInfo.getArea())) {
String[] regionArr = subjectInfo.getArea().split(",");
for (String regionId : regionArr) {
BoolQueryBuilder nestedRegionBoolQueryBuilder = QueryBuilders.boolQuery();
TermQueryBuilder relationIdQuery = QueryBuilders.termQuery("labels.relationId", regionId);
nestedRegionBoolQueryBuilder.should(QueryBuilders.nestedQuery("labels", relationIdQuery, ScoreMode.None));
boolQuery.must(nestedRegionBoolQueryBuilder);
}
}
if (subjectInfo.getLabelName()!=null) {
BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
TermQueryBuilder relationIdQuery = QueryBuilders.termQuery("labels.relationName", subjectInfo.getLabelName());
nestedBoolQueryBuilder.should(QueryBuilders.nestedQuery("labels", relationIdQuery, ScoreMode.None));
boolQuery.must(nestedBoolQueryBuilder);
}
if (StringUtils.isNotBlank(sourceId)) {
boolQuery.must(QueryBuilders.termQuery("sid", sourceId));
}
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
SearchHit[] searchHits = searchResponse.getHits().getHits();
List<DisplayInfo> list = new ArrayList<>();
List<LabelModelVo> labelModelVos = eventService.selectLabelModelBySubjectId(subjectIdList);
Map<String, List<LabelModelVo>> modelMap = labelModelVos.stream().collect(Collectors.groupingBy(LabelModelVo::getSubjectId));
for (SearchHit hit : searchHits) {
String queryInfo = hit.getSourceAsString();
DisplayInfo info = JSONUtil.toBean(queryInfo, DisplayInfo.class);
info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
QueryWrapper<CollectionMap> query = Wrappers.query();
query.eq("user_id", userId);
query.eq("article_id", info.getId());
int count = collectionMapService.count(query);
if (count > 0) {
info.setYnCollect(true);
}
//标签处理
List<LabelModelVo> modelVoList = modelMap.get(info.getSubjectId());
formatLabel(modelVoList, info);
String index = hit.getIndex();
info.setIndex(index);
list.add(info);
}
IPage<DisplayInfo> pageData = new Page<>(offset, pageSize, searchResponse.getHits().getTotalHits().value);
pageData.setRecords(list);
return pageData;
}
/**
* 获取信息源时间段内的采集量
*
* @param sid 信息源id
* @param startDate 开始日期
* @param endDate 结束日期
* @param type 分类(1-不聚合;2-聚合)
* @author lkg
* @date 2024/4/25
*/
public NumVO count(String sid, String startDate, String endDate, Integer type) {
NumVO countVO = new NumVO();
SearchRequest searchRequest = new SearchRequest(Constants.COLLECT_INDEX);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//默认最大数量是10000,设置为true后,显示准确数量
searchSourceBuilder.trackTotalHits(true);
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.termQuery("sid", sid));
if (StringUtils.isNotBlank(startDate) || StringUtils.isNotBlank(endDate)) {
RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("createDate");
if (StringUtils.isNotBlank(startDate)) {
rangeQueryBuilder.from(EsDateUtil.esFieldDateFormat(startDate));
}
if (StringUtils.isNotBlank(endDate)) {
rangeQueryBuilder.to(EsDateUtil.esFieldDateFormat(endDate));
}
boolQuery.filter(rangeQueryBuilder);
}
searchSourceBuilder.query(boolQuery);
if (type == 2) {
DateHistogramAggregationBuilder aggregation = AggregationBuilders.dateHistogram("group_date")
.field("createDate")
.calendarInterval(DateHistogramInterval.DAY)
.format("yyyy-MM-dd");
searchSourceBuilder.aggregation(aggregation);
}
searchRequest.source(searchSourceBuilder);
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
long num = searchResponse.getHits().getTotalHits().value;
countVO.setCount(num);
if (type == 2) {
List<NumVO> list = new ArrayList<>();
Aggregations aggregations = searchResponse.getAggregations();
ParsedDateHistogram groupHour = aggregations.get("group_date");
List<? extends Histogram.Bucket> buckets = groupHour.getBuckets();
if (org.apache.commons.collections4.CollectionUtils.isNotEmpty(buckets)) {
for (Histogram.Bucket bucket : buckets) {
NumVO vo = new NumVO();
vo.setKey(bucket.getKeyAsString());
vo.setCount(bucket.getDocCount());
list.add(vo);
}
}
countVO.setChildren(list);
}
} catch (IOException e) {
e.printStackTrace();
}
return countVO;
}
public Map<String, Map<String, Long>> regetFromEs(List<String> sidList, String startDate, String endDate) {
SearchRequest searchRequest = new SearchRequest(Constants.COLLECT_INDEX);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(0);
//时间范围筛选
RangeQueryBuilder rangeQueryBuilder = QueryBuilders
.rangeQuery("createDate")
.from(startDate)
.to(endDate);
searchSourceBuilder.query(rangeQueryBuilder);
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQuery.must(QueryBuilders.termsQuery("sid", sidList));
boolQuery.must(rangeQueryBuilder);
//按时间,sid分组
//es 查询天的时间格式
TermsAggregationBuilder aggregationBuilder = AggregationBuilders.terms("group_sid")
.size(10000)
.field("sid")
.subAggregation(AggregationBuilders.dateHistogram("group_date")
.field("createDate")
.calendarInterval(DateHistogramInterval.DAY)
.format("yyyy-MM-dd"));
searchSourceBuilder.aggregation(aggregationBuilder);
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
Map<String, Map<String, Long>> map = new HashMap<>();
try {
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
//获取分组桶
Aggregations aggregations = searchResponse.getAggregations();
//获取id分组集合
Terms parsedStringTerms = aggregations.get("group_sid");
List<? extends Terms.Bucket> buckets = parsedStringTerms.getBuckets();
for (Terms.Bucket bucket : buckets) {
//key的数据,过滤sid为空的
String key = bucket.getKey().toString();
if (key == null || "null".equals(key) || "".equals(key)) {
continue;
}
//获取时间分组集合
Aggregations bucketAggregations = bucket.getAggregations();
ParsedDateHistogram groupdateAgr = bucketAggregations.get("group_date");
List<? extends Histogram.Bucket> dateBucket = groupdateAgr.getBuckets();
Map<String, Long> temMap = new HashMap<>();
for (Histogram.Bucket date : dateBucket) {
String dateStr = date.getKeyAsString();
long num = date.getDocCount();
temMap.put(dateStr, num);
}
map.put(key, temMap);
}
} catch (IOException e) {
e.printStackTrace();
}
return map;
}
private void formatLabel(List<LabelModelVo> labelModelVos, DisplayInfo info) {
if (CollectionUtils.isNotEmpty(labelModelVos)) {
List<Label> list = info.getLabels();
List<LabelInfo> labelInfos = new ArrayList<>();
//获取专题打的标签
for (LabelModelVo labelModelVo : labelModelVos) {
LabelInfo labelInfo = new LabelInfo();
labelInfo.setLabelId(labelModelVo.getLabelId());
labelInfo.setLabelName(labelModelVo.getLabelName());
labelInfo.setLabelType(labelModelVo.getLabelType());
labelInfo.setLabelMark(labelModelVo.getLabelMark());
List<Label> labelList = new ArrayList<>();
if (list != null && list.size() > 0) {
for (Label label : list) {
if (StringUtils.isNotBlank(label.getLabelMark())) {
if (label.getLabelMark().contains(labelModelVo.getLabelMark()) || ("company_label".equals(labelModelVo.getLabelType()) && "company_label".equals(label.getLabelRemarks()))) {
labelList.add(label);
}
}
}
}
labelInfo.setLabelList(labelList);
labelInfos.add(labelInfo);
}
info.setLabelInfos(labelInfos);
}
}
//格式化数据
private List<SubjectDataVo> formatData(SearchHits searchHits) {
......
......@@ -28,7 +28,7 @@ import java.util.List;
* @date 2022/7/15
*/
@Slf4j
@Component
//@Component
public class KafkaConsumer {
@Autowired
private SubjectAnalysisService subjectAnalysisService;
......@@ -42,7 +42,7 @@ public class KafkaConsumer {
*
* @param record 接受的kafka数据
*/
@KafkaListener(topics = {Constants.VIEWPOINT_RECEIVE_TOPIC})
// @KafkaListener(topics = {Constants.VIEWPOINT_RECEIVE_TOPIC})
public void viewPointAnalysis(ConsumerRecord<String, String> record) {
String value = record.value();
if (StringUtils.isNotEmpty(value)) {
......@@ -74,7 +74,7 @@ public class KafkaConsumer {
*
* @param record 接受的kafka数据
*/
@KafkaListener(topics = {Constants.EVENT_CONTEXT_RECEIVE_TOPIC})
// @KafkaListener(topics = {Constants.EVENT_CONTEXT_RECEIVE_TOPIC})
public void eventContext(ConsumerRecord<String, String> record) {
String value = record.value();
if (StringUtils.isNotEmpty(value)) {
......@@ -98,7 +98,7 @@ public class KafkaConsumer {
*
* @param record 接受的kafka数据
*/
@KafkaListener(topics = {Constants.FAKE_EVENT_CONTEXT_RECEIVE_TOPIC})
// @KafkaListener(topics = {Constants.FAKE_EVENT_CONTEXT_RECEIVE_TOPIC})
public void eventContext_fake(ConsumerRecord<String, String> record) {
String value = record.value();
if (StringUtils.isNotEmpty(value)) {
......@@ -132,7 +132,7 @@ public class KafkaConsumer {
* @author lkg
* @date 2024/4/12
*/
@KafkaListener(topics = {Constants.EVENT_REPORT_RECEIVE_TOPIC})
// @KafkaListener(topics = {Constants.EVENT_REPORT_RECEIVE_TOPIC})
public void eventReport(ConsumerRecord<String, String> record) {
String value = record.value();
EventAnalysisReport eventAnalysisReport = JSONObject.parseObject(value, EventAnalysisReport.class);
......
package com.zzsn.event.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.event.entity.CollectionMap;
import org.apache.ibatis.annotations.Mapper;
/**
* @Description: 文章收藏映射表
* @Author: jeecg-boot
* @Date: 2022-05-17
* @Version: V1.0
*/
@Mapper
public interface CollectionMapMapper extends BaseMapper<CollectionMap> {
}
......@@ -128,6 +128,7 @@ public interface EventMapper extends BaseMapper<Event> {
* @date 2024/4/30
*/
List<EventNewPlatVO> newPlatCustomerPageList(@Param("subjectCondition") SubjectCondition subjectCondition, @Param("offset") Integer offset, @Param("pageSize") Integer pageSize);
/**
* 总数量(客户)-新平台管理
*
......@@ -227,4 +228,97 @@ public interface EventMapper extends BaseMapper<Event> {
*/
List<Node> projectList(@Param("userId") String userId, @Param("customerId") String customerId);
/**
* 分类下的事件id集合
*
* @param userId 用户id
* @param typeIds 分类id集合
* @author lkg
* @date 2024/5/6
*/
List<String> selectSubjectByTypeIds(@Param("userId") String userId, @Param("typeIds") List<String> typeIds);
/**
* 用户下的事件id集合
*
* @param userId 用户id
* @param customerId 客户id
* @author lkg
* @date 2024/5/6
*/
List<String> selectSubjectWithCustomer(@Param("userId") String userId, @Param("customerId") String customerId);
/**
* 专题绑定关键词的id集合
*
* @param subjectIds 专题id集合
* @author lkg
* @date 2024/5/6
*/
List<String> bindKeyWordsIdList(@Param("subjectIds") List<String> subjectIds);
/**
* 专题绑定模型信息列表
*
* @param subjectIds 专题id集合
* @author lkg
* @date 2024/5/6
*/
List<LabelModelVo> selectLabelModelBySubjectId(@Param("subjectIds") List<String> subjectIds);
/**
* 企业标签下的企业信用代码集合
*
* @param labelIds 企业标签id集合
* @author lkg
* @date 2024/5/6
*/
List<String> codesByLabels(@Param("labelIds") List<String> labelIds);
/**
* 判断信息源是否在专题绑定的信息源组下
*
* @param subjectId 专题id
* @param sourceId 信息源id
* @author lkg
* @date 2024/4/24
*/
int ynBelowBindGroup(@Param("subjectId") String subjectId, @Param("sourceId") String sourceId);
/**
* 判断信息源是否在专题排除的信息源组下
*
* @param subjectId 专题id
* @param sourceId 信息源id
* @author lkg
* @date 2024/4/24
*/
int ynBelowExcludeGroup(@Param("subjectId") String subjectId, @Param("sourceId") String sourceId);
/**
* 专题绑定关键词信息-分页列表
*
* @param subjectIds 专题id集合
* @param groupName 词组名称
* @param wordName 关键词名称
* @param offset 偏移量
* @param pageSize 返回条数
* @author lkg
* @date 2024/5/7
*/
List<KeyWordsPage> bindKeyWordsList(@Param("subjectIds") List<String> subjectIds,
@Param("groupName") String groupName, @Param("wordName") String wordName,
@Param("offset") Integer offset, @Param("pageSize") Integer pageSize);
/**
* 专题绑定关键词信息-总数量
*
* @param subjectIds 专题id集合
* @param groupName 词组名称
* @param wordName 关键词名称
* @author lkg
* @date 2024/5/7
*/
Long bindKeyWordsCount(@Param("subjectIds") List<String> subjectIds, @Param("groupName") String groupName, @Param("wordName") String wordName);
}
package com.zzsn.event.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.event.vo.InfoSourceCondition;
import com.zzsn.event.vo.InfoSourceVo;
import com.zzsn.event.xxljob.entity.InfoSource;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
/**
* @Description: 信息源表
* @Author: jeecg-boot
......@@ -14,8 +18,58 @@ import org.apache.ibatis.annotations.Param;
@Mapper
public interface InfoSourceMapper extends BaseMapper<InfoSource> {
String getNatureName(@Param("id") String id);
/**
* 信息源信息列表
*
* @param sourceIdList 信息源id集合
* @author lkg
* @date 2024/5/7
*/
List<InfoSourceVo> queryInfoSource(@Param("sourceIdList") List<String> sourceIdList);
/**
* 专题绑定的信息源集合
*
* @param infoSourceCondition 筛选条件
* @param subjectIds 专题id集合
* @param offset 偏移量
* @param pageSize 返回条数
* @author lkg
* @date 2024/4/24
*/
List<String> bindSourceIdList(@Param("infoSourceVo") InfoSourceCondition infoSourceCondition, @Param("subjectIds") List<String> subjectIds, @Param("offset") Integer offset, @Param("pageSize") Integer pageSize);
/**
* 专题绑定的信息源总数量
*
* @param infoSourceCondition 筛选条件
* @param subjectIds 专题id集合
* @author lkg
* @date 2024/4/24
*/
Long bindSourceIdCount(@Param("infoSourceVo") InfoSourceCondition infoSourceCondition, @Param("subjectIds") List<String> subjectIds);
/**
* 专题直接屏蔽的信息源列表(不包括专题屏蔽的信息源组下的信息源)
*
* @param infoSourceCondition 筛选条件
* @param subjectId 专题id
* @param offset 偏移量
* @param pageSize 返回条数
* @author lkg
* @date 2024/4/30
*/
List<InfoSourceVo> unBindSourcePageList(@Param("infoSourceVo") InfoSourceCondition infoSourceCondition, @Param("subjectId") String subjectId, @Param("offset") Integer offset, @Param("pageSize") Integer pageSize);
/**
* 专题直接屏蔽的信息源数量(不包括专题屏蔽的信息源组下的信息源)
*
* @param infoSourceCondition 筛选条件
* @param subjectId 专题id
* @author lkg
* @date 2024/4/30
*/
Long unBindSourceCount(@Param("infoSourceVo") InfoSourceCondition infoSourceCondition, @Param("subjectId") String subjectId);
}
......@@ -3,6 +3,7 @@ package com.zzsn.event.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.zzsn.event.entity.SubjectInfoSourceMap;
import com.zzsn.event.vo.SubjectStatisticsVo;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
......@@ -33,4 +34,23 @@ public interface SubjectInfoSourceMapMapper extends BaseMapper<SubjectInfoSource
List<String> selectSubjectIdsByGroupId(@Param("infoSourceGroupId") String infoSourceGroupId);
/**
* 专题真正绑定的信息源id集合
*
* @param subjectIds 专题id集合
* @author lkg
* @date 2024/5/6
*/
List<String> subjectRealBindInfoSourceList(@Param("subjectIds") List<String> subjectIds);
/**
* 专题真正绑定的信息源集合
*
* @param subjectIds 专题id集合
* @author lkg
* @date 2024/5/7
*/
List<SubjectStatisticsVo> subjectRealBindInfoSources(@Param("subjectIds") List<String> subjectIds);
}
......@@ -139,7 +139,7 @@
<select id="newPlatPageList" resultType="com.zzsn.event.vo.EventNewPlatVO">
SELECT distinct d.id,d.event_name, m.type_name as subjectTypeName, n.project_name
SELECT distinct d.id,d.event_name,d.start_time,d.end_time, m.type_name as subjectTypeName, n.project_name
from
<choose>
<when test="subjectCondition.userId !=null and subjectCondition.userId != ''">
......@@ -252,8 +252,8 @@
</select>
<select id="newPlatCustomerPageList" resultType="com.zzsn.event.vo.EventNewPlatVO">
select distinct x.id,x.event_name,st.type_name as subjectTypeName,p.project_name from (
select c.id, c.event_name,c.create_time
select distinct x.id,x.event_name,x.start_time,x.end_time,st.type_name as subjectTypeName,p.project_name from (
select c.id, c.event_name,c.create_time,c.start_time,c.end_time,c.sort_order
from event c inner join
(
select a.id, b.permission_id
......@@ -294,7 +294,7 @@
<if test="subjectCondition.projectId!=null and subjectCondition.projectId != ''">
and p.id = #{subjectCondition.projectId}
</if>
order by x.create_time desc
order by x.sort_order, x.create_time desc
limit #{offset},#{pageSize}
</select>
<select id="newPlatCustomerCount" resultType="Integer">
......@@ -489,4 +489,147 @@
where s.is_delete = 0 and s.status = 1
</select>
<select id="selectSubjectByTypeIds" resultType="String">
select distinct a.id from event a
<if test="userId!=null and userId != ''">
inner join sys_user_data_permission b on b.permission_id = a.id and b.category = 'subject' and b.user_id = #{userId}
</if>
inner join subject_type_map c on c.subject_id = a.id
where 1=1
<if test="typeIds!=null and typeIds.size()>0">
and c.type_id in
<foreach collection="typeIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
</select>
<select id="selectSubjectWithCustomer" resultType="String">
select distinct a.id
<choose>
<when test="customerId!=null and customerId != ''">
from customer_data_permission_map b
inner join event a on b.permission_id = a.id
inner join subject_type_map c on c.subject_id = b.permission_id
inner join project_subject_map d on b.permission_id = d.subject_id
inner join customer_project_map e on d.project_id = e.project_id
where b.category = 'subject' and b.customer_id = #{customerId}
</when>
<when test="userId!=null and userId != ''">
from sys_user_data_permission b
inner join event a on b.permission_id = a.id
inner join subject_type_map c on c.subject_id = b.permission_id
inner join project_subject_map d on b.permission_id = d.subject_id
inner join customer_project_map e on d.project_id = e.project_id
where b.category = 'subject' and b.user_id = #{userId}
</when>
<otherwise>
from event a
</otherwise>
</choose>
</select>
<select id="bindKeyWordsIdList" resultType="String">
select distinct b.id from key_words b
left join subject_keywords_map a on a.keywords_id = b.id
left join keywords_type_map c on b.id = c.keywords_id
left join keywords_type d on d.id = c.type_id
where 1=1
<if test="subjectIds != null and subjectIds.size() > 0">
and a.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
</select>
<select id="selectLabelModelBySubjectId" resultType="com.zzsn.event.vo.LabelModelVo">
SELECT distinct a.subject_id,b.id as modelId,b.service_name as modelServiceName,b.label_id as labelId,c.label_mark,c.label_name,c.label_type
FROM subject_model_map a
inner JOIN clb_algorithm_model b ON a.model_id = b.id and a.sign = 1
LEFT JOIN sys_base_label_type c on c.id = b.label_id
WHERE b.service_name is not null and a.type = 3
<if test="subjectIds != null and subjectIds.size() > 0">
and a.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
</select>
<select id="codesByLabels" resultType="String">
select e.social_credit_code from sys_base_enterprise e
inner join sys_base_label_type_map m on e.social_credit_code = m.relation_id
where 1=1
<if test="labelIds!=null and labelIds.size()>0">
and m.label_id in
<foreach collection="labelIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by e.social_credit_code
</select>
<select id="ynBelowBindGroup" resultType="Integer">
select count(1) from subject_info_source_map n
left join info_source_group_map m on n.source_id = m.group_id
where n.type in (2,5) and n.subject_id = #{subjectId} and m.source_id = #{sourceId}
</select>
<select id="ynBelowExcludeGroup" resultType="Integer">
select count(1) from subject_info_source_map n
left join info_source_group_map m on n.source_id = m.group_id
where n.type = 4 and n.subject_id = #{subjectId} and m.source_id = #{sourceId}
</select>
<select id="bindKeyWordsList" resultType="com.zzsn.event.vo.KeyWordsPage">
select x.* from (
SELECT b.*, d.id as keyWordsTypeId, d.type_name as keyWordTypeNames, a.type as type, a.id as subjectKeyWordId
FROM key_words b
LEFT JOIN subject_keywords_map a ON a.keywords_id = b.id
LEFT JOIN keywords_type_map c ON b.id = c.keywords_id
LEFT JOIN keywords_type d ON d.id = c.type_id
where 1=1
<if test="subjectIds != null and subjectIds.size() > 0">
and a.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by b.id
) x
where 1=1
<if test="groupName != null and groupName !=''">
and x.words_name like CONCAT('%',#{groupName},'%')
</if>
<if test="wordName != null and wordName !=''">
and x.key_word like CONCAT('%',#{wordName},'%')
</if>
order by x.create_time desc
limit #{offset},#{pageSize}
</select>
<select id="bindKeyWordsCount" resultType="Long">
select count(1) from (
SELECT b.*, d.id as keyWordsTypeId, d.type_name as keyWordTypeNames, a.type as type, a.id as subjectKeyWordId
FROM key_words b
LEFT JOIN subject_keywords_map a ON a.keywords_id = b.id
LEFT JOIN keywords_type_map c ON b.id = c.keywords_id
LEFT JOIN keywords_type d ON d.id = c.type_id
where 1=1
<if test="subjectIds != null and subjectIds.size() > 0">
and a.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by b.id
) x
where 1=1
<if test="groupName != null and groupName !=''">
and x.words_name like CONCAT('%',#{groupName},'%')
</if>
<if test="wordName != null and wordName !=''">
and x.key_word like CONCAT('%',#{wordName},'%')
</if>
</select>
</mapper>
\ No newline at end of file
......@@ -3,7 +3,205 @@
<mapper namespace="com.zzsn.event.mapper.InfoSourceMapper">
<select id="getNatureName" resultType="String">
select nature_name from info_source_nature where id = #{id}
<select id="queryInfoSource" resultType="com.zzsn.event.vo.InfoSourceVo">
select t.*, g.infoSourceNatureIds as natureIds, g.infoSourceNatureNames, k.infoSourceGroupNames from (
select y.* from (
select a.*, c.type_name as infoSourceTypeName, c.id as infoSourceTypeId from info_source a
LEFT JOIN info_source_type_map b ON b.source_id = a.id
LEFT JOIN info_source_type c ON b.type_id = c.id
) y where 1=1
<if test="sourceIdList != null and sourceIdList.size() > 0">
and y.id in
<foreach collection="sourceIdList" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
)t
LEFT JOIN (
SELECT e.source_id, GROUP_CONCAT(f.nature_name SEPARATOR ',') as infoSourceNatureNames, GROUP_CONCAT(f.id
SEPARATOR ',') as infoSourceNatureIds from info_source_properties_map e
LEFT JOIN info_source_nature f ON f.id = e.nature_id
GROUP BY e.source_id) g ON t.id = g.source_id
LEFT JOIN (
SELECT h.source_id, GROUP_CONCAT(j.group_name SEPARATOR ',') as infoSourceGroupNames, GROUP_CONCAT(j.id
SEPARATOR ',') as infoSourceGroupIds from info_source_group_map h
LEFT JOIN info_source_group j ON j.id = h.group_id
GROUP BY h.source_id) k ON t.id = k.source_id
</select>
<select id="bindSourceIdList" resultType="String">
select distinct x.id from (
select iso.id,iso.web_site_name,iso.site_name,iso.site_uri,iso.`status`,iso.crawl_type,iso.create_time
from info_source_group_map m inner join info_source iso on m.source_id = iso.id
where m.group_id in (
select source_id from subject_info_source_map where type in(2,5)
<if test="subjectIds != null and subjectIds.size() > 0">
and subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by source_id
)
union
select iso.id,iso.web_site_name,iso.site_name,iso.site_uri,iso.`status`,iso.crawl_type,iso.create_time
from subject_info_source_map sm inner join info_source iso on sm.source_id = iso.id where sm.type = 1
<if test="subjectIds != null and subjectIds.size() > 0">
and sm.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
) x where 1=1
<if test="infoSourceVo.webSiteName!=null and infoSourceVo.webSiteName != ''">
and x.web_site_name like CONCAT('%',#{infoSourceVo.webSiteName},'%')
</if>
<if test="infoSourceVo.siteName!=null and infoSourceVo.siteName != ''">
and x.site_name like CONCAT('%',#{infoSourceVo.siteName},'%')
</if>
<if test="infoSourceVo.siteUri!=null and infoSourceVo.siteUri != ''">
and x.site_uri = #{infoSourceVo.siteUri}
</if>
<if test="infoSourceVo.status!=null and infoSourceVo.status != ''">
and x.status = #{infoSourceVo.status}
</if>
<if test="infoSourceVo.crawlType!=null and infoSourceVo.crawlType != 0">
and x.crawl_type = #{infoSourceVo.crawlType}
</if>
and x.id not in(
select distinct y.source_id from (
select m.source_id from info_source_group_map m
where m.group_id in (
select source_id from subject_info_source_map where type = 4
<if test="subjectIds != null and subjectIds.size() > 0">
and subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by source_id
)
union
select sm.source_id from subject_info_source_map sm where type = 3
<if test="subjectIds != null and subjectIds.size() > 0">
and sm.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
) y
)
order by x.create_time
limit #{offset},#{pageSize}
</select>
<select id="bindSourceIdCount" resultType="Long">
select count(1) from
(
select distinct x.id,x.web_site_name,x.site_name,x.site_uri,x.`status`,x.crawl_type from (
select iso.id,iso.web_site_name,iso.site_name,iso.site_uri,iso.`status`,iso.crawl_type
from info_source_group_map m inner join info_source iso on m.source_id = iso.id
where m.group_id in (
select source_id from subject_info_source_map where type in(2,5)
<if test="subjectIds != null and subjectIds.size() > 0">
and subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by source_id
)
union
select iso.id,iso.web_site_name,iso.site_name,iso.site_uri,iso.`status`,iso.crawl_type
from subject_info_source_map sm inner join info_source iso on sm.source_id = iso.id where sm.type = 1
<if test="subjectIds != null and subjectIds.size() > 0">
and sm.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
) x where 1=1
<if test="infoSourceVo.webSiteName!=null and infoSourceVo.webSiteName != ''">
and x.web_site_name like CONCAT('%',#{infoSourceVo.webSiteName},'%')
</if>
<if test="infoSourceVo.siteName!=null and infoSourceVo.siteName != ''">
and x.site_name like CONCAT('%',#{infoSourceVo.siteName},'%')
</if>
<if test="infoSourceVo.siteUri!=null and infoSourceVo.siteUri != ''">
and x.site_uri = #{infoSourceVo.siteUri}
</if>
<if test="infoSourceVo.status!=null and infoSourceVo.status != ''">
and x.status = #{infoSourceVo.status}
</if>
<if test="infoSourceVo.crawlType!=null and infoSourceVo.crawlType != 0">
and x.crawl_type = #{infoSourceVo.crawlType}
</if>
and x.id not in(
select distinct y.source_id from (
select m.source_id from info_source_group_map m
where m.group_id in (
select source_id from subject_info_source_map where type = 4
<if test="subjectIds != null and subjectIds.size() > 0">
and subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by source_id
)
union
select sm.source_id from subject_info_source_map sm where type = 3
<if test="subjectIds != null and subjectIds.size() > 0">
and sm.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
) y
)
) q
</select>
<select id="unBindSourcePageList" resultType="com.zzsn.event.vo.InfoSourceVo">
select s.id,s.info_source_code,s.web_site_name,s.site_name,s.site_uri,s.status,s.create_time from info_source s
inner join subject_info_source_map sm on s.id = sm.source_id
where sm.type = 3 and sm.subject_id = #{subjectId}
<if test="infoSourceVo.webSiteName!=null and infoSourceVo.webSiteName != ''">
and s.web_site_name like CONCAT('%',#{infoSourceVo.webSiteName},'%')
</if>
<if test="infoSourceVo.siteName!=null and infoSourceVo.siteName != ''">
and s.site_name like CONCAT('%',#{infoSourceVo.siteName},'%')
</if>
<if test="infoSourceVo.siteUri!=null and infoSourceVo.siteUri != ''">
and s.site_uri = #{infoSourceVo.siteUri}
</if>
<if test="infoSourceVo.status!=null and infoSourceVo.status != ''">
and s.status = #{infoSourceVo.status}
</if>
<if test="infoSourceVo.crawlType!=null and infoSourceVo.crawlType != 0">
and s.crawl_type = #{infoSourceVo.crawlType}
</if>
order by s.create_time desc
limit #{offset},#{pageSize}
</select>
<select id="unBindSourceCount" resultType="Long">
select count(1) from info_source s
inner join subject_info_source_map sm on s.id = sm.source_id
where sm.type = 3 and sm.subject_id = #{subjectId}
<if test="infoSourceVo.webSiteName!=null and infoSourceVo.webSiteName != ''">
and s.web_site_name like CONCAT('%',#{infoSourceVo.webSiteName},'%')
</if>
<if test="infoSourceVo.siteName!=null and infoSourceVo.siteName != ''">
and s.site_name like CONCAT('%',#{infoSourceVo.siteName},'%')
</if>
<if test="infoSourceVo.siteUri!=null and infoSourceVo.siteUri != ''">
and s.site_uri = #{infoSourceVo.siteUri}
</if>
<if test="infoSourceVo.status!=null and infoSourceVo.status != ''">
and s.status = #{infoSourceVo.status}
</if>
<if test="infoSourceVo.crawlType!=null and infoSourceVo.crawlType != 0">
and s.crawl_type = #{infoSourceVo.crawlType}
</if>
</select>
</mapper>
......@@ -79,4 +79,101 @@
where type = 1 and subject_id = #{subjectId} and source_id in (select source_id from info_source_group_map where group_id = #{infoSourceGroupId})
</update>
<select id="subjectRealBindInfoSourceList" resultType="String">
select distinct x.id from (
select iso.id
from info_source_group_map m inner join info_source iso on m.source_id = iso.id
where m.group_id in (
select source_id from subject_info_source_map where type in(2,5)
<if test="subjectIds != null and subjectIds.size() > 0">
and subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by source_id
)
union
select iso.id
from subject_info_source_map sm inner join info_source iso on sm.source_id = iso.id where sm.type = 1
<if test="subjectIds != null and subjectIds.size() > 0">
and sm.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
) x
where x.id not in(
select distinct y.source_id from (
select m.source_id from info_source_group_map m
where m.group_id in (
select source_id from subject_info_source_map where type = 4
<if test="subjectIds != null and subjectIds.size() > 0">
and subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by source_id
)
union
select sm.source_id from subject_info_source_map sm where type = 3
<if test="subjectIds != null and subjectIds.size() > 0">
and sm.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
) y
)
</select>
<select id="subjectRealBindInfoSources" resultType="com.zzsn.event.vo.SubjectStatisticsVo">
select distinct x.* from (
select iso.id,iso.web_site_name,iso.site_name,iso.site_uri
from info_source_group_map m inner join info_source iso on m.source_id = iso.id
where m.group_id in (
select source_id from subject_info_source_map where type in(2,5)
<if test="subjectIds != null and subjectIds.size() > 0">
and subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by source_id
)
union
select iso.id,iso.web_site_name,iso.site_name,iso.site_uri
from subject_info_source_map sm inner join info_source iso on sm.source_id = iso.id where sm.type = 1
<if test="subjectIds != null and subjectIds.size() > 0">
and sm.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
) x
where x.id not in(
select distinct y.source_id from (
select m.source_id from info_source_group_map m
where m.group_id in (
select source_id from subject_info_source_map where type = 4
<if test="subjectIds != null and subjectIds.size() > 0">
and subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
group by source_id
)
union
select sm.source_id from subject_info_source_map sm where type = 3
<if test="subjectIds != null and subjectIds.size() > 0">
and sm.subject_id in
<foreach collection="subjectIds" item="item" open="(" close=")" separator=",">
#{item}
</foreach>
</if>
) y
)
</select>
</mapper>
......@@ -15,7 +15,9 @@
<select id="subjectAndTypeTree" resultType="com.zzsn.event.vo.SubjectTreeVO">
select s.id,s.type_name as name,s.pid,'false' as ynSubject from subject_type s
select x.* from (
select s.id,s.type_name as name,s.pid,'false' as ynSubject,null as startTime,null as endTime,s.create_time
from subject_type s
<if test="userId !=null and userId != ''">
inner join sys_user_data_permission dp on s.id=dp.permission_id and dp.user_id = #{userId}
</if>
......@@ -24,7 +26,7 @@
</if>
where s.category = 2 and s.status = 1
union
select n.id,n.name,m.id as pid,'true' as ynSubject from
select n.id,n.name,m.id as pid,'true' as ynSubject,n.start_time,n.end_time,n.create_time from
(
select s.id,s.type_name as name,s.pid from subject_type s
<if test="userId !=null and userId != ''">
......@@ -38,7 +40,7 @@
inner join subject_type_map stm on m.id = stm.type_id
inner join
(
select s.id,s.event_name as name from event s
select s.id,s.event_name as name,s.start_time,s.end_time,s.create_time from event s
<if test="userId !=null and userId != ''">
inner join sys_user_data_permission dp on s.id=dp.permission_id and dp.user_id = #{userId}
</if>
......@@ -47,10 +49,12 @@
</if>
where s.status = 1
) n on stm.subject_id = n.id
) x
order by x.create_time desc
</select>
<select id="subjectAndCustomerTree" resultType="com.zzsn.event.vo.SubjectTreeVO">
select c.id, c.event_name as name, m.id as pid, 'true' as ynSubject
select c.id, c.event_name as name, m.id as pid, 'true' as ynSubject,c.start_time,c.end_time
from event c inner join
(
select a.id, b.permission_id
......@@ -70,6 +74,7 @@
<if test="customerId !=null and customerId != ''">
and m.id = #{customerId}
</if>
order by c.create_time desc
</select>
<select id="enableCustomerList" resultType="com.zzsn.event.vo.SubjectTreeVO">
......
package com.zzsn.event.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.event.entity.CollectionMap;
/**
* @Description: 文章收藏映射表
* @Author: jeecg-boot
* @Date: 2022-05-17
* @Version: V1.0
*/
public interface ICollectionMapService extends IService<CollectionMap> {
}
......@@ -61,7 +61,7 @@ public interface IEventService extends IService<Event> {
* @author lkg
* @date 2024/4/8
*/
IPage<EventFrontVO> frontPageList(String eventName, Integer eventType,String labelField, String labelName, String order, String orderType, Integer pageNo, Integer pageSize);
IPage<EventFrontVO> frontPageList(String eventName, Integer eventType, String labelField, String labelName, String order, String orderType, Integer pageNo, Integer pageSize);
/**
* 分页列表(专题类别)-新平台管理
......@@ -72,7 +72,7 @@ public interface IEventService extends IService<Event> {
* @author lkg
* @date 2024/4/28
*/
IPage<EventNewPlatVO> newPlatPageList(SubjectCondition subjectCondition,Integer pageNo, Integer pageSize);
IPage<EventNewPlatVO> newPlatPageList(SubjectCondition subjectCondition, Integer pageNo, Integer pageSize);
/**
* 分页列表(客户)-新平台管理
......@@ -83,7 +83,7 @@ public interface IEventService extends IService<Event> {
* @author lkg
* @date 2024/4/28
*/
IPage<EventNewPlatVO> newPlatCustomerPageList(SubjectCondition subjectCondition,Integer pageNo, Integer pageSize);
IPage<EventNewPlatVO> newPlatCustomerPageList(SubjectCondition subjectCondition, Integer pageNo, Integer pageSize);
/**
* 热点事件列表-前10
......@@ -151,4 +151,104 @@ public interface IEventService extends IService<Event> {
* @date 2024/4/29
*/
List<Node> projectList(String userId, String customerId);
/**
* 分类下的专题id集合
*
* @param userId 用户id
* @param typeIds 分类id集合
* @author lkg
* @date 2024/5/6
*/
List<String> selectSubjectByTypeIds(String userId, List<String> typeIds);
/**
* 用户下的事件id集合
*
* @param userId 用户id
* @param customerId 客户id
* @author lkg
* @date 2024/5/6
*/
List<String> selectSubjectWithCustomer(String userId, String customerId);
/**
* 专题绑定关键词的id集合
*
* @param subjectIds 专题id集合
* @author lkg
* @date 2024/5/6
*/
List<String> bindKeyWordsIdList(List<String> subjectIds);
/**
* 专题绑定模型信息列表
*
* @param subjectIds 专题id集合
* @author lkg
* @date 2024/5/6
*/
List<LabelModelVo> selectLabelModelBySubjectId(List<String> subjectIds);
/**
* 企业标签下的企业信用代码集合
*
* @param labelIds 企业标签id集合
* @author lkg
* @date 2024/5/6
*/
List<String> codesByLabels(List<String> labelIds);
/**
* 专题直接绑定信息源-批量
*
* @param directBindInfoVO 参数封装
* @author lkg
* @date 2024/4/24
*/
void directBindInfoSource(DirectBindInfoVO directBindInfoVO);
/**
* 专题直接解绑信息源-批量
*
* @param directBindInfoVO 参数封装
* @author lkg
* @date 2024/4/24
*/
void directRemoveInfoSource(DirectBindInfoVO directBindInfoVO);
/**
* 删除专题下已屏蔽的信息源(即恢复绑定关系)
*
* @param subjectId 专题id
* @param infoSourceId 信息源id
* @author lkg
* @date 2024/5/7
*/
void removeUnBindInfoSource(String subjectId,String infoSourceId);
/**
* 专题关联信息源时间段内的采集量统计
*
* @param subjectIds 专题id集合
* @param startDate 开始时间
* @param endDate 结束时间
* @author lkg
* @date 2024/5/7
*/
List<List<String>> subjectStatistics(List<String> subjectIds, String startDate, String endDate);
/**
* 专题绑定的关键词组-分页列表
*
* @param subjectIds 专题id集合
* @param groupName 词组名称
* @param wordName 关键词名称
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2024/5/7
*/
IPage<KeyWordsPage> bindKeyWordsList(List<String> subjectIds, String groupName, String wordName, Integer pageNo, Integer pageSize);
}
package com.zzsn.event.service;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.event.vo.InfoSourceCondition;
import com.zzsn.event.vo.InfoSourceVo;
import com.zzsn.event.xxljob.entity.InfoSource;
import java.util.List;
/**
* @Description: 信息源表
* @Author: jeecg-boot
......@@ -11,6 +16,29 @@ import com.zzsn.event.xxljob.entity.InfoSource;
*/
public interface IInfoSourceService extends IService<InfoSource> {
String getNatureName(String id);
/**
* 专题绑定信息源列表
*
* @param infoSourceCondition 筛选条件
* @param subjectIds 专题id集合
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2024/5/7
*/
IPage<InfoSourceVo> bindInfoSourceList(InfoSourceCondition infoSourceCondition, List<String> subjectIds, Integer pageNo, Integer pageSize);
/**
* 专题直接屏蔽的信息源列表(不包括专题屏蔽的信息源组下的信息源)
*
* @param infoSourceCondition 筛选条件
* @param subjectId 专题id
* @param pageNo 当前页
* @param pageSize 返回条数
* @author lkg
* @date 2024/4/30
*/
IPage<InfoSourceVo> unBindSourcePageList(InfoSourceCondition infoSourceCondition, String subjectId, Integer pageNo, Integer pageSize);
}
......@@ -2,6 +2,7 @@ package com.zzsn.event.service;
import com.baomidou.mybatisplus.extension.service.IService;
import com.zzsn.event.entity.SubjectInfoSourceMap;
import com.zzsn.event.vo.SubjectStatisticsVo;
import java.util.List;
......@@ -54,4 +55,21 @@ public interface ISubjectInfoSourceMapService extends IService<SubjectInfoSource
*/
List<String> querySubjectIdsByGroupId(String infoSourceGroupId);
/**
* 专题真正绑定的信息源id集合
*
* @param subjectIds 专题id集合
* @author lkg
* @date 2024/5/6
*/
List<String> subjectRealBindInfoSourceList(List<String> subjectIds);
/**
* 专题真正绑定的信息源集合
*
* @param subjectIds 专题id集合
* @author lkg
* @date 2024/5/6
*/
List<SubjectStatisticsVo> subjectRealBindInfoSources(List<String> subjectIds);
}
......@@ -7,7 +7,7 @@ import com.zzsn.event.constant.Constants;
import com.zzsn.event.entity.Event;
import com.zzsn.event.entity.SubjectAnalysis;
import com.zzsn.event.service.AnalysisService;
import com.zzsn.event.service.EsService;
import com.zzsn.event.es.EsService;
import com.zzsn.event.service.IEventService;
import com.zzsn.event.service.SubjectAnalysisService;
import com.zzsn.event.util.DateUtil;
......
package com.zzsn.event.service.impl;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.event.entity.CollectionMap;
import com.zzsn.event.mapper.CollectionMapMapper;
import com.zzsn.event.service.ICollectionMapService;
import org.springframework.stereotype.Service;
/**
* @Description: 文章收藏映射表
* @Author: jeecg-boot
* @Date: 2022-05-17
* @Version: V1.0
*/
@Service
public class CollectionMapServiceImpl extends ServiceImpl<CollectionMapMapper, CollectionMap> implements ICollectionMapService {
}
......@@ -2,17 +2,21 @@ package com.zzsn.event.service.impl;
import cn.hutool.core.map.MapUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.event.constant.Constants;
import com.zzsn.event.entity.*;
import com.zzsn.event.enums.BindTypeEnum;
import com.zzsn.event.enums.CodePrefixEnum;
import com.zzsn.event.es.EsService;
import com.zzsn.event.mapper.EventMapper;
import com.zzsn.event.service.*;
import com.zzsn.event.util.CodeGenerateUtil;
import com.zzsn.event.util.CronUtil;
import com.zzsn.event.util.DateUtil;
import com.zzsn.event.util.HanlpUtil;
import com.zzsn.event.util.tree.Node;
import com.zzsn.event.vo.*;
......@@ -296,6 +300,31 @@ public class EventServiceImpl extends ServiceImpl<EventMapper, Event> implements
}
@Override
public List<String> selectSubjectByTypeIds(String userId, List<String> typeIds) {
return baseMapper.selectSubjectByTypeIds(userId,typeIds);
}
@Override
public List<String> codesByLabels(List<String> labelIds) {
return baseMapper.codesByLabels(labelIds);
}
@Override
public List<LabelModelVo> selectLabelModelBySubjectId(List<String> subjectIds) {
return baseMapper.selectLabelModelBySubjectId(subjectIds);
}
@Override
public List<String> bindKeyWordsIdList(List<String> subjectIds) {
return baseMapper.bindKeyWordsIdList(subjectIds);
}
@Override
public List<String> selectSubjectWithCustomer(String userId, String customerId) {
return baseMapper.selectSubjectWithCustomer(userId,customerId);
}
@Override
public List<EventExcelVO> frontList(List<String> eventIdList, Integer size) {
return baseMapper.frontList(eventIdList, size);
}
......@@ -317,6 +346,130 @@ public class EventServiceImpl extends ServiceImpl<EventMapper, Event> implements
return rn;
}
@Override
public void directBindInfoSource(DirectBindInfoVO directBindInfoVO) {
String subjectId = directBindInfoVO.getSubjectId();
List<String> bindList = directBindInfoVO.getBindList();
Integer type = BindTypeEnum.INFO_SOURCE.getvalue();
for (String infoSourceId : bindList) {
/*
* 1.先删除该信息源在专题下的所有数据
* 2.再判断该信息源在专题已绑定的信息源组下,若不在则新增绑定关系;若在则跳过
*/
LambdaQueryWrapper<SubjectInfoSourceMap> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(SubjectInfoSourceMap::getSourceId,infoSourceId)
.eq(SubjectInfoSourceMap::getSubjectId,subjectId);
iSubjectInfoSourceMapService.remove(queryWrapper);
int num = baseMapper.ynBelowBindGroup(subjectId, infoSourceId);
if (num == 0) {
SubjectInfoSourceMap subjectInfoSourceMap = new SubjectInfoSourceMap();
subjectInfoSourceMap.setSourceId(infoSourceId);
subjectInfoSourceMap.setSubjectId(subjectId);
subjectInfoSourceMap.setType(type);
iSubjectInfoSourceMapService.save(subjectInfoSourceMap);
}
}
}
@Override
public void directRemoveInfoSource(DirectBindInfoVO directBindInfoVO) {
String subjectId = directBindInfoVO.getSubjectId();
List<String> excludeList = directBindInfoVO.getExcludeList();
for (String infoSourceId : excludeList) {
/*
* 1.先删除该信息源在专题下的所有数据
* 2.再判断该信息源在专题已排除的信息源组下,若不在则新增绑定关系;若在则跳过
*/
LambdaQueryWrapper<SubjectInfoSourceMap> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(SubjectInfoSourceMap::getSourceId,infoSourceId)
.eq(SubjectInfoSourceMap::getSubjectId,subjectId);
iSubjectInfoSourceMapService.remove(queryWrapper);
int num = baseMapper.ynBelowExcludeGroup(subjectId, infoSourceId);
if (num == 0) {
SubjectInfoSourceMap subjectInfoSourceMap = new SubjectInfoSourceMap();
subjectInfoSourceMap.setSourceId(infoSourceId);
subjectInfoSourceMap.setSubjectId(subjectId);
subjectInfoSourceMap.setType(BindTypeEnum.EXCLUDE_INFO_SOURCE.getvalue());
iSubjectInfoSourceMapService.save(subjectInfoSourceMap);
}
}
}
@Override
public void removeUnBindInfoSource(String subjectId, String infoSourceId) {
int num = baseMapper.ynBelowExcludeGroup(subjectId, infoSourceId);
if (num == 0) {
LambdaUpdateWrapper<SubjectInfoSourceMap> update = Wrappers.lambdaUpdate();
iSubjectInfoSourceMapService.update(update.eq(SubjectInfoSourceMap::getSubjectId,subjectId)
.eq(SubjectInfoSourceMap::getSourceId,infoSourceId)
.eq(SubjectInfoSourceMap::getType,3)
.set(SubjectInfoSourceMap::getType,1));
} else {
LambdaQueryWrapper<SubjectInfoSourceMap> queryWrapper = Wrappers.lambdaQuery();
queryWrapper.eq(SubjectInfoSourceMap::getSourceId, infoSourceId)
.eq(SubjectInfoSourceMap::getSubjectId, subjectId)
.eq(SubjectInfoSourceMap::getType, 3);
iSubjectInfoSourceMapService.remove(queryWrapper);
}
}
@Override
public List<List<String>> subjectStatistics(List<String> subjectIds, String startDate, String endDate) {
List<String> dateList = DateUtil.betweenDate(startDate, endDate);
List<List<String>> dataList = new ArrayList<>();
//专题关联的信息源集合
List<SubjectStatisticsVo> infoSourceList = iSubjectInfoSourceMapService.subjectRealBindInfoSources(subjectIds);
Map<String, Map<String, Long>> stringMapMap = esService.regetFromEs(infoSourceList.stream().map(SubjectStatisticsVo::getId).collect(Collectors.toList()), startDate, endDate);
//封装导出excel需要的数据集合
for (SubjectStatisticsVo subjectStatisticsVo : infoSourceList) {
Map<String, Long> stringLongMap = stringMapMap.get(subjectStatisticsVo.getId());
List<SubjectStatisticsVo> value = new ArrayList<>();
int sum = 0;
if (null != stringLongMap) {
for (String s : stringLongMap.keySet()) {
SubjectStatisticsVo subjectStatisticsVoTemp = new SubjectStatisticsVo();
subjectStatisticsVoTemp.setId(subjectStatisticsVo.getId());
subjectStatisticsVoTemp.setSiteName(subjectStatisticsVo.getSiteName());
subjectStatisticsVoTemp.setDate(s);
int num = Math.toIntExact(stringLongMap.get(s));
subjectStatisticsVoTemp.setCollectCount(num);
value.add(subjectStatisticsVoTemp);
sum += Math.toIntExact(stringLongMap.get(s));
}
}
List<String> list = new ArrayList<>(subjectStatisticsVo.toExcelList());
//信息源在时间段内的采集总量
list.add(String.valueOf(sum));
//信息源采集到数据的日期集合
Set<String> collectDates = value.stream().collect(Collectors.groupingBy(SubjectStatisticsVo::getDate)).keySet();
//信息源未采集到数据的日期集合
List<String> noCollectDates = new ArrayList<>(dateList);
noCollectDates.removeAll(collectDates);
for (String date : noCollectDates) {
SubjectStatisticsVo vo = new SubjectStatisticsVo();
BeanUtils.copyProperties(subjectStatisticsVo, vo);
vo.setCollectCount(0);
vo.setDate(date);
value.add(vo);
}
//根据采集时间排序(正序)
value.sort(Comparator.comparing(SubjectStatisticsVo::getDate));
value.forEach(e -> list.add(String.valueOf(e.getCollectCount())));
dataList.add(list);
}
return dataList;
}
@Override
public IPage<KeyWordsPage> bindKeyWordsList(List<String> subjectIds,String groupName,String wordName,Integer pageNo,Integer pageSize) {
int offset = (pageNo - 1) * pageSize;
List<KeyWordsPage> wordsList = baseMapper.bindKeyWordsList(subjectIds, groupName, wordName, offset, pageSize);
Long count = baseMapper.bindKeyWordsCount(subjectIds, groupName, wordName);
IPage<KeyWordsPage> page = new Page<>(pageNo, pageSize, count);
page.setRecords(wordsList);
return page;
}
private void addRegionMap(String eventId, List<RegionVO> regionList) {
if (CollectionUtils.isNotEmpty(regionList)) {
List<EventRegionMap> dataList = new ArrayList<>();
......
package com.zzsn.event.service.impl;
import cn.hutool.core.date.DateField;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.event.es.EsService;
import com.zzsn.event.mapper.InfoSourceMapper;
import com.zzsn.event.service.IInfoSourceService;
import com.zzsn.event.vo.CountVO;
import com.zzsn.event.vo.InfoSourceCondition;
import com.zzsn.event.vo.InfoSourceVo;
import com.zzsn.event.vo.NumVO;
import com.zzsn.event.xxljob.entity.InfoSource;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
/**
* @Description: 信息源表
* @Author: jeecg-boot
......@@ -16,11 +32,71 @@ import org.springframework.stereotype.Service;
@Service
public class InfoSourceServiceImpl extends ServiceImpl<InfoSourceMapper, InfoSource> implements IInfoSourceService {
@Autowired
private EsService esService;
@Override
public String getNatureName(String id) {
return baseMapper.getNatureName(id);
public IPage<InfoSourceVo> bindInfoSourceList(InfoSourceCondition infoSourceCondition, List<String> subjectIds, Integer pageNo, Integer pageSize) {
int offset = (pageNo - 1) * pageSize;
if (pageNo == 0) {
offset = 0;
}
List<InfoSourceVo> pageList = new ArrayList<>();
List<String> bindList = baseMapper.bindSourceIdList(infoSourceCondition,subjectIds,offset,pageSize);
if (CollectionUtils.isNotEmpty(bindList)) {
pageList = baseMapper.queryInfoSource(bindList);
for (InfoSourceVo infoSourceVo1 : pageList) {
//性质
if (StringUtils.isNotEmpty(infoSourceVo1.getNatureIds())) {
String natureIds = infoSourceVo1.getNatureIds();
List<String> natureIdList = Arrays.asList(natureIds.split(","));
infoSourceVo1.setInfoSourceNatureIds(natureIdList);
}
//采集量
NumVO totalCollectCount = esService.count(infoSourceVo1.getId(), null, null, 1);
//总采集量
infoSourceVo1.setTotalCount(totalCollectCount.getCount());
Date endTime = new Date();
DateTime startTime = DateUtil.offsetDay(endTime, -30);
//月采集量
NumVO monthCollectCount = esService.count(infoSourceVo1.getId(), DateUtil.formatDate(startTime), DateUtil.formatDate(endTime), 2);
infoSourceVo1.setMonthCount(monthCollectCount.getCount());
List<NumVO> children = monthCollectCount.getChildren();
Map<String, List<NumVO>> collect = children.stream().collect(Collectors.groupingBy(NumVO::getKey));
//周采集量
long weekCount = 0L;
List<DateTime> rangeToList = DateUtil.rangeToList(DateUtil.offsetDay(endTime, -7), endTime, DateField.DAY_OF_YEAR);
for (DateTime dateTime : rangeToList) {
List<NumVO> countVOS = collect.get(DateUtil.formatDate(dateTime));
if (CollectionUtils.isNotEmpty(countVOS)) {
weekCount += countVOS.get(0).getCount();
}
}
infoSourceVo1.setWeekCount(weekCount);
//昨日采集量
DateTime yesterday = DateUtil.yesterday();
List<NumVO> countVOS = collect.get(DateUtil.formatDate(yesterday));
if (CollectionUtils.isNotEmpty(countVOS)) {
infoSourceVo1.setYesterdayCount(countVOS.get(0).getCount());
} else {
infoSourceVo1.setYesterdayCount(0L);
}
}
}
Long count = baseMapper.bindSourceIdCount(infoSourceCondition, subjectIds);
IPage<InfoSourceVo> pageData = new Page<>(pageNo, pageSize, count);
pageData.setRecords(pageList);
return pageData;
}
@Override
public IPage<InfoSourceVo> unBindSourcePageList(InfoSourceCondition infoSourceCondition, String subjectId, Integer pageNo, Integer pageSize) {
int offset = (pageNo - 1) * pageSize;
List<InfoSourceVo> pageList = baseMapper.unBindSourcePageList(infoSourceCondition, subjectId, offset, pageSize);
Long count = baseMapper.unBindSourceCount(infoSourceCondition, subjectId);
IPage<InfoSourceVo> pageData = new Page<>(pageNo, pageSize, count);
pageData.setRecords(pageList);
return pageData;
}
}
......@@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.zzsn.event.entity.SubjectInfoSourceMap;
import com.zzsn.event.mapper.SubjectInfoSourceMapMapper;
import com.zzsn.event.service.ISubjectInfoSourceMapService;
import com.zzsn.event.vo.SubjectStatisticsVo;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
......@@ -54,6 +55,16 @@ public class SubjectInfoSourceMapServiceImpl extends ServiceImpl<SubjectInfoSour
}
@Override
public List<SubjectStatisticsVo> subjectRealBindInfoSources(List<String> subjectIds) {
return baseMapper.subjectRealBindInfoSources(subjectIds);
}
@Override
public List<String> subjectRealBindInfoSourceList(List<String> subjectIds) {
return baseMapper.subjectRealBindInfoSourceList(subjectIds);
}
@Override
public void updateMain(SubjectInfoSourceMap subjectInfoSourceMap){
//更新信息源组信息权重
baseMapper.updateById(subjectInfoSourceMap);
......
......@@ -15,6 +15,7 @@ import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
......@@ -90,6 +91,7 @@ public class SubjectTypeServiceImpl extends ServiceImpl<SubjectTypeMapper, Subje
List<SubjectTreeVO> subjectTreeVOS = baseMapper.subjectAndTypeTree(userId, customerId);
List<SubjectTreeVO> tree = TreeUtil.tree(subjectTreeVOS, "0");
tree.forEach(this::subjectNumCount);
tree.sort(Comparator.comparing(SubjectTreeVO::getCreateTime));
return tree;
}
......
......@@ -8,7 +8,7 @@ import com.zzsn.event.constant.Constants;
import com.zzsn.event.entity.Event;
import com.zzsn.event.entity.SubjectAnalysis;
import com.zzsn.event.service.AnalysisService;
import com.zzsn.event.service.EsService;
import com.zzsn.event.es.EsService;
import com.zzsn.event.service.IEventService;
import com.zzsn.event.service.SubjectAnalysisService;
import com.zzsn.event.util.DateUtil;
......
......@@ -4,10 +4,7 @@ import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.*;
public class DateUtil {
......@@ -116,4 +113,31 @@ public class DateUtil {
return dateToString(calendar.getTime(),"yyyy-MM-dd 00:00:00");
}
public static List<String> betweenDate(String startDate,String endDate) {
List<String> dateList = new ArrayList<>();
String formatStr = "yyyy-MM-dd";
try {
Date dayOne = stringToDate(startDate, formatStr);
Date dayTwo = stringToDate(endDate, formatStr);
Calendar calendar = Calendar.getInstance();
calendar.setTime(dayOne);
dateList.add(startDate);
while (dayTwo.after(calendar.getTime())) {
calendar.add(Calendar.DAY_OF_MONTH,1);
dateList.add(format(calendar.getTime(),formatStr));
}
} catch (Exception e) {
e.printStackTrace();
}
return dateList;
}
public static String format(Date d, String format){
if (d == null) {
return "";
}
SimpleDateFormat myFormatter = new SimpleDateFormat(format);
myFormatter.setTimeZone(TimeZone.getTimeZone("Asia/Shanghai"));
return myFormatter.format(d);
}
}
package com.zzsn.event.vo;
import lombok.Data;
@Data
public class ClbNewKeywordsVO {
/**主键*/
private String id;
/**关键词名称*/
private String name;
/**专题id*/
private String subjectId;
/**父分类ID-关联本表ID*/
private String pid;
/**状态(0:未审核,1:通过,2:待定,3:删除,4:修改)*/
private Integer status;
/**创建人*/
private String createBy;
/**创建日期*/
private java.util.Date createTime;
/**更新人*/
private String updateBy;
/**更新日期*/
private java.util.Date updateTime;
/**所属部门*/
private String sysOrgCode;
/**数据类型(1:新词 2:人名 3:热词 4:地名 5:机构名 6:热词)*/
private Integer type;
/**资讯标题*/
private String title;
/**资讯id*/
private String articleId;
/**发布时间*/
private java.util.Date publishDate;
}
package com.zzsn.event.vo;
import lombok.Data;
import java.util.List;
/**
*
*
* @author lkg
* @date 2024/4/24
*/
@Data
public class DirectBindInfoVO {
//专题id
private String subjectId;
//绑定信息源id集合
private List<String> bindList;
//排除信息源id集合
private List<String> excludeList;
}
package com.zzsn.event.vo;
import lombok.Data;
import java.util.List;
@Data
public class DisplayInfo {
//说明:...Raw 表示原文,即原语言
//作者
private String author;
private String authorRaw;
//正文
private String content;
private String contentRaw;
//带标签正文
private String contentWithTag;
private String contentWithTagRaw;
//入库时间
private String createDate;
//信息id
private String id;
//信息来源id(信息源或者关键词)
private String sid;
//语言
private String lang;
private String langRaw;
//来源(信息来源)
private String origin;
private String originRaw;
//发布时间
private String publishDate;
//原始时间(网页上,没有解析之前的时间)
private String originalTime;
//发布地址
private String sourceAddress;
//摘要
private String summary;
private String summaryRaw;
//关键词
private String keyWords;
//标题
private String title;
private String titleRaw;
//采集来源(如通用、定制、微信公众号等)
private String source;
//附加字段
private String type;
//标签信息
private List<Label> labels;
//视频下载链接
private String downLoadUrl;
//视频链接(原链接 网页版)
private String videoUrl;
//视频链接(原链接 手机版)
private String videoPhoneUrl;
//视频时长
private Long videoTime;
//视频第一帧图片
private String videoImg;
//自定义标签
private List<String> customLabel;
private Double score;
//专题库类型(0: 其它 1:政策;2:领导讲话;3:专家观点;4:企业案例)
private Integer classificationType;
//置顶排位(默认为0)
private Integer topNum;
//删除标记(1:删除;0:保留)
private Integer deleteFlag;
private String subjectId;
//审核操作(0:未审核 1:审核通过 2:审核未通过 3:暂定 默认值为0)
private Integer checkStatus;
//阅读数
private Long readNum = 0L;
//是否收藏
private boolean ynCollect = false;
//重复id
private String repeatId;
//(1:主条目 0:非主条目)
private String flag;
//关联的主条目id
private String masterEntryId;
//原始id(去重服务生成的id,因为一条信息可以属于多个专题,原始id会发生改变,所以存储一个原始id,找寻对应关系)
private String originalId;
//快照地址
private String screenShotImg;
//资讯更新时间
private String updateDate;
//信息类别(1:报刊 2:博客 3:客户端 4:论坛 5:视频 6:外媒 7:网站 8:微博 9:微信 10:新闻 11:政务 12:其它)
private String infoSourceType;
//数据类型 qbyw:情报要闻 qbnc:情报内参
private String dataType;
//abi报表地址
private String abiUrl;
//abi报表id
private String abiId;
//栏目code列表
private List<String> programaIds;
//数据库code列表
private List<String> databaseIds;
//是否风险
private Integer isRisk;
//风险类型
private List<String> riskTypes;
//标签集合
private List<LabelInfo> labelInfos;
//是否为资讯(true:资讯 false:附件类)
private Boolean ynArticle = true;
//法规号
private String contentNo;
//来源 (政策发文机关)
private String organ;
//政策文件分类
private String topicClassification;
//发文字号
private String issuedNumber;
//成文时间
private String writtenDate;
//数据所在的索引名称
private String index;
/**
* 是否是主条目,0为非主条目,1为主条目
*/
private Integer ismain;
}
package com.zzsn.event.vo;
import lombok.Data;
import java.util.List;
@Data
public class EventDataCondition {
private String id;
//专题id
private String subjectId;
//文章链接
private String url;
//搜索时,全部情况下,,搜索传值
private String search;
//专题标题
private String title;
//专题摘要
private String summary;
//内容
private String content;
//作者
private String author;
//发布时间
private String publishDate;
//来源
private String origin;
//得分
private String score;
//企业名称
private String enterpriseName;
//倾向性
private String orientation;
//风险类型
private String riskType;
//企业性质
private String enterpriseNature;
//区域
private String area;
//行业
private String trade;
//信息类型
private String InfoType;
//专题库类型
private String libraryType;
//ids
private List<String> ids;
//置顶标识(0取消置顶 1置顶)
private Integer type;
private String sourceAddress;
//开始时间
private String startTime;
//结束时间
private String endTime;
//倾向性(前端交互参数约定 0:中性 1:负面 2:正面)
private Integer tendency;
//专题库类型(1:政策;2:领导讲话;3:专家观点;4:企业案例)
private Integer classificationType;
//审核操作(0:未审核 1:审核通过 2:审核未通过 3:暂定 默认值为0)
private List<Integer> checkStatusList;
private Integer checkStatus;
//删除标记(1:删除;0:保留)
private Integer deleteFlag = 0;
//国家
private String countryIds;
//城市
private String cityIds;
//企业
private String enterpriseIds;
//人物
private String characterIds;
//字段
private String fields;
//操作类型 add .update
private String action;
//日期最大值
private String maxValue;
//关键词检索范围 1标题 2正文 3 全部
private Integer searchScope;
//精确度("精确"| --)
private String searchAccuracy;
//数据所在的索引名称
private String index;
private String operateStartTime;
private String operateEndTime;
private String labelName;
}
package com.zzsn.event.vo;
import lombok.Data;
import java.util.Date;
import java.util.List;
/**
* 数据
*/
@Data
public class EventDataVO {
//说明:...Raw 表示原文,即原语言
//作者
private String author;
private String authorRaw;
//正文
private String content;
private String contentRaw;
//带标签正文
private String contentWithTag;
private String contentWithTagRaw;
//入库时间
private String createDate;
private Date createDateTime;
//信息id
private String id;
//信息来源id(信息源或者关键词)
private String sid;
//语言
private String lang;
private String langRaw;
//来源(信息来源)
private String origin;
private String originRaw;
//发布时间
private String publishDate;
//原始时间(网页上,没有解析之前的时间)
private String originalTime;
//发布地址
private String sourceAddress;
//摘要
private String summary;
private String summaryRaw;
//关键词
private String keyWords;
//标题
private String title;
private String titleRaw;
//采集来源(如通用、定制、微信公众号等)
private String source;
//附加字段
private String type;
//标签信息
private List<Label> labels;
//视频下载链接
private String downLoadUrl;
//视频链接(原链接 网页版)
private String videoUrl;
//视频链接(原链接 手机版)
private String videoPhoneUrl;
//视频时长
private Long videoTime;
//视频第一帧图片
private String videoImg;
//重复id
private String repeatId;
//(1:主条目 0:非主条目)
private String flag;
//关联的主条目id
private String masterEntryId;
//原始id(去重服务生成的id,因为一条信息可以属于多个专题,原始id会发生改变,所以存储一个原始id,找寻对应关系)
private String originalId;
private String subjectId;
//快照地址
private String screenShotImg;
//数据所在的索引名称
private String index;
}
......@@ -12,6 +12,8 @@ import lombok.Data;
public class EventNewPlatVO {
private String id;
private String eventName;
private String startTime;
private String endTime;
private Integer subjectInfoNum;
private Integer unCheckNum;
private Integer infoSourceNum;
......
package com.zzsn.event.vo;
import lombok.Data;
/**
* 信息源搜索条件
*
* @author lkg
* @date 2024/5/7
*/
@Data
public class InfoSourceCondition {
/**信息源名称,微信公众号名称,微博名称*/
private String webSiteName;
/**栏目名称*/
private String siteName;
/**栏目URL,微信公众号URL,微博URL*/
private String siteUri;
/**状态*/
private String status;
/**爬虫类别*/
private int crawlType;
}
......@@ -253,4 +253,13 @@ public class InfoSourceVo {
@JsonFormat(timezone = "GMT+8",pattern = "yyyy-MM-dd HH:mm:ss")
@DateTimeFormat(pattern="yyyy-MM-dd HH:mm:ss")
private Date updateTime;
/**总采集量*/
private Long totalCount;
/**昨日采集量*/
private Long yesterdayCount;
/**近一周采集量*/
private Long weekCount;
/**近一月采集量*/
private Long monthCount;
}
package com.zzsn.event.vo;
import lombok.Data;
import java.util.List;
@Data
public class LabelInfo {
//标签id
private String labelId;
//标签名称
private String labelName;
//标签类别
private String labelType;
//标签标识
private String labelMark;
//标签
private List<Label> labelList;
}
package com.zzsn.event.vo;
import lombok.Data;
@Data
public class LabelModelVo {
/**专题id*/
private String subjectId;
/**模型对应的id*/
private String modelId;
/**模型服务名称*/
private String modelServiceName;
/**标签id*/
private String labelId;
/**模型对应的标签标识*/
private String labelMark;
/**模型对应的标签名称*/
private String labelName;
/**模型类别*/
private String labelType;
}
package com.zzsn.event.vo;
import lombok.Data;
/**
*
*
* @author lkg
* @date 2024/5/7
*/
@Data
public class NewKeywordsCondition {
/**专题id*/
private String subjectId;
/**关键词名称*/
private String name;
/**数据类型(1:新词 2:人名 3:热词 4:地名 5:机构名 6:热词)*/
private Integer type;
/**状态(0:未审核,1:通过,2:待定,3:删除,4:修改)*/
private Integer status;
}
package com.zzsn.event.vo;
import lombok.Data;
import java.util.List;
/**
*
*
* @author lkg
* @date 2024/5/7
*/
@Data
public class NumVO {
//时间或者来源或者项目名称或者热词名称
private String key;
//数量
private long count;
//子集
private List<NumVO> children;
}
package com.zzsn.event.vo;
import lombok.Data;
import java.util.ArrayList;
import java.util.List;
/**
* @author lkg
* @description: 专题关联信息源,采集数量统计封装类
* @date 2022/12/29 11:32
*/
@Data
public class SubjectStatisticsVo {
//信息源id
String id;
//网站名称
String webSiteName;
//栏目名称
String siteName;
//栏目地址
String siteUri;
//采集数量
int collectCount;
//采集日期
String date;
public List<String> toExcelList(){
List<String> list = new ArrayList<>();
list.add(id);
list.add(webSiteName);
list.add(siteName);
list.add(siteUri);
return list;
}
}
......@@ -17,4 +17,8 @@ public class SubjectTreeVO extends Node {
private Boolean ynSubject;
private Integer subjectCount=0;
private String startTime;
private String endTime;
private String createTime;
}
......@@ -110,6 +110,8 @@ spring:
key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
#值的反序列化器类,实现类实现了接口org.apache.kafka.common.serialization.Deserializer
value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
main:
allow-circular-references: true
mybatis-plus:
mapper-locations: classpath*:com/zzsn/event/**/xml/*.xml
configuration:
......
......@@ -107,6 +107,8 @@ spring:
key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
#值的反序列化器类,实现类实现了接口org.apache.kafka.common.serialization.Deserializer
value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
main:
allow-circular-references: true
mybatis-plus:
mapper-locations: classpath*:com/zzsn/event/**/xml/*.xml
configuration:
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论