Skip to content

K
know-base
提交 0c25f2e7 作者: obcy
操作

Merge remote-tracking branch 'origin/master'

上级 7117ce2a d9d646e2
正在显示 包含 516 行增加 和 80 行删除
src/main/java/com/zzsn/knowbase/controller/KnowledgeController.java
...@@ -29,14 +29,12 @@ import javax.servlet.http.HttpServletRequest; ...@@ -29,14 +29,12 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.net.URLEncoder;
import java.util.Date; import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
/** /**
* @Description: 知识 * 知识
* @Author: chenshiqiang * @Author: chenshiqiang
* @Version: V1.0 * @Version: V1.0
*/ */
...@@ -54,6 +52,9 @@ public class KnowledgeController { ...@@ -54,6 +52,9 @@ public class KnowledgeController {
@Autowired @Autowired
private ILocalFileService localFileService; private ILocalFileService localFileService;
@Autowired
private RedisUtil redisUtil;
/** /**
* 分页列表查询 * 分页列表查询
*/ */
...@@ -78,6 +79,9 @@ public class KnowledgeController { ...@@ -78,6 +79,9 @@ public class KnowledgeController {
public Result<?> getById(@RequestParam String id) { public Result<?> getById(@RequestParam String id) {
log.info("{}===query begin", new Date()); log.info("{}===query begin", new Date());
KnowledgeVO knowledgeVO = knowledgeService.getById(id); KnowledgeVO knowledgeVO = knowledgeService.getById(id);
if(knowledgeVO==null){
return Result.OK("nothing");
}
if (null != knowledgeVO.getPublishDate() && (!knowledgeVO.getPublishDate().isEmpty())) { if (null != knowledgeVO.getPublishDate() && (!knowledgeVO.getPublishDate().isEmpty())) {
knowledgeVO.setPublishDate(EsDateUtil.esFieldDateMapping(knowledgeVO.getPublishDate())); knowledgeVO.setPublishDate(EsDateUtil.esFieldDateMapping(knowledgeVO.getPublishDate()));
} }
...@@ -154,7 +158,7 @@ public class KnowledgeController { ...@@ -154,7 +158,7 @@ public class KnowledgeController {
* @return * @return
*/ */
@PostMapping(value = "/delete") @PostMapping(value = "/delete")
public Result<?> delete( @RequestBody DeleteParam deleteParam) { public Result<?> delete(@RequestBody DeleteParam deleteParam) {
knowledgeService.deleteKnowledge(deleteParam.getKnowledgeList()); knowledgeService.deleteKnowledge(deleteParam.getKnowledgeList());
return Result.OK("删除成功!"); return Result.OK("删除成功!");
} }
...@@ -177,6 +181,7 @@ public class KnowledgeController { ...@@ -177,6 +181,7 @@ public class KnowledgeController {
params.put("score_threshold", intelligentQaParam.getScoreThreshold()); params.put("score_threshold", intelligentQaParam.getScoreThreshold());
params.put("question", intelligentQaParam.getQuestion()); params.put("question", intelligentQaParam.getQuestion());
params.put("knowledge_base_id", intelligentQaParam.getKbKnowledgeIds().split(",")); params.put("knowledge_base_id", intelligentQaParam.getKbKnowledgeIds().split(","));
params.put("size", intelligentQaParam.getSize());
String result = HttpUtil.doPost(intelligentQaUrl, params, 120000); String result = HttpUtil.doPost(intelligentQaUrl, params, 120000);
if (!result.isEmpty()) { if (!result.isEmpty()) {
JSONObject jsonObject = JSON.parseObject(result); JSONObject jsonObject = JSON.parseObject(result);
...@@ -194,7 +199,7 @@ public class KnowledgeController { ...@@ -194,7 +199,7 @@ public class KnowledgeController {
public Result<?> importInfo(HttpServletRequest request, HttpServletResponse response) { public Result<?> importInfo(HttpServletRequest request, HttpServletResponse response) {
KbAuthorizedUser userInfo = SpringContextUtils.getUserInfo(); KbAuthorizedUser userInfo = SpringContextUtils.getUserInfo();
assert userInfo != null; assert userInfo != null;
return knowledgeService.doImportInfo(request, userInfo.getId()); return knowledgeService.doImportInfo(request, userInfo.getId());
} }
...@@ -214,8 +219,89 @@ public class KnowledgeController { ...@@ -214,8 +219,89 @@ public class KnowledgeController {
} }
return Result.OK(); return Result.OK();
} }
/**
* 预导出
*
* @param newsIds 资讯知识ids
* @param fileIds 文件知识ids
*/
@PostMapping(value = "/export")
public Result<?> exportCollectionXls(HttpServletRequest request,
String newsIds, String fileIds) {
log.info("newsIds:{},fileIds:{}",newsIds,fileIds);
if(!newsIds.isEmpty()){
knowledgeService.exportExcel(newsIds);
}
if(!fileIds.isEmpty()){
knowledgeService.exportPackage(fileIds);
}
return Result.OK(true);
}
/**
* 导出
*
* @param newsIds 资讯知识ids
* @param fileIds 文件知识ids
*/
@RequestMapping(value = "/downloadFile")
public void downloadXls(HttpServletResponse response, String newsIds, String fileIds) {
response.setHeader("Access-Control-Expose-Headers", "Content-Disposition");
//获取下载路径
Object newsObject = redisUtil.get(newsIds);
Object fileObject = redisUtil.get(fileIds);
String fileName = UUID.randomUUID().toString().replace("-", "");
String newsFilePath = null;
String fileFilePath = null;
if (newsObject != null) {
newsFilePath = newsObject.toString();
}
if (fileObject != null) {
fileFilePath = fileObject.toString();
}
try {
if ((!newsIds.isEmpty())&&(!fileIds.isEmpty())) {
log.info("files and news package downloading ");
List<String> list=new ArrayList<>();
list.add(newsFilePath);
list.add(fileFilePath);
log.info("newsFilePath:{},fileFilePath:{}",newsFilePath,fileFilePath);
String filePath = knowledgeService.comprocessByName(list,fileName );
log.info("filePath:{}",filePath);
response.setHeader("content-Disposition", "attachment;filename=" + URLEncoder.encode(filePath, "UTF-8"));
ExcelExportUtil.download(response, filePath, true);
return;
}
if (fileIds.isEmpty()) {//news
log.info("only news downloading");
redisUtil.del(newsIds);
if(newsFilePath==null){
log.error("nothing");
return;
}
log.info("newsFilePath:{}",newsFilePath);
response.setHeader("content-Disposition", "attachment;filename=" + URLEncoder.encode(newsFilePath, "UTF-8"));
ExcelExportUtil.download(response, newsFilePath, true);
return;
}
log.info("only files downloading");
String s = knowledgeService.exportPackage(fileIds);
log.info("packageFile:{}",s);
response.setHeader("content-Disposition", "attachment;filename=" + URLEncoder.encode(s, "UTF-8"));
ExcelExportUtil.download(response, s, true);
} catch (Exception e) {
e.printStackTrace();
log.error(e.getMessage());
}
}
} }
@Data @Data
class DeleteParam{ class DeleteParam {
private List<Knowledge> knowledgeList; private List<Knowledge> knowledgeList;
} }
\ No newline at end of file
src/main/java/com/zzsn/knowbase/service/IKnowledgeService.java
...@@ -53,4 +53,12 @@ public interface IKnowledgeService { ...@@ -53,4 +53,12 @@ public interface IKnowledgeService {
Result<?> doImportInfo(HttpServletRequest request,String userId); Result<?> doImportInfo(HttpServletRequest request,String userId);
void doExcel(HttpServletRequest request, String fileSuffix,String userId); void doExcel(HttpServletRequest request, String fileSuffix,String userId);
List<KnowledgeVO> listByIds(String ids);
void exportExcel(String ids);
String exportPackage(String ids);
String comprocessByName(List<String> list, String filePath);
} }
src/main/java/com/zzsn/knowbase/service/impl/KnowledgeServiceImpl.java
...@@ -19,6 +19,8 @@ import com.zzsn.knowbase.vo.KnowledgeVO; ...@@ -19,6 +19,8 @@ import com.zzsn.knowbase.vo.KnowledgeVO;
import com.zzsn.knowbase.vo.Result; import com.zzsn.knowbase.vo.Result;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.join.ScoreMode;
import org.apache.pdfbox.pdmodel.PDDocument; import org.apache.pdfbox.pdmodel.PDDocument;
...@@ -26,6 +28,7 @@ import org.apache.pdfbox.pdmodel.PDPage; ...@@ -26,6 +28,7 @@ import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.text.PDFTextStripper; import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.TextPosition; import org.apache.pdfbox.text.TextPosition;
import org.apache.poi.ss.usermodel.Workbook; import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RequestOptions;
...@@ -47,13 +50,12 @@ import org.springframework.web.multipart.MultipartFile; ...@@ -47,13 +50,12 @@ import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest; import org.springframework.web.multipart.MultipartHttpServletRequest;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import java.io.ByteArrayInputStream; import java.io.*;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.*; import java.util.*;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
/** /**
* @Description: 知识 * @Description: 知识
...@@ -85,16 +87,21 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -85,16 +87,21 @@ class KnowledgeServiceImpl implements IKnowledgeService {
@Value("${files.storage}") @Value("${files.storage}")
String filesStorage; String filesStorage;
String tmpFile = "/storage" + Constants.FSP + "brap" + Constants.FSP + "download"
+ Constants.FSP;
@Autowired @Autowired
private AsyncService asyncService; private AsyncService asyncService;
private String TEMP_PATH="/storage/temp/"; private String TEMP_PATH = "/storage/temp/";
@Autowired @Autowired
private StringRedisTemplate stringRedisTemplate; private StringRedisTemplate stringRedisTemplate;
@Override @Override
public void addKnowledge(KnowFile knowFile, Knowledge knowledge, KbAuthorizedUser userInfo) { public void addKnowledge(KnowFile knowFile, Knowledge knowledge, KbAuthorizedUser userInfo) {
log.info("addKnowledge knowledge :{}", knowledge);
log.info("addKnowledge knowFile :{}", knowFile);
knowledge.setAuthor(userInfo.getName()); knowledge.setAuthor(userInfo.getName());
if (null == knowledge.getVerifyStatus()) { if (null == knowledge.getVerifyStatus()) {
knowledge.setVerifyStatus(0); knowledge.setVerifyStatus(0);
...@@ -115,44 +122,49 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -115,44 +122,49 @@ class KnowledgeServiceImpl implements IKnowledgeService {
String html = null; String html = null;
try { try {
String path = filesStorage + knowledge.getFiles().get(0).getFilePath(); String path = filesStorage + knowledge.getFiles().get(0).getFilePath();
String filePath=null; String filePath = null;
if (".pdf".equals(knowFile.getFileType())) { if (".pdf".equals(knowFile.getFileType())) {
log.info("pdf");
try { try {
PDDocument document = PDDocument.load(new File(path)); PDDocument document = PDDocument.load(new File(path));
StringBuilder allBuilder= new StringBuilder(); StringBuilder allBuilder = new StringBuilder();
PDFTextStripper pdfTextStripper = new PDFTextStripper(){ PDFTextStripper pdfTextStripper = new PDFTextStripper() {
private StringBuilder paragraphBuilder= new StringBuilder(); private StringBuilder paragraphBuilder = new StringBuilder();
@Override @Override
protected void startPage(PDPage page) throws IOException{ protected void startPage(PDPage page) throws IOException {
super.startPage(page); super.startPage(page);
paragraphBuilder.setLength(0);//make paragraphBuilder empty paragraphBuilder.setLength(0);//make paragraphBuilder empty
} }
@Override @Override
protected void writeLineSeparator() throws IOException{ protected void writeLineSeparator() throws IOException {
super.writeLineSeparator(); super.writeLineSeparator();
if( if (
paragraphBuilder.toString().endsWith(".")|| paragraphBuilder.toString().endsWith(".") ||
paragraphBuilder.toString().endsWith("。")|| paragraphBuilder.toString().endsWith("。") ||
paragraphBuilder.toString().endsWith("!")|| paragraphBuilder.toString().endsWith("!") ||
paragraphBuilder.toString().endsWith("！") paragraphBuilder.toString().endsWith("！")
){ ) {
paragraphBuilder.append("\n");//mark paragraph paragraphBuilder.append("\n");//mark paragraph
paragraphBuilder.append("😀"); paragraphBuilder.append("😀");
}else { } else {
paragraphBuilder.append("\n");//mark paragraph paragraphBuilder.append("\n");//mark paragraph
} }
} }
@Override @Override
protected void writeString(String string, List<TextPosition> textPositions) throws IOException{ protected void writeString(String string, List<TextPosition> textPositions) throws IOException {
super.writeString(string,textPositions); super.writeString(string, textPositions);
paragraphBuilder.append(string);//add text content paragraphBuilder.append(string);//add text content
} }
@Override @Override
protected void endPage(PDPage page) throws IOException{ protected void endPage(PDPage page) throws IOException {
super.endPage(page); super.endPage(page);
String paragraph= paragraphBuilder.toString().trim();//get paragraph String paragraph = paragraphBuilder.toString().trim();//get paragraph
if(!paragraph.isEmpty()){ if (!paragraph.isEmpty()) {
allBuilder.append(paragraph); allBuilder.append(paragraph);
} }
} }
...@@ -163,17 +175,18 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -163,17 +175,18 @@ class KnowledgeServiceImpl implements IKnowledgeService {
pdfTextStripper.setEndPage(document.getNumberOfPages()); pdfTextStripper.setEndPage(document.getNumberOfPages());
String text = pdfTextStripper.getText(document); String text = pdfTextStripper.getText(document);
knowledge.setContentAll(text); knowledge.setContentAll(text);
log.info("allBuilder:{}",allBuilder); log.info("allBuilder:{}", allBuilder);
contentStringList = Arrays.asList(allBuilder.toString().split("😀")); contentStringList = Arrays.asList(allBuilder.toString().split("😀"));
} catch (Exception e) { } catch (Exception e) {
log.error("parsing pdf error :{}", e.getMessage()); log.error("parsing pdf error :{}", e.getMessage());
} }
}else { } else {
filePath=path; filePath = path;
File file = new File(filePath); File file = new File(filePath);
log.info("word dealing filePath{}", filePath);
html = DocUtil.convertDocStream2Html(new FileInputStream(file)); html = DocUtil.convertDocStream2Html(new FileInputStream(file));
log.info("html :{}", html);
String htmlWithTable = html.replace("</p>", "######</p>"); String htmlWithTable = html.replace("</p>", "######</p>");
htmlWithTable = htmlWithTable.replace("</title>", "######</title>"); htmlWithTable = htmlWithTable.replace("</title>", "######</title>");
htmlWithTable = htmlWithTable.replace("</h1>", "######</h1>"); htmlWithTable = htmlWithTable.replace("</h1>", "######</h1>");
...@@ -181,20 +194,27 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -181,20 +194,27 @@ class KnowledgeServiceImpl implements IKnowledgeService {
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); log.error(e.getMessage());
} }
if (null == knowledge.getTitle()) { if (null == knowledge.getTitle()) {
for (String tem : contentStringList) { // for (String tem : contentStringList) {
String text = Jsoup.parse(tem).text(); // String text = Jsoup.parse(tem).text();
log.info("info:{}", text); // log.info("info:{}", text);
if ((!tem.contains("<title>")) && (!text.isEmpty())) { // if ((!tem.contains("<title>")) && (!text.isEmpty())) {
String[] s = text.split(" "); // String[] s = text.split(" ");
knowledge.setTitle(s[0]); // knowledge.setTitle(s[0]);
break; // break;
} // }
// }
if (knowFile.getFileName() != null && knowFile.getFileName().contains(".")) {
knowledge.setTitle(knowFile.getFileName().substring(0, knowFile.getFileName().lastIndexOf('.')));
} else {
knowledge.setTitle(knowFile.getFileName());
} }
} }
List<String> messageContentList = contentStringList.stream() List<String> messageContentList = contentStringList.stream()
...@@ -221,6 +241,8 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -221,6 +241,8 @@ class KnowledgeServiceImpl implements IKnowledgeService {
knowledge.setContents(contentList); knowledge.setContents(contentList);
knowledgeMessage.setContents(contentList); knowledgeMessage.setContents(contentList);
} }
BeanUtils.copyProperties(knowledge, knowledgeMessage);
knowledgeMessage.setType(knowledge.getTypeId());
//id为空表示新增 //id为空表示新增
if (null == knowledge.getId()) { if (null == knowledge.getId()) {
knowledge.setId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8)); knowledge.setId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8));
...@@ -228,19 +250,32 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -228,19 +250,32 @@ class KnowledgeServiceImpl implements IKnowledgeService {
} else { } else {
//id不为空表示修改数据 //id不为空表示修改数据
esOpUtil.docEditByEntity(Constants.ES_DATA_FOR_KNOWLEDGE, knowledge.getId(), knowledge); esOpUtil.docEditByEntity(Constants.ES_DATA_FOR_KNOWLEDGE, knowledge.getId(), knowledge);
KnowledgeVO knowledgeVO = this.getById(knowledge.getId());
knowledgeMessage.setKbKnowledgeId(knowledgeVO.getKbKnowledgeId());
knowledgeMessage.setKnowledgeProjectId(knowledgeVO.getKnowledgeProjectId());
knowledgeMessage.setType(knowledgeVO.getTypeId());
} }
//删除 //删除
deleteForPython(knowledge.getId(),knowledge.getKnowledgeProjectId()); if (null == knowledge.getKnowledgeProjectId()) {
KnowledgeVO knowledgeVO = this.getById(knowledge.getId());
BeanUtils.copyProperties(knowledge, knowledgeMessage); knowledge.setKnowledgeProjectId(knowledgeVO.getKbKnowledgeId());
knowledgeMessage.setType(knowledge.getTypeId()); }
deleteForPython(knowledge.getId(), knowledge.getKnowledgeProjectId());
log.info("send message:{}", knowledgeMessage);
produceInfo.sendKnowledgeContents(knowledgeMessage); produceInfo.sendKnowledgeContents(knowledgeMessage);
} }
@Override @Override
public void updateKnowledge(Knowledge knowledge) { public void updateKnowledge(Knowledge knowledge) {
esOpUtil.docSavaByEntity(Constants.ES_DATA_FOR_KNOWLEDGE, knowledge.getId(), knowledge); esOpUtil.docSavaByEntity(Constants.ES_DATA_FOR_KNOWLEDGE, knowledge.getId(), knowledge);
//sync data
KbAuthorizedUser userInfo = SpringContextUtils.getUserInfo();
if (null == userInfo) {
return;
}
KnowledgeVO knowledgeVO = this.getById(knowledge.getId());
addKnowledge(knowledgeVO.getFiles().get(0), knowledge, userInfo);
} }
@Override @Override
...@@ -250,20 +285,21 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -250,20 +285,21 @@ class KnowledgeServiceImpl implements IKnowledgeService {
} }
CompletableFuture.runAsync(() -> { CompletableFuture.runAsync(() -> {
for (Knowledge knowledge : knowledgeList) { for (Knowledge knowledge : knowledgeList) {
deleteForPython(knowledge.getId(),knowledge.getKbKnowledgeId()); deleteForPython(knowledge.getId(), knowledge.getKbKnowledgeId());
} }
}); });
} }
public void deleteForPython(String id,String knowledgeBaseId) { public void deleteForPython(String id, String knowledgeBaseId) {
JSONObject params = new JSONObject(); JSONObject params = new JSONObject();
List <String > lidList=new ArrayList<>(); List<String> lidList = new ArrayList<>();
lidList.add(id); lidList.add(id);
params.put("id", lidList); params.put("id", lidList);
params.put("knowledge_base_id", knowledgeBaseId); params.put("knowledge_base_id", knowledgeBaseId);
log.info("delete param:{}", params);
try { try {
HttpUtil.doPost(deleteUrl, params, 120000); HttpUtil.doPost(deleteUrl, params, 120000);
} catch (IOException e) { } catch (IOException e) {
...@@ -310,8 +346,8 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -310,8 +346,8 @@ class KnowledgeServiceImpl implements IKnowledgeService {
searchSourceBuilder.from((pageNo - 1) * pageSize); searchSourceBuilder.from((pageNo - 1) * pageSize);
//默认按照置顶以及时间倒序排列 //默认按照置顶以及时间倒序排列
//根据topNum正序查找，查询置顶数据 //根据topNum正序查找，查询置顶数据
searchSourceBuilder.sort("publishDate", SortOrder.DESC); // searchSourceBuilder.sort("publishDate", SortOrder.DESC);
if ("publishDate".equals(column)) { if ("publishTime".equals(column)) {
if ("desc".equals(order)) { if ("desc".equals(order)) {
searchSourceBuilder.sort("publishDate", SortOrder.DESC); searchSourceBuilder.sort("publishDate", SortOrder.DESC);
} else if ("asc".equals(order)) { } else if ("asc".equals(order)) {
...@@ -384,7 +420,7 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -384,7 +420,7 @@ class KnowledgeServiceImpl implements IKnowledgeService {
boolQuery.must(multiMatchQueryBuilder); boolQuery.must(multiMatchQueryBuilder);
} }
if (StringUtils.isNotEmpty(knowledgeParam.getOrigin())) { if (StringUtils.isNotEmpty(knowledgeParam.getOrigin())) {
boolQuery.must(QueryBuilders.matchPhraseQuery("origin", knowledgeParam.getOrigin())); boolQuery.must(QueryBuilders.matchQuery("origin", knowledgeParam.getOrigin()));
} }
if (StringUtils.isNotEmpty(knowledgeParam.getVerifierName())) { if (StringUtils.isNotEmpty(knowledgeParam.getVerifierName())) {
boolQuery.must(QueryBuilders.matchPhraseQuery("verifierName", knowledgeParam.getVerifierName())); boolQuery.must(QueryBuilders.matchPhraseQuery("verifierName", knowledgeParam.getVerifierName()));
...@@ -415,6 +451,14 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -415,6 +451,14 @@ class KnowledgeServiceImpl implements IKnowledgeService {
if (StringUtils.isNotBlank(knowledgeParam.getEndTime())) { if (StringUtils.isNotBlank(knowledgeParam.getEndTime())) {
boolQuery.filter(QueryBuilders.rangeQuery("publishDate").lte(EsDateUtil.esFieldDateFormat(knowledgeParam.getEndTime()))); boolQuery.filter(QueryBuilders.rangeQuery("publishDate").lte(EsDateUtil.esFieldDateFormat(knowledgeParam.getEndTime())));
} }
//时间过滤筛选
if (StringUtils.isNotBlank(knowledgeParam.getVerifyStartTime())) {
boolQuery.filter(QueryBuilders.rangeQuery("verifyTime").gte(EsDateUtil.esFieldDateFormat(knowledgeParam.getStartTime())));
}
if (StringUtils.isNotBlank(knowledgeParam.getVerifyEndTime())) {
boolQuery.filter(QueryBuilders.rangeQuery("verifyTime").lte(EsDateUtil.esFieldDateFormat(knowledgeParam.getEndTime())));
}
outer.should(boolQuery); outer.should(boolQuery);
searchSourceBuilder.query(outer); searchSourceBuilder.query(outer);
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
...@@ -424,20 +468,24 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -424,20 +468,24 @@ class KnowledgeServiceImpl implements IKnowledgeService {
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
SearchHit[] searchHits = searchResponse.getHits().getHits();
List<KnowledgeVO> list = new ArrayList<>();
for (SearchHit hit : searchHits) {
String queryInfo = hit.getSourceAsString();
KnowledgeVO info = JSONUtil.toBean(queryInfo, KnowledgeVO.class);
info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
info.setVerifyTime(EsDateUtil.esFieldDateMapping(info.getVerifyTime()));
list.add(info);
}
IPage<KnowledgeVO> pageData = new Page<>(pageNo, pageSize, searchResponse.getHits().getTotalHits().value); if (null != searchResponse && null != searchResponse.getHits()) {
SearchHit[] searchHits = searchResponse.getHits().getHits();
List<KnowledgeVO> list = new ArrayList<>();
for (SearchHit hit : searchHits) {
String queryInfo = hit.getSourceAsString();
KnowledgeVO info = JSONUtil.toBean(queryInfo, KnowledgeVO.class);
info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
info.setVerifyTime(EsDateUtil.esFieldDateMapping(info.getVerifyTime()));
list.add(info);
}
pageData.setRecords(list); IPage<KnowledgeVO> pageData = new Page<>(pageNo, pageSize, searchResponse.getHits().getTotalHits().value);
return pageData;
pageData.setRecords(list);
return pageData;
}
return new Page<>(pageNo, pageSize, 0);
} }
@Override @Override
...@@ -561,7 +609,7 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -561,7 +609,7 @@ class KnowledgeServiceImpl implements IKnowledgeService {
String fileSuffix = multipartFile.getOriginalFilename().substring(index + 1); String fileSuffix = multipartFile.getOriginalFilename().substring(index + 1);
if ("xls".equals(fileSuffix) || "xlsx".equals(fileSuffix)) { if ("xls".equals(fileSuffix) || "xlsx".equals(fileSuffix)) {
//文件校验 //文件校验
if(!this.checkExcel(multipartFile)){ if (!this.checkExcel(multipartFile)) {
log.error("excel格式不对"); log.error("excel格式不对");
return Result.error("模版错误，请下载正确模板，再次上传！"); return Result.error("模版错误，请下载正确模板，再次上传！");
} }
...@@ -577,8 +625,8 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -577,8 +625,8 @@ class KnowledgeServiceImpl implements IKnowledgeService {
Workbook workbook = ExcelExportUtil.getWorkbook(multipartFile); Workbook workbook = ExcelExportUtil.getWorkbook(multipartFile);
if (workbook != null) { if (workbook != null) {
List<String> list = ExcelExportUtil.getSheetTitles(workbook); List<String> list = ExcelExportUtil.getSheetTitles(workbook);
return list != null&&list.contains("标题")&&list.contains("正文") return list != null && list.contains("标题") && list.contains("正文")
&&list.contains("作者")&&list.contains("来源")&&list.contains("发布时间"); && list.contains("作者") && list.contains("来源") && list.contains("发布时间");
} }
return false; return false;
} }
...@@ -604,6 +652,101 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -604,6 +652,101 @@ class KnowledgeServiceImpl implements IKnowledgeService {
} }
} }
@Override
public List<KnowledgeVO> listByIds(String ids) {
List<KnowledgeVO> list = new ArrayList<>();
SearchRequest searchRequest = new SearchRequest(Constants.ES_DATA_FOR_KNOWLEDGE);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//默认最大数量是10000，设置为true后，显示准确数量
searchSourceBuilder.trackTotalHits(true);
//创建查询对象
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
if (!ids.isEmpty()) {
List<String> idList = Arrays.asList(ids.split(","));
boolQuery.must(QueryBuilders.termsQuery("id", idList));
}
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = null;
try {
searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
if (null != searchResponse && null != searchResponse.getHits()) {
SearchHit[] searchHits = searchResponse.getHits().getHits();
for (SearchHit hit : searchHits) {
String queryInfo = hit.getSourceAsString();
KnowledgeVO info = JSONUtil.toBean(queryInfo, KnowledgeVO.class);
info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
list.add(info);
}
}
return list;
}
@Override
public void exportExcel(String ids) {
//获取数据
if (redisUtil.get(ids) == null) {
try {
String[] arr = new String[]{"标题", "正文", "作者", "来源", "发布时间"};
List<KnowledgeVO> informationList = this.listByIds(ids);
if (CollectionUtils.isNotEmpty(informationList)) {
//动态补充表头
List<String> headers = Arrays.asList(arr);
//每个sheet页最多5000条数据
List<List<KnowledgeVO>> partition = ListUtils.partition(informationList, 5000);
XSSFWorkbook workbook = new XSSFWorkbook();
for (int i = 0; i < partition.size(); i++) {
List<List<String>> rows = formatData(partition.get(i));
ExcelExportUtil.exportExcelData(workbook, i, headers, rows, "sheet" + (i + 1));
}
File fileDir = ExcelExportUtil.getFileDir();
String filePath = fileDir.getAbsolutePath() + File.separator + UUID.randomUUID().toString().replace("-", "") + "info.xlsx";
FileOutputStream outputStream = new FileOutputStream(filePath);
workbook.write(outputStream);
redisUtil.set(ids, filePath, 3600 * 24);
}
} catch (Exception e) {
e.printStackTrace();
log.error(e.getMessage());
}
}
}
@Override
public String exportPackage(String ids) {
List<KnowledgeVO> knowledgeVOS = this.listByIds(ids);
List<String> pathList = new ArrayList<>(knowledgeVOS.size());
for (KnowledgeVO knowledgeVO : knowledgeVOS) {
pathList.add(filesStorage + knowledgeVO.getFiles().get(0).getFilePath());
}
String filePackage=UUID.randomUUID().toString().replace("-", "");
ZipUtils.compressFiles(tmpFile,filePackage,pathList );
String packagePath=tmpFile+filePackage+".zip";
redisUtil.set(ids, packagePath, 3600 * 24);
return packagePath;
}
/*
* 将所有的文件都压缩到一个文件中去
*/
@Override
public String comprocessByName(List<String> fileNameList, String downloadFileName) {
if (fileNameList.size() == 1) {
return fileNameList.get(0);
}
ZipUtils.compressFiles(tmpFile,downloadFileName,fileNameList);
return tmpFile+downloadFileName+".zip";
}
//导入发布库数据 //导入发布库数据
public void importInfo(List<List<String>> lists, String kbKnowledgeId, String knowledgeProjectId, String typeId, String createBy) { public void importInfo(List<List<String>> lists, String kbKnowledgeId, String knowledgeProjectId, String typeId, String createBy) {
for (List<String> info : lists) { for (List<String> info : lists) {
...@@ -683,4 +826,19 @@ class KnowledgeServiceImpl implements IKnowledgeService { ...@@ -683,4 +826,19 @@ class KnowledgeServiceImpl implements IKnowledgeService {
searchInfo + "#"; searchInfo + "#";
} }
private List<List<String>> formatData(List<KnowledgeVO> knowledgeVOS) {
List<List<String>> list = new ArrayList<>(knowledgeVOS.size());
for (KnowledgeVO knowledgeVO : knowledgeVOS) {
List<String> innerList = new ArrayList<>();
innerList.add(knowledgeVO.getTitle());
innerList.add(knowledgeVO.getContentAll());
innerList.add(knowledgeVO.getAuthor());
innerList.add(knowledgeVO.getOrigin());
innerList.add(knowledgeVO.getPublishDate());
list.add(innerList);
}
return list;
}
} }
src/main/java/com/zzsn/knowbase/service/impl/LocalFileServiceImpl.java
...@@ -112,6 +112,7 @@ public class LocalFileServiceImpl implements ILocalFileService { ...@@ -112,6 +112,7 @@ public class LocalFileServiceImpl implements ILocalFileService {
return result; // create user metadata and return it return result; // create user metadata and return it
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace(); e.printStackTrace();
log.error(e.getMessage());
} }
// if the operation of file uploading is unsuccessful, an error occurs // if the operation of file uploading is unsuccessful, an error occurs
Result result = Result.error("上传文件时出现问题！"); Result result = Result.error("上传文件时出现问题！");
......
src/main/java/com/zzsn/knowbase/util/EsOpUtil.java
...@@ -415,6 +415,7 @@ public class EsOpUtil { ...@@ -415,6 +415,7 @@ public class EsOpUtil {
public Boolean docDeleteById(String index, String id) { public Boolean docDeleteById(String index, String id) {
try { try {
DeleteRequest deleteRequest = new DeleteRequest(index, id); DeleteRequest deleteRequest = new DeleteRequest(index, id);
deleteRequest.setRefreshPolicy("true");
DeleteResponse delete = client.delete(deleteRequest, RequestOptions.DEFAULT); DeleteResponse delete = client.delete(deleteRequest, RequestOptions.DEFAULT);
if (delete.status() == RestStatus.OK) { if (delete.status() == RestStatus.OK) {
log.info("DELETE /{}/_doc/{}/\r\n", index, id); log.info("DELETE /{}/_doc/{}/\r\n", index, id);
......
src/main/java/com/zzsn/knowbase/util/ExcelExportUtil.java
...@@ -12,14 +12,8 @@ import org.apache.commons.lang3.StringUtils; ...@@ -12,14 +12,8 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFCell; import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.hssf.usermodel.HSSFDateUtil; import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import org.apache.poi.hssf.usermodel.HSSFWorkbook; import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.HorizontalAlignment; import org.apache.poi.ss.usermodel.*;
import org.apache.poi.ss.usermodel.Row; import org.apache.poi.xssf.usermodel.*;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFCell;
import org.apache.poi.xssf.usermodel.XSSFRow;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.web.multipart.MultipartFile; import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
...@@ -255,4 +249,90 @@ public class ExcelExportUtil { ...@@ -255,4 +249,90 @@ public class ExcelExportUtil {
} }
//多个sheet
public static void exportExcelData(XSSFWorkbook workbook, int sheetNum, List<String> keyList, List<List<String>> rows, String sheetName) {
Sheet sheet = workbook.createSheet();
//多个sheet
workbook.setSheetName(sheetNum,sheetName);
sheet.setDefaultColumnWidth(20);
sheet.setDefaultRowHeight((short) 400);
XSSFCellStyle titleStyle = ExcelExportUtil.getHeaderStyle(workbook);
XSSFCellStyle rowStrStyle = ExcelExportUtil.getRowStrStyle(workbook);
// 产生表格标题行
Row row = sheet.createRow(0);
row.setHeight((short) 400);
Cell cell;
List<String> colData = new ArrayList<>(keyList);
for (int i = 0; i < colData.size(); i++) {
cell = row.createCell(i, CellType.STRING);
cell.setCellStyle(titleStyle);
cell.setCellValue(new XSSFRichTextString(colData.get(i)));
}
// 数据行
for (int m = 0; m < rows.size(); m++) {
List<String> rowData = rows.get(m);
row = sheet.createRow(m + 1);
row.setHeight((short) 400);
int i=0;
for (String rowDatum : rowData) {
createCell(rowStrStyle, row, i, rowDatum);
i++;
}
}
}
private static void createCell(XSSFCellStyle rowStrStyle, Row row, int index , String value){
Cell cell = row.createCell(index, CellType.STRING);
cell.setCellStyle(rowStrStyle);
cell.setCellValue(value);
}
/**
* 获取标题栏样式
*
* @param workbook
* @return
*/
private static XSSFCellStyle getHeaderStyle(XSSFWorkbook workbook) {
// 生成一个样式
XSSFCellStyle style = workbook.createCellStyle();
// 设置这些样式
style.setFillForegroundColor(IndexedColors.SKY_BLUE.index);
style.setFillPattern(FillPatternType.SOLID_FOREGROUND);
style.setBorderBottom(BorderStyle.THIN);
style.setBorderLeft(BorderStyle.THIN);
style.setBorderRight(BorderStyle.THIN);
style.setBorderTop(BorderStyle.THIN);
style.setAlignment(HorizontalAlignment.CENTER);
// style.setDataFormat(HSSFDataFormat.getBuiltinFormat("0.00"));
// 生成一个字体
XSSFFont font = workbook.createFont();
font.setColor(IndexedColors.BLACK.index);
font.setFontHeightInPoints((short) 9);
// 把字体应用到当前的样式
style.setFont(font);
return style;
}
/**
* 获取字符串型内容栏样式
*
* @return
*/
private static XSSFCellStyle getRowStrStyle(XSSFWorkbook workbook) {
// 生成一个样式
XSSFCellStyle style = workbook.createCellStyle();
// 设置这些样式
style.setBorderBottom(BorderStyle.THIN);
style.setBorderLeft(BorderStyle.THIN);
style.setBorderRight(BorderStyle.THIN);
style.setBorderTop(BorderStyle.THIN);
style.setAlignment(HorizontalAlignment.CENTER);
// 生成一个字体
XSSFFont font = workbook.createFont();
font.setColor(IndexedColors.BLACK.index);
font.setFontHeightInPoints((short) 9);
font.setBold(true);
// 把字体应用到当前的样式
style.setFont(font);
return style;
}
} }
src/main/java/com/zzsn/knowbase/util/ZipUtils.java 0 → 100644
package com.zzsn.knowbase.util;
import lombok.extern.slf4j.Slf4j;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
/**
* @Description
* @ClassName ZipUtils
* @Author yuhuofei
* @Date 2022/8/10 20:37
* @Version 1.0
*/
@Slf4j
public class ZipUtils {
/**
* @param zipPathDir 压缩包路径 ，如 /home/data/zip-folder/
* @param zipFileName 压缩包名称 ，如 测试文件.zip
* @param fileList 要压缩的文件列表（绝对路径），如 /home/person/test/测试.doc，/home/person/haha/测试.doc
* @return
*/
public static void compressFiles(String zipPathDir, String zipFileName, List<String> fileList) {
try (ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(new File(zipPathDir + zipFileName+".zip")))) {
File zipFile = new File(zipPathDir);
if (!zipFile.exists()) {
zipFile.mkdirs();
}
for (String filePath : fileList) {
log.info("file package file :{}",filePath);
File file = new File(filePath);
if (file.exists()) {
int index = file.getName().lastIndexOf('.');
ZipEntry zipEntry = new ZipEntry(file.getName().substring(0, index) + "-" + dateRandom18() + file.getName().substring(index));
zos.putNextEntry(zipEntry);
byte[] buffer = new byte[2048];
compressSingleFile(file, zos, buffer);
}
}
zos.flush();
} catch (Exception e) {
log.error("压缩所有文件成zip包出错{}" , e.getMessage());
}
}
//压缩单个文件
public static void compressSingleFile(File file, ZipOutputStream zos, byte[] buffer) {
int len;
try (FileInputStream fis = new FileInputStream(file)) {
while ((len = fis.read(buffer)) > 0) {
zos.write(buffer, 0, len);
zos.flush();
}
zos.closeEntry();
} catch (IOException e) {
System.out.println("====压缩单个文件异常====" + e);
}
}
//生成随机数
public static String dateRandom18() {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyyMMdd");
String date = simpleDateFormat.format(new Date());
String timeMillis = String.valueOf(System.currentTimeMillis());
String fiveNumber = timeMillis.substring(timeMillis.length() - 6);
String tempRandom = String.valueOf(Math.random());
String number = tempRandom.substring(tempRandom.length() - 4);
return date + fiveNumber + number;
}
//测试方法
public static void main(String[] args) {
String zipPathDir = "D:/";
String zipFileName = "测试文件.zip";
List<String> list = new ArrayList<>();
list.add("D:/a.txt");
list.add("D:/b.txt");
compressFiles(zipPathDir, zipFileName, list);
}
}
src/main/java/com/zzsn/knowbase/vo/IntelligentQaParam.java
...@@ -16,5 +16,6 @@ public class IntelligentQaParam { ...@@ -16,5 +16,6 @@ public class IntelligentQaParam {
private String question; private String question;
private String kbKnowledgeIds; private String kbKnowledgeIds;
private Integer scoreThreshold=700; private Integer scoreThreshold=700;
private Integer size=20;
} }
src/main/java/com/zzsn/knowbase/vo/KnowledgeVO.java
...@@ -27,7 +27,9 @@ public class KnowledgeVO { ...@@ -27,7 +27,9 @@ public class KnowledgeVO {
private String publishDate; private String publishDate;
private String createTime; private String createTime;
private String kbKnowledgeId; private String kbKnowledgeId;
private String KnowledgeProjectId;
private String type; private String type;
private String typeId;
private String verifierName; private String verifierName;
private String origin; private String origin;
private String author; private String author;
......
src/main/resources/application.yml
...@@ -24,7 +24,7 @@ spring: ...@@ -24,7 +24,7 @@ spring:
requiredAcks: 1 requiredAcks: 1
redis: redis:
database: 0 database: 0
host: 114.115.236.206 host: 114.116.90.53
lettuce: lettuce:
pool: pool:
max-active: 8 #最大连接数据库连接数,设 0 为没有限制 max-active: 8 #最大连接数据库连接数,设 0 为没有限制
...@@ -32,7 +32,7 @@ spring: ...@@ -32,7 +32,7 @@ spring:
max-wait: -1ms #最大建立连接等待时间。如果超过此时间将接到异常。设为-1表示无限制。 max-wait: -1ms #最大建立连接等待时间。如果超过此时间将接到异常。设为-1表示无限制。
min-idle: 0 #最小等待连接中的数量,设 0 为没有限制 min-idle: 0 #最小等待连接中的数量,设 0 为没有限制
shutdown-timeout: 100ms shutdown-timeout: 100ms
port: 6379 port: 6380
password: clbzzsn password: clbzzsn
thymeleaf: thymeleaf:
...@@ -58,11 +58,14 @@ know: ...@@ -58,11 +58,14 @@ know:
document: document:
server: server:
host: http://114.116.116.241:9088 host: http://114.116.116.241:9088
# host: http://192.168.1.71:9088
files: files:
storage: /storage/know/ storage: /storage/know/
# storage: C:/know/
docservice: docservice:
url: url:
site: http://114.116.116.241:80/ site: http://114.116.116.241:80/
# site: http://192.168.1.216:80/
converter: ConvertService.ashx converter: ConvertService.ashx
command: coauthoring/CommandService.ashx command: coauthoring/CommandService.ashx
api: web-apps/apps/api/documents/api.js api: web-apps/apps/api/documents/api.js
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论