package com.zzsn.knowbase.service.impl;

import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.zzsn.knowbase.constant.Constants;
import com.zzsn.knowbase.entity.*;
import com.zzsn.knowbase.kafka.message.KnowledgeMessage;
import com.zzsn.knowbase.kafka.producer.ProduceInfo;
import com.zzsn.knowbase.service.IKnowledgeService;
import com.zzsn.knowbase.service.KbAuthorizedUserService;
import com.zzsn.knowbase.service.KbKnowledgeProjectService;
import com.zzsn.knowbase.util.*;
import com.zzsn.knowbase.vo.KnowledgeParam;
import com.zzsn.knowbase.vo.KnowledgeVO;
import com.zzsn.knowbase.vo.Result;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.TextPosition;
import org.apache.poi.hslf.usermodel.HSLFShape;
import org.apache.poi.hslf.usermodel.HSLFSlide;
import org.apache.poi.hslf.usermodel.HSLFSlideShow;
import org.apache.poi.hslf.usermodel.HSLFTextShape;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xslf.usermodel.XMLSlideShow;
import org.apache.poi.xslf.usermodel.XSLFShape;
import org.apache.poi.xslf.usermodel.XSLFSlide;
import org.apache.poi.xslf.usermodel.XSLFTextShape;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.elasticsearch.index.query.NestedQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.jsoup.Jsoup;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;

import javax.servlet.http.HttpServletRequest;
import java.io.*;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;

/**
 * @Description: 知识
 * @Author: chenshiqiang
 * @Version: V1.0
 */
@Service
@Slf4j
class KnowledgeServiceImpl implements IKnowledgeService {
    @Autowired
    private RestHighLevelClient client;
    @Autowired
    private EsOpUtil esOpUtil;
    @Autowired
    private CodeGenerateUtil codeGenerateUtil;
    @Autowired
    private KbKnowledgeProjectService knowledgeProjectService;
    @Autowired
    private KbAuthorizedUserService authorizedUserService;
    @Autowired
    private ProduceInfo produceInfo;
    @Value("${python.searchUrl:}")
    private String searchUrl;
    @Value("${python.deleteUrl:}")
    private String deleteUrl;

    @Autowired
    private RedisUtil redisUtil;
    @Value("${files.storage}")
    String filesStorage;

    String tmpFile = "/storage" + Constants.FSP + "brap" + Constants.FSP + "download"
            + Constants.FSP;

    @Autowired
    private AsyncService asyncService;

    private String TEMP_PATH = "/storage/temp/";
    @Autowired
    private StringRedisTemplate stringRedisTemplate;


    @Override
    public void addKnowledge(KnowFile knowFile, Knowledge knowledge, KbAuthorizedUser userInfo) {
        log.info("addKnowledge knowledge :{}", knowledge);
        log.info("addKnowledge knowFile :{}", knowFile);
        knowledge.setAuthor(userInfo.getName());
        if (null == knowledge.getVerifyStatus()) {
            knowledge.setVerifyStatus(0);
        }
        knowledge.setCreateTime(cn.hutool.core.date.DateUtil.formatDateTime(new Date()).replace(" ", "T"));
        if (null == knowledge.getPublishDate()) {
            knowledge.setPublishDate(cn.hutool.core.date.DateUtil.formatDateTime(new Date()).replace(" ", "T"));
        }

        knowledge.setDeleteFlag(0);
        KnowledgeMessage knowledgeMessage = new KnowledgeMessage();
        if (knowledge.getImportData()!=null && knowledge.getImportData()!=1) {
            //创建临时文件
            File file = null;
            try {
                File dirFile = new File(Constants.TEMP_FILES_DIR);
                dirFile.mkdirs();
                file = File.createTempFile(Constants.TEMP_FILES_DIR, knowFile.getFileType(), dirFile);
            } catch (IOException e) {
                log.info("创建临时文件失败e：{}",e.getMessage(),e);
                return;
            }
            try (FileOutputStream fos = new FileOutputStream(file)){
                String url = MinioUtil.getMinioUrl() + LoadToolUtil.urlEncoderRelative(knowFile.getFilePath());
                try(InputStream inStream = FileUtil.DownloadFileFormServer(url)){
                    int bytesRead = 0;
                    int len = 8129;
                    byte[] buffer = new byte[len];
                    while ((bytesRead = inStream.read(buffer, 0, len)) != -1) {
                        fos.write(buffer, 0, bytesRead);
                    }
                }
                log.info("临时文件本地地址：{}",file.getPath());
                List<KnowFile> knowFileList = new ArrayList<>();
                knowFileList.add(knowFile);
                knowledge.setFiles(knowFileList);
                List<Content> contentList = new ArrayList<>();
                List<String> contentStringList = new ArrayList<>();
                String html = null;
                try {
                    String path = filesStorage + knowledge.getFiles().get(0).getFilePath();
                    String filePath = null;
                    if (".pdf".equals(knowFile.getFileType())) {
                        knowledge.setImportData(3);
                        log.info("pdf");
                        try {
                            PDDocument document = PDDocument.load(new File(path));
                            StringBuilder allBuilder = new StringBuilder();
                            PDFTextStripper pdfTextStripper = new PDFTextStripper() {
                                private StringBuilder paragraphBuilder = new StringBuilder();

                                @Override
                                protected void startPage(PDPage page) throws IOException {
                                    super.startPage(page);
                                    paragraphBuilder.setLength(0);//make paragraphBuilder empty
                                }

                                @Override
                                protected void writeLineSeparator() throws IOException {
                                    super.writeLineSeparator();

                                    if (
                                            paragraphBuilder.toString().endsWith(".") ||
                                                    paragraphBuilder.toString().endsWith("。") ||
                                                    paragraphBuilder.toString().endsWith("!") ||
                                                    paragraphBuilder.toString().endsWith("！")
                                    ) {
                                        paragraphBuilder.append("\n");//mark paragraph
                                        paragraphBuilder.append("😀");
                                    } else {
                                        paragraphBuilder.append("\n");//mark paragraph
                                    }
                                }

                                @Override
                                protected void writeString(String string, List<TextPosition> textPositions) throws IOException {
                                    super.writeString(string, textPositions);
                                    paragraphBuilder.append(string);//add text content
                                }

                                @Override
                                protected void endPage(PDPage page) throws IOException {
                                    super.endPage(page);
                                    String paragraph = paragraphBuilder.toString().trim();//get paragraph
                                    if (!paragraph.isEmpty()) {
                                        allBuilder.append(paragraph);
                                    }
                                }

                            };
                            pdfTextStripper.setSortByPosition(true);
                            pdfTextStripper.setStartPage(0);
                            pdfTextStripper.setEndPage(document.getNumberOfPages());
                            String text = pdfTextStripper.getText(document);
                            //knowledge.setContentAll(text);
                            //log.info("allBuilder:{}", allBuilder.toString().substring(1000));
                            contentStringList = Arrays.asList(allBuilder.toString().split("😀"));
                        } catch (Exception e) {
                            log.error("parsing pdf error :{}", e.getMessage());
                        }
                    } else if(".pptx".equals(knowFile.getFileType())){
                        knowledge.setImportData(2);
                        filePath = path;
                        log.info("pptx dealing filePath{}", filePath);
                        // 加载PPT文件
                        XMLSlideShow ppt = new XMLSlideShow(new FileInputStream(filePath));
                        // 遍历幻灯片
                        for (XSLFSlide slide : ppt.getSlides()) {
                            StringBuffer sb = new StringBuffer();
                            // 遍历形状
                            List<XSLFShape> shapes = slide.getShapes();
                            for (XSLFShape shape : shapes) {
                                // 检查形状是否包含文本
                                if (shape instanceof XSLFTextShape) {
                                    XSLFTextShape textShape = (XSLFTextShape) shape;
                                    String text = textShape.getText();
                                    if (text != null) {
                                        sb.append(text);
                                        log.info("text :{}", text);
                                    }
                                }
                            }
                            contentStringList.add(sb.toString());
                        }
                        ppt.close(); // 关闭PPT
                    } else if(".ppt".equals(knowFile.getFileType())){
                        knowledge.setImportData(2);
                        filePath = path;
                        log.info("pptx dealing filePath{}", filePath);
                        // 加载PPT文件
                        HSLFSlideShow ppt = new HSLFSlideShow(new FileInputStream(filePath));
                        // 遍历幻灯片
                        for (HSLFSlide slide : ppt.getSlides()) {
                            StringBuffer sb = new StringBuffer();
                            // 遍历形状
                            List<HSLFShape> shapes = slide.getShapes();
                            for (HSLFShape shape : shapes) {
                                // 检查形状是否包含文本
                                if (shape instanceof HSLFTextShape) {
                                    HSLFTextShape textShape = (HSLFTextShape) shape;
                                    String text = textShape.getText();
                                    if (text != null) {
                                        sb.append(text);
                                        log.info("text :{}", text);
                                    }
                                }
                            }
                            contentStringList.add(sb.toString());
                        }
                        ppt.close(); // 关闭PPT
                    } else {
                        filePath = path;
                        log.info("word dealing filePath{}", filePath);
                        html = DocUtil.convertDocStream2Html(new FileInputStream(file));
                        //log.info("html :{}", html);
                        String htmlWithTable = html.replace("</p>", "######</p>");
                        htmlWithTable = htmlWithTable.replace("</title>", "######</title>");
                        htmlWithTable = htmlWithTable.replace("</h1>", "######</h1>");
                        contentStringList = Arrays.asList(htmlWithTable.split("######"));
                    }

                } catch (Exception e) {
                    log.error(e.getMessage());
                }


                if (null == knowledge.getTitle()) {
//                for (String tem : contentStringList) {
//                    String text = Jsoup.parse(tem).text();
//                    log.info("info:{}", text);
//                    if ((!tem.contains("<title>")) && (!text.isEmpty())) {
//                        String[] s = text.split(" ");
//                        knowledge.setTitle(s[0]);
//                        break;
//                    }
//                }
                    if (knowFile.getFileName() != null && knowFile.getFileName().contains(".")) {
                        knowledge.setTitle(knowFile.getFileName().substring(0, knowFile.getFileName().lastIndexOf('.')));

                    } else {
                        knowledge.setTitle(knowFile.getFileName());
                    }

                }

                List<String> messageContentList = contentStringList.stream()
                        .filter(item -> !item.contains("<img"))
                        .filter(item -> !item.contains("<table"))
                        .filter(item -> !item.contains("<tr>"))
                        .filter(item -> !item.contains("</tr>"))
                        .filter(item -> !item.contains("<td>"))
                        .filter(item -> !item.contains("</td>"))
                        .map(item -> Jsoup.parse(item).text())
                        .collect(Collectors.toList());

                messageContentList = messageContentList.
                        stream().filter(item -> item.length() > 10)
                        .collect(Collectors.toList());
                if (!messageContentList.isEmpty()) {
                    for (String tem : messageContentList) {
                        contentList.add(Content.builder()
                                .contentId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8))
                                .content(tem)
                                .build());
                    }
                }
                knowledge.setContents(contentList);
                knowledgeMessage.setContents(contentList);
            } catch (Exception e) {
                log.error("本地化并保存附件信息失败，系统异常：{}",e);
            }finally{
                if(file != null){
                    file.delete();
                }
            }
        }
        BeanUtils.copyProperties(knowledge, knowledgeMessage);
        knowledgeMessage.setType(knowledge.getTypeId());
        if(StringUtils.isNotEmpty(knowledge.getContentWithTag())){
            //片段切分
            List<Content> contents = new ArrayList<>();
            List<String> splitContents = HtmlUtil.splitContents(knowledge.getContentWithTag());
            for (String content : splitContents) {
                System.out.println(content.length());
                System.out.println(content);
                contents.add(Content.builder()
                        .contentId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8))
                        .content(content)
                        .build());
            }
            knowledge.setContents(contents);
        }
        //id为空表示新增
        if (null == knowledge.getId()) {
            knowledge.setId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8));
            esOpUtil.docSavaByEntity(Constants.ES_DATA_FOR_KNOWLEDGE, knowledge.getId(), knowledge);
            knowledgeMessage.setId(knowledge.getId());
        } else {
            //id不为空表示修改数据
            esOpUtil.docEditByEntity(Constants.ES_DATA_FOR_KNOWLEDGE, knowledge.getId(), knowledge);
            KnowledgeVO knowledgeVO = this.getById(knowledge.getId());
            knowledgeMessage.setKbKnowledgeId(knowledgeVO.getKbKnowledgeId());
            knowledgeMessage.setKnowledgeProjectId(knowledgeVO.getKnowledgeProjectId());
            knowledgeMessage.setType(knowledgeVO.getTypeId());
            log.info("id:{}",knowledgeVO.getId());
            knowledgeMessage.setId(knowledgeVO.getId());

        }
        //删除
        if (null == knowledge.getKnowledgeProjectId()) {
            KnowledgeVO knowledgeVO = this.getById(knowledge.getId());
            knowledge.setKnowledgeProjectId(knowledgeVO.getKbKnowledgeId());
        }
        deleteForPython(knowledge.getId(), knowledge.getKbKnowledgeId());
        //log.info("send message:{}", knowledgeMessage);
        String msg = JSON.toJSONString(knowledgeMessage);
        log.info(msg);
        produceInfo.sendKnowledgeContents(knowledgeMessage);
    }

    @Override
    public void updateKnowledge(Knowledge knowledge) {
        //sync data
        KbAuthorizedUser userInfo = SpringContextUtils.getUserInfo();
        if (null == userInfo) {
            return;
        }
        esOpUtil.docSavaByEntity(Constants.ES_DATA_FOR_KNOWLEDGE, knowledge.getId(), knowledge);
        KnowledgeVO knowledgeVO = this.getById(knowledge.getId());
        addKnowledge(knowledgeVO.getFiles().get(0), knowledge, userInfo);
    }

    @Override
    public void deleteKnowledge(List<Knowledge> knowledgeList) {
        for (Knowledge knowledge : knowledgeList) {
            knowledge.setDeleteFlag(1);
            //esOpUtil.docDeleteById(Constants.ES_DATA_FOR_KNOWLEDGE, knowledge.getId());
            //修改为标记删除
            esOpUtil.docUpdateById(Constants.ES_DATA_FOR_KNOWLEDGE, knowledge.getId(), JSON.toJSONString(knowledge));
        }
        CompletableFuture.runAsync(() -> {
            for (Knowledge knowledge : knowledgeList) {
                deleteForPython(knowledge.getId(), knowledge.getKbKnowledgeId());
            }
        });
    }

    @Override
    public void deleteForPython(String id, String knowledgeBaseId) {
        JSONObject params = new JSONObject();
        List<String> lidList = new ArrayList<>();
        lidList.add(id);
        params.put("id", lidList);
        params.put("knowledge_base_id", knowledgeBaseId);
        log.info("delete param:{}", params);
        String res = HttpUtil.post(deleteUrl, params.toJSONString(), 5000);
        log.info("delete res:{}", res);

    }

    @Override
    public IPage<KnowledgeVO> queryPageList(KnowledgeParam knowledgeParam, Integer pageNo, Integer pageSize, String column, String order) {
        //分类id列表
        List<String> catList = new ArrayList<>();
        //查看全部且是非管理员的时候
        List<String> knowPermitList = new ArrayList<>();
        if(StringUtils.isNotEmpty(knowledgeParam.getKnowledgeProjectId()) && StringUtils.isEmpty(knowledgeParam.getUserId())){
            List<String> treeList = knowledgeProjectService.getTreeListBy(knowledgeParam.getKnowledgeProjectId())
                    .stream().map(KbKnowledgeProject::getId).collect(Collectors.toList());
            //获取用户拥有的分类
            KbAuthorizedUser userInfo = SpringContextUtils.getUserInfo();
            if (null == userInfo) {
                return null;
            }
            if (!Integer.valueOf("0").equals(userInfo.getIsAll())) {
                Map<String, Boolean> userKnowProjectSigns = authorizedUserService.getUserKnowProjectSigns(treeList, userInfo.getId());
                log.info("userKnowProjectSigns=={}", userKnowProjectSigns.toString());
                //all permit
                Set<Map.Entry<String, Boolean>> entries = userKnowProjectSigns.entrySet();
                List<String> allPermitList = entries.stream().filter(Map.Entry::getValue).map(Map.Entry::getKey).collect(Collectors.toList());
                List<String> halfPermitList = entries.stream().filter(item -> !item.getValue()).map(Map.Entry::getKey).collect(Collectors.toList());
                //know permit
                log.info("halfPermitList=={}", halfPermitList.toString());
                if (!halfPermitList.isEmpty()) {
                    knowPermitList = authorizedUserService.getUserPermissionKnowsByIds(halfPermitList, userInfo.getId());
                }

                log.info("knowPermitList=={}", knowPermitList.toString());
                catList = allPermitList;
            } else {
                catList = treeList;
            }
        }else if(StringUtils.isNotEmpty(knowledgeParam.getKnowledgeProjectId())){
            catList = Arrays.asList(knowledgeParam.getKnowledgeProjectId().split(","));
        }
        SearchRequest searchRequest = new SearchRequest(Constants.ES_DATA_FOR_KNOWLEDGE);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        //设置分页参数
        searchSourceBuilder.size(pageSize);
        searchSourceBuilder.from((pageNo - 1) * pageSize);
        //默认按照置顶以及时间倒序排列
        //根据topNum正序查找，查询置顶数据
//        searchSourceBuilder.sort("publishDate", SortOrder.DESC);
        if ("publishTime".equals(column)) {
            if ("desc".equals(order)) {
                searchSourceBuilder.sort("publishDate", SortOrder.DESC);
            } else if ("asc".equals(order)) {
                searchSourceBuilder.sort("publishDate", SortOrder.ASC);
            }
        }
        if ("verifyTime".equals(column)) {
            if ("desc".equals(order)) {
                searchSourceBuilder.sort("verifyTime", SortOrder.DESC);
            } else if ("asc".equals(order)) {
                searchSourceBuilder.sort("verifyTime", SortOrder.ASC);
            }
        }
        //默认最大数量是10000，设置为true后，显示准确数量
        searchSourceBuilder.trackTotalHits(true);
        //创建查询对象
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();


//        if (StringUtils.isNotEmpty(knowledgeParam.getTitle())) {
//            boolQuery.must(QueryBuilders.matchPhraseQuery("title", knowledgeParam.getTitle()));
//        }
//        if (StringUtils.isNotEmpty(knowledgeParam.getContent())) {
//            BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
//            nestedBoolQueryBuilder.must(QueryBuilders.matchPhraseQuery("contents.content", knowledgeParam.getContent()));
//            NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("contents", nestedBoolQueryBuilder, ScoreMode.None);
//            boolQuery.must(nestedQueryBuilder);
//        }

        //搜索条件
        if (StringUtils.isNotBlank(knowledgeParam.getSearchInfo())) {
            MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(knowledgeParam.getSearchInfo());
            if ("精确".equals(knowledgeParam.getSearchAccuracy())) {
                multiMatchQueryBuilder.type(MultiMatchQueryBuilder.Type.PHRASE);
                if (null != knowledgeParam.getSearchScope() && Integer.valueOf(1).equals(knowledgeParam.getSearchScope())) {
                    multiMatchQueryBuilder.field("title");
                } else if (null != knowledgeParam.getSearchScope() && Integer.valueOf(2).equals(knowledgeParam.getSearchScope())) {
                    BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
                    multiMatchQueryBuilder.field("contents.content");
                    nestedBoolQueryBuilder.must(multiMatchQueryBuilder);
                    NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("contents", nestedBoolQueryBuilder, ScoreMode.None);
                    boolQuery.must(nestedQueryBuilder);

                } else {
                    multiMatchQueryBuilder.field("title", 60);
                    BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
                    multiMatchQueryBuilder.field("contents.content", 20);
                    nestedBoolQueryBuilder.must(multiMatchQueryBuilder);
                    NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("contents", nestedBoolQueryBuilder, ScoreMode.None);
                    boolQuery.must(nestedQueryBuilder);
                }
            } else {
                if (null != knowledgeParam.getSearchScope() && Integer.valueOf(1).equals(knowledgeParam.getSearchScope())) {
                    multiMatchQueryBuilder.field("title");
                } else if (null != knowledgeParam.getSearchScope() && Integer.valueOf(2).equals(knowledgeParam.getSearchScope())) {
                    BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
                    multiMatchQueryBuilder.field("contents.content");
                    nestedBoolQueryBuilder.must(multiMatchQueryBuilder);
                    NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("contents", nestedBoolQueryBuilder, ScoreMode.None);
                    boolQuery.must(nestedQueryBuilder);
                } else {
                    multiMatchQueryBuilder.field("title", 3);
                    BoolQueryBuilder nestedBoolQueryBuilder = QueryBuilders.boolQuery();
                    multiMatchQueryBuilder.field("contents.content", 1);
                    nestedBoolQueryBuilder.must(multiMatchQueryBuilder);
                    NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("contents", nestedBoolQueryBuilder, ScoreMode.None);
                    boolQuery.must(nestedQueryBuilder);
                }
            }
            boolQuery.must(multiMatchQueryBuilder);
        }
        if (StringUtils.isNotEmpty(knowledgeParam.getOrigin())) {
            boolQuery.must(QueryBuilders.matchQuery("origin", knowledgeParam.getOrigin()));
        }
        if (StringUtils.isNotEmpty(knowledgeParam.getUserId())) {
            boolQuery.must(QueryBuilders.matchQuery("userId", knowledgeParam.getUserId()));
        }
        if (StringUtils.isNotEmpty(knowledgeParam.getVerifierName())) {
            boolQuery.must(QueryBuilders.matchPhraseQuery("verifierName", knowledgeParam.getVerifierName()));
        }
        BoolQueryBuilder outer = new BoolQueryBuilder();
        if (!knowPermitList.isEmpty()) {
            BoolQueryBuilder boolQueryColumn = new BoolQueryBuilder();
            boolQueryColumn.must(QueryBuilders.termsQuery("id", knowPermitList));
            outer.should(boolQueryColumn);
        }
        if (!catList.isEmpty()) {
            boolQuery.must(QueryBuilders.termsQuery("knowledgeProjectId", catList));
        }
        if (StringUtils.isNotEmpty(knowledgeParam.getTypeIds())) {
            boolQuery.must(QueryBuilders.termsQuery("typeId", Arrays.asList(knowledgeParam.getTypeIds().split(","))));
        }
        if (StringUtils.isNotEmpty(knowledgeParam.getKbKnowledgeId())) {
            boolQuery.must(QueryBuilders.termsQuery("kbKnowledgeId", Arrays.asList(knowledgeParam.getKbKnowledgeId().split(","))));
        }
        if (null != knowledgeParam.getVerifyStatus()) {
            boolQuery.must(QueryBuilders.termQuery("verifyStatus", knowledgeParam.getVerifyStatus()));
        }

        //时间过滤筛选
        if (StringUtils.isNotBlank(knowledgeParam.getStartTime())) {
            boolQuery.filter(QueryBuilders.rangeQuery("publishDate").gte(EsDateUtil.esFieldDateFormat(knowledgeParam.getStartTime())));
        }
        if (StringUtils.isNotBlank(knowledgeParam.getEndTime())) {
            boolQuery.filter(QueryBuilders.rangeQuery("publishDate").lte(EsDateUtil.esFieldDateFormat(knowledgeParam.getEndTime())));
        }

        //时间过滤筛选
        if (StringUtils.isNotBlank(knowledgeParam.getVerifyStartTime())) {
            boolQuery.filter(QueryBuilders.rangeQuery("verifyTime").gte(EsDateUtil.esFieldDateFormat(knowledgeParam.getStartTime())));
        }
        if (StringUtils.isNotBlank(knowledgeParam.getVerifyEndTime())) {
            boolQuery.filter(QueryBuilders.rangeQuery("verifyTime").lte(EsDateUtil.esFieldDateFormat(knowledgeParam.getEndTime())));
        }
        boolQuery.mustNot(QueryBuilders.termQuery("deleteFlag",1));
        outer.should(boolQuery);
        searchSourceBuilder.query(outer);
        searchRequest.source(searchSourceBuilder);
        SearchResponse searchResponse = null;
        try {
            searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        } catch (IOException e) {
            e.printStackTrace();
        }

        if (null != searchResponse && null != searchResponse.getHits()) {
            SearchHit[] searchHits = searchResponse.getHits().getHits();
            List<KnowledgeVO> list = new ArrayList<>();
            for (SearchHit hit : searchHits) {
                String queryInfo = hit.getSourceAsString();
                KnowledgeVO info = JSON.parseObject(queryInfo, KnowledgeVO.class);
                info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
                info.setVerifyTime(EsDateUtil.esFieldDateMapping(info.getVerifyTime()));
                list.add(info);
            }

            IPage<KnowledgeVO> pageData = new Page<>(pageNo, pageSize, searchResponse.getHits().getTotalHits().value);

            pageData.setRecords(list);
            return pageData;
        }
        return new Page<>(pageNo, pageSize, 0);
    }

    @Override
    public List<KnowledgeVO> queryListByIds(String ids) {
        List<String> idList = Arrays.asList(ids.split(","));
        SearchRequest searchRequest = new SearchRequest(Constants.ES_DATA_FOR_KNOWLEDGE);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        //设置分页参数
        searchSourceBuilder.size(idList.size());
        searchSourceBuilder.from(0);
        //按发布时间倒序
        searchSourceBuilder.sort("publishDate", SortOrder.DESC);
        //默认最大数量是10000，设置为true后，显示准确数量
        searchSourceBuilder.trackTotalHits(true);
        //创建查询对象
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
        boolQuery.must(QueryBuilders.termsQuery("id", idList));
        searchSourceBuilder.query(boolQuery);
        searchRequest.source(searchSourceBuilder);
        SearchResponse searchResponse = null;
        try {
            searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        } catch (IOException e) {
            e.printStackTrace();
        }
        List<KnowledgeVO> list = new ArrayList<>();
        if (null != searchResponse && null != searchResponse.getHits()) {
            SearchHit[] searchHits = searchResponse.getHits().getHits();
            for (SearchHit hit : searchHits) {
                String queryInfo = hit.getSourceAsString();
                KnowledgeVO info = JSON.parseObject(queryInfo, KnowledgeVO.class);
                info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
                info.setVerifyTime(EsDateUtil.esFieldDateMapping(info.getVerifyTime()));
                list.add(info);
            }
        }
        return list;
    }
    @Override
    public IPage<KnowledgeVO> listFromPython(KnowledgeParam knowledgeParam, Integer pageNo, Integer pageSize, String column, String order) {

        Object data = redisUtil.get(genKey(knowledgeParam));
        if (org.springframework.util.StringUtils.isEmpty(data)) {
            List<String> treeList = knowledgeProjectService.getTreeListBy(knowledgeParam.getKnowledgeProjectId())
                    .stream().map(KbKnowledgeProject::getId).collect(Collectors.toList());
            List<String> permitKnowList = new ArrayList<>();
            List<String> catList = new ArrayList<>();
            KbAuthorizedUser userInfo = SpringContextUtils.getUserInfo();
            if (null == userInfo) {
                return null;
            }
            if (!Integer.valueOf("0").equals(userInfo.getIsAll())) {
                List<KbAuthuserKnowledgeprojectMap> permitKnowProjectList = knowledgeProjectService.getKnowledgeProjectListByUserId(userInfo.getId());
                //all permit
                List<String> allPermitList = permitKnowProjectList.stream()
                        .filter(item -> item.getSign() == 0)
                        .map(KbAuthuserKnowledgeprojectMap::getKnowledgeProjectId)
                        .collect(Collectors.toList());
                //know permit
                List<String> knowPermitList = permitKnowProjectList.stream()
                        .filter(item -> item.getSign() == 1)
                        .map(KbAuthuserKnowledgeprojectMap::getKnowledgeProjectId)
                        .collect(Collectors.toList());
                if (!knowPermitList.isEmpty()) {
                    permitKnowList = knowledgeProjectService.getKnowledgeListByUserId(userInfo.getId(), knowPermitList);
                }
                catList = (List<String>) CollectionUtils.intersection(treeList, allPermitList);
            } else {
                catList = treeList;
            }
            List<String> split = Arrays.asList(knowledgeParam.getTypeIds().split(","));
            JSONObject params = new JSONObject();
            params.put("type", split);
            params.put("startTime", knowledgeParam.getStartTime());
            params.put("endTime", knowledgeParam.getEndTime());
            params.put("author", knowledgeParam.getAuthor());
            params.put("origin", knowledgeParam.getOrigin());
            params.put("verifierName", knowledgeParam.getVerifierName());
            params.put("verifyStatus", knowledgeParam.getVerifyStatus());
            params.put("verifyStartTime", knowledgeParam.getVerifyStartTime());
            params.put("knowledge_base_id", knowledgeParam.getKbKnowledgeId().split(","));
            params.put("id", permitKnowList);
            params.put("knowledgeProjectId", catList);
            params.put("verifyEndTime", knowledgeParam.getVerifyEndTime());
            params.put("score_threshold", 600);//score
            params.put("vector_search_top_k", 100);//number

            if (knowledgeParam.getSearchScope().equals(2)) {
                params.put("question", knowledgeParam.getSearchInfo());
            }
            String result = HttpUtil.post(searchUrl, params.toJSONString(), 120000);
            if (result == null || !result.contains("result")) {
                return null;
            }
            JSONObject jsonObject = JSONObject.parseObject(result);
            data = jsonObject.get("result");
            redisUtil.set(genKey(knowledgeParam), data, 1);
        }
        JSONArray jsonArray = JSONObject.parseArray(data.toString());
        List<KnowledgeVO> knowledgeList = new ArrayList<>(jsonArray.size());
        for (Object o : jsonArray) {
            JSONObject one = JSONObject.parseObject(o.toString());

            knowledgeList.add(KnowledgeVO.builder()
                    .id((String) one.get("id"))
                    .importData(Integer.valueOf(one.get("importData").toString()))
                    .title((String) one.get("title"))
                    .author((String) one.get("author"))
                    .score((int) one.get("score"))
                    .verifyStatus("".equals(one.get("verifyStatus")) ? 0 : Integer.parseInt(one.get("verifyStatus").toString()))
                    .verifyTime(one.get("verifyTime").toString())
                    .publishDate(one.get("publishDate").toString())
                    .type(one.get("type").toString())
                    .verifierName(one.get("verifierName").toString())
                    .contents(Collections.singletonList(Content.builder().contentId(one.get("contentId").toString()).content(one.get("content").toString()).build()))
                    .build());
        }
        Map<String, List<KnowledgeVO>> map = knowledgeList.stream().collect(Collectors.groupingBy((KnowledgeVO::getId)));
        Iterator<Map.Entry<String, List<KnowledgeVO>>> iterator = map.entrySet().iterator();
        List<KnowledgeVO> knowledgeListGrouped = new ArrayList<>(map.size());
        while (iterator.hasNext()) {
            Map.Entry<String, List<KnowledgeVO>> next = iterator.next();
            List<KnowledgeVO> value = next.getValue();
            KnowledgeVO knowledgeVO = value.get(0);
            knowledgeVO.setContents(value.stream().map(item -> item.getContents().get(0)).collect(Collectors.toList()));
            knowledgeListGrouped.add(knowledgeVO);
        }
        IPage<KnowledgeVO> pageData = new Page<>(pageNo, pageSize, knowledgeListGrouped.size());
        pageData.setRecords(knowledgeListGrouped);

        return pageData;
    }

    @Override
    public KnowledgeVO getById(String id) {
        Map<String, Object> map = esOpUtil.searchDoc(Constants.ES_DATA_FOR_KNOWLEDGE, id);
        if (map != null) {
            return JSON.parseObject(JSON.toJSONString(map), KnowledgeVO.class);
        }
        return null;
    }

    @Override
    public Result<?> doImportInfo(HttpServletRequest request, String userId) {
        MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
        Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
        if (fileMap.size() < 1) {
            return Result.error("请上传excel文件");
        }
        MultipartFile multipartFile = fileMap.get(new ArrayList<String>(fileMap.keySet()).get(0));
        int index = multipartFile.getOriginalFilename().lastIndexOf(".");
        String fileSuffix = multipartFile.getOriginalFilename().substring(index + 1);
        if ("xls".equals(fileSuffix) || "xlsx".equals(fileSuffix)) {
            //文件校验
            if (!this.checkExcel(multipartFile)) {
                log.error("excel格式不对");
                return Result.error("模版错误，请下载正确模板，再次上传！");
            }
            asyncService.doimport(request, fileSuffix, userId);
            return Result.OK("导入正在后台进行");
        } else {
            return Result.error("不支持的文件类型");
        }

    }

    private boolean checkExcel(MultipartFile multipartFile) {
        Workbook workbook = ExcelExportUtil.getWorkbook(multipartFile);
        if (workbook != null) {
            List<String> list = ExcelExportUtil.getSheetTitles(workbook);
            return list != null && list.contains("标题") && list.contains("正文")
                    && list.contains("作者") && list.contains("来源") && list.contains("发布时间");
        }
        return false;
    }

    @Override
    public void doExcel(HttpServletRequest request, String fileSuffix, String userId) {
        String kbKnowledgeId = request.getParameter("kbKnowledgeId");
        if(request.getRequestURI().contains("/user/")){
            //个人素材上传
            kbKnowledgeId = Constants.SC_USER;
        }
        String knowledgeProjectId = request.getParameter("knowledgeProjectId");
        String typeId = request.getParameter("typeId");
        try {
            MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
            Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
            MultipartFile multipartFile = fileMap.get(new ArrayList<String>(fileMap.keySet()).get(0));
            //1.上传文件到文件服务器
            // 将MultipartFile的文件内容保存到字节数组
            byte[] fileData = multipartFile.getBytes();

            List<List<String>> lists = ExcelExportUtil.readExcel(new ByteArrayInputStream(fileData), 0, 8);
            //存入es
            importInfo(lists, kbKnowledgeId, knowledgeProjectId, typeId, userId);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    @Override
    public List<KnowledgeVO> listByIds(String ids) {
        List<KnowledgeVO> list = new ArrayList<>();
        SearchRequest searchRequest = new SearchRequest(Constants.ES_DATA_FOR_KNOWLEDGE);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        //默认最大数量是10000，设置为true后，显示准确数量
        searchSourceBuilder.trackTotalHits(true);
        //创建查询对象
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
        if (!ids.isEmpty()) {
            List<String> idList = Arrays.asList(ids.split(","));
            boolQuery.must(QueryBuilders.termsQuery("id", idList));
        }
        searchSourceBuilder.query(boolQuery);
        searchRequest.source(searchSourceBuilder);
        SearchResponse searchResponse = null;
        try {
            searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        } catch (IOException e) {
            e.printStackTrace();
        }

        if (null != searchResponse && null != searchResponse.getHits()) {
            SearchHit[] searchHits = searchResponse.getHits().getHits();

            for (SearchHit hit : searchHits) {
                String queryInfo = hit.getSourceAsString();
                KnowledgeVO info = JSON.parseObject(queryInfo, KnowledgeVO.class);
                info.setPublishDate(EsDateUtil.esFieldDateMapping(info.getPublishDate()));
                list.add(info);
            }
        }
        return list;
    }

    @Override
    public void exportExcel(String ids) {
        //获取数据
        if (redisUtil.get(ids) == null) {
            try {
                String[] arr = new String[]{"标题", "正文", "作者", "来源", "发布时间"};
                List<KnowledgeVO> informationList = this.listByIds(ids);
                if (CollectionUtils.isNotEmpty(informationList)) {
                    //动态补充表头
                    List<String> headers = Arrays.asList(arr);
                    //每个sheet页最多5000条数据
                    List<List<KnowledgeVO>> partition = ListUtils.partition(informationList, 5000);
                    XSSFWorkbook workbook = new XSSFWorkbook();
                    for (int i = 0; i < partition.size(); i++) {
                        List<List<String>> rows = formatData(partition.get(i));
                        ExcelExportUtil.exportExcelData(workbook, i, headers, rows, "sheet" + (i + 1));
                    }
                    File fileDir = ExcelExportUtil.getFileDir();
                    String filePath = fileDir.getAbsolutePath() + File.separator + UUID.randomUUID().toString().replace("-", "") + "info.xlsx";
                    FileOutputStream outputStream = new FileOutputStream(filePath);
                    workbook.write(outputStream);
                    redisUtil.set(ids, filePath, 3600 * 24);
                }
            } catch (Exception e) {
                e.printStackTrace();
                log.error(e.getMessage());
            }
        }
    }

    @Override
    public String exportPackage(String ids) {
        List<KnowledgeVO> knowledgeVOS = this.listByIds(ids);
        List<String> pathList = new ArrayList<>(knowledgeVOS.size());
        for (KnowledgeVO knowledgeVO : knowledgeVOS) {
            pathList.add(filesStorage + knowledgeVO.getFiles().get(0).getFilePath());
        }

        String filePackage=UUID.randomUUID().toString().replace("-", "");
        ZipUtils.compressFiles(tmpFile,filePackage,pathList );
        String packagePath=tmpFile+filePackage+".zip";
        redisUtil.set(ids, packagePath, 3600 * 24);
        return packagePath;
    }

    /*
     * 将所有的文件都压缩到一个文件中去
     */
    @Override
    public String comprocessByName(List<String> fileNameList, String downloadFileName) {
        if (fileNameList.size() == 1) {
            return fileNameList.get(0);
        }
        ZipUtils.compressFiles(tmpFile,downloadFileName,fileNameList);
        return tmpFile+downloadFileName+".zip";
    }




    //导入发布库数据
    public void importInfo(List<List<String>> lists, String kbKnowledgeId, String knowledgeProjectId, String typeId, String createBy) {
        List<String> titles = lists.get(0);
        for (int i = 1; i < lists.size(); i++) {
            List<String> info = lists.get(i);
            try {
                Knowledge specialInformation = Knowledge.builder()
                        .kbKnowledgeId(kbKnowledgeId)
                        .knowledgeProjectId(knowledgeProjectId)
                        .typeId(typeId)
                        .createBy(createBy)
                        .verifyStatus(0)
                        .importData(1)
                        .userId(createBy)
                        .build();
                if (titles.contains("id") && StringUtils.isNotEmpty(info.get(titles.indexOf("id")))) {
                    specialInformation.setId(info.get(titles.indexOf("id")));
                }else{
                    specialInformation.setId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8));
                }
                if (StringUtils.isNotEmpty(info.get(titles.indexOf("标题")))) {
                    specialInformation.setTitle(info.get(titles.indexOf("标题")));
                } else {
                    log.error("上传的数据{}标题为空，此条数据忽略", info);
                    continue;
                }
                String contentWithTag = null;
                if(titles.contains("正文（带标签）")){
                    contentWithTag = info.get(titles.indexOf("正文（带标签）"));
                }
                if(StringUtils.isEmpty(contentWithTag)){
                    contentWithTag = info.get(titles.indexOf("正文"));
                }
                if (StringUtils.isNotEmpty(contentWithTag)) {
                    specialInformation.setContentWithTag(contentWithTag);
                    //片段切分
                    List<Content> contents = new ArrayList<>();
                    List<String> splitContents = HtmlUtil.splitContents(specialInformation.getContentWithTag());
                    for (String content : splitContents) {
                        System.out.println(content.length());
                        System.out.println(content);
                        contents.add(Content.builder()
                                .contentId(codeGenerateUtil.geneIdNo(Constants.FINANCE, 8))
                                .content(content)
                                .build());
                    }
                    specialInformation.setContents(contents);
                } else {
                    log.error("上传的数据{}正文为空，此条数据忽略", info);
                    continue;
                }
                if (info.size()>4&&StringUtils.isNotEmpty(info.get(titles.indexOf("作者")))) {
                    specialInformation.setAuthor(info.get(titles.indexOf("作者")));
                }
                if (info.size()>5&&StringUtils.isNotEmpty(info.get(titles.indexOf("来源")))) {
                    specialInformation.setOrigin(info.get(titles.indexOf("来源")));
                }
                if (info.size()>6&&StringUtils.isNotEmpty(info.get(titles.indexOf("发布时间")))) {
                    specialInformation.setPublishDate(EsDateUtil.esFieldDateFormat(info.get(titles.indexOf("发布时间"))));
                }
                if (info.size()>7&&StringUtils.isNotEmpty(info.get(titles.indexOf("原文链接")))) {
                    specialInformation.setSourceAddress(info.get(titles.indexOf("原文链接")));
                }
                specialInformation.setDeleteFlag(0);
                specialInformation.setCreateTime(EsDateUtil.esFieldDateFormat(cn.hutool.core.date.DateUtil.formatDateTime(new Date())));
                esOpUtil.docSavaByEntity(Constants.ES_DATA_FOR_KNOWLEDGE, specialInformation.getId(), specialInformation);
                KnowledgeMessage knowledgeMessage = new KnowledgeMessage();
                BeanUtils.copyProperties(specialInformation, knowledgeMessage);
                knowledgeMessage.setType(specialInformation.getTypeId());
                //produceInfo.sendKnowledgeExcelContents(knowledgeMessage);
                produceInfo.sendKnowledgeContents(knowledgeMessage);
            } catch (NumberFormatException e) {
                log.error("处理异常");
            }

        }
    }


    private String genKey(KnowledgeParam knowledgeParam) {
        String type = (knowledgeParam.getTypeIds() == null ? "" : knowledgeParam.getTypeIds());
        String knowledgeProjectId = (knowledgeParam.getKnowledgeProjectId() == null ? "" : knowledgeParam.getKnowledgeProjectId());
        String startTime = (knowledgeParam.getStartTime() == null ? "" : knowledgeParam.getStartTime());
        String endTime = (knowledgeParam.getEndTime() == null ? "" : knowledgeParam.getEndTime());
        String author = (knowledgeParam.getAuthor() == null ? "" : knowledgeParam.getAuthor());
        String origin = (knowledgeParam.getOrigin() == null ? "" : knowledgeParam.getOrigin());
        String verifierName = (knowledgeParam.getVerifierName() == null ? "" : knowledgeParam.getVerifierName());
        String verifyStatus = (knowledgeParam.getVerifyStatus() == null ? "" : String.valueOf(knowledgeParam.getVerifyStatus()));
        String verifyStartTime = (knowledgeParam.getVerifyStartTime() == null ? "" : knowledgeParam.getVerifyStartTime());
        String verifyEndTime = (knowledgeParam.getVerifyEndTime() == null ? "" : knowledgeParam.getVerifyEndTime());
        String searchInfo = (knowledgeParam.getSearchInfo() == null ? "" : knowledgeParam.getSearchInfo());
        return type + "#" +
                knowledgeProjectId + "#" +
                startTime + "#" +
                endTime + "#" +
                author + "#" +
                origin + "#" +
                verifierName + "#" +
                verifyStatus + "#" +
                verifyStartTime + "#" +
                verifyEndTime + "#" +
                searchInfo + "#";
    }


    private List<List<String>> formatData(List<KnowledgeVO> knowledgeVOS) {
        List<List<String>> list = new ArrayList<>(knowledgeVOS.size());
        for (KnowledgeVO knowledgeVO : knowledgeVOS) {
            List<String> innerList = new ArrayList<>();
            innerList.add(knowledgeVO.getTitle());
            innerList.add(knowledgeVO.getContentAll());
            innerList.add(knowledgeVO.getAuthor());
            innerList.add(knowledgeVO.getOrigin());
            innerList.add(knowledgeVO.getPublishDate());
            list.add(innerList);
        }
        return list;
    }

}
