标题去重更新

config/config.txt

-maxId=23042000000124
+maxId=23101300001722
\ No newline at end of file

pom.xml

@@ -186,6 +186,18 @@
             <artifactId>commons-pool2</artifactId>
             <version>2.6.2</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi-ooxml</artifactId>
+            <version>4.1.2</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi</artifactId>
+            <version>4.1.2</version>
+        </dependency>
+
     </dependencies>
 
     <build>

src/main/java/com/zzsn/controller/BaseDataController.java

+package com.zzsn.controller;
+
+import com.zzsn.entity.*;
+import com.zzsn.service.BaseDataService;
+import com.zzsn.service.impl.DataSyncServiceImpl;
+import com.zzsn.utils.DateUtil;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.*;
+
+import java.util.*;
+
+@RestController
+@RequestMapping("/basedata")
+public class BaseDataController {
+
+
+    @Autowired
+    private BaseDataService baseDataService;
+
+    @Autowired
+    private DataSyncServiceImpl dataSyncService;
+    @RequestMapping(value ="/test", method = RequestMethod.GET)
+    @ResponseBody
+    public String  test(){
+        return "hello!";
+    }
+
+    /**
+     * 新增或修改
+     * @param basedata 数据对象
+     * @return 结果
+     */
+    @PostMapping("/save")
+    public ResultModel saveOrUpdate (@RequestBody Basedatavo basedata){
+
+        String date = DateUtil.format(new Date(), "YYYY-MM-dd HH:MM:ss");
+        String title = basedata.getTitle();
+        String publishDate = basedata.getPublishDate()==null?date:basedata.getPublishDate();
+        String sourceaddress = basedata.getSourceaddress();
+        String content = basedata.getContent();
+        String contentNoTag = basedata.getContentNoTag();
+        String summary = basedata.getSummary();
+        String origin = basedata.getOrigin()==null?"":basedata.getOrigin();
+        String orientation = basedata.getOrientation()==null?"1":basedata.getOrientation();
+
+        Map<String, Object> map=new HashMap<>();
+        Basedata bsdata = new Basedata();
+        bsdata.setTitle(title);
+        bsdata.setPublishDate(publishDate);
+        bsdata.setSourceaddress(sourceaddress);
+        bsdata.setContent(content);
+        bsdata.setContentNoTag(contentNoTag);
+        bsdata.setSummary(summary);
+        bsdata.setOrigin(origin);
+        bsdata.setOrientation(orientation);
+//        bsdata.setKeywords(keywords);
+        String bid = this.baseDataService.getserialno().toString();
+        Long idl =Long.parseLong(bid);
+        bsdata.setId(idl + "");
+        bsdata.setSid(290622l);
+        bsdata.setDelflag(2l);
+        bsdata.setIsdelete(0);
+        bsdata.setSourceType("news");
+        bsdata.setFromWhere("接口录入");
+
+        bsdata.setCreateDate(date);
+        List<Basedata> bsDataList=new ArrayList<>();
+        bsDataList.add(bsdata);
+        map.put("data",bsDataList);
+
+        BasedataType basedataType = new BasedataType();
+        basedataType.setId(Long.parseLong(bid.replace("0000","")));
+        basedataType.setBid(idl);
+        basedataType.setTid(16865l);
+        basedataType.setOrgId(3942l);
+        basedataType.setPublishDate(publishDate);
+        basedataType.setCreateDate(date);
+        basedataType.setDelflag(2l);
+        basedataType.setRelevance(0l);
+        List<BasedataType> typeList=new ArrayList<>();
+        typeList.add(basedataType);
+        //4.插入之前对basedata中的内容包含图片的进行base64转码，先不改变原content中src的内容
+        Map<String, Object> map1 = dataSyncService.imgHandler((List<Basedata>) map.get("data"));
+        try {
+//                dataSyncService.insertBatch((List<Basedata>) secretMap.get("data"), (List<BasedataType>) secretMap.get("typeData"),
+//                        (List<BaseDataImage>) map1.get("imageList"), (List<CesSysWeb>) localMap.get("exitWebList")
+//                        , (List<CesSysWeb>) localMap.get("newWebList"), (List<BaseDataWebMid>) localMap.get("baseDataWebList"));
+            dataSyncService.insertBatch((List<Basedata>) map.get("data"), typeList,
+                    (List<BaseDataImage>) map1.get("imageList"), null
+                    , null, null);
+
+        }catch (Exception e){
+            return ResultModel.OK("插入失败");
+        }
+        String id="插入数据的id: "+bid;
+        return ResultModel.OK(id);
+    }
+
+
+
+
+}

src/main/java/com/zzsn/entity/Basedatavo.java

+package com.zzsn.entity;
+
+import lombok.Data;
+
+
+@Data
+public class Basedatavo {
+
+    String title;
+    String publishDate;
+    String sourceaddress;
+    String content;
+    String contentNoTag ;
+    String summary ;
+    String origin;
+    String orientation ;
+
+}

src/main/java/com/zzsn/mapper/BaseDataMapper.java

@@ -17,6 +17,9 @@ public abstract interface BaseDataMapper extends BaseMapper<Basedata> {
     @DS("master")
     public abstract int queryByAddress(@Param("source") String paramString);
 
+    @DS("master")
+    public abstract int queryByTitle(@Param("title") String title);
+
     public abstract int save(@Param("item") Basedata paramBasedata);
 
     public abstract Basedata getContent();

src/main/java/com/zzsn/mapper/xml/BaseDataMapper.xml

@@ -149,6 +149,14 @@
 			CIS_ANS_BASEDATA
 		WHERE sourceaddress = #{source}
 	</select>
+	<select id="queryByTitle" resultType="java.lang.Integer">
+		SELECT
+			COUNT( ID )
+		FROM
+			CIS_ANS_BASEDATA
+		WHERE TITLE = #{title}
+	</select>
+
 	<select id="getContent" resultMap="basedataMap">
 		select CONTENT from CIS_ANS_BASEDATA where ID =20092900005829
 	</select>

src/main/java/com/zzsn/service/impl/DataSyncServiceImpl.java

@@ -269,12 +269,27 @@ public class DataSyncServiceImpl extends ServiceImpl<BaseDataMapper,Basedata> im
     public boolean isExist(Basedata basedata){
 //        logger.info("查询网址是否已经存在科工局数据库中{}",basedata);
         logger.info("查询网址是否已经存在科工局数据库中{}",basedata.getId());
-        int i = dataMapper.queryByAddress(basedata.getSourceaddress());
+        int i=0;
+        try {
+            i = dataMapper.queryByAddress(basedata.getSourceaddress());
+        }catch (Exception e){
+            i=1;
+        }
 //        QueryWrapper<Basedata> queryWrapper = new QueryWrapper<>();
 //        queryWrapper.eq("sourceaddress",basedata.getSourceaddress());
         if(i>0){
 //            logger.info("该网址已经存在科工局数据库中{}",basedata.getId());
             logger.info("该网址已经存在科工局数据库中{}",basedata.getSourceaddress());
+        }else {
+            try {
+                i = dataMapper.queryByTitle(basedata.getTitle());
+            } catch (Exception e) {
+                i=1;
+            }
+            if(i>0){
+                logger.info("该新闻已经存在科工局数据库中{}",basedata.getTitle());
+            }
+
         }
         return i>0;
     }
@@ -289,7 +304,11 @@ public class DataSyncServiceImpl extends ServiceImpl<BaseDataMapper,Basedata> im
             for (Basedata basedata : data) {
                 Map<String, FileTag> contentFileTag = null;
                 try{
-                    contentFileTag =ContentFileFinder.getContentFileTag(basedata.getContent(), basedata.getSourceaddress());
+                    try {
+                        contentFileTag = ContentFileFinder.getContentFileTag(basedata.getContent(), basedata.getSourceaddress());
+                    }catch (Exception e){
+                        result.add(basedata);
+                    }
                     if(contentFileTag==null || contentFileTag.size()<1){
                         result.add(basedata);
                         continue;
@@ -498,14 +517,15 @@ public class DataSyncServiceImpl extends ServiceImpl<BaseDataMapper,Basedata> im
 
     /**
      * 对content中的视频进行处理
-     * @param data
+     * @param map1
      * @return
      */
-    public Map<String, Object> videoHandler(List<Basedata> data) {
-        logger.info("对content中的视频图片进行处理:{}",data);
-        Map<String, Object> map = new HashMap<>();
+    public Map<String, Object> videoHandler(Map<String, Object> map) {
+        List<Basedata> data=(List<Basedata>)map.get("basedata");
+        logger.info("对content中的视频图片进行处理:{}",data.size());
+//        Map<String, Object> map = new HashMap<>();
         List<Basedata> result = new ArrayList<>();
-        List<BaseDataImage> imageList =new ArrayList<>();
+        List<BaseDataImage> videoList =new ArrayList<>();
         if(data!=null&&data.size()>0){
             for (Basedata basedata : data) {
                 Map<String, FileTag> contentFileTag = null;
@@ -515,17 +535,19 @@ public class DataSyncServiceImpl extends ServiceImpl<BaseDataMapper,Basedata> im
                         result.add(basedata);
                         continue;
                     }
+                    try {
                         Basedata base = new Basedata();
-                    BeanUtils.copyProperties(basedata,base);
+                        BeanUtils.copyProperties(basedata, base);
+                        logger.info("对处理数据的标题:{}",base.getTitle());
                         //遍历map
                         Document document = Jsoup.parse(basedata.getContent());
-                    for (String key:contentFileTag.keySet()){
+                        for (String key : contentFileTag.keySet()) {
                             FileTag value = contentFileTag.get(key);
                             //使用MD5加密：ID+图片的原网址
                             String md5 = DigestUtils.md5DigestAsHex((value.getAbsolutePath()).getBytes(StandardCharsets.UTF_8));
-                        System.out.println("加密后的md5值{}"+md5);
+                            logger.info("加密后的md5值{}" + md5);
                             String savePath = videoPath;
-                        String id = basedata.getId()+"";
+                            String id = basedata.getId() + "";
                             String url = value.getAbsolutePath();
                             String referer = basedata.getSourceaddress();
                             String path = savePath;
@@ -536,19 +558,48 @@ public class DataSyncServiceImpl extends ServiceImpl<BaseDataMapper,Basedata> im
                             String src = keyDoc.select("video").attr("src");
                             Elements elements = document.select("*[src=\"" + src + "\"]");
                             for (Element e : elements) {
-                            e.attr("src","/static/video/" + vname );
+                                e.attr("src", "/video/" + vname);
+                            }
+                            BaseDataImage baseDataImage = new BaseDataImage();
+                            baseDataImage.setId(md5);
+                            baseDataImage.setDataImage(vname);
+                            baseDataImage.setBid(base.getId());
+                            //暂不替换原content中的内容
+                            if(StringUtils.isNotBlank(vname)){
+                                logger.info("替换[{}]中的原图片[{}]为[{}]",base.getId(),key,md5);
+                                System.out.println("视频tihuanchenggong");
+                                videoList.add(baseDataImage);
+                                base.setContent(base.getContent().replace(key,md5));
+                            }else{
+                                System.out.println("视频weipaqu");
+                                base.setContent(base.getContent().replace(key,""));
                             }
                         }
                         base.setContent(document.outerHtml());
                         result.add(base);
                     }catch (Exception e){
+                        result.add(basedata);
+                        logger.info("视频处理解析失败");
+                    }
+                }catch (Exception e){
                     logger.info("[{}]提取img报错：[{}]",basedata.getId(),e.getMessage());
+                    result.add(basedata);
                     continue;
                 }
             }
         }
-        map.put("basedata",result);
-        map.put("imageList",imageList);
+        List<BaseDataImage> imageList=new ArrayList<>();
+        try {
+            imageList = (List<BaseDataImage>) map.get("imageList");
+            if (videoList.size() > 0) {
+                imageList.addAll(videoList);
+            }
+            map.put("basedata", result);
+            map.put("imageList", imageList);
+        }catch (Exception e){
+            map.put("basedata", result);
+            map.put("imageList", imageList);
+        }
         return map;
     }
 

src/main/java/com/zzsn/utils/ContentFileFinder.java

@@ -211,13 +211,26 @@ public class ContentFileFinder {
     * @return
     */
    public static String getSuffix(String uri){
+       String suffix="";
        uri = uri.replaceAll("http://|https://", "");
        Pattern p = Pattern.compile("/.+(\\.\\w{1,4})$");
        Matcher m =  p.matcher(uri);
        if(m.find()){
-           return m.group(1);
+           suffix= m.group(1);
        }
-       return "";
+       if(uri.contains(".jpg")){
+           suffix=".jpg";
+       }
+       if(uri.contains(".png")){
+           suffix=".png";
+       }
+       if(uri.contains(".jpeg")){
+           suffix=".jpeg";
+       }
+       if(uri.contains(".gif")){
+           suffix=".gif";
+       }
+       return suffix;
    }
 
     /**
@@ -271,12 +284,13 @@ public class ContentFileFinder {
             fileTag.setAbsoluteTag(imgTag.outerHtml());
             //图片保存路径
             suffix = ContentFileFinder.getSuffix(absolutePath);
-            fileTag.setSuffix(suffix.substring(1));
-//            if(StringUtils.isNotBlank(suffix)){
 //            fileTag.setSuffix(suffix.substring(1));
-//            }else{
-//                fileTag.setSuffix("");
-//            }
+            if(StringUtils.isNotBlank(suffix)){
+                fileTag.setSuffix(suffix.substring(1));
+            }else{
+                fileTag.setSuffix("");
+                continue;
+            }
             savePath = genImgFileName(suffix);
             fileTag.setSavePath(savePath);
             //图片保存标签

src/main/java/com/zzsn/utils/ImgBaseUtil.java

@@ -514,12 +514,18 @@ public class ImgBaseUtil {
      * @param vname  下载文件名
      */
     public static void sendDownloadInfo(String id,String url,String referer,String path,String vname ){
-        String param="{\"jsonrpc\": \"2.0\", \"id\": \""+id+"\", \"method\": \"aria2.addUri\","
-                + "\"params\": [[\""+url+"\"], {\"referer\":\""+referer+"\" , \"dir\": \""+path+"\",\"out\":\""+vname+"\"}]}";
-        sendPost_body("http://localhost:6800/jsonrpc",param);
+        for (int i = 0; i < 3; i++) {
+            try {
+                String param = "{\"jsonrpc\": \"2.0\", \"id\": \"" + id + "\", \"method\": \"aria2.addUri\","
+                        + "\"params\": [[\"" + url + "\"], {\"referer\":\"" + referer + "\" , \"dir\": \"" + path + "\",\"out\":\"" + vname + "\"}]}";
+                sendPost_body("http://localhost:6800/jsonrpc", param);
+            }catch (Exception e){
+
+            }
+        }
     }
-    public static String sendPost_body(String arl, String s) {
 
+    public static String sendPost_body(String arl, String s) {
         // 创建url资源
         OutputStreamWriter out = null;
         URL url;

src/main/resources/application-pro.yml

@@ -43,8 +43,8 @@ spring:
           # 多数据源配置
         master:
           url: jdbc:oracle:thin:@localhost:1521:orcl
-          username: cis1
-          password: cis1
+          username: cistest
+          password: cistest
           driver-class-name: oracle.jdbc.driver.OracleDriver
 #         slave:
 #           url: jdbc:oracle:thin:@localhost:1521:orcl
@@ -163,3 +163,6 @@ proxy:
 
 video:
   path: /home/ubuntu/video/
+
+excelpath: C:\\Users\\WIN10\\Desktop\\测试数据库环境\\22222.xls
+