package com.zzsn.search.google;

import cn.hutool.core.date.DateUtil;
import com.zzsn.cache.JedisUtil;
import com.zzsn.cache.MemcachedUtils;
import com.zzsn.docinfo.DocInfo;
import com.zzsn.search.db.DBManager;
import com.zzsn.search.db.SnowIdUtils;
import com.zzsn.search.entity.ClbAnsProcessitem;
import com.zzsn.search.entity.KeywordMsg;
import com.zzsn.search.extractor.ContentFileFinder;
import com.zzsn.search.extractor.StandardWebExtractorHandler;
import com.zzsn.utility.index.Constants;
import com.zzsn.utility.model.CatchWebByMetaSearch;
import com.zzsn.utility.model.ContentFileResult;
import com.zzsn.utility.model.FileTag;
import com.zzsn.utility.util.ChromeUtil;
import com.zzsn.utility.util.RequestUtil;
import com.zzsn.utility.util.Utility;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.ssl.SSLContextBuilder;
import org.apache.http.ssl.TrustStrategy;
import org.apache.http.util.EntityUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import javax.net.ssl.SSLContext;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.sql.SQLException;
import java.sql.Types;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@Slf4j
@Data
public class GoogleSearchThread implements Runnable {

    public KeywordMsg keywordMsg;
    public List<String> keywords;
    public Integer threadId;
    String cache_key="google_keyWords";
    static String siddtr="202203070005";
    @Override
    public void run() {
        try {
            crawler();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }


    public void crawler()throws Exception{
//        String keyWord = keywordMsg.getKeyWord();
        List<String> keyWords = keywords;
        log.info("keyWords：" + keyWords);
        for (String kWord :keyWords) {

//            String url1= Constants.META_SEARCH_URL;
            String url = kWord;
//            String url1= Constants.META_SEARCH_URL;
//            String url = url1 + kWord;
            List<String> urlList = new ArrayList<String>();
            log.info("url：" + url);
            urlList.add(url);
//            String charset = locateCharSet(urlList.get(0));
            String charset ="UTF-8";
            Long orgId=202203050005l;
            Long tid=202203050005l;
//            String sid="202202150000";
            String proxyid=Constants.PROXY;
            if(proxyid.equals("1")) {
                CatchWebNews(GoogleRecorderUtil.CatchWebOfGoogle1(urlList, charset, orgId, tid),kWord);
            }else {
                CatchWebNews(GoogleRecorderUtil.CatchWebOfGoogle1(urlList, charset, orgId, tid),kWord);
            }
            JedisUtil.sadd(cache_key,kWord);
        }


    }

    private String locateCharSet(String url) {
        String encoding = "gbk";
        try {
            Connection conn = Jsoup.connect(url);
            conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36)");
            // 伪装成浏览器
            Document doc = conn.ignoreContentType(true).timeout(10000).get();
            Pattern p1 = Pattern.compile("<meta[^>]*>",
                    Pattern.CASE_INSENSITIVE);
            Matcher m1 = p1.matcher(doc.toString());
            while (m1.find()) {
                String str = m1.group();
                Pattern p2 = Pattern.compile("charset[^\\s||\"||;||'||>]*");
                Matcher m2 = p2.matcher(str);
                if (m2.find()) {
                    encoding = m2.group().substring(8);
                    if (encoding.trim().length() == 0) {
                        Pattern p3 = Pattern
                                .compile("charset=\"[^\\s||\"||;||>]*");
                        Matcher m3 = p3.matcher(str);
                        if (m3.find()) {
                            encoding = m3.group().substring(9);
                        }
                        if (encoding.trim().length() == 0) {
                            // encoding = DetectCharSet.detectCharSet(fileName);
                            // if(encoding == null){
                            encoding = "gbk";
                            // }
                        }
                    }

                    return encoding;
                }
            }
        } catch (IOException e) {
            // e.printStackTrace();
//            System.out.println("获取出错编码方式");
            log.info("获取出错编码方式");
            return encoding;
        }

        return encoding;
    }

    // 抓取新闻内容
    private void CatchWebNews(List<CatchWebByMetaSearch> catchWebList,String keyword) {
        try {
            int count = 0;
            int g=catchWebList.size()>5?5:catchWebList.size();
//            for (int i = 0; i < g ; i++) {
            for (int i = 0; i < catchWebList.size(); i++) {
                try {
                    CatchWebByMetaSearch cwbm = catchWebList.get(i);
                    // 判断该网址是否存在于缓存池中
                    String orgId = String.valueOf(cwbm.getOrgId());
                    String key = Constants.SOURCEADDRESS + orgId + cwbm.getTid()+10;
                    Object cacheObj = MemcachedUtils.get(key);
                    List<String> addressList = new ArrayList<String>();
                    if (cacheObj != null && !"null".equals(cacheObj)) {
                        addressList = (List<String>) cacheObj;
                        int flg = 0;
                        //判断缓存池中有没有该网址
                        for (String address : addressList) {
                            if (address.equals(cwbm.getSourceaddress())) {
                                flg = 1;
                                break;
                            }
                        }
                        //如果缓存池中存在该网址，证明已经处理过，不再处理。
                        if (flg == 1) {
                            System.out.println(cwbm.getSourceaddress()+" 数据重复");
                            continue;
                        }
                    }

                    String infourl = cwbm.getSourceaddress();
                    String infodata = "";
                    String charset = "";
                    System.out.println(cwbm.getTitle()+"=="+infourl);
                    if (infourl == null || infourl.contains(".pdf") || infourl.trim().length()==0|| infourl.contains(".PDF")||infourl.contains("download")) {
                        continue;
                    }
//                    infodata=getContentByUrl(infourl);
//					测试获取内容通过模拟浏览器获取
//                    SeleniumTime seleniumTime=new SeleniumTime();
//                    infodata= seleniumTime.getScopehtml(infourl);
                    infodata= ChromeUtil.getChromeDoc(infourl);
                    if(infourl.contains("toutiao.com") &&(null == infodata || infodata.length() < 50)){
                        infodata = RequestUtil.getTaotiaoData(infourl );
                    }
                    if(StringUtils.isEmpty(infodata)){
                        System.out.println("122222222222222222222222/为空，则爬取下一个");
                        //为空，则爬取下一个
                        continue;
                    }
                    String contentCharset = Utility.getWebEncodingByStr(infodata);
                    String content = null;
                    if (infodata != null && charset != null&& contentCharset != null) {
//                        content = Utility.convertCharset(infodata, charset,charset);
                        content=infodata;
                    }
                    if (content != null) {
                        cwbm.setCharset(charset);
                        cwbm.setLastModify("");
                        cwbm.setContent(content);
                    }
                    DocInfo docInfo = new DocInfo();
                    docInfo.setContentType("HTML");
                    docInfo.setOrgId(cwbm.getOrgId());
                    docInfo.setSid(cwbm.getSid());
                    docInfo.setSourceType("News");
                    docInfo.setLastModified(cwbm.getLastModify());
                    docInfo.setCharset("utf-8");
                    docInfo.setSourceaddress(cwbm.getSourceaddress());
                    docInfo.setTitle(cwbm.getTitle().replace("...", ""));
                    docInfo.setAuthor(cwbm.getAuthor());
                    docInfo.setPublishDate(cwbm.getPublishDate());
                    docInfo.setOrigin(cwbm.getSourcesite());
                    docInfo.setKeywords(keyword);
                    docInfo.setSummary(cwbm.getSummary());
                    StandardWebExtractorHandler swe = new StandardWebExtractorHandler();
                    try {
                        swe.doHandler(content, docInfo);
//						判断是否存在对应域名的模板

                    } catch (Exception e1) {
                        log.info("模板解析异常"+e1.getMessage());
                    }
                    System.out.println(docInfo.getTitle()+"---"+docInfo.getSourceaddress());
                    docInfo.setFileDownLoadPath(null);
                    Map<String, String> params = new HashMap<String, String>();
                    params.put("fromWhere", "Google元搜索");
                    if (null!=cwbm.getTid()) {
                        params.put("tid", String.valueOf(cwbm.getTid()));
                    }
                    docInfo.setOtherParams(params);
                    if (docInfo.getTitle() != null
                            && docInfo.getTitle().trim().length() > 0
                            && docInfo.getContentNoTag() != null
                            && docInfo.getContentNoTag().trim().length() > 0) {
//                        String week = DateUtil.getDateBeforeWeek(new Date());
//                        && docInfo.getPublishDate().compareTo(week)>0
                        if (docInfo.getPublishDate()!=null && docInfo.getPublishDate().trim().length()>0
                                ) {
                            ContentFileResult contentFileResult = new ContentFileResult();
                            try {
                                contentFileResult = getContentFile(docInfo.getContentWithTag(),docInfo.getSourceaddress());
//								docInfo.setContentWithTag(contentFileResult.getContentImgCvtTag());
                                docInfo.setContentWithTag(ContentFileFinder.rmHtmlImgOrAtag(contentFileResult.getContentImgCvtTag()));

                                docInfo.setContentImgCvtTag(contentFileResult.getContentImgCvtTag());
                                Map<String, FileTag> imgMap = contentFileResult.getFileMap();
                                //
                                for (String keyImg : imgMap.keySet()) {
                                    FileTag fileTag = imgMap.get(keyImg);
                                    String savePath = fileTag.getSavePath();
                                    InputStream is = getImg(fileTag.getAbsolutePath());
                                    if (is!=null) {
                                        int size = is.available();
                                        if (size>0) {
//                                            mqSender.sendFile(is, true, savePath);
                                        }
                                    }
                                }

                            } catch (Exception e) {
                                // TODO Auto-generated catch block
//								e.printStackTrace();
                                log.info(e.getMessage());
                            }

                            System.out.println(docInfo.getTitle()+"---"+docInfo.getSourceaddress());
                            log.info("title:"+docInfo.getTitle()+"|address:"+docInfo.getSourceaddress()+
                                    "|content:"+(docInfo.getContentNoTag()==null?"":docInfo.getContentNoTag().length()+""));

                            intsertData(docInfo);
                            //信息转换
//                            ClbAnsProcessitem processitem = docInfoTrans2Processitem(docInfo);
//                            ObjectMapper mapper = new ObjectMapper();
//                            String docjson = mapper.writeValueAsString(processitem);
//                            System.out.println(docjson);
//                            kafkaTemplate.send(Constants.KAFKA_PRODUCT_TOPIC, "key", docjson);

                        }else {
                            log.info("资讯发布时间："+docInfo.getPublishDate());
                        }
                        count++;
                    }else {
                        log.info("资讯内容："+docInfo.getContentNoTag());
//                        System.out.println(docInfo.getContentNoTag());
                    }

                    // 加入缓存池中
                    System.out.println("加入缓存池");
                    addressList.add(docInfo.getSourceaddress());
                    MemcachedUtils.set(key,addressList,60*60*24);

                } catch (Exception e) {
                    log.info("访问出错"+e.getMessage());
                    continue;
                }

            }
            System.out.println("本次成功件数：" + count);
            log.info("本次成功件数：" + count);
        } catch (Exception e) {
            log.info("访问出错"+e.getMessage());
        }

    }
    static String insertSql = "insert into cis_ans_processitem " +
            " (id,sid, title,summary,publish_date,origin,author, content,words,keywords,sourceaddress,create_date) " +
            " values(?,?,?,?,?,?,?,?,?,?,?,?)";
    private static void intsertData(DocInfo docInfo)
    {
        long snowID = SnowIdUtils.uniqueLong();
        String id=snowID+"";
        String sid=siddtr;
//        String sid=docInfo.getSid()+"";
        String title=docInfo.getTitle();
            String summ = docInfo.getSummary();
        if(summ!=null && summ.length()>5000){
            summ=summ.substring(0,4900);
        }
        String summary=summ;
        String publish_date=docInfo.getPublishDate();
        String origin=docInfo.getOrigin();
        System.out.println("origin==="+origin);
        String author=docInfo.getAuthor();
        String content=docInfo.getContentNoTag();
        String words=docInfo.getContentWithTag();
        String keywords=docInfo.getKeywords();
        String sourceaddress=docInfo.getSourceaddress();
        String create_date = DateUtil.format(new Date(), "yyyy-MM-dd HH:mm:ss");

        DBManager dm = new DBManager();
        String[] coulmn = new String[]{id,  sid, title,summary,publish_date,origin,author,content,words,keywords,sourceaddress,create_date};
        int[] type = new int[]{Types.CHAR, Types.CHAR, Types.CHAR, Types.CHAR, Types.CHAR, Types.CHAR, Types.CHAR, Types.CHAR, Types.CHAR, Types.CHAR, Types.CHAR, Types.CHAR};

        try {
            boolean flag = dm.updateOrAdd(coulmn, type, insertSql);
            if(flag) {
                System.out.println("插入成功");
            }
        } catch (SQLException e) {
            e.printStackTrace();
        }
    }

    public ClbAnsProcessitem docInfoTrans2Processitem(DocInfo docInfo){
        ClbAnsProcessitem clbAnsProcessitem=new ClbAnsProcessitem();
        clbAnsProcessitem.setSid(docInfo.getSid()+"");
        clbAnsProcessitem.setTitle(docInfo.getTitle());
        clbAnsProcessitem.setContent(docInfo.getContentNoTag());
        clbAnsProcessitem.setContentWithtag(docInfo.getContentWithTag());
        clbAnsProcessitem.setSummary(docInfo.getSummary());
        clbAnsProcessitem.setAuthor(docInfo.getAuthor());
        clbAnsProcessitem.setOrigin(docInfo.getOrigin());
        clbAnsProcessitem.setPublishDate(docInfo.getPublishDate());
        clbAnsProcessitem.setSourceAddress(docInfo.getSourceaddress());

        return clbAnsProcessitem;
    }
    //转换qq新闻链接
    public static String transqqURl(String oldurl){
        String patt="https://new.qq.com/omn/[date]/[pamars].html";
        String b1=oldurl.substring(oldurl.lastIndexOf("/")+1);
        String b2=getNumbers(b1);
        String curl=patt.replace("[date]",b2).replace("[pamars]",b1);
        return curl;
    }
    public static String getNumbers(String content) {

        Pattern pattern = Pattern.compile("\\d+");
        Matcher matcher = pattern.matcher(content);
        while (matcher.find()) {
            return matcher.group(0);
        }
        return "";
    }

    //根据url获取html内容
    private  String getContentByUrl(String infourl){
        String infodata = "";
        String charset="utf-8";
        CloseableHttpClient httpClient =createSSLClientDefault();
        HttpGet httpgeturl = new HttpGet(infourl);// Get请求
        httpgeturl.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 60000);
        httpgeturl.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 60000);
        // 伪装成浏览器
        httpgeturl.setHeader("Content-Type","application/x-www-form-urlencoded;charset=utf-8");
        httpgeturl.setHeader("User-Agent", "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US);");
        httpgeturl.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        HttpResponse httprespse=null;
        try {
            httprespse = httpClient.execute(httpgeturl);
        } catch (Exception e1) {
            System.out.println("请求失败。。更换协议");
            httpClient=createSSLClientDefaulttsl12();
            try {
                httprespse = httpClient.execute(httpgeturl);
            }catch (Exception e2) {
                log.info(e2.getMessage());
                return infodata;
            }
        }
        HttpEntity entitydata = httprespse.getEntity();// 获取返回数据
        // 上次更新时间取得
        Header lastModify = httprespse.getFirstHeader("Last-Modified");
        if (lastModify == null) {
            lastModify = httprespse.getLastHeader("Last-Modified");
        }
        String charstype = EntityUtils.getContentCharSet(entitydata);
        if (charstype != null) {
            charset = charstype;
        } else {
            charset = locateCharSet(infourl);
        }
        charset = Utility.charsetcheck(charset);
        try {
            infodata = EntityUtils.toString(entitydata, charset);
        }catch (Exception e3){
            log.info(e3.getMessage());
            return infodata;
        }
        httpgeturl.releaseConnection();
        return infodata;
    }

    private  CloseableHttpClient createSSLClientDefault(){
        try {
            SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {
                //信任所有
                @Override
                public boolean isTrusted(
                        java.security.cert.X509Certificate[] arg0, String arg1)
                        throws java.security.cert.CertificateException {
                    // TODO Auto-generated method stub
                    return true;
                }

            }).build();
            SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
            return HttpClients.custom().setSSLSocketFactory(sslsf).build();
        } catch (KeyManagementException e) {
            e.printStackTrace();
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        } catch (KeyStoreException e) {
            e.printStackTrace();
        }
        return  HttpClients.createDefault();
    }
    private  CloseableHttpClient createSSLClientDefaulttsl12(){
        try {
            SSLContext sslContext = new SSLContextBuilder().useProtocol("TLSv1.2").loadTrustMaterial(null, new TrustStrategy() {
                //信任所有
                @Override
                public boolean isTrusted(
                        java.security.cert.X509Certificate[] arg0, String arg1)
                        throws java.security.cert.CertificateException {
                    // TODO Auto-generated method stub
                    return true;
                }

            }).build();
            SSLConnectionSocketFactory sslsf = new SSLConnectionSocketFactory(sslContext);
            return HttpClients.custom().setSSLSocketFactory(sslsf).build();
        } catch (KeyManagementException e) {
            e.printStackTrace();
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        } catch (KeyStoreException e) {
            e.printStackTrace();
        }
        return  HttpClients.createDefault();
    }
    /**
     * 爬取图片
     * 创建人:  李东亮
     * 创建时间:  2016-5-10 上午10:57:20
     * @version 1.0
     * @param contentWithTag
     * @param sourceaddress
     * @throws Exception
     */
    private ContentFileResult getContentFile(String contentWithTag,String sourceaddress)throws Exception{
        String contentImgCvtTag = contentWithTag;
        String formatImgContent= contentWithTag;
        Map<String, FileTag> imgDataMap = ContentFileFinder.getContentFileTag(contentWithTag,sourceaddress);
        //key为图片爬取路径，value为图片保存路径
        Map<String, FileTag> imgMap = new HashMap<String, FileTag>();
        for (String key : imgDataMap.keySet()) {
            FileTag fileTag = imgDataMap.get(key);
            while (contentImgCvtTag.contains(key)) {
                //IMG_SERVER开头的路径
                contentImgCvtTag = contentImgCvtTag.replace(key, fileTag.getSaveTag());
            }
//            while (formatImgContent.contains(fileTag.getAbsolutePath())&&!key.equals(fileTag.getAbsoluteTag())) {
//                //转换为绝对路径
//                formatImgContent = formatImgContent.replace(key, fileTag.getAbsoluteTag());
//            }
            imgMap.put(fileTag.getAbsolutePath(), fileTag);
        }

        ContentFileResult cis = new ContentFileResult();
        cis.setContentAbsoulute(formatImgContent);
        cis.setContentImgCvtTag(contentImgCvtTag);
        cis.setFileMap(imgMap);
        return cis;
    }

    public InputStream getImg(String dataUrl){
        CloseableHttpClient httpClient = createSSLClientDefault();

        CloseableHttpResponse response = null;
        InputStream instream =null;
        try {
            HttpGet get = new HttpGet();
            get.setURI(new URI(dataUrl));
            response = httpClient.execute(get);
            HttpEntity entity = response.getEntity();
            if (entity != null) {
                //创建一个输入流对象
                instream = entity.getContent();

//                 BufferedReader reader = new BufferedReader(new InputStreamReader(instream));
//                 StringBuilder sb = new StringBuilder();
//                 String line = null;
//                 while ((line = reader.readLine()) != null) {
//                     sb.append(line + "\n");
//                 }
//                 instream.close();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }

        return instream;
    }

}
