#coding=utf-8
from urllib.parse import urljoin

import pymysql
import requests
import urllib3
from bs4 import BeautifulSoup
from gne import GeneralNewsExtractor
from langid import langid
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import csv
import threading
import time
from lxml import etree
from queue import Queue
import re,sys
import datetime
import redis
from kafka import KafkaProducer
import json
from baseCore import BaseCore
import configparser
urllib3.disable_warnings()
from smart_extractor import SmartExtractor


class SougouSpider(object):
    def __init__(self,searchkw,wordsCode,sid):
        # 创建ConfigParser对象
        self.config = configparser.ConfigParser()
        # 读取配置文件
        self.config.read('config.ini')
        baseCore=BaseCore()
        self.logger=baseCore.getLogger()
        self.url = 'https://www.sogou.com/'
        self.r = redis.Redis(host=self.config.get('redis', 'host'),
                             port=self.config.get('redis', 'port'),
                             password=self.config.get('redis', 'pass'), db=0)
        self.page_num = 1
        chrome_driver =self.config.get('selenium', 'chrome_driver')
        self.kafka_bootstrap_servers = self.config.get('kafka', 'bootstrap_servers')
        path =  Service(chrome_driver)
        chrome_options = webdriver.ChromeOptions()
        chrome_options.binary_location =  self.config.get('selenium', 'binary_location')
        self.driver = webdriver.Chrome(service=path,chrome_options=chrome_options)
        # driver = webdriver.Chrome(chrome_options=chrome_options)
        self.qtitle = Queue()
        self.qurl = Queue()
        self.detailList = Queue()
        self.searchkw = searchkw
        self.wordsCode = wordsCode
        self.sid = sid

    def createDriver(self):
        chrome_driver =self.config.get('selenium', 'chrome_driver')
        path =  Service(chrome_driver)
        chrome_options = webdriver.ChromeOptions()
        chrome_options.binary_location =self.config.get('selenium', 'binary_location')
        # 设置代理
        # proxy = "127.0.0.1:8080"  # 代理地址和端口
        # chrome_options.add_argument('--proxy-server=http://' + proxy)
        self.driver = webdriver.Chrome(service=path,chrome_options=chrome_options)
    #将列表数据插入到表中 meta_search_result
    def itemInsertToTable(self,items):
        try:
            itemdata=[]
            conx,cursorM=self.connMysql()
            for item in items:
                nowtime=self.getNowDate()
                data=(self.sid,self.wordsCode,item['title'],item['detailurl'],item['source'],item['publishtime'],item['content'],item['contentHtml'],'1',item['kword'],nowtime)
                itemdata.append(data)

            sql ="INSERT into meta_search_result (sid,wordsCode,title,detailurl,origin,publishdate,content,content_with_tag,state,keyword,create_time) VALUES (%s, %s,%s, %s, %s, %s, %s, %s, %s, %s, %s)"
            cursorM.executemany(sql, itemdata)
            self.logger.info("数据插入数据库成功！")
            # 定义插入数据的SQL语句
            # 执行插入操作
            conx.commit()
        except Exception as e:
            self.logger.info("数据插入数据库失败！")
        finally:
            self.closeSql(conx,cursorM)

    def connMysql(self):
        # 创建MySQL连接
        conx = pymysql.connect(host=self.config.get('mysql', 'host'),
                               user=self.config.get('mysql', 'username'),
                               password=self.config.get('mysql', 'password'),
                               database=self.config.get('mysql', 'database'))
        # 创建一个游标对象
        cursorM = conx.cursor()
        return conx,cursorM

    def closeSql(self,conx,cursorM):
        # 关闭游标和连接
        cursorM.close()
        conx.close()

    # 解析页面
    def parse_page(self):
        self.logger.info('解析搜狗列表页')
        response = self.driver.page_source
        response = response.replace('<em>', '')
        response = response.replace('</em>', '')
        html = etree.HTML(response)
        lists=self.xpath_paser(html)
        try:
            flag = html.xpath('//a[@id="sogou_next"]')[0]
        except Exception as e:
            flag=''
            lists=[]
        return flag, lists

    def getRealUrl(self,url):
        try:
            header={
                "accept":"*/*",
                "connection":"Keep-Alive",
                "user-agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36"
            }
            # url = 'https://www.sogou.com/link?url=hedJjaC291NbWrwHYHKCyPQj_ei8OKC13fJZ5YRQyvgjcXe6RUhCEXfbi95UdEys0ztd7q5nl6o.'
            url=f"https://www.sogou.com{url}"
            res = requests.get(url,headers=header)
            text=res.text
            # 定义正则表达式
            pattern = r'https?://[^\s<>"]+|www\.[^\s<>"]+'
            # 在给定的字符串中寻找匹配的URL
            urls = re.findall(pattern, text)
            uri=''
            if len(urls)>1:
                uri=urls[0]
        except Exception as e:
            self.logger.info("链接转换异常！")
        return uri

    def xpath_paser(self,html):
        lists=[]
        itemTag=html.xpath('//div[@class="vrwrap"]')
        for itemTag in itemTag:
            try:
                title=itemTag.xpath('.//h3[@class="vr-title"]/a/text()')[0]
            except Exception as e:
                title=''
            try:
                detailUrl=itemTag.xpath('.//h3[@class="vr-title"]/a/@href')[0]
                detailUrl=self.getRealUrl(detailUrl)
            except Exception as e:
                detailUrl=''
            try:
                sourceTag=itemTag.xpath('.//p[@class="news-from text-lightgray"]/span[1]/text()')[0]
            except Exception as e:
                sourceTag=''
            try:
                publishTag=itemTag.xpath('.//p[@class="news-from text-lightgray"]/span[2]/text()')[0]
                publishTag=str(publishTag)
                publishtime=self.paserTime(publishTag)
                publishTag=publishtime.strftime("%Y-%m-%d %H:%M:%S")
            except Exception as e:
                publishTag=''
            detailmsg={
                'title':title,
                'detailUrl':detailUrl,
                'sourceTag':sourceTag,
                'publishTag':publishTag
            }
            lists.append(detailmsg)
        self.logger.info(f'列表获取信息的条数{len(lists)}')
        return lists

        #获取当前时间
    def getNowDate(self):
        # 获取当前时间
        current_time = datetime.datetime.now()
        # 将时间转换为字符串
        currentdate = current_time.strftime("%Y-%m-%d %H:%M:%S")
        return currentdate

    #智能抽取
    def paserDetail(self,detailhtml,detailurl):
        try:
            extractor = GeneralNewsExtractor()
            article_content = extractor.extract(detailhtml,host=detailurl,with_body_html=True)
            # element = html2element(detailhtml)
        except:
            article_content={}

        return article_content
    #解析时间
    def paserTime(self,publishtime):
        timeType=['年前','月前','周前','前天','昨天','天前','今天','小时前','分钟前']
        current_datetime = datetime.datetime.now()
        publishtime=publishtime.strip()
        print(publishtime)
        try:
            if '年前' in publishtime:
                numbers = re.findall(r'\d+', publishtime)
                day=int(numbers[0])
                delta = datetime.timedelta(days=365 * day)
                publishtime = current_datetime - delta
            elif '月前' in publishtime:
                numbers = re.findall(r'\d+', publishtime)
                day=int(numbers[0])
                delta = datetime.timedelta(months= day)
                publishtime = current_datetime - delta
            elif '周前' in publishtime:
                numbers = re.findall(r'\d+', publishtime)
                day=int(numbers[0])
                delta = datetime.timedelta(weeks= day)
                publishtime = current_datetime - delta
            elif '天前' in publishtime:
                numbers = re.findall(r'\d+', publishtime)
                day=int(numbers[0])
                delta = datetime.timedelta(days= day)
                publishtime = current_datetime - delta
            elif '前天' in publishtime:
                delta = datetime.timedelta(days= 2)
                publishtime = current_datetime - delta
            elif '昨天' in publishtime:
                current_datetime = datetime.datetime.now()
                delta = datetime.timedelta(days= 1)
                publishtime = current_datetime - delta
            elif '今天' in publishtime or'小时前' in publishtime or '分钟前' in publishtime :
                delta = datetime.timedelta(hours= 5)
                publishtime = current_datetime - delta
            elif '年' in publishtime and '月' in publishtime :
                time_format = '%Y年%m月%d日'
                publishtime = datetime.datetime.strptime(publishtime, time_format)
            elif '月' in publishtime and '日' in publishtime :
                current_year = current_datetime.year
                time_format = '%Y年%m月%d日'
                publishtime=str(current_year)+'年'+publishtime
                publishtime = datetime.datetime.strptime(publishtime, time_format)
            elif '-' in publishtime:
                time_format = '%Y-%m-%d'
                publishtime = datetime.datetime.strptime(publishtime, time_format)
        except Exception as e:
            print('时间解析异常！！')
        return publishtime

    # 获取每一页数据, 开趴.
    def get_page_html(self):
        self.logger.info("进入搜狗首页...")
        self.driver.get(self.url)
        self.driver.find_element(By.ID, 'query').send_keys(self.searchkw)
        self.driver.find_element(By.ID, 'stb').click()
        wait = WebDriverWait(self.driver, 30)
        wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
        time.sleep(3)
        self.driver.find_element(By.ID, 'sogou_news').click()
        wait = WebDriverWait(self.driver, 30)
        wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
        time.sleep(3)
        self.logger.info("开始抓取首页...")
        try:
            flag, lists = self.parse_page()
            if len(lists)<1:
                return
        except Exception as e:
            time.sleep(5)
            return
        if len(lists)==0:
            time.sleep(5)
        for detail in lists:
            durl=detail['detailUrl']
            is_member = self.r.sismember('pysougou_'+self.wordsCode, durl)
            if is_member:
                continue
            self.detailList.put(detail)

        response = self.driver.page_source
        html = etree.HTML(response)
        hasnext = html.xpath('//a[@id="sogou_next"]//text()')[0]
        hasnext = hasnext.strip()
        timeFlag=False
        while hasnext == '下一页':
            try:
                if self.page_num==2:
                    break
                self.page_num = self.page_num + 1
                self.logger.info("开始抓取第%s页..." % self.page_num)
                try:
                    self.driver.find_element(By.XPATH, '//a[@id="sogou_next"]').click()
                except Exception as e:
                    time.sleep(5)
                    continue
                time.sleep(5)
                flag, lists = self.parse_page()
                if len(lists)<1:
                    break
                for detail in lists:
                    publishTag=detail['publishTag']
                    # if publishTag:
                    #     pubtime = datetime.datetime.strptime(publishTag, "%Y-%m-%d %H:%M:%S")
                    #     needDate='2022-01-01 00:00:00'
                    #     needTime = datetime.datetime.strptime(needDate, "%Y-%m-%d %H:%M:%S")
                    #     if pubtime < needTime:
                    #         timeFlag = True
                    #         break
                    is_member = self.r.sismember('pysougou_'+self.wordsCode, durl)
                    if is_member:
                        continue
                    self.detailList.put(detail)
                if timeFlag:
                    break
                try:
                    response = self.driver.page_source
                    html = etree.HTML(response)
                    hasnext = html.xpath('//a[@id="sogou_next"]//text()')[0]
                    hasnext = hasnext.strip()
                except Exception as e:
                    hasnext=''
            except Exception as e:
                time.sleep(5)
                break
        self.logger.info("抓取完毕")

        #获取资讯内容信息

    def getDetailmsg(self,detailmsg):
        try:
            detailurl=detailmsg['detailUrl']
            title = detailmsg['title']
            content,contentWithTag=self.extractorMsg(detailurl,title)
            contentWithTag=self.rmTagattr(contentWithTag,detailurl)
        except Exception as e:
            content=''
            contentWithTag=''

        currentdate=self.getNowDate()
        kword=self.searchkw
        publishDate=detailmsg['publishTag']
        publishDate=publishDate+''
        # publishtime=self.paserTime(publishtime)
        # publishDate=publishtime.strftime("%Y-%m-%d %H:%M:%S")
        detailmsg={
            'title':detailmsg['title'],
            'source':detailmsg['sourceTag'],
            'detailurl':detailurl,
            'content':content,
            'contentHtml':contentWithTag,
            'publishtime':publishDate,
            'currentdate':currentdate,
            'kword':kword
        }
        return detailmsg

    def webDriver(self,url):
        chrome_driver =self.config.get('selenium', 'chrome_driver')
        path = Service(chrome_driver)
        chrome_options = webdriver.ChromeOptions()
        chrome_options.binary_location =self.config.get('selenium', 'binary_location')
        driver = webdriver.Chrome(service=path,chrome_options=chrome_options)
        html=''
        try:
            driver.get(url)
            # 等待页面加载完成
            # wait = WebDriverWait(self.driver, 20)
            # wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
            time.sleep(2)
            html=driver.page_source
        except Exception as e:
            self.logger.info('请求失败')
        finally:
            driver.quit()

        return html

    def extractorMsg(self,url,title):
        content=''
        contentWithTag=''
        lang=''
        try:
            lang=self.detect_language(title)
            raw_html=self.webDriver(url)
            sm=SmartExtractor(lang)
            article=sm.extract_by_html(raw_html)
            content=article.cleaned_text
            contentWithTag=article.text
        except Exception as e:
            try:
                raw_html=self.webDriver(url)
                sm=SmartExtractor(lang)
                article=sm.extract_by_html(raw_html)
                content=article.cleaned_text
                contentWithTag=article.text
            except Exception as e:
                print('抽取失败！！')

        return content,contentWithTag

    def detect_language(self,html):
        soup = BeautifulSoup(html, "html.parser")
        text = soup.get_text()
        # 使用langid.py判断文本的语言
        lang, confidence = langid.classify(text)
        return lang
    # 获取详情页
    def get_detail_html(self):
        # 获取当前窗口的句柄
        # current_window = self.driver.current_window_handle
        while True:
            if self.detailList.qsize() != 0:
                try:
                    detailmsg=self.detailList.get()
                    title = detailmsg['title']
                    detailUrl = detailmsg['detailUrl']
                    print("%s:%s\n" % (title, detailUrl))
                    # # js = "window.open('"+detailUrl+"')"
                    # # self.driver.execute_script(js)
                    # try:
                    #     self.driver.get(detailUrl)
                    # except Exception as e:
                    #     self.driver.quit()
                    #     self.driver=self.createDriver()
                    #     self.driver.get(detailUrl)
                    #
                    # response = self.driver.page_source
                    # bdetail=self.getDetailmsg(response,detailmsg)
                    bdetail=self.getDetailmsg(detailmsg)
                    processitem=self.getProcessitem(bdetail)
                    try:
                        # self.sendkafka(processitem)
                        self.r.sadd('pysougou_'+self.wordsCode, processitem['sourceAddress'])
                    except Exception as e:
                        self.logger.info("放入kafka失败！")
                    #插入数据库
                    try:
                        items=[]
                        items.append(bdetail)
                        self.itemInsertToTable(items)
                    except Exception as e:
                        self.logger.info("插入数据库失败！")
                    # 关闭当前新窗口
                    # self.driver.close()
                    time.sleep(1)
                except Exception as e:
                    time.sleep(3)
                    self.logger.info("详情页解析异常！"+detailUrl)
            else:
                break
                # time.sleep(5)

    # def getDetailmsg(self,detailhtml,detailmsg):
    #     try:
    #         detailurl=detailmsg['detailUrl']
    #         article_content=self.paserDetail(detailhtml,detailurl)
    #         content=article_content['content']
    #         contentWithTag=article_content['body_html']
    #     except Exception as e:
    #         self.logger.info('内容抽取失败')
    #         content=''
    #         contentWithTag=''
    #     currentdate=self.getNowDate()
    #     kword=self.searchkw
    #     publishtime=detailmsg['publishTag']
    #     publishtime=self.paserTime(publishtime)
    #     publishDate=publishtime.strftime("%Y-%m-%d %H:%M:%S")
    #     detailmsg={
    #         'title':detailmsg['title'],
    #         'source':detailmsg['sourceTag'],
    #         'detailurl':detailurl,
    #         'content':content,
    #         'contentHtml':contentWithTag,
    #         'publishtime':publishDate,
    #         'currentdate':currentdate,
    #         'kword':kword
    #     }
    #     return detailmsg

    def rmTagattr(self,html,url):
        # 使用BeautifulSoup解析网页内容
        # soup = BeautifulSoup(html, 'html.parser')
        soup = self.paserUrl(html,url)
        # 遍历所有标签，并去掉属性
        for tag in soup.find_all(True):
            if tag.name == 'img':
                tag.attrs = {key: value for key, value in tag.attrs.items() if key == 'src'}
            elif tag.name !='img':
                tag.attrs = {key: value for key, value in tag.attrs.items() if key == 'src'}
            else:
                tag.attrs = {key: value for key, value in tag.attrs.items()}
        # 打印去掉属性后的网页内容
        # print(soup.prettify())
        html=soup.prettify()
        return html

    # 将html中的相对地址转换成绝对地址
    def paserUrl(self,html,listurl):
        soup = BeautifulSoup(html, 'html.parser')
        # 获取所有的<a>标签和<img>标签
        links = soup.find_all(['a', 'img'])
        # 遍历标签，将相对地址转换为绝对地址
        for link in links:
            if 'href' in link.attrs:
                link['href'] = urljoin(listurl, link['href'])
            elif 'src' in link.attrs:
                link['src'] = urljoin(listurl, link['src'])

        return soup

    def getProcessitem(self,bdetail):
        nowDate=self.getNowDate()
        content=bdetail['content']
        if content!='':
            processitem={
                "sid":self.sid,
                "source":"5",
                "title":bdetail['title'],
                "content":bdetail['content'],
                "contentWithtag":bdetail['contentHtml'],
                "origin":bdetail['source'],
                "publishDate":bdetail['publishtime'],
                "sourceAddress":bdetail['detailurl'],
                "createDate":nowDate
            }

        return processitem

    def sendkafka(self,processitem):
        try:
            producer = KafkaProducer(bootstrap_servers=[self.kafka_bootstrap_servers])
            content=processitem['content']
            publishDate=str(processitem['publishDate'])
            title=processitem['title']
            if title =='':
                return
            if content=='':
                return
            if publishDate=='':
                return
            kafka_result = producer.send("crawlerInfo", json.dumps(processitem, ensure_ascii=False).encode('utf8'))
            self.logger.info("数据发送kafka成功")
            self.logger.info(kafka_result.get(timeout=10))
        except Exception as e:
            self.logger.info('发送kafka异常')
        finally:
            producer.close()

    def run(self):
        # 获取每页URL
        c = threading.Thread(target=self.get_page_html)
        c.start()
        c.join()
        # 解析详情页
        t = threading.Thread(target=self.get_detail_html)
        t.start()


if __name__ == '__main__':
    zhuce = SougouSpider()
    # zhuce.run()
    # zhuce.driver.close()