from urllib.parse import urljoin

import langid
import pymysql
from gne import GeneralNewsExtractor
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import csv
import threading
import time
from lxml import etree
from queue import Queue
import re,sys
import datetime
import redis
from kafka import KafkaProducer
import json

from baseCore import BaseCore
import configparser
from goose3 import Goose
from goose3.text import StopWordsChinese
from smart_extractor import SmartExtractor
from entity import ExtractionRequest, UrlPickingRequest
import requests
# 从HTML中提取纯文本
from bs4 import BeautifulSoup

class GoogleSpider(object):
    def __init__(self,searchkw,wordsCode,sid):
        # 创建ConfigParser对象
        self.config = configparser.ConfigParser()
        # 读取配置文件
        self.config.read('config.ini')
        baseCore=BaseCore()
        self.logger=baseCore.getLogger()
        # self.url = f'https://www.google.com/search?q={searchkw}&tbm=nws&source=lnms&sa=X&ved=2ahUKEwicke6y37OAAxWJGIgKHQWAASUQ0pQJegQIDRAB&biw=1366&bih=372&dpr=1'
        # self.url = f'https://www.google.com.hk/search?q={searchkw}&sca_esv=555819424&tbs=sbd:1&tbm=nws&ei=CezVZPaGCaqC4-EPqZi_oAk&start=90&sa=N&ved=2ahUKEwi2r_qGk9SAAxUqwTgGHSnMD5QQ8tMDegQIAhAU&biw=1366&bih=619&dpr=1'
        self.url = f'https://www.google.com.hk'
        self.r = redis.Redis(host=self.config.get('redis', 'host'),
                             port=self.config.get('redis', 'port'),
                             password=self.config.get('redis', 'pass'), db=0)
        self.page_num = 1
        chrome_driver =self.config.get('selenium', 'chrome_driver')
        self.kafka_bootstrap_servers = self.config.get('kafka', 'bootstrap_servers')
        path =  Service(chrome_driver)
        chrome_options = webdriver.ChromeOptions()
        chrome_options.binary_location =  self.config.get('selenium', 'binary_location')
        self.driver = webdriver.Chrome(service=path,chrome_options=chrome_options)
        # driver = webdriver.Chrome(chrome_options=chrome_options)
        self.qtitle = Queue()
        self.qurl = Queue()
        self.detailList = Queue()
        self.searchkw = searchkw
        self.wordsCode = wordsCode
        self.sid = sid

    def createDriver(self):
        chrome_driver =self.config.get('selenium', 'chrome_driver')
        path =  Service(chrome_driver)
        chrome_options = webdriver.ChromeOptions()
        chrome_options.binary_location =self.config.get('selenium', 'binary_location')
        # 设置代理
        # proxy = "127.0.0.1:8080"  # 代理地址和端口
        # chrome_options.add_argument('--proxy-server=http://' + proxy)
        self.driver = webdriver.Chrome(service=path,chrome_options=chrome_options)

    #将列表数据插入到表中 baidu_search_result
    def itemInsertToTable(self,items):
        itemdata=[]
        conx,cursorM=self.connMysql()
        for item in items:
            nowtime=self.getNowDate()
            data=(self.sid,self.wordsCode,item['title'],item['detailurl'],item['source'],item['publishtime'],item['content'],item['contentHtml'],'1',item['kword'],nowtime)
            itemdata.append(data)

        sql ="INSERT into google_search_result (sid,wordsCode,title,detailurl,origin,publishdate,content,content_with_tag,state,keyword,create_time) VALUES (%s, %s,%s, %s, %s, %s, %s, %s, %s, %s, %s)"
        cursorM.executemany(sql, itemdata)
        self.logger.info("数据插入数据库成功！")
        # 定义插入数据的SQL语句
        # 执行插入操作
        conx.commit()
        self.closeSql(conx,cursorM)

    def connMysql(self):
        # 创建MySQL连接
        conx = pymysql.connect(host=self.config.get('mysql', 'host'),
                               user=self.config.get('mysql', 'username'),
                               password=self.config.get('mysql', 'password'),
                               database=self.config.get('mysql', 'database'))
        # 创建一个游标对象
        cursorM = conx.cursor()
        return conx,cursorM

    def closeSql(self,conx,cursorM):
        # 关闭游标和连接
        cursorM.close()
        conx.close()

        # 解析页面
    def parse_page(self):
        try:
            response = self.driver.page_source
            html = etree.HTML(response)
            lists=self.xpath_paser(html)
            flag = html.xpath('//tr[@jsname="TeSSVd"]//a[last()]//@class')[0]
        except:
            lists=[]
            flag=''
        return flag, lists

    def xpath_paser(self,html):
        lists=[]
        itemTag=html.xpath('//div[@class="SoaBEf"]')
        for itemTag in itemTag:
            try:
                title=itemTag.xpath('.//div[@class="n0jPhd ynAwRc MBeuO nDgy9d"]/text()')[0]
                title=str(title)
            except Exception as e:
                title=''
            try:
                detailUrl=itemTag.xpath('.//a[@class="WlydOe"]/@href')[0]
                detailUrl=str(detailUrl)
            except Exception as e:
                detailUrl=''
            try:
                sourceTag=itemTag.xpath('.//div[@class="MgUUmf NUnG9d"]//text()')[0]
                sourceTag=str(sourceTag)
            except Exception as e:
                print(e)
                sourceTag=''
            try:
                publishTag=itemTag.xpath('.//div[@class="OSrXXb rbYSKb LfVVr"]/span/text()')[0]
                publishTag=str(publishTag)
                publishtime=self.paserTime(publishTag)
                publishTag=publishtime.strftime("%Y-%m-%d %H:%M:%S")
            except Exception as e:
                publishTag=''
            detailmsg={
                'title':title,
                'detailUrl':detailUrl,
                'sourceTag':sourceTag,
                'publishTag':publishTag
            }
            lists.append(detailmsg)
        return lists

        #获取当前时间

    def getNowDate(self):
        # 获取当前时间
        current_time = datetime.datetime.now()
        # 将时间转换为字符串
        currentdate = current_time.strftime("%Y-%m-%d %H:%M:%S")
        return currentdate

    def webDriver(self,url):
        chrome_driver =self.config.get('selenium', 'chrome_driver')
        path = Service(chrome_driver)
        chrome_options = webdriver.ChromeOptions()
        chrome_options.binary_location =self.config.get('selenium', 'binary_location')
        driver = webdriver.Chrome(service=path,chrome_options=chrome_options)
        html=''
        try:
            driver.get(url)
            # 等待页面加载完成
            wait = WebDriverWait(driver, 20)
            wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
            html=driver.page_source
        except Exception as e:
            self.logger.info('请求失败')
        finally:
            # 关闭driver
            driver.quit()

        return html

    def extractorMsg(self,url,title):
        content=''
        contentWithTag=''
        lang=''
        try:
            lang=self.detect_language(title)
            sm=SmartExtractor(lang)
            article=sm.extract_by_url(url=url)
            content=article.cleaned_text
            contentWithTag=article.text
        except Exception as e:
            try:
                raw_html=self.webDriver(url)
                sm=SmartExtractor(lang)
                article=sm.extract_by_html(raw_html)
                content=article.cleaned_text
                contentWithTag=article.text
            except Exception as e:
                print('抽取失败！！')

        return content,contentWithTag

    def paserTime(self,publishtime):
        timeType=['年前','月前','周前','前天','昨天','天前','今天','小时前','分钟前']
        current_datetime = datetime.datetime.now()
        publishtime=publishtime.strip()
        print(publishtime)
        try:
            if '年前' in publishtime:
                numbers = re.findall(r'\d+', publishtime)
                day=int(numbers[0])
                delta = datetime.timedelta(days=365 * day)
                publishtime = current_datetime - delta
            elif '月前' in publishtime:
                numbers = re.findall(r'\d+', publishtime)
                day=int(numbers[0])
                delta = datetime.timedelta(days= 30*day)
                publishtime = current_datetime - delta
            elif '周前' in publishtime:
                numbers = re.findall(r'\d+', publishtime)
                day=int(numbers[0])
                delta = datetime.timedelta(weeks= day)
                publishtime = current_datetime - delta
            elif '天前' in publishtime:
                numbers = re.findall(r'\d+', publishtime)
                day=int(numbers[0])
                delta = datetime.timedelta(days= day)
                publishtime = current_datetime - delta
            elif '前天' in publishtime:
                delta = datetime.timedelta(days= 2)
                publishtime = current_datetime - delta
            elif '昨天' in publishtime:
                current_datetime = datetime.datetime.now()
                delta = datetime.timedelta(days= 1)
                publishtime = current_datetime - delta
            elif '今天' in publishtime or'小时前' in publishtime or '分钟前' in publishtime :
                delta = datetime.timedelta(hours= 5)
                publishtime = current_datetime - delta
            elif '年' in publishtime and '月' in publishtime :
                time_format = '%Y年%m月%d日'
                publishtime = datetime.datetime.strptime(publishtime, time_format)
            elif '月' in publishtime and '日' in publishtime :
                current_year = current_datetime.year
                time_format = '%Y年%m月%d日'
                publishtime=str(current_year)+'年'+publishtime
                publishtime = datetime.datetime.strptime(publishtime, time_format)
        except Exception as e:
            print('时间解析异常！！')
        return publishtime


    # 获取每一页数据, 开趴.
    def get_page_html(self):
        self.logger.info("进入google首页..."  )
        self.driver.get(self.url)
        # 等待页面加载完成
        time.sleep(3)
        wait = WebDriverWait(self.driver, 20)
        wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
        search_input = self.driver.find_element('xpath', '//textarea[@title="Google 搜索"]')
        search_input.send_keys(self.searchkw)
        search_input.submit()
        try:
            time.sleep(3)
            wait = WebDriverWait(self.driver, 20)
            wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
            self.driver.find_element('xpath', '//div[@class="GKS7s"]/span[text()="新闻"]').click()
            time.sleep(3)
            self.driver.find_element('xpath', '//div[@id="hdtb-tls"]').click()
            time.sleep(2)
            self.driver.find_element('xpath', '//div[@class="hdtb-mn-hd"]/div[text()="按相关性排序"]').click()
            time.sleep(2)
            self.driver.find_element('xpath', '//div[@class="YpcDnf OSrXXb HG1dvd"]/a[text()="按日期排序"]').click()
        except Exception as e:
            print(e)
        self.logger.info("开始抓取首页..." + self.searchkw )
        time.sleep(5)
        flag, lists = self.parse_page()
        if len(lists):
            time.sleep(600)
        for detail in lists:
            durl=detail['detailUrl']
            is_member = self.r.sismember('pygoogle_'+self.wordsCode, durl)
            if is_member:
                continue
            self.detailList.put(detail)

        response = self.driver.page_source
        html = etree.HTML(response)
        hasnext = html.xpath('//table[@class="AaVjTc"]//td[last()]//text()')[0]
        hasnext = hasnext.strip()
        timeFlag = False
        while hasnext == '下一页':
            if self.page_num==29:
                break
            self.page_num = self.page_num + 1
            self.logger.info("开始抓取第%s页..." % self.page_num)
            try:
                self.driver.find_element(By.XPATH, '//a[@id="pnnext"]').click()
            except Exception as e:
                break
            time.sleep(5)
            flag, lists = self.parse_page()
            for detail in lists:
                is_member = self.r.sismember('pygoogle_'+self.wordsCode, durl)
                if is_member:
                    continue
                publishTag=detail['publishTag']
                if publishTag:
                    pubtime = datetime.datetime.strptime(publishTag, "%Y-%m-%d %H:%M:%S")
                    needDate='2022-01-01 00:00:00'
                    needTime = datetime.datetime.strptime(needDate, "%Y-%m-%d %H:%M:%S")
                    if pubtime < needTime:
                        timeFlag = True
                        break
                self.detailList.put(detail)
            if timeFlag:
                break
            try:
                response = self.driver.page_source
                html = etree.HTML(response)
                hasnext = html.xpath('//a[@id="pnnext"]//text()')[0]
                hasnext = hasnext.strip()
                self.logger.info(hasnext)
            except Exception as e:
                hasnext=''
        self.logger.info("抓取完毕")

    def getRequest(self,url):
        html=''
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3',
        }
        try:
            print(url)
            res=requests.get(url=url,timeout=30)
            if res.status_code==200:
                res.encoding = res.apparent_encoding  # 使用自动检测的编码方式
                html=res.text
            else:
                html=''
            if html=='':
                for i in range(1,3):
                    time.sleep(1)
                    html=self.getRequest(url)
        except Exception as e:
            print(e)

        return html

    # 获取详情页
    def get_detail_html(self):
        while True:
            if self.detailList.qsize() != 0:
                try:
                    detailmsg=self.detailList.get()
                    title = detailmsg['title']
                    detailUrl = detailmsg['detailUrl']
                    self.logger.info("%s:%s\n" % (title, detailUrl))
                    # try:
                    #     self.driver.get(detailUrl)
                    # except Exception as e:
                    #     self.driver.quit()
                    #     self.driver=self.createDriver()
                    #     self.driver.get(detailUrl)
                    # response = self.driver.page_source
                    try:
                        bdetail=self.getDetailmsg(detailmsg)
                        # 'content':content,
                        # 'contentHtml':contentWithTag,
                        content=bdetail['content']
                        contentHtml=bdetail['contentHtml']
                        if len(content)<100:
                            continue
                        soup = BeautifulSoup(contentHtml, "html.parser")
                        # 查找所有带有class属性的元素
                        elements_with_class = soup.find_all(class_=True)
                        # 循环遍历元素并去掉class属性
                        for element in elements_with_class:
                            del element.attrs["class"]
                        contentHtml=str(soup)
                        bdetail['content']=content
                        bdetail['contentHtml']=contentHtml
                    except Exception as e:
                        self.logger.info('详情解析失败')
                        continue

                    processitem=self.getProcessitem(bdetail)
                    try:
                        self.sendkafka(processitem)
                        self.r.sadd('pygoogle_'+self.wordsCode, processitem['sourceAddress'])
                    except Exception as e:
                        self.logger.info("放入kafka失败！")
                    #插入数据库
                    try:
                        items=[]
                        items.append(bdetail)
                        self.itemInsertToTable(items)
                    except Exception as e:
                        self.logger.info(f"插入数据库失败==={e}")
                    # 关闭当前新窗口
                    # self.driver.close()
                    time.sleep(1)
                except Exception as e:
                    time.sleep(5)
                    self.logger.info("详情页解析异常！"+detailUrl)
            else:
                break
                # time.sleep(5)

    def rmTagattr(self,html,url):
        # 使用BeautifulSoup解析网页内容
        # soup = BeautifulSoup(html, 'html.parser')
        soup = self.paserUrl(html,url)
        # 遍历所有标签，并去掉属性
        for tag in soup.find_all(True):
            if tag.name == 'img':
                tag.attrs = {key: value for key, value in tag.attrs.items() if key == 'src'}
            elif tag.name !='img':
                tag.attrs = {key: value for key, value in tag.attrs.items() if key == 'src'}
            else:
                tag.attrs = {key: value for key, value in tag.attrs.items()}
        # 打印去掉属性后的网页内容
        # print(soup.prettify())
        html=soup.prettify()
        return html

    # 将html中的相对地址转换成绝对地址
    def paserUrl(self,html,listurl):
        soup = BeautifulSoup(html, 'html.parser')
        # 获取所有的<a>标签和<img>标签
        links = soup.find_all(['a', 'img'])
        # 遍历标签，将相对地址转换为绝对地址
        for link in links:
            if 'href' in link.attrs:
                link['href'] = urljoin(listurl, link['href'])
            elif 'src' in link.attrs:
                link['src'] = urljoin(listurl, link['src'])

        return soup

    #获取资讯内容信息
    def getDetailmsg(self,detailmsg):
        try:
            detailurl=detailmsg['detailUrl']
            title = detailmsg['title']
            content,contentWithTag=self.extractorMsg(detailurl,title)
            contentWithTag=self.rmTagattr(contentWithTag)
        except Exception as e:
            content=''
            contentWithTag=''

        currentdate=self.getNowDate()
        kword=self.searchkw
        publishDate=detailmsg['publishTag']
        publishDate=publishDate+''
        # publishtime=self.paserTime(publishtime)
        # publishDate=publishtime.strftime("%Y-%m-%d %H:%M:%S")
        detailmsg={
            'title':detailmsg['title'],
            'source':detailmsg['sourceTag'],
            'detailurl':detailurl,
            'content':content,
            'contentHtml':contentWithTag,
            'publishtime':publishDate,
            'currentdate':currentdate,
            'kword':kword
        }
        return detailmsg

    def getProcessitem(self,bdetail):
        nowDate=self.getNowDate()
        content=bdetail['content']
        if content!='':
            processitem={
                "sid":self.sid,
                "source":"4",
                "title":bdetail['title'],
                "content":bdetail['content'],
                "contentWithtag":bdetail['contentHtml'],
                "origin":bdetail['source'],
                "publishDate":bdetail['publishtime'],
                "sourceAddress":bdetail['detailurl'],
                "createDate":nowDate
            }

        return processitem

    def sendkafka(self,processitem):
        try:
            producer = KafkaProducer(bootstrap_servers=[self.kafka_bootstrap_servers])
            content=processitem['content']
            publishDate=str(processitem['publishDate'])
            title=processitem['title']
            if title =='':
                return
            if content=='':
                return
            if publishDate=='':
                return
            kafka_result = producer.send("crawlerInfo", json.dumps(processitem, ensure_ascii=False).encode('utf8'))
            self.logger.info("数据发送kafka成功")
            self.logger.info(kafka_result.get(timeout=10))
        except Exception as e:
            self.logger.info('发送kafka异常')
        finally:
            producer.close()

    def run(self):
        # 获取每页URL
        c = threading.Thread(target=self.get_page_html)
        c.start()
        c.join()
        # 解析详情页
        t = threading.Thread(target=self.get_detail_html)
        t.start()

    def detect_language(self,html):
        soup = BeautifulSoup(html, "html.parser")
        text = soup.get_text()
        # 使用langid.py判断文本的语言
        lang, confidence = langid.classify(text)
        return lang

if __name__ == '__main__':
    searchkw='kw'
    wordsCode='wordsCode'
    sid='sid'
    zhuce=GoogleSpider(searchkw,wordsCode,sid)
    # zhuce.run()
    url='https://vostok.today/46962-fesco-i-rzhd-rasshirjat-propusknuju-sposobnost-vladivostokskogo-morskogo-torgovogo-porta.html'
    zhuce.driver.get(url)
    time.sleep(20)
    html=zhuce.driver.page_source
    print(html)
    lang=zhuce.detect_language(html)
    print(lang)
    print('++++++++++++++++++')

    sm=SmartExtractor(lang)
    article=sm.extract_by_html(html)
    # article=sm.extract_by_url(url)
    content=article.cleaned_text
    text=article.text
    print(content)
    print(text)
    # raw_html = article.raw_html
    # html=zhuce.getRequest(url)
    # article_content=zhuce.extract_article(html,url)
    # print(article_content)


