#企业动态 从redis中获取数据
import json
import os
import random
import subprocess

import requests, time, pymysql
import jieba
import sys

from bs4 import BeautifulSoup
from kafka import KafkaProducer
from getTycId import getTycIdByXYDM
# from base.BaseCore import BaseCore
# from base.smart import smart_extractor
sys.path.append('D:\\kkwork\\zzsn_spider\\base')
import BaseCore
from smart import smart_extractor
import urllib3

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
jieba.cut("必须加载jieba")
# 初始化，设置中文分词
smart =smart_extractor.SmartExtractor('cn')
baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
cnx = pymysql.connect(host='114.116.44.11', user='caiji', password='f7s0&7qqtK', db='dbScore', charset='utf8mb4')
cursor = cnx.cursor()
pageSize = 10
log.info(f'======================当前脚本进程为{baseCore.getPID()}==============================')

headers = {
        'Accept': 'application/json, text/plain, */*',
        'Accept-Encoding': 'gzip, deflate, br',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Content-Type': 'application/json',
        'Host': 'capi.tianyancha.com',
        'Origin': 'https://www.tianyancha.com',
        'Referer': 'https://www.tianyancha.com/',
        'Sec-Fetch-Dest': 'empty',
        'Sec-Fetch-Mode': 'cors',
        'Sec-Fetch-Site': 'same-site',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36',
        'X-AUTH-TOKEN': 'eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiIxODcwMzc1MjYwMCIsImlhdCI6MTY5OTkyNTk5NywiZXhwIjoxNzAyNTE3OTk3fQ.9iXmxFEiBdu2WYa7RwdU0xKKx7v_wBe9-QipH0TNKp9Dzk_2cZK1ESsmO1o8ICrddb5sx2cl5pjOBoaaf_9Qsg',
        'X-TYCID': '6f6298905d3011ee96146793e725899d',
        'sec-ch-ua': '"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"',
        'sec-ch-ua-mobile': '?0',
        'sec-ch-ua-platform': '"Windows"',
        'version': 'TYC-Web'
        }
cnx_ = baseCore.cnx
cursor_ = baseCore.cursor

taskType = '企业动态/天眼查/'

def reqDetailmsg(url,headers):

    # proxy = {'https': 'http://127.0.0.1:1080', 'http': 'http://127.0.0.1:1080'}
    for i in range(0,1):
        try:
            response=requests.get(url=url,headers=headers,timeout=8,verify=False)
            response.encoding = response.apparent_encoding
            htmltext=response.text
        except Exception as e:
            htmltext=''
            log.info(f'{url}---详情请求失败--{e}')
        if htmltext:
            log.info(f'{url}---详情请求成功')
            break
    return htmltext

def beinWork(tyc_code, social_code,start_time):

    time.sleep(3)
    # retData={'up_state':False,'total':0,'up_okCount':0,'up_errorCount':0,'up_repetCount':0}
    retData = {'total': 0, 'up_okCount': 0, 'up_errorCount': 0, 'up_repetCount': 0}
    t = time.time()
    url = f'https://capi.tianyancha.com/cloud-yq-news/company/detail/publicmsg/news/webSimple?_={t}&id={tyc_code}&ps={pageSize}&pn=1&emotion=-100&event=-100'
    try:
        for m in range(0,3):
            ip = baseCore.get_proxy()
            headers['User-Agent'] = baseCore.getRandomUserAgent()
            response = requests.get(url=url, headers=headers, verify=False)
            time.sleep(random.randint(3, 5))
            break
        if (response.status_code == 200):
            pass
    except Exception as e:
        #todo:重新放入redis中
        baseCore.rePutIntoR('NewsResend:newsInfo',social_code)
        log.error(f"{tyc_code}-----获取总数接口失败")
        error = '获取总数接口失败'
        state = 0
        takeTime = baseCore.getTimeCost(start_time, time.time())
        baseCore.recordLog(social_code, taskType, state, takeTime, url, f'{error}----{e}')
        #获取当前进程pid
        current_pid = baseCore.getPID()
        #todo: 重新启动新进程，杀死当前进程
        subprocess.Popen([sys.executable] + sys.argv)
        os.kill(current_pid,9)
        return retData
    try:
        json_1 = json.loads(response.content.decode('utf-8'))
        total = json_1['data']['total']
    except:
        log.error(f"{tyc_code}-----获取总数失败")
        e = '获取总数失败'
        state = 0
        takeTime = baseCore.getTimeCost(start_time, time.time())
        baseCore.recordLog(social_code, taskType, state, takeTime, url, e)
        return retData
    if (total > 0):
        if (total % pageSize == 0):
            totalPage = total // pageSize
        else:
            totalPage = total // pageSize + 1
    else:
        log.error(f"{tyc_code}--------总数为0")
        retData['state'] = True
        return retData
    log.info(f"{tyc_code}-------总数：{total}----总页数:{totalPage}")

    retData['total'] = total
    up_okCount = 0
    up_errorCount = 0
    up_repetCount = 0
    for num in range(1, 10):
        time.sleep(3)
        log.info(f"获取分页数据--{tyc_code}----分页{num}----开始")
        start_page = time.time()
        url_page = f'https://capi.tianyancha.com/cloud-yq-news/company/detail/publicmsg/news/webSimple?_={time.time()}&id={tyc_code}&ps={pageSize}&pn={num}&emotion=-100&event=-100'
        for m in range(0, 3):
            try:
                ip = baseCore.get_proxy()
                headers['User-Agent'] = baseCore.getRandomUserAgent()
                response_page = requests.get(url=url_page, headers=headers, verify=False)
                # time.sleep(3)
                break
            except:
                pass

        if (response_page.status_code == 200):
            pass
        else:
            log.error(f"{tyc_code}--{num}页---获取分页数据失败")
            e = '获取分页数据失败'
            state = 0
            takeTime = baseCore.getTimeCost(start_time, time.time())
            baseCore.recordLog(social_code, taskType, state, takeTime, url_page, e)
            up_errorCount = up_errorCount + pageSize
            continue
        try:
            json_page = json.loads(response_page.content.decode('utf-8'))
            info_list_page = json_page['data']['items']
        except:
            log.error(f"{tyc_code}--{num}页---获取分页数据失败")
            e = '获取分页数据失败'
            state = 0
            takeTime = baseCore.getTimeCost(start_time, time.time())
            baseCore.recordLog(social_code, taskType, state, takeTime, url_page, e)
            up_errorCount = up_errorCount + pageSize
            continue
        pageIndex = 0
        for info_page in info_list_page:
            pageIndex = pageIndex + 1
            title = info_page['title']
            source = info_page['website']
            link = info_page['uri']
            try:
                time_struct = time.localtime(int(info_page['rtm'] / 1000))  # 首先把时间戳转换为结构化时间
                time_format = time.strftime("%Y-%m-%d %H:%M:%S", time_struct)  # 把结构化时间转换为格式化时间
                if time_format > '2023-12-10 00:00':
                    pass
                else:
                    retData['up_okCount'] = up_okCount
                    retData['up_errorCount'] = up_errorCount
                    retData['up_repetCount'] = up_repetCount
                    return retData
            except:
                time_format = baseCore.getNowTime(1)
            try:
                # 开始进行智能解析
                # lang = baseCore.detect_language(title)
                # smart = smart_extractor.SmartExtractor(lang)
                #带标签正文
                contentText = smart.extract_by_url(link).text
                #不带标签正文
                content = smart.extract_by_url(link).cleaned_text
                if len(content) < 300:
                    continue
                # time.sleep(3)
            except Exception as e:
                contentText = ''
            if contentText == '':
                log.error(f'获取正文失败：--------{tyc_code}--------{num}--------{link}')
                e = '获取正文失败'
                state = 0
                takeTime = baseCore.getTimeCost(start_time, time.time())
                baseCore.recordLog(social_code, taskType, state, takeTime, link, e)
                up_errorCount = up_errorCount + 1
                try:
                    insert_err_sql = f"insert into dt_err(xydm,`from`,url,title,pub_date,zhaiyao,create_date,state,pageNo,pageIndex) values('{social_code}','{source}','{link}','{title}','{time_format}','{info_page['abstracts']}',now(),1,{num},{pageIndex})"
                    cursor.execute(insert_err_sql)
                    cnx.commit()
                except:
                    pass
                continue
            try:
                insert_sql = '''insert into brpa_source_article_news(social_credit_code,source_address,origin,type,publish_time,content,create_time) values(%s,%s,%s,%s,%s,%s,now())'''
                # 动态信息列表
                up_okCount = up_okCount + 1
                list_info = [
                    social_code,
                    link,
                    f'天眼查-{source}',
                    '2',
                    time_format,
                    content[:500]
                ]
                cursor_.execute(insert_sql, tuple(list_info))
                cnx_.commit()
                # 采集一条资讯记录一条，记录该企业采到了多少的资讯
                log.info(f'{social_code}----{link}:新增一条')
                time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
                # todo:插入一条数据，并传入kafka
                dic_news = {
                    'attachmentIds': '',
                    'author': '',
                    'content': content,
                    'contentWithTag': contentText,
                    'createDate': time_now,
                    'deleteFlag': '0',
                    'id': '',
                    'keyWords': '',
                    'lang': 'zh',
                    'origin': source,
                    'publishDate': time_format,
                    # 'sid': '1684032033495392257',
                    'sid': '1714852232679067649',
                    'sourceAddress': link,  # 原文链接
                    'summary': info_page['abstracts'],
                    'title': title,
                    'type': 2,
                    'socialCreditCode': social_code,
                    'year': time_format[:4]
                }
            except Exception as e:
                log.info(f'传输失败:{social_code}----{link}')
                error = '数据库传输失败'
                state = 0
                takeTime = baseCore.getTimeCost(start_time, time.time())
                baseCore.recordLog(social_code, taskType, state, takeTime, link, f'{error}----{e}')
                continue
                # print(dic_news)
                # 将相应字段通过kafka传输保存
            try:
                producer = KafkaProducer(bootstrap_servers=['114.115.159.144:9092'])
                kafka_result = producer.send("researchReportDynamicTopic",
                                             json.dumps(dic_news, ensure_ascii=False).encode('utf8'))

                print(kafka_result.get(timeout=10))
                dic_result = {
                    'success': 'ture',
                    'message': '操作成功',
                    'code': '200',
                }
                log.info(dic_result)
                # 传输成功,写入日志中
                state = 1
                takeTime = baseCore.getTimeCost(start_time, time.time())
                baseCore.recordLog(social_code, taskType, state, takeTime, link, '')
                # return True
            except Exception as e:
                dic_result = {
                    'success': 'false',
                    'message': '操作失败',
                    'code': '204',
                    'e': e
                }
                log.error(dic_result)
                e = 'Kafka操作失败'
                state = 0
                takeTime = baseCore.getTimeCost(start_time, time.time())
                baseCore.recordLog(social_code, taskType, state, takeTime, link, e)

        log.info(f"获取分页数据--{tyc_code}----分页{num}，耗时{baseCore.getTimeCost(start_page, time.time())}")
    retData['up_okCount'] = up_okCount
    retData['up_errorCount'] = up_errorCount
    retData['up_repetCount'] = up_repetCount
    return retData

# 日志信息保存至现已创建好数据库中,因此并没有再对此前保存日志信息数据库进行保存
def doJob():
    while True:
        # 根据从Redis中拿到的社会信用代码,在数据库中获取对应基本信息
        social_code = baseCore.redicPullData('NewsResend:newsInfo')
        # social_code = '91320000733334390E'
        # 判断 如果Redis中已经没有数据，则等待
        if social_code == None:
            time.sleep(20)
            continue
        start = time.time()
        try:
            data = baseCore.getInfomation(social_code)
            if len(data) != 0:
                pass
            else:
                #数据重新塞入redis
                baseCore.rePutIntoR('NewsResend:newsInfo',social_code)
                continue
            id = data[0]
            com_name= data[1]
            xydm = data[2]
            tycid = data[11]
            if tycid == None or tycid == '':
                try:
                    retData = getTycIdByXYDM(com_name)
                    if retData['tycData'] and retData['reput']:
                        tycid = retData['tycData']['id']
                        # todo:写入数据库
                        updateSql = f"update EnterpriseInfo set TYCID = '{tycid}' where SocialCode = '{xydm}'"
                        cursor_.execute(updateSql)
                        cnx_.commit()
                    elif not retData['tycData'] and retData['reput']:
                        state = 0
                        takeTime = baseCore.getTimeCost(start, time.time())
                        baseCore.recordLog(social_code, taskType, state, takeTime, '', '获取天眼查id失败')
                        log.info(f'======={social_code}====重新放入redis====')
                        baseCore.rePutIntoR('NewsResend:newsInfo', social_code)
                        continue
                    elif not retData['reput'] and not retData['tycData']:
                        continue
                except:
                    state = 0
                    takeTime = baseCore.getTimeCost(start, time.time())
                    baseCore.recordLog(social_code, taskType, state, takeTime, '', '获取天眼查id失败')
                    baseCore.rePutIntoR('NewsResend:newsInfo', social_code)
                    continue
            count = data[17]
            log.info(f"{id}---{xydm}----{tycid}----开始处理")
            start_time = time.time()
            # 开始采集企业动态
            retData = beinWork(tycid, xydm,start_time)
            # 信息采集完成后将该企业的采集次数更新
            runType = 'NewsRunCount'
            count += 1
            baseCore.updateRun(social_code, runType, count)
            total = retData['total']
            up_okCount = retData['up_okCount']
            up_errorCount = retData['up_errorCount']
            up_repetCount = retData['up_repetCount']
            log.info(
                f"{id}---{xydm}----{tycid}----结束处理，耗时{baseCore.getTimeCost(start_time, time.time())}---总数:{total}---成功数:{up_okCount}----失败数:{up_errorCount}--重复数:{up_repetCount}")
        except Exception as e:
            log.info(f'==={social_code}=====获取企业信息失败====')
            #重新塞入redis
            baseCore.rePutIntoR('NewsResend:newsInfo',social_code)
            state = 0
            takeTime = baseCore.getTimeCost(start, time.time())
            baseCore.recordLog(social_code, taskType, state, takeTime, '', f'获取企业信息失败--{e}')
            time.sleep(5)

    cursor.close()
    cnx.close()
    # 释放资源
    baseCore.close()

if __name__ == '__main__':
    log.info(f'当前进程id为{baseCore.getPID()}')
    doJob()
