#补充剩余核心人员信息
#先采集天眼查id，再通过id采集核心人员信息
import datetime
import json
import os
import subprocess
import sys

import requests,time,random
import pandas as pd
from bs4 import BeautifulSoup
import urllib3
from retry import retry

from base.BaseCore import BaseCore
from getTycId import getTycIdByXYDM
baseCore = BaseCore()
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
log = baseCore.getLogger()
headers = {
    'Cookie': 'HWWAFSESID=38a70202d86311cd90f; HWWAFSESTIME=1706662296323; jsid=SEO-BING-ALL-SY-000001; TYCID=e35f3910bfd211eeac66555a29ade465; ssuid=6800091776; sajssdk_2015_cross_new_user=1; csrfToken=e85dxv9-DXNUkQ7yuzIgZrbs; bannerFlag=true; Hm_lvt_e92c8d65d92d534b0fc290df538b4758=1706662300; _ga=GA1.2.1071312772.1706662301; _gid=GA1.2.1602571847.1706662301; tyc-user-info={%22state%22:%220%22%2C%22vipManager%22:%220%22%2C%22mobile%22:%2217103126138%22%2C%22userId%22:%22304029617%22}; tyc-user-info-save-time=1706662339304; auth_token=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiIxNzEwMzEyNjEzOCIsImlhdCI6MTcwNjY2MjMzOCwiZXhwIjoxNzA5MjU0MzM4fQ.z9cOzr0YWyU_rxTZNn8ojsxfMAdre4NbQLzwgKAGdI-CCcfPvuBBrL4tFP5HmR5pDv204e4P4k4Ll4kKPhBQTg; tyc-user-phone=%255B%252217103126138%2522%255D; searchSessionId=1706667106.29658260; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22304029617%22%2C%22first_id%22%3A%2218d5d0009e8153-01c79a4d65a09f9-4c657b58-921600-18d5d0009e914e%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22identities%22%3A%22eyIkaWRlbnRpdHlfY29va2llX2lkIjoiMThkNWQwMDA5ZTgxNTMtMDFjNzlhNGQ2NWEwOWY5LTRjNjU3YjU4LTkyMTYwMC0xOGQ1ZDAwMDllOTE0ZSIsIiRpZGVudGl0eV9sb2dpbl9pZCI6IjMwNDAyOTYxNyJ9%22%2C%22history_login_id%22%3A%7B%22name%22%3A%22%24identity_login_id%22%2C%22value%22%3A%22304029617%22%7D%2C%22%24device_id%22%3A%2218d5d0009e8153-01c79a4d65a09f9-4c657b58-921600-18d5d0009e914e%22%7D; Hm_lpvt_e92c8d65d92d534b0fc290df538b4758=1706667529',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
}
cnx_ = baseCore.cnx
cursor_ = baseCore.cursor

cnx = baseCore.cnx_
cursor = baseCore.cursor_

list_all_1 = []
list_all_2 = []
taskType = '天眼查/核心人员'
ip_num = 0

def get_proxy(ip_num):
    sql = "select proxy from clb_proxy"
    cursor_.execute(sql)
    proxy_lists = cursor_.fetchall()
    cnx_.commit()
    ip_list = []
    for proxy_ in proxy_lists:
        ip_list.append(str(proxy_).replace("('", '').replace("',)", ''))
    proxy_list = []
    for str_ip in ip_list:
        str_ip_list = str_ip.split('-')
        proxyMeta = "http://%(host)s:%(port)s" % {
            "host": str_ip_list[0],
            "port": str_ip_list[1],
        }
        proxy = {
            "http": proxyMeta,
            "https": proxyMeta
        }
        proxy_list.append(proxy)
    return proxy_list[ip_num]

@retry(tries=3, delay=1)
def get_html(tycid, ip_num):
    url = f"https://www.tianyancha.com/company/{tycid}"
    ip = get_proxy(ip_num)
    response = requests.get(url=url, headers=headers, proxies=ip)
    if response.status_code == 200:
        pass
    else:
        ip_num += 1
        raise
        # return -1
    soup = BeautifulSoup(response.content, 'html.parser')
    try:
        tmp_field = soup.find('div', class_='dim-tab-root').find('span').text
        if '最新公示' in tmp_field:
            total = soup.find('div', class_='dim-tab-root').find('span').get_text().split('最新公示')[1].replace(' ', '')
            return int(total)
        else:
            return 0
    except:
        return 0

@retry(tries=3, delay=1)
def get_page(url, ip_num):
    ip = get_proxy(ip_num)
    res = requests.get(url=url, headers=headers, proxies=ip)
    if res.status_code == 200:
        pass
    else:
        ip_num += 1
        raise
    time.sleep(1)
    total_page_ = res.json()['data']['total']
    return total_page_

def doJob():
    while True:
        # 根据从Redis中拿到的社会信用代码,在数据库中获取对应基本信息
        # social_code = baseCore.redicPullData('CorPersonEnterprise:gnqy_socialCode')
        # 判断 如果Redis中已经没有数据，则等待
        social_code = '91320691550279691N'
        if social_code == None:
            time.sleep(20)
            continue
        start = time.time()
        try:
            data = baseCore.getInfomation(social_code)
            if len(data) != 0:
                id = data[0]
                com_name = data[1]
                xydm = data[2]
                tycid = data[11]
                count = data[17]
            else:
                #数据重新塞入redis
                # log.info(f'数据库中无该企业{social_code}')
                sql = f"SELECT * FROM sys_base_enterprise WHERE social_credit_code = '{social_code}'"
                cursor.execute(sql)
                data = cursor.fetchone()
                id = data[0]
                com_name = data[3]
                xydm = data[1]
                conut = 0
                # 写入数据库
                insert = "INSERT INTO EnterpriseInfo(com_name, xydm, social_credit_code) VALUES (%s, %s, %s)"
                cursor_.execute(insert, (com_name, xydm, social_code))
                cnx_.commit()
                tycid = ''
                # baseCore.rePutIntoR('CorPersonEnterpriseNone:gnqy_socialCode', social_code)
                # continue

            if tycid == None or tycid == '':
                try:
                    retData = getTycIdByXYDM(com_name)
                    if retData['state']:
                        tycid = retData['tycData']['id']
                        # # todo:写入数据库
                        updateSql = f"update EnterpriseInfo set TYCID = '{tycid}' where SocialCode = '{xydm}'"
                        cursor_.execute(updateSql)
                        cnx_.commit()
                    else:
                        state = 0
                        takeTime = baseCore.getTimeCost(start, time.time())
                        baseCore.recordLog(social_code, taskType, state, takeTime, '', '获取天眼查id失败')
                        log.info(f'======={social_code}====重新放入redis====')
                        baseCore.rePutIntoR('CorPersonEnterpriseError:gnqy_socialCode', social_code)
                        continue
                except:
                    state = 0
                    takeTime = baseCore.getTimeCost(start, time.time())
                    baseCore.recordLog(social_code, taskType, state, takeTime, '', '获取天眼查id失败')
                    baseCore.rePutIntoR('CorPersonEnterpriseError:gnqy_socialCode', social_code)
                    continue

            log.info(f"{id}---{xydm}----{tycid}----开始采集核心人员")
            list_one_info = []
            num = 1

            #todo:先确定接口走哪个
            try:
                charge = get_html(tycid, ip_num)
            except Exception as e:
                charge = -1
                log.info(e)
            total_page = 0
            t = int(time.time() * 1000)
            if charge == -1:
                # 重新塞入redis
                baseCore.rePutIntoR('CorPersonEnterpriseError:gnqy_socialCode', social_code)
                log.info(f'==={social_code}=====页面请求失败===重新放入redis====')
                log.info(f"{id}---{xydm}----{tycid}----请求失败")
                # 获取当前进程pid
                current_pid = baseCore.getPID()
                # todo: 重新启动新进程，杀死当前进程
                subprocess.Popen([sys.executable] + sys.argv)
                os.kill(current_pid, 9)
                continue
            elif charge == 0:
                log.info(f"{id}---{xydm}----{tycid}----没有最新公示")
                url1 = f'https://capi.tianyancha.com/cloud-company-background/company/dim/staff?_={t}&gid={tycid}&pageSize=20&pageNum=1'
                try:
                    total_page1 = get_page(url1, ip_num)
                except:
                    total_page1 = 0
                url = 'https://capi.tianyancha.com/cloud-company-background/company/dim/staff?_={}&gid={}&pageSize=20&pageNum={}'
                total_page = total_page1
                flag = 2
            else:
                log.info(f"{id}---{xydm}----{tycid}----有最新公示")
                url2 = f'https://capi.tianyancha.com/cloud-listed-company/listed/noRepeatSeniorExecutive?_={t}&gid={tycid}&pageSize=20&pageNum=1'
                url3 = f'https://capi.tianyancha.com/cloud-listed-company/listed/getHkNoRepeatSeniorExecutive?_={t}&gid={tycid}&pageSize=20&pageNum=1'
                try:
                    total_page2 = get_page(url2, ip_num)
                except:
                    total_page2 = 0
                time.sleep(2)
                try:
                    total_page3 = get_page(url3, ip_num)
                except:
                    total_page3 = 0
                if total_page2 == charge:
                    url = 'https://capi.tianyancha.com/cloud-listed-company/listed/noRepeatSeniorExecutive?_={}&gid={}&pageSize=20&pageNum={}'
                    total_page = total_page2
                    flag = 1
                else:
                    if total_page3 == charge:
                        url = 'https://capi.tianyancha.com/cloud-listed-company/listed/getHkNoRepeatSeniorExecutive?_={}&gid={}&pageSize=20&pageNum={}'
                        total_page = total_page3
                        flag = 3
                    else:
                        total_page = 0
                        flag = 0
                        log.info(f'{id}---{xydm}----{tycid}----没有高管信息')
                        continue
            if total_page == 0:
                # 重新塞入redis
                baseCore.rePutIntoR('CorPersonEnterpriseError:gnqy_socialCode', social_code)
                log.info(f'==={social_code}=====总数请求失败===重新放入redis====')
                continue
            #todo:获取页数
            time.sleep(2)
            for page in range(1, int((total_page/20) + 1)+1):
                for c in range(3):
                    ip = baseCore.get_proxy()
                    url_ = url.format(t, tycid, page)
                    res = requests.get(url_, headers=headers, proxies=ip)  # ,verify=False
                    time.sleep(1)
                    if res.status_code == 200:
                        break
                    else:
                        if c == 2:
                            res = ''
                            break
                        continue
                if res:
                    pass
                else:
                    # 重新塞入redis
                    baseCore.rePutIntoR('CorPersonEnterpriseError:gnqy_socialCode', social_code)
                    log.info(f'{id}---{xydm}----{tycid}----高管信息请求失败')
                    continue
                try:
                    list_all = res.json()['data']['dataList']
                except:
                    list_all = res.json()['data']['result']
                if list_all:
                    pass
                else:
                    log.info(f'{id}---{xydm}----{tycid}----没有高管信息')
                if flag == 1:
                    for one_info in list_all:
                        name = one_info['name']
                        sex = one_info['sex']
                        education = one_info['education']
                        position = one_info['position']
                        Salary = one_info['salary']
                        #todo:获取当前年份
                        now = datetime.datetime.now()
                        year = now.year
                        try:
                            birthYear = year - int(one_info['age'])
                        except:
                            birthYear = ''
                        StockKeepings = one_info['numberOfShares']
                        currentTerm = one_info['term']
                        personInfo = one_info['resume']

                        try:
                            person_img = one_info['logo']
                        except:
                            person_img = '--'
                        dic_json = {
                            "socialCreditCode":social_code,
                            "name":name,
                            "sex":sex,
                            "education":education,
                            "position":position,
                            "salary":Salary,
                            "birthYear":birthYear,
                            "shareNum":StockKeepings,
                            "shareRatio":'',
                            "benefitShare":'',
                            "currentTerm":currentTerm,
                            "personInfo":personInfo,
                            "sort":str(num)
                        }
                        dic_json_img = {
                            "socialCreditCode":social_code,
                            "name":name,
                            "sex":sex,
                            "education":education,
                            "position":position,
                            "salary":Salary,
                            "birthYear":birthYear,
                            "shareNum":StockKeepings,
                            "shareRatio":'',
                            "benefitShare":'',
                            "currentTerm":currentTerm,
                            "personInfo":personInfo,
                            "头像":person_img,
                            "sort":str(num)
                        }
                        num = num+1
                        list_one_info.append(dic_json)
                        # list_all_2.append(dic_json_img)
                elif flag == 3:
                    for one_info in list_all:
                        name = one_info['personal_name']
                        try:
                            sex = one_info['gender2']
                        except:
                            sex = ''
                        education = ''
                        position = one_info['position_name']
                        Salary = ''
                        try:
                            birthYear = one_info['year_of_birth']
                        except:
                            birthYear = ''
                        personInfo = one_info['resume_cn']
                        timestamp = int(int(one_info['employ_date'])/10000)
                        currentTerm = time.strftime("%Y-%m-%d", time.localtime(timestamp))
                        dic_json = {
                            "socialCreditCode": social_code,
                            "name": name,
                            "sex": sex,
                            "education": education,
                            "position": position,
                            "salary": Salary,
                            "birthYear": birthYear,
                            "shareNum": '',
                            "shareRatio": '',
                            "benefitShare": '',
                            "currentTerm": currentTerm+'至-',
                            "personInfo": personInfo,
                            "sort": str(num)
                        }
                        num = num + 1
                        list_one_info.append(dic_json)
                else:
                    for one_info in list_all:
                        name = one_info['name']
                        try:
                            position = one_info['typeSore']
                        except:
                            position = ''

                        person_id = one_info['id']
                        person_url = f'https://www.tianyancha.com/human/{person_id}-c{tycid}'
                        # person_res = requests.get(person_url, headers=headers, proxies=ip)
                        person_res = requests.get(person_url, headers=headers)
                        person_soup = BeautifulSoup(person_res.content, 'html.parser')
                        try:
                            personInfo = person_soup.find('span', {'class': '_56d0a'}).text.strip()
                        except:
                            personInfo = ''
                        try:
                            person_img = one_info['logo']
                        except:
                            person_img = '--'
                        dic_json = {
                            "socialCreditCode": social_code,
                            "name": name,
                            "sex": '',
                            "education": '',
                            "position": position,
                            "salary": '',
                            "birthYear": '',
                            "shareNum": '',
                            "shareRatio": '',
                            "benefitShare": '',
                            "currentTerm": '',
                            "personInfo": personInfo,
                            "sort": str(num)
                        }
                        dic_json_img = {
                            "socialCreditCode": social_code,
                            "name": name,
                            "sex": '',
                            "education": '',
                            "position": position,
                            "salary": '',
                            "birthYear": '',
                            "shareNum": '',
                            "shareRatio": '',
                            "benefitShare": '',
                            "currentTerm": '',
                            "personInfo": personInfo,
                            "头像": person_img,
                            "sort": str(num)
                        }
                        num = num + 1
                        list_one_info.append(dic_json)
            # print(list_one_info)
            json_updata = json.dumps(list_one_info)
            if json_updata == '[]':
                continue
            else:
                pass
            response = requests.post('http://114.115.236.206:8088/sync/executive',data=json_updata,timeout=300, verify=False)
            print(response.text)
            log.info('=========成功======')
        except Exception as e:
            log.info(f'==={social_code}=====企业核心人员采集失败===重新放入redis====')
            log.info(e)
            # 重新塞入redis
            baseCore.rePutIntoR('CorPersonEnterpriseError:gnqy_socialCode', social_code)
            state = 0
            takeTime = baseCore.getTimeCost(start, time.time())
            baseCore.recordLog(social_code, taskType, state, takeTime, '', f'获取企业信息失败--{e}')
            time.sleep(5)
        # break
# df_img = pd.DataFrame(list_all_2)
# df_img.to_excel('企业主要人员-头像.xlsx',index=False)
if __name__ == "__main__":
    doJob()