"""
es查询出的数据放到redis中，从redis中取出数据，
重新下载链接，更新附件表中的文件路径 更新es中的content字段
"""

import json
import os
import threading
import time
import uuid

import redis
import requests
from bs4 import BeautifulSoup
from obs import ObsClient
from retry import retry
from elasticsearch import Elasticsearch
from base import BaseCore

baseCore = BaseCore.BaseCore()
log = baseCore.getLogger()
baseCore = BaseCore.BaseCore()
import os
import subprocess
import uuid

import requests,time, json, sys
from retry import retry

from obs import ObsClient
import fitz
from urllib.parse import unquote
cnx_ = baseCore.cnx_
cursor_ = cnx_.cursor()

lock = threading.Lock()
obsClient = ObsClient(
        access_key_id='VEHN7D0TJ9316H8AHCAV',  # 你的华为云的ak码
        secret_access_key='heR353lvSWVPNU8pe2QxDtd8GDsO5L6PGH5eUoQY',  # 你的华为云的sk
        server='https://obs.cn-north-1.myhuaweicloud.com'  # 你的桶的地址
    )
es = Elasticsearch(['http://114.116.19.92:9700'], http_auth=('elastic', 'zzsn9988'), timeout=300)
index_name = 'researchreportdata'
taskType = '公告处理错误数据'
pathType = 'QYNotice/'

def updateaunn(index_name,id,content):
    body = {
        'doc': {
            'content': [str(content)]
        }
    }
    result = es.update(index=index_name
                            ,id=id
                            ,body=body)
    log.info('更新结果:%s' % result)

#获取文件大小
def convert_size(size_bytes):
    # 定义不同单位的转换值
    units = ['bytes', 'KB', 'MB', 'GB', 'TB']
    i = 0
    while size_bytes >= 1024 and i < len(units)-1:
        size_bytes /= 1024
        i += 1
    return f"{size_bytes:.2f} {units[i]}"

def getuuid():
    get_timestamp_uuid = uuid.uuid1()  # 根据 时间戳生成 uuid , 保证全球唯一
    return get_timestamp_uuid

def uptoOBS(pdf_url,type_id,social_code):
    headers = {}
    category = os.path.splitext(pdf_url)[1]
    retData = {'state': False, 'type_id': type_id, 'item_id': social_code, 'group_name': '', 'path': '',
               'full_path': '',
               'category': category, 'file_size': '', 'status': 1, 'create_by': 'XueLingKun',
               'create_time': '', 'page_size': '', 'content': ''}
    headers['User-Agent'] = baseCore.getRandomUserAgent()
    if category == '.pdf':
        try:
            response = requests.get(pdf_url, headers=headers, verify=False, timeout=20)
            if response.status_code != 200:
                return retData
            file_size = int(response.headers.get('Content-Length'))
            with fitz.open(stream=response.content, filetype='pdf') as doc:
                page_size = doc.page_count
                for page in doc.pages():
                    retData['content'] += page.get_text()
            # todo:判断内容是否成功
            if '<div class="K">403</div>' in retData['content'] or 'Error Times: ' in retData['content']:
                return retData
            else:
                pass
        except:
            log.error(f'文件损坏')
            return retData
    else:
        for i in range(0, 3):
            try:
                page_size = 1
                response = requests.get(pdf_url, headers=headers,verify=False, timeout=20)
                if response.status_code != 200:
                    return retData
                file_size = int(response.headers.get('Content-Length'))
                retData['content'] = response.text
                #todo:判断内容是否成功
                if '<div class="K">403</div>' in retData['content'] or 'Error Times: ' in retData['content']:
                    return retData
                else:
                    break
            except:
                time.sleep(3)
                continue
    name = str(getuuid()) + category
    try:
        result = getOBSres(pathType, name, response)
    except:
        log.error(f'OBS发送失败')
        return retData

    if page_size < 1:
        # pdf解析失败
        # print(f'======pdf解析失败=====')
        return retData
    else:
        try:
            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
            retData['state'] = True
            retData['path'] = unquote(result['body']['objectUrl'].split('.com')[1])
            retData['full_path'] = unquote(result['body']['objectUrl'])
            retData['file_size'] = convert_size(file_size)
            retData['create_time'] = time_now
            retData['page_size'] = page_size
        except Exception as e:
            state = 0
            takeTime = baseCore.getTimeCost(start_time, time.time())
            baseCore.recordLog(social_code, taskType, state, takeTime, pdf_url, f'{e}')
            return retData

        return retData

@retry(tries=3, delay=1)
def getOBSres(pathType,name, response):
    result = obsClient.putContent('zzsn', pathType + name, content=response.content)
    # resp = obsClient.putFile('zzsn', pathType + name, file_path='要上传的那个文件的本地路径')
    return result

def tableUpdate(retData,att_id):
    path = retData['path']
    full_path = retData['full_path']
    category = retData['category']
    file_size = retData['file_size']
    page_size = retData['page_size']
    try:
        # Upsql = '''insert into clb_sys_attachment(year,name,type_id,item_id,group_name,path,full_path,category,file_size,order_by,status,create_by,create_time,page_size,object_key,bucket_name) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'''
        Upsql = """update clb_sys_attachment set path= %s, full_path= %s, category=%s,file_size=%s,page_size=%s,object_key=%s,bucket_name=%s where id=%s"""
        values = (
            path, full_path, category, file_size, page_size,full_path.split('https://zzsn.obs.cn-north-1.myhuaweicloud.com/')[1],'zzsn', att_id)
        # lock.acquire()
        cursor_.execute(Upsql, values)  # 插入
        cnx_.commit()  # 提交
        # lock.release()
    except Exception as e:
        log.info(e)
        return False
    log.info(f"更新完成:{att_id}====")
    return True

def GetContent(pdf_url, social_code, start_time,att_id):
    # 上传至华为云服务器
    retData = uptoOBS(pdf_url, 8, social_code)
    # 附件插入att数据库
    if retData['state']:
        pass
    else:
        log.info(f'====pdf解析失败====')
        # 获取当前进程pid
        current_pid = baseCore.getPID()
        # todo: 重新启动新进程，杀死当前进程
        subprocess.Popen([sys.executable] + sys.argv)
        os.kill(current_pid, 9)
        return False

    att_id = tableUpdate(retData, att_id)
    if att_id:
        pass
    else:
        return False
    content = retData['content']
    return content

if __name__ == '__main__':
    while True:
        item = '23121403366|1731946331417709349|https://data.eastmoney.com/notices/detail/CG/AN202311081609279378.html|ZZSN230824151229535'
        # item = baseCore.redicPullData('Notice:id')
        log.info(item)
        if item:
            es_id = item.split('|')[0]
            att_id = item.split('|')[1]
            # 原文链接
            href = item.split('|')[2]
            social_code = item.split('|')[3]
            art_code = href.split('/')[-1].split('.')[0]
            t = int(time.time() * 1000)
            json_url = f'https://np-cnotice-stock.eastmoney.com/api/content/ann?art_code={art_code}&client_source=web&page_index=1&_={t}'

            for n1 in range(0, 3):
                try:
                    ip = baseCore.get_proxy()
                    json_2 = requests.get(json_url, proxies=ip, verify=False).json()
                    break
                except:
                    if n1 == 2:
                        sys.exit(0)
                    time.sleep(60)
                    continue
            try:
                pdf_url = json_2['data']['attach_url']
            except:
                pdf_url = ''
            if pdf_url:
                # todo: 下载pdf文件
                start_time = time.time()
                content = GetContent(pdf_url, social_code, start_time, att_id)
                if content:
                    # 公告信息列表
                    log.info(f'{att_id}==============解析传输操作成功')
                    state = 1
                    takeTime = baseCore.getTimeCost(start_time, time.time())
                    baseCore.recordLog(social_code, taskType, state, takeTime, pdf_url, '成功')
                    # todo: 更新es
                    updateaunn(index_name, es_id, content)
                    time.sleep(2)
            else:
                baseCore.rePutIntoR('Noticeerror:id', item)
        else:
            log.info('######已无数据######')
        break


