import datetime
import json
import random
import re

import numpy as np
import pandas as pd
import pymongo
import requests
import time

from retry import retry
from selenium.webdriver.common.by import By

from base import BaseCore

db_storage_1 = pymongo.MongoClient('mongodb://192.168.1.36:27017', username='admin', password='zzsn@9988').RESCenter[
    'REITsProdOverview']
db_storage_2 = pymongo.MongoClient('mongodb://192.168.1.36:27017', username='admin', password='zzsn@9988').RESCenter[
    'REITsFinancing']
db_storage_3 = pymongo.MongoClient('mongodb://192.168.1.36:27017', username='admin', password='zzsn@9988').RESCenter[
    'REITsEquityDist']
db_storage_4 = pymongo.MongoClient('mongodb://192.168.1.36:27017', username='admin', password='zzsn@9988').RESCenter[
    'REITsShareStruct']
db_storage_5 = pymongo.MongoClient('mongodb://192.168.1.36:27017', username='admin', password='zzsn@9988').RESCenter[
    'REITsNetWorth']

baseCore = BaseCore.BaseCore()

log = baseCore.getLogger()

headers = {
    'Accept': '*/*',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh-TW;q=0.9,zh;q=0.8',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Pragma': 'no-cache',
    'Referer': 'http://www.sse.com.cn/',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0'
}


@retry(tries=4, delay=20)
def getDataJson(url):
    # ip = baseCore.get_proxy()
    req = requests.get(url, headers=headers)
    req.encoding = req.apparent_encoding
    data_json = re.findall('\((.*)\)', req.text)[0]
    data_json = json.loads(data_json)
    req.close()
    return data_json


# 获取基金代码列表
@retry(tries=3, delay=10)
def getCode():
    code_list = []
    url = f'http://yunhq.sse.com.cn:32041/v1/sh1/list/exchange/reits?callback=jQuery112407214866998855156_1699360786929&select=code%2Cname%2Clast%2Cchg_rate%2Cchange%2Cvolume%2Camount%2Cprev_close%2Copen%2Chigh%2Clow%2Camp_rate%2Ccpxxtype%2Ccpxxsubtype%2Ccpxxextendname&order=code%2Case&begin=0&end=25&_={int(time.time())}'
    ip = baseCore.get_proxy()
    req = requests.get(url, headers=headers, proxies=ip)
    req.encoding = req.apparent_encoding
    data_json = re.findall('\((.*)\)', req.text)[0]
    data_json = json.loads(data_json)['list']
    for data in data_json:
        code_list.append([data[0], data[-1]])
    req.close()
    return code_list


# 产品概况
def productOverview(code):
    url = f'http://query.sse.com.cn/commonSoaQuery.do?jsonCallBack=jsonpCallback3638&isPagination=false&sqlId=FUND_BASIC_INFO&fundCode={code}&_={int(time.time())}'
    try:
        data_json = getDataJson(url)['result'][0]
    except:
        log.error(f'{code}===产品概况连接失败')
        time.sleep(1)
        return
    if db_storage_1.find_one({'code': data_json['fundCode'], 'exchange': '上海证券交易所'}):
        log.info(f"{data_json['fundCode']}===产品概况已采集")
        time.sleep(1)
        return
    dic_info = {
        'code': data_json['fundCode'],  # 代码
        'shortName': data_json['fundExpansionAbbr'],  # 扩位简称
        'office': data_json['lawFirm'],  # 律师事务所
        'caretaker': data_json['companyName'],  # 管理人
        'caretakerPhone': data_json['contactMobile'],  # 管理人联系方式
        'custodian': data_json['trusteeName'],  # 托管人
        'country': '中国',  # 国家
        'exchange': '上海证券交易所'  # 交易所
    }
    try:
        db_storage_1.insert_one(dic_info)
        log.info(f"{data_json['fundCode']}===产品概况采集成功")
    except:
        log.info(f"{data_json['fundCode']}===产品概况保存失败")
    time.sleep(1)


# 筹资情况
def financing(code, name):
    url = f'http://query.sse.com.cn/commonSoaQuery.do?jsonCallBack=jsonpCallback74728&isPagination=true&sqlId=REITS_FXYKM&fundCode={code}&pageHelp.pageSize=10&pageHelp.cacheSize=1&pageHelp.pageNo=1&pageHelp.beginPage=1&pageHelp.endPage=5&_={int(time.time())}'
    try:
        data_jsons = getDataJson(url)['result']
    except:
        log.error(f'{code}===筹资情况连接失败')
        time.sleep(1)
        return
    for data_json in data_jsons:
        saleStartDate = datetime.datetime.strptime(data_json['saleStartDate'], '%Y-%m-%d')
        saleEndDate = datetime.datetime.strptime(data_json['saleEndDate'], '%Y-%m-%d')
        try:
            listingDate = datetime.datetime.strptime(data_json['listingDate'], '%Y-%m-%d')
        except:
            listingDate = ''
        if db_storage_2.find_one(
                {'code': code, 'saleStartDate': saleStartDate, 'saleEndDate': saleEndDate, 'exchange': '上海证券交易所'}):
            log.info(f"{code}===筹资情况已采集")
            time.sleep(1)
            continue
        dic_info = {
            'code': code,  # 代码
            'shortName': name,  # 简称
            'price': data_json['salePrice'],  # 发售价格
            'saleStartDate': saleStartDate,  # 发售起始日期
            'saleEndDate': saleEndDate,  # 发售终止日期
            'saleCopies': data_json['saleCopies'],  # 发售总份数(亿份)
            'listingDate': listingDate,  # 上市日期
            'exchange': '上海证券交易所'  # 交易所
        }
        try:
            db_storage_2.insert_one(dic_info)
            log.info(f'{code}===筹资情况采集成功')
        except:
            log.error(f'{code}===筹资情况保存失败')
        time.sleep(1)


# 权益分配
def equityDistribution(code, name):
    url = f'http://query.sse.com.cn/commonSoaQuery.do?jsonCallBack=jsonpCallback10108&isPagination=true&sqlId=REITS_FH&fundCode={code}&pageHelp.pageSize=10&pageHelp.cacheSize=1&pageHelp.pageNo=1&pageHelp.beginPage=1&pageHelp.endPage=5&_={int(time.time())}'
    try:
        data_jsons = getDataJson(url)['result']
    except:
        log.error(f'{code}===权益分配连接失败')
        time.sleep(1)
        return
    for data_json in data_jsons:
        rightsRegistDate = datetime.datetime.strptime(data_json['rightsRegistDate'], '%Y-%m-%d')
        exrightDate = datetime.datetime.strptime(data_json['exrightDate'], '%Y-%m-%d')
        if db_storage_3.find_one(
                {'code': code, 'rightsRegistDate': rightsRegistDate, 'exrightDate': exrightDate,
                 'exchange': '上海证券交易所'}):
            log.info(f"{code}==={rightsRegistDate}===权益分配已采集")
            time.sleep(1)
            continue
        dic_info = {
            'code': code,  # 代码
            'shortName': name,  # 简称
            'year': data_json['year'],  # 年份
            'fundDividends': data_json['fundDividends'],  # 红利(元)
            'rightsRegistDate': rightsRegistDate,  # 权益登记日
            'exrightDate': exrightDate,  # 除权基准日
            'exchange': '上海证券交易所'  # 交易所
        }
        try:
            db_storage_3.insert_one(dic_info)
            log.info(f'{code}==={rightsRegistDate}===权益分配采集成功')
        except:
            log.error(f'{code}==={rightsRegistDate}===权益分配保存失败')
        time.sleep(1)


# 份额结构
def shareStructure(code, name):
    url = f'http://query.sse.com.cn/commonQuery.do?jsonCallBack=jsonpCallback66502&isPagination=true&sqlId=COMMON_SSE_SJ_JJSJ_JJGM_REITSGM_L&FUND_CODE={code}&pageHelp.pageSize=10&pageHelp.cacheSize=1&pageHelp.pageNo=1&pageHelp.beginPage=1&pageHelp.endPage=5&FUND_TYPE=01&_={int(time.time())}'
    try:
        total = getDataJson(url)['pageHelp']['pageCount']
    except:
        log.error(f'{code}===份额结构总数获取失败')
        time.sleep(1)
        return
    for page in range(1, total + 1):
        url = f'http://query.sse.com.cn/commonQuery.do?jsonCallBack=jsonpCallback66502&isPagination=true&sqlId=COMMON_SSE_SJ_JJSJ_JJGM_REITSGM_L&FUND_CODE={code}&pageHelp.pageSize=10&pageHelp.cacheSize=1&pageHelp.pageNo={page}&pageHelp.beginPage={page}&pageHelp.endPage=5&FUND_TYPE=01&_={int(time.time())}'
        try:
            data_jsons = getDataJson(url)['result']
        except:
            log.error(f'{code}===份额结构第{page}页连接失败')
            time.sleep(1)
            continue
        for data_json in data_jsons:
            tradeDate = datetime.datetime.strptime(str(data_json['TRADE_DATE']), '%Y%m%d')
            if db_storage_4.find_one(
                    {'code': code, 'tradeDate': tradeDate,
                     'exchange': '上海证券交易所'}):
                log.info(f"{code}==={tradeDate}===份额结构已采集")
                time.sleep(1)
                continue
            dic_info = {
                'code': code,  # 代码
                'shortName': name,  # 简称
                'limitVol': data_json['LIMIT_VOL'],  # 场内限售份额（万份）
                'unlimitVol': data_json['UNLIMIT_VOL'],  # 场内非限售份额（万份）
                'totalVol': data_json['TOTAL_VOL'],  # 场内总份额（万份）
                'tradeDate': tradeDate,  # 最新份额日期
                'sellVol': data_json['SELL_VOL'],  # 总份额（万份）
                'exchange': '上海证券交易所'  # 交易所
            }
            try:
                db_storage_4.insert_one(dic_info)
                log.info(f'{code}==={tradeDate}===份额结构采集成功')
            except:
                log.error(f'{code}==={tradeDate}===份额结构保存失败')
            time.sleep(1)


# 净值
def netWorth(code, name):
    url = f'http://query.sse.com.cn/commonSoaQuery.do?jsonCallBack=jsonpCallback66035&isPagination=true&sqlId=REITS_JZ&fundCode={code}&order=appraiseDate%7Cdesc&pageHelp.pageSize=10&pageHelp.cacheSize=1&pageHelp.pageNo=1&pageHelp.beginPage=1&pageHelp.endPage=5&_={int(time.time())}'
    try:
        data_jsons = getDataJson(url)['result']
    except:
        log.error(f'{code}===净值连接失败')
        time.sleep(1)
        return
    for data_json in data_jsons:
        if not data_json['fundUnitnetWorth']:
            continue
        if int(data_json['fundUnitnetWorth']) == 0:
            continue
        appraiseDate = datetime.datetime.strptime(data_json['appraiseDate'], '%Y-%m-%d')
        if db_storage_5.find_one(
                {'code': code, 'appraiseDate': appraiseDate,
                 'exchange': '上海证券交易所'}):
            log.info(f"{code}==={appraiseDate}===净值已采集")
            time.sleep(1)
            continue
        dic_info = {
            'code': code,  # 代码
            'shortName': name,  # 简称
            'appraiseDate': appraiseDate,  # 估值日期
            'fundUnitnetWorth': data_json['fundUnitnetWorth'],  # REITs单位净值（元）
            'exchange': '上海证券交易所'  # 交易所
        }
        try:
            db_storage_5.insert_one(dic_info)
            log.info(f'{code}==={appraiseDate}===净值采集成功')
        except:
            log.error(f'{code}==={appraiseDate}===净值保存失败')
        time.sleep(1)


def getInfo():
    codes_list = getCode()
    for codes in codes_list:
        code = codes[0]
        name = codes[1]
        log.info(f'{code}==={name}===开始采集')
        productOverview(code)
        financing(code, name)
        equityDistribution(code, name)
        shareStructure(code, name)
        netWorth(code, name)
        log.info(f'{code}==={name}===采集结束')
        time.sleep(5)


if __name__ == '__main__':
    getInfo()